From bae9ec8daa600144f2620b04386c2d969993b010 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Wed, 14 Aug 2019 08:26:55 -0600 Subject: [PATCH 01/24] Implement merging code --- JXFormToMysql/main.cpp | 542 +++++++++++++++++++++++++++++++++++------ 1 file changed, 469 insertions(+), 73 deletions(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index b094df0bc..3d5780346 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -60,7 +60,11 @@ int numColumns; int numColumnsInData; QStringList duplicatedTables; bool justCheck; +bool merge; QStringList requiredFiles; +QDomElement Insertroot; +QDomElement Createroot; +QStringList merge_lkptable_errors; //********************************************Global structures**************************************** @@ -110,6 +114,8 @@ struct fieldDef QString formula; QString rTable; //Related table QString rField; //Related field + QString rName; //Contraint name + bool mergeFine = false; bool key; //Whether the field is key QString xmlCode; //The field XML code /xx/xx/xx/xx QString xmlFullPath; //The field XML code /xx/xx/xx/xx with groups @@ -183,6 +189,7 @@ struct tableDef typedef tableDef TtableDef; QList tables; //List of tables +QList merging_tables; //List of tables struct duplicatedSelectValue { @@ -1279,6 +1286,16 @@ QString getDefLanguage() return ""; } +QString getDefLanguageCode() +{ + for (int pos = 0; pos <= languages.count()-1; pos++) + { + if (languages[pos].deflang == true) + return languages[pos].code; + } + return ""; +} + //This function returns the language code for a given name QString getLanguageCode(QString languageName) { @@ -1290,10 +1307,43 @@ QString getLanguageCode(QString languageName) return ""; } +int getMaxDescLength(QList values) +{ + int res; + res = 0; + int pos; + int lng; + for (pos = 0; pos <= values.count()-1;pos++) + { + for (lng = 0; lng < values[pos].desc.count();lng++) + { + if (values[pos].desc[lng].desc.length() >= res) + res = values[pos].desc[lng].desc.length(); + } + } + return res; +} + +//Return the maximum lenght of the values in a lookup table so the size is not excesive for primary keys +int getMaxValueLength(QList values) +{ + int res; + res = 0; + int pos; + for (pos = 0; pos <= values.count()-1;pos++) + { + if (values[pos].code.length() >= res) + res = values[pos].code.length(); + } + return res; +} + //This fuction checkd wheter a lookup table is duplicated. //If there is a match then returns such table -TtableDef getDuplicatedLkpTable(QList thisValues) +TtableDef getDuplicatedLkpTable(QString lkptname, QList thisValues, bool &changeSize, int &newCodeSize, bool &changedRel, bool useMergedTables = true) { + changeSize = false; + newCodeSize = -1; int pos2; bool same; TtableDef empty; @@ -1304,41 +1354,157 @@ TtableDef getDuplicatedLkpTable(QList thisValues) QList currentValues; QString thisDesc; QString currenDesc; - + changedRel = false; //Move the new list of values to a new list and sort it by code qSort(thisValues.begin(),thisValues.end(),lkpComp); QString defLangCode; defLangCode = getLanguageCode(getDefLanguage()); - - for (int pos = 0; pos <= tables.count()-1; pos++) + bool table_found; + table_found = false; + if (merge && useMergedTables) { - if (tables[pos].islookup) + for (int pos = 0; pos <= merging_tables.count()-1; pos++) { - //Move the current list of values to a new list and sort it by code - currentValues.clear(); - currentValues.append(tables[pos].lkpValues); - qSort(currentValues.begin(),currentValues.end(),lkpComp); - - if (currentValues.count() == thisValues.count()) //Same number of values - { - same = true; - for (pos2 = 0; pos2 <= tables[pos].lkpValues.count()-1;pos2++) + if (merging_tables[pos].islookup) + { + if (merging_tables[pos].name == lkptname) { - //Compares if an item in the list dont have same code or same description - thisDesc = getDescForLanguage(thisValues[pos2].desc,defLangCode); - currenDesc = getDescForLanguage(currentValues[pos2].desc,defLangCode); + table_found = true; + currentValues.clear(); + currentValues.append(merging_tables[pos].lkpValues); + qSort(currentValues.begin(),currentValues.end(),lkpComp); + if (currentValues.count() == thisValues.count()) //Same number of values + { + same = true; + for (pos2 = 0; pos2 <= merging_tables[pos].lkpValues.count()-1;pos2++) + { + //Compares if an item in the list dont have same code or same description + thisDesc = getDescForLanguage(thisValues[pos2].desc,defLangCode); + currenDesc = getDescForLanguage(currentValues[pos2].desc,defLangCode); - if ((currentValues[pos2].code.simplified().toLower() != thisValues[pos2].code.simplified().toLower()) || - (currenDesc.simplified().toLower() != thisDesc.simplified().toLower())) - { - same = false; - break; + if ((currentValues[pos2].code.simplified().toLower() != thisValues[pos2].code.simplified().toLower()) || + (currenDesc.simplified().toLower() != thisDesc.simplified().toLower())) + { + merge_lkptable_errors.append(lkptname); + same = false; + break; + } + } + if (!same) + { + merge_lkptable_errors.append(lkptname); + return empty; + } + else + { + tables.append(merging_tables[pos]); + return merging_tables[pos]; + } + } + else + { + if (currentValues.count() > thisValues.count()) + { + merge_lkptable_errors.append(lkptname); + return empty; + } + else + { + bool found; + for (pos2 = 0; pos2 < currentValues.count(); pos2++) + { + //Compares if an item in the current values does not exist in the new list + currenDesc = getDescForLanguage(currentValues[pos2].desc,defLangCode); + found = false; + for (int pos3 = 0; pos3 < thisValues.count(); pos3++) + { + thisDesc = getDescForLanguage(thisValues[pos3].desc,defLangCode); + if ((thisValues[pos3].code == currentValues[pos2].code) && (currenDesc == thisDesc)) + found = true; + } + if (!found) + break; + } + if (!found) + { + merge_lkptable_errors.append(lkptname); + return empty; + } + else + { + //Look for values that are not part of current + bool found; + for (pos2 = 0; pos2 < thisValues.count(); pos2++) + { + found = false; + for (int pos3 = 0; pos3 < currentValues.count(); pos3++) + { + if (thisValues[pos2].code == currentValues[pos3].code) + found = true; + } + if (!found) + { + merging_tables[pos].lkpValues.append(thisValues[pos2]); + } + } + bool nchangeSize; + int nnewCodeSize; + bool nchangedRel; + TtableDef existing_table = getDuplicatedLkpTable(lkptname, merging_tables[pos].lkpValues,nchangeSize,nnewCodeSize,nchangedRel,false); + if (existing_table.name != "EMPTY") + { + changeSize = true; + changedRel = true; + newCodeSize = getMaxValueLength(existing_table.lkpValues); + return existing_table; + } + changeSize = true; + newCodeSize = getMaxValueLength(merging_tables[pos].lkpValues); + int newDescSize; + newDescSize = getMaxDescLength(merging_tables[pos].lkpValues); + merging_tables[pos].fields[0].size = newCodeSize; + merging_tables[pos].fields[1].size = newDescSize; + tables.append(merging_tables[pos]); + return merging_tables[pos]; + } + } } } - if (same) + } + } + } + if (!table_found) + { + for (int pos = 0; pos <= tables.count()-1; pos++) + { + if (tables[pos].islookup) + { + //Move the current list of values to a new list and sort it by code + currentValues.clear(); + currentValues.append(tables[pos].lkpValues); + qSort(currentValues.begin(),currentValues.end(),lkpComp); + + if (currentValues.count() == thisValues.count()) //Same number of values { - return tables[pos]; + same = true; + for (pos2 = 0; pos2 <= tables[pos].lkpValues.count()-1;pos2++) + { + //Compares if an item in the list dont have same code or same description + thisDesc = getDescForLanguage(thisValues[pos2].desc,defLangCode); + currenDesc = getDescForLanguage(currentValues[pos2].desc,defLangCode); + + if ((currentValues[pos2].code.simplified().toLower() != thisValues[pos2].code.simplified().toLower()) || + (currenDesc.simplified().toLower() != thisDesc.simplified().toLower())) + { + same = false; + break; + } + } + if (same) + { + return tables[pos]; + } } } } @@ -1766,10 +1932,13 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr { createFieldNode.setAttribute("rtable",prefix + tables[pos].fields[clm].rTable); createFieldNode.setAttribute("rfield",tables[pos].fields[clm].rField); + createFieldNode.setAttribute("rname","fk_" + tables[pos].fields[clm].rName); if (isRelatedTableLookUp(tables[pos].fields[clm].rTable)) { createFieldNode.setAttribute("rlookup","true"); + if (tables[pos].fields[clm].mergeFine) + createFieldNode.setAttribute("mergeFine","true"); } } @@ -1794,10 +1963,13 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr if (tables[pos].fields[clm].rTable != "") { createFieldNode.setAttribute("rtable",prefix + tables[pos].fields[clm].rTable); - createFieldNode.setAttribute("rfield",tables[pos].fields[clm].rField); + createFieldNode.setAttribute("rfield",tables[pos].fields[clm].rField); + createFieldNode.setAttribute("rname","fk_" + tables[pos].fields[clm].rName); if (isRelatedTableLookUp(tables[pos].fields[clm].rTable)) { createFieldNode.setAttribute("rlookup","true"); + if (tables[pos].fields[clm].mergeFine) + createFieldNode.setAttribute("mergeFine","true"); } } tables[pos].tableCreteElement.appendChild(createFieldNode); @@ -1824,9 +1996,12 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr { createFieldNode.setAttribute("rtable",prefix + tables[pos].fields[clm].rTable); createFieldNode.setAttribute("rfield",tables[pos].fields[clm].rField); + createFieldNode.setAttribute("rname","fk_" + tables[pos].fields[clm].rName); if (isRelatedTableLookUp(tables[pos].fields[clm].rTable)) { createFieldNode.setAttribute("rlookup","true"); + if (tables[pos].fields[clm].mergeFine) + createFieldNode.setAttribute("mergeFine","true"); } } tables[pos].tableCreteElement.appendChild(createFieldNode); @@ -1852,7 +2027,8 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr if (tables[pos].fields[clm].rTable != "") { createFieldNode.setAttribute("rtable",prefix + tables[pos].fields[clm].rTable); - createFieldNode.setAttribute("rfield",tables[pos].fields[clm].rField); + createFieldNode.setAttribute("rfield",tables[pos].fields[clm].rField); + createFieldNode.setAttribute("rname","fk_" + tables[pos].fields[clm].rName); } tables[pos].tableCreteElement.appendChild(createFieldNode); } @@ -1898,13 +2074,13 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr if (!tables[pos].fields[clm].rTable.isEmpty()) { if (isRelatedTableLookUp(tables[pos].fields[clm].rTable)) - { + { idx++; - index = "INDEX fk" + QString::number(idx) + "_" + prefix + tables[pos].name.toLower() + "_" + prefix + tables[pos].fields[clm].rTable.toLower() ; - indexes << index.left(64) + " (" + tables[pos].fields[clm].name.toLower() + ") , " << "\n"; + index = "INDEX fk_" + tables[pos].fields[clm].rName ; + indexes << index + " (" + tables[pos].fields[clm].name.toLower() + ") , " << "\n"; - constraint = "CONSTRAINT fk" + QString::number(idx) + "_" + prefix + tables[pos].name.toLower() + "_" + prefix + tables[pos].fields[clm].rTable.toLower(); - rels << constraint.left(64) << "\n"; + constraint = "CONSTRAINT fk_" + tables[pos].fields[clm].rName; + rels << constraint << "\n"; rels << "FOREIGN KEY (" + tables[pos].fields[clm].name.toLower() + ")" << "\n"; rels << "REFERENCES " + prefix + tables[pos].fields[clm].rTable.toLower() + " (" + tables[pos].fields[clm].rField.toLower() + ")" << "\n"; @@ -2496,38 +2672,6 @@ int getMaxMSelValueLength(QList values) return res.length(); } -//Return the maximum lenght of the values in a lookup table so the size is not excesive for primary keys -int getMaxValueLength(QList values) -{ - int res; - res = 0; - int pos; - for (pos = 0; pos <= values.count()-1;pos++) - { - if (values[pos].code.length() >= res) - res = values[pos].code.length(); - } - return res; -} - -int getMaxDescLength(QList values) -{ - int res; - res = 0; - int pos; - int lng; - for (pos = 0; pos <= values.count()-1;pos++) - { - for (lng = 0; lng < values[pos].desc.count();lng++) - { - if (values[pos].desc[lng].desc.length() >= res) - res = values[pos].desc[lng].desc.length(); - } - } - return res; -} - - //Returns wheter or not a the values of a lookup table are yes/no values. Used in combination to yes/no lookup values to //create or not a lookup table of yes/no. // NOTE this is a very crude function and only catches a couple of cases... @@ -3277,6 +3421,16 @@ void getReferenceForSelectAt(QString calculateExpresion,QString &fieldType, int } } +QString getUUIDCode(bool last12 = false) +{ + QUuid triggerUUID=QUuid::createUuid(); + QString strTriggerUUID=triggerUUID.toString().replace("{","").replace("}","").replace("-","_"); + if (last12) + return strTriggerUUID.left(12); + else + return strTriggerUUID; +} + //Adds OSM tags as columns to a OSM table void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) { @@ -3314,13 +3468,17 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) //Creating the lookp table if its neccesary if (isLookUpYesNo(values) == false) //Do not process yes/no values as lookup tables { - TtableDef lkpTable = getDuplicatedLkpTable(values); + QString table_name = "lkp" + fixField(variableName.toLower()); + bool changeSize; + int newCodeSize; + bool changedRel; + TtableDef lkpTable = getDuplicatedLkpTable(table_name,values,changeSize,newCodeSize,changedRel); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; if (lkpTable.name == "EMPTY") { - lkpTable.name = "lkp" + fixField(variableName.toLower()); + lkpTable.name = table_name; for (int lang = 0; lang < aField.desc.count(); lang++) { TlngLkpDesc fieldDesc; @@ -3370,6 +3528,7 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) lkpTable.fields.append(lkpDesc); aField.rTable = lkpTable.name; aField.rField = lkpCode.name; + aField.rName = getUUIDCode(); tables.append(lkpTable); } else @@ -3378,6 +3537,18 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) log("Lookup table for field " + variableName + " is the same as " + lkpTable.name + ". Using " + lkpTable.name); aField.rTable = lkpTable.name; aField.rField = getKeyField(lkpTable.name); + aField.rName = getUUIDCode(); + if (merge) + { + if (changeSize) + { + aField.size = newCodeSize; + } + if (changedRel) + { + aField.mergeFine = true; + } + } } } } @@ -3687,13 +3858,17 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q //Creating the lookp table if its neccesary if (isLookUpYesNo(values) == false) //Do not process yes/no values as lookup tables { - TtableDef lkpTable = getDuplicatedLkpTable(values); + QString table_name = "lkp" + fixField(variableName.toLower()); + bool changeSize; + int newCodeSize; + bool changedRel; + TtableDef lkpTable = getDuplicatedLkpTable(table_name, values, changeSize, newCodeSize, changedRel); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; if (lkpTable.name == "EMPTY") { - lkpTable.name = "lkp" + fixField(variableName.toLower()); + lkpTable.name = table_name; for (int lang = 0; lang < aField.desc.count(); lang++) { TlngLkpDesc fieldDesc; @@ -3743,6 +3918,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q lkpTable.fields.append(lkpDesc); aField.rTable = lkpTable.name; aField.rField = lkpCode.name; + aField.rName = getUUIDCode(); tables.append(lkpTable); } else @@ -3750,7 +3926,17 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q if (outputType == "h") log("Lookup table for field " + variableName + " is the same as " + lkpTable.name + ". Using " + lkpTable.name); aField.rTable = lkpTable.name; - aField.rField = getKeyField(lkpTable.name); + aField.rField = getKeyField(lkpTable.name); + aField.rName = getUUIDCode(); + if (merge) + { + if (changeSize) + { + aField.size = newCodeSize; + } + if (changedRel) + aField.mergeFine = true; + } } } } @@ -3853,6 +4039,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q relField.sensitive = false; relField.rTable = tables[tblIndex].name; relField.rField = tables[tblIndex].fields[field].name; + relField.rName = getUUIDCode(); relField.xmlCode = "NONE"; relField.isMultiSelect = false; relField.formula = ""; @@ -3898,13 +4085,17 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q mselKeyField.size = getMaxValueLength(values); mselKeyField.decSize = 0; //Processing the lookup table if neccesary - TtableDef lkpTable = getDuplicatedLkpTable(values); + QString table_name = "lkp" + fixField(variableName.toLower()); + bool changeSize; + int newCodeSize; + bool changedRel; + TtableDef lkpTable = getDuplicatedLkpTable(table_name,values,changeSize,newCodeSize,changedRel); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; if (lkpTable.name == "EMPTY") { - lkpTable.name = "lkp" + fixField(variableName.toLower()); + lkpTable.name = table_name; for (int lang = 0; lang < aField.desc.count(); lang++) { @@ -3957,6 +4148,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q //Linking the multiselect field to this lookup table mselKeyField.rTable = lkpTable.name; mselKeyField.rField = lkpCode.name; + mselKeyField.rName = getUUIDCode(); lkpTable.lkpValues.append(values); tables.append(lkpTable); //Append the lookup table to the list of tables } @@ -3967,6 +4159,16 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q //Linkin the multiselect table to the existing lookup table mselKeyField.rTable = lkpTable.name; mselKeyField.rField = getKeyField(lkpTable.name); + mselKeyField.rName = getUUIDCode(); + if (merge) + { + if (changeSize) + { + aField.size = getMaxMSelValueLength(lkpTable.lkpValues); + } + if (changedRel) + aField.mergeFine = true; + } } mselTable.fields.append(mselKeyField); //Add the multiselect key now linked to the looktable to the multiselect table tables.append(mselTable); //Append the multiselect to the list of tables @@ -4121,6 +4323,7 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f relField.sensitive = false; relField.rTable = parentTable.name; relField.rField = parentTable.fields[field].name; + relField.rName = getUUIDCode(); relField.xmlCode = "NONE"; relField.xmlFullPath = "NONE"; relField.selectSource = "NONE"; @@ -4208,13 +4411,17 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f //Creating the lookp table if its neccesary if (isLookUpYesNo(values) == false) //Do not process yes/no values as lookup tables { - TtableDef lkpTable = getDuplicatedLkpTable(values); + QString table_name = "lkp" + fixField(variableName.toLower()); + bool changeSize; + int newCodeSize; + bool changedRel; + TtableDef lkpTable = getDuplicatedLkpTable(table_name, values, changeSize, newCodeSize, changedRel); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; if (lkpTable.name == "EMPTY") { - lkpTable.name = "lkp" + fixField(variableName.toLower()); + lkpTable.name = table_name; for (int lang = 0; lang < aField.desc.count(); lang++) { TlngLkpDesc fieldDesc; @@ -4264,6 +4471,7 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f lkpTable.fields.append(lkpDesc); aField.rTable = lkpTable.name; aField.rField = lkpCode.name; + aField.rName = getUUIDCode(); tables.append(lkpTable); } else @@ -4272,6 +4480,16 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f log("Lookup table for field " + variableName + " is the same as " + lkpTable.name + ". Using " + lkpTable.name); aField.rTable = lkpTable.name; aField.rField = getKeyField(lkpTable.name); + aField.rName = getUUIDCode(); + if (merge) + { + if (changeSize) + { + aField.size = newCodeSize; + } + if (changedRel) + aField.mergeFine = true; + } } } } @@ -4630,6 +4848,109 @@ void reportSelectDuplicates() log(XMLResult.toString()); } +QList getValuesFromInsertFile(QString tableName, QDomNode startNode, QString &clmCode, QString &cmlDesc) +{ + QDomNode lkptable = startNode; + QList res; + while (!lkptable.isNull()) + { + if (lkptable.toElement().attribute("name") == tableName) + { + clmCode = lkptable.toElement().attribute("clmcode"); + cmlDesc = lkptable.toElement().attribute("clmdesc"); + QDomNode nodeValue = lkptable.firstChild(); + while (!nodeValue.isNull()) + { + TlkpValue value; + value.code = nodeValue.toElement().attribute("code"); + TlngLkpDesc valueDesc; + valueDesc.langCode = getDefLanguageCode(); + valueDesc.desc = nodeValue.toElement().attribute("description"); + value.desc.append(valueDesc); + res.append(value); + nodeValue = nodeValue.nextSibling(); + } + } + lkptable = lkptable.nextSibling(); + } + return res; +} + +void loadMergingCreateFile(QDomNode lkptable, QDomNode insertStartNode) +{ + while (!lkptable.isNull()) + { + QString tableName; + QString tableDesc; + tableName = lkptable.toElement().attribute("name"); + tableDesc = lkptable.toElement().attribute("desc"); + TtableDef lkpTable; + lkpTable.isLoop = false; + lkpTable.isOSM = false; + lkpTable.isGroup = false; + + lkpTable.name = tableName; + + TlngLkpDesc fieldDesc; + fieldDesc.langCode = getDefLanguageCode(); + fieldDesc.desc = tableDesc; + lkpTable.desc.append(fieldDesc); + + lkpTable.pos = -1; + lkpTable.islookup = true; + lkpTable.isOneToOne = false; + QString clmCode; + QString cmlDesc; + lkpTable.lkpValues.append(getValuesFromInsertFile(tableName, insertStartNode, clmCode, cmlDesc)); + QDomNode lkpField = lkptable.firstChild(); + while (!lkpField.isNull()) + { + if (lkpField.toElement().attribute("name") == clmCode) + { + TfieldDef lkpCode; + lkpCode.name = clmCode; + lkpCode.selectSource = "NONE"; + lkpCode.selectListName = "NONE"; + + TlngLkpDesc langDesc; + langDesc.langCode = getDefLanguageCode(); + langDesc.desc = lkpField.toElement().attribute("desc"); + lkpCode.desc.append(langDesc); + + lkpCode.key = true; + lkpCode.sensitive = false; + lkpCode.type = lkpField.toElement().attribute("type"); + lkpCode.size = lkpField.toElement().attribute("size").toInt(); + lkpCode.decSize = lkpField.toElement().attribute("decsize").toInt(); + lkpTable.fields.append(lkpCode); + } + if (lkpField.toElement().attribute("name") == cmlDesc) + { + TfieldDef lkpDesc; + lkpDesc.name = cmlDesc; + lkpDesc.selectSource = "NONE"; + lkpDesc.selectListName = "NONE"; + + TlngLkpDesc langDesc; + langDesc.langCode = getDefLanguageCode(); + langDesc.desc = lkpField.toElement().attribute("desc"); + lkpDesc.desc.append(langDesc); + + lkpDesc.key = false; + lkpDesc.sensitive = false; + lkpDesc.type = lkpField.toElement().attribute("type"); + lkpDesc.size = lkpField.toElement().attribute("size").toInt(); + lkpDesc.decSize = lkpField.toElement().attribute("decsize").toInt(); + lkpTable.fields.append(lkpDesc); + } + + lkpField = lkpField.nextSibling(); + } + merging_tables.append(lkpTable); + lkptable = lkptable.nextSibling(); + } +} + //Reads the input JSON file and converts it to a MySQL database int processJSON(QString inputFile, QString mainTable, QString mainField, QDir dir, QSqlDatabase database) { @@ -4792,6 +5113,11 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di exit(4); } } + if (merge) + { + loadMergingCreateFile(Createroot.firstChild().firstChild(),Insertroot.firstChild()); + } + int lang; //Creating the main table tableIndex = tableIndex + 1; @@ -5190,6 +5516,9 @@ int main(int argc, char *argv[]) TCLAP::ValueArg tempDirArg("e","tempdirectory","Temporary directory. ./tmp by default",false,"./tmp","string"); TCLAP::ValueArg outputTypeArg("o","outputtype","Output type: (h)uman or (m)achine readble. Machine readble by default",false,"m","string"); TCLAP::SwitchArg justCheckSwitch("K","justCheck","Just check of main inconsistencies and report back", cmd, false); + TCLAP::SwitchArg mergeSwitch("M","merge","Process the JXForm for merging. If true the MC and MI parameters cannot be empty", cmd, false); + TCLAP::ValueArg mergeCreateArg("r","mergecreatefile","Merge with create file",false,"","string"); + TCLAP::ValueArg mergeInsertArg("n","mergeinsertfile","Merge with insert file",false,"","string"); TCLAP::UnlabeledMultiArg suppFiles("supportFile", "support files", false, "string"); debug = false; @@ -5215,7 +5544,11 @@ int main(int argc, char *argv[]) cmd.add(transFileArg); cmd.add(yesNoStringArg); cmd.add(tempDirArg); - cmd.add(outputTypeArg); + cmd.add(outputTypeArg); + + cmd.add(mergeCreateArg); + cmd.add(mergeInsertArg); + cmd.add(suppFiles); //Parsing the command lines @@ -5236,6 +5569,7 @@ int main(int argc, char *argv[]) } loadInvalidFieldNames(); justCheck = justCheckSwitch.getValue(); + merge = mergeSwitch.getValue(); //Getting the variables from the command QString input = QString::fromUtf8(inputArg.getValue().c_str()); QString ddl = QString::fromUtf8(ddlArg.getValue().c_str()); @@ -5252,6 +5586,10 @@ int main(int argc, char *argv[]) QString transFile = QString::fromUtf8(transFileArg.getValue().c_str()); QString yesNoString = QString::fromUtf8(yesNoStringArg.getValue().c_str()); QString tempDirectory = QString::fromUtf8(tempDirArg.getValue().c_str()); + + QString mergeCreateFile = QString::fromUtf8(mergeCreateArg.getValue().c_str()); + QString mergeInsertFile = QString::fromUtf8(mergeInsertArg.getValue().c_str()); + outputType = QString::fromUtf8(outputTypeArg.getValue().c_str()); mainVar = mainVar.trimmed().toLower(); QDir currdir("."); @@ -5286,6 +5624,64 @@ int main(int argc, char *argv[]) } dir.setPath(tempDirectory); + if (merge) + { + if (!QFile::exists(mergeCreateFile)) + { + log("Create file used for merging does not exists"); + return 1; + } + if (!QFile::exists(mergeInsertFile)) + { + log("Insert file used for merging does not exists"); + return 1; + } + //Openning and parsing create XML file + QDomDocument mergingCrate("create"); + QFile mergingFile(mergeCreateFile); + if (!mergingFile.open(QIODevice::ReadOnly)) + { + log("Cannot open create file used for merging"); + return 1; + } + if (!mergingCrate.setContent(&mergingFile)) + { + log("Cannot parse create file used for merging"); + mergingFile.close(); + return 1; + } + mergingFile.close(); + Createroot = mergingCrate.documentElement(); + if (Createroot.tagName() != "XMLSchemaStructure") + { + log("Create file used for merging is not valid"); + mergingFile.close(); + return 1; + } + + QDomDocument mergingInsert("insert"); + mergingFile.setFileName(mergeInsertFile); + if (!mergingFile.open(QIODevice::ReadOnly)) + { + log("Cannot open insert file used for merging"); + return 1; + } + if (!mergingInsert.setContent(&mergingFile)) + { + log("Cannot parse insert file used for merging"); + mergingFile.close(); + return 1; + } + mergingFile.close(); + Insertroot = mergingInsert.documentElement(); + if (Insertroot.tagName() != "insertValuesXML") + { + log("Insert file used for merging is not valid"); + mergingFile.close(); + return 1; + } + } + //Unzip any zip files in the temporary directory QStringList zipFiles; for (int pos = 0; pos <= supportFiles.count()-1; pos++) From 92f171c3d6d1e496f7c3daf7521e66a2d98dfe13 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Wed, 14 Aug 2019 08:27:23 -0600 Subject: [PATCH 02/24] Fix merging process --- utilities/compareCreateXML/main.cpp | 303 ++++++++++++---------------- 1 file changed, 132 insertions(+), 171 deletions(-) diff --git a/utilities/compareCreateXML/main.cpp b/utilities/compareCreateXML/main.cpp index 98d2f7694..f4d873394 100644 --- a/utilities/compareCreateXML/main.cpp +++ b/utilities/compareCreateXML/main.cpp @@ -23,6 +23,7 @@ License along with CompareCreateXML. If not, see #include #include +#include bool fatalError; QStringList diff; @@ -306,96 +307,73 @@ QDomNode findTable(QDomDocument docB,QString tableName) return null; } -bool compareFields(QDomElement a, QDomElement b, bool &increased, int &newSize, int &newDec, bool &critical) -{ - bool same; - increased = false; +QString compareFields(QDomElement a, QDomElement b, int &newSize, int &newDec) +{ newSize = a.attribute("size","0").toInt(); newDec = a.attribute("decsize","0").toInt(); - same = true; - critical = true; - bool isKey; - isKey = false; - if (a.attribute("key","false") == "true") - isKey = true; - if (a.attribute("key","") != b.attribute("key","")) - return false; - + return "KNS"; if (a.attribute("rtable","") != b.attribute("rtable","")) - return false; + return "RTNS"; if (a.attribute("rfield","") != b.attribute("rfield","")) - return false; - + return "RFNS"; if (a.attribute("type","") != b.attribute("type","")) - return false; + return "TNS"; if (a.attribute("size","") != b.attribute("size","")) - { - critical = false; - if (a.attribute("size","0").toInt() >= b.attribute("size","0").toInt()) + { + if (a.attribute("size","0").toInt() > b.attribute("size","0").toInt()) { if (a.attribute("type","") == "decimal") { - if (a.attribute("decsize","0").toInt() >= b.attribute("decsize","0").toInt()) + if (a.attribute("decsize","0").toInt() != b.attribute("decsize","0").toInt()) { - if ((a.attribute("size","0").toInt() - a.attribute("decsize","0").toInt()) >= (b.attribute("size","0").toInt() - b.attribute("decsize","0").toInt())) - { - increased = true; + if (a.attribute("decsize","0").toInt() > b.attribute("decsize","0").toInt()) + { newSize = a.attribute("size","0").toInt(); newDec = a.attribute("decsize","0").toInt(); - if (isKey == false) - same = true; - else - same = false; + return "FIC"; } else { - same = false; + if (a.attribute("decsize","0").toInt() < b.attribute("decsize","0").toInt()) + return "FDC"; } } else { - same = false; + if (a.attribute("size","0").toInt() < b.attribute("size","0").toInt()) + return "FDC"; } } else - { - increased = true; + { newSize = a.attribute("size","0").toInt(); - if (isKey == false) - same = true; - else - same = false; + return "FIC"; } - } else - same = false; + { + if (a.attribute("size","0").toInt() < b.attribute("size","0").toInt()) + return "FDC"; + } } if (a.attribute("decsize","") != b.attribute("decsize","")) - { - critical = false; - if (a.attribute("decsize","0").toInt() >= b.attribute("decsize","0").toInt()) + { + if (a.attribute("decsize","0").toInt() > b.attribute("decsize","0").toInt()) { - if ((a.attribute("size","0").toInt() - a.attribute("decsize","0").toInt()) >= (b.attribute("size","0").toInt() - b.attribute("decsize","0").toInt())) - { - increased = true; - newSize = a.attribute("size","0").toInt(); - newDec = a.attribute("decsize","0").toInt(); - if (isKey == false) - same = true; - else - same = false; - } - else - same = false; + newSize = a.attribute("size","0").toInt(); + newDec = a.attribute("decsize","0").toInt(); + return "FIC"; } else - same = false; + { + if (a.attribute("decsize","0").toInt() < b.attribute("decsize","0").toInt()) + return "FDC"; + } } - return same; + return ""; } QString getFieldDefinition(QDomElement field) @@ -410,6 +388,57 @@ QString getFieldDefinition(QDomElement field) return result; } +void checkField(QDomNode eTable, QDomElement a, QDomElement b) +{ + int newSize; + int newDec; + QString result; + result = compareFields(a,b,newSize,newDec); + if (result != "") + { + if (result == "FIC") + { + if (outputType == "h") + { + if (a.attribute("type","") != "decimal") + log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0")); + else + log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0") + ". Decimal size from " + b.attribute("decsize",0) + " to " + a.attribute("decsize","0")); + } + b.setAttribute("size",newSize); + b.setAttribute("decsize",newDec); + addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); + } + else + { + if (outputType == "h") + { + if (result == "KNS") + fatal("KNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed key from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); + if (result == "RTNS") + fatal("RTNS: The relational table for field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); + if (result == "RFNS") + fatal("RFNS: The relational field for field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); + if (result == "TNS") + fatal("TNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); + if (result == "FDC") + log("FDC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " has descreased in size. This change will be ignored"); + } + else + { + TcompError error; + error.code = result; + error.table = eTable.toElement().attribute("name",""); + error.field = a.attribute("name",""); + error.desc = ""; + error.from = getFieldDefinition(b); + error.to = getFieldDefinition(a); + errorList.append(error); + } + } + } +} + void compareLKPTables(QDomNode table,QDomDocument &docB) { QDomNode node; @@ -424,56 +453,24 @@ void compareLKPTables(QDomNode table,QDomDocument &docB) while (!field.isNull()) { QDomElement eField = field.toElement(); - QDomNode fieldFound = findField(tablefound,eField.attribute("name","")); - if (!fieldFound.isNull()) + if (eField.tagName() == "table") + compareLKPTables(field,docB); + else { - bool increased; - int newSize; - int newDec; - bool critical; - if (compareFields(eField,fieldFound.toElement(),increased,newSize,newDec,critical) == false) + QDomNode fieldFound = findField(tablefound,eField.attribute("name","")); + if (!fieldFound.isNull()) { - if (outputType == "h") - { - if (critical) - fatal("FNS:Field " + eField.attribute("name","") + " in lookup table " + node.toElement().attribute("name","") + " from A is not the same in B. You need to fix it in the XLSX file"); - } - else - { - if (critical) - { - TcompError error; - error.code = "FNS"; - error.table = node.toElement().attribute("name",""); - error.field = eField.attribute("name",""); - error.desc = "Field " + eField.attribute("name","") + " in lookup table " + node.toElement().attribute("name","") + " from A is not the same in B"; - error.from = getFieldDefinition(fieldFound.toElement()); - error.to = getFieldDefinition(eField); - errorList.append(error); - } - } + checkField(tablefound,field.toElement(),fieldFound.toElement()); } else { - if (increased) - { - if (outputType == "h") - { - if (eField.attribute("type","") != "decimal") - log("FIC: Field " + eField.attribute("name","") + " in lookup table " + node.toElement().attribute("name","") + " will be increased from " + fieldFound.toElement().attribute("size",0) + " to " + eField.toElement().attribute("size","0")); - else - log("FIC: Field " + eField.attribute("name","") + " in lookup table " + node.toElement().attribute("name","") + " will be increased from " + fieldFound.toElement().attribute("size",0) + " to " + eField.toElement().attribute("size","0") + ". Decimal size from " + fieldFound.toElement().attribute("decsize",0) + " to " + eField.toElement().attribute("decsize","0")); - } - addAlterFieldToDiff(node.toElement().attribute("name",""),eField,newSize,newDec); - } + if (outputType == "h") + log("FNF:Field " + eField.attribute("name","") + " in table " + tablefound.toElement().attribute("name","") + " from A is not found in B"); + addFieldToDiff(tablefound.toElement().attribute("name",""),eField); + tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); } } - else - { - if (outputType == "h") - log("FNF:Field " + eField.attribute("name","") + " in lookup table " + node.toElement().attribute("name","") + " from A is not found in B"); - addFieldToDiff(node.toElement().attribute("name",""),eField); - } + field = field.nextSibling(); } } @@ -511,55 +508,17 @@ void compareTables(QDomNode table,QDomDocument &docB) QDomNode fieldFound = findField(tablefound,eField.attribute("name","")); if (!fieldFound.isNull()) { - bool increased; - int newSize; - int newDec; - bool critical; - if (compareFields(eField,fieldFound.toElement(),increased,newSize,newDec,critical) == false) - { - if (outputType == "h") - { - if (critical) - fatal("FNS:Field " + eField.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " from A is not the same in B. You need to fix it in the XLSX file"); - } - else - { - if (critical) - { - TcompError error; - error.code = "FNS"; - error.table = eTable.toElement().attribute("name",""); - error.field = eField.attribute("name",""); - error.desc = "Field " + eField.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " from A is not the same in B"; - error.from = getFieldDefinition(fieldFound.toElement()); - error.to = getFieldDefinition(eField); - errorList.append(error); - } - } - } - else - { - if (increased) - { - if (outputType == "h") - { - if (eField.attribute("type","") != "decimal") - log("FIC: Field " + eField.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + fieldFound.toElement().attribute("size",0) + " to " + eField.toElement().attribute("size","0")); - else - log("FIC: Field " + eField.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + fieldFound.toElement().attribute("size",0) + " to " + eField.toElement().attribute("size","0") + ". Decimal size from " + fieldFound.toElement().attribute("decsize",0) + " to " + eField.toElement().attribute("decsize","0")); - } - addAlterFieldToDiff(eTable.attribute("name",""),eField,newSize,newDec); - } - } + checkField(tablefound,field.toElement(),fieldFound.toElement()); } else { if (outputType == "h") - log("FNF:Field " + eField.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " from A is not found in B"); - addFieldToDiff( eTable.toElement().attribute("name",""),eField); + log("FNF:Field " + eField.attribute("name","") + " in table " + tablefound.toElement().attribute("name","") + " from A is not found in B"); + addFieldToDiff(tablefound.toElement().attribute("name",""),eField); tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); } } + field = field.nextSibling(); } } @@ -617,37 +576,39 @@ int main(int argc, char *argv[]) { QString title; title = title + "********************************************************************* \n"; - title = title + " * Compare Create XML * \n"; - title = title + " * This tool compares two create XML files (A and B) for incremental * \n"; - title = title + " * changes. A is consider an incremental version of B . * \n"; - title = title + " * * \n"; - title = title + " * The tool informs of tables and variables in A that are not in B. * \n"; - title = title + " * The tool can also create a combined file C that appends * \n"; - title = title + " * not found tables and variables with the following conditions: * \n"; - title = title + " * 1) If a table in A is not found in B then it will be added to B * \n"; - title = title + " * only if its parent exists in B. * \n"; - title = title + " * * \n"; - title = title + " * C will have all of B plus all of A. * \n"; - title = title + " * * \n"; - title = title + " * The tool WILL NOT fix the following: * \n"; - title = title + " * 1) Inconsistencies in field definition like size, type, * \n"; - title = title + " * parent table and parent field. * \n"; - title = title + " * 2) Tables that do not share the same parent. * \n"; - title = title + " * * \n"; - title = title + " * Nomenclature: * \n"; - title = title + " * TNF: Table not found. * \n"; - title = title + " * TNS: The table does not have the same parent table. * \n"; - title = title + " * FNF: Field not found. * \n"; - title = title + " * FNS: The field is not the same. * \n"; - title = title + " * * \n"; - title = title + " * This tool is usefull when dealing with multiple versions of an * \n"; - title = title + " * ODK survey that must be combined in one common database. * \n"; - title = title + " * * \n"; - title = title + " * If a TNS nor a FNS are NOT encounter then this means that B can * \n"; - title = title + " * merge into A and a diff SQL script is issued. * \n"; - title = title + " * * \n"; - title = title + " * Decrimental changes are not taken into account because this means * \n"; - title = title + " * losing data between versions. * \n"; + title = title + " * Compare Create XML * \n"; + title = title + " * This tool compares two create XML files (A and B) for incremental * \n"; + title = title + " * changes. A is consider an incremental version of B . * \n"; + title = title + " * * \n"; + title = title + " * The tool informs of tables and variables in A that are not in B. * \n"; + title = title + " * The tool can also create a combined file C that appends * \n"; + title = title + " * not found tables and variables with the following conditions: * \n"; + title = title + " * 1) If a table in A is not found in B then it will be added to B * \n"; + title = title + " * only if its parent exists in B. * \n"; + title = title + " * * \n"; + title = title + " * C will have all of B plus all of A. * \n"; + title = title + " * * \n"; + title = title + " * The tool WILL NOT fix the following: * \n"; + title = title + " * 1) Inconsistencies in field definition like size, type, * \n"; + title = title + " * parent table and parent field. * \n"; + title = title + " * 2) Tables that do not share the same parent. * \n"; + title = title + " * * \n"; + title = title + " * Nomenclature: * \n"; + title = title + " * TNF: Table not found. * \n"; + title = title + " * TNS: The table does not have the same parent table. (Cannot merge) * \n"; + title = title + " * FNF: Field not found. * \n"; + title = title + " * FNS: The field is not the same. (Cannot merge) * \n"; + title = title + " * FIC: The field will increase in size. * \n"; + title = title + " * FDC: The field decreased in in size. (Cannot merge) * \n"; + title = title + " * * \n"; + title = title + " * This tool is usefull when dealing with multiple versions of an * \n"; + title = title + " * ODK survey that must be combined in one common database. * \n"; + title = title + " * * \n"; + title = title + " * If a TNS nor a FNS are NOT encounter then this means that A can * \n"; + title = title + " * merge into B and a diff SQL script is issued. * \n"; + title = title + " * * \n"; + title = title + " * Decrimental changes are not taken into account because this means * \n"; + title = title + " * losing data between versions. * \n"; title = title + " ********************************************************************* \n"; TCLAP::CmdLine cmd(title.toUtf8().constData(), ' ', "2.0"); From 1af84c20b4405e9b2a2e42c49bf8372ae342eed6 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Fri, 16 Aug 2019 11:15:03 -0600 Subject: [PATCH 03/24] Bump create schema to 2.0. Fix some attributes --- JXFormToMysql/main.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index 3d5780346..89503f0f8 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -1654,7 +1654,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr XMLSchemaStructure = QDomDocument("XMLSchemaStructure"); QDomElement XMLRoot; XMLRoot = XMLSchemaStructure.createElement("XMLSchemaStructure"); - XMLRoot.setAttribute("version", "1.0"); + XMLRoot.setAttribute("version", "2.0"); XMLSchemaStructure.appendChild(XMLRoot); QDomElement XMLLKPTables; @@ -1938,7 +1938,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr { createFieldNode.setAttribute("rlookup","true"); if (tables[pos].fields[clm].mergeFine) - createFieldNode.setAttribute("mergeFine","true"); + createFieldNode.setAttribute("mergefine","true"); } } @@ -1969,7 +1969,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr { createFieldNode.setAttribute("rlookup","true"); if (tables[pos].fields[clm].mergeFine) - createFieldNode.setAttribute("mergeFine","true"); + createFieldNode.setAttribute("mergefine","true"); } } tables[pos].tableCreteElement.appendChild(createFieldNode); @@ -2001,7 +2001,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr { createFieldNode.setAttribute("rlookup","true"); if (tables[pos].fields[clm].mergeFine) - createFieldNode.setAttribute("mergeFine","true"); + createFieldNode.setAttribute("mergefine","true"); } } tables[pos].tableCreteElement.appendChild(createFieldNode); @@ -3528,7 +3528,7 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) lkpTable.fields.append(lkpDesc); aField.rTable = lkpTable.name; aField.rField = lkpCode.name; - aField.rName = getUUIDCode(); + aField.rName = getUUIDCode(); tables.append(lkpTable); } else @@ -3918,7 +3918,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q lkpTable.fields.append(lkpDesc); aField.rTable = lkpTable.name; aField.rField = lkpCode.name; - aField.rName = getUUIDCode(); + aField.rName = getUUIDCode(); tables.append(lkpTable); } else @@ -4887,7 +4887,7 @@ void loadMergingCreateFile(QDomNode lkptable, QDomNode insertStartNode) TtableDef lkpTable; lkpTable.isLoop = false; lkpTable.isOSM = false; - lkpTable.isGroup = false; + lkpTable.isGroup = false; lkpTable.name = tableName; @@ -5658,6 +5658,12 @@ int main(int argc, char *argv[]) mergingFile.close(); return 1; } + if (Createroot.attribute("version","1.0") != "2.0") + { + log("Cannot merge files. Only Version 2 are allowed for merging"); + mergingFile.close(); + return 1; + } QDomDocument mergingInsert("insert"); mergingFile.setFileName(mergeInsertFile); From b9d1693c2c685a484ca072f960987180eaf687d2 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Fri, 16 Aug 2019 11:26:08 -0600 Subject: [PATCH 04/24] Support more merging options --- utilities/compareCreateXML/main.cpp | 182 ++++++++++++++++++++-------- 1 file changed, 132 insertions(+), 50 deletions(-) diff --git a/utilities/compareCreateXML/main.cpp b/utilities/compareCreateXML/main.cpp index f4d873394..b044fc0f0 100644 --- a/utilities/compareCreateXML/main.cpp +++ b/utilities/compareCreateXML/main.cpp @@ -32,8 +32,7 @@ int idx; struct compError { - QString code; - QString desc; + QString code; QString table; QString field; QString from; @@ -60,6 +59,7 @@ typedef rtableDef TrtableDef; QList rtables; +QStringList dropTables; void addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int newDec) { @@ -83,6 +83,32 @@ void addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int new } } +void addTableToDrop(QString name) +{ + bool found = false; + if (dropTables.indexOf(name) >= 0) + found = true; + if (!found) + { + dropTables.append(name); + } +} + + +void changeLookupRelationship(QString table, QDomElement a, QDomElement b) +{ + QString sql; + QString oldcntname; + QString newcntname; + oldcntname = b.attribute("rname"); + newcntname = a.attribute("rname"); + sql = "ALTER TABLE " + table + " DROP CONSTRAINT " + oldcntname + ";\n\n"; + diff.append(sql); + sql = "ALTER TABLE " + table + " ADD CONSTRAINT " + newcntname + " FOREIGN KEY (" + a.attribute("name") + ") REFERENCES " + a.attribute("rtable") + " (" + a.attribute("rfield") + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n\n"; + diff.append(sql); + addTableToDrop(b.attribute("rtable")); +} + //This adds the modifications to a table to the diff list void addFieldToDiff(QString table, QDomElement eField) { @@ -99,13 +125,10 @@ void addFieldToDiff(QString table, QDomElement eField) } diff.append(sql); if (eField.attribute("rtable","") != "") - { - QUuid recordUUID=QUuid::createUuid(); - QString strRecordUUID=recordUUID.toString().replace("{","").replace("}","").right(12); - - sql = "ALTER TABLE " + table + " ADD INDEX DIDX" + strRecordUUID + " (" + eField.attribute("name","") + ");\n"; + { + sql = "ALTER TABLE " + table + " ADD INDEX " + eField.attribute("rname","") + " (" + eField.attribute("name","") + ");\n"; diff.append(sql); - sql = "ALTER TABLE " + table + " ADD CONSTRAINT DFK" + strRecordUUID + " FOREIGN KEY ("; + sql = "ALTER TABLE " + table + " ADD CONSTRAINT " + eField.attribute("rname","") + " FOREIGN KEY ("; sql = sql + eField.attribute("name","") + ") REFERENCES " + eField.attribute("rtable","") + "("; sql = sql + eField.attribute("rfield","") + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n" ; diff.append(sql); @@ -256,11 +279,15 @@ void addTableToSDiff(QDomNode table, bool lookUp) if (hasRelatedTables) sql = sql.left(sql.length()-2); sql = sql + ")\n ENGINE = InnoDB CHARSET=utf8 COMMENT = \"" + eTable.attribute("desc","") + "\";\n"; - sql = sql + "CREATE UNIQUE INDEX DXROWUUID" + strRecordUUID + " ON " + eTable.attribute("name","") + "(rowuuid);\n"; - if (lookUp) - sql = sql + "CREATE TRIGGER uudi_" + eTable.attribute("name","") + " BEFORE INSERT ON " + eTable.attribute("name","") + " FOR EACH ROW SET new.rowuuid = uuid();\n\n"; - else - sql = sql + "\n"; + sql = sql + "CREATE UNIQUE INDEX DXROWUUID" + strRecordUUID + " ON " + eTable.attribute("name","") + "(rowuuid);\n\n"; + + QUuid TriggerUUID=QUuid::createUuid(); + QString strTriggerUUID=TriggerUUID.toString().replace("{","").replace("}",""); + + sql = sql + "delimiter $$\n\n"; + sql = sql + "CREATE TRIGGER T" + strTriggerUUID + " BEFORE INSERT ON " + eTable.attribute("name","") + " FOR EACH ROW BEGIN IF (new.rowuuid IS NULL) THEN SET new.rowuuid = uuid(); ELSE IF (new.rowuuid NOT REGEXP '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}') THEN SET new.rowuuid = uuid(); END IF; END IF; END;$$\n\n"; + sql = sql + "delimiter ;\n\n"; + diff.append(sql); for (int pos = 0; pos <= childTables.count()-1;pos++) addTableToSDiff(childTables[pos],lookUp); @@ -310,13 +337,18 @@ QDomNode findTable(QDomDocument docB,QString tableName) QString compareFields(QDomElement a, QDomElement b, int &newSize, int &newDec) { newSize = a.attribute("size","0").toInt(); - newDec = a.attribute("decsize","0").toInt(); + newDec = a.attribute("decsize","0").toInt(); if (a.attribute("key","") != b.attribute("key","")) return "KNS"; - if (a.attribute("rtable","") != b.attribute("rtable","")) - return "RTNS"; - if (a.attribute("rfield","") != b.attribute("rfield","")) - return "RFNS"; + if ((a.attribute("rtable","") != b.attribute("rtable","")) || (a.attribute("rfield","") != b.attribute("rfield",""))) + { + if (a.attribute("mergefine","false") == "true") + { + return "CHR"; + } + else + return "RNS"; + } if (a.attribute("type","") != b.attribute("type","")) return "TNS"; @@ -396,18 +428,29 @@ void checkField(QDomNode eTable, QDomElement a, QDomElement b) result = compareFields(a,b,newSize,newDec); if (result != "") { - if (result == "FIC") + if ((result == "FIC") || (result == "CHR")) { - if (outputType == "h") + if (result == "FIC") { - if (a.attribute("type","") != "decimal") - log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0")); - else - log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0") + ". Decimal size from " + b.attribute("decsize",0) + " to " + a.attribute("decsize","0")); + if (outputType == "h") + { + if (a.attribute("type","") != "decimal") + log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0")); + else + log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0") + ". Decimal size from " + b.attribute("decsize",0) + " to " + a.attribute("decsize","0")); + } + b.setAttribute("size",newSize); + b.setAttribute("decsize",newDec); + addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); + } + else + { + if (outputType == "h") + { + log("CHR: The relationship of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be changed from " + b.attribute("rtable") + "." + b.attribute("rfield") + " to " + a.attribute("rtable") + "." + a.attribute("rfield")); + } + changeLookupRelationship(eTable.toElement().attribute("name",""),a,b); } - b.setAttribute("size",newSize); - b.setAttribute("decsize",newDec); - addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); } else { @@ -415,26 +458,23 @@ void checkField(QDomNode eTable, QDomElement a, QDomElement b) { if (result == "KNS") fatal("KNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed key from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); - if (result == "RTNS") - fatal("RTNS: The relational table for field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); - if (result == "RFNS") - fatal("RFNS: The relational field for field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); + if (result == "RNS") + fatal("RNS: The relationship for field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); if (result == "TNS") fatal("TNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); if (result == "FDC") log("FDC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " has descreased in size. This change will be ignored"); - } - else - { - TcompError error; - error.code = result; - error.table = eTable.toElement().attribute("name",""); - error.field = a.attribute("name",""); - error.desc = ""; - error.from = getFieldDefinition(b); - error.to = getFieldDefinition(a); - errorList.append(error); - } + } + } + if (outputType != "h") + { + TcompError error; + error.code = result; + error.table = eTable.toElement().attribute("name",""); + error.field = a.attribute("name",""); + error.from = getFieldDefinition(b); + error.to = getFieldDefinition(a); + errorList.append(error); } } } @@ -466,6 +506,16 @@ void compareLKPTables(QDomNode table,QDomDocument &docB) { if (outputType == "h") log("FNF:Field " + eField.attribute("name","") + " in table " + tablefound.toElement().attribute("name","") + " from A is not found in B"); + else + { + TcompError error; + error.code = "FNF"; + error.table = tablefound.toElement().attribute("name",""); + error.field = eField.attribute("name",""); + error.from = "NULL"; + error.to = getFieldDefinition(eField); + errorList.append(error); + } addFieldToDiff(tablefound.toElement().attribute("name",""),eField); tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); } @@ -478,6 +528,16 @@ void compareLKPTables(QDomNode table,QDomDocument &docB) { if (outputType == "h") log("TNF:Lookup table " + node.toElement().attribute("name","") + " from A not found in B"); + else + { + TcompError error; + error.code = "TNF"; + error.table = "NA"; + error.field = "NA"; + error.from = "NULL"; + error.to = node.toElement().attribute("name",""); + errorList.append(error); + } addTableToSDiff(node,true); //Now adds the lookup table docB.documentElement().firstChild().appendChild(node.cloneNode(true)); @@ -514,6 +574,16 @@ void compareTables(QDomNode table,QDomDocument &docB) { if (outputType == "h") log("FNF:Field " + eField.attribute("name","") + " in table " + tablefound.toElement().attribute("name","") + " from A is not found in B"); + else + { + TcompError error; + error.code = "FNF"; + error.table = tablefound.toElement().attribute("name",""); + error.field = eField.attribute("name",""); + error.from = "NULL"; + error.to = getFieldDefinition(eField); + errorList.append(error); + } addFieldToDiff(tablefound.toElement().attribute("name",""),eField); tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); } @@ -531,8 +601,7 @@ void compareTables(QDomNode table,QDomDocument &docB) TcompError error; error.code = "TNS"; error.table = "NA"; - error.field = "NA"; - error.desc = "Table " + eTable.toElement().attribute("name","") + " from A does not have the same parent in B"; + error.field = "NA"; error.from = tablefound.parentNode().toElement().attribute("name",""); error.to = table.parentNode().toElement().attribute("name",""); errorList.append(error); @@ -550,20 +619,29 @@ void compareTables(QDomNode table,QDomDocument &docB) { if (outputType == "h") log("TNF:Table " + eTable.toElement().attribute("name","") + " from A not found in B"); + else + { + TcompError error; + error.code = "TNF"; + error.table = "NA"; + error.field = "NA"; + error.from = "NULL"; + error.to = eTable.toElement().attribute("name",""); + errorList.append(error); + } addTableToSDiff(eTable,false); parentfound.appendChild(table.cloneNode(true)); } else { if (outputType == "h") - fatal("TNF:Table " + eTable.toElement().attribute("name","") + " from A not found in B. Its parent in A is not found in B"); + fatal("TWP:Table " + eTable.toElement().attribute("name","") + " from A not found in B. Its parent in A is not found in B"); else { TcompError error; - error.code = "TNF"; + error.code = "TWP"; error.table = "NA"; error.field = "NA"; - error.desc = "Table " + eTable.toElement().attribute("name","") + " from A not found in B. Its parent in A is not found in B"; error.from = "NULL"; error.to = parentTableName; errorList.append(error); @@ -683,6 +761,11 @@ int main(int argc, char *argv[]) compareLKPTables(rootA.firstChild().firstChild(),docB); //Comparing tables compareTables(rootA.firstChild().nextSibling().firstChild(),docB); + //Process drops + for (int pos = 0; pos < dropTables.count(); pos++) + { + diff.append("DROP TABLE IF EXISTS " + dropTables[pos] + ";\n\n"); + } if (outputType == "m") { @@ -702,8 +785,7 @@ int main(int argc, char *argv[]) anError = XMLResult.createElement("error"); anError.setAttribute("table",errorList[pos].table); anError.setAttribute("field",errorList[pos].field); - anError.setAttribute("code",errorList[pos].code); - anError.setAttribute("desc",errorList[pos].desc); + anError.setAttribute("code",errorList[pos].code); anError.setAttribute("from",errorList[pos].from); anError.setAttribute("to",errorList[pos].to); eErrors.appendChild(anError); From 1e142b7f15866f13af7a0075c31464316e7a8a56 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Fri, 16 Aug 2019 11:26:35 -0600 Subject: [PATCH 05/24] Allow audit for certaint tables --- utilities/createAuditTriggers/main.cpp | 5 ++++- utilities/createAuditTriggers/mainclass.cpp | 12 ++++++++++-- utilities/createAuditTriggers/mainclass.h | 3 ++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/utilities/createAuditTriggers/main.cpp b/utilities/createAuditTriggers/main.cpp index a6c554163..2d04b3226 100644 --- a/utilities/createAuditTriggers/main.cpp +++ b/utilities/createAuditTriggers/main.cpp @@ -21,6 +21,7 @@ int main(int argc, char *argv[]) TCLAP::ValueArg passArg("p","password","MySQL Password",true,"","string"); TCLAP::ValueArg schemaArg("s","schema","MySQL Schema",true,"","string"); TCLAP::ValueArg outputArg("o","output","Output directory",false,".","string"); + TCLAP::ValueArg tablesArg("t","tables","Coma separated list of tables to generare audit. Empty (default) means all",false,"","string"); //These two parameters should be removed once the external script code works @@ -33,6 +34,7 @@ int main(int argc, char *argv[]) cmd.add(passArg); cmd.add(schemaArg); cmd.add(outputArg); + cmd.add(tablesArg); //Parsing the command lines cmd.parse( argc, argv ); @@ -45,10 +47,11 @@ int main(int argc, char *argv[]) QString password = QString::fromUtf8(passArg.getValue().c_str()); QString schema = QString::fromUtf8(schemaArg.getValue().c_str()); QString output = QString::fromUtf8(outputArg.getValue().c_str()); + QString tables = QString::fromUtf8(tablesArg.getValue().c_str()); mainClass *task = new mainClass(&app); - task->setParameters(host,port,user,password,schema,output); + task->setParameters(host,port,user,password,schema,output,tables.split(",",QString::SkipEmptyParts)); QObject::connect(task, SIGNAL(finished()), &app, SLOT(quit())); diff --git a/utilities/createAuditTriggers/mainclass.cpp b/utilities/createAuditTriggers/mainclass.cpp index 77fc2f4fa..7eaeab98b 100644 --- a/utilities/createAuditTriggers/mainclass.cpp +++ b/utilities/createAuditTriggers/mainclass.cpp @@ -9,7 +9,7 @@ mainClass::mainClass(QObject *parent) : QObject(parent) returnCode = 0; } -void mainClass::setParameters(QString host, QString port, QString user, QString pass, QString schema, QString outputDirectory) +void mainClass::setParameters(QString host, QString port, QString user, QString pass, QString schema, QString outputDirectory, QStringList tables) { this->host = host; this->port = port; @@ -17,6 +17,8 @@ void mainClass::setParameters(QString host, QString port, QString user, QString this->pass = pass; this->schema = schema; this->outputDirectory = outputDirectory; + for (int pos = 0; pos < tables.count(); pos++) + this->tables.append("'" + tables[pos] + "'"); } void mainClass::log(QString message) @@ -72,7 +74,13 @@ int mainClass::createAudit(QSqlDatabase mydb, QString auditDir, QStringList igno TriggerLite << "PRIMARY KEY (audit_id) );"; TriggerLite << ""; - sql = "SELECT table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE' and TABLE_SCHEMA = '" + schema + "'"; //Excluse views in audit + if (tables.count() == 0) + sql = "SELECT table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE' and TABLE_SCHEMA = '" + schema + "'"; //Excluse views in audit + else + { + sql = "SELECT table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE' and TABLE_SCHEMA = '" + schema + "'"; + sql = sql + " AND table_name IN (" + tables.join(",") + ")"; + } if (query.exec(sql)) { while (query.next()) diff --git a/utilities/createAuditTriggers/mainclass.h b/utilities/createAuditTriggers/mainclass.h index 28576a40b..fb12bcbb4 100644 --- a/utilities/createAuditTriggers/mainclass.h +++ b/utilities/createAuditTriggers/mainclass.h @@ -10,7 +10,7 @@ class mainClass : public QObject Q_OBJECT public: explicit mainClass(QObject *parent = nullptr); - void setParameters(QString host, QString port, QString user, QString pass, QString schema, QString outputDirectory); + void setParameters(QString host, QString port, QString user, QString pass, QString schema, QString outputDirectory, QStringList tables); int returnCode; signals: void finished(); @@ -23,6 +23,7 @@ public slots: QString pass; QString schema; QString outputDirectory; + QStringList tables; void log(QString message); int createAudit(QSqlDatabase mydb, QString auditDir, QStringList ignoreTables); From 8e65862a19e287f8bc6d35acdd4a1783919274e2 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Sat, 17 Aug 2019 21:41:10 -0600 Subject: [PATCH 06/24] Fix merge bugs --- JXFormToMysql/main.cpp | 67 ++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index 89503f0f8..fce4f1ea5 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -1338,11 +1338,28 @@ int getMaxValueLength(QList values) return res; } +//Return whether the values of a lookup table are numbers or strings. Used to determine the type of variables in the lookup tables +bool areValuesStrings(QList values) +{ + bool ok; + int pos; + for (pos = 0; pos <= values.count()-1;pos++) + { + values[pos].code.toInt(&ok,10); + if (!ok) + { + return true; + } + } + return false; +} + //This fuction checkd wheter a lookup table is duplicated. //If there is a match then returns such table -TtableDef getDuplicatedLkpTable(QString lkptname, QList thisValues, bool &changeSize, int &newCodeSize, bool &changedRel, bool useMergedTables = true) +TtableDef getDuplicatedLkpTable(QString lkptname, QList thisValues, bool &changeSize, int &newCodeSize, bool &changedRel, QString newCodeType, bool useMergedTables = true) { changeSize = false; + newCodeType = ""; newCodeSize = -1; int pos2; bool same; @@ -1451,19 +1468,29 @@ TtableDef getDuplicatedLkpTable(QString lkptname, QList thisValues, b bool nchangeSize; int nnewCodeSize; bool nchangedRel; - TtableDef existing_table = getDuplicatedLkpTable(lkptname, merging_tables[pos].lkpValues,nchangeSize,nnewCodeSize,nchangedRel,false); + QString nnewCodeType; + TtableDef existing_table = getDuplicatedLkpTable(lkptname, merging_tables[pos].lkpValues,nchangeSize,nnewCodeSize,nchangedRel,nnewCodeType,false); if (existing_table.name != "EMPTY") { changeSize = true; changedRel = true; newCodeSize = getMaxValueLength(existing_table.lkpValues); + if (areValuesStrings(existing_table.lkpValues)) + newCodeType = "varchar"; + else + newCodeType = "int"; return existing_table; } changeSize = true; + if (areValuesStrings(merging_tables[pos].lkpValues)) + newCodeType = "varchar"; + else + newCodeType = "int"; newCodeSize = getMaxValueLength(merging_tables[pos].lkpValues); int newDescSize; newDescSize = getMaxDescLength(merging_tables[pos].lkpValues); merging_tables[pos].fields[0].size = newCodeSize; + merging_tables[pos].fields[0].type = newCodeType; merging_tables[pos].fields[1].size = newDescSize; tables.append(merging_tables[pos]); return merging_tables[pos]; @@ -2645,22 +2672,6 @@ TfieldMap mapODKFieldTypeToMySQL(QString ODKFieldType) return result; } -//Return whether the values of a lookup table are numbers or strings. Used to determine the type of variables in the lookup tables -bool areValuesStrings(QList values) -{ - bool ok; - int pos; - for (pos = 0; pos <= values.count()-1;pos++) - { - values[pos].code.toInt(&ok,10); - if (!ok) - { - return true; - } - } - return false; -} - int getMaxMSelValueLength(QList values) { QString res; @@ -3472,7 +3483,8 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) bool changeSize; int newCodeSize; bool changedRel; - TtableDef lkpTable = getDuplicatedLkpTable(table_name,values,changeSize,newCodeSize,changedRel); + QString newCodeType; + TtableDef lkpTable = getDuplicatedLkpTable(table_name,values,changeSize,newCodeSize,changedRel,newCodeType); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -3540,6 +3552,8 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) aField.rName = getUUIDCode(); if (merge) { + if (newCodeType != "") + aField.type = newCodeType; if (changeSize) { aField.size = newCodeSize; @@ -3862,7 +3876,8 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q bool changeSize; int newCodeSize; bool changedRel; - TtableDef lkpTable = getDuplicatedLkpTable(table_name, values, changeSize, newCodeSize, changedRel); + QString newCodeType; + TtableDef lkpTable = getDuplicatedLkpTable(table_name, values, changeSize, newCodeSize, changedRel, newCodeType); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -3930,6 +3945,8 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q aField.rName = getUUIDCode(); if (merge) { + if (newCodeType != "") + aField.type = newCodeType; if (changeSize) { aField.size = newCodeSize; @@ -4089,7 +4106,8 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q bool changeSize; int newCodeSize; bool changedRel; - TtableDef lkpTable = getDuplicatedLkpTable(table_name,values,changeSize,newCodeSize,changedRel); + QString newCodeType; + TtableDef lkpTable = getDuplicatedLkpTable(table_name,values,changeSize,newCodeSize,changedRel,newCodeType); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -4162,6 +4180,8 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q mselKeyField.rName = getUUIDCode(); if (merge) { + if (newCodeType != "") + aField.type = newCodeType; if (changeSize) { aField.size = getMaxMSelValueLength(lkpTable.lkpValues); @@ -4415,7 +4435,8 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f bool changeSize; int newCodeSize; bool changedRel; - TtableDef lkpTable = getDuplicatedLkpTable(table_name, values, changeSize, newCodeSize, changedRel); + QString newCodeType; + TtableDef lkpTable = getDuplicatedLkpTable(table_name, values, changeSize, newCodeSize, changedRel, newCodeType); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -4483,6 +4504,8 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f aField.rName = getUUIDCode(); if (merge) { + if (newCodeType != "") + aField.type = newCodeType; if (changeSize) { aField.size = newCodeSize; From 339fc50ad274d5d622a95643119fcb326eb97026 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Sat, 17 Aug 2019 21:41:33 -0600 Subject: [PATCH 07/24] Fix merge bugs --- utilities/compareCreateXML/main.cpp | 72 ++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/utilities/compareCreateXML/main.cpp b/utilities/compareCreateXML/main.cpp index b044fc0f0..7c3996995 100644 --- a/utilities/compareCreateXML/main.cpp +++ b/utilities/compareCreateXML/main.cpp @@ -102,7 +102,12 @@ void changeLookupRelationship(QString table, QDomElement a, QDomElement b) QString newcntname; oldcntname = b.attribute("rname"); newcntname = a.attribute("rname"); - sql = "ALTER TABLE " + table + " DROP CONSTRAINT " + oldcntname + ";\n\n"; + sql = "ALTER TABLE " + table + " DROP CONSTRAINT " + oldcntname + ";\n"; + diff.append(sql); + sql = "ALTER TABLE " + table + " DROP INDEX " + oldcntname + ";\n\n"; + diff.append(sql); + addAlterFieldToDiff(table,a,a.attribute("size","0").toInt(),0); + sql = "ALTER TABLE " + table + " ADD INDEX " + newcntname + ";\n"; diff.append(sql); sql = "ALTER TABLE " + table + " ADD CONSTRAINT " + newcntname + " FOREIGN KEY (" + a.attribute("name") + ") REFERENCES " + a.attribute("rtable") + " (" + a.attribute("rfield") + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n\n"; diff.append(sql); @@ -282,7 +287,7 @@ void addTableToSDiff(QDomNode table, bool lookUp) sql = sql + "CREATE UNIQUE INDEX DXROWUUID" + strRecordUUID + " ON " + eTable.attribute("name","") + "(rowuuid);\n\n"; QUuid TriggerUUID=QUuid::createUuid(); - QString strTriggerUUID=TriggerUUID.toString().replace("{","").replace("}",""); + QString strTriggerUUID=TriggerUUID.toString().replace("{","").replace("}","").replace("-","_"); sql = sql + "delimiter $$\n\n"; sql = sql + "CREATE TRIGGER T" + strTriggerUUID + " BEFORE INSERT ON " + eTable.attribute("name","") + " FOR EACH ROW BEGIN IF (new.rowuuid IS NULL) THEN SET new.rowuuid = uuid(); ELSE IF (new.rowuuid NOT REGEXP '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}') THEN SET new.rowuuid = uuid(); END IF; END IF; END;$$\n\n"; @@ -350,7 +355,24 @@ QString compareFields(QDomElement a, QDomElement b, int &newSize, int &newDec) return "RNS"; } if (a.attribute("type","") != b.attribute("type","")) - return "TNS"; + { + if ((b.attribute("type") == "int") && (a.attribute("type") == "varchar")) + { + if (a.attribute("size","0").toInt() >= b.attribute("size","0").toInt()) + { + if (a.attribute("decsize","0").toInt() > b.attribute("decsize","0").toInt()) + { + newSize = a.attribute("size","0").toInt(); + newDec = a.attribute("decsize","0").toInt(); + } + return "FTC"; + } + else + return "FNS"; + } + else + return "FNS"; + } if (a.attribute("size","") != b.attribute("size","")) { @@ -428,7 +450,7 @@ void checkField(QDomNode eTable, QDomElement a, QDomElement b) result = compareFields(a,b,newSize,newDec); if (result != "") { - if ((result == "FIC") || (result == "CHR")) + if ((result == "FIC") || (result == "CHR") || (result == "FTC")) { if (result == "FIC") { @@ -445,11 +467,29 @@ void checkField(QDomNode eTable, QDomElement a, QDomElement b) } else { - if (outputType == "h") + if (result == "CHR") + { + if (outputType == "h") + { + log("CHR: The relationship of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be changed from " + b.attribute("rtable") + "." + b.attribute("rfield") + " to " + a.attribute("rtable") + "." + a.attribute("rfield")); + } + changeLookupRelationship(eTable.toElement().attribute("name",""),a,b); + b.setAttribute("rtable",a.attribute("rtable")); + b.setAttribute("rfield",a.attribute("rfield")); + b.setAttribute("type",a.attribute("type")); + b.setAttribute("size",a.attribute("size")); + } + else { - log("CHR: The relationship of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be changed from " + b.attribute("rtable") + "." + b.attribute("rfield") + " to " + a.attribute("rtable") + "." + a.attribute("rfield")); + if (outputType == "h") + { + log("FTC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from " + b.attribute("type") + " to " + a.attribute("type") + " which is allowed"); + b.setAttribute("size",newSize); + b.setAttribute("decsize",newDec); + b.setAttribute("type",a.attribute("type")); + addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); + } } - changeLookupRelationship(eTable.toElement().attribute("name",""),a,b); } } else @@ -460,8 +500,8 @@ void checkField(QDomNode eTable, QDomElement a, QDomElement b) fatal("KNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed key from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); if (result == "RNS") fatal("RNS: The relationship for field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); - if (result == "TNS") - fatal("TNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); + if (result == "FNS") + fatal("FNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); if (result == "FDC") log("FDC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " has descreased in size. This change will be ignored"); } @@ -674,10 +714,11 @@ int main(int argc, char *argv[]) title = title + " * Nomenclature: * \n"; title = title + " * TNF: Table not found. * \n"; title = title + " * TNS: The table does not have the same parent table. (Cannot merge) * \n"; + title = title + " * TWP: The parent table is not found in B. (Cannot merge). * \n"; title = title + " * FNF: Field not found. * \n"; title = title + " * FNS: The field is not the same. (Cannot merge) * \n"; title = title + " * FIC: The field will increase in size. * \n"; - title = title + " * FDC: The field decreased in in size. (Cannot merge) * \n"; + title = title + " * FDC: The field decreased in in size. (Ignore) * \n"; title = title + " * * \n"; title = title + " * This tool is usefull when dealing with multiple versions of an * \n"; title = title + " * ODK survey that must be combined in one common database. * \n"; @@ -762,9 +803,18 @@ int main(int argc, char *argv[]) //Comparing tables compareTables(rootA.firstChild().nextSibling().firstChild(),docB); //Process drops + QDomNode lkpTables = rootB.firstChild(); for (int pos = 0; pos < dropTables.count(); pos++) { - diff.append("DROP TABLE IF EXISTS " + dropTables[pos] + ";\n\n"); + diff.append("DROP TABLE IF EXISTS " + dropTables[pos] + ";\n\n"); + QDomNode a_table = rootB.firstChild().firstChild(); + while (!a_table.isNull()) + { + if (a_table.toElement().attribute("name") == dropTables[pos]) + break; + a_table = a_table.nextSibling(); + } + lkpTables.removeChild(a_table); } if (outputType == "m") From 807a04fea9af69b98f9ed7d68baf2690f9bd631d Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Wed, 21 Aug 2019 08:03:54 -0600 Subject: [PATCH 08/24] New tool to merge versions. Replaces CompareCreate and CompareInsert --- .gitignore | 1 + utilities/mergeVersions/compareinsert.cpp | 386 ++++++++++ utilities/mergeVersions/compareinsert.h | 63 ++ utilities/mergeVersions/generic.h | 45 ++ utilities/mergeVersions/main.cpp | 121 +++ utilities/mergeVersions/mainclass.cpp | 124 +++ utilities/mergeVersions/mainclass.h | 50 ++ utilities/mergeVersions/mergeVersions.pro | 29 + utilities/mergeVersions/mergecreate.cpp | 892 ++++++++++++++++++++++ utilities/mergeVersions/mergecreate.h | 91 +++ utilities/utilities.pro | 7 +- 11 files changed, 1805 insertions(+), 4 deletions(-) create mode 100644 utilities/mergeVersions/compareinsert.cpp create mode 100644 utilities/mergeVersions/compareinsert.h create mode 100644 utilities/mergeVersions/generic.h create mode 100644 utilities/mergeVersions/main.cpp create mode 100644 utilities/mergeVersions/mainclass.cpp create mode 100644 utilities/mergeVersions/mainclass.h create mode 100644 utilities/mergeVersions/mergeVersions.pro create mode 100644 utilities/mergeVersions/mergecreate.cpp create mode 100644 utilities/mergeVersions/mergecreate.h diff --git a/.gitignore b/.gitignore index b1bbc6d7d..2d5a15e5a 100644 --- a/.gitignore +++ b/.gitignore @@ -118,6 +118,7 @@ jxformtomysql createaudittriggers createdummyjson mysqltosqlite +mergeversions #Other directories old diff --git a/utilities/mergeVersions/compareinsert.cpp b/utilities/mergeVersions/compareinsert.cpp new file mode 100644 index 000000000..1bd7a29d6 --- /dev/null +++ b/utilities/mergeVersions/compareinsert.cpp @@ -0,0 +1,386 @@ +/* +Merge Versions + +Copyright (C) 2019 QLands Technology Consultants. +Author: Carlos Quiros (cquiros_at_qlands.com) + +Merge Versions is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. + +Merge Versions is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with Merge Versions. If not, see . +*/ + +#include "compareinsert.h" +#include + +compareInsert::compareInsert(QObject *parent) : QObject(parent) +{ + fatalError = false; +} + +void compareInsert::setFiles(QString insertA, QString insertB, QString insertC, QString diffSQL, QString outputType) +{ + inputA = insertA; + inputB = insertB; + outputC = insertC; + outputD = diffSQL; + this->outputType = outputType; +} + +QList compareInsert::getDiffs() +{ + QList res; + res.append(diff); + return res; +} + +void compareInsert::addDiffToTable(QString table, QString sql) +{ + bool found; + found = false; + for (int pos = 0; pos < diff.count(); pos++) + { + if (diff[pos].table == table) + { + diff[pos].diff.append(sql); + found = true; + break; + } + } + if (!found) + { + TtableDiff a_table; + a_table.table = table; + a_table.diff.append(sql); + diff.append(a_table); + } +} + +void compareInsert::setAsParsed(QString table) +{ + for (int pos = 0; pos < diff.count(); pos++) + { + if (diff[pos].table == table) + { + diff[pos].parsed = true; + } + } +} + +int compareInsert::createCFile() +{ + if (!fatalError) + { + //Create the insert XML file. If exist it get overwriten + if (QFile::exists(outputC)) + QFile::remove(outputC); + QFile file(outputC); + if (file.open(QIODevice::WriteOnly | QIODevice::Text)) + { + QTextStream out(&file); + out.setCodec("UTF-8"); + docB.save(out,1,QDomNode::EncodingFromTextStream); + file.close(); + } + else + { + log("Error: Cannot create XML combined file"); + return 1; + } + return 0; + } + else + return 1; +} + +int compareInsert::createDiffFile() +{ + if (!fatalError) + { + if (QFile::exists(outputD)) + QFile::remove(outputD); + QFile dfile(outputD); + if (dfile.open(QIODevice::WriteOnly | QIODevice::Text)) + { + QTextStream outD(&dfile); + outD.setCodec("UTF-8"); + for (int tpos = 0; tpos < diff.count();tpos++) + { + if (diff[tpos].parsed == false) + { + diff[tpos].parsed = true; + for (int dpos = 0; dpos < diff[tpos].diff.count(); dpos++) + { + outD << diff[tpos].diff[dpos] + "\n"; + } + } + } + dfile.close(); + return 0; + } + else + { + log("Error: Cannot create Diff file"); + return 1; + } + } + else + return 1; +} + +QList compareInsert::getErrorList() +{ + return errorList; +} + +int compareInsert::compare() +{ + fatalError = false; + if (inputA != inputB) + { + if ((QFile::exists(inputA)) && (QFile::exists(inputB))) + { + //Openning and parsing input file A + QDomDocument docA("inputA"); + QFile fileA(inputA); + if (!fileA.open(QIODevice::ReadOnly)) + { + log("Cannot open input file A"); + return 1; + } + if (!docA.setContent(&fileA)) + { + log("Cannot parse document for input file A"); + fileA.close(); + return 1; + } + fileA.close(); + + //Openning and parsing input file B + QFile fileB(inputB); + if (!fileB.open(QIODevice::ReadOnly)) + { + log("Cannot open input file B"); + return 1; + } + if (!docB.setContent(&fileB)) + { + log("Cannot parse document for input file B"); + fileB.close(); + return 1; + } + fileB.close(); + + QDomElement rootA = docA.documentElement(); + QDomElement rootB = docB.documentElement(); + if ((rootA.tagName() == "insertValuesXML") && (rootB.tagName() == "insertValuesXML")) + { + compareLKPTables(rootA.firstChild(),docB); + } + else + { + if (!(rootA.tagName() == "ODKImportXML")) + { + log("Input document A is not a insert XML file"); + return 1; + } + if (!(rootB.tagName() == "ODKImportXML")) + { + log("Input document B is not a insert XML file"); + return 1; + } + } + + } + else + { + if (!QFile::exists(inputA)) + { + log("Input file A does not exists"); + return 1; + } + if (!QFile::exists(inputB)) + { + log("Input file B does not exists"); + return 1; + } + } + } + else + { + log("Input files A and B are the same. No point in comparing them."); + return 1; + } + if (!fatalError) + return 0; + else + return 1; +} + +//This logs messages to the terminal. We use printf because qDebug does not log in relase +void compareInsert::log(QString message) +{ + QString temp; + temp = message + "\n"; + printf("%s",temp.toUtf8().data()); +} + +void compareInsert::fatal(QString message) +{ + fatalError = true; + fprintf(stderr, "\033[31m%s\033[0m \n", message.toUtf8().data()); +} + +QDomNode compareInsert::findTable(QDomDocument docB,QString tableName) +{ + QDomNodeList tables; + tables = docB.elementsByTagName("table"); + for (int pos = 0; pos < tables.count();pos++) + { + if (tables.item(pos).toElement().attribute("name","") == tableName) + return tables.item(pos); + } + QDomNode null; + return null; +} + +QDomNode compareInsert::findValue(QDomNode table,QString code) +{ + QDomNode node; + node = table.firstChild(); + while (!node.isNull()) + { + if (node.toElement().attribute("code","") == code) + return node; + node = node.nextSibling(); + } + QDomNode null; + return null; +} + +void compareInsert::addValueToDiff(QDomElement table, QDomElement field) +{ + QString sql; + sql = "INSERT INTO " + table.attribute("name","") + " ("; + sql = sql + table.attribute("clmcode","") + ","; + sql = sql + table.attribute("clmdesc","") + ") VALUES ('"; + sql = sql + field.attribute("code","") + "','"; + sql = sql + field.attribute("description","") + "');"; + addDiffToTable(table.attribute("name",""),sql); +} + +void compareInsert::UpdateValue(QDomElement table, QDomElement field) +{ + QString sql; + sql = "UPDATE " + table.attribute("name","") + " SET "; + sql = sql + table.attribute("clmdesc","") + " = '"; + sql = sql + field.attribute("description","") + "' WHERE "; + sql = sql + table.attribute("clmcode","") + " = '"; + sql = sql + field.attribute("code","") + "';"; + addDiffToTable(table.attribute("name",""),sql); +} + +void compareInsert::addTableToDiff(QDomElement table) +{ + QDomNode field; + field = table.firstChild(); + while (!field.isNull()) + { + addValueToDiff(table,field.toElement()); + field = field.nextSibling(); + } +} + +void compareInsert::changeValueInC(QDomNode table, QString code, QString newDescription) +{ + QDomNode field; + field = table.firstChild(); + while (!field.isNull()) + { + QDomElement efield; + efield = field.toElement(); + if (efield.attribute("code","") == code) + { + efield.setAttribute("description",newDescription); + return; + } + field = field.nextSibling(); + } +} + +void compareInsert::compareLKPTables(QDomNode table,QDomDocument &docB) +{ + QDomNode node; + node = table; + while (!node.isNull()) + { + QDomNode tableFound = findTable(docB,node.toElement().attribute("name","")); + if (!tableFound.isNull()) + { + QDomNode field = node.firstChild(); + while (!field.isNull()) + { + QDomNode fieldFound = findValue(tableFound,field.toElement().attribute("code","")); + if (!fieldFound.isNull()) + { + if (field.toElement().attribute("description","") != fieldFound.toElement().attribute("description","")) + { + if (outputType == "h") + { + fatal("VNS:Value " + field.toElement().attribute("code","") + " of lookup table " + node.toElement().attribute("name","") + " has changed from \"" + fieldFound.toElement().attribute("description","") + "\" to \"" + field.toElement().attribute("description","") + "\""); + log("Do you want to change this value in the database? Y/N"); + std::string line; + std::getline(std::cin, line); + QString result = QString::fromStdString(line); + if (std::cin.eof() || result.toLower() == "y") + { + fatalError = false; + UpdateValue(node.toElement(),field.toElement()); + changeValueInC(tableFound,field.toElement().attribute("code",""),field.toElement().attribute("description","")); + } + else + { + fatalError = true; + } + } + { + TcompError error; + error.code = "VNS"; + error.desc = "Value " + field.toElement().attribute("code","") + " of lookup table " + node.toElement().attribute("name","") + " from A not the same in B"; + error.table = node.toElement().attribute("name",""); + error.value = field.toElement().attribute("code",""); + error.from = fieldFound.toElement().attribute("description",""); + error.to = field.toElement().attribute("description",""); + errorList.append(error); + } + } + } + else + { + if (outputType == "h") + log("VNF:Value " + field.toElement().attribute("code","") + "(" + field.toElement().attribute("description","") + ") will be included in table " + node.toElement().attribute("name","")); + tableFound.appendChild(field.cloneNode(true)); + addValueToDiff(node.toElement(),field.toElement()); + } + field = field.nextSibling(); + } + } + else + { + if (outputType == "h") + log("TNF:The lookup table " + node.toElement().attribute("name","") + " will be included in the database."); + //Now adds the lookup table + addTableToDiff(node.toElement()); + docB.documentElement().appendChild(node.cloneNode(true)); + } + node = node.nextSibling(); + } +} diff --git a/utilities/mergeVersions/compareinsert.h b/utilities/mergeVersions/compareinsert.h new file mode 100644 index 000000000..64f3ba3f2 --- /dev/null +++ b/utilities/mergeVersions/compareinsert.h @@ -0,0 +1,63 @@ +/* +Merge Versions + +Copyright (C) 2019 QLands Technology Consultants. +Author: Carlos Quiros (cquiros_at_qlands.com) + +Merge Versions is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. + +Merge Versions is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with Merge Versions. If not, see . +*/ + +#ifndef COMPAREINSERT_H +#define COMPAREINSERT_H + +#include +#include +#include +#include "generic.h" +#include + +class compareInsert : public QObject +{ +public: + explicit compareInsert(QObject *parent = nullptr); + void setFiles(QString insertA, QString insertB, QString insertC, QString diffSQL, QString outputType); + int compare(); + int createCFile(); + int createDiffFile(); + void setAsParsed(QString table); + QList getDiffs(); + QList getErrorList(); +private: + QString inputA; + QString inputB; + QString outputC; + QString outputD; + QString outputType; + bool fatalError; + QDomDocument docB; + QList diff; + QList errorList; + void log(QString message); + void fatal(QString message); + QDomNode findTable(QDomDocument docB,QString tableName); + QDomNode findValue(QDomNode table,QString code); + void addValueToDiff(QDomElement table, QDomElement field); + void UpdateValue(QDomElement table, QDomElement field); + void addTableToDiff(QDomElement table); + void changeValueInC(QDomNode table, QString code, QString newDescription); + void compareLKPTables(QDomNode table,QDomDocument &docB); + void addDiffToTable(QString table, QString sql); +}; + +#endif // COMPAREINSERT_H diff --git a/utilities/mergeVersions/generic.h b/utilities/mergeVersions/generic.h new file mode 100644 index 000000000..3187f0689 --- /dev/null +++ b/utilities/mergeVersions/generic.h @@ -0,0 +1,45 @@ +/* +Merge Versions + +Copyright (C) 2019 QLands Technology Consultants. +Author: Carlos Quiros (cquiros_at_qlands.com) + +Merge Versions is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. + +Merge Versions is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with Merge Versions. If not, see . +*/ + +#ifndef GENERIC_H +#define GENERIC_H +#include + +struct tableDiff +{ + QString table; + QStringList diff; + bool parsed = false; +}; +typedef tableDiff TtableDiff; + +struct compError +{ + QString code; + QString desc; + QString table; + QString field; + QString value; + QString from; + QString to; +}; +typedef compError TcompError; + +#endif // GENERIC_H diff --git a/utilities/mergeVersions/main.cpp b/utilities/mergeVersions/main.cpp new file mode 100644 index 000000000..ef95117f4 --- /dev/null +++ b/utilities/mergeVersions/main.cpp @@ -0,0 +1,121 @@ +/* +Merge Versions + +Copyright (C) 2019 QLands Technology Consultants. +Author: Carlos Quiros (cquiros_at_qlands.com) + +Merge Versions is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. + +Merge Versions is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with Merge Versions. If not, see . +*/ + +#include +#include +#include +#include "mainclass.h" + +int main(int argc, char *argv[]) +{ + QCoreApplication a(argc, argv); + + QString title; + title = title + "************************************************************************ "; + title = title + " * Merge Versions. Version 2.0 * "; + title = title + " * This tool merges two versions of an ODK (A & B) using the XML * "; + title = title + " * files created by JXFormToMySQL (create.xml and insert.xml). * "; + title = title + " * * "; + title = title + " * A is considered an incremental version of B. The tool informs * "; + title = title + " * of tables, variables and values in A that are not in B and adds them * "; + title = title + " * in a file called C that will have all of B + all of A. * "; + title = title + " * * "; + title = title + " * Nomenclature: * "; + title = title + " * Changes allowed from B to A: * "; + title = title + " * TNF: Table in A not found in B. It will be added to C. * "; + title = title + " * FNF: Field in A not found in B. It will be addded to C. * "; + title = title + " * VNF: Lookup value in A not found in B. It will be addded to C. * "; + title = title + " * FIC: The field size in A is bigger than in B. It will be increased * "; + title = title + " * in C to match the size of in A. * "; + title = title + " * FDC: The field in A is smaller than in B. This is a decrimental * "; + title = title + " * change thus will be ignored. * "; + title = title + " * CHR: The referenced lookup table+field in A is different than in B.* "; + title = title + " * however the new looup table in A has the same values or more * "; + title = title + " * than the previous one in B. The referenced lookup table+field * "; + title = title + " * will be changed in C. * "; + title = title + " * FTC: Field type changed from int in B to varchar A. This is * "; + title = title + " * allowed beause a varchar can hold numeric characters. The * "; + title = title + " * field will be changed to varchar in C. * "; + title = title + " * * "; + title = title + " * Changes NOT allowed from B to A: * "; + title = title + " * TNS: The table does not have the same parent table. * "; + title = title + " * TWP: The parent table is not found in B. * "; + title = title + " * FNS: The field is not the same and such change will generate * "; + title = title + " * iconsistencies in the data. An example is to change a * "; + title = title + " * variable from categorical to continuous. * "; + title = title + " * VNS: The description of a lookup value changed from B to A. For * "; + title = title + " * example 1-Male in B changed to 1-Female in A. * "; + title = title + " * * "; + title = title + " * This tool is usefull when dealing with multiple versions of an * "; + title = title + " * ODK survey that must be combined in one common database. * "; + title = title + " * * "; + title = title + " * If all changes are allowed, this means that A can merge into B and * "; + title = title + " * a C file is created. Also a diff SQL script is issued. * "; + title = title + " * * "; + title = title + " * Decrimental changes are not taken into account because this means * "; + title = title + " * losing data between versions. * "; + title = title + " ************************************************************************ "; + + TCLAP::CmdLine cmd(title.toUtf8().constData(), ' ', "2.0"); + + TCLAP::ValueArg aArg("a","inputa","Input create XML file A (later)",true,"","string"); + TCLAP::ValueArg bArg("b","inputb","Input create XML file B (former)",true,"","string"); + TCLAP::ValueArg AArg("A","inputA","Input insert XML file A (later)",true,"","string"); + TCLAP::ValueArg BArg("B","inputB","Input insert XML file B (former)",true,"","string"); + TCLAP::ValueArg cArg("c","outputc","Output create XML file C",false,"./combined-create.xml","string"); + TCLAP::ValueArg CArg("C","outputC","Output insert XML file C",false,"./combined-insert.xml","string"); + TCLAP::ValueArg dArg("d","diff","Output diff SQL script for create",false,"./diff-create.sql","string"); + TCLAP::ValueArg DArg("D","diff","Output diff SQL script for insert",false,"./diff-insert.sql","string"); + TCLAP::ValueArg oArg("o","outputype","Output type: (h)uman readble or (m)achine readble",false,"m","string"); + + cmd.add(aArg); + cmd.add(bArg); + cmd.add(AArg); + cmd.add(BArg); + cmd.add(cArg); + cmd.add(CArg); + cmd.add(dArg); + cmd.add(oArg); + + + //Parsing the command lines + cmd.parse( argc, argv ); + + //Getting the variables from the command + QString a_createXML = QString::fromUtf8(aArg.getValue().c_str()); + QString b_createXML = QString::fromUtf8(bArg.getValue().c_str()); + QString a_insertXML = QString::fromUtf8(AArg.getValue().c_str()); + QString b_insertXML = QString::fromUtf8(BArg.getValue().c_str()); + + QString c_createXML = QString::fromUtf8(cArg.getValue().c_str()); + QString c_insertXML = QString::fromUtf8(CArg.getValue().c_str()); + QString outputd = QString::fromUtf8(dArg.getValue().c_str()); + QString outputD = QString::fromUtf8(DArg.getValue().c_str()); + QString outputType = QString::fromUtf8(oArg.getValue().c_str()); + + mainClass *task = new mainClass(&a); + task->setParameters(a_createXML,b_createXML,a_insertXML,b_insertXML,c_createXML,c_insertXML,outputd,outputD,outputType); + QObject::connect(task, SIGNAL(finished()), &a, SLOT(quit())); + + QTimer::singleShot(0, task, SLOT(run())); + + return a.exec(); + return task->returnCode; +} diff --git a/utilities/mergeVersions/mainclass.cpp b/utilities/mergeVersions/mainclass.cpp new file mode 100644 index 000000000..55f7f6560 --- /dev/null +++ b/utilities/mergeVersions/mainclass.cpp @@ -0,0 +1,124 @@ +/* +Merge Versions + +Copyright (C) 2019 QLands Technology Consultants. +Author: Carlos Quiros (cquiros_at_qlands.com) + +Merge Versions is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. + +Merge Versions is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with Merge Versions. If not, see . +*/ + +#include "mainclass.h" +#include "mergecreate.h" +#include "compareinsert.h" + +mainClass::mainClass(QObject *parent) : QObject(parent) +{ + returnCode = 0; +} + +void mainClass::log(QString message) +{ + QString temp; + temp = message + "\n"; + printf("%s",temp.toUtf8().data()); +} + +void mainClass::run() +{ + QDomDocument XMLResult; + QDomElement XMLRoot; + QDomElement eErrors; + if (output_type != "h") + { + XMLResult = QDomDocument("XMLResult"); + XMLRoot = XMLResult.createElement("XMLResult"); + XMLResult.appendChild(XMLRoot); + eErrors = XMLResult.createElement("errors"); + XMLRoot.appendChild(eErrors); + } + + compareInsert insert; + insert.setFiles(a_insertXML,b_insertXML,c_insertXML,d_insertSQL,output_type); + returnCode = insert.compare(); + if (output_type != "h") + { + QList errorList = insert.getErrorList(); + for (int pos = 0; pos < errorList.count(); pos++) + { + QDomElement anError; + anError = XMLResult.createElement("error"); + anError.setAttribute("table",errorList[pos].table); + anError.setAttribute("field",errorList[pos].field); + anError.setAttribute("code",errorList[pos].code); + anError.setAttribute("from",errorList[pos].from); + anError.setAttribute("to",errorList[pos].to); + eErrors.appendChild(anError); + } + } + if (returnCode == 0) + { + returnCode = insert.createCFile(); + if (returnCode == 0) + { + mergeCreate create; + create.setFiles(a_createXML,b_createXML,c_createXML,d_createSQL,output_type); + create.setInsertDiff(insert.getDiffs()); + returnCode = create.compare(); + if (output_type != "h") + { + QList errorList = create.getErrorList(); + for (int pos = 0; pos < errorList.count(); pos++) + { + QDomElement anError; + anError = XMLResult.createElement("error"); + anError.setAttribute("table",errorList[pos].table); + anError.setAttribute("field",errorList[pos].field); + anError.setAttribute("code",errorList[pos].code); + anError.setAttribute("from",errorList[pos].from); + anError.setAttribute("to",errorList[pos].to); + eErrors.appendChild(anError); + } + } + if (returnCode == 0) + { + QStringList insertsUsed = create.getInsertTablesUsed(); + for (int pos = 0; pos < insertsUsed.count(); pos++) + { + insert.setAsParsed(insertsUsed[pos]); + } + } + } + } + if (returnCode == 0) + { + returnCode = insert.createDiffFile(); + } + if (output_type != "h") + { + log(XMLResult.toString()); + } + emit finished(); +} +void mainClass::setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType) +{ + a_createXML = createA; + b_createXML = createB; + a_insertXML = insertA; + b_insertXML = insertB; + c_createXML = createC; + c_insertXML = insertC; + d_createSQL = diffCreate; + d_insertSQL = diffInsert; + output_type = outputType; +} diff --git a/utilities/mergeVersions/mainclass.h b/utilities/mergeVersions/mainclass.h new file mode 100644 index 000000000..6508faa8d --- /dev/null +++ b/utilities/mergeVersions/mainclass.h @@ -0,0 +1,50 @@ +/* +Merge Versions + +Copyright (C) 2019 QLands Technology Consultants. +Author: Carlos Quiros (cquiros_at_qlands.com) + +Merge Versions is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. + +Merge Versions is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with Merge Versions. If not, see . +*/ + +#ifndef MAINCLASS_H +#define MAINCLASS_H + +#include + +class mainClass : public QObject +{ + Q_OBJECT +public: + explicit mainClass(QObject *parent = nullptr); + int returnCode; +signals: + void finished(); +public slots: + void run(); + void setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType); +private: + QString a_createXML; + QString b_createXML ; + QString a_insertXML; + QString b_insertXML; + QString c_createXML; + QString c_insertXML ; + QString d_createSQL; + QString d_insertSQL; + QString output_type; + void log(QString message); +}; + +#endif // MAINCLASS_H diff --git a/utilities/mergeVersions/mergeVersions.pro b/utilities/mergeVersions/mergeVersions.pro new file mode 100644 index 000000000..40fa5d754 --- /dev/null +++ b/utilities/mergeVersions/mergeVersions.pro @@ -0,0 +1,29 @@ +QT -= gui +QT += core xml +CONFIG += c++11 console +CONFIG -= app_bundle +TARGET = mergeversions + +# The following define makes your compiler emit warnings if you use +# any feature of Qt which as been marked deprecated (the exact warnings +# depend on your compiler). Please consult the documentation of the +# deprecated API in order to know how to port your code away from it. +DEFINES += QT_DEPRECATED_WARNINGS + +# You can also make your code fail to compile if you use deprecated APIs. +# In order to do so, uncomment the following line. +# You can also select to disable deprecated APIs only up to a certain version of Qt. +#DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000 # disables all the APIs deprecated before Qt 6.0.0 + +INCLUDEPATH += ../../3rdparty + +SOURCES += main.cpp \ + mainclass.cpp \ + mergecreate.cpp \ + compareinsert.cpp + +HEADERS += \ + mainclass.h \ + mergecreate.h \ + compareinsert.h \ + generic.h diff --git a/utilities/mergeVersions/mergecreate.cpp b/utilities/mergeVersions/mergecreate.cpp new file mode 100644 index 000000000..16790a26b --- /dev/null +++ b/utilities/mergeVersions/mergecreate.cpp @@ -0,0 +1,892 @@ +/* +Merge Versions + +Copyright (C) 2019 QLands Technology Consultants. +Author: Carlos Quiros (cquiros_at_qlands.com) + +Merge Versions is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. + +Merge Versions is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with Merge Versions. If not, see . +*/ + +#include "mergecreate.h" + +mergeCreate::mergeCreate(QObject *parent) : QObject(parent) +{ + fatalError = false; + idx = 1; +} + +void mergeCreate::setInsertDiff(QList diff) +{ + insert_diff.append(diff); +} + +QStringList mergeCreate::getInsertTablesUsed() +{ + return insertTablesUsed; +} + +int mergeCreate::compare() +{ + fatalError = false; + idx = 1; + + if (inputA != inputB) + { + if ((QFile::exists(inputA)) && (QFile::exists(inputB))) + { + //Openning and parsing input file A + QDomDocument docA("inputA"); + QFile fileA(inputA); + if (!fileA.open(QIODevice::ReadOnly)) + { + log("Cannot open input file A"); + return 1; + } + if (!docA.setContent(&fileA)) + { + log("Cannot parse document for input file A"); + fileA.close(); + return 1; + } + fileA.close(); + + //Openning and parsing input file B + QDomDocument docB("inputB"); + QFile fileB(inputB); + if (!fileB.open(QIODevice::ReadOnly)) + { + log("Cannot open input file B"); + return 1; + } + if (!docB.setContent(&fileB)) + { + log("Cannot parse document for input file B"); + fileB.close(); + return 1; + } + fileB.close(); + + QDomElement rootA = docA.documentElement(); + QDomElement rootB = docB.documentElement(); + if ((rootA.tagName() == "XMLSchemaStructure") && (rootB.tagName() == "XMLSchemaStructure")) + { + //Comparing lookup tables + compareLKPTables(rootA.firstChild().firstChild(),docB); + //Comparing tables + compareTables(rootA.firstChild().nextSibling().firstChild(),docB); + //Process drops + QDomNode lkpTables = rootB.firstChild(); + for (int pos = 0; pos < dropTables.count(); pos++) + { + diff.append("DROP TABLE IF EXISTS " + dropTables[pos] + ";\n\n"); + QDomNode a_table = rootB.firstChild().firstChild(); + while (!a_table.isNull()) + { + if (a_table.toElement().attribute("name") == dropTables[pos]) + break; + a_table = a_table.nextSibling(); + } + lkpTables.removeChild(a_table); + } + + //Do not create C and the Diff SQL if the merge cannot be done + if (fatalError == false) + { + //Create the output file. If exist it get overwriten + if (QFile::exists(outputC)) + QFile::remove(outputC); + QFile file(outputC); + if (file.open(QIODevice::WriteOnly | QIODevice::Text)) + { + QTextStream out(&file); + out.setCodec("UTF-8"); + docB.save(out,1,QDomNode::EncodingFromTextStream); + file.close(); + } + else + { + log("Error: Cannot create XML combined file"); + return 1; + } + + if (QFile::exists(outputD)) + QFile::remove(outputD); + QFile dfile(outputD); + if (dfile.open(QIODevice::WriteOnly | QIODevice::Text)) + { + QTextStream outD(&dfile); + outD.setCodec("UTF-8"); + for (int dpos = 0; dpos < diff.count();dpos++) + { + outD << diff[dpos]; + } + file.close(); + } + else + { + log("Error: Cannot create Diff file"); + return 1; + } + } + else + return 1; + } + else + { + if (!(rootA.tagName() == "ODKImportXML")) + { + log("Input document A is not a create XML file"); + return 1; + } + if (!(rootB.tagName() == "ODKImportXML")) + { + log("Input document B is not a create XML file"); + return 1; + } + } + + } + else + { + if (!QFile::exists(inputA)) + { + log("Input file A does not exists"); + return 1; + } + if (!QFile::exists(inputB)) + { + log("Input file B does not exists"); + return 1; + } + } + } + else + { + log("Input files A and B are the same. No point in comparing them."); + return 1; + } + if (!fatalError) + return 0; + else + return 1; +} + +void mergeCreate::setFiles(QString createA, QString createB, QString createC, QString diffSQL, QString outputType) +{ + inputA = createA; + inputB = createB; + outputC = createC; + outputD = diffSQL; + this->outputType = outputType; +} + +void mergeCreate::addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int newDec) +{ + QString sql; + QString fieldName; + fieldName = eField.attribute("name",""); + if (eField.attribute("type","") == "varchar") + { + sql = "ALTER TABLE " + table + " MODIFY " + fieldName + " varchar(" + QString::number(newSize) + ");\n"; + diff.append(sql); + } + if (eField.attribute("type","") == "int") + { + sql = "ALTER TABLE " + table + " MODIFY " + fieldName + " int(" + QString::number(newSize) + ");\n"; + diff.append(sql); + } + if (eField.attribute("type","") == "decimal") + { + sql = "ALTER TABLE " + table + " MODIFY " + fieldName + " decimal(" + QString::number(newSize) + "," + QString::number(newDec) + ");\n"; + diff.append(sql); + } +} + +void mergeCreate::ddTableToDrop(QString name) +{ + bool found = false; + if (dropTables.indexOf(name) >= 0) + found = true; + if (!found) + { + dropTables.append(name); + } +} + +void mergeCreate::changeLookupRelationship(QString table, QDomElement a, QDomElement b) +{ + QString sql; + QString oldcntname; + QString newcntname; + oldcntname = b.attribute("rname"); + newcntname = a.attribute("rname"); + sql = "ALTER TABLE " + table + " DROP CONSTRAINT " + oldcntname + ";\n"; + diff.append(sql); + sql = "ALTER TABLE " + table + " DROP INDEX " + oldcntname + ";\n\n"; + diff.append(sql); + addAlterFieldToDiff(table,a,a.attribute("size","0").toInt(),0); + sql = "ALTER TABLE " + table + " ADD INDEX " + newcntname + ";\n"; + diff.append(sql); + for (int pos = 0; pos < insert_diff.count(); pos++) + { + if (insert_diff[pos].table == a.attribute("rtable")) + { + diff.append("\n"); + for (int pos2 = 0; pos2 < insert_diff[pos].diff.count(); pos2++) + { + diff.append(insert_diff[pos].diff[pos2] + "\n"); + } + insertTablesUsed.append(a.attribute("rtable")); + diff.append("\n"); + } + } + sql = "ALTER TABLE " + table + " ADD CONSTRAINT " + newcntname + " FOREIGN KEY (" + a.attribute("name") + ") REFERENCES " + a.attribute("rtable") + " (" + a.attribute("rfield") + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n\n"; + diff.append(sql); + addTableToDrop(b.attribute("rtable")); +} + +void mergeCreate::addTableToDrop(QString name) +{ + bool found = false; + if (dropTables.indexOf(name) >= 0) + found = true; + if (!found) + { + dropTables.append(name); + } +} + +//This adds the modifications to a table to the diff list +void mergeCreate::addFieldToDiff(QString table, QDomElement eField) +{ + QString sql; + sql = "ALTER TABLE " + table + " ADD COLUMN " + eField.attribute("name",""); + if (eField.attribute("type","") == "decimal") + sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + "," + eField.attribute("decsize","0") + ");\n"; + else + { + if (eField.attribute("type","") != "text") + sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + ");\n"; + else + sql = sql + " " + eField.attribute("type","") + ";\n"; + } + diff.append(sql); + if (eField.attribute("rtable","") != "") + { + sql = "ALTER TABLE " + table + " ADD INDEX " + eField.attribute("rname","") + " (" + eField.attribute("name","") + ");\n"; + diff.append(sql); + sql = "ALTER TABLE " + table + " ADD CONSTRAINT " + eField.attribute("rname","") + " FOREIGN KEY ("; + sql = sql + eField.attribute("name","") + ") REFERENCES " + eField.attribute("rtable","") + "("; + sql = sql + eField.attribute("rfield","") + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n" ; + diff.append(sql); + idx++; + } +} + +void mergeCreate::addFieldToRTables(QString parentTable, QString rTable, QString field, QString rField, QString rname, bool isLookUp) +{ + int tidx; + tidx = -1; + int pos; + for (pos = 0; pos < rtables.count(); pos++) + { + if ((rtables[pos].parentTable == parentTable) && (rtables[pos].name == rTable)) + tidx = pos; + } + if (tidx != -1) + { + TrfieldDef aField; + aField.name = field; + aField.rname = rField; + aField.rcode = rname; + aField.isLookUp = isLookUp; + rtables[tidx].rfields.append(aField); + } + else + { + TrtableDef table; + table.parentTable = parentTable; + table.name = rTable; + TrfieldDef aField; + aField.name = field; + aField.rname = rField; + aField.rcode = rname; + aField.isLookUp = isLookUp; + table.rfields.append(aField); + rtables.append(table); + } +} + +//This adds a table to the diff list +void mergeCreate::addTableToSDiff(QDomNode table, bool lookUp) +{ + QUuid recordUUID=QUuid::createUuid(); + QString strRecordUUID=recordUUID.toString().replace("{","").replace("}","").right(12); + + QDomElement eTable = table.toElement(); + QString sql; + sql = "CREATE TABLE IF NOT EXISTS " + eTable.attribute("name","") + "(\n"; + QDomNode node = table.firstChild(); + QStringList keys; + QList childTables; + while (!node.isNull()) + { + QDomElement field = node.toElement(); + if (field.tagName() == "field") + { + sql = sql + field.attribute("name",""); + + if (field.attribute("type","") == "decimal") + sql = sql + " " + field.attribute("type","") + " (" + field.attribute("size","0") + "," + field.attribute("decsize","0") + ")"; + else + { + if (field.attribute("type","") != "text") + sql = sql + " " + field.attribute("type","") + " (" + field.attribute("size","0") + ")"; + else + sql = sql + " " + field.attribute("type",""); + } + if (field.attribute("key","") == "true") + { + sql = sql + "NOT NULL COMMENT \"" + field.attribute("desc","Without description") + "\",\n"; + keys.append(field.attribute("name","")); + } + else + sql = sql + " COMMENT \"" + field.attribute("desc","Without description") + "\",\n"; + if (field.attribute("rtable","") != "") + { + bool isLookup = false; + if (field.attribute("rlookup","false") == "true") + isLookup = true; + addFieldToRTables(eTable.attribute("name",""),field.attribute("rtable",""),field.attribute("name",""),field.attribute("rfield",""),field.attribute("rname",""),isLookup); + } + } + else + { + childTables.append(node); + } + node = node.nextSibling(); + } + //sql = sql + "rowuuid varchar(80) COMMENT \"Unique Row Identifier (UUID)\",\n"; + int pos; + int pos2; + //Add the keys + sql = sql + "PRIMARY KEY ("; + for (pos = 0; pos < keys.count();pos++) + { + sql = sql + keys[pos] + ","; + } + bool hasRelatedTables; + hasRelatedTables = false; + for (pos = 0; pos < rtables.count();pos++) + { + if (rtables[pos].parentTable == eTable.attribute("name","")) + { + hasRelatedTables = true; + break; + } + } + if (hasRelatedTables) + sql = sql.left(sql.length()-1) + "),\n"; + else + sql = sql.left(sql.length()-1) + ")"; + //Add the indexes + for (pos = 0; pos < rtables.count();pos++) + { + if (rtables[pos].parentTable == eTable.attribute("name","")) + { + bool isLookup = false; + QString rname; + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + if (rtables[pos].rfields[pos2].isLookUp) + { + isLookup = true; + rname = rtables[pos].rfields[pos2].rcode; + break; + } + } + + if (!isLookup) + { + QUuid idxUUID=QUuid::createUuid(); + QString strIdxUUID=idxUUID.toString().replace("{","").replace("}","").right(12); + sql = sql + "INDEX DIDX" + strIdxUUID + "("; + } + else + { + sql = sql + "INDEX " + rname + "("; + } + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + sql = sql + rtables[pos].rfields[pos2].name + ","; + } + sql = sql.left(sql.length()-1) + "),\n"; + idx++; + + } + } + //Add foreign keys + for (pos = 0; pos < rtables.count();pos++) + { + if (rtables[pos].parentTable == eTable.attribute("name","")) + { + bool isLookup = false; + QString rname; + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + if (rtables[pos].rfields[pos2].isLookUp) + { + isLookup = true; + rname = rtables[pos].rfields[pos2].rcode; + break; + } + } + if (!isLookup) + { + QUuid cntUUID=QUuid::createUuid(); + QString strCntUUID=cntUUID.toString().replace("{","").replace("}","").right(12); + sql = sql + "CONSTRAINT DFK" + strCntUUID + "\n"; + } + else + { + sql = sql + "CONSTRAINT " + rname + "\n"; + } + sql = sql + "FOREIGN KEY ("; + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + sql = sql + rtables[pos].rfields[pos2].name + ","; + } + sql = sql.left(sql.length()-1) + ")\n"; + sql = sql + "REFERENCES " + rtables[pos].name + "("; + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + sql = sql + rtables[pos].rfields[pos2].rname + ","; + } + sql = sql.left(sql.length()-1) + ")\nON DELETE RESTRICT\nON UPDATE NO ACTION,\n"; + idx++; + } + } + if (hasRelatedTables) + sql = sql.left(sql.length()-2); + sql = sql + ")\n ENGINE = InnoDB CHARSET=utf8 COMMENT = \"" + eTable.attribute("desc","") + "\";\n"; + sql = sql + "CREATE UNIQUE INDEX DXROWUUID" + strRecordUUID + " ON " + eTable.attribute("name","") + "(rowuuid);\n\n"; + + QUuid TriggerUUID=QUuid::createUuid(); + QString strTriggerUUID=TriggerUUID.toString().replace("{","").replace("}","").replace("-","_"); + + sql = sql + "delimiter $$\n\n"; + sql = sql + "CREATE TRIGGER T" + strTriggerUUID + " BEFORE INSERT ON " + eTable.attribute("name","") + " FOR EACH ROW BEGIN IF (new.rowuuid IS NULL) THEN SET new.rowuuid = uuid(); ELSE IF (new.rowuuid NOT REGEXP '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}') THEN SET new.rowuuid = uuid(); END IF; END IF; END;$$\n\n"; + sql = sql + "delimiter ;\n\n"; + + diff.append(sql); + for (int pos = 0; pos <= childTables.count()-1;pos++) + addTableToSDiff(childTables[pos],lookUp); +} + +void mergeCreate::log(QString message) +{ + QString temp; + temp = message + "\n"; + printf("%s",temp.toUtf8().data()); +} + +QList mergeCreate::getErrorList() +{ + return errorList; +} + +void mergeCreate::fatal(QString message) +{ + fprintf(stderr, "\033[31m%s\033[0m \n", message.toUtf8().data()); + fatalError = true; +} + +QDomNode mergeCreate::findField(QDomNode table,QString field) +{ + QDomNode node; + node = table.firstChild(); + while (!node.isNull()) + { + if (node.toElement().attribute("name","") == field) + return node; + node = node.nextSibling(); + } + QDomNode null; + return null; +} + +QDomNode mergeCreate::findTable(QDomDocument docB,QString tableName) +{ + QDomNodeList tables; + tables = docB.elementsByTagName("table"); + for (int pos = 0; pos < tables.count();pos++) + { + if (tables.item(pos).toElement().attribute("name","") == tableName) + return tables.item(pos); + } + QDomNode null; + return null; +} + +QString mergeCreate::compareFields(QDomElement a, QDomElement b, int &newSize, int &newDec) +{ + newSize = a.attribute("size","0").toInt(); + newDec = a.attribute("decsize","0").toInt(); + if (a.attribute("key","") != b.attribute("key","")) + return "KNS"; + if ((a.attribute("rtable","") != b.attribute("rtable","")) || (a.attribute("rfield","") != b.attribute("rfield",""))) + { + if (a.attribute("mergefine","false") == "true") + { + return "CHR"; + } + else + return "RNS"; + } + if (a.attribute("type","") != b.attribute("type","")) + { + if ((b.attribute("type") == "int") && (a.attribute("type") == "varchar")) + { + if (a.attribute("size","0").toInt() >= b.attribute("size","0").toInt()) + { + if (a.attribute("decsize","0").toInt() > b.attribute("decsize","0").toInt()) + { + newSize = a.attribute("size","0").toInt(); + newDec = a.attribute("decsize","0").toInt(); + } + return "FTC"; + } + else + return "FNS"; + } + else + return "FNS"; + } + + if (a.attribute("size","") != b.attribute("size","")) + { + if (a.attribute("size","0").toInt() > b.attribute("size","0").toInt()) + { + if (a.attribute("type","") == "decimal") + { + if (a.attribute("decsize","0").toInt() != b.attribute("decsize","0").toInt()) + { + if (a.attribute("decsize","0").toInt() > b.attribute("decsize","0").toInt()) + { + newSize = a.attribute("size","0").toInt(); + newDec = a.attribute("decsize","0").toInt(); + return "FIC"; + } + else + { + if (a.attribute("decsize","0").toInt() < b.attribute("decsize","0").toInt()) + return "FDC"; + } + } + else + { + if (a.attribute("size","0").toInt() < b.attribute("size","0").toInt()) + return "FDC"; + } + } + else + { + newSize = a.attribute("size","0").toInt(); + return "FIC"; + } + } + else + { + if (a.attribute("size","0").toInt() < b.attribute("size","0").toInt()) + return "FDC"; + } + } + if (a.attribute("decsize","") != b.attribute("decsize","")) + { + if (a.attribute("decsize","0").toInt() > b.attribute("decsize","0").toInt()) + { + newSize = a.attribute("size","0").toInt(); + newDec = a.attribute("decsize","0").toInt(); + return "FIC"; + } + else + { + if (a.attribute("decsize","0").toInt() < b.attribute("decsize","0").toInt()) + return "FDC"; + } + } + + return ""; +} + +QString mergeCreate::getFieldDefinition(QDomElement field) +{ + QString result; + result = "[Key=" + field.attribute("key","false") + ","; + result = result + "Type=" + field.attribute("type","") + ","; + result = result + "Size=" + field.attribute("size","0") + ","; + result = result + "DecimalSize=" + field.attribute("decsize","0") + ","; + result = result + "RelatedTable=" + field.attribute("rtable","None") + ","; + result = result + "RelatedField=" + field.attribute("rfield","None") + "]"; + return result; +} + +void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b) +{ + int newSize; + int newDec; + QString result; + result = compareFields(a,b,newSize,newDec); + if (result != "") + { + if ((result == "FIC") || (result == "CHR") || (result == "FTC")) + { + if (result == "FIC") + { + if (outputType == "h") + { + if (a.attribute("type","") != "decimal") + log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0")); + else + log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0") + ". Decimal size from " + b.attribute("decsize",0) + " to " + a.attribute("decsize","0")); + } + b.setAttribute("size",newSize); + b.setAttribute("decsize",newDec); + addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); + } + else + { + if (result == "CHR") + { + if (outputType == "h") + { + log("CHR: The relationship of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be changed from " + b.attribute("rtable") + "." + b.attribute("rfield") + " to " + a.attribute("rtable") + "." + a.attribute("rfield")); + } + changeLookupRelationship(eTable.toElement().attribute("name",""),a,b); + b.setAttribute("rtable",a.attribute("rtable")); + b.setAttribute("rfield",a.attribute("rfield")); + b.setAttribute("type",a.attribute("type")); + b.setAttribute("size",a.attribute("size")); + b.setAttribute("rname",a.attribute("rname")); + } + else + { + if (outputType == "h") + { + log("FTC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from " + b.attribute("type") + " to " + a.attribute("type") + " which is allowed"); + b.setAttribute("size",newSize); + b.setAttribute("decsize",newDec); + b.setAttribute("type",a.attribute("type")); + addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); + } + } + } + } + else + { + if (outputType == "h") + { + if (result == "KNS") + fatal("KNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed key from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); + if (result == "RNS") + fatal("RNS: The relationship for field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); + if (result == "FNS") + fatal("FNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); + if (result == "FDC") + log("FDC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " has descreased in size. This change will be ignored"); + } + } + if (outputType != "h") + { + TcompError error; + error.code = result; + error.table = eTable.toElement().attribute("name",""); + error.field = a.attribute("name",""); + error.from = getFieldDefinition(b); + error.to = getFieldDefinition(a); + errorList.append(error); + } + } +} + +void mergeCreate::compareLKPTables(QDomNode table,QDomDocument &docB) +{ + QDomNode node; + node = table; + while (!node.isNull()) + { + QDomNode tablefound; + tablefound = findTable(docB,node.toElement().attribute("name","")); + if (!tablefound.isNull()) + { + QDomNode field = node.firstChild(); + while (!field.isNull()) + { + QDomElement eField = field.toElement(); + if (eField.tagName() == "table") + compareLKPTables(field,docB); + else + { + QDomNode fieldFound = findField(tablefound,eField.attribute("name","")); + if (!fieldFound.isNull()) + { + checkField(tablefound,field.toElement(),fieldFound.toElement()); + } + else + { + if (outputType == "h") + log("FNF:Field " + eField.attribute("name","") + " in table " + tablefound.toElement().attribute("name","") + " from A is not found in B"); + else + { + TcompError error; + error.code = "FNF"; + error.table = tablefound.toElement().attribute("name",""); + error.field = eField.attribute("name",""); + error.from = "NULL"; + error.to = getFieldDefinition(eField); + errorList.append(error); + } + addFieldToDiff(tablefound.toElement().attribute("name",""),eField); + tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); + } + } + + field = field.nextSibling(); + } + } + else + { + if (outputType == "h") + log("TNF:Lookup table " + node.toElement().attribute("name","") + " from A not found in B"); + else + { + TcompError error; + error.code = "TNF"; + error.table = "NA"; + error.field = "NA"; + error.from = "NULL"; + error.to = node.toElement().attribute("name",""); + errorList.append(error); + } + addTableToSDiff(node,true); + //Now adds the lookup table + docB.documentElement().firstChild().appendChild(node.cloneNode(true)); + } + node = node.nextSibling(); + } +} + +void mergeCreate::compareTables(QDomNode table,QDomDocument &docB) +{ + QDomElement eTable = table.toElement(); + QDomNode tablefound; + //log(eTable.toElement().attribute("name","")); + tablefound = findTable(docB,eTable.toElement().attribute("name","")); + if (!tablefound.isNull()) + { + if (table.parentNode().toElement().attribute("name","") == + tablefound.parentNode().toElement().attribute("name","")) + { + QDomNode field = table.firstChild(); + while (!field.isNull()) + { + QDomElement eField = field.toElement(); + if (eField.tagName() == "table") + compareTables(field,docB); + else + { + QDomNode fieldFound = findField(tablefound,eField.attribute("name","")); + if (!fieldFound.isNull()) + { + checkField(tablefound,field.toElement(),fieldFound.toElement()); + } + else + { + if (outputType == "h") + log("FNF:Field " + eField.attribute("name","") + " in table " + tablefound.toElement().attribute("name","") + " from A is not found in B"); + else + { + TcompError error; + error.code = "FNF"; + error.table = tablefound.toElement().attribute("name",""); + error.field = eField.attribute("name",""); + error.from = "NULL"; + error.to = getFieldDefinition(eField); + errorList.append(error); + } + addFieldToDiff(tablefound.toElement().attribute("name",""),eField); + tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); + } + } + + field = field.nextSibling(); + } + } + else + { + if (outputType == "h") + fatal("TNS:Table " + eTable.toElement().attribute("name","") + " from A does not have the same parent in B"); + else + { + TcompError error; + error.code = "TNS"; + error.table = "NA"; + error.field = "NA"; + error.from = tablefound.parentNode().toElement().attribute("name",""); + error.to = table.parentNode().toElement().attribute("name",""); + errorList.append(error); + } + } + } + else + { + //Now adds the table to doc2 + QDomNode parentfound; + QString parentTableName; + parentTableName = table.parentNode().toElement().attribute("name",""); + parentfound = findTable(docB,parentTableName); + if (!parentfound.isNull()) + { + if (outputType == "h") + log("TNF:Table " + eTable.toElement().attribute("name","") + " from A not found in B"); + else + { + TcompError error; + error.code = "TNF"; + error.table = "NA"; + error.field = "NA"; + error.from = "NULL"; + error.to = eTable.toElement().attribute("name",""); + errorList.append(error); + } + addTableToSDiff(eTable,false); + parentfound.appendChild(table.cloneNode(true)); + } + else + { + if (outputType == "h") + fatal("TWP:Table " + eTable.toElement().attribute("name","") + " from A not found in B. Its parent in A is not found in B"); + else + { + TcompError error; + error.code = "TWP"; + error.table = "NA"; + error.field = "NA"; + error.from = "NULL"; + error.to = parentTableName; + errorList.append(error); + } + } + } +} diff --git a/utilities/mergeVersions/mergecreate.h b/utilities/mergeVersions/mergecreate.h new file mode 100644 index 000000000..a6970d701 --- /dev/null +++ b/utilities/mergeVersions/mergecreate.h @@ -0,0 +1,91 @@ +/* +Merge Versions + +Copyright (C) 2019 QLands Technology Consultants. +Author: Carlos Quiros (cquiros_at_qlands.com) + +Merge Versions is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. + +Merge Versions is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with Merge Versions. If not, see . +*/ + +#ifndef MERGECREATE_H +#define MERGECREATE_H + +#include +#include +#include +#include +#include +#include "generic.h" + + + +class mergeCreate : public QObject +{ + struct rfieldDef + { + QString name; + QString rname; + QString rcode; + bool isLookUp = false; + }; + typedef rfieldDef TrfieldDef; + + struct rtableDef + { + QString parentTable; + QString name; + QList rfields; + }; + typedef rtableDef TrtableDef; + +public: + explicit mergeCreate(QObject *parent = nullptr); + int compare(); + void setFiles(QString createA, QString createB, QString createC, QString diffSQL, QString outputType); + void setInsertDiff(QList diff); + QStringList getInsertTablesUsed(); + QList getErrorList(); +private: + QString inputA; + QString inputB ; + QString outputC; + QString outputD; + bool fatalError; + QStringList diff; + QList insert_diff; + QString outputType; + int idx; + QList errorList; + QList rtables; + QStringList dropTables; + QStringList insertTablesUsed; + void addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int newDec); + void ddTableToDrop(QString name); + void changeLookupRelationship(QString table, QDomElement a, QDomElement b); + void addFieldToDiff(QString table, QDomElement eField); + void addFieldToRTables(QString parentTable, QString rTable, QString field, QString rField, QString rname, bool isLookUp); + void addTableToSDiff(QDomNode table, bool lookUp); + void log(QString message); + void fatal(QString message); + QDomNode findField(QDomNode table,QString field); + QDomNode findTable(QDomDocument docB,QString tableName); + QString compareFields(QDomElement a, QDomElement b, int &newSize, int &newDec); + QString getFieldDefinition(QDomElement field); + void checkField(QDomNode eTable, QDomElement a, QDomElement b); + void compareLKPTables(QDomNode table,QDomDocument &docB); + void compareTables(QDomNode table,QDomDocument &docB); + void addTableToDrop(QString name); +}; + +#endif // MERGECREATE_H diff --git a/utilities/utilities.pro b/utilities/utilities.pro index e4d5b3fec..233cdeb62 100644 --- a/utilities/utilities.pro +++ b/utilities/utilities.pro @@ -2,13 +2,12 @@ TEMPLATE = subdirs CONFIG += debug -SUBDIRS = compareCreateXML \ - compareInsertXML \ - createFromXML \ +SUBDIRS = createFromXML \ insertFromXML \ MySQLDenormalize \ JSONToCSV \ MySQLToXLSX \ createAuditTriggers \ createDummyJSON \ - MySQLToSQLite + MySQLToSQLite \ + mergeVersions From e56b7ee9b0dec067fb87c30e6b55c5d62b57be76 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Wed, 21 Aug 2019 08:04:41 -0600 Subject: [PATCH 09/24] Remove old compare tools. This is now replaced by the mergeVersions tool --- .../compareCreateXML/compareCreateXML.pro | 19 - utilities/compareCreateXML/main.cpp | 926 ------------------ .../compareInsertXML/compareInsertXML.pro | 19 - utilities/compareInsertXML/main.cpp | 394 -------- 4 files changed, 1358 deletions(-) delete mode 100644 utilities/compareCreateXML/compareCreateXML.pro delete mode 100644 utilities/compareCreateXML/main.cpp delete mode 100644 utilities/compareInsertXML/compareInsertXML.pro delete mode 100644 utilities/compareInsertXML/main.cpp diff --git a/utilities/compareCreateXML/compareCreateXML.pro b/utilities/compareCreateXML/compareCreateXML.pro deleted file mode 100644 index 38b7c43ec..000000000 --- a/utilities/compareCreateXML/compareCreateXML.pro +++ /dev/null @@ -1,19 +0,0 @@ -#------------------------------------------------- -# -# Project created by QtCreator 2015-09-22T09:49:34 -# -#------------------------------------------------- - -QT += core xml - -QT -= gui - -TARGET = comparecreatexml -CONFIG += console -CONFIG -= app_bundle - -TEMPLATE = app - -INCLUDEPATH += ../../3rdparty - -SOURCES += main.cpp diff --git a/utilities/compareCreateXML/main.cpp b/utilities/compareCreateXML/main.cpp deleted file mode 100644 index 7c3996995..000000000 --- a/utilities/compareCreateXML/main.cpp +++ /dev/null @@ -1,926 +0,0 @@ -/* -CompareCreateXML - -Copyright (C) 2015-2017 International Livestock Research Institute. -Author: Carlos Quiros (cquiros_at_qlands.com / c.f.quiros_at_cgiar.org) - -CompareCreateXML is free software: you can redistribute it and/or modify -it under the terms of the GNU Lesser General Public License as -published by the Free Software Foundation, either version 3 of -the License, or (at your option) any later version. - -CompareCreateXML is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with CompareCreateXML. If not, see . -*/ - -#include -#include -#include -#include -#include -#include - -bool fatalError; -QStringList diff; -QString outputType; -int idx; - -struct compError -{ - QString code; - QString table; - QString field; - QString from; - QString to; -}; -typedef compError TcompError; - -QList errorList; - -struct rfieldDef -{ - QString name; - QString rname; -}; -typedef rfieldDef TrfieldDef; - -struct rtableDef -{ - QString parentTable; - QString name; - QList rfields; -}; -typedef rtableDef TrtableDef; - -QList rtables; - -QStringList dropTables; - -void addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int newDec) -{ - QString sql; - QString fieldName; - fieldName = eField.attribute("name",""); - if (eField.attribute("type","") == "varchar") - { - sql = "ALTER TABLE " + table + " MODIFY " + fieldName + " varchar(" + QString::number(newSize) + ");\n"; - diff.append(sql); - } - if (eField.attribute("type","") == "int") - { - sql = "ALTER TABLE " + table + " MODIFY " + fieldName + " int(" + QString::number(newSize) + ");\n"; - diff.append(sql); - } - if (eField.attribute("type","") == "decimal") - { - sql = "ALTER TABLE " + table + " MODIFY " + fieldName + " decimal(" + QString::number(newSize) + "," + QString::number(newDec) + ");\n"; - diff.append(sql); - } -} - -void addTableToDrop(QString name) -{ - bool found = false; - if (dropTables.indexOf(name) >= 0) - found = true; - if (!found) - { - dropTables.append(name); - } -} - - -void changeLookupRelationship(QString table, QDomElement a, QDomElement b) -{ - QString sql; - QString oldcntname; - QString newcntname; - oldcntname = b.attribute("rname"); - newcntname = a.attribute("rname"); - sql = "ALTER TABLE " + table + " DROP CONSTRAINT " + oldcntname + ";\n"; - diff.append(sql); - sql = "ALTER TABLE " + table + " DROP INDEX " + oldcntname + ";\n\n"; - diff.append(sql); - addAlterFieldToDiff(table,a,a.attribute("size","0").toInt(),0); - sql = "ALTER TABLE " + table + " ADD INDEX " + newcntname + ";\n"; - diff.append(sql); - sql = "ALTER TABLE " + table + " ADD CONSTRAINT " + newcntname + " FOREIGN KEY (" + a.attribute("name") + ") REFERENCES " + a.attribute("rtable") + " (" + a.attribute("rfield") + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n\n"; - diff.append(sql); - addTableToDrop(b.attribute("rtable")); -} - -//This adds the modifications to a table to the diff list -void addFieldToDiff(QString table, QDomElement eField) -{ - QString sql; - sql = "ALTER TABLE " + table + " ADD COLUMN " + eField.attribute("name",""); - if (eField.attribute("type","") == "decimal") - sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + "," + eField.attribute("decsize","0") + ");\n"; - else - { - if (eField.attribute("type","") != "text") - sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + ");\n"; - else - sql = sql + " " + eField.attribute("type","") + ";\n"; - } - diff.append(sql); - if (eField.attribute("rtable","") != "") - { - sql = "ALTER TABLE " + table + " ADD INDEX " + eField.attribute("rname","") + " (" + eField.attribute("name","") + ");\n"; - diff.append(sql); - sql = "ALTER TABLE " + table + " ADD CONSTRAINT " + eField.attribute("rname","") + " FOREIGN KEY ("; - sql = sql + eField.attribute("name","") + ") REFERENCES " + eField.attribute("rtable","") + "("; - sql = sql + eField.attribute("rfield","") + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n" ; - diff.append(sql); - idx++; - } -} - -void addFieldToRTables(QString parentTable, QString rTable, QString field, QString rField) -{ - int tidx; - tidx = -1; - int pos; - for (pos = 0; pos < rtables.count(); pos++) - { - if ((rtables[pos].parentTable == parentTable) && (rtables[pos].name == rTable)) - tidx = pos; - } - if (tidx != -1) - { - TrfieldDef aField; - aField.name = field; - aField.rname = rField; - rtables[tidx].rfields.append(aField); - } - else - { - TrtableDef table; - table.parentTable = parentTable; - table.name = rTable; - TrfieldDef aField; - aField.name = field; - aField.rname = rField; - table.rfields.append(aField); - rtables.append(table); - } -} - -//This adds a table to the diff list -void addTableToSDiff(QDomNode table, bool lookUp) -{ - QUuid recordUUID=QUuid::createUuid(); - QString strRecordUUID=recordUUID.toString().replace("{","").replace("}","").right(12); - - QDomElement eTable = table.toElement(); - QString sql; - sql = "CREATE TABLE IF NOT EXISTS " + eTable.attribute("name","") + "(\n"; - QDomNode node = table.firstChild(); - QStringList keys; - QList childTables; - while (!node.isNull()) - { - QDomElement field = node.toElement(); - if (field.tagName() == "field") - { - sql = sql + field.attribute("name",""); - - if (field.attribute("type","") == "decimal") - sql = sql + " " + field.attribute("type","") + " (" + field.attribute("size","0") + "," + field.attribute("decsize","0") + ")"; - else - { - if (field.attribute("type","") != "text") - sql = sql + " " + field.attribute("type","") + " (" + field.attribute("size","0") + ")"; - else - sql = sql + " " + field.attribute("type",""); - } - if (field.attribute("key","") == "true") - { - sql = sql + "NOT NULL COMMENT \"" + field.attribute("desc","Without description") + "\",\n"; - keys.append(field.attribute("name","")); - } - else - sql = sql + " COMMENT \"" + field.attribute("desc","Without description") + "\",\n"; - if (field.attribute("rtable","") != "") - { - addFieldToRTables(eTable.attribute("name",""),field.attribute("rtable",""),field.attribute("name",""),field.attribute("rfield","")); - } - } - else - { - childTables.append(node); - } - node = node.nextSibling(); - } - //sql = sql + "rowuuid varchar(80) COMMENT \"Unique Row Identifier (UUID)\",\n"; - int pos; - int pos2; - //Add the keys - sql = sql + "PRIMARY KEY ("; - for (pos = 0; pos < keys.count();pos++) - { - sql = sql + keys[pos] + ","; - } - bool hasRelatedTables; - hasRelatedTables = false; - for (pos = 0; pos < rtables.count();pos++) - { - if (rtables[pos].parentTable == eTable.attribute("name","")) - { - hasRelatedTables = true; - break; - } - } - if (hasRelatedTables) - sql = sql.left(sql.length()-1) + "),\n"; - else - sql = sql.left(sql.length()-1) + ")"; - //Add the indexes - for (pos = 0; pos < rtables.count();pos++) - { - if (rtables[pos].parentTable == eTable.attribute("name","")) - { - QUuid idxUUID=QUuid::createUuid(); - QString strIdxUUID=idxUUID.toString().replace("{","").replace("}","").right(12); - - sql = sql + "INDEX DIDX" + strIdxUUID + "("; - for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) - { - sql = sql + rtables[pos].rfields[pos2].name + ","; - } - sql = sql.left(sql.length()-1) + "),\n"; - idx++; - } - } - //Add foreign keys - for (pos = 0; pos < rtables.count();pos++) - { - if (rtables[pos].parentTable == eTable.attribute("name","")) - { - QUuid cntUUID=QUuid::createUuid(); - QString strCntUUID=cntUUID.toString().replace("{","").replace("}","").right(12); - - sql = sql + "CONSTRAINT DFK" + strCntUUID + "\n"; - sql = sql + "FOREIGN KEY ("; - for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) - { - sql = sql + rtables[pos].rfields[pos2].name + ","; - } - sql = sql.left(sql.length()-1) + ")\n"; - sql = sql + "REFERENCES " + rtables[pos].name + "("; - for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) - { - sql = sql + rtables[pos].rfields[pos2].rname + ","; - } - sql = sql.left(sql.length()-1) + ")\nON DELETE RESTRICT\nON UPDATE NO ACTION,\n"; - idx++; - } - } - if (hasRelatedTables) - sql = sql.left(sql.length()-2); - sql = sql + ")\n ENGINE = InnoDB CHARSET=utf8 COMMENT = \"" + eTable.attribute("desc","") + "\";\n"; - sql = sql + "CREATE UNIQUE INDEX DXROWUUID" + strRecordUUID + " ON " + eTable.attribute("name","") + "(rowuuid);\n\n"; - - QUuid TriggerUUID=QUuid::createUuid(); - QString strTriggerUUID=TriggerUUID.toString().replace("{","").replace("}","").replace("-","_"); - - sql = sql + "delimiter $$\n\n"; - sql = sql + "CREATE TRIGGER T" + strTriggerUUID + " BEFORE INSERT ON " + eTable.attribute("name","") + " FOR EACH ROW BEGIN IF (new.rowuuid IS NULL) THEN SET new.rowuuid = uuid(); ELSE IF (new.rowuuid NOT REGEXP '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}') THEN SET new.rowuuid = uuid(); END IF; END IF; END;$$\n\n"; - sql = sql + "delimiter ;\n\n"; - - diff.append(sql); - for (int pos = 0; pos <= childTables.count()-1;pos++) - addTableToSDiff(childTables[pos],lookUp); -} - -//This logs messages to the terminal. We use printf because qDebug does not log in relase -void log(QString message) -{ - QString temp; - temp = message + "\n"; - printf("%s",temp.toUtf8().data()); -} - -void fatal(QString message) -{ - fprintf(stderr, "\033[31m%s\033[0m \n", message.toUtf8().data()); - fatalError = true; -} - -QDomNode findField(QDomNode table,QString field) -{ - QDomNode node; - node = table.firstChild(); - while (!node.isNull()) - { - if (node.toElement().attribute("name","") == field) - return node; - node = node.nextSibling(); - } - QDomNode null; - return null; -} - -QDomNode findTable(QDomDocument docB,QString tableName) -{ - QDomNodeList tables; - tables = docB.elementsByTagName("table"); - for (int pos = 0; pos < tables.count();pos++) - { - if (tables.item(pos).toElement().attribute("name","") == tableName) - return tables.item(pos); - } - QDomNode null; - return null; -} - -QString compareFields(QDomElement a, QDomElement b, int &newSize, int &newDec) -{ - newSize = a.attribute("size","0").toInt(); - newDec = a.attribute("decsize","0").toInt(); - if (a.attribute("key","") != b.attribute("key","")) - return "KNS"; - if ((a.attribute("rtable","") != b.attribute("rtable","")) || (a.attribute("rfield","") != b.attribute("rfield",""))) - { - if (a.attribute("mergefine","false") == "true") - { - return "CHR"; - } - else - return "RNS"; - } - if (a.attribute("type","") != b.attribute("type","")) - { - if ((b.attribute("type") == "int") && (a.attribute("type") == "varchar")) - { - if (a.attribute("size","0").toInt() >= b.attribute("size","0").toInt()) - { - if (a.attribute("decsize","0").toInt() > b.attribute("decsize","0").toInt()) - { - newSize = a.attribute("size","0").toInt(); - newDec = a.attribute("decsize","0").toInt(); - } - return "FTC"; - } - else - return "FNS"; - } - else - return "FNS"; - } - - if (a.attribute("size","") != b.attribute("size","")) - { - if (a.attribute("size","0").toInt() > b.attribute("size","0").toInt()) - { - if (a.attribute("type","") == "decimal") - { - if (a.attribute("decsize","0").toInt() != b.attribute("decsize","0").toInt()) - { - if (a.attribute("decsize","0").toInt() > b.attribute("decsize","0").toInt()) - { - newSize = a.attribute("size","0").toInt(); - newDec = a.attribute("decsize","0").toInt(); - return "FIC"; - } - else - { - if (a.attribute("decsize","0").toInt() < b.attribute("decsize","0").toInt()) - return "FDC"; - } - } - else - { - if (a.attribute("size","0").toInt() < b.attribute("size","0").toInt()) - return "FDC"; - } - } - else - { - newSize = a.attribute("size","0").toInt(); - return "FIC"; - } - } - else - { - if (a.attribute("size","0").toInt() < b.attribute("size","0").toInt()) - return "FDC"; - } - } - if (a.attribute("decsize","") != b.attribute("decsize","")) - { - if (a.attribute("decsize","0").toInt() > b.attribute("decsize","0").toInt()) - { - newSize = a.attribute("size","0").toInt(); - newDec = a.attribute("decsize","0").toInt(); - return "FIC"; - } - else - { - if (a.attribute("decsize","0").toInt() < b.attribute("decsize","0").toInt()) - return "FDC"; - } - } - - return ""; -} - -QString getFieldDefinition(QDomElement field) -{ - QString result; - result = "[Key=" + field.attribute("key","false") + ","; - result = result + "Type=" + field.attribute("type","") + ","; - result = result + "Size=" + field.attribute("size","0") + ","; - result = result + "DecimalSize=" + field.attribute("decsize","0") + ","; - result = result + "RelatedTable=" + field.attribute("rtable","None") + ","; - result = result + "RelatedField=" + field.attribute("rfield","None") + "]"; - return result; -} - -void checkField(QDomNode eTable, QDomElement a, QDomElement b) -{ - int newSize; - int newDec; - QString result; - result = compareFields(a,b,newSize,newDec); - if (result != "") - { - if ((result == "FIC") || (result == "CHR") || (result == "FTC")) - { - if (result == "FIC") - { - if (outputType == "h") - { - if (a.attribute("type","") != "decimal") - log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0")); - else - log("FIC: The size of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be increased from " + b.attribute("size",0) + " to " + a.attribute("size","0") + ". Decimal size from " + b.attribute("decsize",0) + " to " + a.attribute("decsize","0")); - } - b.setAttribute("size",newSize); - b.setAttribute("decsize",newDec); - addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); - } - else - { - if (result == "CHR") - { - if (outputType == "h") - { - log("CHR: The relationship of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be changed from " + b.attribute("rtable") + "." + b.attribute("rfield") + " to " + a.attribute("rtable") + "." + a.attribute("rfield")); - } - changeLookupRelationship(eTable.toElement().attribute("name",""),a,b); - b.setAttribute("rtable",a.attribute("rtable")); - b.setAttribute("rfield",a.attribute("rfield")); - b.setAttribute("type",a.attribute("type")); - b.setAttribute("size",a.attribute("size")); - } - else - { - if (outputType == "h") - { - log("FTC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from " + b.attribute("type") + " to " + a.attribute("type") + " which is allowed"); - b.setAttribute("size",newSize); - b.setAttribute("decsize",newDec); - b.setAttribute("type",a.attribute("type")); - addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); - } - } - } - } - else - { - if (outputType == "h") - { - if (result == "KNS") - fatal("KNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed key from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); - if (result == "RNS") - fatal("RNS: The relationship for field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); - if (result == "FNS") - fatal("FNS: Field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from B to A. ODK Tools cannot fix this. You need to fix it in the Excel file"); - if (result == "FDC") - log("FDC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " has descreased in size. This change will be ignored"); - } - } - if (outputType != "h") - { - TcompError error; - error.code = result; - error.table = eTable.toElement().attribute("name",""); - error.field = a.attribute("name",""); - error.from = getFieldDefinition(b); - error.to = getFieldDefinition(a); - errorList.append(error); - } - } -} - -void compareLKPTables(QDomNode table,QDomDocument &docB) -{ - QDomNode node; - node = table; - while (!node.isNull()) - { - QDomNode tablefound; - tablefound = findTable(docB,node.toElement().attribute("name","")); - if (!tablefound.isNull()) - { - QDomNode field = node.firstChild(); - while (!field.isNull()) - { - QDomElement eField = field.toElement(); - if (eField.tagName() == "table") - compareLKPTables(field,docB); - else - { - QDomNode fieldFound = findField(tablefound,eField.attribute("name","")); - if (!fieldFound.isNull()) - { - checkField(tablefound,field.toElement(),fieldFound.toElement()); - } - else - { - if (outputType == "h") - log("FNF:Field " + eField.attribute("name","") + " in table " + tablefound.toElement().attribute("name","") + " from A is not found in B"); - else - { - TcompError error; - error.code = "FNF"; - error.table = tablefound.toElement().attribute("name",""); - error.field = eField.attribute("name",""); - error.from = "NULL"; - error.to = getFieldDefinition(eField); - errorList.append(error); - } - addFieldToDiff(tablefound.toElement().attribute("name",""),eField); - tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); - } - } - - field = field.nextSibling(); - } - } - else - { - if (outputType == "h") - log("TNF:Lookup table " + node.toElement().attribute("name","") + " from A not found in B"); - else - { - TcompError error; - error.code = "TNF"; - error.table = "NA"; - error.field = "NA"; - error.from = "NULL"; - error.to = node.toElement().attribute("name",""); - errorList.append(error); - } - addTableToSDiff(node,true); - //Now adds the lookup table - docB.documentElement().firstChild().appendChild(node.cloneNode(true)); - } - node = node.nextSibling(); - } -} - -void compareTables(QDomNode table,QDomDocument &docB) -{ - QDomElement eTable = table.toElement(); - QDomNode tablefound; - //log(eTable.toElement().attribute("name","")); - tablefound = findTable(docB,eTable.toElement().attribute("name","")); - if (!tablefound.isNull()) - { - if (table.parentNode().toElement().attribute("name","") == - tablefound.parentNode().toElement().attribute("name","")) - { - QDomNode field = table.firstChild(); - while (!field.isNull()) - { - QDomElement eField = field.toElement(); - if (eField.tagName() == "table") - compareTables(field,docB); - else - { - QDomNode fieldFound = findField(tablefound,eField.attribute("name","")); - if (!fieldFound.isNull()) - { - checkField(tablefound,field.toElement(),fieldFound.toElement()); - } - else - { - if (outputType == "h") - log("FNF:Field " + eField.attribute("name","") + " in table " + tablefound.toElement().attribute("name","") + " from A is not found in B"); - else - { - TcompError error; - error.code = "FNF"; - error.table = tablefound.toElement().attribute("name",""); - error.field = eField.attribute("name",""); - error.from = "NULL"; - error.to = getFieldDefinition(eField); - errorList.append(error); - } - addFieldToDiff(tablefound.toElement().attribute("name",""),eField); - tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); - } - } - - field = field.nextSibling(); - } - } - else - { - if (outputType == "h") - fatal("TNS:Table " + eTable.toElement().attribute("name","") + " from A does not have the same parent in B"); - else - { - TcompError error; - error.code = "TNS"; - error.table = "NA"; - error.field = "NA"; - error.from = tablefound.parentNode().toElement().attribute("name",""); - error.to = table.parentNode().toElement().attribute("name",""); - errorList.append(error); - } - } - } - else - { - //Now adds the table to doc2 - QDomNode parentfound; - QString parentTableName; - parentTableName = table.parentNode().toElement().attribute("name",""); - parentfound = findTable(docB,parentTableName); - if (!parentfound.isNull()) - { - if (outputType == "h") - log("TNF:Table " + eTable.toElement().attribute("name","") + " from A not found in B"); - else - { - TcompError error; - error.code = "TNF"; - error.table = "NA"; - error.field = "NA"; - error.from = "NULL"; - error.to = eTable.toElement().attribute("name",""); - errorList.append(error); - } - addTableToSDiff(eTable,false); - parentfound.appendChild(table.cloneNode(true)); - } - else - { - if (outputType == "h") - fatal("TWP:Table " + eTable.toElement().attribute("name","") + " from A not found in B. Its parent in A is not found in B"); - else - { - TcompError error; - error.code = "TWP"; - error.table = "NA"; - error.field = "NA"; - error.from = "NULL"; - error.to = parentTableName; - errorList.append(error); - } - } - } -} - -int main(int argc, char *argv[]) -{ - QString title; - title = title + "********************************************************************* \n"; - title = title + " * Compare Create XML * \n"; - title = title + " * This tool compares two create XML files (A and B) for incremental * \n"; - title = title + " * changes. A is consider an incremental version of B . * \n"; - title = title + " * * \n"; - title = title + " * The tool informs of tables and variables in A that are not in B. * \n"; - title = title + " * The tool can also create a combined file C that appends * \n"; - title = title + " * not found tables and variables with the following conditions: * \n"; - title = title + " * 1) If a table in A is not found in B then it will be added to B * \n"; - title = title + " * only if its parent exists in B. * \n"; - title = title + " * * \n"; - title = title + " * C will have all of B plus all of A. * \n"; - title = title + " * * \n"; - title = title + " * The tool WILL NOT fix the following: * \n"; - title = title + " * 1) Inconsistencies in field definition like size, type, * \n"; - title = title + " * parent table and parent field. * \n"; - title = title + " * 2) Tables that do not share the same parent. * \n"; - title = title + " * * \n"; - title = title + " * Nomenclature: * \n"; - title = title + " * TNF: Table not found. * \n"; - title = title + " * TNS: The table does not have the same parent table. (Cannot merge) * \n"; - title = title + " * TWP: The parent table is not found in B. (Cannot merge). * \n"; - title = title + " * FNF: Field not found. * \n"; - title = title + " * FNS: The field is not the same. (Cannot merge) * \n"; - title = title + " * FIC: The field will increase in size. * \n"; - title = title + " * FDC: The field decreased in in size. (Ignore) * \n"; - title = title + " * * \n"; - title = title + " * This tool is usefull when dealing with multiple versions of an * \n"; - title = title + " * ODK survey that must be combined in one common database. * \n"; - title = title + " * * \n"; - title = title + " * If a TNS nor a FNS are NOT encounter then this means that A can * \n"; - title = title + " * merge into B and a diff SQL script is issued. * \n"; - title = title + " * * \n"; - title = title + " * Decrimental changes are not taken into account because this means * \n"; - title = title + " * losing data between versions. * \n"; - title = title + " ********************************************************************* \n"; - - TCLAP::CmdLine cmd(title.toUtf8().constData(), ' ', "2.0"); - - TCLAP::ValueArg aArg("a","inputa","Input create XML file A (later)",true,"","string"); - TCLAP::ValueArg bArg("b","inputb","Input create XML file B (former)",true,"","string"); - TCLAP::ValueArg cArg("c","outputc","Output create XML file C",false,"./combined-create.xml","string"); - TCLAP::ValueArg dArg("d","diff","Output diff SQL script",false,"./diff.sql","string"); - TCLAP::ValueArg oArg("o","outputype","Output type: (h)uman readble or (m)achine readble",false,"m","string"); - - cmd.add(aArg); - cmd.add(bArg); - cmd.add(cArg); - cmd.add(dArg); - cmd.add(oArg); - - - //Parsing the command lines - cmd.parse( argc, argv ); - - //Getting the variables from the command - QString inputA = QString::fromUtf8(aArg.getValue().c_str()); - QString inputB = QString::fromUtf8(bArg.getValue().c_str()); - QString outputC = QString::fromUtf8(cArg.getValue().c_str()); - QString outputD = QString::fromUtf8(dArg.getValue().c_str()); - outputType = QString::fromUtf8(oArg.getValue().c_str()); - - fatalError = false; - idx = 1; - - if (inputA != inputB) - { - if ((QFile::exists(inputA)) && (QFile::exists(inputB))) - { - //Openning and parsing input file A - QDomDocument docA("inputA"); - QFile fileA(inputA); - if (!fileA.open(QIODevice::ReadOnly)) - { - log("Cannot open input file A"); - return 1; - } - if (!docA.setContent(&fileA)) - { - log("Cannot parse document for input file A"); - fileA.close(); - return 1; - } - fileA.close(); - - //Openning and parsing input file B - QDomDocument docB("inputB"); - QFile fileB(inputB); - if (!fileB.open(QIODevice::ReadOnly)) - { - log("Cannot open input file B"); - return 1; - } - if (!docB.setContent(&fileB)) - { - log("Cannot parse document for input file B"); - fileB.close(); - return 1; - } - fileB.close(); - - QDomElement rootA = docA.documentElement(); - QDomElement rootB = docB.documentElement(); - if ((rootA.tagName() == "XMLSchemaStructure") && (rootB.tagName() == "XMLSchemaStructure")) - { - //Comparing lookup tables - compareLKPTables(rootA.firstChild().firstChild(),docB); - //Comparing tables - compareTables(rootA.firstChild().nextSibling().firstChild(),docB); - //Process drops - QDomNode lkpTables = rootB.firstChild(); - for (int pos = 0; pos < dropTables.count(); pos++) - { - diff.append("DROP TABLE IF EXISTS " + dropTables[pos] + ";\n\n"); - QDomNode a_table = rootB.firstChild().firstChild(); - while (!a_table.isNull()) - { - if (a_table.toElement().attribute("name") == dropTables[pos]) - break; - a_table = a_table.nextSibling(); - } - lkpTables.removeChild(a_table); - } - - if (outputType == "m") - { - QDomDocument XMLResult; - XMLResult = QDomDocument("XMLResult"); - QDomElement XMLRoot; - XMLRoot = XMLResult.createElement("XMLResult"); - XMLResult.appendChild(XMLRoot); - QDomElement eErrors; - eErrors = XMLResult.createElement("errors"); - XMLRoot.appendChild(eErrors); - if (errorList.count() > 0) - fatalError = true; - for (int pos = 0; pos <= errorList.count()-1; pos++) - { - QDomElement anError; - anError = XMLResult.createElement("error"); - anError.setAttribute("table",errorList[pos].table); - anError.setAttribute("field",errorList[pos].field); - anError.setAttribute("code",errorList[pos].code); - anError.setAttribute("from",errorList[pos].from); - anError.setAttribute("to",errorList[pos].to); - eErrors.appendChild(anError); - } - log(XMLResult.toString()); - } - - //Do not create C and the Diff SQL if the merge cannot be done - if (fatalError == false) - { - //Create the output file. If exist it get overwriten - if (QFile::exists(outputC)) - QFile::remove(outputC); - QFile file(outputC); - if (file.open(QIODevice::WriteOnly | QIODevice::Text)) - { - QTextStream out(&file); - out.setCodec("UTF-8"); - docB.save(out,1,QDomNode::EncodingFromTextStream); - file.close(); - } - else - { - log("Error: Cannot create XML combined file"); - return 1; - } - - if (QFile::exists(outputD)) - QFile::remove(outputD); - QFile dfile(outputD); - if (dfile.open(QIODevice::WriteOnly | QIODevice::Text)) - { - QTextStream outD(&dfile); - outD.setCodec("UTF-8"); - for (int dpos = 0; dpos < diff.count();dpos++) - { - outD << diff[dpos]; - } - file.close(); - } - else - { - log("Error: Cannot create Diff file"); - return 1; - } - } - else - return 1; - } - else - { - if (!(rootA.tagName() == "ODKImportXML")) - { - log("Input document A is not a create XML file"); - return 1; - } - if (!(rootB.tagName() == "ODKImportXML")) - { - log("Input document B is not a create XML file"); - return 1; - } - } - - } - else - { - if (!QFile::exists(inputA)) - { - log("Input file A does not exists"); - return 1; - } - if (!QFile::exists(inputB)) - { - log("Input file B does not exists"); - return 1; - } - } - } - else - { - log("Input files A and B are the same. No point in comparing them."); - return 1; - } - if (!fatalError) - return 0; - else - return 1; -} diff --git a/utilities/compareInsertXML/compareInsertXML.pro b/utilities/compareInsertXML/compareInsertXML.pro deleted file mode 100644 index e09a23258..000000000 --- a/utilities/compareInsertXML/compareInsertXML.pro +++ /dev/null @@ -1,19 +0,0 @@ -#------------------------------------------------- -# -# Project created by QtCreator 2015-09-23T15:35:18 -# -#------------------------------------------------- - -QT += core xml - -QT -= gui - -TARGET = compareinsertxml -CONFIG += console -CONFIG -= app_bundle - -TEMPLATE = app - -INCLUDEPATH += ../../3rdparty - -SOURCES += main.cpp diff --git a/utilities/compareInsertXML/main.cpp b/utilities/compareInsertXML/main.cpp deleted file mode 100644 index ccadf70d6..000000000 --- a/utilities/compareInsertXML/main.cpp +++ /dev/null @@ -1,394 +0,0 @@ -/* -CompareInsertXML - -Copyright (C) 2015-2017 International Livestock Research Institute. -Author: Carlos Quiros (cquiros_at_qlands.com / c.f.quiros_at_cgiar.org) - -CompareInsertXML is free software: you can redistribute it and/or modify -it under the terms of the GNU Lesser General Public License as -published by the Free Software Foundation, either version 3 of -the License, or (at your option) any later version. - -CompareInsertXML is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with CompareInsertXML. If not, see . -*/ - -#include -#include -#include - -QString outputType; -bool fatalError; -QStringList diff; - -struct compError -{ - QString code; - QString desc; - QString table; - QString value; - QString from; - QString to; -}; -typedef compError TcompError; - -QList errorList; - -//This logs messages to the terminal. We use printf because qDebug does not log in relase -void log(QString message) -{ - QString temp; - temp = message + "\n"; - printf("%s",temp.toUtf8().data()); -} - -void fatal(QString message) -{ - fatalError = true; - fprintf(stderr, "\033[31m%s\033[0m \n", message.toUtf8().data()); -} - -QDomNode findTable(QDomDocument docB,QString tableName) -{ - QDomNodeList tables; - tables = docB.elementsByTagName("table"); - for (int pos = 0; pos < tables.count();pos++) - { - if (tables.item(pos).toElement().attribute("name","") == tableName) - return tables.item(pos); - } - QDomNode null; - return null; -} - -QDomNode findValue(QDomNode table,QString code) -{ - QDomNode node; - node = table.firstChild(); - while (!node.isNull()) - { - if (node.toElement().attribute("code","") == code) - return node; - node = node.nextSibling(); - } - QDomNode null; - return null; -} - -void addValueToDiff(QDomElement table, QDomElement field) -{ - QString sql; - sql = "INSERT INTO " + table.attribute("name","") + " ("; - sql = sql + table.attribute("clmcode","") + ","; - sql = sql + table.attribute("clmdesc","") + ") VALUES ('"; - sql = sql + field.attribute("code","") + "','"; - sql = sql + field.attribute("description","") + "');"; - diff.append(sql); -} - -void UpdateValue(QDomElement table, QDomElement field) -{ - QString sql; - sql = "UPDATE " + table.attribute("name","") + " SET "; - sql = sql + table.attribute("clmdesc","") + " = '"; - sql = sql + field.attribute("description","") + "' WHERE "; - sql = sql + table.attribute("clmcode","") + " = '"; - sql = sql + field.attribute("code","") + "';"; - diff.append(sql); - -} - -void addTableToDiff(QDomElement table) -{ - QDomNode field; - field = table.firstChild(); - while (!field.isNull()) - { - addValueToDiff(table,field.toElement()); - field = field.nextSibling(); - } -} - -void changeValueInC(QDomNode table, QString code, QString newDescription) -{ - QDomNode field; - field = table.firstChild(); - while (!field.isNull()) - { - QDomElement efield; - efield = field.toElement(); - if (efield.attribute("code","") == code) - { - efield.setAttribute("description",newDescription); - return; - } - field = field.nextSibling(); - } -} - -void compareLKPTables(QDomNode table,QDomDocument &docB) -{ - QDomNode node; - node = table; - while (!node.isNull()) - { - QDomNode tableFound = findTable(docB,node.toElement().attribute("name","")); - if (!tableFound.isNull()) - { - QDomNode field = node.firstChild(); - while (!field.isNull()) - { - QDomNode fieldFound = findValue(tableFound,field.toElement().attribute("code","")); - if (!fieldFound.isNull()) - { - if (field.toElement().attribute("description","") != fieldFound.toElement().attribute("description","")) - { - if (outputType == "h") - { - fatal("VNS:Value " + field.toElement().attribute("code","") + " of lookup table " + node.toElement().attribute("name","") + " has changed from \"" + fieldFound.toElement().attribute("description","") + "\" to \"" + field.toElement().attribute("description","") + "\""); - log("Do you want to change this value in the database? Y/N"); - std::string line; - std::getline(std::cin, line); - QString result = QString::fromStdString(line); - if (std::cin.eof() || result.toLower() == "y") - { - fatalError = false; - UpdateValue(node.toElement(),field.toElement()); - changeValueInC(tableFound,field.toElement().attribute("code",""),field.toElement().attribute("description","")); - } - else - { - fatalError = true; - } - } - { - TcompError error; - error.code = "VNS"; - error.desc = "Value " + field.toElement().attribute("code","") + " of lookup table " + node.toElement().attribute("name","") + " from A not the same in B"; - error.table = node.toElement().attribute("name",""); - error.value = field.toElement().attribute("code",""); - error.from = fieldFound.toElement().attribute("description",""); - error.to = field.toElement().attribute("description",""); - errorList.append(error); - } - } - } - else - { - if (outputType == "h") - log("VNF:Value " + field.toElement().attribute("code","") + "(" + field.toElement().attribute("description","") + ") will be included in table " + node.toElement().attribute("name","")); - tableFound.appendChild(field.cloneNode(true)); - addValueToDiff(node.toElement(),field.toElement()); - } - field = field.nextSibling(); - } - } - else - { - if (outputType == "h") - log("TNF:The lookup table " + node.toElement().attribute("name","") + " will be included in the database."); - //Now adds the lookup table - addTableToDiff(node.toElement()); - docB.documentElement().appendChild(node.cloneNode(true)); - } - node = node.nextSibling(); - } -} - -int main(int argc, char *argv[]) -{ - QString title; - title = title + "********************************************************************* \n"; - title = title + " * Compare Insert XML * \n"; - title = title + " * This tool compares two insert XML files (A and B) for incremental * \n"; - title = title + " * changes. * \n"; - title = title + " * * \n"; - title = title + " * The tool informs of lookup values in A that are not in B. * \n"; - title = title + " * The tool can also create a combined file C that appends * \n"; - title = title + " * not found values. * \n"; - title = title + " * * \n"; - title = title + " * Nomenclature: * \n"; - title = title + " * TNF: Lookup table not found. * \n"; - title = title + " * VNF: Value not found. * \n"; - title = title + " * VNS: The values is not the same. * \n"; - title = title + " * * \n"; - title = title + " * This tool is usefull when dealing with multiple versions of an * \n"; - title = title + " * ODK survey that must be combined in one common database. * \n"; - title = title + " ********************************************************************* \n"; - - TCLAP::CmdLine cmd(title.toUtf8().constData(), ' ', "2.0"); - - TCLAP::ValueArg aArg("a","inputa","Input insert XML file A",true,"","string"); - TCLAP::ValueArg bArg("b","inputb","Input insert XML file B",true,"","string"); - TCLAP::ValueArg cArg("c","outputc","Output insert XML file C",false,"./combined-insert.xml","string"); - TCLAP::ValueArg dArg("d","diff","Output diff SQL script",false,"./diff.sql","string"); - TCLAP::ValueArg oArg("o","outputype","Output type: (h)uman readble or (m)achine readble",false,"m","string"); - - cmd.add(aArg); - cmd.add(bArg); - cmd.add(cArg); - cmd.add(dArg); - cmd.add(oArg); - - - //Parsing the command lines - cmd.parse( argc, argv ); - - //Getting the variables from the command - QString inputA = QString::fromUtf8(aArg.getValue().c_str()); - QString inputB = QString::fromUtf8(bArg.getValue().c_str()); - QString outputC = QString::fromUtf8(cArg.getValue().c_str()); - QString outputD = QString::fromUtf8(dArg.getValue().c_str()); - outputType = QString::fromUtf8(oArg.getValue().c_str()); - - fatalError = false; - if (inputA != inputB) - { - if ((QFile::exists(inputA)) && (QFile::exists(inputB))) - { - //Openning and parsing input file A - QDomDocument docA("inputA"); - QFile fileA(inputA); - if (!fileA.open(QIODevice::ReadOnly)) - { - log("Cannot open input file A"); - return 1; - } - if (!docA.setContent(&fileA)) - { - log("Cannot parse document for input file A"); - fileA.close(); - return 1; - } - fileA.close(); - - //Openning and parsing input file B - QDomDocument docB("inputB"); - QFile fileB(inputB); - if (!fileB.open(QIODevice::ReadOnly)) - { - log("Cannot open input file B"); - return 1; - } - if (!docB.setContent(&fileB)) - { - log("Cannot parse document for input file B"); - fileB.close(); - return 1; - } - fileB.close(); - - QDomElement rootA = docA.documentElement(); - QDomElement rootB = docB.documentElement(); - if ((rootA.tagName() == "insertValuesXML") && (rootB.tagName() == "insertValuesXML")) - { - compareLKPTables(rootA.firstChild(),docB); - - if (outputType == "m") - { - QDomDocument XMLResult; - XMLResult = QDomDocument("XMLResult"); - QDomElement XMLRoot; - XMLRoot = XMLResult.createElement("XMLResult"); - XMLResult.appendChild(XMLRoot); - QDomElement eErrors; - eErrors = XMLResult.createElement("errors"); - XMLRoot.appendChild(eErrors); - if (errorList.count() > 0) - fatalError = true; - for (int pos = 0; pos <= errorList.count()-1; pos++) - { - QDomElement anError; - anError = XMLResult.createElement("error"); - anError.setAttribute("table",errorList[pos].table); - anError.setAttribute("value",errorList[pos].value); - anError.setAttribute("code",errorList[pos].code); - anError.setAttribute("desc",errorList[pos].desc); - anError.setAttribute("from",errorList[pos].from); - anError.setAttribute("to",errorList[pos].to); - eErrors.appendChild(anError); - } - log(XMLResult.toString()); - } - - - //Create the manifext file. If exist it get overwriten - if (QFile::exists(outputC)) - QFile::remove(outputC); - QFile file(outputC); - if (file.open(QIODevice::WriteOnly | QIODevice::Text)) - { - QTextStream out(&file); - out.setCodec("UTF-8"); - docB.save(out,1,QDomNode::EncodingFromTextStream); - file.close(); - } - else - { - log("Error: Cannot create XML combined file"); - return 1; - } - - if (QFile::exists(outputD)) - QFile::remove(outputD); - QFile dfile(outputD); - if (dfile.open(QIODevice::WriteOnly | QIODevice::Text)) - { - QTextStream outD(&dfile); - outD.setCodec("UTF-8"); - for (int dpos = 0; dpos < diff.count();dpos++) - { - outD << diff[dpos] + "\n"; - } - file.close(); - } - else - { - log("Error: Cannot create Diff file"); - return 1; - } - } - else - { - if (!(rootA.tagName() == "ODKImportXML")) - { - log("Input document A is not a insert XML file"); - return 1; - } - if (!(rootB.tagName() == "ODKImportXML")) - { - log("Input document B is not a insert XML file"); - return 1; - } - } - - } - else - { - if (!QFile::exists(inputA)) - { - log("Input file A does not exists"); - return 1; - } - if (!QFile::exists(inputB)) - { - log("Input file B does not exists"); - return 1; - } - } - } - else - { - log("Input files A and B are the same. No point in comparing them."); - return 1; - } - if (!fatalError) - return 0; - else - return 1; -} From 7863006d98f65789d2ed331dc58bab60cb02f658 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Wed, 21 Aug 2019 10:25:37 -0600 Subject: [PATCH 10/24] Add support to ignore VNS --- utilities/mergeVersions/compareinsert.cpp | 57 ++++++++++++++--------- utilities/mergeVersions/compareinsert.h | 4 +- utilities/mergeVersions/generic.h | 7 +++ utilities/mergeVersions/main.cpp | 30 +++++++++++- utilities/mergeVersions/mainclass.cpp | 9 ++-- utilities/mergeVersions/mainclass.h | 4 +- 6 files changed, 82 insertions(+), 29 deletions(-) diff --git a/utilities/mergeVersions/compareinsert.cpp b/utilities/mergeVersions/compareinsert.cpp index 1bd7a29d6..a806aba15 100644 --- a/utilities/mergeVersions/compareinsert.cpp +++ b/utilities/mergeVersions/compareinsert.cpp @@ -26,13 +26,14 @@ compareInsert::compareInsert(QObject *parent) : QObject(parent) fatalError = false; } -void compareInsert::setFiles(QString insertA, QString insertB, QString insertC, QString diffSQL, QString outputType) +void compareInsert::setFiles(QString insertA, QString insertB, QString insertC, QString diffSQL, QString outputType, QList toIgnore) { inputA = insertA; inputB = insertB; outputC = insertC; - outputD = diffSQL; + outputD = diffSQL; this->outputType = outputType; + valuesToIgnore = toIgnore; } QList compareInsert::getDiffs() @@ -316,6 +317,24 @@ void compareInsert::changeValueInC(QDomNode table, QString code, QString newDesc } } +bool compareInsert::ignoreChange(QString table, QString value) +{ + for (int pos =0; pos < valuesToIgnore.count(); pos++) + { + if (valuesToIgnore[pos].table == table) + { + for (int pos2 =0; pos2 < valuesToIgnore[pos].values.count(); pos2++) + { + if (valuesToIgnore[pos].values[pos2] == value) + { + return true; + } + } + } + } + return false; +} + void compareInsert::compareLKPTables(QDomNode table,QDomDocument &docB) { QDomNode node; @@ -333,33 +352,27 @@ void compareInsert::compareLKPTables(QDomNode table,QDomDocument &docB) { if (field.toElement().attribute("description","") != fieldFound.toElement().attribute("description","")) { - if (outputType == "h") + if (!ignoreChange(node.toElement().attribute("name",""),field.toElement().attribute("code",""))) { - fatal("VNS:Value " + field.toElement().attribute("code","") + " of lookup table " + node.toElement().attribute("name","") + " has changed from \"" + fieldFound.toElement().attribute("description","") + "\" to \"" + field.toElement().attribute("description","") + "\""); - log("Do you want to change this value in the database? Y/N"); - std::string line; - std::getline(std::cin, line); - QString result = QString::fromStdString(line); - if (std::cin.eof() || result.toLower() == "y") - { - fatalError = false; - UpdateValue(node.toElement(),field.toElement()); - changeValueInC(tableFound,field.toElement().attribute("code",""),field.toElement().attribute("description","")); - } + if (outputType == "h") + fatal("VNS:Value " + field.toElement().attribute("code","") + " of lookup table " + node.toElement().attribute("name","") + " has changed from \"" + fieldFound.toElement().attribute("description","") + "\" to \"" + field.toElement().attribute("description","") + "\""); else { + TcompError error; + error.code = "VNS"; + error.desc = "Value " + field.toElement().attribute("code","") + " of lookup table " + node.toElement().attribute("name","") + " from A not the same in B"; + error.table = node.toElement().attribute("name",""); + error.value = field.toElement().attribute("code",""); + error.from = fieldFound.toElement().attribute("description",""); + error.to = field.toElement().attribute("description",""); + errorList.append(error); fatalError = true; } } + else { - TcompError error; - error.code = "VNS"; - error.desc = "Value " + field.toElement().attribute("code","") + " of lookup table " + node.toElement().attribute("name","") + " from A not the same in B"; - error.table = node.toElement().attribute("name",""); - error.value = field.toElement().attribute("code",""); - error.from = fieldFound.toElement().attribute("description",""); - error.to = field.toElement().attribute("description",""); - errorList.append(error); + UpdateValue(node.toElement(),field.toElement()); + changeValueInC(tableFound,field.toElement().attribute("code",""),field.toElement().attribute("description","")); } } } diff --git a/utilities/mergeVersions/compareinsert.h b/utilities/mergeVersions/compareinsert.h index 64f3ba3f2..d5816c852 100644 --- a/utilities/mergeVersions/compareinsert.h +++ b/utilities/mergeVersions/compareinsert.h @@ -31,7 +31,7 @@ class compareInsert : public QObject { public: explicit compareInsert(QObject *parent = nullptr); - void setFiles(QString insertA, QString insertB, QString insertC, QString diffSQL, QString outputType); + void setFiles(QString insertA, QString insertB, QString insertC, QString diffSQL, QString outputType, QList toIgnore); int compare(); int createCFile(); int createDiffFile(); @@ -44,6 +44,7 @@ class compareInsert : public QObject QString outputC; QString outputD; QString outputType; + QList valuesToIgnore; bool fatalError; QDomDocument docB; QList diff; @@ -58,6 +59,7 @@ class compareInsert : public QObject void changeValueInC(QDomNode table, QString code, QString newDescription); void compareLKPTables(QDomNode table,QDomDocument &docB); void addDiffToTable(QString table, QString sql); + bool ignoreChange(QString table, QString value); }; #endif // COMPAREINSERT_H diff --git a/utilities/mergeVersions/generic.h b/utilities/mergeVersions/generic.h index 3187f0689..cd61ca0c9 100644 --- a/utilities/mergeVersions/generic.h +++ b/utilities/mergeVersions/generic.h @@ -42,4 +42,11 @@ struct compError }; typedef compError TcompError; +struct ignoreTableValues +{ + QString table; + QStringList values; +}; +typedef ignoreTableValues TignoreTableValues; + #endif // GENERIC_H diff --git a/utilities/mergeVersions/main.cpp b/utilities/mergeVersions/main.cpp index ef95117f4..1e4902691 100644 --- a/utilities/mergeVersions/main.cpp +++ b/utilities/mergeVersions/main.cpp @@ -22,6 +22,8 @@ License along with Merge Versions. If not, see #include #include "mainclass.h" +#include "generic.h" +#include int main(int argc, char *argv[]) { @@ -84,6 +86,7 @@ int main(int argc, char *argv[]) TCLAP::ValueArg dArg("d","diff","Output diff SQL script for create",false,"./diff-create.sql","string"); TCLAP::ValueArg DArg("D","diff","Output diff SQL script for insert",false,"./diff-insert.sql","string"); TCLAP::ValueArg oArg("o","outputype","Output type: (h)uman readble or (m)achine readble",false,"m","string"); + TCLAP::ValueArg iArg("i","ignore","Ignore changes in value descriptions. Indicated like Table1:value1,value2,...;Table2:value1,value2,..;..",false,"","string"); cmd.add(aArg); cmd.add(bArg); @@ -93,6 +96,7 @@ int main(int argc, char *argv[]) cmd.add(CArg); cmd.add(dArg); cmd.add(oArg); + cmd.add(iArg); //Parsing the command lines @@ -109,13 +113,35 @@ int main(int argc, char *argv[]) QString outputd = QString::fromUtf8(dArg.getValue().c_str()); QString outputD = QString::fromUtf8(DArg.getValue().c_str()); QString outputType = QString::fromUtf8(oArg.getValue().c_str()); + QString ignoreString = QString::fromUtf8(iArg.getValue().c_str()); + + QStringList ignoreTables; + QList valuesToIgnore; + if (ignoreString != "") + { + ignoreTables = ignoreString.split(";",QString::SkipEmptyParts); + for (int pos =0; pos < ignoreTables.count(); pos++) + { + if (ignoreTables[pos].indexOf(":") > 0) + { + QStringList parts = ignoreTables[pos].split(":",QString::SkipEmptyParts); + if (parts.count() == 2) + { + TignoreTableValues aIgnoreValue; + aIgnoreValue.table = parts[0]; + aIgnoreValue.values = parts[1].split(",",QString::SkipEmptyParts); + valuesToIgnore.append(aIgnoreValue); + } + } + } + } mainClass *task = new mainClass(&a); - task->setParameters(a_createXML,b_createXML,a_insertXML,b_insertXML,c_createXML,c_insertXML,outputd,outputD,outputType); + task->setParameters(a_createXML,b_createXML,a_insertXML,b_insertXML,c_createXML,c_insertXML,outputd,outputD,outputType,valuesToIgnore); QObject::connect(task, SIGNAL(finished()), &a, SLOT(quit())); QTimer::singleShot(0, task, SLOT(run())); - return a.exec(); + a.exec(); return task->returnCode; } diff --git a/utilities/mergeVersions/mainclass.cpp b/utilities/mergeVersions/mainclass.cpp index 55f7f6560..b2b0027bb 100644 --- a/utilities/mergeVersions/mainclass.cpp +++ b/utilities/mergeVersions/mainclass.cpp @@ -49,7 +49,7 @@ void mainClass::run() } compareInsert insert; - insert.setFiles(a_insertXML,b_insertXML,c_insertXML,d_insertSQL,output_type); + insert.setFiles(a_insertXML,b_insertXML,c_insertXML,d_insertSQL,output_type,valuesToIgnore); returnCode = insert.compare(); if (output_type != "h") { @@ -61,6 +61,7 @@ void mainClass::run() anError.setAttribute("table",errorList[pos].table); anError.setAttribute("field",errorList[pos].field); anError.setAttribute("code",errorList[pos].code); + anError.setAttribute("value",errorList[pos].value); anError.setAttribute("from",errorList[pos].from); anError.setAttribute("to",errorList[pos].to); eErrors.appendChild(anError); @@ -85,6 +86,7 @@ void mainClass::run() anError.setAttribute("table",errorList[pos].table); anError.setAttribute("field",errorList[pos].field); anError.setAttribute("code",errorList[pos].code); + anError.setAttribute("value",errorList[pos].value); anError.setAttribute("from",errorList[pos].from); anError.setAttribute("to",errorList[pos].to); eErrors.appendChild(anError); @@ -107,10 +109,10 @@ void mainClass::run() if (output_type != "h") { log(XMLResult.toString()); - } + } emit finished(); } -void mainClass::setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType) +void mainClass::setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType, QList toIgnore) { a_createXML = createA; b_createXML = createB; @@ -121,4 +123,5 @@ void mainClass::setParameters(QString createA, QString createB, QString insertA, d_createSQL = diffCreate; d_insertSQL = diffInsert; output_type = outputType; + valuesToIgnore = toIgnore; } diff --git a/utilities/mergeVersions/mainclass.h b/utilities/mergeVersions/mainclass.h index 6508faa8d..fa90827db 100644 --- a/utilities/mergeVersions/mainclass.h +++ b/utilities/mergeVersions/mainclass.h @@ -22,6 +22,7 @@ License along with Merge Versions. If not, see +#include "generic.h" class mainClass : public QObject { @@ -33,7 +34,7 @@ class mainClass : public QObject void finished(); public slots: void run(); - void setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType); + void setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType, QList toIgnore); private: QString a_createXML; QString b_createXML ; @@ -45,6 +46,7 @@ public slots: QString d_insertSQL; QString output_type; void log(QString message); + QList valuesToIgnore; }; #endif // MAINCLASS_H From 6662b97316c32b58ac05f09151076a9f836b50db Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Tue, 27 Aug 2019 11:13:25 -0600 Subject: [PATCH 11/24] Fixes in merge files --- utilities/mergeVersions/main.cpp | 44 ++++++-- utilities/mergeVersions/mainclass.cpp | 21 +++- utilities/mergeVersions/mainclass.h | 4 +- utilities/mergeVersions/mergecreate.cpp | 144 ++++++++++++++++++------ utilities/mergeVersions/mergecreate.h | 22 +++- 5 files changed, 183 insertions(+), 52 deletions(-) diff --git a/utilities/mergeVersions/main.cpp b/utilities/mergeVersions/main.cpp index 1e4902691..bcb1eefab 100644 --- a/utilities/mergeVersions/main.cpp +++ b/utilities/mergeVersions/main.cpp @@ -83,10 +83,12 @@ int main(int argc, char *argv[]) TCLAP::ValueArg BArg("B","inputB","Input insert XML file B (former)",true,"","string"); TCLAP::ValueArg cArg("c","outputc","Output create XML file C",false,"./combined-create.xml","string"); TCLAP::ValueArg CArg("C","outputC","Output insert XML file C",false,"./combined-insert.xml","string"); - TCLAP::ValueArg dArg("d","diff","Output diff SQL script for create",false,"./diff-create.sql","string"); - TCLAP::ValueArg DArg("D","diff","Output diff SQL script for insert",false,"./diff-insert.sql","string"); - TCLAP::ValueArg oArg("o","outputype","Output type: (h)uman readble or (m)achine readble",false,"m","string"); + TCLAP::ValueArg dArg("d","diffc","Output diff SQL script for create",false,"./diff-create.sql","string"); + TCLAP::ValueArg DArg("D","diffC","Output diff SQL script for insert",false,"./diff-insert.sql","string"); + TCLAP::ValueArg tArg("t","outputype","Output type: (h)uman readble or (m)achine readble",false,"m","string"); + TCLAP::ValueArg oArg("o","erroroutput","Error file ",false,"./error.xml","string"); TCLAP::ValueArg iArg("i","ignore","Ignore changes in value descriptions. Indicated like Table1:value1,value2,...;Table2:value1,value2,..;..",false,"","string"); + TCLAP::SwitchArg saveErrorSwitch("s","saveerror","Save errors to file", cmd, false); cmd.add(aArg); cmd.add(bArg); @@ -95,8 +97,10 @@ int main(int argc, char *argv[]) cmd.add(cArg); cmd.add(CArg); cmd.add(dArg); - cmd.add(oArg); + cmd.add(DArg); + cmd.add(tArg); cmd.add(iArg); + cmd.add(oArg); //Parsing the command lines @@ -112,8 +116,9 @@ int main(int argc, char *argv[]) QString c_insertXML = QString::fromUtf8(CArg.getValue().c_str()); QString outputd = QString::fromUtf8(dArg.getValue().c_str()); QString outputD = QString::fromUtf8(DArg.getValue().c_str()); - QString outputType = QString::fromUtf8(oArg.getValue().c_str()); + QString outputType = QString::fromUtf8(tArg.getValue().c_str()); QString ignoreString = QString::fromUtf8(iArg.getValue().c_str()); + QString errorFile = QString::fromUtf8(oArg.getValue().c_str()); QStringList ignoreTables; QList valuesToIgnore; @@ -127,17 +132,36 @@ int main(int argc, char *argv[]) QStringList parts = ignoreTables[pos].split(":",QString::SkipEmptyParts); if (parts.count() == 2) { - TignoreTableValues aIgnoreValue; - aIgnoreValue.table = parts[0]; - aIgnoreValue.values = parts[1].split(",",QString::SkipEmptyParts); - valuesToIgnore.append(aIgnoreValue); + int index; + index = -1; + for (int tbl = 0; tbl < valuesToIgnore.count(); tbl++) + { + if (valuesToIgnore[tbl].table == parts[0]) + { + index = tbl; + break; + } + } + if (index == -1) + { + TignoreTableValues aIgnoreValue; + aIgnoreValue.table = parts[0]; + aIgnoreValue.values = parts[1].split(",",QString::SkipEmptyParts); + valuesToIgnore.append(aIgnoreValue); + } + else + { + valuesToIgnore[index].values.append(parts[1].split(",",QString::SkipEmptyParts)); + } } } } } + bool saveToFile = saveErrorSwitch.getValue(); + mainClass *task = new mainClass(&a); - task->setParameters(a_createXML,b_createXML,a_insertXML,b_insertXML,c_createXML,c_insertXML,outputd,outputD,outputType,valuesToIgnore); + task->setParameters(a_createXML,b_createXML,a_insertXML,b_insertXML,c_createXML,c_insertXML,outputd,outputD,outputType,valuesToIgnore,saveToFile,errorFile); QObject::connect(task, SIGNAL(finished()), &a, SLOT(quit())); QTimer::singleShot(0, task, SLOT(run())); diff --git a/utilities/mergeVersions/mainclass.cpp b/utilities/mergeVersions/mainclass.cpp index b2b0027bb..ac50739c1 100644 --- a/utilities/mergeVersions/mainclass.cpp +++ b/utilities/mergeVersions/mainclass.cpp @@ -109,10 +109,25 @@ void mainClass::run() if (output_type != "h") { log(XMLResult.toString()); - } + if (save_to_file) + { + if (QFile::exists(error_file)) + QFile::remove(error_file); + QFile file(error_file); + if (file.open(QIODevice::WriteOnly | QIODevice::Text)) + { + QTextStream out(&file); + out.setCodec("UTF-8"); + XMLResult.save(out,1,QDomNode::EncodingFromTextStream); + file.close(); + } + else + log("Error: Cannot create xml error file"); + } + } emit finished(); } -void mainClass::setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType, QList toIgnore) +void mainClass::setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType, QList toIgnore, bool saveToFile, QString errorFile) { a_createXML = createA; b_createXML = createB; @@ -124,4 +139,6 @@ void mainClass::setParameters(QString createA, QString createB, QString insertA, d_insertSQL = diffInsert; output_type = outputType; valuesToIgnore = toIgnore; + error_file = errorFile; + save_to_file = saveToFile; } diff --git a/utilities/mergeVersions/mainclass.h b/utilities/mergeVersions/mainclass.h index fa90827db..5a8233125 100644 --- a/utilities/mergeVersions/mainclass.h +++ b/utilities/mergeVersions/mainclass.h @@ -34,7 +34,7 @@ class mainClass : public QObject void finished(); public slots: void run(); - void setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType, QList toIgnore); + void setParameters(QString createA, QString createB, QString insertA, QString insertB, QString createC, QString insertC, QString diffCreate, QString diffInsert, QString outputType, QList toIgnore, bool saveToFile, QString errorFile); private: QString a_createXML; QString b_createXML ; @@ -45,6 +45,8 @@ public slots: QString d_createSQL; QString d_insertSQL; QString output_type; + QString error_file; + bool save_to_file; void log(QString message); QList valuesToIgnore; }; diff --git a/utilities/mergeVersions/mergecreate.cpp b/utilities/mergeVersions/mergecreate.cpp index 16790a26b..a0b92d41d 100644 --- a/utilities/mergeVersions/mergecreate.cpp +++ b/utilities/mergeVersions/mergecreate.cpp @@ -19,6 +19,7 @@ License along with Merge Versions. If not, see mergeCreate::mergeCreate(QObject *parent) : QObject(parent) { @@ -77,14 +78,22 @@ int mergeCreate::compare() } fileB.close(); - QDomElement rootA = docA.documentElement(); - QDomElement rootB = docB.documentElement(); + rootA = docA.documentElement(); + rootB = docB.documentElement(); if ((rootA.tagName() == "XMLSchemaStructure") && (rootB.tagName() == "XMLSchemaStructure")) { //Comparing lookup tables compareLKPTables(rootA.firstChild().firstChild(),docB); //Comparing tables compareTables(rootA.firstChild().nextSibling().firstChild(),docB); + // Add foreign keys that were dropped earlier + diff.append("\n"); + for (int pos = 0; pos < create_lookup_rels.count(); pos++) + { + diff.append("ALTER TABLE " + create_lookup_rels[pos].table_name + " ADD INDEX " + create_lookup_rels[pos].rel_name + " (" + create_lookup_rels[pos].field_name + ");\n"); + diff.append("ALTER TABLE " + create_lookup_rels[pos].table_name + " ADD CONSTRAINT " + create_lookup_rels[pos].rel_name + " FOREIGN KEY (" + create_lookup_rels[pos].field_name + ") REFERENCES " + create_lookup_rels[pos].rel_table + " (" + create_lookup_rels[pos].rel_field + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n\n"); + } + //Process drops QDomNode lkpTables = rootB.firstChild(); for (int pos = 0; pos < dropTables.count(); pos++) @@ -100,6 +109,7 @@ int mergeCreate::compare() lkpTables.removeChild(a_table); } + //Do not create C and the Diff SQL if the merge cannot be done if (fatalError == false) { @@ -191,11 +201,61 @@ void mergeCreate::setFiles(QString createA, QString createB, QString createC, QS this->outputType = outputType; } -void mergeCreate::addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int newDec) +void mergeCreate::replace_lookup_relationships(QString table, QString field) +{ + QDomNodeList fields; + fields = rootB.elementsByTagName("field"); + QString sql; + for (int pos = 0; pos < fields.count(); pos++) + { + if ((fields.item(pos).toElement().attribute("rtable","") == table) && ((fields.item(pos).toElement().attribute("rfield","") == field))) + { + dropped_rels.append(fields.item(pos).toElement().attribute("rname","")); + sql = "ALTER TABLE " + fields.item(pos).parentNode().toElement().attribute("name") + " DROP FOREIGN KEY " + fields.item(pos).toElement().attribute("rname","") + ";\n"; + diff.append(sql); + sql = "ALTER TABLE " + fields.item(pos).parentNode().toElement().attribute("name") + " DROP INDEX " + fields.item(pos).toElement().attribute("rname","") + ";\n"; + diff.append(sql); + } + } + fields = rootA.elementsByTagName("field"); + for (int pos = 0; pos < fields.count(); pos++) + { + if ((fields.item(pos).toElement().attribute("rtable","") == table) && ((fields.item(pos).toElement().attribute("rfield","") == field))) + { + TreplaceRef a_replace; + a_replace.table_name = fields.item(pos).parentNode().toElement().attribute("name"); + a_replace.rel_name = fields.item(pos).toElement().attribute("rname",""); + a_replace.field_name = fields.item(pos).toElement().attribute("name",""); + a_replace.rel_table = fields.item(pos).toElement().attribute("rtable",""); + a_replace.rel_field = fields.item(pos).toElement().attribute("rfield",""); + create_lookup_rels.append(a_replace); + } + } +} + +bool mergeCreate::relation_is_dropped(QString name) +{ + for (int pos = 0; pos < dropped_rels.count() ; pos++) + { + if (dropped_rels[pos] == name) + { + return true; + } + } + return false; +} + +void mergeCreate::addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int newDec, bool islookup) { QString sql; QString fieldName; fieldName = eField.attribute("name",""); + + if (islookup && (eField.attribute("key","false") == "true")) + { + replace_lookup_relationships(table,fieldName); + } + if (eField.attribute("type","") == "varchar") { sql = "ALTER TABLE " + table + " MODIFY " + fieldName + " varchar(" + QString::number(newSize) + ");\n"; @@ -224,20 +284,29 @@ void mergeCreate::ddTableToDrop(QString name) } } -void mergeCreate::changeLookupRelationship(QString table, QDomElement a, QDomElement b) +void mergeCreate::changeLookupRelationship(QString table, QDomElement a, QDomElement b, bool islookup) { QString sql; QString oldcntname; QString newcntname; oldcntname = b.attribute("rname"); newcntname = a.attribute("rname"); - sql = "ALTER TABLE " + table + " DROP CONSTRAINT " + oldcntname + ";\n"; - diff.append(sql); - sql = "ALTER TABLE " + table + " DROP INDEX " + oldcntname + ";\n\n"; - diff.append(sql); - addAlterFieldToDiff(table,a,a.attribute("size","0").toInt(),0); - sql = "ALTER TABLE " + table + " ADD INDEX " + newcntname + ";\n"; - diff.append(sql); + bool create_new_rel; + create_new_rel = false; + if (!relation_is_dropped(oldcntname)) + { + create_new_rel = true; + sql = "ALTER TABLE " + table + " DROP FOREIGN KEY " + oldcntname + ";\n"; + diff.append(sql); + sql = "ALTER TABLE " + table + " DROP INDEX " + oldcntname + ";\n\n"; + diff.append(sql); + } + addAlterFieldToDiff(table,a,a.attribute("size","0").toInt(),0,islookup); + if (create_new_rel) + { + sql = "ALTER TABLE " + table + " ADD INDEX " + newcntname + " (" + a.attribute("name") + ");\n"; + diff.append(sql); + } for (int pos = 0; pos < insert_diff.count(); pos++) { if (insert_diff[pos].table == a.attribute("rtable")) @@ -251,8 +320,11 @@ void mergeCreate::changeLookupRelationship(QString table, QDomElement a, QDomEle diff.append("\n"); } } - sql = "ALTER TABLE " + table + " ADD CONSTRAINT " + newcntname + " FOREIGN KEY (" + a.attribute("name") + ") REFERENCES " + a.attribute("rtable") + " (" + a.attribute("rfield") + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n\n"; - diff.append(sql); + if (create_new_rel) + { + sql = "ALTER TABLE " + table + " ADD CONSTRAINT " + newcntname + " FOREIGN KEY (" + a.attribute("name") + ") REFERENCES " + a.attribute("rtable") + " (" + a.attribute("rfield") + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n\n"; + diff.append(sql); + } addTableToDrop(b.attribute("rtable")); } @@ -642,7 +714,7 @@ QString mergeCreate::getFieldDefinition(QDomElement field) return result; } -void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b) +void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b, bool islookup = false) { int newSize; int newDec; @@ -650,6 +722,16 @@ void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b) result = compareFields(a,b,newSize,newDec); if (result != "") { + if (outputType != "h") + { + TcompError error; + error.code = result; + error.table = eTable.toElement().attribute("name",""); + error.field = a.attribute("name",""); + error.from = getFieldDefinition(b); + error.to = getFieldDefinition(a); + errorList.append(error); + } if ((result == "FIC") || (result == "CHR") || (result == "FTC")) { if (result == "FIC") @@ -663,7 +745,7 @@ void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b) } b.setAttribute("size",newSize); b.setAttribute("decsize",newDec); - addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); + addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec,islookup); } else { @@ -673,7 +755,7 @@ void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b) { log("CHR: The relationship of field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " will be changed from " + b.attribute("rtable") + "." + b.attribute("rfield") + " to " + a.attribute("rtable") + "." + a.attribute("rfield")); } - changeLookupRelationship(eTable.toElement().attribute("name",""),a,b); + changeLookupRelationship(eTable.toElement().attribute("name",""),a,b,islookup); b.setAttribute("rtable",a.attribute("rtable")); b.setAttribute("rfield",a.attribute("rfield")); b.setAttribute("type",a.attribute("type")); @@ -684,12 +766,12 @@ void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b) { if (outputType == "h") { - log("FTC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from " + b.attribute("type") + " to " + a.attribute("type") + " which is allowed"); - b.setAttribute("size",newSize); - b.setAttribute("decsize",newDec); - b.setAttribute("type",a.attribute("type")); - addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec); + log("FTC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " changed type from " + b.attribute("type") + " to " + a.attribute("type") + " which is allowed"); } + b.setAttribute("size",newSize); + b.setAttribute("decsize",newDec); + b.setAttribute("type",a.attribute("type")); + addAlterFieldToDiff(eTable.toElement().attribute("name",""),a,newSize,newDec,islookup); } } } @@ -707,16 +789,6 @@ void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b) log("FDC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " has descreased in size. This change will be ignored"); } } - if (outputType != "h") - { - TcompError error; - error.code = result; - error.table = eTable.toElement().attribute("name",""); - error.field = a.attribute("name",""); - error.from = getFieldDefinition(b); - error.to = getFieldDefinition(a); - errorList.append(error); - } } } @@ -741,7 +813,7 @@ void mergeCreate::compareLKPTables(QDomNode table,QDomDocument &docB) QDomNode fieldFound = findField(tablefound,eField.attribute("name","")); if (!fieldFound.isNull()) { - checkField(tablefound,field.toElement(),fieldFound.toElement()); + checkField(tablefound,field.toElement(),fieldFound.toElement(),true); } else { @@ -841,7 +913,7 @@ void mergeCreate::compareTables(QDomNode table,QDomDocument &docB) { TcompError error; error.code = "TNS"; - error.table = "NA"; + error.table = eTable.toElement().attribute("name",""); error.field = "NA"; error.from = tablefound.parentNode().toElement().attribute("name",""); error.to = table.parentNode().toElement().attribute("name",""); @@ -881,10 +953,10 @@ void mergeCreate::compareTables(QDomNode table,QDomDocument &docB) { TcompError error; error.code = "TWP"; - error.table = "NA"; + error.table = eTable.toElement().attribute("name",""); error.field = "NA"; - error.from = "NULL"; - error.to = parentTableName; + error.from = parentTableName; + error.to = "NULL"; errorList.append(error); } } diff --git a/utilities/mergeVersions/mergecreate.h b/utilities/mergeVersions/mergecreate.h index a6970d701..66c7ebb19 100644 --- a/utilities/mergeVersions/mergecreate.h +++ b/utilities/mergeVersions/mergecreate.h @@ -49,6 +49,16 @@ class mergeCreate : public QObject }; typedef rtableDef TrtableDef; + struct replaceRef + { + QString table_name; + QString rel_name; + QString field_name; + QString rel_table; + QString rel_field; + }; + typedef replaceRef TreplaceRef; + public: explicit mergeCreate(QObject *parent = nullptr); int compare(); @@ -57,6 +67,8 @@ class mergeCreate : public QObject QStringList getInsertTablesUsed(); QList getErrorList(); private: + QDomElement rootA; + QDomElement rootB; QString inputA; QString inputB ; QString outputC; @@ -70,9 +82,11 @@ class mergeCreate : public QObject QList rtables; QStringList dropTables; QStringList insertTablesUsed; - void addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int newDec); + QList create_lookup_rels; + QStringList dropped_rels; + void addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int newDec, bool islookup); void ddTableToDrop(QString name); - void changeLookupRelationship(QString table, QDomElement a, QDomElement b); + void changeLookupRelationship(QString table, QDomElement a, QDomElement b, bool islookup); void addFieldToDiff(QString table, QDomElement eField); void addFieldToRTables(QString parentTable, QString rTable, QString field, QString rField, QString rname, bool isLookUp); void addTableToSDiff(QDomNode table, bool lookUp); @@ -82,10 +96,12 @@ class mergeCreate : public QObject QDomNode findTable(QDomDocument docB,QString tableName); QString compareFields(QDomElement a, QDomElement b, int &newSize, int &newDec); QString getFieldDefinition(QDomElement field); - void checkField(QDomNode eTable, QDomElement a, QDomElement b); + void checkField(QDomNode eTable, QDomElement a, QDomElement b, bool islookup); void compareLKPTables(QDomNode table,QDomDocument &docB); void compareTables(QDomNode table,QDomDocument &docB); void addTableToDrop(QString name); + void replace_lookup_relationships(QString table, QString field); + bool relation_is_dropped(QString name); }; #endif // MERGECREATE_H From 3015f562e2d46ed94047fb1780fc2007ded400a8 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Sat, 31 Aug 2019 23:21:36 -0600 Subject: [PATCH 12/24] Fix several issues --- JXFormToMysql/main.cpp | 919 +++++++++++++++++------------------------ 1 file changed, 389 insertions(+), 530 deletions(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index fce4f1ea5..0d0f2f495 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -36,6 +36,7 @@ License along with JXFormToMySQL. If not, see #include #include +#include #include #include @@ -60,17 +61,22 @@ int numColumns; int numColumnsInData; QStringList duplicatedTables; bool justCheck; -bool merge; QStringList requiredFiles; -QDomElement Insertroot; -QDomElement Createroot; -QStringList merge_lkptable_errors; //********************************************Global structures**************************************** //Structure that holds the description of each lkpvalue separated by language +struct duplicatedLookUp +{ + QStringList tables; + QString sameas; +}; +typedef duplicatedLookUp TduplicatedLookUp; + +QList< TduplicatedLookUp> duplicated_lookups; + struct tblwitherror { QString name; @@ -115,7 +121,6 @@ struct fieldDef QString rTable; //Related table QString rField; //Related field QString rName; //Contraint name - bool mergeFine = false; bool key; //Whether the field is key QString xmlCode; //The field XML code /xx/xx/xx/xx QString xmlFullPath; //The field XML code /xx/xx/xx/xx with groups @@ -1063,6 +1068,20 @@ void log(QString message) printf("%s", temp.toUtf8().data()); } +void report_file_error(QString file_name) +{ + QDomDocument XMLResult; + XMLResult = QDomDocument("XMLResult"); + QDomElement XMLRoot; + XMLRoot = XMLResult.createElement("XMLResult"); + XMLResult.appendChild(XMLRoot); + QDomElement eFileError; + eFileError = XMLResult.createElement("file"); + eFileError.setAttribute("name",file_name); + XMLRoot.appendChild(eFileError); + log(XMLResult.toString()); +} + void cb1(void *s, size_t, void *) { char* charData; @@ -1133,7 +1152,7 @@ void cb2(int , void *) } int convertCSVToSQLite(QString fileName, QDir tempDirectory, QSqlDatabase database) -{ +{ FILE *fp; struct csv_parser p; char buf[4096]; @@ -1180,7 +1199,12 @@ int convertCSVToSQLite(QString fileName, QDir tempDirectory, QSqlDatabase databa if (CSVColumError) { - log("The CSV \"" + fileName + "\" has invalid characters. Only : and _ are allowed"); + if (outputType == "h") + log("The CSV \"" + fileName + "\" has invalid characters. Only : and _ are allowed"); + else + { + report_file_error(fileName); + } exit(14); } @@ -1354,158 +1378,44 @@ bool areValuesStrings(QList values) return false; } +QString get_related_usage(QString table) +{ + for (int pos = 0; pos < tables.count(); pos++) + { + for (int pos2 = 0; pos2 < tables[pos].fields.count(); pos2++) + { + if (tables[pos].fields[pos2].rTable == table) + return tables[pos].fields[pos2].name; + } + } + return ""; +} + //This fuction checkd wheter a lookup table is duplicated. //If there is a match then returns such table -TtableDef getDuplicatedLkpTable(QString lkptname, QList thisValues, bool &changeSize, int &newCodeSize, bool &changedRel, QString newCodeType, bool useMergedTables = true) -{ - changeSize = false; - newCodeType = ""; - newCodeSize = -1; - int pos2; - bool same; +TtableDef checkDuplicatedLkpTable(QString table, QList thisValues) +{ TtableDef empty; empty.name = "EMPTY"; empty.isLoop = false; empty.isOSM = false; empty.isGroup = false; + + bool found; QList currentValues; QString thisDesc; QString currenDesc; - changedRel = false; //Move the new list of values to a new list and sort it by code qSort(thisValues.begin(),thisValues.end(),lkpComp); QString defLangCode; defLangCode = getLanguageCode(getDefLanguage()); - bool table_found; - table_found = false; - if (merge && useMergedTables) - { - for (int pos = 0; pos <= merging_tables.count()-1; pos++) - { - if (merging_tables[pos].islookup) - { - if (merging_tables[pos].name == lkptname) - { - table_found = true; - currentValues.clear(); - currentValues.append(merging_tables[pos].lkpValues); - qSort(currentValues.begin(),currentValues.end(),lkpComp); - if (currentValues.count() == thisValues.count()) //Same number of values - { - same = true; - for (pos2 = 0; pos2 <= merging_tables[pos].lkpValues.count()-1;pos2++) - { - //Compares if an item in the list dont have same code or same description - thisDesc = getDescForLanguage(thisValues[pos2].desc,defLangCode); - currenDesc = getDescForLanguage(currentValues[pos2].desc,defLangCode); - if ((currentValues[pos2].code.simplified().toLower() != thisValues[pos2].code.simplified().toLower()) || - (currenDesc.simplified().toLower() != thisDesc.simplified().toLower())) - { - merge_lkptable_errors.append(lkptname); - same = false; - break; - } - } - if (!same) - { - merge_lkptable_errors.append(lkptname); - return empty; - } - else - { - tables.append(merging_tables[pos]); - return merging_tables[pos]; - } - } - else - { - if (currentValues.count() > thisValues.count()) - { - merge_lkptable_errors.append(lkptname); - return empty; - } - else - { - bool found; - for (pos2 = 0; pos2 < currentValues.count(); pos2++) - { - //Compares if an item in the current values does not exist in the new list - currenDesc = getDescForLanguage(currentValues[pos2].desc,defLangCode); - found = false; - for (int pos3 = 0; pos3 < thisValues.count(); pos3++) - { - thisDesc = getDescForLanguage(thisValues[pos3].desc,defLangCode); - if ((thisValues[pos3].code == currentValues[pos2].code) && (currenDesc == thisDesc)) - found = true; - } - if (!found) - break; - } - if (!found) - { - merge_lkptable_errors.append(lkptname); - return empty; - } - else - { - //Look for values that are not part of current - bool found; - for (pos2 = 0; pos2 < thisValues.count(); pos2++) - { - found = false; - for (int pos3 = 0; pos3 < currentValues.count(); pos3++) - { - if (thisValues[pos2].code == currentValues[pos3].code) - found = true; - } - if (!found) - { - merging_tables[pos].lkpValues.append(thisValues[pos2]); - } - } - bool nchangeSize; - int nnewCodeSize; - bool nchangedRel; - QString nnewCodeType; - TtableDef existing_table = getDuplicatedLkpTable(lkptname, merging_tables[pos].lkpValues,nchangeSize,nnewCodeSize,nchangedRel,nnewCodeType,false); - if (existing_table.name != "EMPTY") - { - changeSize = true; - changedRel = true; - newCodeSize = getMaxValueLength(existing_table.lkpValues); - if (areValuesStrings(existing_table.lkpValues)) - newCodeType = "varchar"; - else - newCodeType = "int"; - return existing_table; - } - changeSize = true; - if (areValuesStrings(merging_tables[pos].lkpValues)) - newCodeType = "varchar"; - else - newCodeType = "int"; - newCodeSize = getMaxValueLength(merging_tables[pos].lkpValues); - int newDescSize; - newDescSize = getMaxDescLength(merging_tables[pos].lkpValues); - merging_tables[pos].fields[0].size = newCodeSize; - merging_tables[pos].fields[0].type = newCodeType; - merging_tables[pos].fields[1].size = newDescSize; - tables.append(merging_tables[pos]); - return merging_tables[pos]; - } - } - } - } - } - } - } - if (!table_found) - { - for (int pos = 0; pos <= tables.count()-1; pos++) + for (int pos = 0; pos < tables.count(); pos++) + { + if (tables[pos].islookup) { - if (tables[pos].islookup) + if (tables[pos].name != table) { //Move the current list of values to a new list and sort it by code currentValues.clear(); @@ -1514,8 +1424,8 @@ TtableDef getDuplicatedLkpTable(QString lkptname, QList thisValues, b if (currentValues.count() == thisValues.count()) //Same number of values { - same = true; - for (pos2 = 0; pos2 <= tables[pos].lkpValues.count()-1;pos2++) + found = true; + for (int pos2 = 0; pos2 < currentValues.count(); pos2++) { //Compares if an item in the list dont have same code or same description thisDesc = getDescForLanguage(thisValues[pos2].desc,defLangCode); @@ -1524,16 +1434,62 @@ TtableDef getDuplicatedLkpTable(QString lkptname, QList thisValues, b if ((currentValues[pos2].code.simplified().toLower() != thisValues[pos2].code.simplified().toLower()) || (currenDesc.simplified().toLower() != thisDesc.simplified().toLower())) { - same = false; + found = false; break; } } - if (same) + if (found) { - return tables[pos]; + int idx; + idx = -1; + for (int pos2=0; pos2 < duplicated_lookups.count(); pos2++) + { + if (duplicated_lookups[pos2].sameas == tables[pos].name) + { + idx = pos2; + break; + } + found = false; + for (int pos3 = 0; pos3 < duplicated_lookups[pos2].tables.count(); pos3++) + { + if (duplicated_lookups[pos2].tables[pos3] == table) + { + idx = pos2; + break; + } + } + if (found) + break; + } + if (idx == -1) + { + found = false; + TduplicatedLookUp duplicated; + duplicated.sameas = tables[pos].name; + duplicated.tables.append(table); + duplicated_lookups.append(duplicated); + } + else + { + found = false; + for (int pos2 = 0; pos2 < duplicated_lookups[idx].tables.count(); pos2++) + { + if (duplicated_lookups[idx].tables[pos2] == table) + { + found = true; + break; + } + } + if (!found) + duplicated_lookups[idx].tables.append(table); + } } } } + else + { + return tables[pos]; + } } } return empty; @@ -1963,9 +1919,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr if (isRelatedTableLookUp(tables[pos].fields[clm].rTable)) { - createFieldNode.setAttribute("rlookup","true"); - if (tables[pos].fields[clm].mergeFine) - createFieldNode.setAttribute("mergefine","true"); + createFieldNode.setAttribute("rlookup","true"); } } @@ -1995,8 +1949,6 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr if (isRelatedTableLookUp(tables[pos].fields[clm].rTable)) { createFieldNode.setAttribute("rlookup","true"); - if (tables[pos].fields[clm].mergeFine) - createFieldNode.setAttribute("mergefine","true"); } } tables[pos].tableCreteElement.appendChild(createFieldNode); @@ -2026,9 +1978,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr createFieldNode.setAttribute("rname","fk_" + tables[pos].fields[clm].rName); if (isRelatedTableLookUp(tables[pos].fields[clm].rTable)) { - createFieldNode.setAttribute("rlookup","true"); - if (tables[pos].fields[clm].mergeFine) - createFieldNode.setAttribute("mergefine","true"); + createFieldNode.setAttribute("rlookup","true"); } } tables[pos].tableCreteElement.appendChild(createFieldNode); @@ -2222,7 +2172,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr // Append UUIDs triggers to the file but only if the UUID is null or if it is not an uuid sql = sql + "delimiter $$\n\n"; - sql = sql + "CREATE TRIGGER T" + strTriggerUUID + " BEFORE INSERT ON " + tables[pos].name + " FOR EACH ROW BEGIN IF (new.rowuuid IS NULL) THEN SET new.rowuuid = uuid(); ELSE IF (new.rowuuid NOT REGEXP '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}') THEN SET new.rowuuid = uuid(); END IF; END IF; END;$$\n"; + sql = sql + "CREATE TRIGGER T" + strTriggerUUID + " BEFORE INSERT ON " + tables[pos].name + " FOR EACH ROW BEGIN IF (new.rowuuid IS NULL) THEN SET new.rowuuid = uuid(); ELSE IF (new.rowuuid NOT REGEXP '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{}') THEN SET new.rowuuid = uuid(); END IF; END IF; END;$$\n"; sql = sql + "delimiter ;\n\n"; sqlCreateStrm << sql; //Add the triggers to the SQL DDL file @@ -2683,41 +2633,23 @@ int getMaxMSelValueLength(QList values) return res.length(); } -//Returns wheter or not a the values of a lookup table are yes/no values. Used in combination to yes/no lookup values to -//create or not a lookup table of yes/no. -// NOTE this is a very crude function and only catches a couple of cases... +//Returns wheter or not a the values of a lookup table are yes/no values. bool isLookUpYesNo(QList values) { bool hasNo; bool hasYes; - int lkp; - QString lkpValue; - QString defLang; + int lkp; + QString defLang; defLang = getLanguageCode(getDefLanguage()); if (values.count() == 2 ) { hasNo = false; hasYes = false; for (lkp = 0; lkp <= values.count()-1;lkp++) - { - lkpValue = getDescForLanguage(values[lkp].desc,defLang); - - //Evaluate 0 = No, 1 = yes - if ((values[lkp].code.toInt() == 0) && ((lkpValue.trimmed().toLower() == strNo.toLower()) || (lkpValue.trimmed().toLower() == "0-" + strNo.toLower()) || (lkpValue.trimmed().toLower() == "0- " + strNo.toLower()))) - hasNo = true; - if ((values[lkp].code.toInt() == 1) && ((lkpValue.trimmed().toLower() == strYes.toLower()) || (lkpValue.trimmed().toLower() == "1-" + strYes.toLower()) || (lkpValue.trimmed().toLower() == "1- " + strYes.toLower()))) - hasYes = true; - - //Evalues 1 = yes, 2 = no - if ((values[lkp].code.toInt() == 1) && ((lkpValue.trimmed().toLower() == strYes.toLower()) || (lkpValue.trimmed().toLower() == "1-"+ strYes.toLower()) || (lkpValue.trimmed().toLower() == "1- "+ strYes.toLower()))) - hasNo = true; - if ((values[lkp].code.toInt() == 2) && ((lkpValue.trimmed().toLower() == strNo.toLower()) || (lkpValue.trimmed().toLower() == "2-" + strNo.toLower()) || (lkpValue.trimmed().toLower() == "2- " + strNo.toLower()))) - hasYes = true; - - //Evalues 1 = no, 2 = Yes - if ((values[lkp].code.toInt() == 2) && ((lkpValue.trimmed().toLower() == strYes.toLower()) || (lkpValue.trimmed().toLower() == "2-"+ strYes.toLower()) || (lkpValue.trimmed().toLower() == "2- "+ strYes.toLower()))) + { + if (values[lkp].code.trimmed().toLower() == strYes.trimmed().toLower()) hasNo = true; - if ((values[lkp].code.toInt() == 1) && ((lkpValue.trimmed().toLower() == strNo.toLower()) || (lkpValue.trimmed().toLower() == "1-" + strNo.toLower()) || (lkpValue.trimmed().toLower() == "1- " + strNo.toLower()))) + if (values[lkp].code.trimmed().toLower() == strNo.trimmed().toLower()) hasYes = true; } if (hasYes && hasNo) @@ -2984,14 +2916,14 @@ QList getSelectValuesFromXML(QString variableName, QString fileName, if (!file.open(QIODevice::ReadOnly)) { log("Cannot open XML resource file: "+ xmlFile); - result = 21; + exit(1); } if (!xmlDocument.setContent(&file)) { log("Cannot parse XML resource file: "+ xmlFile); file.close(); - result = 21; + exit(1); } file.close(); QDomNodeList items; @@ -3026,10 +2958,26 @@ QList getSelectValuesFromXML(QString variableName, QString fileName, value.desc.append(desc); } else - { - if (!justCheck) + { + if (outputType == "m") log("Language " + parts[1] + " was not found in the parameters. Please indicate it as default language (-d) or as other lannguage (-l)"); - result = 4; + else + { + QDomDocument XMLResult; + XMLResult = QDomDocument("XMLResult"); + QDomElement XMLRoot; + XMLRoot = XMLResult.createElement("XMLResult"); + XMLResult.appendChild(XMLRoot); + QDomElement eLanguages; + eLanguages = XMLResult.createElement("languages"); + QDomElement eLanguage; + eLanguage = XMLResult.createElement("language"); + eLanguage.setAttribute("name",parts[1]); + eLanguages.appendChild(eLanguage); + XMLRoot.appendChild(eLanguages); + log(XMLResult.toString()); + } + exit(4); } } else @@ -3045,10 +2993,14 @@ QList getSelectValuesFromXML(QString variableName, QString fileName, res.append(value); } else - { - if (!justCheck) + { + if (outputType == "h") log("Unable to find the column called name in the XML file \"" + fileName + "\". Is this a valid XML resource"); - result = 12; + else + { + report_file_error(fileName); + } + exit(12); } } if (hasOrOther) @@ -3070,17 +3022,28 @@ QList getSelectValuesFromXML(QString variableName, QString fileName, } } else - { - if (!justCheck) + { + if (outputType == "h") log("Unable to retreive items from the XML file \"" + fileName + "\". Is this a valid XML resource"); - result = 12; + else + { + report_file_error(fileName); + } + exit(12); } } else { if (!justCheck) - log("There is no XML file for \"" + fileName + "\". Did you add it when you ran JXFormToMySQL?"); - result = 13; + { + if (outputType == "h") + log("There is no XML file for \"" + fileName + "\". Did you add it when you ran JXFormToMySQL?"); + else + { + report_file_error(fileName); + } + exit(11); + } } return res; } @@ -3131,47 +3094,70 @@ QList getSelectValuesFromCSV2(QString variableName, QString fileName, TlkpValue value; value.code = query.value(fixColumnName(codeColumn)).toString(); for (int pos = 0; pos <= descColumns.count()-1; pos++) - { - queryValue = query.value(fixColumnName(descColumns[pos])); - if (queryValue.isValid()) + { + if (query.record().contains(fixColumnName(descColumns[pos]))) { - if (descColumns[pos].indexOf("::") >= 0) + queryValue = query.value(fixColumnName(descColumns[pos])); + if (queryValue.isValid()) { - QStringList parts; - parts = descColumns[pos].split("::",QString::SkipEmptyParts); - QString langCode; - langCode = getLanguageCode(parts[1]); - if (langCode != "") + if (descColumns[pos].indexOf("::") >= 0) { - TlngLkpDesc desc; - desc.desc = queryValue.toString(); - desc.langCode = langCode; - value.desc.append(desc); + QStringList parts; + parts = descColumns[pos].split("::",QString::SkipEmptyParts); + QString langCode; + langCode = getLanguageCode(parts[1]); + if (langCode != "") + { + TlngLkpDesc desc; + desc.desc = queryValue.toString(); + desc.langCode = langCode; + value.desc.append(desc); + } + else + { + if (outputType == "m") + log("Language " + parts[1] + " was not found in the parameters. Please indicate it as default language (-d) or as other lannguage (-l)"); + else + { + QDomDocument XMLResult; + XMLResult = QDomDocument("XMLResult"); + QDomElement XMLRoot; + XMLRoot = XMLResult.createElement("XMLResult"); + XMLResult.appendChild(XMLRoot); + QDomElement eLanguages; + eLanguages = XMLResult.createElement("languages"); + QDomElement eLanguage; + eLanguage = XMLResult.createElement("language"); + eLanguage.setAttribute("name",parts[1]); + eLanguages.appendChild(eLanguage); + XMLRoot.appendChild(eLanguages); + log(XMLResult.toString()); + } + exit(4); + } } else { - if (!justCheck) - log("Language " + parts[1] + " was not found in the parameters. Please indicate it as default language (-d) or as other lannguage (-l)"); - result = 4; + TlngLkpDesc desc; + desc.desc = queryValue.toString(); + desc.langCode = getLanguageCode(getDefLanguage()); + value.desc.append(desc); } } - else - { - TlngLkpDesc desc; - desc.desc = queryValue.toString(); - desc.langCode = getLanguageCode(getDefLanguage()); - value.desc.append(desc); - } } } checkSelectValue(variableName,res,value.code); res.append(value); } else - { - if (!justCheck) + { + if (outputType == "h") log("The file " + fileName + "is not a valid ODK resource file"); - result = 20; + else + { + report_file_error(fileName); + } + exit(15); } } if (hasOrOther) @@ -3195,24 +3181,34 @@ QList getSelectValuesFromCSV2(QString variableName, QString fileName, return res; } else - { - if (!justCheck) + { + if (outputType == "h") log("Unable to retreive data for search \"" + fileName + "\". Reason: " + query.lastError().databaseText() + ". Maybe the \"name column\" or any of the \"labels columns\" do not exist in the CSV?"); - result = 12; + else + { + report_file_error(fileName); + } + exit(15); } } else - { - if (!justCheck) - log("Cannot create SQLite database " + sqliteFile); - result = 1; + { + log("Cannot create SQLite database " + sqliteFile); + exit(1); } } else { if (!justCheck) - log("There is no SQLite file for file \"" + fileName + "\". Did you add it as CSV when you ran JXFormToMySQL?"); - result = 13; + { + if (outputType == "h") + log("There is no SQLite file for file \"" + fileName + "\". Did you add it as CSV when you ran JXFormToMySQL?"); + else + { + report_file_error(fileName); + } + exit(13); + } } return res; } @@ -3313,30 +3309,56 @@ QList getSelectValuesFromCSV(QString searchExpresion, QJsonArray choi { if (!justCheck) log("Unable to retreive data for search \"" + file + "\". Reason: " + query.lastError().databaseText() + ". Maybe the \"name column\" or any of the \"labels columns\" do not exist in the CSV?"); - result = 12; + exit(15); } } else - { - if (!justCheck) - log("Cannot create SQLite database " + sqliteFile); - result = 1; + { + log("Cannot create SQLite database " + sqliteFile); + exit(1); } } else { if (!justCheck) - log("There is no SQLite file for search \"" + file + "\". Did you add it as CSV when you ran JXFormToMySQL?"); - result = 13; + { + if (outputType == "h") + log("There is no SQLite file for search \"" + file + "\". Did you add it as CSV when you ran JXFormToMySQL?"); + else + { + report_file_error(file); + } + exit(13); + } } } else { - if (codeColumn != "") - log("Cannot locate the code column for the search select \"" + variableName + "\""); - if (descColumns.count() == 0) - log("Cannot locate a description column for the search select \"" + variableName + "\""); - result = 16; + if (outputType == "h") + { + if (codeColumn != "") + log("Cannot locate the code column for the search select \"" + variableName + "\""); + if (descColumns.count() == 0) + log("Cannot locate a description column for the search select \"" + variableName + "\""); + } + else + { + QDomDocument XMLResult; + XMLResult = QDomDocument("XMLResult"); + QDomElement XMLRoot; + XMLRoot = XMLResult.createElement("XMLResult"); + XMLResult.appendChild(XMLRoot); + QDomElement eFileError; + eFileError = XMLResult.createElement("search"); + eFileError.setAttribute("name",variableName); + if (codeColumn != "") + eFileError.setAttribute("codenotfound","true"); + if (descColumns.count() == 0) + eFileError.setAttribute("descnotfound","true"); + XMLRoot.appendChild(eFileError); + log(XMLResult.toString()); + } + exit(16); } return res; } @@ -3432,11 +3454,11 @@ void getReferenceForSelectAt(QString calculateExpresion,QString &fieldType, int } } -QString getUUIDCode(bool last12 = false) +QString getUUIDCode(bool last = false) { QUuid triggerUUID=QUuid::createUuid(); QString strTriggerUUID=triggerUUID.toString().replace("{","").replace("}","").replace("-","_"); - if (last12) + if (last) return strTriggerUUID.left(12); else return strTriggerUUID; @@ -3452,7 +3474,6 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) { QList values; values.append(getSelectValues(variableName,fieldObject.value("choices").toArray(),false)); - TfieldDef aField; aField.selectSource = "NONE"; aField.name = fixField(variableName.toLower()); @@ -3479,12 +3500,8 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) //Creating the lookp table if its neccesary if (isLookUpYesNo(values) == false) //Do not process yes/no values as lookup tables { - QString table_name = "lkp" + fixField(variableName.toLower()); - bool changeSize; - int newCodeSize; - bool changedRel; - QString newCodeType; - TtableDef lkpTable = getDuplicatedLkpTable(table_name,values,changeSize,newCodeSize,changedRel,newCodeType); + QString table_name = "lkp" + fixField(variableName.toLower()); + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -3540,29 +3557,14 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) lkpTable.fields.append(lkpDesc); aField.rTable = lkpTable.name; aField.rField = lkpCode.name; - aField.rName = getUUIDCode(); + aField.rName = getUUIDCode(); tables.append(lkpTable); } else { - if (outputType == "h") - log("Lookup table for field " + variableName + " is the same as " + lkpTable.name + ". Using " + lkpTable.name); + aField.rName = getUUIDCode(); aField.rTable = lkpTable.name; aField.rField = getKeyField(lkpTable.name); - aField.rName = getUUIDCode(); - if (merge) - { - if (newCodeType != "") - aField.type = newCodeType; - if (changeSize) - { - aField.size = newCodeSize; - } - if (changedRel) - { - aField.mergeFine = true; - } - } } } } @@ -3581,6 +3583,7 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) aField.selectSource = "NONE"; aField.selectListName = "NONE"; aField.odktype = "NONE"; + aField.key = false; aField.type = "text"; aField.size = 255; aField.decSize = 0; @@ -3803,7 +3806,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q int result; values.append(getSelectValuesFromCSV2(fixField(variableName),"itemsets.csv",selectHasOrOther(variableType),result,dir,database,queryField)); if (result != 0) - exit(19); + exit(15); } } @@ -3872,12 +3875,14 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q //Creating the lookp table if its neccesary if (isLookUpYesNo(values) == false) //Do not process yes/no values as lookup tables { - QString table_name = "lkp" + fixField(variableName.toLower()); - bool changeSize; - int newCodeSize; - bool changedRel; - QString newCodeType; - TtableDef lkpTable = getDuplicatedLkpTable(table_name, values, changeSize, newCodeSize, changedRel, newCodeType); + QString listName; + QJsonValue listValue = fieldObject.value("query"); + if (!listValue.isUndefined()) + listName = fixField(listValue.toString().toLower()); + else + listName = fixField(variableName.toLower()); + QString table_name = "lkp" + listName; + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -3933,27 +3938,14 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q lkpTable.fields.append(lkpDesc); aField.rTable = lkpTable.name; aField.rField = lkpCode.name; - aField.rName = getUUIDCode(); + aField.rName = getUUIDCode(); tables.append(lkpTable); } else { - if (outputType == "h") - log("Lookup table for field " + variableName + " is the same as " + lkpTable.name + ". Using " + lkpTable.name); + aField.rName = getUUIDCode(); aField.rTable = lkpTable.name; aField.rField = getKeyField(lkpTable.name); - aField.rName = getUUIDCode(); - if (merge) - { - if (newCodeType != "") - aField.type = newCodeType; - if (changeSize) - { - aField.size = newCodeSize; - } - if (changedRel) - aField.mergeFine = true; - } } } } @@ -3988,7 +3980,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q oField.multiSelectTable = ""; oField.rField = ""; oField.rTable = ""; - oField.size = 120; + oField.size = 0; oField.type = "varchar"; tables[tblIndex].fields.append(oField); } @@ -4102,19 +4094,21 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q mselKeyField.size = getMaxValueLength(values); mselKeyField.decSize = 0; //Processing the lookup table if neccesary - QString table_name = "lkp" + fixField(variableName.toLower()); - bool changeSize; - int newCodeSize; - bool changedRel; - QString newCodeType; - TtableDef lkpTable = getDuplicatedLkpTable(table_name,values,changeSize,newCodeSize,changedRel,newCodeType); + QString listName; + QJsonValue listValue = fieldObject.value("query"); + if (!listValue.isUndefined()) + listName = fixField(listValue.toString().toLower()); + else + listName = fixField(variableName.toLower()); + QString table_name = "lkp" + listName; + + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; if (lkpTable.name == "EMPTY") { lkpTable.name = table_name; - for (int lang = 0; lang < aField.desc.count(); lang++) { TlngLkpDesc fieldDesc; @@ -4161,37 +4155,23 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q lkpDesc.sensitive = false; lkpDesc.type = "varchar"; lkpDesc.size = getMaxDescLength(values); - lkpDesc.decSize = 0; - lkpTable.fields.append(lkpDesc); + lkpDesc.decSize = 0; + lkpTable.fields.append(lkpDesc); //Linking the multiselect field to this lookup table mselKeyField.rTable = lkpTable.name; mselKeyField.rField = lkpCode.name; mselKeyField.rName = getUUIDCode(); lkpTable.lkpValues.append(values); tables.append(lkpTable); //Append the lookup table to the list of tables + mselTable.fields.append(mselKeyField); //Add the multiselect key now linked to the looktable to the multiselect table + tables.append(mselTable); //Append the multiselect to the list of tables } else { - if (outputType == "h") - log("Lookup table for field " + variableName + " is the same as " + lkpTable.name + ". Using " + lkpTable.name); - //Linkin the multiselect table to the existing lookup table mselKeyField.rTable = lkpTable.name; mselKeyField.rField = getKeyField(lkpTable.name); mselKeyField.rName = getUUIDCode(); - if (merge) - { - if (newCodeType != "") - aField.type = newCodeType; - if (changeSize) - { - aField.size = getMaxMSelValueLength(lkpTable.lkpValues); - } - if (changedRel) - aField.mergeFine = true; - } } - mselTable.fields.append(mselKeyField); //Add the multiselect key now linked to the looktable to the multiselect table - tables.append(mselTable); //Append the multiselect to the list of tables } else { @@ -4228,7 +4208,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q oField.multiSelectTable = ""; oField.rField = ""; oField.rTable = ""; - oField.size = 120; + oField.size = 0; oField.type = "varchar"; tables[tblIndex].fields.append(oField); } @@ -4431,12 +4411,14 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f //Creating the lookp table if its neccesary if (isLookUpYesNo(values) == false) //Do not process yes/no values as lookup tables { - QString table_name = "lkp" + fixField(variableName.toLower()); - bool changeSize; - int newCodeSize; - bool changedRel; - QString newCodeType; - TtableDef lkpTable = getDuplicatedLkpTable(table_name, values, changeSize, newCodeSize, changedRel, newCodeType); + QString listName; + QJsonValue listValue = tableObject.value("query"); + if (!listValue.isUndefined()) + listName = fixField(listValue.toString().toLower()); + else + listName = fixField(variableName.toLower()); + QString table_name = "lkp" + listName; + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -4497,22 +4479,9 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f } else { - if (outputType == "h") - log("Lookup table for field " + variableName + " is the same as " + lkpTable.name + ". Using " + lkpTable.name); + aField.rName = getUUIDCode(); aField.rTable = lkpTable.name; aField.rField = getKeyField(lkpTable.name); - aField.rName = getUUIDCode(); - if (merge) - { - if (newCodeType != "") - aField.type = newCodeType; - if (changeSize) - { - aField.size = newCodeSize; - } - if (changedRel) - aField.mergeFine = true; - } } } } @@ -4899,81 +4868,6 @@ QList getValuesFromInsertFile(QString tableName, QDomNode startNode, return res; } -void loadMergingCreateFile(QDomNode lkptable, QDomNode insertStartNode) -{ - while (!lkptable.isNull()) - { - QString tableName; - QString tableDesc; - tableName = lkptable.toElement().attribute("name"); - tableDesc = lkptable.toElement().attribute("desc"); - TtableDef lkpTable; - lkpTable.isLoop = false; - lkpTable.isOSM = false; - lkpTable.isGroup = false; - - lkpTable.name = tableName; - - TlngLkpDesc fieldDesc; - fieldDesc.langCode = getDefLanguageCode(); - fieldDesc.desc = tableDesc; - lkpTable.desc.append(fieldDesc); - - lkpTable.pos = -1; - lkpTable.islookup = true; - lkpTable.isOneToOne = false; - QString clmCode; - QString cmlDesc; - lkpTable.lkpValues.append(getValuesFromInsertFile(tableName, insertStartNode, clmCode, cmlDesc)); - QDomNode lkpField = lkptable.firstChild(); - while (!lkpField.isNull()) - { - if (lkpField.toElement().attribute("name") == clmCode) - { - TfieldDef lkpCode; - lkpCode.name = clmCode; - lkpCode.selectSource = "NONE"; - lkpCode.selectListName = "NONE"; - - TlngLkpDesc langDesc; - langDesc.langCode = getDefLanguageCode(); - langDesc.desc = lkpField.toElement().attribute("desc"); - lkpCode.desc.append(langDesc); - - lkpCode.key = true; - lkpCode.sensitive = false; - lkpCode.type = lkpField.toElement().attribute("type"); - lkpCode.size = lkpField.toElement().attribute("size").toInt(); - lkpCode.decSize = lkpField.toElement().attribute("decsize").toInt(); - lkpTable.fields.append(lkpCode); - } - if (lkpField.toElement().attribute("name") == cmlDesc) - { - TfieldDef lkpDesc; - lkpDesc.name = cmlDesc; - lkpDesc.selectSource = "NONE"; - lkpDesc.selectListName = "NONE"; - - TlngLkpDesc langDesc; - langDesc.langCode = getDefLanguageCode(); - langDesc.desc = lkpField.toElement().attribute("desc"); - lkpDesc.desc.append(langDesc); - - lkpDesc.key = false; - lkpDesc.sensitive = false; - lkpDesc.type = lkpField.toElement().attribute("type"); - lkpDesc.size = lkpField.toElement().attribute("size").toInt(); - lkpDesc.decSize = lkpField.toElement().attribute("decsize").toInt(); - lkpTable.fields.append(lkpDesc); - } - - lkpField = lkpField.nextSibling(); - } - merging_tables.append(lkpTable); - lkptable = lkptable.nextSibling(); - } -} - //Reads the input JSON file and converts it to a MySQL database int processJSON(QString inputFile, QString mainTable, QString mainField, QDir dir, QSqlDatabase database) { @@ -5058,7 +4952,7 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di if (invalidKeyTypes.indexOf(surveyVariables[mainFieldIndex].object.value("type").toString()) >= 0) { log("The type of the primary key field is invalid"); - exit(18); + exit(17); } } @@ -5085,9 +4979,7 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di { QDomElement eLanguage; eLanguage = XMLResult.createElement("language"); - QDomText vSepFile; - vSepFile = XMLResult.createTextNode(ODKLanguages[pos]); - eLanguage.appendChild(vSepFile); + eLanguage.setAttribute("name",ODKLanguages[pos]); eLanguages.appendChild(eLanguage); } XMLRoot.appendChild(eLanguages); @@ -5120,9 +5012,7 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di languageNotFound = true; QDomElement eLanguage; eLanguage = XMLResult.createElement("language"); - QDomText vSepFile; - vSepFile = XMLResult.createTextNode(ODKLanguages[lng]); - eLanguage.appendChild(vSepFile); + eLanguage.setAttribute("name",ODKLanguages[lng]); eLanguages.appendChild(eLanguage); } } @@ -5135,11 +5025,7 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di log(XMLResult.toString()); exit(4); } - } - if (merge) - { - loadMergingCreateFile(Createroot.firstChild().firstChild(),Insertroot.firstChild()); - } + } int lang; //Creating the main table @@ -5226,7 +5112,7 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di f_submitted_by.desc.append(langDesc); } f_submitted_by.type = "varchar"; - f_submitted_by.size = 120; + f_submitted_by.size = 0; f_submitted_by.decSize = 0; f_submitted_by.rField = ""; f_submitted_by.rTable = ""; @@ -5252,7 +5138,7 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di f_submitted_date.desc.append(langDesc); } f_submitted_date.type = "datetime"; - f_submitted_date.size = 120; + f_submitted_date.size = 0; f_submitted_date.decSize = 0; f_submitted_date.rField = ""; f_submitted_date.rTable = ""; @@ -5325,12 +5211,12 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di if (duplicatedTables.count() > 0) { reportDuplicatedTables(); - exit(22); + exit(18); } if (duplicatedFields.count() > 0) { reportDuplicatedFields(); - exit(23); + exit(19); } if (duplicatedSelectValues.count() > 0) { @@ -5461,8 +5347,7 @@ int addLanguage(QString langCode, bool defLang) QString code = reEngLetterOnly.cap(1); if (genLangIndex(code) == -1) { - QString name = reEngLetterOnly.cap(2); - name = name; + QString name = reEngLetterOnly.cap(2); TlangDef language; language.code = code.toLower(); language.desc = name.toLower(); @@ -5517,6 +5402,31 @@ int main(int argc, char *argv[]) title = title + " * JSON XForm To MySQL * \n"; title = title + " * This tool generates a MySQL schema from a PyXForm JSON file. * \n"; title = title + " * JXFormToMySQL generates full relational MySQL databases. * \n"; + title = title + " * * \n"; + title = title + " * Exit codes: * \n"; + title = title + " * 1: General processing error. * \n"; + title = title + " * 2: Tables with more than 64 relationhips. (XML) * \n"; + title = title + " * 3: otherlanguages option is empty (XML). * \n"; + title = title + " * 4: Language not found. (XML) * \n"; + title = title + " * 5: Malformed deflanguage option. * \n"; + title = title + " * 6: Malformed otherlanguages option. * \n"; + title = title + " * 7: English is not default but yesnostring option is not set. * \n"; + title = title + " * 8: Malformed yesnostring option. * \n"; + title = title + " * 9: The ODK has duplicated choice options (XML). * \n"; + title = title + " * 10: Primary key not found. * \n"; + title = title + " * 11: Resource XML file was not attached (XML). * \n"; + title = title + " * 12: Error parsing resource XML file (XML). * \n"; + title = title + " * 13: Resource CSV file was not attached (XML). * \n"; + title = title + " * 14: Resource CSV file has invalid characters (XML). * \n"; + title = title + " * 15: Error parsing CSV file (XML). * \n"; + title = title + " * 16: Error parsing search expression (XML). * \n"; + title = title + " * 17: Primary key is invalid. * \n"; + title = title + " * 18: Duplicated tables (XML). * \n"; + title = title + " * 19: Duplicated field (XML). * \n"; + title = title + " * 20: Invalid fields (XML). * \n"; + title = title + " * 21: Duplicated lookups (XML). * \n"; + title = title + " * * \n"; + title = title + " * XML = XML oputput is available. * \n"; title = title + " ********************************************************************* \n"; TCLAP::CmdLine cmd(title.toUtf8().constData(), ' ', "2.0"); @@ -5535,13 +5445,10 @@ int main(int argc, char *argv[]) TCLAP::ValueArg langArg("l","otherlanguages","Other languages. For example: (en)English,(es)Español. Required if ODK form has multiple languages",false,"","string"); TCLAP::ValueArg defLangArg("d","deflanguage","Default language. For example: (en)English. If not indicated then English will be asumed",false,"(en)English","string"); TCLAP::ValueArg transFileArg("T","translationfile","Output translation file",false,"./iso639.sql","string"); - TCLAP::ValueArg yesNoStringArg("y","yesnostring","Yes and No strings in the default language in the format \"String|String\". This will allow the tool to identify Yes/No lookup tables and exclude them. This is not case sensitive. For example, if the default language is Spanish this value should be indicated as \"Si|No\". If empty English \"Yes|No\" will be assumed",false,"Yes|No","string"); + TCLAP::ValueArg yesNoStringArg("y","yesnostring","Yes and No values in the format \"Value|Value\". This will allow the tool to identify Yes/No lookup tables and exclude them. For example, \"1|0\".",true,"","string"); TCLAP::ValueArg tempDirArg("e","tempdirectory","Temporary directory. ./tmp by default",false,"./tmp","string"); TCLAP::ValueArg outputTypeArg("o","outputtype","Output type: (h)uman or (m)achine readble. Machine readble by default",false,"m","string"); - TCLAP::SwitchArg justCheckSwitch("K","justCheck","Just check of main inconsistencies and report back", cmd, false); - TCLAP::SwitchArg mergeSwitch("M","merge","Process the JXForm for merging. If true the MC and MI parameters cannot be empty", cmd, false); - TCLAP::ValueArg mergeCreateArg("r","mergecreatefile","Merge with create file",false,"","string"); - TCLAP::ValueArg mergeInsertArg("n","mergeinsertfile","Merge with insert file",false,"","string"); + TCLAP::SwitchArg justCheckSwitch("K","justCheck","Just check of main inconsistencies and report back", cmd, false); TCLAP::UnlabeledMultiArg suppFiles("supportFile", "support files", false, "string"); debug = false; @@ -5568,10 +5475,6 @@ int main(int argc, char *argv[]) cmd.add(yesNoStringArg); cmd.add(tempDirArg); cmd.add(outputTypeArg); - - cmd.add(mergeCreateArg); - cmd.add(mergeInsertArg); - cmd.add(suppFiles); //Parsing the command lines @@ -5591,8 +5494,7 @@ int main(int argc, char *argv[]) } } loadInvalidFieldNames(); - justCheck = justCheckSwitch.getValue(); - merge = mergeSwitch.getValue(); + justCheck = justCheckSwitch.getValue(); //Getting the variables from the command QString input = QString::fromUtf8(inputArg.getValue().c_str()); QString ddl = QString::fromUtf8(ddlArg.getValue().c_str()); @@ -5610,8 +5512,9 @@ int main(int argc, char *argv[]) QString yesNoString = QString::fromUtf8(yesNoStringArg.getValue().c_str()); QString tempDirectory = QString::fromUtf8(tempDirArg.getValue().c_str()); - QString mergeCreateFile = QString::fromUtf8(mergeCreateArg.getValue().c_str()); - QString mergeInsertFile = QString::fromUtf8(mergeInsertArg.getValue().c_str()); + lang = lang.replace("'",""); + defLang = defLang.replace("'",""); + yesNoString = yesNoString.replace("'",""); outputType = QString::fromUtf8(outputTypeArg.getValue().c_str()); mainVar = mainVar.trimmed().toLower(); @@ -5647,70 +5550,6 @@ int main(int argc, char *argv[]) } dir.setPath(tempDirectory); - if (merge) - { - if (!QFile::exists(mergeCreateFile)) - { - log("Create file used for merging does not exists"); - return 1; - } - if (!QFile::exists(mergeInsertFile)) - { - log("Insert file used for merging does not exists"); - return 1; - } - //Openning and parsing create XML file - QDomDocument mergingCrate("create"); - QFile mergingFile(mergeCreateFile); - if (!mergingFile.open(QIODevice::ReadOnly)) - { - log("Cannot open create file used for merging"); - return 1; - } - if (!mergingCrate.setContent(&mergingFile)) - { - log("Cannot parse create file used for merging"); - mergingFile.close(); - return 1; - } - mergingFile.close(); - Createroot = mergingCrate.documentElement(); - if (Createroot.tagName() != "XMLSchemaStructure") - { - log("Create file used for merging is not valid"); - mergingFile.close(); - return 1; - } - if (Createroot.attribute("version","1.0") != "2.0") - { - log("Cannot merge files. Only Version 2 are allowed for merging"); - mergingFile.close(); - return 1; - } - - QDomDocument mergingInsert("insert"); - mergingFile.setFileName(mergeInsertFile); - if (!mergingFile.open(QIODevice::ReadOnly)) - { - log("Cannot open insert file used for merging"); - return 1; - } - if (!mergingInsert.setContent(&mergingFile)) - { - log("Cannot parse insert file used for merging"); - mergingFile.close(); - return 1; - } - mergingFile.close(); - Insertroot = mergingInsert.documentElement(); - if (Insertroot.tagName() != "insertValuesXML") - { - log("Insert file used for merging is not valid"); - mergingFile.close(); - return 1; - } - } - //Unzip any zip files in the temporary directory QStringList zipFiles; for (int pos = 0; pos <= supportFiles.count()-1; pos++) @@ -5795,8 +5634,8 @@ int main(int argc, char *argv[]) prefix = QString::fromUtf8(prefixArg.getValue().c_str()); prefix = prefix + "_"; - strYes = "Yes"; - strNo = "No"; + strYes = "1"; + strNo = "0"; if (prefix == "_") prefix = ""; @@ -5804,7 +5643,7 @@ int main(int argc, char *argv[]) tableIndex = 0; if (addLanguage(defLang.replace("'",""),true) != 0) - return 1; + exit(5); QStringList othLanguages; othLanguages = lang.split(",",QString::SkipEmptyParts); @@ -5812,31 +5651,24 @@ int main(int argc, char *argv[]) if (addLanguage(othLanguages[lng].replace("'",""),false) != 0) exit(6); - if (isDefaultLanguage("english") == false) + if (yesNoString == "") { - if (yesNoString == "") - { - log("English is not the default language. You need to specify Yes and No values with the -y parameter in " + getDefLanguage()); - exit(7); - } - if (yesNoString.indexOf("|") == -1) - { - log("Malformed yes|no language string1"); - exit(8); - } - if (yesNoString.length() == 1) - { - log("Malformed yes|no language string2"); - exit(8); - } - strYes = yesNoString.split("|",QString::SkipEmptyParts)[0]; - strNo = yesNoString.split("|",QString::SkipEmptyParts)[1]; + log("You need to specify Yes and No values with the -y parameter"; + exit(7); } - else + + if (yesNoString.indexOf("|") == -1) + { + log("Malformed yes|no language string"); + exit(8); + } + if (yesNoString.length() == 1) { - strYes = yesNoString.split("|",QString::SkipEmptyParts)[0]; - strNo = yesNoString.split("|",QString::SkipEmptyParts)[1]; + log("Malformed yes|no language string"); + exit(8); } + strYes = yesNoString.split("|",QString::SkipEmptyParts)[0]; + strNo = yesNoString.split("|",QString::SkipEmptyParts)[1]; int returnValue; returnValue = processJSON(input,mTable.trimmed(),mainVar.trimmed(),dir,dblite); @@ -5880,9 +5712,36 @@ int main(int argc, char *argv[]) } log(XMLResult.toString()); } - exit(24); + exit(20); } + + if (duplicated_lookups.count() > 0) + { + QDomDocument XMLResult; + XMLResult = QDomDocument("XMLResult"); + QDomElement XMLRoot; + XMLRoot = XMLResult.createElement("XMLResult"); + XMLResult.appendChild(XMLRoot); + for (int item = 0; item < duplicated_lookups.count(); item++) + { + QDomElement eDuplicatedTable; + eDuplicatedTable = XMLResult.createElement("table"); + eDuplicatedTable.setAttribute("name",duplicated_lookups[item].sameas.right(duplicated_lookups[item].sameas.length()-3)); + for (int item2 = 0; item2 < duplicated_lookups[item].tables.count(); item2++) + { + QDomElement eDuplicatedItem; + eDuplicatedItem = XMLResult.createElement("duplicate"); + eDuplicatedItem.setAttribute("name",duplicated_lookups[item].tables[item2].right(duplicated_lookups[item].tables[item2].length()-3)); + eDuplicatedTable.appendChild(eDuplicatedItem); + } + XMLRoot.appendChild(eDuplicatedTable); + } + log(XMLResult.toString()); + exit(21); + } + + if (justCheck) { if (requiredFiles.count() > 0) From 9c38b56fc262ae9010495f30b4a4d8e8dd7dde74 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Sat, 31 Aug 2019 23:21:48 -0600 Subject: [PATCH 13/24] Fix merging issues --- utilities/mergeVersions/main.cpp | 1 + utilities/mergeVersions/mergecreate.cpp | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/utilities/mergeVersions/main.cpp b/utilities/mergeVersions/main.cpp index bcb1eefab..2ff9ed4a1 100644 --- a/utilities/mergeVersions/main.cpp +++ b/utilities/mergeVersions/main.cpp @@ -64,6 +64,7 @@ int main(int argc, char *argv[]) title = title + " * variable from categorical to continuous. * "; title = title + " * VNS: The description of a lookup value changed from B to A. For * "; title = title + " * example 1-Male in B changed to 1-Female in A. * "; + title = title + " * RNS: The related lookup table changed from B to A. * "; title = title + " * * "; title = title + " * This tool is usefull when dealing with multiple versions of an * "; title = title + " * ODK survey that must be combined in one common database. * "; diff --git a/utilities/mergeVersions/mergecreate.cpp b/utilities/mergeVersions/mergecreate.cpp index a0b92d41d..cc8076b69 100644 --- a/utilities/mergeVersions/mergecreate.cpp +++ b/utilities/mergeVersions/mergecreate.cpp @@ -617,6 +617,17 @@ QString mergeCreate::compareFields(QDomElement a, QDomElement b, int &newSize, i newDec = a.attribute("decsize","0").toInt(); if (a.attribute("key","") != b.attribute("key","")) return "KNS"; +// if (a.attribute("key","true") == "true") +// { +// if (a.attribute("type") != b.attribute("type")) +// return "KNS"; +// if (a.attribute("size") != b.attribute("size")) +// return "KNS"; +// if (a.attribute("rtable") != b.attribute("rtable")) +// return "KNS"; +// if (a.attribute("rfield") != b.attribute("rfield")) +// return "KNS"; +// } if ((a.attribute("rtable","") != b.attribute("rtable","")) || (a.attribute("rfield","") != b.attribute("rfield",""))) { if (a.attribute("mergefine","false") == "true") @@ -788,6 +799,11 @@ void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b, bool if (result == "FDC") log("FDC: The field " + a.attribute("name","") + " in table " + eTable.toElement().attribute("name","") + " has descreased in size. This change will be ignored"); } + else + { + if (result != "FDC") + fatalError = true; + } } } } From eda5f12a0ef835ce004ab6f841a9c32ef3526ba0 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Sat, 31 Aug 2019 23:49:19 -0600 Subject: [PATCH 14/24] Remove yes/no code --- JXFormToMysql/main.cpp | 460 ++++++++++++++++++----------------------- 1 file changed, 200 insertions(+), 260 deletions(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index 0d0f2f495..7aaf7815d 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -45,8 +45,6 @@ bool debug; QString command; QString outputType; QString default_language; -QString strYes; //Yes string for comparing Yes/No lookup tables -QString strNo; //No string for comparing Yes/No lookup tables QStringList variableStack; //This is a stack of groups or repeats for a variable. Used to get /xxx/xxx/xxx structures QStringList repeatStack; //This is a stack of repeats. So we know in which repeat we are QString prefix; //Table prefix @@ -2633,34 +2631,6 @@ int getMaxMSelValueLength(QList values) return res.length(); } -//Returns wheter or not a the values of a lookup table are yes/no values. -bool isLookUpYesNo(QList values) -{ - bool hasNo; - bool hasYes; - int lkp; - QString defLang; - defLang = getLanguageCode(getDefLanguage()); - if (values.count() == 2 ) - { - hasNo = false; - hasYes = false; - for (lkp = 0; lkp <= values.count()-1;lkp++) - { - if (values[lkp].code.trimmed().toLower() == strYes.trimmed().toLower()) - hasNo = true; - if (values[lkp].code.trimmed().toLower() == strNo.trimmed().toLower()) - hasYes = true; - } - if (hasYes && hasNo) - return true; - else - return false; - } - else - return false; -} - //Return the index of table in the list using its name int getTableIndex(QString name) { @@ -3498,75 +3468,73 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) if (values.count() > 0) //If the lookup table has values { //Creating the lookp table if its neccesary - if (isLookUpYesNo(values) == false) //Do not process yes/no values as lookup tables + QString table_name = "lkp" + fixField(variableName.toLower()); + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); + lkpTable.isLoop = false; + lkpTable.isOSM = false; + lkpTable.isGroup = false; + if (lkpTable.name == "EMPTY") { - QString table_name = "lkp" + fixField(variableName.toLower()); - TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); - lkpTable.isLoop = false; - lkpTable.isOSM = false; - lkpTable.isGroup = false; - if (lkpTable.name == "EMPTY") + lkpTable.name = table_name; + for (int lang = 0; lang < aField.desc.count(); lang++) { - lkpTable.name = table_name; - for (int lang = 0; lang < aField.desc.count(); lang++) - { - TlngLkpDesc fieldDesc; - fieldDesc.langCode = aField.desc[lang].langCode; - fieldDesc.desc = "Lookup table (" + aField.desc[lang].desc + ")"; - lkpTable.desc.append(fieldDesc); - } - lkpTable.pos = -1; - lkpTable.islookup = true; - lkpTable.isOneToOne = false; - lkpTable.lkpValues.append(values); - //Creates the field for code in the lookup - TfieldDef lkpCode; - lkpCode.name = fixField(variableName.toLower()) + "_cod"; - lkpCode.selectSource = "NONE"; - lkpCode.selectListName = "NONE"; - for (int lang = 0; lang <= languages.count()-1;lang++) - { - TlngLkpDesc langDesc; - langDesc.langCode = languages[lang].code; - langDesc.desc = "Code"; - lkpCode.desc.append(langDesc); - } - lkpCode.key = true; - lkpCode.sensitive = false; - lkpCode.type = aField.type; - lkpCode.size = aField.size; - lkpCode.decSize = aField.decSize; - lkpTable.fields.append(lkpCode); - //Creates the field for description in the lookup - TfieldDef lkpDesc; - lkpDesc.name = fixField(variableName.toLower()) + "_des"; - lkpDesc.selectSource = "NONE"; - lkpDesc.selectListName = "NONE"; - for (int lang = 0; lang <= languages.count()-1;lang++) - { - TlngLkpDesc langDesc; - langDesc.langCode = languages[lang].code; - langDesc.desc = "Description"; - lkpDesc.desc.append(langDesc); - } - lkpDesc.key = false; - lkpDesc.sensitive = false; - lkpDesc.type = "varchar"; - lkpDesc.size = getMaxDescLength(values); - lkpDesc.decSize = 0; - lkpTable.fields.append(lkpDesc); - aField.rTable = lkpTable.name; - aField.rField = lkpCode.name; - aField.rName = getUUIDCode(); - tables.append(lkpTable); + TlngLkpDesc fieldDesc; + fieldDesc.langCode = aField.desc[lang].langCode; + fieldDesc.desc = "Lookup table (" + aField.desc[lang].desc + ")"; + lkpTable.desc.append(fieldDesc); } - else + lkpTable.pos = -1; + lkpTable.islookup = true; + lkpTable.isOneToOne = false; + lkpTable.lkpValues.append(values); + //Creates the field for code in the lookup + TfieldDef lkpCode; + lkpCode.name = fixField(variableName.toLower()) + "_cod"; + lkpCode.selectSource = "NONE"; + lkpCode.selectListName = "NONE"; + for (int lang = 0; lang <= languages.count()-1;lang++) { - aField.rName = getUUIDCode(); - aField.rTable = lkpTable.name; - aField.rField = getKeyField(lkpTable.name); + TlngLkpDesc langDesc; + langDesc.langCode = languages[lang].code; + langDesc.desc = "Code"; + lkpCode.desc.append(langDesc); } + lkpCode.key = true; + lkpCode.sensitive = false; + lkpCode.type = aField.type; + lkpCode.size = aField.size; + lkpCode.decSize = aField.decSize; + lkpTable.fields.append(lkpCode); + //Creates the field for description in the lookup + TfieldDef lkpDesc; + lkpDesc.name = fixField(variableName.toLower()) + "_des"; + lkpDesc.selectSource = "NONE"; + lkpDesc.selectListName = "NONE"; + for (int lang = 0; lang <= languages.count()-1;lang++) + { + TlngLkpDesc langDesc; + langDesc.langCode = languages[lang].code; + langDesc.desc = "Description"; + lkpDesc.desc.append(langDesc); + } + lkpDesc.key = false; + lkpDesc.sensitive = false; + lkpDesc.type = "varchar"; + lkpDesc.size = getMaxDescLength(values); + lkpDesc.decSize = 0; + lkpTable.fields.append(lkpDesc); + aField.rTable = lkpTable.name; + aField.rField = lkpCode.name; + aField.rName = getUUIDCode(); + tables.append(lkpTable); + } + else + { + aField.rName = getUUIDCode(); + aField.rTable = lkpTable.name; + aField.rField = getKeyField(lkpTable.name); } + } else { @@ -3873,81 +3841,80 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q if (values.count() > 0) //If the lookup table has values { //Creating the lookp table if its neccesary - if (isLookUpYesNo(values) == false) //Do not process yes/no values as lookup tables + + QString listName; + QJsonValue listValue = fieldObject.value("query"); + if (!listValue.isUndefined()) + listName = fixField(listValue.toString().toLower()); + else + listName = fixField(variableName.toLower()); + QString table_name = "lkp" + listName; + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); + lkpTable.isLoop = false; + lkpTable.isOSM = false; + lkpTable.isGroup = false; + if (lkpTable.name == "EMPTY") { - QString listName; - QJsonValue listValue = fieldObject.value("query"); - if (!listValue.isUndefined()) - listName = fixField(listValue.toString().toLower()); - else - listName = fixField(variableName.toLower()); - QString table_name = "lkp" + listName; - TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); - lkpTable.isLoop = false; - lkpTable.isOSM = false; - lkpTable.isGroup = false; - if (lkpTable.name == "EMPTY") + lkpTable.name = table_name; + for (int lang = 0; lang < aField.desc.count(); lang++) { - lkpTable.name = table_name; - for (int lang = 0; lang < aField.desc.count(); lang++) - { - TlngLkpDesc fieldDesc; - fieldDesc.langCode = aField.desc[lang].langCode; - fieldDesc.desc = "Lookup table (" + aField.desc[lang].desc + ")"; - lkpTable.desc.append(fieldDesc); - } - lkpTable.pos = -1; - lkpTable.islookup = true; - lkpTable.isOneToOne = false; - lkpTable.lkpValues.append(values); - //Creates the field for code in the lookup - TfieldDef lkpCode; - lkpCode.name = fixField(variableName.toLower()) + "_cod"; - lkpCode.selectSource = "NONE"; - lkpCode.selectListName = "NONE"; - for (int lang = 0; lang <= languages.count()-1;lang++) - { - TlngLkpDesc langDesc; - langDesc.langCode = languages[lang].code; - langDesc.desc = "Code"; - lkpCode.desc.append(langDesc); - } - lkpCode.key = true; - lkpCode.sensitive = false; - lkpCode.type = aField.type; - lkpCode.size = aField.size; - lkpCode.decSize = aField.decSize; - lkpTable.fields.append(lkpCode); - //Creates the field for description in the lookup - TfieldDef lkpDesc; - lkpDesc.name = fixField(variableName.toLower()) + "_des"; - lkpDesc.selectSource = "NONE"; - lkpDesc.selectListName = "NONE"; - for (int lang = 0; lang <= languages.count()-1;lang++) - { - TlngLkpDesc langDesc; - langDesc.langCode = languages[lang].code; - langDesc.desc = "Description"; - lkpDesc.desc.append(langDesc); - } - lkpDesc.key = false; - lkpDesc.sensitive = false; - lkpDesc.type = "varchar"; - lkpDesc.size = getMaxDescLength(values); - lkpDesc.decSize = 0; - lkpTable.fields.append(lkpDesc); - aField.rTable = lkpTable.name; - aField.rField = lkpCode.name; - aField.rName = getUUIDCode(); - tables.append(lkpTable); + TlngLkpDesc fieldDesc; + fieldDesc.langCode = aField.desc[lang].langCode; + fieldDesc.desc = "Lookup table (" + aField.desc[lang].desc + ")"; + lkpTable.desc.append(fieldDesc); } - else + lkpTable.pos = -1; + lkpTable.islookup = true; + lkpTable.isOneToOne = false; + lkpTable.lkpValues.append(values); + //Creates the field for code in the lookup + TfieldDef lkpCode; + lkpCode.name = fixField(variableName.toLower()) + "_cod"; + lkpCode.selectSource = "NONE"; + lkpCode.selectListName = "NONE"; + for (int lang = 0; lang <= languages.count()-1;lang++) { - aField.rName = getUUIDCode(); - aField.rTable = lkpTable.name; - aField.rField = getKeyField(lkpTable.name); + TlngLkpDesc langDesc; + langDesc.langCode = languages[lang].code; + langDesc.desc = "Code"; + lkpCode.desc.append(langDesc); } + lkpCode.key = true; + lkpCode.sensitive = false; + lkpCode.type = aField.type; + lkpCode.size = aField.size; + lkpCode.decSize = aField.decSize; + lkpTable.fields.append(lkpCode); + //Creates the field for description in the lookup + TfieldDef lkpDesc; + lkpDesc.name = fixField(variableName.toLower()) + "_des"; + lkpDesc.selectSource = "NONE"; + lkpDesc.selectListName = "NONE"; + for (int lang = 0; lang <= languages.count()-1;lang++) + { + TlngLkpDesc langDesc; + langDesc.langCode = languages[lang].code; + langDesc.desc = "Description"; + lkpDesc.desc.append(langDesc); + } + lkpDesc.key = false; + lkpDesc.sensitive = false; + lkpDesc.type = "varchar"; + lkpDesc.size = getMaxDescLength(values); + lkpDesc.decSize = 0; + lkpTable.fields.append(lkpDesc); + aField.rTable = lkpTable.name; + aField.rField = lkpCode.name; + aField.rName = getUUIDCode(); + tables.append(lkpTable); + } + else + { + aField.rName = getUUIDCode(); + aField.rTable = lkpTable.name; + aField.rField = getKeyField(lkpTable.name); } + } else { @@ -4409,81 +4376,80 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f if (values.count() > 0) //If the lookup table has values { //Creating the lookp table if its neccesary - if (isLookUpYesNo(values) == false) //Do not process yes/no values as lookup tables + + QString listName; + QJsonValue listValue = tableObject.value("query"); + if (!listValue.isUndefined()) + listName = fixField(listValue.toString().toLower()); + else + listName = fixField(variableName.toLower()); + QString table_name = "lkp" + listName; + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); + lkpTable.isLoop = false; + lkpTable.isOSM = false; + lkpTable.isGroup = false; + if (lkpTable.name == "EMPTY") { - QString listName; - QJsonValue listValue = tableObject.value("query"); - if (!listValue.isUndefined()) - listName = fixField(listValue.toString().toLower()); - else - listName = fixField(variableName.toLower()); - QString table_name = "lkp" + listName; - TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); - lkpTable.isLoop = false; - lkpTable.isOSM = false; - lkpTable.isGroup = false; - if (lkpTable.name == "EMPTY") + lkpTable.name = table_name; + for (int lang = 0; lang < aField.desc.count(); lang++) { - lkpTable.name = table_name; - for (int lang = 0; lang < aField.desc.count(); lang++) - { - TlngLkpDesc fieldDesc; - fieldDesc.langCode = aField.desc[lang].langCode; - fieldDesc.desc = "Lookup table (" + aField.desc[lang].desc + ")"; - lkpTable.desc.append(fieldDesc); - } - lkpTable.pos = -1; - lkpTable.islookup = true; - lkpTable.isOneToOne = false; - lkpTable.lkpValues.append(values); - //Creates the field for code in the lookup - TfieldDef lkpCode; - lkpCode.name = fixField(variableName.toLower()) + "_cod"; - lkpCode.selectSource = "NONE"; - lkpCode.selectListName = "NONE"; - for (int lang = 0; lang <= languages.count()-1;lang++) - { - TlngLkpDesc langDesc; - langDesc.langCode = languages[lang].code; - langDesc.desc = "Code"; - lkpCode.desc.append(langDesc); - } - lkpCode.key = true; - lkpCode.sensitive = false; - lkpCode.type = aField.type; - lkpCode.size = aField.size; - lkpCode.decSize = aField.decSize; - lkpTable.fields.append(lkpCode); - //Creates the field for description in the lookup - TfieldDef lkpDesc; - lkpDesc.name = fixField(variableName.toLower()) + "_des"; - lkpDesc.selectSource = "NONE"; - lkpDesc.selectListName = "NONE"; - for (int lang = 0; lang <= languages.count()-1;lang++) - { - TlngLkpDesc langDesc; - langDesc.langCode = languages[lang].code; - langDesc.desc = "Description"; - lkpDesc.desc.append(langDesc); - } - lkpDesc.key = false; - lkpDesc.sensitive = false; - lkpDesc.type = "varchar"; - lkpDesc.size = getMaxDescLength(values); - lkpDesc.decSize = 0; - lkpTable.fields.append(lkpDesc); - aField.rTable = lkpTable.name; - aField.rField = lkpCode.name; - aField.rName = getUUIDCode(); - tables.append(lkpTable); + TlngLkpDesc fieldDesc; + fieldDesc.langCode = aField.desc[lang].langCode; + fieldDesc.desc = "Lookup table (" + aField.desc[lang].desc + ")"; + lkpTable.desc.append(fieldDesc); } - else + lkpTable.pos = -1; + lkpTable.islookup = true; + lkpTable.isOneToOne = false; + lkpTable.lkpValues.append(values); + //Creates the field for code in the lookup + TfieldDef lkpCode; + lkpCode.name = fixField(variableName.toLower()) + "_cod"; + lkpCode.selectSource = "NONE"; + lkpCode.selectListName = "NONE"; + for (int lang = 0; lang <= languages.count()-1;lang++) { - aField.rName = getUUIDCode(); - aField.rTable = lkpTable.name; - aField.rField = getKeyField(lkpTable.name); + TlngLkpDesc langDesc; + langDesc.langCode = languages[lang].code; + langDesc.desc = "Code"; + lkpCode.desc.append(langDesc); } + lkpCode.key = true; + lkpCode.sensitive = false; + lkpCode.type = aField.type; + lkpCode.size = aField.size; + lkpCode.decSize = aField.decSize; + lkpTable.fields.append(lkpCode); + //Creates the field for description in the lookup + TfieldDef lkpDesc; + lkpDesc.name = fixField(variableName.toLower()) + "_des"; + lkpDesc.selectSource = "NONE"; + lkpDesc.selectListName = "NONE"; + for (int lang = 0; lang <= languages.count()-1;lang++) + { + TlngLkpDesc langDesc; + langDesc.langCode = languages[lang].code; + langDesc.desc = "Description"; + lkpDesc.desc.append(langDesc); + } + lkpDesc.key = false; + lkpDesc.sensitive = false; + lkpDesc.type = "varchar"; + lkpDesc.size = getMaxDescLength(values); + lkpDesc.decSize = 0; + lkpTable.fields.append(lkpDesc); + aField.rTable = lkpTable.name; + aField.rField = lkpCode.name; + aField.rName = getUUIDCode(); + tables.append(lkpTable); + } + else + { + aField.rName = getUUIDCode(); + aField.rTable = lkpTable.name; + aField.rField = getKeyField(lkpTable.name); } + } else { @@ -5444,8 +5410,7 @@ int main(int argc, char *argv[]) TCLAP::ValueArg prefixArg("p","prefix","Prefix for each table. _ is added to the prefix. Default no prefix",false,"","string"); TCLAP::ValueArg langArg("l","otherlanguages","Other languages. For example: (en)English,(es)Español. Required if ODK form has multiple languages",false,"","string"); TCLAP::ValueArg defLangArg("d","deflanguage","Default language. For example: (en)English. If not indicated then English will be asumed",false,"(en)English","string"); - TCLAP::ValueArg transFileArg("T","translationfile","Output translation file",false,"./iso639.sql","string"); - TCLAP::ValueArg yesNoStringArg("y","yesnostring","Yes and No values in the format \"Value|Value\". This will allow the tool to identify Yes/No lookup tables and exclude them. For example, \"1|0\".",true,"","string"); + TCLAP::ValueArg transFileArg("T","translationfile","Output translation file",false,"./iso639.sql","string"); TCLAP::ValueArg tempDirArg("e","tempdirectory","Temporary directory. ./tmp by default",false,"./tmp","string"); TCLAP::ValueArg outputTypeArg("o","outputtype","Output type: (h)uman or (m)achine readble. Machine readble by default",false,"m","string"); TCLAP::SwitchArg justCheckSwitch("K","justCheck","Just check of main inconsistencies and report back", cmd, false); @@ -5471,8 +5436,7 @@ int main(int argc, char *argv[]) cmd.add(prefixArg); cmd.add(langArg); cmd.add(defLangArg); - cmd.add(transFileArg); - cmd.add(yesNoStringArg); + cmd.add(transFileArg); cmd.add(tempDirArg); cmd.add(outputTypeArg); cmd.add(suppFiles); @@ -5508,13 +5472,11 @@ int main(int argc, char *argv[]) QString mainVar = QString::fromUtf8(mainVarArg.getValue().c_str()); QString lang = QString::fromUtf8(langArg.getValue().c_str()); QString defLang = QString::fromUtf8(defLangArg.getValue().c_str()); - QString transFile = QString::fromUtf8(transFileArg.getValue().c_str()); - QString yesNoString = QString::fromUtf8(yesNoStringArg.getValue().c_str()); + QString transFile = QString::fromUtf8(transFileArg.getValue().c_str()); QString tempDirectory = QString::fromUtf8(tempDirArg.getValue().c_str()); lang = lang.replace("'",""); defLang = defLang.replace("'",""); - yesNoString = yesNoString.replace("'",""); outputType = QString::fromUtf8(outputTypeArg.getValue().c_str()); mainVar = mainVar.trimmed().toLower(); @@ -5634,9 +5596,6 @@ int main(int argc, char *argv[]) prefix = QString::fromUtf8(prefixArg.getValue().c_str()); prefix = prefix + "_"; - strYes = "1"; - strNo = "0"; - if (prefix == "_") prefix = ""; @@ -5651,25 +5610,6 @@ int main(int argc, char *argv[]) if (addLanguage(othLanguages[lng].replace("'",""),false) != 0) exit(6); - if (yesNoString == "") - { - log("You need to specify Yes and No values with the -y parameter"; - exit(7); - } - - if (yesNoString.indexOf("|") == -1) - { - log("Malformed yes|no language string"); - exit(8); - } - if (yesNoString.length() == 1) - { - log("Malformed yes|no language string"); - exit(8); - } - strYes = yesNoString.split("|",QString::SkipEmptyParts)[0]; - strNo = yesNoString.split("|",QString::SkipEmptyParts)[1]; - int returnValue; returnValue = processJSON(input,mTable.trimmed(),mainVar.trimmed(),dir,dblite); if (returnValue == 0) From 85fd5d05ac958021747df831be77a53eecc1824b Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Tue, 3 Sep 2019 11:20:30 -0600 Subject: [PATCH 15/24] Fix error codes in help --- JXFormToMysql/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index 7aaf7815d..f197dc83b 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -5376,8 +5376,8 @@ int main(int argc, char *argv[]) title = title + " * 4: Language not found. (XML) * \n"; title = title + " * 5: Malformed deflanguage option. * \n"; title = title + " * 6: Malformed otherlanguages option. * \n"; - title = title + " * 7: English is not default but yesnostring option is not set. * \n"; - title = title + " * 8: Malformed yesnostring option. * \n"; + title = title + " * 7: Not used. * \n"; + title = title + " * 8: Not used. * \n"; title = title + " * 9: The ODK has duplicated choice options (XML). * \n"; title = title + " * 10: Primary key not found. * \n"; title = title + " * 11: Resource XML file was not attached (XML). * \n"; From 766fc3cb495f9bd3e0d9b218b648ec42391b522f Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Thu, 5 Sep 2019 12:17:24 -0600 Subject: [PATCH 16/24] Fix dependency --- utilities/MySQLToSQLite/MySQLToSQLite.pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/MySQLToSQLite/MySQLToSQLite.pro b/utilities/MySQLToSQLite/MySQLToSQLite.pro index 0d0db98fc..2a342d1cd 100644 --- a/utilities/MySQLToSQLite/MySQLToSQLite.pro +++ b/utilities/MySQLToSQLite/MySQLToSQLite.pro @@ -14,7 +14,7 @@ CONFIG -= app_bundle TEMPLATE = app -INCLUDEPATH += ../3rdparty +INCLUDEPATH += ../../3rdparty SOURCES += \ main.cpp From b9d3ce9dfd91d66f47c481fd3703322756f03206 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Fri, 6 Sep 2019 19:47:12 -0600 Subject: [PATCH 17/24] Change query to list_name --- JXFormToMysql/main.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index f197dc83b..ceb7ed26a 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -3843,7 +3843,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q //Creating the lookp table if its neccesary QString listName; - QJsonValue listValue = fieldObject.value("query"); + QJsonValue listValue = fieldObject.value("list_name"); if (!listValue.isUndefined()) listName = fixField(listValue.toString().toLower()); else @@ -4062,7 +4062,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q mselKeyField.decSize = 0; //Processing the lookup table if neccesary QString listName; - QJsonValue listValue = fieldObject.value("query"); + QJsonValue listValue = fieldObject.value("list_name"); if (!listValue.isUndefined()) listName = fixField(listValue.toString().toLower()); else @@ -4378,7 +4378,7 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f //Creating the lookp table if its neccesary QString listName; - QJsonValue listValue = tableObject.value("query"); + QJsonValue listValue = tableObject.value("list_name"); if (!listValue.isUndefined()) listName = fixField(listValue.toString().toLower()); else From 5741e04e189b85c95ca0a2edec3ec06c498085e7 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Sun, 8 Sep 2019 22:32:50 -0600 Subject: [PATCH 18/24] Fix bug with lookup table fields. Use search if no list_name is present --- JXFormToMysql/main.cpp | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index ceb7ed26a..9d67fa762 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -3847,7 +3847,13 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q if (!listValue.isUndefined()) listName = fixField(listValue.toString().toLower()); else - listName = fixField(variableName.toLower()); + { + listValue = fieldObject.value("query"); + if (!listValue.isUndefined()) + listName = fixField(listValue.toString().toLower()); + else + listName = fixField(variableName.toLower()); + } QString table_name = "lkp" + listName; TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); lkpTable.isLoop = false; @@ -3860,7 +3866,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q { TlngLkpDesc fieldDesc; fieldDesc.langCode = aField.desc[lang].langCode; - fieldDesc.desc = "Lookup table (" + aField.desc[lang].desc + ")"; + fieldDesc.desc = "Lookup table for list name \"" + listName + "\""; lkpTable.desc.append(fieldDesc); } lkpTable.pos = -1; @@ -3869,7 +3875,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q lkpTable.lkpValues.append(values); //Creates the field for code in the lookup TfieldDef lkpCode; - lkpCode.name = fixField(variableName.toLower()) + "_cod"; + lkpCode.name = fixField(listName.toLower()) + "_cod"; lkpCode.selectSource = "NONE"; lkpCode.selectListName = "NONE"; for (int lang = 0; lang <= languages.count()-1;lang++) @@ -3887,7 +3893,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q lkpTable.fields.append(lkpCode); //Creates the field for description in the lookup TfieldDef lkpDesc; - lkpDesc.name = fixField(variableName.toLower()) + "_des"; + lkpDesc.name = fixField(listName.toLower()) + "_des"; lkpDesc.selectSource = "NONE"; lkpDesc.selectListName = "NONE"; for (int lang = 0; lang <= languages.count()-1;lang++) @@ -4066,7 +4072,13 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q if (!listValue.isUndefined()) listName = fixField(listValue.toString().toLower()); else - listName = fixField(variableName.toLower()); + { + listValue = fieldObject.value("query"); + if (!listValue.isUndefined()) + listName = fixField(listValue.toString().toLower()); + else + listName = fixField(variableName.toLower()); + } QString table_name = "lkp" + listName; TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); @@ -4080,7 +4092,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q { TlngLkpDesc fieldDesc; fieldDesc.langCode = aField.desc[lang].langCode; - fieldDesc.desc = "Lookup table (" + aField.desc[lang].desc + ")"; + fieldDesc.desc = "Lookup table for list name \"" + listName + "\""; lkpTable.desc.append(fieldDesc); } lkpTable.pos = -1; @@ -4088,7 +4100,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q lkpTable.isOneToOne = false; //Creates the field for code in the lookup TfieldDef lkpCode; - lkpCode.name = fixField(variableName.toLower()) + "_cod"; + lkpCode.name = fixField(listName.toLower()) + "_cod"; lkpCode.selectSource = "NONE"; lkpCode.selectListName = "NONE"; int lang; @@ -4107,7 +4119,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q lkpTable.fields.append(lkpCode); //Creates the field for description in the lookup TfieldDef lkpDesc; - lkpDesc.name = fixField(variableName.toLower()) + "_des"; + lkpDesc.name = fixField(listName.toLower()) + "_des"; lkpDesc.selectSource = "NONE"; lkpDesc.selectListName = "NONE"; for (lang = 0; lang <= languages.count()-1;lang++) @@ -4382,7 +4394,13 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f if (!listValue.isUndefined()) listName = fixField(listValue.toString().toLower()); else - listName = fixField(variableName.toLower()); + { + listValue = tableObject.value("query"); + if (!listValue.isUndefined()) + listName = fixField(listValue.toString().toLower()); + else + listName = fixField(variableName.toLower()); + } QString table_name = "lkp" + listName; TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); lkpTable.isLoop = false; @@ -4395,7 +4413,7 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f { TlngLkpDesc fieldDesc; fieldDesc.langCode = aField.desc[lang].langCode; - fieldDesc.desc = "Lookup table (" + aField.desc[lang].desc + ")"; + fieldDesc.desc = "Lookup table for list name \"" + listName + "\""; lkpTable.desc.append(fieldDesc); } lkpTable.pos = -1; @@ -4404,7 +4422,7 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f lkpTable.lkpValues.append(values); //Creates the field for code in the lookup TfieldDef lkpCode; - lkpCode.name = fixField(variableName.toLower()) + "_cod"; + lkpCode.name = fixField(listName.toLower()) + "_cod"; lkpCode.selectSource = "NONE"; lkpCode.selectListName = "NONE"; for (int lang = 0; lang <= languages.count()-1;lang++) @@ -4422,7 +4440,7 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f lkpTable.fields.append(lkpCode); //Creates the field for description in the lookup TfieldDef lkpDesc; - lkpDesc.name = fixField(variableName.toLower()) + "_des"; + lkpDesc.name = fixField(listName.toLower()) + "_des"; lkpDesc.selectSource = "NONE"; lkpDesc.selectListName = "NONE"; for (int lang = 0; lang <= languages.count()-1;lang++) From 0d19f4a603d273a477f9e77071f57d9b8ca33247 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Sun, 8 Sep 2019 22:33:38 -0600 Subject: [PATCH 19/24] Fix some bugs --- utilities/mergeVersions/mergecreate.cpp | 113 +++++++++++++----------- utilities/mergeVersions/mergecreate.h | 1 + 2 files changed, 60 insertions(+), 54 deletions(-) diff --git a/utilities/mergeVersions/mergecreate.cpp b/utilities/mergeVersions/mergecreate.cpp index cc8076b69..def11febe 100644 --- a/utilities/mergeVersions/mergecreate.cpp +++ b/utilities/mergeVersions/mergecreate.cpp @@ -25,6 +25,7 @@ mergeCreate::mergeCreate(QObject *parent) : QObject(parent) { fatalError = false; idx = 1; + create_lookup_rels.clear(); } void mergeCreate::setInsertDiff(QList diff) @@ -89,7 +90,7 @@ int mergeCreate::compare() // Add foreign keys that were dropped earlier diff.append("\n"); for (int pos = 0; pos < create_lookup_rels.count(); pos++) - { + { diff.append("ALTER TABLE " + create_lookup_rels[pos].table_name + " ADD INDEX " + create_lookup_rels[pos].rel_name + " (" + create_lookup_rels[pos].field_name + ");\n"); diff.append("ALTER TABLE " + create_lookup_rels[pos].table_name + " ADD CONSTRAINT " + create_lookup_rels[pos].rel_name + " FOREIGN KEY (" + create_lookup_rels[pos].field_name + ") REFERENCES " + create_lookup_rels[pos].rel_table + " (" + create_lookup_rels[pos].rel_field + ") ON DELETE RESTRICT ON UPDATE NO ACTION;\n\n"); } @@ -203,6 +204,9 @@ void mergeCreate::setFiles(QString createA, QString createB, QString createC, QS void mergeCreate::replace_lookup_relationships(QString table, QString field) { + QStringList affected_tables; + + QDomNodeList fields; fields = rootB.elementsByTagName("field"); QString sql; @@ -210,6 +214,10 @@ void mergeCreate::replace_lookup_relationships(QString table, QString field) { if ((fields.item(pos).toElement().attribute("rtable","") == table) && ((fields.item(pos).toElement().attribute("rfield","") == field))) { + QString affected_table = fields.item(pos).parentNode().toElement().attribute("name"); + if (affected_tables.indexOf(affected_table) == -1) + affected_tables.append(affected_table); + dropped_rels.append(fields.item(pos).toElement().attribute("rname","")); sql = "ALTER TABLE " + fields.item(pos).parentNode().toElement().attribute("name") + " DROP FOREIGN KEY " + fields.item(pos).toElement().attribute("rname","") + ";\n"; diff.append(sql); @@ -222,13 +230,17 @@ void mergeCreate::replace_lookup_relationships(QString table, QString field) { if ((fields.item(pos).toElement().attribute("rtable","") == table) && ((fields.item(pos).toElement().attribute("rfield","") == field))) { - TreplaceRef a_replace; - a_replace.table_name = fields.item(pos).parentNode().toElement().attribute("name"); - a_replace.rel_name = fields.item(pos).toElement().attribute("rname",""); - a_replace.field_name = fields.item(pos).toElement().attribute("name",""); - a_replace.rel_table = fields.item(pos).toElement().attribute("rtable",""); - a_replace.rel_field = fields.item(pos).toElement().attribute("rfield",""); - create_lookup_rels.append(a_replace); + QString affected_table = fields.item(pos).parentNode().toElement().attribute("name"); + if (affected_tables.indexOf(affected_table) >= 0) + { + TreplaceRef a_replace; + a_replace.table_name = fields.item(pos).parentNode().toElement().attribute("name"); + a_replace.rel_name = fields.item(pos).toElement().attribute("rname",""); + a_replace.field_name = fields.item(pos).toElement().attribute("name",""); + a_replace.rel_table = fields.item(pos).toElement().attribute("rtable",""); + a_replace.rel_field = fields.item(pos).toElement().attribute("rfield",""); + create_lookup_rels.append(a_replace); + } } } } @@ -390,6 +402,7 @@ void mergeCreate::addFieldToRTables(QString parentTable, QString rTable, QString TrtableDef table; table.parentTable = parentTable; table.name = rTable; + table.isLookUp = isLookUp; TrfieldDef aField; aField.name = field; aField.rname = rField; @@ -476,34 +489,27 @@ void mergeCreate::addTableToSDiff(QDomNode table, bool lookUp) for (pos = 0; pos < rtables.count();pos++) { if (rtables[pos].parentTable == eTable.attribute("name","")) - { - bool isLookup = false; - QString rname; - for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) - { - if (rtables[pos].rfields[pos2].isLookUp) - { - isLookup = true; - rname = rtables[pos].rfields[pos2].rcode; - break; - } - } + { - if (!isLookup) + if (!rtables[pos].isLookUp) { QUuid idxUUID=QUuid::createUuid(); QString strIdxUUID=idxUUID.toString().replace("{","").replace("}","").right(12); sql = sql + "INDEX DIDX" + strIdxUUID + "("; + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + sql = sql + rtables[pos].rfields[pos2].name + ","; + } + sql = sql.left(sql.length()-1) + "),\n"; } else { - sql = sql + "INDEX " + rname + "("; - } - for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) - { - sql = sql + rtables[pos].rfields[pos2].name + ","; - } - sql = sql.left(sql.length()-1) + "),\n"; + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + sql = sql + "INDEX " + rtables[pos].rfields[pos2].rcode + " ("; + sql = sql + rtables[pos].rfields[pos2].name + "),\n"; + } + } idx++; } @@ -512,40 +518,39 @@ void mergeCreate::addTableToSDiff(QDomNode table, bool lookUp) for (pos = 0; pos < rtables.count();pos++) { if (rtables[pos].parentTable == eTable.attribute("name","")) - { - bool isLookup = false; - QString rname; - for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) - { - if (rtables[pos].rfields[pos2].isLookUp) - { - isLookup = true; - rname = rtables[pos].rfields[pos2].rcode; - break; - } - } - if (!isLookup) + { + if (!rtables[pos].isLookUp) { QUuid cntUUID=QUuid::createUuid(); QString strCntUUID=cntUUID.toString().replace("{","").replace("}","").right(12); sql = sql + "CONSTRAINT DFK" + strCntUUID + "\n"; + + sql = sql + "FOREIGN KEY ("; + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + sql = sql + rtables[pos].rfields[pos2].name + ","; + } + sql = sql.left(sql.length()-1) + ")\n"; + sql = sql + "REFERENCES " + rtables[pos].name + "("; + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + sql = sql + rtables[pos].rfields[pos2].rname + ","; + } + sql = sql.left(sql.length()-1) + ")\nON DELETE CASCADE\nON UPDATE NO ACTION,\n"; + } else { - sql = sql + "CONSTRAINT " + rname + "\n"; - } - sql = sql + "FOREIGN KEY ("; - for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) - { - sql = sql + rtables[pos].rfields[pos2].name + ","; - } - sql = sql.left(sql.length()-1) + ")\n"; - sql = sql + "REFERENCES " + rtables[pos].name + "("; - for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) - { - sql = sql + rtables[pos].rfields[pos2].rname + ","; + for (pos2 = 0; pos2 < rtables[pos].rfields.count();pos2++) + { + sql = sql + "CONSTRAINT " + rtables[pos].rfields[pos2].rcode + "\n"; + sql = sql + "FOREIGN KEY ("; + sql = sql + rtables[pos].rfields[pos2].name + ")\n"; + sql = sql + "REFERENCES " + rtables[pos].name + "("; + sql = sql + rtables[pos].rfields[pos2].rname + ")\nON DELETE RESTRICT\nON UPDATE NO ACTION,\n"; + } } - sql = sql.left(sql.length()-1) + ")\nON DELETE RESTRICT\nON UPDATE NO ACTION,\n"; + idx++; } } diff --git a/utilities/mergeVersions/mergecreate.h b/utilities/mergeVersions/mergecreate.h index 66c7ebb19..a073286c8 100644 --- a/utilities/mergeVersions/mergecreate.h +++ b/utilities/mergeVersions/mergecreate.h @@ -45,6 +45,7 @@ class mergeCreate : public QObject { QString parentTable; QString name; + bool isLookUp = false; QList rfields; }; typedef rtableDef TrtableDef; From 9b382d94427229ea36318a7a506d0a758d5eb28d Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Tue, 10 Sep 2019 22:54:08 -0600 Subject: [PATCH 20/24] Fix _submitted_by field size --- JXFormToMysql/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index 9d67fa762..be6797d33 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -5096,7 +5096,7 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di f_submitted_by.desc.append(langDesc); } f_submitted_by.type = "varchar"; - f_submitted_by.size = 0; + f_submitted_by.size = 255; f_submitted_by.decSize = 0; f_submitted_by.rField = ""; f_submitted_by.rTable = ""; From dfc6c52ee760bb69f73c42a0d6800400296bef4e Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Wed, 11 Sep 2019 11:08:58 -0600 Subject: [PATCH 21/24] Add _xform_id_string to schema --- JXFormToMysql/main.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index be6797d33..0fb162a14 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -5111,6 +5111,32 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di f_submitted_by.selectListName = "NONE"; maintable.fields.append(f_submitted_by); + //Append the _xform_id_stringy field to the main table + TfieldDef f_xform_id_string; + f_xform_id_string.name = "_xform_id_string"; + for (lang = 0; lang <= languages.count()-1;lang++) + { + TlngLkpDesc langDesc; + langDesc.langCode = languages[lang].code; + langDesc.desc = "XForm ID"; + f_xform_id_string.desc.append(langDesc); + } + f_xform_id_string.type = "varchar"; + f_xform_id_string.size = 255; + f_xform_id_string.decSize = 0; + f_xform_id_string.rField = ""; + f_xform_id_string.rTable = ""; + f_xform_id_string.key = false; + f_xform_id_string.sensitive = false; + f_xform_id_string.isMultiSelect = false; + f_xform_id_string.xmlCode = "_submitted_by"; + f_xform_id_string.odktype = "text"; + f_xform_id_string.calculateWithSelect = false; + f_xform_id_string.formula = ""; + f_xform_id_string.selectSource = "NONE"; + f_xform_id_string.selectListName = "NONE"; + maintable.fields.append(f_xform_id_string); + //Append the Submmited date field to the main table TfieldDef f_submitted_date; f_submitted_date.name = "_submitted_date"; From 3c35cc5b482d92d02701a2e52acfe064215c2a39 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Wed, 11 Sep 2019 15:22:31 -0600 Subject: [PATCH 22/24] Fix _xform_id_string xml code --- JXFormToMysql/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index 0fb162a14..7a0ce099f 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -5129,7 +5129,7 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di f_xform_id_string.key = false; f_xform_id_string.sensitive = false; f_xform_id_string.isMultiSelect = false; - f_xform_id_string.xmlCode = "_submitted_by"; + f_xform_id_string.xmlCode = "_xform_id_string"; f_xform_id_string.odktype = "text"; f_xform_id_string.calculateWithSelect = false; f_xform_id_string.formula = ""; From f3819e58be2dc277c0e72495ac6e2058324e7540 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Wed, 11 Sep 2019 15:23:08 -0600 Subject: [PATCH 23/24] Properly place new variables in C --- utilities/mergeVersions/mergecreate.cpp | 109 +++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/utilities/mergeVersions/mergecreate.cpp b/utilities/mergeVersions/mergecreate.cpp index def11febe..825fe484a 100644 --- a/utilities/mergeVersions/mergecreate.cpp +++ b/utilities/mergeVersions/mergecreate.cpp @@ -813,6 +813,30 @@ void mergeCreate::checkField(QDomNode eTable, QDomElement a, QDomElement b, bool } } +QDomNode getLastField(QDomNode table) +{ + QDomNode child = table.firstChild(); + QDomNode res; + while (!child.isNull()) + { + res = child; + child = child.nextSibling(); + } + return res; +} + +QDomNode findNodeByName(QDomNode table, QString name) +{ + QDomNode child = table.firstChild(); + while (!child.isNull()) + { + if (child.toElement().attribute("name","") == name) + return child; + child = child.nextSibling(); + } + return QDomNode(); +} + void mergeCreate::compareLKPTables(QDomNode table,QDomDocument &docB) { QDomNode node; @@ -851,7 +875,46 @@ void mergeCreate::compareLKPTables(QDomNode table,QDomDocument &docB) errorList.append(error); } addFieldToDiff(tablefound.toElement().attribute("name",""),eField); - tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); + QDomNode node_before = eField.previousSibling(); + if (!node_before.isNull()) + { + if (node_before.toElement().tagName() != "field") + node_before = QDomNode(); + } + QDomNode node_after = eField.previousSibling(); + if (!node_after.isNull()) + { + if (node_after.toElement().tagName() != "field") + node_after = QDomNode(); + } + QDomNode reference; + bool after = true; + if (node_before.isNull() && node_after.isNull()) + reference = getLastField(tablefound); + else + { + if (!node_before.isNull()) + { + reference = findNodeByName(tablefound,node_before.toElement().attribute("name")); + if (reference.isNull()) + reference = getLastField(tablefound); + } + else + { + reference = findNodeByName(tablefound,node_after.toElement().attribute("name")); + if (reference.isNull()) + { + reference = getLastField(tablefound); + } + else + after = false; + } + } + + if (after) + tablefound.insertAfter(eField.cloneNode(true),reference); + else + tablefound.insertBefore(eField.cloneNode(true),reference); } } @@ -902,7 +965,7 @@ void mergeCreate::compareTables(QDomNode table,QDomDocument &docB) QDomNode fieldFound = findField(tablefound,eField.attribute("name","")); if (!fieldFound.isNull()) { - checkField(tablefound,field.toElement(),fieldFound.toElement()); + checkField(tablefound,field.toElement(),fieldFound.toElement(),false); } else { @@ -919,7 +982,47 @@ void mergeCreate::compareTables(QDomNode table,QDomDocument &docB) errorList.append(error); } addFieldToDiff(tablefound.toElement().attribute("name",""),eField); - tablefound.insertBefore(eField.cloneNode(true),tablefound.firstChild()); + QDomNode node_before = eField.previousSibling(); + if (!node_before.isNull()) + { + if (node_before.toElement().tagName() != "field") + node_before = QDomNode(); + } + QDomNode node_after = eField.previousSibling(); + if (!node_after.isNull()) + { + if (node_after.toElement().tagName() != "field") + node_after = QDomNode(); + } + QDomNode reference; + bool after = true; + if (node_before.isNull() && node_after.isNull()) + reference = getLastField(tablefound); + else + { + if (!node_before.isNull()) + { + reference = findNodeByName(tablefound,node_before.toElement().attribute("name")); + if (reference.isNull()) + reference = getLastField(tablefound); + } + else + { + reference = findNodeByName(tablefound,node_after.toElement().attribute("name")); + if (reference.isNull()) + { + reference = getLastField(tablefound); + } + else + after = false; + } + } + + if (after) + tablefound.insertAfter(eField.cloneNode(true),reference); + else + tablefound.insertBefore(eField.cloneNode(true),reference); + } } From 1c8572373646b02b65e8a6ad448b19657f11e6f7 Mon Sep 17 00:00:00 2001 From: Carlos Quiros Date: Thu, 12 Sep 2019 00:13:05 -0600 Subject: [PATCH 24/24] Fix bug in sql --- utilities/mergeVersions/mergecreate.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/utilities/mergeVersions/mergecreate.cpp b/utilities/mergeVersions/mergecreate.cpp index 825fe484a..5b76b3ff8 100644 --- a/utilities/mergeVersions/mergecreate.cpp +++ b/utilities/mergeVersions/mergecreate.cpp @@ -356,15 +356,26 @@ void mergeCreate::addFieldToDiff(QString table, QDomElement eField) { QString sql; sql = "ALTER TABLE " + table + " ADD COLUMN " + eField.attribute("name",""); - if (eField.attribute("type","") == "decimal") - sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + "," + eField.attribute("decsize","0") + ");\n"; + + if ((eField.attribute("type","") == "varchar") || (eField.attribute("type","") == "int")) + sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + ");\n"; else { - if (eField.attribute("type","") != "text") - sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + ");\n"; + if (eField.attribute("type","") == "decimal") + sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + "," + eField.attribute("decsize","0") + ");\n"; else sql = sql + " " + eField.attribute("type","") + ";\n"; } + +// if (eField.attribute("type","") == "decimal") +// sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + "," + eField.attribute("decsize","0") + ");\n"; +// else +// { +// if (eField.attribute("type","") != "text") +// sql = sql + " " + eField.attribute("type","") + " (" + eField.attribute("size","0") + ");\n"; +// else +// sql = sql + " " + eField.attribute("type","") + ";\n"; +// } diff.append(sql); if (eField.attribute("rtable","") != "") {