Line data Source code
1 : #include "otsdaq/MessageFacility/MessageFacility.h"
2 :
3 : #include <dirent.h>
4 : #include <cassert>
5 : #include <iostream>
6 : #include <memory>
7 : #include <string>
8 :
9 : #include "otsdaq/ConfigurationInterface/ConfigurationInterface.h"
10 : #include "otsdaq/ConfigurationInterface/ConfigurationManagerRW.h"
11 :
12 : // Shared test utilities
13 : #include "otsdaq/Macros/TestUtilities.h"
14 :
15 : /// Imports all groups and group/table aliases (from backbone group) found at path
16 : /// to the current database and active backbone group. New table version and group keys will be assigned
17 : /// to the imported tables and groups.
18 : /// usage:
19 : /// otsdaq_import_groups_from_export_path <path to import from>
20 : ///
21 : ///
22 : using namespace ots;
23 :
24 0 : void ImportTableGroupsFromPath(int argc, char* argv[])
25 : {
26 0 : std::cout << "=================================================\n";
27 0 : std::cout << "=================================================\n";
28 0 : std::cout << "=================================================\n";
29 0 : __COUT__ << "\nImporting Groups!" << std::endl;
30 :
31 0 : std::cout << "\n\nusage: One argument:\n\t <import path> \n\n" << std::endl;
32 :
33 0 : std::cout << "argc = " << argc << std::endl;
34 0 : for(int i = 0; i < argc; i++)
35 0 : std::cout << "argv[" << i << "] = " << argv[i] << std::endl;
36 :
37 0 : if(argc < 2)
38 : {
39 0 : std::cout << "Error! Must provide at least one parameter.\n\n" << std::endl;
40 0 : return;
41 : }
42 :
43 0 : std::string importPath = argv[1];
44 0 : std::string prepend = argc > 2 ? argv[2] : ""; //get prepend arg or empty default
45 0 : bool forceBackboneSave =
46 0 : argc > 3 ? true : false; //get force backbone save arg or false
47 0 : int flatVersion = 0;
48 :
49 0 : __COUTV__(importPath);
50 0 : __COUTV__(flatVersion);
51 0 : __COUTV__(prepend);
52 0 : __COUTV__(forceBackboneSave);
53 :
54 : //==============================================================================
55 : // Steps:
56 : //
57 : // -- create empty map of import alias to original groupName & groupKey
58 : // -- create empty map of original groupName & groupKey to new groupKey
59 : //
60 : // -- create empty map of import alias to original tableName & tableVersion
61 : // -- create empty map of original tableName & tableVersion to new tableVersion
62 : //
63 : // -- check import basename+alias for collision with existing group aliases and throw
64 : // error if collision
65 : // -- check import basename+alias for collision with existing table aliases and throw
66 : // error if collision
67 : //
68 : // -- track map of import alias --> import group name/key --> new group key
69 : // -- track map of import table name --> import version --> new table version, to prevent looking multiple times for duplicate tables
70 : // -- for each group to import
71 : // - for each table in group to import
72 : // . load json into table view, check that table view is unique
73 : // . if not unique update member map version
74 : // - save (modified) member map as new group
75 : // - report to user import alias --> import group/table name/key --> new group/table key
76 : //
77 : // -- reload active backbone group
78 : // -- insert new aliases for imported groups in current active backbone
79 : // - should be basename+alias connection to (hop through maps) new groupName & groupKey
80 : // -- insert new aliases for imported tables
81 : // - should be basename+alias connection to (hop through maps) new tableName & tableVersion
82 : // -- save new backbone tables and save new backbone group
83 : // -- backup the file ConfigurationManager::ACTIVE_GROUPS_FILENAME with time
84 : // -- activate the new backbone group
85 : //
86 : //==============================================================================
87 :
88 : // clang-format off
89 : std::map<std::string /*importGroupAlias*/,
90 0 : /*original*/ std::pair<std::string /*groupName*/, TableGroupKey /*origKey*/>> importGroupAliasMap;
91 : std::map<std::pair<std::string /*groupName*/, TableGroupKey /*origKey*/>,
92 0 : TableGroupKey /*newKey*/> importGroupMap;
93 : std::map<std::string /*importTableAlias*/,
94 : /*original*/ std::pair<std::string /*tableName*/,
95 0 : TableVersion /*origVersion*/>> importTableAliasMap;
96 : std::map</*original*/ std::pair<std::string /*tableName*/, TableVersion /*origVersion*/>,
97 0 : /*newVersion*/ TableVersion> importTableMap;
98 : // clang-format on
99 :
100 0 : std::string importedBackboneGroupName = "";
101 0 : TableGroupKey importedBackboneGroupKey;
102 :
103 : // add groups to vector list from directory
104 : {
105 : DIR* dp;
106 : struct dirent* dirp;
107 0 : if((dp = opendir(importPath.c_str())) == 0)
108 : {
109 0 : __COUT_ERR__ << "ERROR:(" << errno
110 0 : << "). Can't open directory: " << importPath << std::endl;
111 0 : exit(0);
112 : }
113 :
114 0 : const unsigned char isDir = 0x4;
115 0 : while((dirp = readdir(dp)) != 0)
116 0 : if(dirp->d_type == isDir && dirp->d_name[0] != '.') //if a directory type
117 : {
118 0 : __COUT__ << "Directory: " << dirp->d_name << std::endl;
119 :
120 0 : auto split = StringMacros::getVectorFromString(dirp->d_name, {'_'});
121 :
122 0 : if(split.size() != 2)
123 0 : continue;
124 :
125 0 : importGroupMap.insert(
126 0 : std::pair<
127 : std::pair<std::string /*groupName*/, TableGroupKey /*origKey*/>,
128 : TableGroupKey /*newKey*/>(
129 0 : std::pair<std::string, TableGroupKey>(split[0], split[1]),
130 0 : TableGroupKey()));
131 :
132 : //check if group is the active backbone to import
133 : std::string groupIsBackbonePath =
134 0 : importPath + "/" + dirp->d_name + "/" + "groupIsBackbone.txt";
135 0 : __COUTV__(groupIsBackbonePath);
136 0 : FILE* fp = std::fopen(groupIsBackbonePath.c_str(), "r");
137 0 : if(fp)
138 : {
139 0 : fclose(fp);
140 0 : __COUT_INFO__ << "Found imported Backbone as " << split[0] << " ("
141 0 : << split[1] << ")" << __E__;
142 0 : importedBackboneGroupName = split[0];
143 0 : importedBackboneGroupKey = TableGroupKey(split[1]);
144 : }
145 0 : } //end found group directory handling
146 :
147 0 : closedir(dp);
148 : }
149 :
150 0 : __COUT__ << "Identified groups to import:" << std::endl;
151 0 : for(auto& group : importGroupMap)
152 0 : __COUT__ << " ==> Group to import: " << group.first.first << " ("
153 0 : << group.first.second << ")" << std::endl;
154 0 : __COUTV__(importedBackboneGroupName);
155 :
156 0 : if(!importGroupMap.size())
157 : {
158 0 : __SS__ << "No groups identified to import!" << __E__;
159 0 : __SS_THROW__;
160 0 : }
161 : // return;
162 :
163 : //==============================================================================
164 : // Define environment variables
165 : // Note: normally these environment variables are set by ots script
166 :
167 0 : test::util::check_and_make_envs();
168 :
169 : ////////////////////////////////////////////////////
170 :
171 : // get prepared with initial source db
172 :
173 : // ConfigurationManager instance immediately loads active groups
174 0 : __COUT__ << "Loading active Backbone..." << std::endl;
175 :
176 0 : std::string ARTDAQ_DATABASE_URI = __ENV__("ARTDAQ_DATABASE_URI");
177 0 : __COUTV__(ARTDAQ_DATABASE_URI);
178 :
179 : // return;
180 :
181 0 : ConfigurationManagerRW cfgMgrInst("import_admin");
182 0 : ConfigurationManagerRW* cfgMgr = &cfgMgrInst;
183 :
184 : //get all table/group info
185 : {
186 0 : std::string accumulatedWarnings;
187 : const std::map<std::string, TableInfo>& allTableInfo =
188 0 : cfgMgr->getAllTableInfo(true /* refresh */,
189 : &accumulatedWarnings,
190 : "" /* errorFilterName */,
191 : true /* getGroupKeys*/,
192 : false /* getGroupInfo */,
193 : true /* initializeActiveGroups */);
194 0 : __COUTV__(allTableInfo.size());
195 0 : auto groups = cfgMgr->getAllGroupInfo();
196 0 : __COUTTV__(groups.size());
197 0 : if(TTEST(1))
198 0 : for(auto& group : groups)
199 : {
200 0 : __COUTTV__(group.first);
201 : }
202 0 : }
203 :
204 0 : ConfigurationInterface* theInterface_ = ConfigurationInterface::getInstance(
205 : ConfigurationInterface::CONFIGURATION_MODE::ARTDAQ_DATABASE);
206 :
207 : if(0)
208 : {
209 : __COUT__ << "Test finding equivalent backbone table versions" << __E__;
210 :
211 : auto table =
212 : cfgMgr->getTableByName(ConfigurationManager::VERSION_ALIASES_TABLE_NAME);
213 : __COUTV__(table->getTableName());
214 : __COUTV__(table->getViewVersion());
215 :
216 : auto cfgView = table->getViewP();
217 : unsigned int col0 = cfgView->findCol("VersionAlias");
218 : unsigned int col1 = cfgView->findCol("TableName");
219 : unsigned int col2 = cfgView->findCol("Version");
220 :
221 : unsigned int row;
222 :
223 : cfgView->print();
224 :
225 : TableVersion duplicateVersion;
226 :
227 : // -- insert new aliases for imported tables
228 : // - should be basename+alias connection to (hop through maps) new
229 : // tableName & tableVersion
230 : for(auto& aliasPair : importTableAliasMap)
231 : {
232 : __COUT__ << "\t" << aliasPair.first << " ==> " << aliasPair.second.first
233 : << "-" << aliasPair.second.second << std::endl;
234 :
235 : auto tableIt = importTableMap.find(std::pair<std::string, TableVersion>(
236 : aliasPair.second.first, aliasPair.second.second));
237 :
238 : if(tableIt == importTableMap.end())
239 : {
240 : __COUT__ << "Error! Could not find the new entry for the original table "
241 : << aliasPair.second.first << "(" << aliasPair.second.second
242 : << ")" << __E__;
243 : continue;
244 : }
245 : row =
246 : cfgView->addRow(prepend + "import_aliases", true /*incrementUniqueData*/);
247 : cfgView->setValue(prepend + aliasPair.first, row, col0);
248 : cfgView->setValue(aliasPair.second.first, row, col1);
249 : cfgView->setValue(tableIt->second.toString(), row, col2);
250 : } // end group alias edit
251 :
252 : if(!importTableAliasMap.size())
253 : duplicateVersion =
254 : table->getViewVersion(); //mark duplicate as self if nothing to add
255 :
256 : cfgView->print();
257 : TableVersion originalVersion =
258 : table
259 : ->getViewVersion(); //save version because cache fill will change active version
260 :
261 : if(duplicateVersion.isInvalid())
262 : {
263 : auto tableName = ConfigurationManager::VERSION_ALIASES_TABLE_NAME;
264 : __COUT__ << "Checking for duplicate '" << tableName << "' tables..." << __E__;
265 :
266 : {
267 : //"DEEP" checking
268 : // load into cache 'recent' versions for this table
269 : // 'recent' := those already in cache, plus highest version numbers not in cache
270 : const std::map<std::string, TableInfo>& allTableInfo =
271 : cfgMgr->getAllTableInfo(); // do not refresh
272 :
273 : auto versionReverseIterator =
274 : allTableInfo.at(tableName)
275 : .versions_.rbegin(); // get reverse iterator
276 : __COUT__ << "Filling up '" << tableName << "' cache from "
277 : << table->getNumberOfStoredViews() << " to max count of "
278 : << table->MAX_VIEWS_IN_CACHE << __E__;
279 : for(;
280 : table->getNumberOfStoredViews() < table->MAX_VIEWS_IN_CACHE &&
281 : versionReverseIterator != allTableInfo.at(tableName).versions_.rend();
282 : ++versionReverseIterator)
283 : {
284 : __COUTT__ << "'" << tableName << "' versions in reverse order "
285 : << *versionReverseIterator << __E__;
286 : try
287 : {
288 : cfgMgr->getVersionedTableByName(
289 : tableName,
290 : *versionReverseIterator); // load to cache
291 : }
292 : catch(const std::runtime_error& e)
293 : {
294 : // ignore error
295 : __COUTT__ << "'" << tableName << "' version failed to load: "
296 : << *versionReverseIterator << __E__;
297 : }
298 : }
299 : }
300 :
301 : __COUT__ << "Checking '" << tableName << "' for duplicate..." << __E__;
302 : duplicateVersion =
303 : table->checkForDuplicate(originalVersion,
304 : TableVersion()); // then all versions in search
305 :
306 : } //end check duplicate
307 :
308 : //return the original version to active
309 : table->setActiveView(originalVersion);
310 :
311 : if(!duplicateVersion.isInvalid())
312 : {
313 : // found an equivalent!
314 : __COUT__ << "Equivalent table found in version v" << duplicateVersion
315 : << __E__;
316 : }
317 : else
318 : {
319 : __COUTV__(table->getViewVersion());
320 : auto newVersion =
321 : TableVersion::getNextVersion(theInterface_->findLatestVersion(table));
322 : __COUTV__(newVersion);
323 : // cfgView->setVersion(newVersion);
324 : // theInterface_->saveActiveVersion(table);
325 : }
326 :
327 : } //end test version alias new version
328 :
329 : // return;
330 :
331 : //if missing Active Groups File, create empty backbone group (e.g., to seed a new database)
332 0 : FILE* fp = nullptr;
333 0 : if(!(fp = fopen(ConfigurationManager::ACTIVE_GROUPS_FILENAME.c_str(), "r")))
334 : {
335 0 : __COUT_INFO__ << "Identified missing Active Groups File: "
336 0 : << ConfigurationManager::ACTIVE_GROUPS_FILENAME << __E__;
337 0 : __COUT_INFO__ << "Creating an empty Backbone group... group name will match "
338 0 : "imported backbone name: "
339 0 : << importedBackboneGroupName << __E__;
340 :
341 0 : std::map<std::string, TableVersion> backboneMemberMap;
342 0 : for(auto& memberName : ConfigurationManager::getBackboneMemberNames())
343 : {
344 : //create empty mockup version
345 : // if mockup, then generate a new persistent version to use based on mockup
346 0 : TableBase* table = cfgMgr->getTableByName(memberName);
347 : // create a temporary version from the mockup as source version
348 0 : TableVersion temporaryVersion = table->createTemporaryView();
349 0 : __COUT__ << "\t\ttemporaryVersion: " << temporaryVersion << __E__;
350 :
351 : // if other versions exist check for another mockup, and use that instead
352 0 : __COUT__ << "Creating version from mock-up for name: " << memberName
353 0 : << " temporaryVersion: " << temporaryVersion << __E__;
354 :
355 : // set table comment
356 : table->getTemporaryView(temporaryVersion)
357 0 : ->setComment("Auto-generated from mock-up.");
358 :
359 : // finish off the version creation
360 : bool foundEquivalent;
361 : auto newAssignedVersion =
362 : cfgMgr->saveModifiedVersion(memberName,
363 0 : TableVersion() /*original source is mockup*/,
364 : false /*makeTemporary*/,
365 : table,
366 : temporaryVersion,
367 : false /*ignoreDuplicates*/,
368 : true /*lookForEquivalent*/,
369 0 : &foundEquivalent);
370 0 : if(foundEquivalent)
371 0 : __COUT__ << "Found equivalent version: " << newAssignedVersion << __E__;
372 : else
373 0 : __COUT__ << "Created new version: " << newAssignedVersion << __E__;
374 :
375 0 : backboneMemberMap[memberName] = newAssignedVersion;
376 0 : }
377 0 : __COUTV__(StringMacros::mapToString(backboneMemberMap));
378 :
379 0 : bool foundExistingEmptyBackbone = false;
380 : { //check if group is a duplicate
381 0 : __COUT__ << "Checking for duplicate groups..." << __E__;
382 : try
383 : {
384 : TableGroupKey foundKey = cfgMgr->findTableGroup(
385 : importedBackboneGroupName,
386 0 : backboneMemberMap); //, memberTableAliases); std::map<std::string /*name*/, std::string /*alias*/> memberTableAliases
387 :
388 0 : if(!foundKey.isInvalid())
389 : {
390 0 : __COUT_WARN__ << "Found equivalent empty backbone group key ("
391 0 : << foundKey << ") for " << importedBackboneGroupName
392 0 : << ". Skipping creation and activating existing key!"
393 0 : << __E__;
394 0 : foundExistingEmptyBackbone = true;
395 :
396 : // -- activate the existing empty backbone group
397 0 : cfgMgr->activateTableGroup(
398 : importedBackboneGroupName,
399 : foundKey); // and write to active group file
400 : }
401 :
402 0 : __COUT__ << "Check for empty backbone duplicate groups complete."
403 0 : << __E__;
404 0 : }
405 0 : catch(...)
406 : {
407 0 : __COUT_WARN__ << "Ignoring errors looking for empty backbone duplicate "
408 : "groups! Proceeding "
409 0 : "with new group creation."
410 0 : << __E__;
411 0 : }
412 : } //end check if group is a duplicate
413 0 : __COUTV__(foundExistingEmptyBackbone);
414 :
415 0 : if(!foundExistingEmptyBackbone) //save new empty backbone
416 : {
417 : TableGroupKey newKey = TableGroupKey::getNextKey(
418 0 : theInterface_->findLatestGroupKey(importedBackboneGroupName));
419 :
420 : // save group, and retry on save collision
421 0 : uint16_t retries = 0;
422 : while(1)
423 : {
424 0 : __COUT__ << "New Key for empty backbone group: "
425 0 : << importedBackboneGroupName << " found as " << newKey << __E__;
426 :
427 : try
428 : {
429 0 : theInterface_->saveTableGroup(backboneMemberMap,
430 0 : TableGroupKey::getFullGroupString(
431 : importedBackboneGroupName, newKey));
432 : }
433 0 : catch(const std::runtime_error& e)
434 : {
435 0 : __COUT__ << "Caught runtime_error exception during group save."
436 0 : << __E__;
437 0 : if(std::string(e.what()).find("there was a collision") !=
438 : std::string::npos)
439 : {
440 0 : __COUT_WARN__
441 0 : << "There was a collision saving the new group "
442 0 : << importedBackboneGroupName << "(" << newKey
443 0 : << "), trying incremented group key... retries=" << retries
444 0 : << __E__;
445 0 : if(++retries > 3) //give up
446 0 : throw;
447 0 : newKey = TableGroupKey::getNextKey(newKey); //increment group key
448 0 : __COUT__ << "New Key for group: " << importedBackboneGroupName
449 0 : << " found as " << newKey << __E__;
450 0 : continue;
451 0 : }
452 : else
453 0 : throw;
454 0 : }
455 0 : __COUT__ << "Created new empty backbone table group: "
456 0 : << importedBackboneGroupName << "(" << newKey << ")" << __E__;
457 0 : break;
458 0 : } //end collission retry loop
459 :
460 : // -- activate the new empty backbone group
461 0 : cfgMgr->activateTableGroup(importedBackboneGroupName,
462 : newKey); // and write to active group file
463 0 : }
464 :
465 0 : __COUT_INFO__ << "Empty backbone group now activated!" << __E__;
466 : // return;
467 0 : }
468 0 : else if(fp)
469 0 : fclose(fp);
470 :
471 : //load active backbone
472 : {
473 0 : std::string accumulatedWarnings;
474 0 : cfgMgr->restoreActiveTableGroups(
475 : false /*throwErrors*/,
476 : "" /*pathToActiveGroupsFile*/,
477 : ConfigurationManager::LoadGroupType::ONLY_BACKBONE_TYPE,
478 : &accumulatedWarnings);
479 :
480 0 : __COUT__ << "Done Loading active backbone." << std::endl;
481 0 : }
482 :
483 : // return;
484 :
485 : /* <tableName, <origVersion, newVersion> >*/
486 0 : std::map<std::pair<std::string, TableVersion>, TableVersion> modifiedTables;
487 0 : std::map<std::string, std::pair<TableGroupKey, TableGroupKey>> activeGroupKeys;
488 0 : std::map<std::pair<std::string, TableGroupKey>, std::string> groupErrors;
489 :
490 0 : std::string activeBackboneGroupName = "";
491 0 : std::string activeContextGroupName = "";
492 0 : std::string activeIterateGroupName = "";
493 0 : std::string activeConfigGroupName = "";
494 :
495 0 : std::string nowTime = std::to_string(time(0));
496 :
497 : // Find active backbone ---------------
498 : std::map<std::string, std::pair<std::string, TableGroupKey>> activeGroupsMap =
499 0 : cfgMgr->getActiveTableGroups();
500 :
501 0 : bool foundAnyActiveGroups = false;
502 :
503 0 : for(const auto& activeGroup : activeGroupsMap)
504 : {
505 0 : if(activeGroup.second.second.TableGroupKey::isInvalid())
506 0 : continue;
507 :
508 0 : activeGroupKeys.insert(
509 0 : std::pair<std::string, std::pair<TableGroupKey, TableGroupKey>>(
510 0 : activeGroup.second.first,
511 0 : std::pair<TableGroupKey, TableGroupKey>(activeGroup.second.second,
512 0 : TableGroupKey())));
513 :
514 0 : if(activeGroup.first == ConfigurationManager::GROUP_TYPE_NAME_BACKBONE)
515 : {
516 0 : activeBackboneGroupName = activeGroup.second.first;
517 0 : __COUT__ << "found activeBackboneGroupName = " << activeBackboneGroupName
518 0 : << std::endl;
519 0 : foundAnyActiveGroups = true;
520 : }
521 0 : else if(activeGroup.first == ConfigurationManager::GROUP_TYPE_NAME_CONTEXT)
522 : {
523 0 : activeContextGroupName = activeGroup.second.first;
524 0 : __COUT__ << "found activeContextGroupName = " << activeContextGroupName
525 0 : << std::endl;
526 0 : foundAnyActiveGroups = true;
527 : }
528 0 : else if(activeGroup.first == ConfigurationManager::GROUP_TYPE_NAME_ITERATE)
529 : {
530 0 : activeIterateGroupName = activeGroup.second.first;
531 0 : __COUT__ << "found activeIterateGroupName = " << activeIterateGroupName
532 0 : << std::endl;
533 0 : foundAnyActiveGroups = true;
534 : }
535 0 : else if(activeGroup.first == ConfigurationManager::GROUP_TYPE_NAME_CONFIGURATION)
536 : {
537 0 : activeConfigGroupName = activeGroup.second.first;
538 0 : __COUT__ << "found activeConfigGroupName = " << activeConfigGroupName
539 0 : << std::endl;
540 0 : foundAnyActiveGroups = true;
541 : }
542 : }
543 0 : __COUT__ << "Identified active groups: " << activeGroupsMap.size() << std::endl;
544 0 : for(auto& group : activeGroupsMap)
545 0 : __COUT__ << " ==> Active Group of type " << group.first << ": "
546 0 : << group.second.first << " (" << group.second.second << ")" << std::endl;
547 :
548 0 : __COUT__ << "Identified groups to import: " << importGroupMap.size() << std::endl;
549 0 : for(auto& group : importGroupMap)
550 0 : __COUT__ << " ==> Group to import: " << group.first.first << " ("
551 0 : << group.first.second << ")" << std::endl;
552 0 : __COUTV__(importedBackboneGroupName);
553 0 : __COUTV__(prepend);
554 :
555 0 : if(activeBackboneGroupName == "" || !foundAnyActiveGroups)
556 : {
557 0 : __SS__
558 : << "Did not find valid active groups in current database as starting point "
559 : "for import! "
560 : "Must have a backbone at least. Is the current database URI correct? "
561 : "ARTDAQ_DATABASE_URI = "
562 : << ARTDAQ_DATABASE_URI
563 : << "\n\n*** Note: The active groups are specified by the Active Groups file: "
564 : << ConfigurationManager::ACTIVE_GROUPS_FILENAME
565 : << "\n*** If you want an empty backbone to be generated for you as a "
566 0 : "starting point, please delete the Active Groups file."
567 0 : << std::endl;
568 0 : __SS_THROW__;
569 0 : }
570 : //return; //comment for production functionality
571 :
572 : // ------------
573 : // At this point, all groups to import have been identified! and which import is Backbone has been identified ------------
574 : // now check for any alias conflicts between current Backbone and Backbone-to-Import
575 :
576 : { //check import backbone group and table aliases for uniqueness
577 0 : std::string importBackbonePath = importPath + "/" + importedBackboneGroupName +
578 0 : "_" + importedBackboneGroupKey.str();
579 0 : __COUTV__(importBackbonePath);
580 :
581 : std::map<std::string /* tableName */, TableVersion /* tableVersion */>
582 0 : memberTableSet;
583 : { //get member table file names
584 : DIR* dp;
585 : struct dirent* dirp;
586 0 : if((dp = opendir(importBackbonePath.c_str())) == 0)
587 : {
588 0 : __COUT_ERR__ << "ERROR:(" << errno
589 0 : << "). Can't open directory: " << importBackbonePath
590 0 : << std::endl;
591 0 : exit(0);
592 : }
593 :
594 0 : const unsigned char isDir = 0x4;
595 0 : while((dirp = readdir(dp)) != 0)
596 0 : if(dirp->d_type != isDir && strlen(dirp->d_name) > 5 &&
597 0 : dirp->d_name[strlen(dirp->d_name) - 5] == '.' &&
598 0 : dirp->d_name[strlen(dirp->d_name) - 4] == 'j' &&
599 0 : dirp->d_name[strlen(dirp->d_name) - 3] == 's' &&
600 0 : dirp->d_name[strlen(dirp->d_name) - 2] == 'o' &&
601 0 : dirp->d_name[strlen(dirp->d_name) - 1] ==
602 : 'n') //if not directory w/extension .json
603 : {
604 0 : __COUT__ << dirp->d_name << std::endl;
605 :
606 0 : auto split = StringMacros::getVectorFromString(dirp->d_name, {'_'});
607 :
608 0 : if(split.size() != 2)
609 0 : continue;
610 :
611 0 : memberTableSet.insert(std::pair<std::string, TableVersion>(
612 0 : split[0], split[1].substr(1, split[1].size() - 6)));
613 0 : } //end found group directory handling
614 :
615 0 : closedir(dp);
616 : } //end load of backbone alias table names/versions from directory
617 :
618 : { //verify group aliases are unique
619 : std::vector<std::pair<std::string, ConfigurationTree>> currentAliasNodePairs =
620 0 : cfgMgr->getNode(ConfigurationManager::GROUP_ALIASES_TABLE_NAME)
621 0 : .getChildren();
622 0 : std::set<std::string /* current aliases */> currentAliases;
623 :
624 0 : __COUTV__(cfgMgr->getNode(ConfigurationManager::GROUP_ALIASES_TABLE_NAME)
625 0 : .getTableName());
626 0 : __COUTV__(cfgMgr->getNode(ConfigurationManager::GROUP_ALIASES_TABLE_NAME)
627 0 : .getTableVersion());
628 :
629 0 : __COUT__ << "Existing group aliases:" << __E__;
630 0 : for(auto& groupPair : currentAliasNodePairs)
631 : {
632 0 : __COUT__ << " ==> Current group alias "
633 0 : << groupPair.second.getNode("GroupKeyAlias").getValueAsString()
634 0 : << ": "
635 0 : << groupPair.second.getNode("GroupName").getValueAsString()
636 0 : << " ("
637 0 : << TableGroupKey(
638 0 : groupPair.second.getNode("GroupKey").getValueAsString())
639 0 : << ")" << __E__;
640 :
641 0 : currentAliases.emplace(
642 0 : groupPair.second.getNode("GroupKeyAlias").getValueAsString());
643 : }
644 :
645 : std::string fullpath =
646 0 : importBackbonePath + "/" +
647 0 : ConfigurationManager::GROUP_ALIASES_TABLE_NAME + "_v" +
648 0 : memberTableSet.at(ConfigurationManager::GROUP_ALIASES_TABLE_NAME).str() +
649 0 : ".json";
650 :
651 0 : std::string json;
652 0 : std::FILE* fp = std::fopen(fullpath.c_str(), "rb");
653 0 : if(!fp)
654 : {
655 0 : __SS__ << "Could not open file at " << fullpath << ". Error: " << errno
656 0 : << " - " << strerror(errno) << __E__;
657 0 : __SS_THROW__;
658 0 : }
659 0 : std::fseek(fp, 0, SEEK_END);
660 0 : json.resize(std::ftell(fp));
661 0 : std::rewind(fp);
662 0 : std::fread(&json[0], 1, json.size(), fp);
663 0 : std::fclose(fp);
664 :
665 0 : __COUTV__(json);
666 :
667 : auto aliasTable =
668 0 : cfgMgr->getTableByName(ConfigurationManager::GROUP_ALIASES_TABLE_NAME);
669 0 : auto temporaryVersion = aliasTable->createTemporaryView();
670 0 : auto aliasView = aliasTable->getViewP(temporaryVersion);
671 :
672 0 : aliasView->fillFromJSON(json);
673 0 : aliasView->print();
674 :
675 0 : auto aliasCol = aliasView->findCol("GroupKeyAlias");
676 0 : auto groupNameCol = aliasView->findCol("GroupName");
677 0 : auto groupKeyCol = aliasView->findCol("GroupKey");
678 :
679 0 : __COUT__ << "Existing group aliases: " << currentAliases.size() << __E__;
680 0 : for(auto& currentAlias : currentAliases)
681 0 : __COUTV__(currentAlias);
682 :
683 0 : __COUT__ << "Checking that group aliases to import are unique:" << __E__;
684 0 : for(size_t row = 0; row < aliasView->getNumberOfRows(); ++row)
685 : {
686 0 : std::string aliasName = aliasView->getValueAsString(row, aliasCol);
687 0 : __COUT__ << " ==> Group alias to import at row " << row << ": "
688 0 : << aliasName << __E__;
689 0 : if(currentAliases.find(prepend + aliasName) != currentAliases.end())
690 : {
691 0 : __SS__ << "The imported prepend + group alias of '" << prepend
692 : << "' + '" << aliasName
693 : << "' already exists as a group alias in the current database."
694 : << " Please modify the prepend string to make the imported "
695 0 : "alias names unique."
696 0 : << __E__;
697 0 : __SS_THROW__;
698 0 : }
699 0 : importGroupAliasMap[aliasName] =
700 0 : std::pair<std::string /*groupName*/, TableGroupKey /*origKey*/>(
701 0 : aliasView->getValueAsString(row, groupNameCol),
702 0 : aliasView->getValueAsString(row, groupKeyCol));
703 0 : }
704 0 : __COUT__ << "Verified group aliases to import are unique." << __E__;
705 0 : } //end verify group aliases are unique
706 :
707 : { //verify table aliases are unique
708 : std::vector<std::pair<std::string, ConfigurationTree>> currentAliasNodePairs =
709 0 : cfgMgr->getNode(ConfigurationManager::VERSION_ALIASES_TABLE_NAME)
710 0 : .getChildren();
711 0 : std::set<std::string /* current aliases */> currentAliases;
712 :
713 0 : __COUTV__(cfgMgr->getNode(ConfigurationManager::VERSION_ALIASES_TABLE_NAME)
714 0 : .getTableName());
715 0 : __COUTV__(cfgMgr->getNode(ConfigurationManager::VERSION_ALIASES_TABLE_NAME)
716 0 : .getTableVersion());
717 :
718 0 : __COUT__ << "Existing table aliases: " << currentAliasNodePairs.size()
719 0 : << __E__;
720 0 : for(auto& groupPair : currentAliasNodePairs)
721 : {
722 0 : __COUT__ << " ==> Current table alias "
723 0 : << groupPair.second.getNode("VersionAlias").getValueAsString()
724 0 : << ": "
725 0 : << groupPair.second.getNode("TableName").getValueAsString()
726 0 : << " ("
727 0 : << TableGroupKey(
728 0 : groupPair.second.getNode("Version").getValueAsString())
729 0 : << ")" << __E__;
730 :
731 0 : currentAliases.emplace(
732 0 : groupPair.second.getNode("VersionAlias").getValueAsString());
733 : }
734 :
735 : std::string fullpath =
736 0 : importBackbonePath + "/" +
737 0 : ConfigurationManager::VERSION_ALIASES_TABLE_NAME + "_v" +
738 0 : memberTableSet.at(ConfigurationManager::VERSION_ALIASES_TABLE_NAME)
739 0 : .str() +
740 0 : ".json";
741 :
742 0 : std::string json;
743 0 : std::FILE* fp = std::fopen(fullpath.c_str(), "rb");
744 0 : if(!fp)
745 : {
746 0 : __SS__ << "Could not open file at " << fullpath << ". Error: " << errno
747 0 : << " - " << strerror(errno) << __E__;
748 0 : __SS_THROW__;
749 0 : }
750 0 : std::fseek(fp, 0, SEEK_END);
751 0 : json.resize(std::ftell(fp));
752 0 : std::rewind(fp);
753 0 : std::fread(&json[0], 1, json.size(), fp);
754 0 : std::fclose(fp);
755 :
756 0 : __COUTV__(json);
757 :
758 : auto aliasTable =
759 0 : cfgMgr->getTableByName(ConfigurationManager::VERSION_ALIASES_TABLE_NAME);
760 0 : auto temporaryVersion = aliasTable->createTemporaryView();
761 0 : auto aliasView = aliasTable->getViewP(temporaryVersion);
762 :
763 0 : aliasView->fillFromJSON(json);
764 0 : aliasView->print();
765 :
766 0 : auto aliasCol = aliasView->findCol("VersionAlias");
767 0 : auto tablepNameCol = aliasView->findCol("TableName");
768 0 : auto tableVersionCol = aliasView->findCol("Version");
769 :
770 0 : __COUT__ << "Existing table aliases:" << __E__;
771 0 : for(auto& currentAlias : currentAliases)
772 0 : __COUTV__(currentAlias);
773 :
774 0 : __COUT__ << "Checking that table aliases to import are unique:" << __E__;
775 0 : for(size_t row = 0; row < aliasView->getNumberOfRows(); ++row)
776 : {
777 0 : std::string aliasName = aliasView->getValueAsString(row, aliasCol);
778 0 : __COUT__ << " ==> Table alias to import at row " << row << ": "
779 0 : << aliasName << __E__;
780 0 : if(currentAliases.find(prepend + aliasName) != currentAliases.end())
781 : {
782 0 : __SS__ << "The imported prepend + table alias of '" << prepend
783 : << "' + '" << aliasName
784 : << "' already exists as a table alias in the current database."
785 : << " Please modify the prepend string to make the imported "
786 0 : "alias names unique."
787 0 : << __E__;
788 0 : __SS_THROW__;
789 0 : }
790 0 : importTableAliasMap[aliasName] =
791 0 : std::pair<std::string /*tableName*/, TableVersion /*origVersion*/>(
792 0 : aliasView->getValueAsString(row, tablepNameCol),
793 0 : aliasView->getValueAsString(row, tableVersionCol));
794 0 : }
795 0 : __COUT__ << "Verified table aliases to import are unique." << __E__;
796 0 : } //end verify table aliases are unique
797 :
798 0 : } //end check import backbone group aliases for uniqueness
799 :
800 0 : __COUT__ << "Identified group aliases to import:" << std::endl;
801 0 : for(auto& groupAlias : importGroupAliasMap)
802 0 : __COUT__ << "\t" << groupAlias.first << " ==> " << groupAlias.second.first << " ("
803 0 : << groupAlias.second.second << ")" << std::endl;
804 0 : __COUT__ << "Identified table aliases to import:" << std::endl;
805 0 : for(auto& tableAlias : importTableAliasMap)
806 0 : __COUT__ << "\t" << tableAlias.first << " ==> " << tableAlias.second.first << "-v"
807 0 : << tableAlias.second.second << std::endl;
808 : // return;
809 :
810 : //Now...
811 : // -- for each group to import
812 : // - for each table in group to import
813 : // . load json into table view, check that table view is unique
814 : // . if not unique update member map version
815 : // - save (modified) member map as new group
816 : // - report to user import alias --> import group name/key --> new group key
817 : //
818 0 : bool anyNewGroupSaved = false;
819 0 : __COUT__ << "Importing member tables..." << __E__;
820 0 : for(auto& group : importGroupMap)
821 : {
822 0 : __COUT__ << " ==> Group to import: " << group.first.first << " ("
823 0 : << group.first.second << ")" << std::endl;
824 :
825 : std::string importGroupPath =
826 0 : importPath + "/" + group.first.first + "_" + group.first.second.str();
827 0 : __COUTV__(importGroupPath);
828 :
829 : std::map<std::string /* tableName */, TableVersion /* tableVersion */>
830 0 : memberTableSet;
831 : { //get member table file names
832 : DIR* dp;
833 : struct dirent* dirp;
834 0 : if((dp = opendir(importGroupPath.c_str())) == 0)
835 : {
836 0 : __COUT_ERR__ << "ERROR:(" << errno
837 0 : << "). Can't open directory: " << importGroupPath
838 0 : << std::endl;
839 0 : exit(0);
840 : }
841 :
842 0 : const unsigned char isDir = 0x4;
843 0 : while((dirp = readdir(dp)) != 0)
844 0 : if(dirp->d_type != isDir && strlen(dirp->d_name) > 5 &&
845 0 : dirp->d_name[strlen(dirp->d_name) - 5] == '.' &&
846 0 : dirp->d_name[strlen(dirp->d_name) - 4] == 'j' &&
847 0 : dirp->d_name[strlen(dirp->d_name) - 3] == 's' &&
848 0 : dirp->d_name[strlen(dirp->d_name) - 2] == 'o' &&
849 0 : dirp->d_name[strlen(dirp->d_name) - 1] ==
850 : 'n') //if not directory w/extension .json
851 : {
852 : // __COUT__ << dirp->d_name << std::endl;
853 :
854 0 : auto split = StringMacros::getVectorFromString(dirp->d_name, {'_'});
855 :
856 0 : if(split.size() != 2)
857 0 : continue;
858 :
859 0 : memberTableSet.insert(std::pair<std::string, TableVersion>(
860 0 : split[0], split[1].substr(1, split[1].size() - 6)));
861 0 : } //end found group directory handling
862 :
863 0 : closedir(dp);
864 : } //end load of group member table names/versions from directory
865 :
866 0 : std::map<std::string, TableVersion> groupMembers;
867 0 : for(auto& member : memberTableSet)
868 : {
869 0 : __COUT__ << " ==> Member table to import: " << member.first << " v"
870 0 : << member.second << std::endl;
871 :
872 0 : std::string fullpath = importGroupPath + "/" + member.first + "_v" +
873 0 : member.second.str() + ".json";
874 :
875 0 : std::string json;
876 0 : std::FILE* fp = std::fopen(fullpath.c_str(), "rb");
877 0 : if(!fp)
878 : {
879 0 : __SS__ << "Could not open file at " << fullpath << ". Error: " << errno
880 0 : << " - " << strerror(errno) << __E__;
881 0 : __SS_THROW__;
882 0 : }
883 0 : std::fseek(fp, 0, SEEK_END);
884 0 : json.resize(std::ftell(fp));
885 0 : std::rewind(fp);
886 0 : std::fread(&json[0], 1, json.size(), fp);
887 0 : std::fclose(fp);
888 :
889 0 : __COUTV__(json);
890 :
891 0 : auto memberTable = member.first == TableBase::GROUP_METADATA_TABLE_NAME
892 0 : ? cfgMgr->getMetadataTable()
893 0 : : cfgMgr->getTableByName(member.first);
894 :
895 0 : TableVersion newAssignedVersion;
896 0 : if(member.first == TableBase::GROUP_METADATA_TABLE_NAME)
897 : {
898 : //only one view ever for meta data table
899 0 : auto tableView = memberTable->getViewP();
900 :
901 0 : tableView->print();
902 : //clear all data from table (fill does not clear)
903 0 : while(tableView->getNumberOfRows() > 0)
904 0 : tableView->deleteRow(0);
905 0 : tableView->print();
906 :
907 0 : tableView->fillFromJSON(json);
908 0 : tableView->print();
909 :
910 : // set metadata table version to first available persistent version
911 0 : newAssignedVersion = TableVersion::getNextVersion(
912 0 : theInterface_->findLatestVersion(memberTable));
913 0 : __COUTV__(newAssignedVersion);
914 0 : tableView->setVersion(newAssignedVersion);
915 0 : memberTable->getViewP()->print();
916 :
917 : // save table, and retry on save collision
918 0 : uint16_t retries = 0;
919 : while(1)
920 : {
921 : try
922 : {
923 0 : theInterface_->saveActiveVersion(memberTable);
924 : }
925 0 : catch(const std::runtime_error& e)
926 : {
927 0 : __COUT__ << "Caught runtime_error exception during table save: "
928 0 : << e.what() << __E__;
929 0 : if(std::string(e.what()).find("there was a collision") !=
930 : std::string::npos)
931 : {
932 0 : __COUT_WARN__
933 0 : << "There was a collision saving the new table "
934 0 : << *tableView << "(" << newAssignedVersion
935 0 : << "), trying incremented table version... retries="
936 0 : << retries << __E__;
937 0 : if(++retries > 3) //give up
938 0 : throw;
939 0 : newAssignedVersion = TableVersion::getNextVersion(
940 0 : newAssignedVersion); //increment table version
941 0 : tableView->setVersion(newAssignedVersion);
942 0 : __COUT__ << "New version for table: " << *tableView
943 0 : << " found as " << newAssignedVersion << __E__;
944 0 : continue;
945 0 : }
946 : else
947 0 : throw;
948 0 : }
949 :
950 0 : __COUT__ << "Created table: " << *tableView << "-v"
951 0 : << newAssignedVersion << __E__;
952 0 : break;
953 0 : } //end collission retry loop
954 :
955 0 : __COUTV__(memberTable->getViewVersion());
956 : }
957 : else //else normal member table
958 : {
959 0 : auto temporaryVersion = memberTable->createTemporaryView();
960 0 : auto tableView = memberTable->getViewP(temporaryVersion);
961 :
962 0 : tableView->fillFromJSON(json);
963 0 : tableView->print();
964 :
965 : bool foundEquivalent;
966 : newAssignedVersion =
967 0 : cfgMgr->saveModifiedVersion(member.first,
968 : temporaryVersion,
969 : false /*makeTemporary*/,
970 : memberTable,
971 : temporaryVersion,
972 : false /*ignoreDuplicates*/,
973 : true /*lookForEquivalent*/,
974 0 : &foundEquivalent);
975 0 : if(foundEquivalent)
976 0 : __COUT__ << "Found equivalent version: " << newAssignedVersion
977 0 : << __E__;
978 0 : }
979 0 : __COUTV__(newAssignedVersion);
980 :
981 : //assembly importTableMap at this point
982 0 : importTableMap[std::make_pair(member.first, member.second)] =
983 0 : newAssignedVersion;
984 0 : groupMembers[member.first] = newAssignedVersion;
985 : // return;
986 0 : } //end member table import and find loop
987 :
988 0 : __COUT__ << "Tables imported so far: " << importTableMap.size() << std::endl;
989 0 : for(auto& table : importTableMap)
990 0 : __COUT__ << " ==> Member table imported: " << table.first.first << " v"
991 0 : << table.first.second << " ==> v" << table.second << std::endl;
992 :
993 0 : __COUT__ << "Saving group '" << group.first.first
994 0 : << "' members: " << groupMembers.size() << std::endl;
995 :
996 0 : std::map<std::string, TableVersion> groupMembersWithoutMeta;
997 0 : for(auto& table : groupMembers)
998 : {
999 0 : __COUT__ << " ==> Saving group w/Member table: " << table.first << " v"
1000 0 : << table.second << std::endl;
1001 :
1002 0 : if(table.first != TableBase::GROUP_METADATA_TABLE_NAME)
1003 0 : groupMembersWithoutMeta[table.first] = table.second;
1004 : }
1005 :
1006 : { //check if group is a duplicate
1007 0 : __COUT__ << "Checking for duplicate groups..." << __E__;
1008 : try
1009 : {
1010 : TableGroupKey foundKey = cfgMgr->findTableGroup(
1011 0 : group.first.first,
1012 0 : groupMembersWithoutMeta); //, memberTableAliases); std::map<std::string /*name*/, std::string /*alias*/> memberTableAliases
1013 :
1014 0 : if(!foundKey.isInvalid())
1015 : {
1016 0 : __COUT_WARN__ << "Found equivalent group key (" << foundKey
1017 0 : << ") for " << group.first.first << ". Skipping import!"
1018 0 : << __E__;
1019 : //update key import transformation map
1020 0 : importGroupMap.at(
1021 0 : std::make_pair(group.first.first, group.first.second)) = foundKey;
1022 0 : continue;
1023 0 : }
1024 :
1025 0 : __COUT__ << "Check for duplicate groups complete." << __E__;
1026 0 : }
1027 0 : catch(...)
1028 : {
1029 0 : __COUT_WARN__
1030 : << "Ignoring errors looking for duplicate groups! Proceeding "
1031 0 : "with new group creation."
1032 0 : << __E__;
1033 0 : }
1034 : } //end check if group is a duplicate
1035 : // return;
1036 :
1037 : TableGroupKey newKey = TableGroupKey::getNextKey(
1038 0 : theInterface_->findLatestGroupKey(group.first.first));
1039 0 : __COUT__ << "New Key for group: " << group.first.first << " found as " << newKey
1040 0 : << __E__;
1041 :
1042 : // save group, and retry on save collision
1043 0 : uint16_t retries = 0;
1044 : while(1)
1045 : {
1046 : try
1047 : {
1048 0 : theInterface_->saveTableGroup(
1049 : groupMembers,
1050 0 : TableGroupKey::getFullGroupString(group.first.first, newKey));
1051 : }
1052 0 : catch(const std::runtime_error& e)
1053 : {
1054 0 : __COUT__ << "Caught runtime_error exception during group save: "
1055 0 : << e.what() << __E__;
1056 0 : if(std::string(e.what()).find("there was a collision") !=
1057 : std::string::npos)
1058 : {
1059 0 : __COUT_WARN__
1060 0 : << "There was a collision saving the new group "
1061 0 : << group.first.first << "(" << newKey
1062 0 : << "), trying incremented group key... retries=" << retries
1063 0 : << __E__;
1064 0 : if(++retries > 3) //give up
1065 0 : throw;
1066 0 : newKey = TableGroupKey::getNextKey(newKey); //increment group key
1067 0 : __COUT__ << "New Key for group: " << group.first.first << " found as "
1068 0 : << newKey << __E__;
1069 0 : continue;
1070 0 : }
1071 : else
1072 0 : throw;
1073 0 : }
1074 0 : __COUT__ << "Created table group: " << group.first.first << "(" << newKey
1075 0 : << ")" << __E__;
1076 0 : break;
1077 0 : } //end collission retry loop
1078 :
1079 0 : anyNewGroupSaved = true;
1080 :
1081 : //update key import transformation map
1082 0 : importGroupMap.at(std::make_pair(group.first.first, group.first.second)) = newKey;
1083 : // return;
1084 0 : } //end group import loop
1085 :
1086 0 : if(!forceBackboneSave && !anyNewGroupSaved)
1087 : {
1088 0 : __SS__ << "All groups to import already exist in current db! Was the wrong db "
1089 0 : "selected from which to import?"
1090 0 : << __E__;
1091 0 : __SS_THROW__;
1092 0 : }
1093 :
1094 : // - report to user import alias --> import group name/key --> new group key
1095 0 : __COUT__ << "Tables imported summary: " << importTableMap.size() << std::endl;
1096 0 : for(auto& table : importTableMap)
1097 0 : __COUT__ << " ==> Table imported: " << table.first.first << " v"
1098 0 : << table.first.second << " ==> v" << table.second << std::endl;
1099 :
1100 0 : __COUT__ << "Groups imported summary: " << importGroupMap.size() << std::endl;
1101 0 : for(auto& group : importGroupMap)
1102 0 : __COUT__ << " ==> Group imported: " << group.first.first << " ("
1103 0 : << group.first.second << ") ==> (" << group.second << ")" << std::endl;
1104 :
1105 : // return;
1106 :
1107 : // Done making groups, now...
1108 : // -- insert new aliases for imported groups/tables in current active backbone
1109 : // - should be basename+alias connection to (hop through maps) new groupName & groupKey
1110 : { //insert new aliases for imported groups in current active backbone
1111 : auto table =
1112 0 : cfgMgr->getTableByName(ConfigurationManager::GROUP_ALIASES_TABLE_NAME);
1113 0 : __COUTV__(table->getTableName());
1114 0 : __COUTV__(table->getViewVersion());
1115 :
1116 : } //end insert new aliases for imported groups in current active backbone
1117 : { //insert new aliases for imported tables in current active backbone
1118 : auto table =
1119 0 : cfgMgr->getTableByName(ConfigurationManager::VERSION_ALIASES_TABLE_NAME);
1120 0 : __COUTV__(table->getTableName());
1121 0 : __COUTV__(table->getViewVersion());
1122 :
1123 : } //end insert new aliases for imported tables in current active backbone
1124 :
1125 0 : std::map<std::string, TableVersion> backboneMemberMap;
1126 : {
1127 0 : std::string accumulatedWarnings, accumulateErrors;
1128 0 : cfgMgr->restoreActiveTableGroups(
1129 : false /*throwErrors*/,
1130 : "" /*pathToActiveGroupsFile*/,
1131 : ConfigurationManager::LoadGroupType::
1132 : ALL_TYPES, //must do ALL_TYPES to not affect activeTablesGroup file
1133 : &accumulatedWarnings);
1134 :
1135 : activeBackboneGroupName =
1136 0 : cfgMgr->getActiveGroupName(ConfigurationManager::GroupType::BACKBONE_TYPE);
1137 0 : cfgMgr->loadTableGroup(
1138 : activeBackboneGroupName,
1139 0 : cfgMgr->getActiveGroupKey(ConfigurationManager::GroupType::BACKBONE_TYPE),
1140 : true,
1141 : &backboneMemberMap,
1142 : 0,
1143 : &accumulateErrors);
1144 0 : __COUT__ << "Done re-loading active backbone: " << activeBackboneGroupName << " ("
1145 0 : << cfgMgr->getActiveGroupKey(
1146 0 : ConfigurationManager::GroupType::BACKBONE_TYPE)
1147 0 : << ")" << std::endl;
1148 0 : }
1149 :
1150 0 : __COUT__ << "Modifying the active Backbone table to reflect new table versions and "
1151 0 : "group keys."
1152 0 : << std::endl;
1153 :
1154 : // Done making groups, now...
1155 : // -- insert new aliases for imported groups/tables in current active backbone
1156 : // - should be basename+alias connection to (hop through maps) new groupName & groupKey
1157 : { //insert new aliases for imported groups in current active backbone
1158 : auto table =
1159 0 : cfgMgr->getTableByName(ConfigurationManager::GROUP_ALIASES_TABLE_NAME);
1160 0 : __COUTV__(table->getTableName());
1161 0 : __COUTV__(table->getViewVersion());
1162 :
1163 0 : auto cfgView = table->getViewP();
1164 :
1165 0 : unsigned int col0 = cfgView->findCol("GroupKeyAlias");
1166 0 : unsigned int col1 = cfgView->findCol("GroupName");
1167 0 : unsigned int col2 = cfgView->findCol("GroupKey");
1168 : unsigned int row;
1169 :
1170 0 : cfgView->print();
1171 :
1172 0 : TableVersion duplicateVersion;
1173 :
1174 : // -- insert new aliases for imported groups
1175 : // - should be basename+alias connection to (hop through maps) new
1176 : // groupName & groupKey
1177 0 : for(auto& aliasPair : importGroupAliasMap)
1178 : {
1179 0 : __COUT__ << "\t" << aliasPair.first << " ==> " << aliasPair.second.first
1180 0 : << "-v" << aliasPair.second.second << std::endl;
1181 :
1182 0 : auto groupIt = importGroupMap.find(std::pair<std::string, TableGroupKey>(
1183 0 : aliasPair.second.first, aliasPair.second.second));
1184 :
1185 0 : if(groupIt == importGroupMap.end())
1186 : {
1187 0 : __COUT__ << "Error! Could not find the new entry for the original group "
1188 0 : << aliasPair.second.first << "(" << aliasPair.second.second
1189 0 : << ")" << __E__;
1190 0 : continue;
1191 0 : }
1192 : row =
1193 0 : cfgView->addRow(prepend + "import_aliases", true /*incrementUniqueData*/);
1194 0 : cfgView->setValue(prepend + aliasPair.first, row, col0);
1195 0 : cfgView->setValue(aliasPair.second.first, row, col1);
1196 0 : cfgView->setValue(groupIt->second.toString(), row, col2);
1197 : } // end group alias edit
1198 :
1199 0 : if(!importGroupAliasMap.size())
1200 : duplicateVersion =
1201 0 : table->getViewVersion(); //mark duplicate as self if nothing to add
1202 :
1203 0 : cfgView->print();
1204 :
1205 : TableVersion originalVersion =
1206 : table
1207 0 : ->getViewVersion(); //save version because cache fill will change active version
1208 0 : if(duplicateVersion.isInvalid())
1209 : {
1210 0 : auto tableName = ConfigurationManager::GROUP_ALIASES_TABLE_NAME;
1211 0 : __COUT__ << "Checking for duplicate '" << tableName << "' tables..." << __E__;
1212 :
1213 : {
1214 : //"DEEP" checking
1215 : // load into cache 'recent' versions for this table
1216 : // 'recent' := those already in cache, plus highest version numbers not in cache
1217 : const std::map<std::string, TableInfo>& allTableInfo =
1218 0 : cfgMgr->getAllTableInfo(); // do not refresh
1219 :
1220 : auto versionReverseIterator =
1221 0 : allTableInfo.at(tableName)
1222 0 : .versions_.rbegin(); // get reverse iterator
1223 0 : __COUT__ << "Filling up '" << tableName << "' cache from "
1224 0 : << table->getNumberOfStoredViews() << " to max count of "
1225 0 : << table->MAX_VIEWS_IN_CACHE << __E__;
1226 0 : for(;
1227 0 : table->getNumberOfStoredViews() < table->MAX_VIEWS_IN_CACHE &&
1228 0 : versionReverseIterator != allTableInfo.at(tableName).versions_.rend();
1229 0 : ++versionReverseIterator)
1230 : {
1231 0 : __COUTT__ << "'" << tableName << "' versions in reverse order "
1232 0 : << *versionReverseIterator << __E__;
1233 : try
1234 : {
1235 0 : cfgMgr->getVersionedTableByName(
1236 : tableName,
1237 0 : *versionReverseIterator); // load to cache
1238 : }
1239 0 : catch(const std::runtime_error& e)
1240 : {
1241 : // ignore error
1242 0 : __COUTT__ << "'" << tableName << "' version failed to load: "
1243 0 : << *versionReverseIterator << __E__;
1244 0 : }
1245 : }
1246 : }
1247 :
1248 0 : __COUT__ << "Checking '" << tableName << "' for duplicate..." << __E__;
1249 : duplicateVersion =
1250 0 : table->checkForDuplicate(originalVersion,
1251 0 : TableVersion()); // then all versions in search
1252 :
1253 0 : } //end check duplicate
1254 :
1255 : //return the original version to active
1256 0 : table->setActiveView(originalVersion);
1257 :
1258 0 : if(!duplicateVersion.isInvalid())
1259 : {
1260 : // found an equivalent!
1261 0 : __COUT__ << "Equivalent " << ConfigurationManager::GROUP_ALIASES_TABLE_NAME
1262 0 : << " table found in version v" << duplicateVersion << __E__;
1263 0 : backboneMemberMap.at(ConfigurationManager::GROUP_ALIASES_TABLE_NAME) =
1264 0 : duplicateVersion;
1265 : }
1266 : else
1267 : {
1268 : auto newVersion =
1269 0 : TableVersion::getNextVersion(theInterface_->findLatestVersion(table));
1270 0 : __COUTV__(newVersion);
1271 0 : cfgView->setVersion(newVersion);
1272 :
1273 : // save table, and retry on save collision
1274 0 : uint16_t retries = 0;
1275 : while(1)
1276 : {
1277 : try
1278 : {
1279 0 : theInterface_->saveActiveVersion(table);
1280 : }
1281 0 : catch(const std::runtime_error& e)
1282 : {
1283 0 : __COUT__ << "Caught runtime_error exception during table save."
1284 0 : << __E__;
1285 0 : if(std::string(e.what()).find("there was a collision") !=
1286 : std::string::npos)
1287 : {
1288 0 : __COUT_WARN__ << "There was a collision saving the new table "
1289 0 : << *cfgView << "(" << newVersion
1290 0 : << "), trying incremented table version... retries="
1291 0 : << retries << __E__;
1292 0 : if(++retries > 3) //give up
1293 0 : throw;
1294 0 : newVersion = TableVersion::getNextVersion(
1295 0 : newVersion); //increment table version
1296 0 : cfgView->setVersion(newVersion);
1297 0 : __COUT__ << "New version for table: " << *cfgView << " found as "
1298 0 : << newVersion << __E__;
1299 0 : continue;
1300 0 : }
1301 : else
1302 0 : throw;
1303 0 : }
1304 :
1305 0 : __COUT__ << "Created table: " << *cfgView << "-v" << newVersion << __E__;
1306 0 : break;
1307 0 : } //end collission retry loop
1308 :
1309 0 : __COUT__ << "Updated backbone table "
1310 0 : << ConfigurationManager::GROUP_ALIASES_TABLE_NAME << " from v"
1311 0 : << backboneMemberMap.at(
1312 0 : ConfigurationManager::GROUP_ALIASES_TABLE_NAME)
1313 0 : << " to v" << newVersion << std::endl;
1314 0 : backboneMemberMap.at(ConfigurationManager::GROUP_ALIASES_TABLE_NAME) =
1315 0 : newVersion; // change version in the member map
1316 0 : }
1317 :
1318 0 : } //end insert new aliases for imported groups in current active backbone
1319 : { //insert new aliases for imported tables in current active backbone
1320 : auto table =
1321 0 : cfgMgr->getTableByName(ConfigurationManager::VERSION_ALIASES_TABLE_NAME);
1322 0 : __COUTV__(table->getTableName());
1323 0 : __COUTV__(table->getViewVersion());
1324 :
1325 0 : auto cfgView = table->getViewP();
1326 0 : unsigned int col0 = cfgView->findCol("VersionAlias");
1327 0 : unsigned int col1 = cfgView->findCol("TableName");
1328 0 : unsigned int col2 = cfgView->findCol("Version");
1329 0 : __COUTV__(table->getViewVersion());
1330 :
1331 : unsigned int row;
1332 :
1333 0 : cfgView->print();
1334 :
1335 0 : TableVersion duplicateVersion;
1336 :
1337 : // -- insert new aliases for imported tables
1338 : // - should be basename+alias connection to (hop through maps) new
1339 : // tableName & tableVersion
1340 0 : for(auto& aliasPair : importTableAliasMap)
1341 : {
1342 0 : __COUT__ << "\t" << aliasPair.first << " ==> " << aliasPair.second.first
1343 0 : << "-" << aliasPair.second.second << std::endl;
1344 :
1345 0 : auto tableIt = importTableMap.find(std::pair<std::string, TableVersion>(
1346 0 : aliasPair.second.first, aliasPair.second.second));
1347 :
1348 0 : if(tableIt == importTableMap.end())
1349 : {
1350 0 : __COUT__ << "Error! Could not find the new entry for the original table "
1351 0 : << aliasPair.second.first << "(" << aliasPair.second.second
1352 0 : << ")" << __E__;
1353 0 : continue;
1354 0 : }
1355 : row =
1356 0 : cfgView->addRow(prepend + "import_aliases", true /*incrementUniqueData*/);
1357 0 : cfgView->setValue(prepend + aliasPair.first, row, col0);
1358 0 : cfgView->setValue(aliasPair.second.first, row, col1);
1359 0 : cfgView->setValue(tableIt->second.toString(), row, col2);
1360 : } // end group alias edit
1361 0 : __COUTV__(table->getViewVersion());
1362 :
1363 0 : if(!importTableAliasMap.size())
1364 : duplicateVersion =
1365 0 : table->getViewVersion(); //mark duplicate as self if nothing to add
1366 :
1367 0 : __COUTV__(table->getViewVersion());
1368 0 : cfgView->print();
1369 0 : __COUTV__(table->getViewVersion());
1370 :
1371 : TableVersion originalVersion =
1372 : table
1373 0 : ->getViewVersion(); //save version because cache fill will change active version
1374 0 : if(duplicateVersion.isInvalid())
1375 : {
1376 0 : auto tableName = ConfigurationManager::VERSION_ALIASES_TABLE_NAME;
1377 0 : __COUT__ << "Checking for duplicate '" << tableName << "' tables..." << __E__;
1378 :
1379 : {
1380 : //"DEEP" checking
1381 : // load into cache 'recent' versions for this table
1382 : // 'recent' := those already in cache, plus highest version numbers not in cache
1383 : const std::map<std::string, TableInfo>& allTableInfo =
1384 0 : cfgMgr->getAllTableInfo(); // do not refresh
1385 0 : __COUTV__(table->getViewVersion());
1386 :
1387 : auto versionReverseIterator =
1388 0 : allTableInfo.at(tableName)
1389 0 : .versions_.rbegin(); // get reverse iterator
1390 0 : __COUT__ << "Filling up '" << tableName << "' cache from "
1391 0 : << table->getNumberOfStoredViews() << " to max count of "
1392 0 : << table->MAX_VIEWS_IN_CACHE << __E__;
1393 0 : for(;
1394 0 : table->getNumberOfStoredViews() < table->MAX_VIEWS_IN_CACHE &&
1395 0 : versionReverseIterator != allTableInfo.at(tableName).versions_.rend();
1396 0 : ++versionReverseIterator)
1397 : {
1398 0 : __COUTT__ << "'" << tableName << "' versions in reverse order "
1399 0 : << *versionReverseIterator << __E__;
1400 : try
1401 : {
1402 0 : cfgMgr->getVersionedTableByName(
1403 : tableName,
1404 0 : *versionReverseIterator); // load to cache
1405 : }
1406 0 : catch(const std::runtime_error& e)
1407 : {
1408 : // ignore error
1409 0 : __COUTT__ << "'" << tableName << "' version failed to load: "
1410 0 : << *versionReverseIterator << __E__;
1411 0 : }
1412 0 : __COUTV__(table->getViewVersion());
1413 : }
1414 : }
1415 :
1416 0 : __COUT__ << "Checking '" << tableName << "' for duplicate..." << __E__;
1417 : duplicateVersion =
1418 0 : table->checkForDuplicate(originalVersion,
1419 0 : TableVersion()); // then all versions in search
1420 :
1421 0 : __COUTV__(table->getViewVersion());
1422 0 : } //end check duplicate
1423 :
1424 : //return the original version to active
1425 0 : table->setActiveView(originalVersion);
1426 :
1427 0 : if(!duplicateVersion.isInvalid())
1428 : {
1429 : // found an equivalent!
1430 0 : __COUT__ << "Equivalent " << ConfigurationManager::VERSION_ALIASES_TABLE_NAME
1431 0 : << " table found in version v" << duplicateVersion << __E__;
1432 0 : backboneMemberMap.at(ConfigurationManager::VERSION_ALIASES_TABLE_NAME) =
1433 0 : duplicateVersion; // change version in the member map
1434 : }
1435 : else
1436 : {
1437 : auto newVersion =
1438 0 : TableVersion::getNextVersion(theInterface_->findLatestVersion(table));
1439 0 : cfgView->setVersion(newVersion);
1440 : // table->setActiveView(newVersion);
1441 :
1442 : // save table, and retry on save collision
1443 0 : uint16_t retries = 0;
1444 : while(1)
1445 : {
1446 : try
1447 : {
1448 0 : theInterface_->saveActiveVersion(table);
1449 : }
1450 0 : catch(const std::runtime_error& e)
1451 : {
1452 0 : __COUT__ << "Caught runtime_error exception during table save."
1453 0 : << __E__;
1454 0 : if(std::string(e.what()).find("there was a collision") !=
1455 : std::string::npos)
1456 : {
1457 0 : __COUT_WARN__ << "There was a collision saving the new table "
1458 0 : << *cfgView << "(" << newVersion
1459 0 : << "), trying incremented table version... retries="
1460 0 : << retries << __E__;
1461 0 : if(++retries > 3) //give up
1462 0 : throw;
1463 0 : newVersion = TableVersion::getNextVersion(
1464 0 : newVersion); //increment table version
1465 0 : cfgView->setVersion(newVersion);
1466 0 : __COUT__ << "New version for table: " << *cfgView << " found as "
1467 0 : << newVersion << __E__;
1468 0 : continue;
1469 0 : }
1470 : else
1471 0 : throw;
1472 0 : }
1473 :
1474 0 : __COUT__ << "Created table: " << *cfgView << "-v" << newVersion << __E__;
1475 0 : break;
1476 0 : } //end collission retry loop
1477 :
1478 0 : __COUT__ << "Updated backbone table "
1479 0 : << ConfigurationManager::VERSION_ALIASES_TABLE_NAME << " from v"
1480 0 : << backboneMemberMap.at(
1481 0 : ConfigurationManager::VERSION_ALIASES_TABLE_NAME)
1482 0 : << " to v" << newVersion << std::endl;
1483 0 : backboneMemberMap.at(ConfigurationManager::VERSION_ALIASES_TABLE_NAME) =
1484 0 : newVersion; // change version in the member map
1485 0 : }
1486 :
1487 0 : } //end insert new aliases for imported tables in current active backbone
1488 :
1489 0 : __COUT_INFO__ << "Done inserting new aliases in active Backbone." << __E__;
1490 :
1491 : // return;
1492 :
1493 : // -- save new backbone tables and save new backbone group, then activate
1494 : {
1495 0 : __COUT__ << "Backbone member map to create:" << std::endl;
1496 0 : for(auto& member : backboneMemberMap)
1497 0 : __COUT__ << " ==> Member to create: " << member.first << "-v" << member.second
1498 0 : << std::endl;
1499 :
1500 0 : bool foundExistingEmptyBackbone = false;
1501 : { //check if group is a duplicate
1502 0 : __COUT__ << "Checking for duplicate backbone groups..." << __E__;
1503 : try
1504 : {
1505 : TableGroupKey foundKey = cfgMgr->findTableGroup(
1506 : activeBackboneGroupName,
1507 0 : backboneMemberMap); //, memberTableAliases); std::map<std::string /*name*/, std::string /*alias*/> memberTableAliases
1508 :
1509 0 : if(!foundKey.isInvalid())
1510 : {
1511 0 : __COUT_WARN__ << "Found equivalent empty backbone group key ("
1512 0 : << foundKey << ") for " << activeBackboneGroupName
1513 0 : << ". Skipping creation and activating existing key!"
1514 0 : << __E__;
1515 0 : foundExistingEmptyBackbone = true;
1516 :
1517 : // -- backup the file ConfigurationManager::ACTIVE_GROUPS_FILENAME with time
1518 : std::string renameFile =
1519 0 : ConfigurationManager::ACTIVE_GROUPS_FILENAME + "." + nowTime;
1520 0 : rename(ConfigurationManager::ACTIVE_GROUPS_FILENAME.c_str(),
1521 : renameFile.c_str());
1522 :
1523 0 : __COUT__ << "Backing up '"
1524 0 : << ConfigurationManager::ACTIVE_GROUPS_FILENAME
1525 0 : << "' to ... '" << renameFile << "'" << std::endl;
1526 :
1527 : // -- activate the existing empty backbone group
1528 0 : cfgMgr->activateTableGroup(
1529 : activeBackboneGroupName,
1530 : foundKey); // and write to active group file
1531 0 : }
1532 :
1533 0 : __COUT__ << "Check for existing backbone duplicate groups complete."
1534 0 : << __E__;
1535 0 : }
1536 0 : catch(...)
1537 : {
1538 0 : __COUT_WARN__ << "Ignoring errors looking for existing backbone "
1539 : "duplicate groups! Proceeding "
1540 0 : "with new group creation."
1541 0 : << __E__;
1542 0 : }
1543 : } //end check if group is a duplicate
1544 0 : __COUTV__(foundExistingEmptyBackbone);
1545 :
1546 0 : if(!foundExistingEmptyBackbone)
1547 : {
1548 : auto newKey = TableGroupKey::getNextKey(
1549 0 : theInterface_->findLatestGroupKey(activeBackboneGroupName));
1550 :
1551 0 : __COUT__ << "Updating backbone group from ("
1552 0 : << cfgMgr->getActiveGroupKey(
1553 0 : ConfigurationManager::GroupType::BACKBONE_TYPE)
1554 0 : << ") to (" << newKey << ")" << __E__;
1555 :
1556 : // memberMap should now consist of members with new flat version, so save
1557 :
1558 : // save group, and retry on save collision
1559 0 : uint16_t retries = 0;
1560 : while(1)
1561 : {
1562 : try
1563 : {
1564 0 : theInterface_->saveTableGroup(backboneMemberMap,
1565 0 : TableGroupKey::getFullGroupString(
1566 : activeBackboneGroupName, newKey));
1567 : }
1568 0 : catch(const std::runtime_error& e)
1569 : {
1570 0 : __COUT__ << "Caught runtime_error exception during group save."
1571 0 : << __E__;
1572 0 : if(std::string(e.what()).find("there was a collision") !=
1573 : std::string::npos)
1574 : {
1575 0 : __COUT_WARN__
1576 0 : << "There was a collision saving the new group "
1577 0 : << activeBackboneGroupName << "(" << newKey
1578 0 : << "), trying incremented group key... retries=" << retries
1579 0 : << __E__;
1580 0 : if(++retries > 3) //give up
1581 0 : throw;
1582 0 : newKey = TableGroupKey::getNextKey(newKey); //increment group key
1583 0 : __COUT__ << "New Key for group: " << activeBackboneGroupName
1584 0 : << " found as " << newKey << __E__;
1585 0 : continue;
1586 0 : }
1587 : else
1588 0 : throw;
1589 0 : }
1590 :
1591 0 : __COUT__ << "Created table group: " << activeBackboneGroupName << "("
1592 0 : << newKey << ")" << __E__;
1593 0 : break;
1594 0 : } //end collission retry loop
1595 :
1596 : // -- backup the file ConfigurationManager::ACTIVE_GROUPS_FILENAME with time
1597 : std::string renameFile =
1598 0 : ConfigurationManager::ACTIVE_GROUPS_FILENAME + "." + nowTime;
1599 0 : rename(ConfigurationManager::ACTIVE_GROUPS_FILENAME.c_str(),
1600 : renameFile.c_str());
1601 :
1602 0 : __COUT__ << "Backing up '" << ConfigurationManager::ACTIVE_GROUPS_FILENAME
1603 0 : << "' to ... '" << renameFile << "'" << std::endl;
1604 :
1605 : // -- activate the new backbone group
1606 0 : cfgMgr->activateTableGroup(activeBackboneGroupName,
1607 : newKey); // and write to active group file
1608 0 : }
1609 : }
1610 :
1611 0 : __COUT__ << "End of Importing Table Groups from path!\n\n\n" << std::endl;
1612 :
1613 0 : } //end ImportTableGroupsFromPath()
1614 :
1615 0 : int main(int argc, char* argv[])
1616 : {
1617 : try
1618 : {
1619 0 : ImportTableGroupsFromPath(argc, argv);
1620 : }
1621 0 : catch(...)
1622 : {
1623 0 : __COUT_ERR__ << "Unhandled exception caught in main()!" << __E__
1624 0 : << StringMacros::stackTrace() << std::endl;
1625 0 : throw;
1626 0 : }
1627 :
1628 0 : return 0;
1629 : }
1630 : // BOOST_AUTO_TEST_SUITE_END()
|
