Line data Source code
1 : #include "otsdaq/MessageFacility/MessageFacility.h"
2 :
3 : #include <dirent.h>
4 : #include <cassert>
5 : #include <iostream>
6 : #include <memory>
7 : #include <string>
8 :
9 : #include "otsdaq/ConfigurationInterface/ConfigurationInterface.h"
10 : #include "otsdaq/ConfigurationInterface/ConfigurationManagerRW.h"
11 : // #include "artdaq-database/StorageProviders/FileSystemDB/provider_filedb_index.h"
12 : // #include "artdaq-database/JsonDocument/JSONDocument.h"
13 :
14 : // Shared test utilities
15 : #include "otsdaq/Macros/TestUtilities.h"
16 :
17 : /// Exports all active groups and system alias groups (specified by the active backbone)
18 : /// to the path specified. Each group's member tables will be organized in a folder with metadata.
19 : /// Each table is exported as a json format text file.
20 : /// usage:
21 : /// otsdaq_export_system_aliases <path to export to>
22 : ///
23 : ///
24 : using namespace ots;
25 :
26 0 : void ExportActiveSystemAliasTableGroups(int argc, char* argv[])
27 : {
28 0 : std::cout << "=================================================\n";
29 0 : std::cout << "=================================================\n";
30 0 : std::cout << "=================================================\n";
31 0 : __COUT__ << "\nExporting Active System Aliases!" << std::endl;
32 :
33 0 : std::cout << "\n\nusage: One argument:\n\t <export path> \n\n" << std::endl;
34 :
35 0 : std::cout << "argc = " << argc << std::endl;
36 0 : for(int i = 0; i < argc; i++)
37 0 : std::cout << "argv[" << i << "] = " << argv[i] << std::endl;
38 :
39 0 : if(argc != 2)
40 : {
41 0 : std::cout << "Error! Must provide one parameter.\n\n" << std::endl;
42 0 : return;
43 : }
44 :
45 0 : std::string exportPath = argv[1];
46 0 : __COUTV__(exportPath);
47 :
48 : // make directory just in case
49 0 : mkdir(exportPath.c_str(), 0755);
50 :
51 : DIR* dp;
52 0 : if((dp = opendir(exportPath.c_str())) == 0)
53 : {
54 0 : __COUT_ERR__ << "ERROR:(" << errno
55 0 : << "). Can't open directory for export: " << exportPath
56 0 : << std::endl;
57 0 : exit(0);
58 : }
59 0 : closedir(dp);
60 :
61 : // return;
62 :
63 : //==============================================================================
64 : // Define environment variables
65 : // Note: normally these environment variables are set by ots script
66 :
67 0 : test::util::check_and_make_envs();
68 : ////////////////////////////////////////////////////
69 :
70 : //==============================================================================
71 : // get prepared with initial source db
72 :
73 : // ConfigurationManager instance immediately loads active groups
74 0 : __COUT__ << "Loading active Aliases..." << std::endl;
75 :
76 0 : std::string ARTDAQ_DATABASE_URI = __ENV__("ARTDAQ_DATABASE_URI");
77 0 : __COUTV__(ARTDAQ_DATABASE_URI);
78 :
79 : // return;
80 :
81 0 : ConfigurationManagerRW cfgMgrInst("export_admin");
82 0 : ConfigurationManagerRW* cfgMgr = &cfgMgrInst;
83 :
84 : if(0)
85 : {
86 : std::string accumulatedWarnings;
87 : const std::map<std::string, TableInfo>& allTableInfo =
88 : cfgMgr->getAllTableInfo(true /* refresh */,
89 : &accumulatedWarnings,
90 : "" /* errorFilterName */,
91 : true /* getGroupKeys*/,
92 : false /* getGroupInfo */,
93 : true /* initializeActiveGroups */);
94 : __COUTV__(allTableInfo.size());
95 : auto groups = cfgMgr->getAllGroupInfo();
96 : __COUTV__(groups.size());
97 : for(auto& group : groups)
98 : {
99 : __COUTV__(group.first);
100 : }
101 : }
102 :
103 : if(1)
104 : {
105 0 : std::string accumulatedWarnings;
106 0 : cfgMgr->restoreActiveTableGroups(false /*throwErrors*/,
107 : "" /*pathToActiveGroupsFile*/,
108 : ConfigurationManager::LoadGroupType::ALL_TYPES,
109 : &accumulatedWarnings);
110 :
111 0 : __COUT__ << "Done Loading active groups." << std::endl;
112 0 : }
113 : // return;
114 :
115 : // create set of groups to persist
116 : // include active context
117 : // include active backbone
118 : // include active iterate group
119 : // include active config group
120 : // (keep key translation separate activeGroupKeys)
121 : // include all groups with system aliases
122 :
123 : // for each group in set
124 : // load/activate group
125 : // export each table as json doc to a folder <group name>_<export time>/
126 :
127 : /* map<<groupName, origKey>, newKey> */
128 0 : std::map<std::pair<std::string, TableGroupKey>, TableGroupKey> groupSet;
129 : /* <tableName, <origVersion, newVersion> >*/
130 0 : std::map<std::pair<std::string, TableVersion>, TableVersion> modifiedTables;
131 0 : std::map<std::string, std::pair<TableGroupKey, TableGroupKey>> activeGroupKeys;
132 0 : std::map<std::pair<std::string, TableGroupKey>, std::string> groupErrors;
133 :
134 0 : std::string activeBackboneGroupName = "";
135 0 : std::string activeContextGroupName = "";
136 0 : std::string activeIterateGroupName = "";
137 0 : std::string activeConfigGroupName = "";
138 :
139 0 : std::string nowTime = std::to_string(time(0));
140 :
141 : // add active groups to set
142 : std::map<std::string, std::pair<std::string, TableGroupKey>> activeGroupsMap =
143 0 : cfgMgr->getActiveTableGroups();
144 :
145 0 : bool foundAnyActiveGroups = false;
146 :
147 0 : for(const auto& activeGroup : activeGroupsMap)
148 : {
149 0 : if(activeGroup.second.second.TableGroupKey::isInvalid())
150 0 : continue;
151 :
152 0 : groupSet.insert(std::pair<std::pair<std::string, TableGroupKey>, TableGroupKey>(
153 0 : std::pair<std::string, TableGroupKey>(activeGroup.second.first,
154 0 : activeGroup.second.second),
155 0 : TableGroupKey()));
156 0 : activeGroupKeys.insert(
157 0 : std::pair<std::string, std::pair<TableGroupKey, TableGroupKey>>(
158 0 : activeGroup.second.first,
159 0 : std::pair<TableGroupKey, TableGroupKey>(activeGroup.second.second,
160 0 : TableGroupKey())));
161 :
162 0 : if(activeGroup.first == ConfigurationManager::GROUP_TYPE_NAME_BACKBONE)
163 : {
164 0 : activeBackboneGroupName = activeGroup.second.first;
165 0 : __COUT__ << "found activeBackboneGroupName = " << activeBackboneGroupName
166 0 : << std::endl;
167 0 : foundAnyActiveGroups = true;
168 : }
169 0 : else if(activeGroup.first == ConfigurationManager::GROUP_TYPE_NAME_CONTEXT)
170 : {
171 0 : activeContextGroupName = activeGroup.second.first;
172 0 : __COUT__ << "found activeContextGroupName = " << activeContextGroupName
173 0 : << std::endl;
174 0 : foundAnyActiveGroups = true;
175 : }
176 0 : else if(activeGroup.first == ConfigurationManager::GROUP_TYPE_NAME_ITERATE)
177 : {
178 0 : activeIterateGroupName = activeGroup.second.first;
179 0 : __COUT__ << "found activeIterateGroupName = " << activeIterateGroupName
180 0 : << std::endl;
181 0 : foundAnyActiveGroups = true;
182 : }
183 0 : else if(activeGroup.first == ConfigurationManager::GROUP_TYPE_NAME_CONFIGURATION)
184 : {
185 0 : activeConfigGroupName = activeGroup.second.first;
186 0 : __COUT__ << "found activeConfigGroupName = " << activeConfigGroupName
187 0 : << std::endl;
188 0 : foundAnyActiveGroups = true;
189 : }
190 : }
191 :
192 0 : if(activeBackboneGroupName == "" || !foundAnyActiveGroups)
193 : {
194 0 : __SS__ << "Did not find valid active groups to export! "
195 : "Must have a backbone at least. Is the current database URI correct? "
196 : "ARTDAQ_DATABASE_URI = "
197 0 : << ARTDAQ_DATABASE_URI << std::endl;
198 0 : __SS_THROW__;
199 0 : }
200 :
201 0 : __COUT__ << "Identified active groups:" << std::endl;
202 0 : for(auto& group : groupSet)
203 0 : __COUT__ << " ==> Group to export: " << group.first.first << " ("
204 0 : << group.first.second << ")" << std::endl;
205 : // return;
206 :
207 : // add system alias groups to set
208 : const std::string groupAliasesTableName =
209 0 : ConfigurationManager::GROUP_ALIASES_TABLE_NAME;
210 0 : std::map<std::string, TableVersion> activeVersions = cfgMgr->getActiveVersions();
211 0 : if(activeVersions.find(groupAliasesTableName) == activeVersions.end())
212 : {
213 0 : __SS__ << "\nActive version of " << groupAliasesTableName << " missing! "
214 : << groupAliasesTableName
215 : << " is a required member of the Backbone configuration group."
216 0 : << "\n\nLikely you need to activate a valid Backbone group." << std::endl;
217 0 : __SS_THROW__;
218 0 : }
219 :
220 : std::vector<std::pair<std::string, ConfigurationTree>> aliasNodePairs =
221 0 : cfgMgr->getNode(groupAliasesTableName).getChildren();
222 0 : for(auto& groupPair : aliasNodePairs)
223 0 : groupSet.insert(std::pair<std::pair<std::string, TableGroupKey>, TableGroupKey>(
224 0 : std::pair<std::string, TableGroupKey>(
225 0 : groupPair.second.getNode("GroupName").getValueAsString(),
226 0 : TableGroupKey(groupPair.second.getNode("GroupKey").getValueAsString())),
227 0 : TableGroupKey()));
228 :
229 : //at this point, all groups to export have been identified! ------------
230 0 : __COUT__ << "All identified groups:" << std::endl;
231 0 : for(auto& group : groupSet)
232 0 : __COUT__ << " ==> Group to export: " << group.first.first << " ("
233 0 : << group.first.second << ")" << std::endl;
234 0 : __COUT__ << std::endl;
235 0 : __COUT__ << std::endl;
236 :
237 0 : if(!groupSet.size())
238 : {
239 0 : __SS__ << "No groups identified to export!" << __E__;
240 0 : __SS_THROW__;
241 0 : }
242 :
243 0 : ConfigurationInterface* theInterface_ = ConfigurationInterface::getInstance(
244 : ConfigurationInterface::CONFIGURATION_MODE::ARTDAQ_DATABASE);
245 :
246 : //now, for each group in set
247 : // load/activate group
248 : // export each table as json doc to a folder <group name>_<export time>/
249 :
250 0 : bool errDetected = false;
251 0 : for(auto& group : groupSet)
252 : {
253 0 : __COUT__ << " ==> Group to export: " << group.first.first << " ("
254 0 : << group.first.second << ")" << std::endl;
255 :
256 : std::string groupPath =
257 0 : exportPath + "/" + group.first.first + "_" + group.first.second.str();
258 0 : __COUTV__(groupPath);
259 : //check if output directory already exists, and do not overwrite throw error
260 : {
261 : DIR* dp;
262 0 : if((dp = opendir(groupPath.c_str())) != 0)
263 : {
264 0 : closedir(dp);
265 0 : __COUT__ << "ERROR: Can't export to directory (already exists! Please "
266 0 : "choose a different export path.): "
267 0 : << groupPath << std::endl;
268 0 : exit(0);
269 : }
270 : }
271 0 : mkdir(groupPath.c_str(), 0755);
272 :
273 0 : if(group.first.first ==
274 : activeBackboneGroupName) //create special metadata file to flag the active Backbone group
275 : {
276 0 : std::string groupIsBackbonePath = groupPath + "/" + "groupIsBackbone.txt";
277 0 : __COUTV__(groupIsBackbonePath);
278 0 : FILE* fp = std::fopen(groupIsBackbonePath.c_str(), "w");
279 0 : if(!fp)
280 : {
281 0 : __COUT_ERR__ << "\n\nERROR! Could not open file at "
282 0 : << groupIsBackbonePath << ". Error: " << errno << " - "
283 0 : << strerror(errno) << __E__;
284 0 : return;
285 : }
286 0 : fputs("1", fp);
287 0 : fclose(fp);
288 0 : }
289 :
290 0 : std::string accumulateErrors = "";
291 0 : std::map<std::string /*name*/, TableVersion> memberMap;
292 0 : std::map<std::string /*name*/, std::string /*alias*/> groupAliases;
293 0 : std::string groupComment;
294 0 : std::string groupAuthor;
295 0 : std::string groupCreateTime;
296 0 : std::string groupTypeString;
297 :
298 : //=========================
299 : // load group, group metadata, and tables from original DB
300 : try
301 : {
302 0 : accumulateErrors = "";
303 0 : cfgMgr->loadTableGroup(group.first.first,
304 0 : group.first.second,
305 : true /*doActivate*/,
306 : 0, //&memberMap /*memberMap*/,
307 : 0 /*progressBar*/,
308 : &accumulateErrors,
309 : &groupComment,
310 : &groupAuthor,
311 : &groupCreateTime,
312 : false /*doNotLoadMember*/,
313 : &groupTypeString /*groupTypeString*/,
314 : &groupAliases);
315 : }
316 0 : catch(std::runtime_error& e)
317 : {
318 0 : __COUT__ << "Error was caught loading members for " << group.first.first
319 0 : << "(" << group.first.second << ")" << std::endl;
320 0 : __COUT__ << e.what() << std::endl;
321 0 : errDetected = true;
322 0 : }
323 0 : catch(...)
324 : {
325 0 : __COUT__ << "Error was caught loading members for " << group.first.first
326 0 : << "(" << group.first.second << ")" << std::endl;
327 0 : errDetected = true;
328 0 : }
329 :
330 : //Record if group is a Backbone group, because Aliases need to be handled specially on import
331 0 : if(groupTypeString == ConfigurationManager::GROUP_TYPE_NAME_BACKBONE)
332 : {
333 0 : __COUT__ << "\t\tFound backbone type group!" << __E__;
334 : }
335 :
336 : //get full member map with Metadata table
337 : {
338 : // std::map<std::string /*name*/, TableVersion /*version*/> memberMap =
339 0 : memberMap = theInterface_->getTableGroupMembers(
340 0 : TableGroupKey::getFullGroupString(group.first.first, group.first.second),
341 0 : true /*include meta data table*/);
342 :
343 : // save meta data table separately, since there is no table definition, and then remove from member map
344 0 : auto metaTablePair = memberMap.find(TableBase::GROUP_METADATA_TABLE_NAME);
345 0 : if(metaTablePair != memberMap.end())
346 : {
347 0 : __COUT__ << TableBase::GROUP_METADATA_TABLE_NAME << ":v"
348 0 : << metaTablePair->second << std::endl;
349 :
350 0 : std::string tablePath = groupPath + "/" +
351 0 : TableBase::GROUP_METADATA_TABLE_NAME + "_v" +
352 0 : metaTablePair->second.str() + ".json";
353 0 : __COUTV__(tablePath);
354 :
355 0 : auto groupMetadataTable = cfgMgr->getMetadataTable(metaTablePair->second);
356 :
357 0 : __COUTV__(tablePath);
358 0 : std::stringstream json;
359 0 : groupMetadataTable->getViewP()->printJSON(json);
360 :
361 0 : __COUTV__(tablePath);
362 0 : FILE* fp = std::fopen(tablePath.c_str(), "w");
363 0 : if(!fp)
364 : {
365 0 : __COUT_ERR__ << "\n\nERROR! Could not open file at " << tablePath
366 0 : << ". Error: " << errno << " - " << strerror(errno)
367 0 : << __E__;
368 0 : return;
369 : }
370 0 : fputs(json.str().c_str(), fp);
371 0 : fclose(fp);
372 :
373 0 : memberMap.erase(
374 : metaTablePair); // remove from member map that is returned
375 :
376 0 : } // end metadata handling
377 : else
378 : {
379 0 : __COUT_ERR__ << "Ignoring that groupMetadataTable_ is missing for group '"
380 0 : << group.first.first << "(" << group.first.second
381 0 : << "). Going with anonymous defaults." << __E__;
382 : }
383 : }
384 :
385 : // TableView* cfgView;
386 : // TableBase* config;
387 : //=========================
388 : // export the group tables to json documents!
389 : try
390 : {
391 : // saving tables
392 0 : for(auto& memberPair : memberMap)
393 : {
394 0 : __COUT__ << memberPair.first << ":v" << memberPair.second << std::endl;
395 :
396 0 : std::string tablePath = groupPath + "/" + memberPair.first + "_v" +
397 0 : memberPair.second.str() + ".json";
398 0 : __COUTV__(tablePath);
399 :
400 0 : std::stringstream json;
401 0 : cfgMgr->getTableByName(memberPair.first)->getViewP()->printJSON(json);
402 :
403 0 : FILE* fp = std::fopen(tablePath.c_str(), "w");
404 0 : if(!fp)
405 : {
406 0 : __COUT_ERR__ << "\n\nERROR! Could not open file at " << tablePath
407 0 : << ". Error: " << errno << " - " << strerror(errno)
408 0 : << __E__;
409 0 : return;
410 : }
411 0 : fputs(json.str().c_str(), fp);
412 0 : fclose(fp);
413 :
414 : // change the version of the active view to flatVersion and save it
415 : // config = cfgMgr->getTableByName(memberPair.first);
416 : // cfgView = config->getViewP();
417 : // cfgView->setVersion(TableVersion(flatVersion));
418 : // theInterface_->saveActiveVersion(config);
419 :
420 0 : } //end member table loop
421 :
422 : // Note: this code copies actions in ConfigurationManagerRW::saveNewTableGroup
423 :
424 : // add meta data
425 0 : __COUTV__(StringMacros::mapToString(groupAliases));
426 0 : __COUTV__(groupComment);
427 0 : __COUTV__(groupAuthor);
428 0 : __COUTV__(groupCreateTime);
429 : // time_t groupCreateTime_t;
430 : // sscanf(groupCreateTime.c_str(), "%ld", &groupCreateTime_t);
431 : // __COUTV__(groupCreateTime_t);
432 :
433 : // theInterface_->saveActiveVersion(groupMetadataTable);
434 :
435 : // memberMap should now consist of members with new flat version, so save
436 : // group
437 : // theInterface_->saveTableGroup(
438 : // memberMap,
439 : // TableGroupKey::getFullGroupString(groupPair.first.first,
440 : // TableGroupKey(flatVersion)));
441 : }
442 0 : catch(std::runtime_error& e)
443 : {
444 0 : __COUT__ << "Error was caught exporting group " << group.first.first << " ("
445 0 : << group.first.second << ") " << std::endl;
446 0 : __COUT__ << e.what() << std::endl;
447 :
448 0 : groupErrors.insert(
449 0 : std::pair<std::pair<std::string, TableGroupKey>, std::string>(
450 0 : std::pair<std::string, TableGroupKey>(group.first.first,
451 0 : group.first.second),
452 : "Error caught exporting the group."));
453 0 : }
454 0 : catch(...)
455 : {
456 0 : __COUT__ << "Error was caught saving group " << group.first.first << " ("
457 0 : << group.first.second << ") " << std::endl;
458 :
459 0 : groupErrors.insert(
460 0 : std::pair<std::pair<std::string, TableGroupKey>, std::string>(
461 0 : std::pair<std::string, TableGroupKey>(group.first.first,
462 0 : group.first.second),
463 : "Error caught exporting the group."));
464 0 : }
465 0 : } //end group export loop
466 :
467 0 : if(errDetected)
468 : {
469 0 : __COUT_ERR__ << "There was an error detected while exporting groups." << __E__;
470 : }
471 :
472 0 : __COUT_INFO__ << "Exported group summary: " << groupSet.size() << std::endl;
473 0 : for(auto& group : groupSet)
474 0 : __COUT_INFO__ << " ==> Group: " << group.first.first << " (" << group.first.second
475 0 : << ")" << std::endl;
476 :
477 0 : __COUT_INFO__ << "****************************" << std::endl;
478 0 : __COUT_INFO__ << "There were " << groupSet.size()
479 0 : << " groups considered, and there were " << groupErrors.size()
480 : << " errors found handling those groups. The groups were exported in "
481 0 : "text/json format to the export path: "
482 0 : << exportPath << std::endl;
483 0 : if(groupErrors.size())
484 : {
485 0 : __COUT_ERR__ << "There were " << groupErrors.size()
486 : << " errors found while loading and exporting groups. The "
487 0 : "following errors were found handling the groups:"
488 0 : << std::endl;
489 0 : for(auto& groupErr : groupErrors)
490 0 : __COUT_ERR__ << "\t" << groupErr.first.first << " " << groupErr.first.second
491 0 : << ": \t" << groupErr.second << std::endl;
492 0 : __COUT_ERR__ << "End of errors.\n\n" << std::endl;
493 : }
494 : else
495 0 : __COUT_INFO__ << "There were NO ERRORS found while loading and exporting groups."
496 0 : << __E__;
497 :
498 0 : } //end ExportActiveSystemAliasTableGroups()
499 :
500 0 : int main(int argc, char* argv[])
501 : {
502 0 : ExportActiveSystemAliasTableGroups(argc, argv);
503 0 : return 0;
504 : }
505 : // BOOST_AUTO_TEST_SUITE_END()
|
