Line data Source code
1 :
2 :
3 : #include "dtcInterfaceLib/otsStyleCoutMacros.h"
4 :
5 : #include "Mu2eUtil.h"
6 :
7 0 : void DTCLib::Mu2eUtil::parse_arguments(int argc, char** argv)
8 : {
9 0 : for (auto optind = 1; optind < argc; ++optind)
10 : {
11 0 : if (argv[optind][0] == '-')
12 : {
13 0 : switch (argv[optind][1])
14 : {
15 0 : case 'i':
16 0 : incrementTimestamp = false;
17 0 : break;
18 0 : case 'd':
19 0 : delay = DTCLib::Utilities::getOptionValue(&optind, &argv);
20 0 : break;
21 0 : case 'D':
22 0 : cfodelay = DTCLib::Utilities::getOptionValue(&optind, &argv);
23 0 : break;
24 0 : case 'S':
25 0 : syncRequests = true;
26 0 : break;
27 0 : case 'n':
28 0 : number = DTCLib::Utilities::getOptionValue(&optind, &argv);
29 0 : break;
30 0 : case 'o':
31 0 : timestampOffset = DTCLib::Utilities::getOptionValueLong(&optind, &argv) & 0x0000FFFFFFFFFFFF; // Timestamps are 48 bits
32 0 : break;
33 0 : case 'c':
34 0 : packetCount = DTCLib::Utilities::getOptionValue(&optind, &argv);
35 0 : break;
36 0 : case 'N':
37 0 : forceNoDebug = true;
38 0 : break;
39 0 : case 'b':
40 0 : blockCount = DTCLib::Utilities::getOptionValue(&optind, &argv);
41 0 : break;
42 0 : case 'E':
43 0 : eventCount = DTCLib::Utilities::getOptionValue(&optind, &argv);
44 0 : break;
45 0 : case 'a':
46 0 : requestsAhead = DTCLib::Utilities::getOptionValue(&optind, &argv);
47 0 : break;
48 0 : case 'q':
49 0 : quiet = true;
50 0 : quietCount = DTCLib::Utilities::getOptionValue(&optind, &argv);
51 0 : break;
52 0 : case 'p':
53 0 : useSimFile = true;
54 0 : break;
55 0 : case 'P':
56 0 : useSimFile = true;
57 0 : simFile = DTCLib::Utilities::getOptionValue(&optind, &argv);
58 0 : break;
59 0 : case 'G':
60 0 : readGenerated = true;
61 0 : break;
62 0 : case 'Q':
63 0 : quiet = true;
64 0 : reallyQuiet = true;
65 0 : break;
66 0 : case 's':
67 0 : checkSERDES = true;
68 0 : break;
69 0 : case 'e':
70 0 : useCFOEmulator = false;
71 0 : break;
72 0 : case 'f':
73 0 : rawOutput = true;
74 0 : rawOutputFile = DTCLib::Utilities::getOptionString(&optind, &argv);
75 0 : break;
76 0 : case 'H':
77 0 : writeDMAHeadersToOutput = true;
78 0 : break;
79 0 : case 't':
80 0 : debugType = DTC_DebugType_ExternalSerialWithReset;
81 0 : stickyDebugType = false;
82 0 : break;
83 0 : case 'T':
84 0 : val = DTCLib::Utilities::getOptionValue(&optind, &argv);
85 0 : if (val < static_cast<int>(DTC_DebugType_Invalid))
86 : {
87 0 : stickyDebugType = true;
88 0 : debugType = static_cast<DTC_DebugType>(val);
89 0 : break;
90 : }
91 0 : TLOG(TLVL_ERROR) << "Invalid Debug Type passed to -T!" << std::endl;
92 0 : printHelpMsg();
93 0 : break;
94 0 : case 'r':
95 0 : rocMask = DTCLib::Utilities::getOptionValue(&optind, &argv);
96 0 : break;
97 0 : case 'C':
98 0 : clockToProgram = DTCLib::Utilities::getOptionValue(&optind, &argv) % 3;
99 0 : break;
100 0 : case 'F':
101 0 : targetFrequency = DTCLib::Utilities::getOptionValue(&optind, &argv);
102 0 : break;
103 0 : case 'v':
104 0 : expectedDesignVersion = DTCLib::Utilities::getOptionString(&optind, &argv);
105 0 : break;
106 0 : case 'V':
107 0 : skipVerify = true;
108 0 : break;
109 0 : case '-': // Long option
110 : {
111 0 : auto option = DTCLib::Utilities::getLongOptionOption(&optind, &argv);
112 0 : if (option == "--timestamp-list")
113 : {
114 0 : timestampFile = DTCLib::Utilities::getLongOptionString(&optind, &argv);
115 : }
116 0 : else if (option == "--dtc")
117 : {
118 0 : dtc = DTCLib::Utilities::getLongOptionValue(&optind, &argv);
119 : }
120 0 : else if (option == "--cfoDRP")
121 : {
122 0 : useCFODRP = true;
123 : }
124 0 : else if (option == "--heartbeats")
125 : {
126 0 : heartbeatsAfter = DTCLib::Utilities::getLongOptionValue(&optind, &argv);
127 : }
128 0 : else if (option == "--binary-file-mode")
129 : {
130 0 : binaryFileOutput = true;
131 0 : rawOutput = true;
132 0 : rawOutputFile = DTCLib::Utilities::getLongOptionString(&optind, &argv);
133 : }
134 0 : else if (option == "--stop-verify")
135 : {
136 0 : stopOnVerifyFailure = true;
137 : }
138 0 : else if (option == "--stop-on-timeout")
139 : {
140 0 : stopOnTimeout = true;
141 : }
142 0 : else if (option == "--extra-reads")
143 : {
144 0 : extraReads = DTCLib::Utilities::getLongOptionValue(&optind, &argv);
145 : }
146 0 : else if (option == "--help")
147 : {
148 0 : printHelpMsg();
149 : }
150 0 : break;
151 0 : }
152 0 : default:
153 0 : TLOG(TLVL_ERROR) << "Unknown option: " << argv[optind] << std::endl;
154 0 : printHelpMsg();
155 0 : break;
156 0 : case 'h':
157 0 : printHelpMsg();
158 0 : break;
159 : }
160 : }
161 : else
162 : {
163 0 : op = std::string(argv[optind]);
164 : }
165 : }
166 :
167 0 : TLOG(TLVL_DEBUG) << "Options are: " << std::boolalpha
168 0 : << "Operation: " << std::string(op) << ", DTC: " << dtc << ", Num: " << number << ", Delay: " << delay
169 0 : << ", CFO Delay: " << cfodelay << ", TS Offset: " << timestampOffset << ", PacketCount: " << packetCount
170 0 : << ", Force NO Debug Flag: " << forceNoDebug << ", DataBlock Count: " << blockCount;
171 0 : TLOG(TLVL_DEBUG) << std::boolalpha << ", Event Count: " << eventCount << ", Requests Ahead of Reads: " << requestsAhead
172 0 : << ", Synchronous Request Mode: " << syncRequests << ", Use DTC CFO Emulator: " << useCFOEmulator << (useCFODRP ? " (CFO Emulator DRPs)" : " (DTC DRPs)")
173 0 : << ", Increment TS: " << incrementTimestamp << ", Quiet Mode: " << quiet << " (" << quietCount << ")"
174 0 : << ", Really Quiet Mode: " << reallyQuiet << ", Check SERDES Error Status: " << checkSERDES;
175 0 : TLOG(TLVL_DEBUG) << std::boolalpha << ", Read Data from DDR: " << readGenerated
176 0 : << ", Use Sim File: " << useSimFile << ", Skip Verify: " << skipVerify << ", ROC Mask: " << std::hex
177 0 : << rocMask << ", Debug Type: " << DTC_DebugTypeConverter(debugType).toString()
178 0 : << ", Target Frequency: " << std::dec << targetFrequency;
179 0 : TLOG(TLVL_DEBUG) << std::boolalpha << ", Clock To Program: " << (clockToProgram == 0 ? "SERDES" : (clockToProgram == 1 ? "DDR" : "Timing"))
180 0 : << ", Expected Design Version: " << expectedDesignVersion << ", Heartbeats: " << heartbeatsAfter;
181 0 : if (rawOutput)
182 : {
183 0 : TLOG(TLVL_DEBUG) << ", Raw output file: " << rawOutputFile;
184 : }
185 0 : if (simFile.size() > 0)
186 : {
187 0 : TLOG(TLVL_DEBUG) << ", Sim file: " << simFile;
188 : }
189 0 : }
190 :
191 0 : void DTCLib::Mu2eUtil::read_data()
192 : {
193 0 : TLOG(TLVL_DEBUG) << "Operation \"read_data\"" << std::endl;
194 0 : auto thisDTC = new DTC(DTC_SimMode_NoCFO, dtc, rocMask, expectedDesignVersion);
195 :
196 0 : auto device = thisDTC->GetDevice();
197 0 : if (readGenerated)
198 : {
199 0 : thisDTC->EnableDetectorEmulatorMode();
200 0 : thisDTC->SetDetectorEmulationDMACount(number);
201 0 : thisDTC->EnableDetectorEmulator();
202 : }
203 0 : for (unsigned ii = 0; ii < number; ++ii)
204 : {
205 0 : TLOG((reallyQuiet ? TLVL_DEBUG + 4 : TLVL_INFO)) << "Buffer Read " << ii << std::endl;
206 : mu2e_databuff_t* buffer;
207 0 : auto tmo_ms = 1500;
208 0 : auto sts = device->read_data(DTC_DMA_Engine_DAQ, reinterpret_cast<void**>(&buffer), tmo_ms);
209 :
210 0 : TLOG(TLVL_TRACE) << "util - read for DAQ - ii=" << ii << ", sts=" << sts << ", buffer=" << (void*)buffer;
211 0 : if (sts > 0)
212 : {
213 0 : auto bufSize = static_cast<uint16_t>(*reinterpret_cast<uint64_t*>(&buffer[0]));
214 0 : TLOG(TLVL_TRACE) << "util - bufSize is " << bufSize;
215 :
216 0 : if (!reallyQuiet)
217 : {
218 0 : DTCLib::Utilities::PrintBuffer(buffer, bufSize, quietCount);
219 : }
220 : }
221 0 : device->read_release(DTC_DMA_Engine_DAQ, 1);
222 0 : if (delay > 0) usleep(delay);
223 : }
224 0 : delete thisDTC;
225 0 : }
226 :
227 0 : void DTCLib::Mu2eUtil::toggle_serdes()
228 : {
229 0 : TLOG(TLVL_DEBUG) << "Swapping SERDES Oscillator Clock" << std::endl;
230 0 : auto thisDTC = new DTC(DTC_SimMode_NoCFO, dtc, rocMask, expectedDesignVersion);
231 0 : auto clock = thisDTC->ReadSERDESOscillatorClock();
232 0 : if (clock == DTC_SerdesClockSpeed_3125Gbps)
233 : {
234 0 : TLOG(TLVL_INFO) << "Setting SERDES Oscillator Clock to 2.5 Gbps" << std::endl;
235 0 : thisDTC->SetSERDESOscillatorClock(DTC_SerdesClockSpeed_25Gbps);
236 : }
237 0 : else if (clock == DTC_SerdesClockSpeed_25Gbps)
238 : {
239 0 : TLOG(TLVL_INFO) << "Setting SERDES Oscillator Clock to 3.125 Gbps" << std::endl;
240 0 : thisDTC->SetSERDESOscillatorClock(DTC_SerdesClockSpeed_3125Gbps);
241 : }
242 : else
243 : {
244 0 : TLOG(TLVL_ERROR) << "Error: SERDES clock not recognized value!";
245 : }
246 0 : delete thisDTC;
247 0 : }
248 :
249 0 : void DTCLib::Mu2eUtil::reset_ddr()
250 : {
251 0 : TLOG(TLVL_DEBUG) << "Resetting DDR Address" << std::endl;
252 0 : auto thisDTC = new DTC(DTC_SimMode_NoCFO, dtc, rocMask, expectedDesignVersion);
253 :
254 0 : thisDTC->ResetDDRInterface();
255 0 : thisDTC->ResetDDRReadAddress();
256 0 : thisDTC->ResetDDRWriteAddress();
257 0 : thisDTC->ResetDDR();
258 0 : delete thisDTC;
259 0 : }
260 :
261 0 : void DTCLib::Mu2eUtil::reset_detector_emulator()
262 : {
263 0 : TLOG(TLVL_DEBUG) << "Resetting Detector Emulator" << std::endl;
264 0 : auto thisDTC = new DTC(DTC_SimMode_NoCFO, dtc, rocMask, expectedDesignVersion);
265 0 : thisDTC->ClearDetectorEmulatorInUse();
266 0 : thisDTC->ResetDDR();
267 0 : thisDTC->SoftReset();
268 0 : delete thisDTC;
269 0 : }
270 :
271 0 : void DTCLib::Mu2eUtil::verify_sim_file()
272 : {
273 0 : TLOG(TLVL_DEBUG) << "Operation \"verify_simfile\"" << std::endl;
274 0 : auto thisDTC = new DTC(DTC_SimMode_NoCFO, dtc, rocMask, expectedDesignVersion);
275 0 : auto device = thisDTC->GetDevice();
276 :
277 0 : device->ResetDeviceTime();
278 :
279 0 : if (useSimFile)
280 : {
281 0 : thisDTC->DisableDetectorEmulator();
282 0 : thisDTC->EnableDetectorEmulatorMode();
283 0 : thisDTC->VerifySimFileInDTC(simFile, rawOutputFile);
284 : }
285 0 : }
286 :
287 0 : void DTCLib::Mu2eUtil::verify_stream()
288 : {
289 0 : TLOG(TLVL_DEBUG) << "Operation \"verify_stream\"" << std::endl;
290 0 : auto startTime = std::chrono::steady_clock::now();
291 0 : auto thisDTC = new DTC(DTC_SimMode_NoCFO, dtc, rocMask, expectedDesignVersion);
292 0 : auto device = thisDTC->GetDevice();
293 :
294 0 : auto initTime = device->GetDeviceTime();
295 0 : device->ResetDeviceTime();
296 0 : auto afterInit = std::chrono::steady_clock::now();
297 :
298 0 : DTCSoftwareCFO cfo(thisDTC, useCFOEmulator, packetCount, debugType, stickyDebugType, quiet, false, forceNoDebug, useCFODRP);
299 :
300 0 : if (useSimFile)
301 : {
302 0 : auto overwrite = false;
303 0 : if (simFile.size() > 0) overwrite = true;
304 0 : thisDTC->WriteSimFileToDTC(simFile, false, overwrite, rawOutputFile, skipVerify);
305 0 : if (readGenerated)
306 : {
307 0 : exit(0);
308 : }
309 : }
310 0 : else if (readGenerated)
311 : {
312 0 : thisDTC->DisableDetectorEmulator();
313 0 : thisDTC->EnableDetectorEmulatorMode();
314 0 : thisDTC->SetDetectorEmulationDMACount(number);
315 0 : thisDTC->EnableDetectorEmulator();
316 : }
317 :
318 0 : if (thisDTC->GetSimMode() != DTC_SimMode_Loopback && timestampFile != "")
319 : {
320 0 : syncRequests = false;
321 0 : std::set<DTC_EventWindowTag> timestamps;
322 0 : std::ifstream is(timestampFile);
323 : uint64_t a;
324 0 : while (is >> a)
325 : {
326 0 : timestamps.insert(DTC_EventWindowTag(a));
327 : }
328 0 : number = timestamps.size();
329 0 : cfo.SendRequestsForList(timestamps, cfodelay, heartbeatsAfter);
330 0 : }
331 0 : else if (thisDTC->GetSimMode() != DTC_SimMode_Loopback && !syncRequests)
332 : {
333 0 : cfo.SendRequestsForRange(number, DTC_EventWindowTag(timestampOffset), incrementTimestamp, cfodelay, requestsAhead, heartbeatsAfter);
334 : }
335 0 : else if (thisDTC->GetSimMode() == DTC_SimMode_Loopback)
336 : {
337 0 : uint64_t ts = timestampOffset;
338 0 : DTC_DataHeaderPacket header(DTC_Link_0, static_cast<uint16_t>(0), DTC_DataStatus_Valid, 0, DTC_Subsystem_Other, 0, DTC_EventWindowTag(ts));
339 0 : TLOG(TLVL_INFO) << "Request: " << header.toJSON() << std::endl;
340 0 : thisDTC->WriteDMAPacket(header);
341 0 : }
342 :
343 0 : auto readoutRequestTime = device->GetDeviceTime();
344 0 : device->ResetDeviceTime();
345 0 : auto afterRequests = std::chrono::steady_clock::now();
346 :
347 0 : DTC_Data_Verifier verifier;
348 :
349 0 : for (unsigned ii = 0; ii < number + extraReads; ++ii)
350 : {
351 0 : if (syncRequests && ii < number)
352 : {
353 0 : auto startRequest = std::chrono::steady_clock::now();
354 0 : cfo.SendRequestForTimestamp(DTC_EventWindowTag(timestampOffset + (incrementTimestamp ? ii : 0), heartbeatsAfter));
355 0 : auto endRequest = std::chrono::steady_clock::now();
356 0 : readoutRequestTime +=
357 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(endRequest - startRequest).count();
358 : }
359 0 : TLOG((reallyQuiet ? TLVL_DEBUG + 4 : TLVL_INFO)) << "Buffer Read " << std::dec << ii << std::endl;
360 :
361 0 : bool readSuccess = false;
362 0 : bool timeout = false;
363 0 : bool verified = false;
364 0 : size_t sts = 0;
365 0 : unsigned buffers_read = 1;
366 0 : mu2e_databuff_t* buffer = readDTCBuffer(device, readSuccess, timeout, sts, false);
367 :
368 0 : if (!readSuccess && checkSERDES)
369 : break;
370 0 : else if (!readSuccess)
371 0 : continue;
372 :
373 0 : void* readPtr = &buffer[0];
374 0 : uint16_t bufSize = static_cast<uint16_t>(*static_cast<uint64_t*>(readPtr));
375 0 : readPtr = static_cast<uint8_t*>(readPtr) + 8;
376 :
377 0 : if (timeout)
378 : {
379 0 : if (stopOnTimeout)
380 : {
381 0 : TLOG(TLVL_ERROR) << "Timeout detected and stop-on-timeout mode enabled. Stopping after " << ii << " events!";
382 0 : break;
383 : }
384 :
385 0 : TLOG(TLVL_WARNING) << "Timeout detected, moving to next read";
386 0 : continue;
387 0 : }
388 :
389 0 : DTC_Event evt(readPtr);
390 0 : size_t eventByteCount = evt.GetEventByteCount();
391 0 : size_t subEventCount = 0;
392 0 : if (eventByteCount > bufSize - 8U)
393 : {
394 0 : DTC_Event newEvt(eventByteCount);
395 0 : memcpy(const_cast<void*>(newEvt.GetRawBufferPointer()), evt.GetRawBufferPointer(), bufSize - 8);
396 0 : size_t newEvtSize = bufSize - 8;
397 0 : while (newEvtSize < eventByteCount)
398 : {
399 0 : TLOG(TLVL_TRACE) << "Reading continued DMA, current size " << newEvtSize << " / " << eventByteCount;
400 0 : buffer = readDTCBuffer(device, readSuccess, timeout, sts, true);
401 0 : if (!readSuccess)
402 : {
403 0 : TLOG(TLVL_ERROR) << "Unable to receive continued DMA! Aborting!";
404 0 : break;
405 : }
406 0 : readPtr = &buffer[0];
407 0 : bufSize = static_cast<uint16_t>(*static_cast<uint64_t*>(readPtr));
408 0 : readPtr = static_cast<uint8_t*>(readPtr) + 8;
409 0 : buffers_read++;
410 :
411 0 : size_t bytes_to_read = bufSize - 8;
412 0 : if (newEvtSize + bufSize - 8 > eventByteCount) { bytes_to_read = eventByteCount - newEvtSize; }
413 :
414 0 : memcpy(const_cast<uint8_t*>(static_cast<const uint8_t*>(newEvt.GetRawBufferPointer()) + newEvtSize), readPtr, bytes_to_read);
415 0 : newEvtSize += bufSize - 8;
416 : }
417 :
418 0 : if (!readSuccess && checkSERDES)
419 : break;
420 0 : else if (!readSuccess)
421 0 : continue;
422 :
423 0 : newEvt.SetupEvent();
424 0 : subEventCount = newEvt.GetSubEventCount();
425 :
426 0 : if (thisDTC->GetSimMode() == DTC_SimMode_ROCEmulator ||
427 0 : thisDTC->GetSimMode() == DTC_SimMode_Performance)
428 : {
429 0 : auto roc_mask_tmp = rocMask;
430 0 : size_t num_rocs = 0;
431 :
432 0 : while (roc_mask_tmp != 0)
433 : {
434 0 : if ((roc_mask_tmp & 0x1) == 1)
435 : {
436 0 : num_rocs++;
437 : }
438 0 : roc_mask_tmp = roc_mask_tmp >> 4;
439 : }
440 :
441 0 : size_t expectedEventSize = sizeof(DTC_EventHeader) + sizeof(DTC_SubEventHeader) + num_rocs * ((packetCount + 1) * 16);
442 0 : if (newEvt.GetEventByteCount() != expectedEventSize)
443 : {
444 0 : TLOG(TLVL_WARNING) << "DTC_Event size mismatch! Expected size was " << expectedEventSize << ", actual " << newEvt.GetEventByteCount();
445 : }
446 : }
447 :
448 0 : verified = verifier.VerifyEvent(newEvt);
449 0 : }
450 : else
451 : {
452 0 : evt.SetupEvent();
453 0 : subEventCount = evt.GetSubEventCount();
454 0 : if (thisDTC->GetSimMode() == DTC_SimMode_ROCEmulator ||
455 0 : thisDTC->GetSimMode() == DTC_SimMode_Performance)
456 : {
457 0 : auto roc_mask_tmp = rocMask;
458 0 : size_t num_rocs = 0;
459 :
460 0 : while (roc_mask_tmp != 0)
461 : {
462 0 : if ((roc_mask_tmp & 0x1) == 1)
463 : {
464 0 : num_rocs++;
465 : }
466 0 : roc_mask_tmp = roc_mask_tmp >> 4;
467 : }
468 :
469 0 : size_t expectedEventSize = sizeof(DTC_EventHeader) + sizeof(DTC_SubEventHeader) + num_rocs * ((packetCount + 1) * 16);
470 0 : if (evt.GetEventByteCount() != expectedEventSize)
471 : {
472 0 : TLOG(TLVL_WARNING) << "DTC_Event size mismatch! Expected size was " << expectedEventSize << ", actual " << evt.GetEventByteCount();
473 : }
474 : }
475 0 : verified = verifier.VerifyEvent(evt);
476 : }
477 0 : if (!reallyQuiet)
478 : {
479 0 : if (verified)
480 : {
481 0 : TLOG(TLVL_INFO) << "Event verified successfully, " << subEventCount << " sub-events";
482 : }
483 : else
484 : {
485 0 : TLOG(TLVL_WARNING) << "Event verification failed!";
486 : }
487 : }
488 :
489 0 : if (stopOnVerifyFailure && !verified)
490 : {
491 0 : TLOG(TLVL_ERROR) << "Event verification error encountered and stop-verify mode enabled. Stopping after " << ii << " events!";
492 0 : break;
493 : }
494 :
495 0 : device->read_release(DTC_DMA_Engine_DAQ, buffers_read);
496 0 : if (delay > 0) usleep(delay);
497 0 : }
498 0 : device->release_all(DTC_DMA_Engine_DAQ);
499 :
500 0 : auto readDevTime = device->GetDeviceTime();
501 0 : auto doneTime = std::chrono::steady_clock::now();
502 0 : auto totalBytesRead = device->GetReadSize();
503 0 : auto totalBytesWritten = device->GetWriteSize();
504 : auto totalTime =
505 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(doneTime - startTime).count();
506 : auto totalInitTime =
507 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(afterInit - startTime).count();
508 : auto totalRequestTime =
509 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(afterRequests - afterInit).count();
510 : auto totalReadTime =
511 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(doneTime - afterRequests).count();
512 :
513 0 : TLOG(TLVL_INFO) << "Total Elapsed Time: " << Utilities::FormatTimeString(totalTime) << "." << std::endl
514 0 : << "Total Init Time: " << Utilities::FormatTimeString(totalInitTime) << "." << std::endl
515 0 : << "Total Readout Request Time: " << Utilities::FormatTimeString(totalRequestTime) << "." << std::endl
516 0 : << "Total Read Time: " << Utilities::FormatTimeString(totalReadTime) << "." << std::endl;
517 0 : TLOG(TLVL_INFO) << "Device Init Time: " << Utilities::FormatTimeString(initTime) << "." << std::endl
518 0 : << "Device Request Time: " << Utilities::FormatTimeString(readoutRequestTime) << "." << std::endl
519 0 : << "Device Read Time: " << Utilities::FormatTimeString(readDevTime) << "." << std::endl;
520 0 : TLOG(TLVL_INFO) << "Total Bytes Written: " << Utilities::FormatByteString(static_cast<double>(totalBytesWritten), "")
521 0 : << "." << std::endl
522 0 : << "Total Bytes Read: " << Utilities::FormatByteString(static_cast<double>(totalBytesRead), "") << "."
523 0 : << std::endl;
524 0 : TLOG(TLVL_INFO) << "Total PCIe Rate: "
525 0 : << Utilities::FormatByteString((totalBytesWritten + totalBytesRead) / totalTime, "/s") << std::endl
526 0 : << "Read Rate: " << Utilities::FormatByteString(totalBytesRead / totalReadTime, "/s") << std::endl
527 0 : << "Device Read Rate: " << Utilities::FormatByteString(totalBytesRead / readDevTime, "/s") << std::endl;
528 0 : }
529 :
530 0 : void DTCLib::Mu2eUtil::buffer_test()
531 : {
532 0 : TLOG(TLVL_DEBUG) << "Operation \"buffer_test\"" << std::endl;
533 0 : auto startTime = std::chrono::steady_clock::now(); // get the beggining of the test
534 0 : auto thisDTC = new DTC(DTC_SimMode_NoCFO, dtc, rocMask, expectedDesignVersion);
535 0 : auto device = thisDTC->GetDevice();
536 :
537 0 : auto initTime = device->GetDeviceTime();
538 0 : device->ResetDeviceTime();
539 0 : auto afterInit = std::chrono::steady_clock::now();
540 :
541 : // Emulated CFO
542 : DTCSoftwareCFO cfo(thisDTC,
543 0 : useCFOEmulator,
544 0 : packetCount,
545 : debugType,
546 0 : stickyDebugType,
547 0 : quiet,
548 : false,
549 0 : forceNoDebug,
550 0 : useCFODRP);
551 :
552 0 : if (useSimFile)
553 : {
554 0 : auto overwrite = false;
555 0 : if (simFile.size() > 0) overwrite = true;
556 0 : thisDTC->WriteSimFileToDTC(simFile, false, overwrite, rawOutputFile, skipVerify);
557 0 : if (readGenerated)
558 : {
559 0 : exit(0);
560 : }
561 : }
562 0 : else if (readGenerated)
563 : {
564 0 : thisDTC->DisableDetectorEmulator();
565 0 : thisDTC->EnableDetectorEmulatorMode();
566 0 : thisDTC->SetDetectorEmulationDMACount(number);
567 0 : thisDTC->EnableDetectorEmulator();
568 : }
569 :
570 0 : if (thisDTC->GetSimMode() != DTC_SimMode_Loopback && timestampFile != "")
571 : {
572 0 : syncRequests = false;
573 0 : std::set<DTC_EventWindowTag> timestamps;
574 0 : std::ifstream is(timestampFile);
575 : uint64_t a;
576 0 : while (is >> a)
577 : {
578 0 : timestamps.insert(DTC_EventWindowTag(a));
579 : }
580 0 : number = timestamps.size();
581 0 : cfo.SendRequestsForList(timestamps, cfodelay, heartbeatsAfter);
582 0 : }
583 0 : else if (thisDTC->GetSimMode() != DTC_SimMode_Loopback && !syncRequests)
584 : {
585 0 : cfo.SendRequestsForRange(number,
586 0 : DTC_EventWindowTag(timestampOffset),
587 0 : incrementTimestamp,
588 : cfodelay,
589 : requestsAhead,
590 : heartbeatsAfter);
591 : }
592 0 : else if (thisDTC->GetSimMode() == DTC_SimMode_Loopback)
593 : {
594 0 : uint64_t ts = timestampOffset;
595 0 : DTC_DataHeaderPacket header(DTC_Link_0, static_cast<uint16_t>(0), DTC_DataStatus_Valid, 0, DTC_Subsystem_Other, 0, DTC_EventWindowTag(ts));
596 0 : TLOG(TLVL_INFO) << "Request: " << header.toJSON() << std::endl;
597 0 : thisDTC->WriteDMAPacket(header);
598 0 : }
599 :
600 : // record the response time
601 0 : auto readoutRequestTime = device->GetDeviceTime();
602 0 : device->ResetDeviceTime();
603 0 : auto afterRequests = std::chrono::steady_clock::now();
604 :
605 0 : for (unsigned ii = 0; ii < number + extraReads; ++ii)
606 : {
607 0 : if (syncRequests && ii < number)
608 : {
609 0 : auto startRequest = std::chrono::steady_clock::now();
610 0 : cfo.SendRequestForTimestamp(DTC_EventWindowTag(timestampOffset + (incrementTimestamp ? ii : 0), heartbeatsAfter));
611 0 : auto endRequest = std::chrono::steady_clock::now();
612 0 : readoutRequestTime +=
613 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(endRequest - startRequest).count();
614 : }
615 0 : TLOG((reallyQuiet ? TLVL_DEBUG + 4 : TLVL_INFO)) << "Buffer Read " << std::dec << ii << std::endl;
616 :
617 0 : bool readSuccess = false;
618 0 : bool timeout = false;
619 0 : size_t sts = 0;
620 0 : mu2e_databuff_t* buffer = readDTCBuffer(device, readSuccess, timeout, sts, false);
621 :
622 : // check if the read of the buffer is succesful
623 0 : if (!readSuccess && checkSERDES)
624 : break;
625 0 : else if (!readSuccess)
626 0 : continue;
627 :
628 0 : if (stopOnTimeout && timeout)
629 : {
630 0 : TLOG(TLVL_ERROR) << "Timeout detected and stop-on-timeout mode enabled. Stopping after " << ii << " events!";
631 0 : break;
632 : }
633 :
634 0 : if (!reallyQuiet)
635 : {
636 0 : DTCLib::Utilities::PrintBuffer(buffer, sts, quietCount);
637 : }
638 :
639 0 : device->read_release(DTC_DMA_Engine_DAQ, 1);
640 0 : if (delay > 0) usleep(delay);
641 : }
642 0 : device->release_all(DTC_DMA_Engine_DAQ);
643 :
644 0 : auto readDevTime = device->GetDeviceTime();
645 0 : auto doneTime = std::chrono::steady_clock::now();
646 0 : auto totalBytesRead = device->GetReadSize();
647 0 : auto totalBytesWritten = device->GetWriteSize();
648 : auto totalTime =
649 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(doneTime - startTime).count();
650 : auto totalInitTime =
651 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(afterInit - startTime).count();
652 : auto totalRequestTime =
653 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(afterRequests - afterInit).count();
654 : auto totalReadTime =
655 0 : std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(doneTime - afterRequests).count();
656 :
657 0 : TLOG(TLVL_INFO) << "Total Elapsed Time: " << Utilities::FormatTimeString(totalTime) << "." << std::endl
658 0 : << "Total Init Time: " << Utilities::FormatTimeString(totalInitTime) << "." << std::endl
659 0 : << "Total Readout Request Time: " << Utilities::FormatTimeString(totalRequestTime) << "." << std::endl
660 0 : << "Total Read Time: " << Utilities::FormatTimeString(totalReadTime) << "." << std::endl;
661 0 : TLOG(TLVL_INFO) << "Device Init Time: " << Utilities::FormatTimeString(initTime) << "." << std::endl
662 0 : << "Device Request Time: " << Utilities::FormatTimeString(readoutRequestTime) << "." << std::endl
663 0 : << "Device Read Time: " << Utilities::FormatTimeString(readDevTime) << "." << std::endl;
664 0 : TLOG(TLVL_INFO) << "Total Bytes Written: " << Utilities::FormatByteString(static_cast<double>(totalBytesWritten), "")
665 0 : << "." << std::endl
666 0 : << "Total Bytes Read: " << Utilities::FormatByteString(static_cast<double>(totalBytesRead), "") << "."
667 0 : << std::endl;
668 0 : TLOG(TLVL_INFO) << "Total PCIe Rate: "
669 0 : << Utilities::FormatByteString((totalBytesWritten + totalBytesRead) / totalTime, "/s") << std::endl
670 0 : << "Read Rate: " << Utilities::FormatByteString(totalBytesRead / totalReadTime, "/s") << std::endl
671 0 : << "Device Read Rate: " << Utilities::FormatByteString(totalBytesRead / readDevTime, "/s") << std::endl;
672 0 : }
673 :
674 0 : void DTCLib::Mu2eUtil::read_release()
675 : {
676 0 : TLOG(TLVL_DEBUG) << "Operation \"read_release\"";
677 0 : mu2edev device;
678 0 : device.init(DTCLib::DTC_SimMode_Disabled, dtc);
679 0 : for (unsigned ii = 0; ii < number; ++ii)
680 : {
681 : void* buffer;
682 0 : auto tmo_ms = 0;
683 0 : auto stsRD = device.read_data(DTC_DMA_Engine_DAQ, &buffer, tmo_ms);
684 0 : auto stsRL = device.read_release(DTC_DMA_Engine_DAQ, 1);
685 0 : TLOG(TLVL_TRACE + 10) << "util - release/read for DAQ and DCS ii=" << ii << ", stsRD=" << stsRD << ", stsRL=" << stsRL << ", buffer=" << buffer;
686 0 : if (delay > 0) usleep(delay);
687 : }
688 0 : }
689 :
690 0 : void DTCLib::Mu2eUtil::program_clock()
691 : {
692 0 : TLOG(TLVL_DEBUG) << "Operation \"program_clock\"";
693 0 : auto thisDTC = new DTC(DTC_SimMode_NoCFO, dtc, rocMask, expectedDesignVersion, true);
694 0 : auto oscillator = clockToProgram == 0 ? DTC_OscillatorType_SERDES
695 0 : : (clockToProgram == 1 ? DTC_OscillatorType_DDR : DTC_OscillatorType_Timing);
696 0 : thisDTC->SetNewOscillatorFrequency(oscillator, targetFrequency);
697 0 : delete thisDTC;
698 0 : }
699 :
700 0 : void DTCLib::Mu2eUtil::dma_info()
701 : {
702 0 : TLOG(TLVL_DEBUG) << "Opearation \"dma_info\"";
703 0 : if (dtc == -1)
704 : {
705 0 : auto dtcE = getenv("DTCLIB_DTC");
706 0 : if (dtcE != nullptr)
707 : {
708 0 : dtc = atoi(dtcE);
709 : }
710 : else
711 0 : dtc = 0;
712 : }
713 :
714 0 : mu2edev device;
715 0 : device.init(DTCLib::DTC_SimMode_Disabled, dtc);
716 0 : device.meta_dump();
717 0 : }
718 :
719 0 : void DTCLib::Mu2eUtil::run()
720 : {
721 0 : if (rawOutput) outputStream.open(rawOutputFile, std::ios::out | std::ios::app | std::ios::binary);
722 :
723 0 : if (op == "read_data")
724 : {
725 0 : read_data();
726 : }
727 0 : else if (op == "toggle_serdes")
728 : {
729 0 : toggle_serdes();
730 : }
731 0 : else if (op == "reset_ddr")
732 : {
733 0 : reset_ddr();
734 : }
735 0 : else if (op == "reset_detemu")
736 : {
737 0 : reset_detector_emulator();
738 : }
739 0 : else if (op == "verify_simfile")
740 : {
741 0 : verify_sim_file();
742 : }
743 0 : else if (op == "verify_stream")
744 : {
745 0 : verify_stream();
746 : }
747 0 : else if (op == "buffer_test")
748 : {
749 0 : buffer_test();
750 : }
751 0 : else if (op == "read_release")
752 : {
753 0 : read_release();
754 : }
755 0 : else if (op == "program_clock")
756 : {
757 0 : program_clock();
758 : }
759 0 : else if (op == "dma_info")
760 : {
761 0 : dma_info();
762 : }
763 : else
764 : {
765 0 : TLOG(TLVL_ERROR) << "Unrecognized operation: " << op << std::endl;
766 0 : printHelpMsg();
767 : }
768 :
769 0 : if (rawOutput)
770 : {
771 0 : outputStream.flush();
772 0 : outputStream.close();
773 : }
774 0 : }
775 :
776 0 : void DTCLib::Mu2eUtil::printHelpMsg()
777 : {
778 : std::cout << "Usage: mu2eUtil [options] "
779 0 : "[read_data,reset_ddrread,reset_detemu,toggle_serdes,loopback,verify_stream,buffer_test,read_release,program_clock,verify_simfile,dma_info]"
780 0 : << std::endl;
781 : std::cout
782 0 : << "Options are:" << std::endl
783 0 : << " -h, --help: This message." << std::endl
784 0 : << " -n: Number of times to repeat test. (Default: 1)" << std::endl
785 0 : << " -o: Starting Timestamp offest. (Default: 1)." << std::endl
786 0 : << " -i: Do not increment Timestamps." << std::endl
787 0 : << " -S: Synchronous Timestamp mode (1 RR & DR per Read operation)" << std::endl
788 0 : << " -d: Delay between tests, in us (Default: 0)." << std::endl
789 0 : << " -D: CFO Request delay interval (Default: 1000 (minimum)." << std::endl
790 0 : << " -c: Number of Debug Packets to request (Default: 0)." << std::endl
791 0 : << " -N: Do NOT set the Debug flag in generated Data Request packets" << std::endl
792 0 : << " -b: Number of Data Blocks to generate per Event (Default: 1)." << std::endl
793 0 : << " -E: Number of Events to generate per DMA block (Default: 1)." << std::endl
794 0 : << " -a: Number of Readout Request/Data Requests to send before starting to read data (Default: 0)."
795 0 : << std::endl
796 : << " -q: Quiet mode (Don't print requests) Additionally, for buffer_test mode, limits to N (Default 1) "
797 0 : "packets at the beginning and end of the buffer."
798 0 : << std::endl
799 0 : << " -Q: Really Quiet mode (Try not to print anything)" << std::endl
800 0 : << " -s: Stop on SERDES Error." << std::endl
801 0 : << " -e: Use DTCLib's SoftwareCFO instead of the DTC CFO Emulator" << std::endl
802 0 : << " -t: Use DebugType flag (1st request gets ExternalDataWithFIFOReset, the rest get ExternalData)"
803 0 : << std::endl
804 0 : << " -T: Set DebugType flag for ALL requests (0, 1, or 2)" << std::endl
805 0 : << " -f: RAW Output file path" << std::endl
806 0 : << " -H: Write DMA headers to raw output file (when -f is used with -g)" << std::endl
807 0 : << " -p: Send DTCLIB_SIM_FILE to DTC and enable Detector Emulator mode" << std::endl
808 0 : << " -P: Send <file> to DTC and enable Detector Emulator mode (Default: \"\")" << std::endl
809 0 : << " -G: Enable Detector Emulator Mode" << std::endl
810 : << " -r: # of rocs to enable. Hexadecimal, each digit corresponds to a link. ROC_0: 1, ROC_1: 3, ROC_2: 5, "
811 0 : "ROC_3: 7, ROC_4: 9, ROC_5: B (Default 0x1, All possible: 0xBBBBBB)"
812 0 : << std::endl
813 0 : << " -F: Frequency to program (in Hz, sorry...Default 166666667 Hz)" << std::endl
814 0 : << " -C: Clock to program (0: SERDES, 1: DDR, 2: Timing, Default 0)" << std::endl
815 0 : << " -v: Expected DTC Design version string (Default: \"\")" << std::endl
816 0 : << " -V: Do NOT attempt to verify that the sim file landed in DTC memory correctly" << std::endl
817 0 : << " --timestamp-list: Read <file> for timestamps to request (CFO will generate heartbeats for all timestamps in range spanned by file)" << std::endl
818 0 : << " --dtc: Use dtc <num> (Defaults to DTCLIB_DTC if set, 0 otherwise, see ls /dev/mu2e* for available DTCs)" << std::endl
819 0 : << " --cfoDRP: Send DRPs from the DTC CFO Emulator instead of from the DTC itself" << std::endl
820 0 : << " --heartbeats: Send <int> heartbeats after each DataRequestPacket" << std::endl
821 0 : << " --binary-file-mode: Write DMA sizes to <file> along with read data, to generate a new binary file for detector emulator mode (not compatible with -f)" << std::endl
822 0 : << " --stop-verify: If a verify_stream mode error occurs, stop processing" << std::endl
823 0 : << " --stop-on-timeout: Stop verify_stream or buffer_test mode if a timeout is detected (0xCAFE in first packet of buffer)" << std::endl
824 0 : << " --extra-reads: Number of extra DMA reads to attempt in verify_stream and buffer_test modes (Default: 1)" << std::endl;
825 :
826 0 : exit(0);
827 : }
828 :
829 0 : mu2e_databuff_t* DTCLib::Mu2eUtil::readDTCBuffer(mu2edev* device, bool& readSuccess, bool& timeout, size_t& sts, bool continuedMode)
830 : {
831 : mu2e_databuff_t* buffer;
832 0 : auto tmo_ms = 1500;
833 0 : readSuccess = false;
834 0 : TLOG(TLVL_TRACE) << "util - before read for DAQ";
835 0 : sts = device->read_data(DTC_DMA_Engine_DAQ, reinterpret_cast<void**>(&buffer), tmo_ms);
836 0 : TLOG(TLVL_TRACE) << "util - after read for DAQ sts=" << sts << ", buffer=" << (void*)buffer;
837 :
838 0 : if (sts > 0)
839 : {
840 0 : readSuccess = true;
841 0 : void* readPtr = &buffer[0];
842 0 : uint16_t bufSize = static_cast<uint16_t>(*static_cast<uint64_t*>(readPtr));
843 0 : readPtr = static_cast<uint8_t*>(readPtr) + 8;
844 0 : TLOG((reallyQuiet ? TLVL_DEBUG + 4 : TLVL_INFO)) << "Buffer reports DMA size of " << std::dec << bufSize << " bytes. Device driver reports read of "
845 0 : << sts << " bytes," << std::endl;
846 :
847 0 : TLOG(TLVL_TRACE) << "util - bufSize is " << bufSize;
848 0 : if (binaryFileOutput)
849 : {
850 0 : uint64_t dmaWriteSize = sts + 8;
851 0 : outputStream.write(reinterpret_cast<char*>(&dmaWriteSize), sizeof(dmaWriteSize));
852 0 : outputStream.write(reinterpret_cast<char*>(&buffer[0]), sts);
853 : }
854 0 : else if (rawOutput)
855 : {
856 0 : outputStream.write(static_cast<char*>(readPtr), sts - 8);
857 : }
858 :
859 0 : timeout = false;
860 0 : if (!continuedMode && sts > sizeof(DTC_EventHeader) + sizeof(DTC_SubEventHeader) + 8)
861 : {
862 : // Check for dead or cafe in first packet
863 0 : readPtr = static_cast<uint8_t*>(readPtr) + sizeof(DTC_EventHeader) + sizeof(DTC_SubEventHeader);
864 0 : std::vector<size_t> wordsToCheck{1, 2, 3, 7, 8};
865 0 : for (auto& word : wordsToCheck)
866 : {
867 0 : auto wordPtr = static_cast<uint16_t*>(readPtr) + (word - 1);
868 0 : TLOG(TLVL_TRACE + 1) << word << (word == 1 ? "st" : word == 2 ? "nd"
869 0 : : word == 3 ? "rd"
870 0 : : "th")
871 0 : << " word of buffer: " << *wordPtr;
872 0 : if (*wordPtr == 0xcafe || *wordPtr == 0xdead)
873 : {
874 0 : TLOG(TLVL_WARNING) << "Buffer Timeout detected! " << word << (word == 1 ? "st" : word == 2 ? "nd"
875 0 : : word == 3 ? "rd"
876 0 : : "th")
877 0 : << " word of buffer is 0x" << std::hex << *wordPtr;
878 0 : DTCLib::Utilities::PrintBuffer(readPtr, 16, 0, TLVL_TRACE + 3);
879 0 : timeout = true;
880 0 : break;
881 : }
882 : }
883 0 : }
884 : }
885 0 : return buffer;
886 : }
|
