Coverage for python/lsst/cp/pipe/runEotestTask.py : 3%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Calibration products production task code."""
23from __future__ import absolute_import, division, print_function
25import os
26import glob
27import sys
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31import lsst.log as lsstLog
32import lsst.eotest.sensor as sensorTest
35class RunEotestConfig(pexConfig.Config):
36 """Config class for the calibration products production (CP) task."""
38 ccdKey = pexConfig.Field(
39 dtype=str,
40 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
41 default='ccd',
42 )
43 fe55 = pexConfig.ConfigurableField(
44 target=sensorTest.Fe55Task,
45 doc="The Fe55 analysis task.",
46 )
47 doFe55 = pexConfig.Field(
48 dtype=bool,
49 doc="Measure gains using Fe55?",
50 default=True,
51 )
52 readNoise = pexConfig.ConfigurableField(
53 target=sensorTest.ReadNoiseTask,
54 doc="The read noise task.",
55 )
56 doReadNoise = pexConfig.Field(
57 dtype=bool,
58 doc="Measure the read-noise?",
59 default=True,
60 )
61 brightPixels = pexConfig.ConfigurableField(
62 target=sensorTest.BrightPixelsTask,
63 doc="The bright pixel/column finding task.",
64 )
65 doBrightPixels = pexConfig.Field(
66 dtype=bool,
67 doc="Find bright pixels?",
68 default=True,
69 )
70 darkPixels = pexConfig.ConfigurableField(
71 target=sensorTest.DarkPixelsTask,
72 doc="The dark pixel/column finding task.",
73 )
74 doDarkPixels = pexConfig.Field(
75 dtype=bool,
76 doc="Find dark pixels?",
77 default=True,
78 )
79 traps = pexConfig.ConfigurableField(
80 target=sensorTest.TrapTask,
81 doc="The trap-finding task.",
82 )
83 doTraps = pexConfig.Field(
84 dtype=bool,
85 doc="Find traps using pocket-pumping exposures?",
86 default=True,
87 )
88 cte = pexConfig.ConfigurableField(
89 target=sensorTest.CteTask,
90 doc="The CTE analysis task.",
91 )
92 doCTE = pexConfig.Field(
93 dtype=bool,
94 doc="Measure the charge transfer efficiency?",
95 default=True,
96 )
97 ptc = pexConfig.ConfigurableField(
98 target=sensorTest.PtcTask,
99 doc="The PTC analysis task.",
100 )
101 doPTC = pexConfig.Field(
102 dtype=bool,
103 doc="Measure the photon transfer curve?",
104 default=True,
105 )
106 flatPair = pexConfig.ConfigurableField(
107 target=sensorTest.FlatPairTask,
108 doc="The flat-pair analysis task.",
109 )
110 doFlatPair = pexConfig.Field(
111 dtype=bool,
112 doc="Measure the detector response vs incident flux using flat pairs?",
113 default=True,
114 )
115 eotestOutputPath = pexConfig.Field(
116 dtype=str,
117 doc="Path to which to write the eotest output results. Madatory runtime arg for running eotest.",
118 default='',
119 )
120 requireAllEOTests = pexConfig.Field(
121 dtype=bool,
122 doc="If True, all tests are required to be runnable, and will Raise if data is missing. If False, "
123 "processing will continue if a previous part failed due to the input dataset being incomplete.",
124 default=True,
125 )
126 flatPairMaxPdFracDev = pexConfig.Field(
127 dtype=float,
128 doc="Maximum allowed fractional deviation between photodiode currents for the eotest flatPair task. "
129 "This value is passed to the task's run() method at runtime rather than being stored in the task's"
130 "own pexConfig field.",
131 default=0.05,
132 )
134 def setDefaults(self):
135 """Set default config options for the subTasks."""
136 # TODO: Set to proper values - DM-12939
137 self.fe55.temp_set_point = -100
138 self.fe55.temp_set_point_tol = 20
140 # TODO: Set to proper values - DM-12939
141 self.readNoise.temp_set_point = -100
142 self.readNoise.temp_set_point_tol = 20
144 # TODO: make this work - DM-12939
145 self.brightPixels.temp_set_point = -100
146 self.brightPixels.temp_set_point_tol = 20
148 # TODO: find the proper settings for flatPairTask to work. This will mean either using the expTime,
149 # or working out what's wrong with the MONDIODE values (debug that anyway as a large difference might
150 # indicate something else going on). Note that this task doesn't really use much in its config class,
151 # but passes in things at runtime param to its run() method, hence putting in a slot for them here.
152 # DM-12939
153 self.flatPairMaxPdFracDev = 0.99
155 def validate(self):
156 """Override of the valiate() method.
158 The pexConfigs of the subTasks here cannot be validated in the normal way, as they are the configs
159 for eotest, which does illegal things, and this would require an upstream PR to fix. Therefore, we
160 override the validate() method here, and use it to set the output directory for each of the tasks
161 based on the legal pexConfig parameter for the main task.
162 """
163 log = lsstLog.Log.getLogger("cp.pipe.runEotestConfig")
164 if not self.eotestOutputPath:
165 raise RuntimeError("Must supply an output path for eotest data. "
166 "Please set config.eotestOutputPath.")
168 taskList = ['fe55', 'brightPixels', 'darkPixels', 'readNoise', 'traps', 'cte', 'flatPair', 'ptc']
169 for task in taskList:
170 if getattr(self, task).output_dir != '.':
171 # Being thorough here: '.' is the eotest default. If this is not the value then the user has
172 # specified something, and we're going to clobber it, so raise a warning. Unlike to happen.
173 log.warn("OVERWRITING: Found a user defined output path of %s for %sTask. "
174 "This has been overwritten with %s, as individually specified output paths for "
175 "subTasks are not supported at present" % (getattr(self, task).output_dir,
176 task, self.eotestOutputPath))
177 getattr(self, task).output_dir = self.eotestOutputPath
180class RunEotestTask(pipeBase.CmdLineTask):
181 """
182 Task to run test stand data through eotest using a butler.
184 This task is used to produce an eotest report (the project's sensor
185 acceptance testing package)
186 Examples of some of its operations are as follows:
187 * Given a set of flat-field images, find the dark pixels and columns.
188 * Given a set of darks, find the bright pixels and columns.
189 * Given a set of Fe55 exposures, calulate the gain of the readout chain,
190 in e-/ADU
191 * Given a set of Fe55 exposures, calulate the instrinsic PSF of the silicon,
192 and the degradation of
193 * the PSF due to CTE.
194 * Given a set of flat-pairs, measure the photon transfer curve (PTC).
195 * Given a set of bias frames, calculate the read noise of the system in e-.
196 * Given a set of pocket-pumping exposures, find charge-traps in the silicon.
198 The RunEotestTask.runEotestDirect() is only applicable to LSST sensors, and
199 only for a specific type of dataset. This method takes a
200 dafPersistance.Butler corresponding to a repository in which a full eotest
201 run has been taken and ingested, and runs each of the tasks in eotest
202 directly, allowing for bitwise comparison with results given by the camera
203 team.
205 See http://ls.st/ldm-151 Chapter 4, Calibration Products Production for
206 further details regarding the inputs and outputs.
207 """
209 ConfigClass = RunEotestConfig
210 _DefaultName = "runEotest"
212 def __init__(self, *args, **kwargs):
213 """Constructor for the RunEotestTask."""
214 if 'lsst.eotest.sensor' not in sys.modules: # check we have eotest before going further
215 raise RuntimeError('eotest failed to import')
217 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
219 # Note - we can't currently call validate on the subTask configs, as they are NOT valid
220 # due to state of eotest. However, we override validate() and call it here
221 # and use it to set the output dir config parameter in the subTasks.
222 self.config.validate()
223 self.config.freeze()
225 self.makeSubtask("fe55")
226 self.makeSubtask("readNoise")
227 self.makeSubtask("brightPixels")
228 self.makeSubtask("darkPixels")
229 self.makeSubtask("traps")
230 self.makeSubtask("cte")
231 self.makeSubtask("flatPair")
232 self.makeSubtask("ptc")
234 def _getMaskFiles(self, path, ccd):
235 """Get all available eotest mask files for a given ccd.
237 Each stage of the processing generates more mask files, so this allows each to be picked up
238 as more and more tests run, and saves having to have clever logic for if some tasks fail.
240 Parameters
241 ----------
242 path : `str`
243 Path on which to find the mask files
244 ccd : `string` or `int`
245 Name/identifier of the CCD
247 Returns
248 -------
249 maskFiles : iterable of `str`
250 List of mask files, or an empty tuple if none are found
251 """
252 pattern = '*' + str(ccd) + '*mask*' # the cast to str supports obs_auxTel
253 maskFiles = glob.glob(os.path.join(path, pattern))
254 return maskFiles if len(maskFiles) > 0 else () # eotest wants an empty tuple here
256 def _cleanupEotest(self, path):
257 """Delete all the medianed files left behind after eotest has run.
259 Running eotest generates a lot of interim medianed files, so this just cleans them up.
261 Parameters
262 ----------
263 path : `str`
264 Path on which to delete all the eotest medianed files.
265 """
266 for filename in glob.glob(os.path.join(path, '*_median_*.fits')):
267 os.remove(filename)
269 def makeEotestReport(self, butler):
270 """After running eotest, generate pdf(s) of the results.
272 Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD.
273 The pdf file(s), along with the .tex file(s) and the individual plots are written
274 to the eotestOutputPath.
275 .pdf generation requires a TeX distro including pdflatex to be installed.
276 """
277 ccds = butler.queryMetadata('raw', self.config.ccdKey)
278 for ccd in ccds:
279 self.log.info("Starting test report generation for %s"%ccd)
280 try:
281 plotPath = os.path.join(self.config.eotestOutputPath, 'plots')
282 if not os.path.exists(plotPath):
283 os.makedirs(plotPath)
284 plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath)
285 eoTestReport = sensorTest.EOTestReport(plots, wl_dir='')
286 eoTestReport.make_figures()
287 eoTestReport.make_pdf()
288 except Exception as e:
289 self.log.warn("Failed to make eotest report for %s: %s"%(ccd, e))
290 self.log.info("Finished test report generation.")
292 @pipeBase.timeMethod
293 def runEotestDirect(self, butler, run=None):
294 """
295 Generate calibration products using eotest algorithms.
297 Generate all calibration products possible using the vanilla eotest implementation,
298 given a butler for a TS8 (raft-test) repo. It can contain multiple runs, but must correspond to
299 only a single raft/RTM.
301 - Run all eotest tasks possible, using the butler to gather the data
302 - Write outputs in eotest format
304 In order to replicate the canonical eotest analysis, the tasks should be run in a specific order.
305 This is given/defined in the "Steps" section here:
306 http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayProcess.jsp?processPath=1179
308 But is replicated here for conveniece:
309 * 55Fe Analysis
310 * CCD Read Noise Analysis
311 * Bright Defects Analysis
312 * Dark Defects Analysis
313 * Traps Finding
314 * Dark Current X - will not be implemented here
315 * Charge Transfer Efficiencies
316 * Photo-response analysis X - will not be implemented here
317 * Flat Pairs Analysis
318 * Photon Transfer Curve
319 * Quantum Efficiency X - will not be implemented here
321 List of tasks that exist in the eotest package but aren't mentioned on the above link:
322 * linearityTask()
323 * fe55CteTask()
324 * eperTask()
325 * crosstalkTask()
326 * persistenceTask()
328 # TODO: For each eotest task, find out what the standard raft testing does for the optional params.
329 i.e. many have optional params for gains, bias-frames etc - if we want bitwise identicallity then we
330 need to know what is typically provided to these tasks when the camera team runs this code.
331 This can probably be worked out from https://github.com/lsst-camera-dh/lcatr-harness
332 but it sounds like Jim Chiang doesn't recommend trying to do that.
333 DM-12939
335 Parameters
336 ----------
337 butler : `lsst.daf.persistence.butler`
338 Butler for the repo containg the eotest data to be used
339 run : `str` or `int`
340 Optional run number, to be used for repos containing multiple runs
341 """
342 self.log.info("Running eotest routines direct")
344 # Input testing to check that run is in the repo
345 runs = butler.queryMetadata('raw', ['run'])
346 if run is None:
347 if len(runs) == 1:
348 run = runs[0]
349 else:
350 raise RuntimeError("Butler query found %s for runs. eotest datasets must have a run number,"
351 "and you must specify which run to use if a respoitory contains several."
352 % runs)
353 else:
354 run = str(run)
355 if run not in runs:
356 raise RuntimeError("Butler query found %s for runs, but the run specified (%s) "
357 "was not among them." % (runs, run))
358 del runs # we have run defined now, so remove this to avoid potential confusion later
360 if not os.path.exists(self.config.eotestOutputPath):
361 os.makedirs(self.config.eotestOutputPath)
363 ccds = butler.queryMetadata('raw', self.config.ccdKey)
364 imTypes = butler.queryMetadata('raw', ['imageType'])
365 testTypes = butler.queryMetadata('raw', ['testType'])
367 ################################
368 ################################
369 # Run the Fe55 task
370 if self.config.doFe55:
371 fe55TaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'FE55'}
372 self.log.info("Starting Fe55 pixel task")
373 for ccd in ccds:
374 if 'FE55' not in testTypes:
375 msg = "No Fe55 tests found. Available data: %s" % testTypes
376 if self.config.requireAllEOTests:
377 raise RuntimeError(msg)
378 else:
379 self.log.warn(msg + "\nSkipping Fe55 task")
380 break
381 fe55Filenames = [butler.get('raw_filename', dataId={'visit': visit,
382 self.config.ccdKey: ccd})[0][:-3]
383 for visit in butler.queryMetadata('raw', ['visit'], dataId=fe55TaskDataId)]
384 self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(fe55Filenames)))
385 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
386 gains = self.fe55.run(sensor_id=ccd, infiles=fe55Filenames, mask_files=maskFiles)
387 # gainsPropSet = dafBase.PropertySet()
388 # for amp, gain in gains.items(): # there is no propSet.fromDict() method so make like this
389 # gainsPropSet.addDouble(str(amp), gain)
390 butler.put(gains, 'eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
391 del fe55TaskDataId
393 # TODO: validate the results above, and/or change code to (be able to) always run
394 # over all files instead of stopping at the "required accuracy"
395 # This will require making changes to the eotest code.
396 # DM-12939
398 ################################
399 ################################
400 # Run the Noise task
401 if self.config.doReadNoise:
402 # note that LCA-10103 defines the Fe55 bias frames as the ones to use here
403 self.log.info("Starting readNoise task")
404 noiseTaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'BIAS'}
405 for ccd in ccds:
406 if ('FE55' not in testTypes) or ('BIAS' not in imTypes):
407 msg = "Required data for readNoise unavailable. Available data:\
408 \ntestTypes: %s\nimageTypes: %s" % (testTypes, imTypes)
409 if self.config.requireAllEOTests:
410 raise RuntimeError(msg)
411 else:
412 self.log.warn(msg + "\nSkipping noise task")
413 noiseFilenames = [butler.get('raw_filename', dataId={'visit': visit,
414 self.config.ccdKey: ccd})[0][:-3]
415 for visit in butler.queryMetadata('raw', ['visit'],
416 dataId=noiseTaskDataId)]
417 self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(noiseFilenames)))
418 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
419 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
420 self.readNoise.run(sensor_id=ccd, bias_files=noiseFilenames,
421 gains=gains, mask_files=maskFiles)
422 del noiseTaskDataId
424 ################################
425 ################################
426 # Run the bright task
427 if self.config.doBrightPixels:
428 self.log.info("Starting bright pixel task")
429 brightTaskDataId = {'run': run, 'testType': 'DARK', 'imageType': 'DARK'}
430 for ccd in ccds:
431 if 'DARK' not in testTypes:
432 msg = "No dark tests found. Available data: %s" % testTypes
433 if self.config.requireAllEOTests:
434 raise RuntimeError(msg)
435 else:
436 self.log.warn(msg + "\nSkipping bright pixel task")
437 break
438 darkFilenames = [butler.get('raw_filename', dataId={'visit': visit,
439 self.config.ccdKey: ccd})[0][:-3]
440 for visit in butler.queryMetadata('raw', ['visit'],
441 dataId=brightTaskDataId)]
442 self.log.trace("BrightTask: Processing %s with %s files" % (ccd, len(darkFilenames)))
443 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
444 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
445 self.brightPixels.run(sensor_id=ccd, dark_files=darkFilenames,
446 mask_files=maskFiles, gains=gains)
447 del brightTaskDataId
449 ################################
450 ################################
451 # Run the dark task
452 if self.config.doDarkPixels:
453 self.log.info("Starting dark pixel task")
454 darkTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
455 for ccd in ccds:
456 if 'SFLAT_500' not in testTypes:
457 msg = "No superflats found. Available data: %s" % testTypes
458 if self.config.requireAllEOTests:
459 raise RuntimeError(msg)
460 else:
461 self.log.warn(msg + "\nSkipping dark pixel task")
462 break
463 sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
464 self.config.ccdKey: ccd})[0][:-3]
465 for visit in butler.queryMetadata('raw', ['visit'],
466 dataId=darkTaskDataId)]
467 self.log.trace("DarkTask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
468 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
469 self.darkPixels.run(sensor_id=ccd, sflat_files=sflatFilenames, mask_files=maskFiles)
470 del darkTaskDataId
472 ################################
473 ################################
474 # Run the trap task
475 if self.config.doTraps:
476 self.log.info("Starting trap task")
477 trapTaskDataId = {'run': run, 'testType': 'TRAP', 'imageType': 'PPUMP'}
478 for ccd in ccds:
479 if ('TRAP' not in testTypes) and ('PPUMP' not in imTypes):
480 msg = "No pocket pumping exposures found. Available data: %s" % testTypes
481 if self.config.requireAllEOTests:
482 raise RuntimeError(msg)
483 else:
484 self.log.warn(msg + "\nSkipping trap task")
485 break
486 trapFilenames = [butler.get('raw_filename', dataId={'visit': visit,
487 self.config.ccdKey: ccd})[0][:-3]
488 for visit in butler.queryMetadata('raw', ['visit'], dataId=trapTaskDataId)]
489 if len(trapFilenames) != 1: # eotest can't handle more than one
490 msg = "Trap Task: Found more than one ppump trap file: %s" % trapFilenames
491 msg += " Running using only the first one found."
492 self.log.warn(msg)
493 self.log.trace("Trap Task: Processing %s with %s files" % (ccd, len(trapFilenames)))
494 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
495 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
496 self.traps.run(sensor_id=ccd, pocket_pumped_file=trapFilenames[0],
497 mask_files=maskFiles, gains=gains)
498 del trapTaskDataId
500 ################################
501 ################################
502 # Run the CTE task
503 if self.config.doCTE:
504 self.log.info("Starting CTE task")
505 cteTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
506 for ccd in ccds:
507 if 'SFLAT_500' not in testTypes:
508 msg = "No superflats found. Available data: %s" % testTypes
509 if self.config.requireAllEOTests:
510 raise RuntimeError(msg)
511 else:
512 self.log.warn(msg + "\nSkipping CTE task")
513 break
514 sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
515 self.config.ccdKey: ccd})[0][:-3]
516 for visit in butler.queryMetadata('raw', ['visit'], dataId=cteTaskDataId)]
517 self.log.trace("CTETask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
518 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
519 self.cte.run(sensor_id=ccd, superflat_files=sflatFilenames, mask_files=maskFiles)
520 del cteTaskDataId
522 ################################
523 ################################
524 # Run the flatPair task
525 if self.config.doFlatPair:
526 self.log.info("Starting flatPair task")
527 flatPairDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
528 for ccd in ccds:
529 if 'FLAT' not in testTypes:
530 msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
531 if self.config.requireAllEOTests:
532 raise RuntimeError(msg)
533 else:
534 self.log.warn(msg + "\nSkipping flatPair task")
535 break
536 flatPairFilenames = [butler.get('raw_filename', dataId={'visit': visit,
537 self.config.ccdKey: ccd})[0][:-3]
538 for visit in butler.queryMetadata('raw', ['visit'],
539 dataId=flatPairDataId)]
540 # Note that eotest needs the original filename as written by the test-stand data acquisition
541 # system, as that is the only place the flat pair-number is recorded, so we have to resolve
542 # sym-links and pass in the *original* paths/filenames here :(
543 # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
544 # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
545 # isn't totally thorough on rejecting the wrong types of data here)
546 # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
547 # by the butler instead of here. DM-12939
548 flatPairFilenames = [os.path.realpath(f) for f in flatPairFilenames if
549 os.path.realpath(f).find('flat1') != -1
550 or os.path.realpath(f).find('flat2') != -1]
551 if not flatPairFilenames:
552 raise RuntimeError("No flatPair files found.")
553 self.log.trace("FlatPairTask: Processing %s with %s files" % (ccd, len(flatPairFilenames)))
554 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
555 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
556 self.flatPair.run(sensor_id=ccd, infiles=flatPairFilenames, mask_files=maskFiles,
557 gains=gains, max_pd_frac_dev=self.config.flatPairMaxPdFracDev)
558 del flatPairDataId
560 ################################
561 ################################
562 # Run the PTC task
563 if self.config.doPTC:
564 self.log.info("Starting PTC task")
565 ptcDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
566 for ccd in ccds:
567 if 'FLAT' not in testTypes:
568 msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
569 if self.config.requireAllEOTests:
570 raise RuntimeError(msg)
571 else:
572 self.log.warn(msg + "\nSkipping PTC task")
573 break
574 ptcFilenames = [butler.get('raw_filename', dataId={'visit': visit,
575 self.config.ccdKey: ccd})[0][:-3]
576 for visit in butler.queryMetadata('raw', ['visit'], dataId=ptcDataId)]
577 # Note that eotest needs the original filename as written by the test-stand data acquisition
578 # system, as that is the only place the flat pair-number is recorded, so we have to resolve
579 # sym-links and pass in the *original* paths/filenames here :(
580 # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
581 # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
582 # isn't totally thorough on rejecting the wrong types of data here)
583 # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
584 # by the butler instead of here. DM-12939
585 ptcFilenames = [os.path.realpath(f) for f in ptcFilenames if
586 os.path.realpath(f).find('flat1') != -1
587 or os.path.realpath(f).find('flat2') != -1]
588 if not ptcFilenames:
589 raise RuntimeError("No flatPair files found")
590 self.log.trace("PTCTask: Processing %s with %s files" % (ccd, len(ptcFilenames)))
591 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
592 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
593 self.ptc.run(sensor_id=ccd, infiles=ptcFilenames, mask_files=maskFiles, gains=gains)
594 del ptcDataId
596 self._cleanupEotest(self.config.eotestOutputPath)
597 self.log.info("Finished running EOTest")