lsst.cp.pipe  master-gcb87133ace+5
cpTask.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 """Calibration products production task code."""
25 from __future__ import absolute_import, division, print_function
26 
27 import os
28 import glob
29 import sys
30 
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.log as lsstLog
34 import lsst.eotest.sensor as sensorTest
35 
36 
37 class CpTaskConfig(pexConfig.Config):
38  """Config class for the calibration products production (CP) task."""
39 
40  fe55 = pexConfig.ConfigurableField(
41  target=sensorTest.Fe55Task,
42  doc="The Fe55 analysis task.",
43  )
44  doFe55 = pexConfig.Field(
45  dtype=bool,
46  doc="Measure gains using Fe55?",
47  default=True,
48  )
49  readNoise = pexConfig.ConfigurableField(
50  target=sensorTest.ReadNoiseTask,
51  doc="The read noise task.",
52  )
53  doReadNoise = pexConfig.Field(
54  dtype=bool,
55  doc="Measure the read-noise?",
56  default=True,
57  )
58  brightPixels = pexConfig.ConfigurableField(
59  target=sensorTest.BrightPixelsTask,
60  doc="The bright pixel/column finding task.",
61  )
62  doBrightPixels = pexConfig.Field(
63  dtype=bool,
64  doc="Find bright pixels?",
65  default=True,
66  )
67  darkPixels = pexConfig.ConfigurableField(
68  target=sensorTest.DarkPixelsTask,
69  doc="The dark pixel/column finding task.",
70  )
71  doDarkPixels = pexConfig.Field(
72  dtype=bool,
73  doc="Find dark pixels?",
74  default=True,
75  )
76  traps = pexConfig.ConfigurableField(
77  target=sensorTest.TrapTask,
78  doc="The trap-finding task.",
79  )
80  doTraps = pexConfig.Field(
81  dtype=bool,
82  doc="Find traps using pocket-pumping exposures?",
83  default=True,
84  )
85  cte = pexConfig.ConfigurableField(
86  target=sensorTest.CteTask,
87  doc="The CTE analysis task.",
88  )
89  doCTE = pexConfig.Field(
90  dtype=bool,
91  doc="Measure the charge transfer efficiency?",
92  default=True,
93  )
94  ptc = pexConfig.ConfigurableField(
95  target=sensorTest.PtcTask,
96  doc="The PTC analysis task.",
97  )
98  doPTC = pexConfig.Field(
99  dtype=bool,
100  doc="Measure the photon transfer curve?",
101  default=True,
102  )
103  flatPair = pexConfig.ConfigurableField(
104  target=sensorTest.FlatPairTask,
105  doc="The flat-pair analysis task.",
106  )
107  doFlatPair = pexConfig.Field(
108  dtype=bool,
109  doc="Measure the detector response vs incident flux using flat pairs?",
110  default=True,
111  )
112  eotestOutputPath = pexConfig.Field(
113  dtype=str,
114  doc="Path to which to write the eotest output results. Madatory runtime arg for running eotest.",
115  default='',
116  )
117  requireAllEOTests = pexConfig.Field(
118  dtype=bool,
119  doc="If True, all tests are required to be runnable, and will Raise if data is missing. If False, "
120  "processing will continue if a previous part failed due to the input dataset being incomplete.",
121  default=True,
122  )
123  flatPairMaxPdFracDev = pexConfig.Field(
124  dtype=float,
125  doc="Maximum allowed fractional deviation between photodiode currents for the eotest flatPair task. "
126  "This value is passed to the task's run() method at runtime rather than being stored in the task's"
127  "own pexConfig field.",
128  default=0.05,
129  )
130 
131  def setDefaults(self):
132  """Set default config options for the subTasks."""
133  # TODO: Set to proper values - DM-12939
134  self.fe55.temp_set_point = -100
135  self.fe55.temp_set_point_tol = 20
136 
137  # TODO: Set to proper values - DM-12939
138  self.readNoise.temp_set_point = -100
139  self.readNoise.temp_set_point_tol = 20
140 
141  # TODO: make this work - DM-12939
142  self.brightPixels.temp_set_point = -100
143  self.brightPixels.temp_set_point_tol = 20
144 
145  # TODO: find the proper settings for flatPairTask to work. This will mean either using the expTime,
146  # or working out what's wrong with the MONDIODE values (debug that anyway as a large difference might
147  # indicate something else going on). Note that this task doesn't really use much in its config class,
148  # but passes in things at runtime param to its run() method, hence putting in a slot for them here.
149  # DM-12939
150  self.flatPairMaxPdFracDev = 0.99
151 
152  def validate(self):
153  """Override of the valiate() method.
154 
155  The pexConfigs of the subTasks here cannot be validated in the normal way, as they are the configs
156  for eotest, which does illegal things, and this would require an upstream PR to fix. Therefore, we
157  override the validate() method here, and use it to set the output directory for each of the tasks
158  based on the legal pexConfig parameter for the main task.
159  """
160  log = lsstLog.Log.getLogger("cp.pipe.cpTaskConfig")
161  if not self.eotestOutputPath:
162  raise RuntimeError("Must supply an output path for eotest data. "
163  "Please set config.eotestOutputPath.")
164 
165  taskList = ['fe55', 'brightPixels', 'darkPixels', 'readNoise', 'traps', 'cte', 'flatPair', 'ptc']
166  for task in taskList:
167  if getattr(self, task).output_dir != '.':
168  # Being thorough here: '.' is the eotest default. If this is not the value then the user has
169  # specified something, and we're going to clobber it, so raise a warning. Unlike to happen.
170  log.warn("OVERWRITING: Found a user defined output path of %s for %sTask. "
171  "This has been overwritten with %s, as individually specified output paths for "
172  "subTasks are not supported at present" % (getattr(self, task).output_dir,
173  task, self.eotestOutputPath))
174  getattr(self, task).output_dir = self.eotestOutputPath
175 
176 
177 class CpTask(pipeBase.CmdLineTask):
178  """
179  Calibration (Products) Production (CP) task.
180 
181  This task is used to produce the calibration products required to calibrate cameras.
182  Examples of such operations are as follows:
183  * Given a set of flat-field images, find the dark pixels and columns.
184  * Given a set of darks, find the bright pixels and columns.
185  * Given a set of Fe55 exposures, calulate the gain of the readout chain, in e-/ADU
186  * Given a set of Fe55 exposures, calulate the instrinsic PSF of the silicon, and the degradation of
187  * the PSF due to CTE.
188  * Given a set of flat-pairs, measure the photon transfer curve (PTC).
189  * Given a set of bias frames, calculate the read noise of the system in e-.
190  * Given a set of pocket-pumping exposures, find charge-traps in the silicon.
191 
192  The CpTask.runEotestDirect() is only applicable to LSST sensors, and only for a specific type of dataset
193  This method takes a dafPersistance.Butler corresponding to a repository in which a full eotest run has
194  been taken and ingested, and runs each of the tasks in eotest directly, allowing for bitwise comparison
195  with results given by the camera team.
196 
197  See http://ls.st/ldm-151 Chapter 4, Calibration Products Production for further details
198  regarding the inputs and outputs.
199  """
200 
201  ConfigClass = CpTaskConfig
202  _DefaultName = "cp"
203 
204  def __init__(self, *args, **kwargs):
205  """Constructor for the CpTask."""
206  if 'lsst.eotest.sensor' not in sys.modules: # check we have eotest before going further
207  raise RuntimeError('eotest failed to import')
208 
209  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
210 
211  # Note - we can't currently call validate on the subTask configs, as they are NOT valid
212  # due to state of eotest. However, we override validate() and call it here
213  # and use it to set the output dir config parameter in the subTasks.
214  self.config.validate()
215  self.config.freeze()
216 
217  self.makeSubtask("fe55")
218  self.makeSubtask("readNoise")
219  self.makeSubtask("brightPixels")
220  self.makeSubtask("darkPixels")
221  self.makeSubtask("traps")
222  self.makeSubtask("cte")
223  self.makeSubtask("flatPair")
224  self.makeSubtask("ptc")
225 
226  def _getMaskFiles(self, path, ccd):
227  """Get all available eotest mask files for a given ccd.
228 
229  Each stage of the processing generates more mask files, so this allows each to be picked up
230  as more and more tests run, and saves having to have clever logic for if some tasks fail.
231 
232  Parameters
233  ----------
234  path : `str`
235  Path on which to find the mask files
236  ccd : `string` or `int`
237  Name/identifier of the CCD
238 
239  Returns
240  -------
241  maskFiles : iterable of `str`
242  List of mask files, or an empty tuple if none are found
243  """
244  pattern = '*' + str(ccd) + '*mask*' # the cast to str supports obs_auxTel
245  maskFiles = glob.glob(os.path.join(path, pattern))
246  return maskFiles if len(maskFiles) > 0 else () # eotest wants an empty tuple here
247 
248  def _cleanupEotest(self, path):
249  """Delete all the medianed files left behind after eotest has run.
250 
251  Running eotest generates a lot of interim medianed files, so this just cleans them up.
252 
253  Parameters
254  ----------
255  path : `str`
256  Path on which to delete all the eotest medianed files.
257  """
258  for filename in glob.glob(os.path.join(path, '*_median_*.fits')):
259  os.remove(filename)
260 
261  def makeEotestReport(self, butler):
262  """After running eotest, generate pdf(s) of the results.
263 
264  Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD.
265  The pdf file(s), along with the .tex file(s) and the individual plots are written
266  to the eotestOutputPath.
267  .pdf generation requires a TeX distro including pdflatex to be installed.
268  """
269  ccds = butler.queryMetadata('raw', ['ccd'])
270  for ccd in ccds:
271  self.log.info("Starting test report generation for %s"%ccd)
272  try:
273  plotPath = os.path.join(self.config.eotestOutputPath, 'plots')
274  if not os.path.exists(plotPath):
275  os.makedirs(plotPath)
276  plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath)
277  eoTestReport = sensorTest.EOTestReport(plots, wl_dir='')
278  eoTestReport.make_figures()
279  eoTestReport.make_pdf()
280  except Exception as e:
281  self.log.warn("Failed to make eotest report for %s: %s"%(ccd, e))
282  self.log.info("Finished test report generation.")
283 
284  @pipeBase.timeMethod
285  def runEotestDirect(self, butler, run=None):
286  """
287  Generate calibration products using eotest algorithms.
288 
289  Generate all calibration products possible using the vanilla eotest implementation,
290  given a butler for a TS8 (raft-test) repo. It can contain multiple runs, but must correspond to
291  only a single raft/RTM.
292 
293  - Run all eotest tasks possible, using the butler to gather the data
294  - Write outputs in eotest format
295 
296  In order to replicate the canonical eotest analysis, the tasks should be run in a specific order.
297  This is given/defined in the "Steps" section here:
298  http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayProcess.jsp?processPath=1179
299 
300  But is replicated here for conveniece:
301  * 55Fe Analysis
302  * CCD Read Noise Analysis
303  * Bright Defects Analysis
304  * Dark Defects Analysis
305  * Traps Finding
306  * Dark Current X - will not be implemented here
307  * Charge Transfer Efficiencies
308  * Photo-response analysis X - will not be implemented here
309  * Flat Pairs Analysis
310  * Photon Transfer Curve
311  * Quantum Efficiency X - will not be implemented here
312 
313  List of tasks that exist in the eotest package but aren't mentioned on the above link:
314  * linearityTask()
315  * fe55CteTask()
316  * eperTask()
317  * crosstalkTask()
318  * persistenceTask()
319 
320  # TODO: For each eotest task, find out what the standard raft testing does for the optional params.
321  i.e. many have optional params for gains, bias-frames etc - if we want bitwise identicallity then we
322  need to know what is typically provided to these tasks when the camera team runs this code.
323  This can probably be worked out from https://github.com/lsst-camera-dh/lcatr-harness
324  but it sounds like Jim Chiang doesn't recommend trying to do that.
325  DM-12939
326 
327  Parameters
328  ----------
329  butler : `lsst.daf.persistence.butler`
330  Butler for the repo containg the eotest data to be used
331  run : `str` or `int`
332  Optional run number, to be used for repos containing multiple runs
333  """
334  self.log.info("Running eotest routines direct")
335 
336  # Input testing to check that run is in the repo
337  runs = butler.queryMetadata('raw', ['run'])
338  if run is None:
339  if len(runs) == 1:
340  run = runs[0]
341  else:
342  raise RuntimeError("Butler query found %s for runs. eotest datasets must have a run number,"
343  "and you must specify which run to use if a respoitory contains several."
344  % runs)
345  else:
346  run = str(run)
347  if run not in runs:
348  raise RuntimeError("Butler query found %s for runs, but the run specified (%s) "
349  "was not among them." % (runs, run))
350  del runs # we have run defined now, so remove this to avoid potential confusion later
351 
352  if not os.path.exists(self.config.eotestOutputPath):
353  os.makedirs(self.config.eotestOutputPath)
354 
355  ccds = butler.queryMetadata('raw', ['ccd'])
356  imTypes = butler.queryMetadata('raw', ['imageType'])
357  testTypes = butler.queryMetadata('raw', ['testType'])
358 
359 
362  if self.config.doFe55:
363  fe55TaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'FE55'}
364  self.log.info("Starting Fe55 pixel task")
365  for ccd in ccds:
366  if 'FE55' not in testTypes:
367  msg = "No Fe55 tests found. Available data: %s" % testTypes
368  if self.config.requireAllEOTests:
369  raise RuntimeError(msg)
370  else:
371  self.log.warn(msg + "\nSkipping Fe55 task")
372  break
373  fe55Filenames = [butler.get('raw_filename', dataId={'visit': visit,
374  'ccd': ccd})[0][:-3]
375  for visit in butler.queryMetadata('raw', ['visit'], dataId=fe55TaskDataId)]
376  self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(fe55Filenames)))
377  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
378  gains = self.fe55.run(sensor_id=ccd, infiles=fe55Filenames, mask_files=maskFiles)
379  # gainsPropSet = dafBase.PropertySet()
380  # for amp, gain in gains.items(): # there is no propSet.fromDict() method so make like this
381  # gainsPropSet.addDouble(str(amp), gain)
382  butler.put(gains, 'eotest_gain', dataId={'ccd': ccd, 'run': run})
383  del fe55TaskDataId
384 
385  # TODO: validate the results above, and/or change code to (be able to) always run
386  # over all files instead of stopping at the "required accuracy"
387  # This will require making changes to the eotest code.
388  # DM-12939
389 
390 
393  if self.config.doReadNoise:
394  # note that LCA-10103 defines the Fe55 bias frames as the ones to use here
395  self.log.info("Starting readNoise task")
396  noiseTaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'BIAS'}
397  for ccd in ccds:
398  if ('FE55' not in testTypes) or ('BIAS' not in imTypes):
399  msg = "Required data for readNoise unavailable. Available data:\
400  \ntestTypes: %s\nimageTypes: %s" % (testTypes, imTypes)
401  if self.config.requireAllEOTests:
402  raise RuntimeError(msg)
403  else:
404  self.log.warn(msg + "\nSkipping noise task")
405  noiseFilenames = [butler.get('raw_filename', dataId={'visit': visit,
406  'ccd': ccd})[0][:-3]
407  for visit in butler.queryMetadata('raw', ['visit'],
408  dataId=noiseTaskDataId)]
409  self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(noiseFilenames)))
410  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
411  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
412  self.readNoise.run(sensor_id=ccd, bias_files=noiseFilenames,
413  gains=gains, mask_files=maskFiles)
414  del noiseTaskDataId
415 
416 
419  if self.config.doBrightPixels:
420  self.log.info("Starting bright pixel task")
421  brightTaskDataId = {'run': run, 'testType': 'DARK', 'imageType': 'DARK'}
422  for ccd in ccds:
423  if 'DARK' not in testTypes:
424  msg = "No dark tests found. Available data: %s" % testTypes
425  if self.config.requireAllEOTests:
426  raise RuntimeError(msg)
427  else:
428  self.log.warn(msg + "\nSkipping bright pixel task")
429  break
430  darkFilenames = [butler.get('raw_filename', dataId={'visit': visit,
431  'ccd': ccd})[0][:-3]
432  for visit in butler.queryMetadata('raw', ['visit'],
433  dataId=brightTaskDataId)]
434  self.log.trace("BrightTask: Processing %s with %s files" % (ccd, len(darkFilenames)))
435  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
436  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
437  self.brightPixels.run(sensor_id=ccd, dark_files=darkFilenames,
438  mask_files=maskFiles, gains=gains)
439  del brightTaskDataId
440 
441 
444  if self.config.doDarkPixels:
445  self.log.info("Starting dark pixel task")
446  darkTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
447  for ccd in ccds:
448  if 'SFLAT_500' not in testTypes:
449  msg = "No superflats found. Available data: %s" % testTypes
450  if self.config.requireAllEOTests:
451  raise RuntimeError(msg)
452  else:
453  self.log.warn(msg + "\nSkipping dark pixel task")
454  break
455  sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
456  'ccd': ccd})[0][:-3]
457  for visit in butler.queryMetadata('raw', ['visit'],
458  dataId=darkTaskDataId)]
459  self.log.trace("DarkTask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
460  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
461  self.darkPixels.run(sensor_id=ccd, sflat_files=sflatFilenames, mask_files=maskFiles)
462  del darkTaskDataId
463 
464 
467  if self.config.doTraps:
468  self.log.info("Starting trap task")
469  trapTaskDataId = {'run': run, 'testType': 'TRAP', 'imageType': 'PPUMP'}
470  for ccd in ccds:
471  if ('TRAP' not in testTypes) and ('PPUMP' not in imTypes):
472  msg = "No pocket pumping exposures found. Available data: %s" % testTypes
473  if self.config.requireAllEOTests:
474  raise RuntimeError(msg)
475  else:
476  self.log.warn(msg + "\nSkipping trap task")
477  break
478  trapFilenames = [butler.get('raw_filename', dataId={'visit': visit,
479  'ccd': ccd})[0][:-3]
480  for visit in butler.queryMetadata('raw', ['visit'], dataId=trapTaskDataId)]
481  if len(trapFilenames) != 1: # eotest can't handle more than one
482  self.log.fatal("Trap Task: Found more than one ppump trap file: %s" % trapFilenames)
483  self.log.trace("Trap Task: Processing %s with %s files" % (ccd, len(trapFilenames)))
484  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
485  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
486  self.traps.run(sensor_id=ccd, pocket_pumped_file=trapFilenames[0],
487  mask_files=maskFiles, gains=gains)
488  del trapTaskDataId
489 
490 
493  if self.config.doCTE:
494  self.log.info("Starting CTE task")
495  cteTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
496  for ccd in ccds:
497  if 'SFLAT_500' not in testTypes:
498  msg = "No superflats found. Available data: %s" % testTypes
499  if self.config.requireAllEOTests:
500  raise RuntimeError(msg)
501  else:
502  self.log.warn(msg + "\nSkipping CTE task")
503  break
504  sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
505  'ccd': ccd})[0][:-3]
506  for visit in butler.queryMetadata('raw', ['visit'], dataId=cteTaskDataId)]
507  self.log.trace("CTETask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
508  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
509  self.cte.run(sensor_id=ccd, superflat_files=sflatFilenames, mask_files=maskFiles)
510  del cteTaskDataId
511 
512 
515  if self.config.doFlatPair:
516  self.log.info("Starting flatPair task")
517  flatPairDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
518  for ccd in ccds:
519  if 'FLAT' not in testTypes:
520  msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
521  if self.config.requireAllEOTests:
522  raise RuntimeError(msg)
523  else:
524  self.log.warn(msg + "\nSkipping flatPair task")
525  break
526  flatPairFilenames = [butler.get('raw_filename', dataId={'visit': visit,
527  'ccd': ccd})[0][:-3]
528  for visit in butler.queryMetadata('raw', ['visit'],
529  dataId=flatPairDataId)]
530  # Note that eotest needs the original filename as written by the test-stand data acquisition
531  # system, as that is the only place the flat pair-number is recorded, so we have to resolve
532  # sym-links and pass in the *original* paths/filenames here :(
533  # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
534  # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
535  # isn't totally thorough on rejecting the wrong types of data here)
536  # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
537  # by the butler instead of here. DM-12939
538  flatPairFilenames = [os.path.realpath(f) for f in flatPairFilenames if
539  os.path.realpath(f).find('flat1') != -1 or
540  os.path.realpath(f).find('flat2') != -1]
541  if not flatPairFilenames:
542  raise RuntimeError("No flatPair files found.")
543  self.log.trace("FlatPairTask: Processing %s with %s files" % (ccd, len(flatPairFilenames)))
544  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
545  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
546  self.flatPair.run(sensor_id=ccd, infiles=flatPairFilenames, mask_files=maskFiles,
547  gains=gains, max_pd_frac_dev=self.config.flatPairMaxPdFracDev)
548  del flatPairDataId
549 
550 
553  if self.config.doPTC:
554  self.log.info("Starting PTC task")
555  ptcDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
556  for ccd in ccds:
557  if 'FLAT' not in testTypes:
558  msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
559  if self.config.requireAllEOTests:
560  raise RuntimeError(msg)
561  else:
562  self.log.warn(msg + "\nSkipping PTC task")
563  break
564  ptcFilenames = [butler.get('raw_filename', dataId={'visit': visit,
565  'ccd': ccd})[0][:-3]
566  for visit in butler.queryMetadata('raw', ['visit'], dataId=ptcDataId)]
567  # Note that eotest needs the original filename as written by the test-stand data acquisition
568  # system, as that is the only place the flat pair-number is recorded, so we have to resolve
569  # sym-links and pass in the *original* paths/filenames here :(
570  # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
571  # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
572  # isn't totally thorough on rejecting the wrong types of data here)
573  # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
574  # by the butler instead of here. DM-12939
575  ptcFilenames = [os.path.realpath(f) for f in ptcFilenames if
576  os.path.realpath(f).find('flat1') != -1 or
577  os.path.realpath(f).find('flat2') != -1]
578  if not ptcFilenames:
579  raise RuntimeError("No flatPair files found")
580  self.log.trace("PTCTask: Processing %s with %s files" % (ccd, len(ptcFilenames)))
581  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
582  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
583  self.ptc.run(sensor_id=ccd, infiles=ptcFilenames, mask_files=maskFiles, gains=gains)
584  del ptcDataId
585 
586  self._cleanupEotest(self.config.eotestOutputPath)
587  self.log.info("Finished running EOTest")
def _getMaskFiles(self, path, ccd)
Definition: cpTask.py:226
def runEotestDirect(self, butler, run=None)
Definition: cpTask.py:285
def _cleanupEotest(self, path)
Definition: cpTask.py:248
def __init__(self, args, kwargs)
Definition: cpTask.py:204
def makeEotestReport(self, butler)
Definition: cpTask.py:261