lsst.cp.pipe  master-ga5d0207a7d+31
cpTask.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 """Calibration products production task code."""
25 from __future__ import absolute_import, division, print_function
26 
27 import os
28 import glob
29 import sys
30 
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.log as lsstLog
34 import lsst.eotest.sensor as sensorTest
35 
36 
37 class CpTaskConfig(pexConfig.Config):
38  """Config class for the calibration products production (CP) task."""
39 
40  fe55 = pexConfig.ConfigurableField(
41  target=sensorTest.Fe55Task,
42  doc="The Fe55 analysis task.",
43  )
44  doFe55 = pexConfig.Field(
45  dtype=bool,
46  doc="Measure gains using Fe55?",
47  default=True,
48  )
49  readNoise = pexConfig.ConfigurableField(
50  target=sensorTest.ReadNoiseTask,
51  doc="The read noise task.",
52  )
53  doReadNoise = pexConfig.Field(
54  dtype=bool,
55  doc="Measure the read-noise?",
56  default=True,
57  )
58  brightPixels = pexConfig.ConfigurableField(
59  target=sensorTest.BrightPixelsTask,
60  doc="The bright pixel/column finding task.",
61  )
62  doBrightPixels = pexConfig.Field(
63  dtype=bool,
64  doc="Find bright pixels?",
65  default=True,
66  )
67  darkPixels = pexConfig.ConfigurableField(
68  target=sensorTest.DarkPixelsTask,
69  doc="The dark pixel/column finding task.",
70  )
71  doDarkPixels = pexConfig.Field(
72  dtype=bool,
73  doc="Find dark pixels?",
74  default=True,
75  )
76  traps = pexConfig.ConfigurableField(
77  target=sensorTest.TrapTask,
78  doc="The trap-finding task.",
79  )
80  doTraps = pexConfig.Field(
81  dtype=bool,
82  doc="Find traps using pocket-pumping exposures?",
83  default=True,
84  )
85  cte = pexConfig.ConfigurableField(
86  target=sensorTest.CteTask,
87  doc="The CTE analysis task.",
88  )
89  doCTE = pexConfig.Field(
90  dtype=bool,
91  doc="Measure the charge transfer efficiency?",
92  default=True,
93  )
94  ptc = pexConfig.ConfigurableField(
95  target=sensorTest.PtcTask,
96  doc="The PTC analysis task.",
97  )
98  doPTC = pexConfig.Field(
99  dtype=bool,
100  doc="Measure the photon transfer curve?",
101  default=True,
102  )
103  flatPair = pexConfig.ConfigurableField(
104  target=sensorTest.FlatPairTask,
105  doc="The flat-pair analysis task.",
106  )
107  doFlatPair = pexConfig.Field(
108  dtype=bool,
109  doc="Measure the detector response vs incident flux using flat pairs?",
110  default=True,
111  )
112  eotestOutputPath = pexConfig.Field(
113  dtype=str,
114  doc="Path to which to write the eotest output results. Madatory runtime arg for running eotest.",
115  default='',
116  )
117  requireAllEOTests = pexConfig.Field(
118  dtype=bool,
119  doc="If True, all tests are required to be runnable, and will Raise if data is missing. If False, "
120  "processing will continue if a previous part failed due to the input dataset being incomplete.",
121  default=True,
122  )
123  flatPairMaxPdFracDev = pexConfig.Field(
124  dtype=float,
125  doc="Maximum allowed fractional deviation between photodiode currents for the eotest flatPair task. "
126  "This value is passed to the task's run() method at runtime rather than being stored in the task's"
127  "own pexConfig field.",
128  default=0.05,
129  )
130 
131  def setDefaults(self):
132  """Set default config options for the subTasks."""
133  # TODO: Set to proper values - DM-12939
134  self.fe55.temp_set_point = -100
135  self.fe55.temp_set_point_tol = 20
136 
137  # TODO: Set to proper values - DM-12939
138  self.readNoise.temp_set_point = -100
139  self.readNoise.temp_set_point_tol = 20
140 
141  # TODO: make this work - DM-12939
142  self.brightPixels.temp_set_point = -100
143  self.brightPixels.temp_set_point_tol = 20
144 
145  # TODO: find the proper settings for flatPairTask to work. This will mean either using the expTime,
146  # or working out what's wrong with the MONDIODE values (debug that anyway as a large difference might
147  # indicate something else going on). Note that this task doesn't really use much in its config class,
148  # but passes in things at runtime param to its run() method, hence putting in a slot for them here.
149  # DM-12939
150  self.flatPairMaxPdFracDev = 0.99
151 
152  def validate(self):
153  """Override of the valiate() method.
154 
155  The pexConfigs of the subTasks here cannot be validated in the normal way, as they are the configs
156  for eotest, which does illegal things, and this would require an upstream PR to fix. Therefore, we
157  override the validate() method here, and use it to set the output directory for each of the tasks
158  based on the legal pexConfig parameter for the main task.
159  """
160  log = lsstLog.Log.getLogger("cp.pipe.cpTaskConfig")
161  if not self.eotestOutputPath:
162  raise RuntimeError("Must supply an output path for eotest data. "
163  "Please set config.eotestOutputPath.")
164 
165  taskList = ['fe55', 'brightPixels', 'darkPixels', 'readNoise', 'traps', 'cte', 'flatPair', 'ptc']
166  for task in taskList:
167  if getattr(self, task).output_dir != '.':
168  # Being thorough here: '.' is the eotest default. If this is not the value then the user has
169  # specified something, and we're going to clobber it, so raise a warning. Unlike to happen.
170  log.warn("OVERWRITING: Found a user defined output path of %s for %sTask. "
171  "This has been overwritten with %s, as individually specified output paths for "
172  "subTasks are not supported at present" % (getattr(self, task).output_dir,
173  task, self.eotestOutputPath))
174  getattr(self, task).output_dir = self.eotestOutputPath
175 
176 
177 class CpTask(pipeBase.CmdLineTask):
178  """
179  Calibration (Products) Production (CP) task.
180 
181  This task is used to produce the calibration products required to calibrate cameras.
182  Examples of such operations are as follows:
183  * Given a set of flat-field images, find the dark pixels and columns.
184  * Given a set of darks, find the bright pixels and columns.
185  * Given a set of Fe55 exposures, calulate the gain of the readout chain, in e-/ADU
186  * Given a set of Fe55 exposures, calulate the instrinsic PSF of the silicon, and the degradation of
187  * the PSF due to CTE.
188  * Given a set of flat-pairs, measure the photon transfer curve (PTC).
189  * Given a set of bias frames, calculate the read noise of the system in e-.
190  * Given a set of pocket-pumping exposures, find charge-traps in the silicon.
191 
192  The CpTask.runEotestDirect() is only applicable to LSST sensors, and only for a specific type of dataset
193  This method takes a dafPersistance.Butler corresponding to a repository in which a full eotest run has
194  been taken and ingested, and runs each of the tasks in eotest directly, allowing for bitwise comparison
195  with results given by the camera team.
196 
197  See http://ls.st/ldm-151 Chapter 4, Calibration Products Production for further details
198  regarding the inputs and outputs.
199  """
200 
201  ConfigClass = CpTaskConfig
202  _DefaultName = "cp"
203 
204  def __init__(self, *args, **kwargs):
205  """Constructor for the CpTask."""
206  if 'lsst.eotest.sensor' not in sys.modules: # check we have eotest before going further
207  raise RuntimeError('eotest failed to import')
208 
209  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
210 
211  # Note - we can't currently call validate on the subTask configs, as they are NOT valid
212  # due to state of eotest. However, we override validate() and call it here
213  # and use it to set the output dir config parameter in the subTasks.
214  self.config.validate()
215  self.config.freeze()
216 
217  self.makeSubtask("fe55")
218  self.makeSubtask("readNoise")
219  self.makeSubtask("brightPixels")
220  self.makeSubtask("darkPixels")
221  self.makeSubtask("traps")
222  self.makeSubtask("cte")
223  self.makeSubtask("flatPair")
224  self.makeSubtask("ptc")
225 
226  def _getMaskFiles(self, path, ccd):
227  """Get all available eotest mask files for a given ccd.
228 
229  Each stage of the processing generates more mask files, so this allows each to be picked up
230  as more and more tests run, and saves having to have clever logic for if some tasks fail.
231 
232  Parameters
233  ----------
234  path : `str`
235  Path on which to find the mask files
236  ccd : `string` or `int`
237  Name/identifier of the CCD
238 
239  Returns
240  -------
241  maskFiles : iterable of `str`
242  List of mask files, or an empty tuple if none are found
243  """
244  pattern = '*' + str(ccd) + '*mask*' # the cast to str supports obs_auxTel
245  maskFiles = glob.glob(os.path.join(path, pattern))
246  return maskFiles if len(maskFiles) > 0 else () # eotest wants an empty tuple here
247 
248  def _cleanupEotest(self, path):
249  """Delete all the medianed files left behind after eotest has run.
250 
251  Running eotest generates a lot of interim medianed files, so this just cleans them up.
252 
253  Parameters
254  ----------
255  path : `str`
256  Path on which to delete all the eotest medianed files.
257  """
258  for filename in glob.glob(os.path.join(path, '*_median_*.fits')):
259  os.remove(filename)
260 
261  def makeEotestReport(self, butler):
262  """After running eotest, generate pdf(s) of the results.
263 
264  Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD.
265  The pdf file(s), along with the .tex file(s) and the individual plots are written
266  to the eotestOutputPath.
267  .pdf generation requires a TeX distro including pdflatex to be installed.
268  """
269  ccds = butler.queryMetadata('raw', ['ccd'])
270  for ccd in ccds:
271  self.log.info("Starting test report generation for %s"%ccd)
272  try:
273  plotPath = os.path.join(self.config.eotestOutputPath, 'plots')
274  if not os.path.exists(plotPath):
275  os.makedirs(plotPath)
276  plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath)
277  eoTestReport = sensorTest.EOTestReport(plots, wl_dir='')
278  eoTestReport.make_figures()
279  eoTestReport.make_pdf()
280  except Exception as e:
281  self.log.warn("Failed to make eotest report for %s: %s"%(ccd, e))
282  self.log.info("Finished test report generation.")
283 
284  @pipeBase.timeMethod
285  def runEotestDirect(self, butler, run=None):
286  """
287  Generate calibration products using eotest algorithms.
288 
289  Generate all calibration products possible using the vanilla eotest implementation,
290  given a butler for a TS8 (raft-test) repo. It can contain multiple runs, but must correspond to
291  only a single raft/RTM.
292 
293  - Run all eotest tasks possible, using the butler to gather the data
294  - Write outputs in eotest format
295 
296  In order to replicate the canonical eotest analysis, the tasks should be run in a specific order.
297  This is given/defined in the "Steps" section here:
298  http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayProcess.jsp?processPath=1179
299 
300  But is replicated here for conveniece:
301  * 55Fe Analysis
302  * CCD Read Noise Analysis
303  * Bright Defects Analysis
304  * Dark Defects Analysis
305  * Traps Finding
306  * Dark Current X - will not be implemented here
307  * Charge Transfer Efficiencies
308  * Photo-response analysis X - will not be implemented here
309  * Flat Pairs Analysis
310  * Photon Transfer Curve
311  * Quantum Efficiency X - will not be implemented here
312 
313  List of tasks that exist in the eotest package but aren't mentioned on the above link:
314  --------
315  # linearityTask()
316  # fe55CteTask()
317  # eperTask()
318  # crosstalkTask()
319  # persistenceTask()
320 
321  # TODO: For each eotest task, find out what the standard raft testing does for the optional params.
322  i.e. many have optional params for gains, bias-frames etc - if we want bitwise identicallity then we
323  need to know what is typically provided to these tasks when the camera team runs this code.
324  This can probably be worked out from https://github.com/lsst-camera-dh/lcatr-harness
325  but it sounds like Jim Chiang doesn't recommend trying to do that.
326  DM-12939
327 
328  Parameters
329  ----------
330  butler : `lsst.daf.persistence.butler`
331  Butler for the repo containg the eotest data to be used
332  run : `str` or `int`
333  Optional run number, to be used for repos containing multiple runs
334  """
335  self.log.info("Running eotest routines direct")
336 
337  # Input testing to check that run is in the repo
338  runs = butler.queryMetadata('raw', ['run'])
339  if run is None:
340  if len(runs) == 1:
341  run = runs[0]
342  else:
343  raise RuntimeError("Butler query found %s for runs. eotest datasets must have a run number,"
344  "and you must specify which run to use if a respoitory contains several."
345  % runs)
346  else:
347  run = str(run)
348  if run not in runs:
349  raise RuntimeError("Butler query found %s for runs, but the run specified (%s) "
350  "was not among them." % (runs, run))
351  del runs # we have run defined now, so remove this to avoid potential confusion later
352 
353  if not os.path.exists(self.config.eotestOutputPath):
354  os.makedirs(self.config.eotestOutputPath)
355 
356  ccds = butler.queryMetadata('raw', ['ccd'])
357  imTypes = butler.queryMetadata('raw', ['imageType'])
358  testTypes = butler.queryMetadata('raw', ['testType'])
359 
360 
363  if self.config.doFe55:
364  fe55TaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'FE55'}
365  self.log.info("Starting Fe55 pixel task")
366  for ccd in ccds:
367  if 'FE55' not in testTypes:
368  msg = "No Fe55 tests found. Available data: %s" % testTypes
369  if self.config.requireAllEOTests:
370  raise RuntimeError(msg)
371  else:
372  self.log.warn(msg + "\nSkipping Fe55 task")
373  break
374  fe55Filenames = [butler.get('raw_filename', dataId={'visit': visit,
375  'ccd': ccd})[0][:-3]
376  for visit in butler.queryMetadata('raw', ['visit'], dataId=fe55TaskDataId)]
377  self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(fe55Filenames)))
378  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
379  gains = self.fe55.run(sensor_id=ccd, infiles=fe55Filenames, mask_files=maskFiles)
380  # gainsPropSet = dafBase.PropertySet()
381  # for amp, gain in gains.items(): # there is no propSet.fromDict() method so make like this
382  # gainsPropSet.addDouble(str(amp), gain)
383  butler.put(gains, 'eotest_gain', dataId={'ccd': ccd, 'run': run})
384  del fe55TaskDataId
385 
386  # TODO: validate the results above, and/or change code to (be able to) always run
387  # over all files instead of stopping at the "required accuracy"
388  # This will require making changes to the eotest code.
389  # DM-12939
390 
391 
394  if self.config.doReadNoise:
395  # note that LCA-10103 defines the Fe55 bias frames as the ones to use here
396  self.log.info("Starting readNoise task")
397  noiseTaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'BIAS'}
398  for ccd in ccds:
399  if ('FE55' not in testTypes) or ('BIAS' not in imTypes):
400  msg = "Required data for readNoise unavailable. Available data:\
401  \ntestTypes: %s\nimageTypes: %s" % (testTypes, imTypes)
402  if self.config.requireAllEOTests:
403  raise RuntimeError(msg)
404  else:
405  self.log.warn(msg + "\nSkipping noise task")
406  noiseFilenames = [butler.get('raw_filename', dataId={'visit': visit,
407  'ccd': ccd})[0][:-3]
408  for visit in butler.queryMetadata('raw', ['visit'],
409  dataId=noiseTaskDataId)]
410  self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(noiseFilenames)))
411  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
412  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
413  self.readNoise.run(sensor_id=ccd, bias_files=noiseFilenames,
414  gains=gains, mask_files=maskFiles)
415  del noiseTaskDataId
416 
417 
420  if self.config.doBrightPixels:
421  self.log.info("Starting bright pixel task")
422  brightTaskDataId = {'run': run, 'testType': 'DARK', 'imageType': 'DARK'}
423  for ccd in ccds:
424  if 'DARK' not in testTypes:
425  msg = "No dark tests found. Available data: %s" % testTypes
426  if self.config.requireAllEOTests:
427  raise RuntimeError(msg)
428  else:
429  self.log.warn(msg + "\nSkipping bright pixel task")
430  break
431  darkFilenames = [butler.get('raw_filename', dataId={'visit': visit,
432  'ccd': ccd})[0][:-3]
433  for visit in butler.queryMetadata('raw', ['visit'],
434  dataId=brightTaskDataId)]
435  self.log.trace("BrightTask: Processing %s with %s files" % (ccd, len(darkFilenames)))
436  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
437  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
438  self.brightPixels.run(sensor_id=ccd, dark_files=darkFilenames,
439  mask_files=maskFiles, gains=gains)
440  del brightTaskDataId
441 
442 
445  if self.config.doDarkPixels:
446  self.log.info("Starting dark pixel task")
447  darkTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
448  for ccd in ccds:
449  if 'SFLAT_500' not in testTypes:
450  msg = "No superflats found. Available data: %s" % testTypes
451  if self.config.requireAllEOTests:
452  raise RuntimeError(msg)
453  else:
454  self.log.warn(msg + "\nSkipping dark pixel task")
455  break
456  sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
457  'ccd': ccd})[0][:-3]
458  for visit in butler.queryMetadata('raw', ['visit'],
459  dataId=darkTaskDataId)]
460  self.log.trace("DarkTask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
461  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
462  self.darkPixels.run(sensor_id=ccd, sflat_files=sflatFilenames, mask_files=maskFiles)
463  del darkTaskDataId
464 
465 
468  if self.config.doTraps:
469  self.log.info("Starting trap task")
470  trapTaskDataId = {'run': run, 'testType': 'TRAP', 'imageType': 'PPUMP'}
471  for ccd in ccds:
472  if ('TRAP' not in testTypes) and ('PPUMP' not in imTypes):
473  msg = "No pocket pumping exposures found. Available data: %s" % testTypes
474  if self.config.requireAllEOTests:
475  raise RuntimeError(msg)
476  else:
477  self.log.warn(msg + "\nSkipping trap task")
478  break
479  trapFilenames = [butler.get('raw_filename', dataId={'visit': visit,
480  'ccd': ccd})[0][:-3]
481  for visit in butler.queryMetadata('raw', ['visit'], dataId=trapTaskDataId)]
482  if len(trapFilenames) != 1: # eotest can't handle more than one
483  self.log.fatal("Trap Task: Found more than one ppump trap file: %s" % trapFilenames)
484  self.log.trace("Trap Task: Processing %s with %s files" % (ccd, len(trapFilenames)))
485  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
486  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
487  self.traps.run(sensor_id=ccd, pocket_pumped_file=trapFilenames[0],
488  mask_files=maskFiles, gains=gains)
489  del trapTaskDataId
490 
491 
494  if self.config.doCTE:
495  self.log.info("Starting CTE task")
496  cteTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
497  for ccd in ccds:
498  if 'SFLAT_500' not in testTypes:
499  msg = "No superflats found. Available data: %s" % testTypes
500  if self.config.requireAllEOTests:
501  raise RuntimeError(msg)
502  else:
503  self.log.warn(msg + "\nSkipping CTE task")
504  break
505  sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
506  'ccd': ccd})[0][:-3]
507  for visit in butler.queryMetadata('raw', ['visit'], dataId=cteTaskDataId)]
508  self.log.trace("CTETask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
509  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
510  self.cte.run(sensor_id=ccd, superflat_files=sflatFilenames, mask_files=maskFiles)
511  del cteTaskDataId
512 
513 
516  if self.config.doFlatPair:
517  self.log.info("Starting flatPair task")
518  flatPairDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
519  for ccd in ccds:
520  if 'FLAT' not in testTypes:
521  msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
522  if self.config.requireAllEOTests:
523  raise RuntimeError(msg)
524  else:
525  self.log.warn(msg + "\nSkipping flatPair task")
526  break
527  flatPairFilenames = [butler.get('raw_filename', dataId={'visit': visit,
528  'ccd': ccd})[0][:-3]
529  for visit in butler.queryMetadata('raw', ['visit'],
530  dataId=flatPairDataId)]
531  # Note that eotest needs the original filename as written by the test-stand data acquisition
532  # system, as that is the only place the flat pair-number is recorded, so we have to resolve
533  # sym-links and pass in the *original* paths/filenames here :(
534  # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
535  # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
536  # isn't totally thorough on rejecting the wrong types of data here)
537  # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
538  # by the butler instead of here. DM-12939
539  flatPairFilenames = [os.path.realpath(f) for f in flatPairFilenames if
540  os.path.realpath(f).find('flat1') != -1 or
541  os.path.realpath(f).find('flat2') != -1]
542  if not flatPairFilenames:
543  raise RuntimeError("No flatPair files found.")
544  self.log.trace("FlatPairTask: Processing %s with %s files" % (ccd, len(flatPairFilenames)))
545  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
546  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
547  self.flatPair.run(sensor_id=ccd, infiles=flatPairFilenames, mask_files=maskFiles,
548  gains=gains, max_pd_frac_dev=self.config.flatPairMaxPdFracDev)
549  del flatPairDataId
550 
551 
554  if self.config.doPTC:
555  self.log.info("Starting PTC task")
556  ptcDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
557  for ccd in ccds:
558  if 'FLAT' not in testTypes:
559  msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
560  if self.config.requireAllEOTests:
561  raise RuntimeError(msg)
562  else:
563  self.log.warn(msg + "\nSkipping PTC task")
564  break
565  ptcFilenames = [butler.get('raw_filename', dataId={'visit': visit,
566  'ccd': ccd})[0][:-3]
567  for visit in butler.queryMetadata('raw', ['visit'], dataId=ptcDataId)]
568  # Note that eotest needs the original filename as written by the test-stand data acquisition
569  # system, as that is the only place the flat pair-number is recorded, so we have to resolve
570  # sym-links and pass in the *original* paths/filenames here :(
571  # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
572  # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
573  # isn't totally thorough on rejecting the wrong types of data here)
574  # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
575  # by the butler instead of here. DM-12939
576  ptcFilenames = [os.path.realpath(f) for f in ptcFilenames if
577  os.path.realpath(f).find('flat1') != -1 or
578  os.path.realpath(f).find('flat2') != -1]
579  if not ptcFilenames:
580  raise RuntimeError("No flatPair files found")
581  self.log.trace("PTCTask: Processing %s with %s files" % (ccd, len(ptcFilenames)))
582  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
583  gains = butler.get('eotest_gain', dataId={'ccd': ccd, 'run': run})
584  self.ptc.run(sensor_id=ccd, infiles=ptcFilenames, mask_files=maskFiles, gains=gains)
585  del ptcDataId
586 
587  self._cleanupEotest(self.config.eotestOutputPath)
588  self.log.info("Finished running EOTest")
def _getMaskFiles(self, path, ccd)
Definition: cpTask.py:226
def runEotestDirect(self, butler, run=None)
Definition: cpTask.py:285
def _cleanupEotest(self, path)
Definition: cpTask.py:248
def __init__(self, args, kwargs)
Definition: cpTask.py:204
def makeEotestReport(self, butler)
Definition: cpTask.py:261