lsst.cp.pipe  16.0-3-g324faa9+1
cpTask.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 """Calibration products production task code."""
25 from __future__ import absolute_import, division, print_function
26 
27 import os
28 import glob
29 import sys
30 
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 import lsst.log as lsstLog
34 import lsst.eotest.sensor as sensorTest
35 
36 
37 class CpTaskConfig(pexConfig.Config):
38  """Config class for the calibration products production (CP) task."""
39 
40  ccdKey = pexConfig.Field(
41  dtype=str,
42  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
43  default='ccd',
44  )
45  fe55 = pexConfig.ConfigurableField(
46  target=sensorTest.Fe55Task,
47  doc="The Fe55 analysis task.",
48  )
49  doFe55 = pexConfig.Field(
50  dtype=bool,
51  doc="Measure gains using Fe55?",
52  default=True,
53  )
54  readNoise = pexConfig.ConfigurableField(
55  target=sensorTest.ReadNoiseTask,
56  doc="The read noise task.",
57  )
58  doReadNoise = pexConfig.Field(
59  dtype=bool,
60  doc="Measure the read-noise?",
61  default=True,
62  )
63  brightPixels = pexConfig.ConfigurableField(
64  target=sensorTest.BrightPixelsTask,
65  doc="The bright pixel/column finding task.",
66  )
67  doBrightPixels = pexConfig.Field(
68  dtype=bool,
69  doc="Find bright pixels?",
70  default=True,
71  )
72  darkPixels = pexConfig.ConfigurableField(
73  target=sensorTest.DarkPixelsTask,
74  doc="The dark pixel/column finding task.",
75  )
76  doDarkPixels = pexConfig.Field(
77  dtype=bool,
78  doc="Find dark pixels?",
79  default=True,
80  )
81  traps = pexConfig.ConfigurableField(
82  target=sensorTest.TrapTask,
83  doc="The trap-finding task.",
84  )
85  doTraps = pexConfig.Field(
86  dtype=bool,
87  doc="Find traps using pocket-pumping exposures?",
88  default=True,
89  )
90  cte = pexConfig.ConfigurableField(
91  target=sensorTest.CteTask,
92  doc="The CTE analysis task.",
93  )
94  doCTE = pexConfig.Field(
95  dtype=bool,
96  doc="Measure the charge transfer efficiency?",
97  default=True,
98  )
99  ptc = pexConfig.ConfigurableField(
100  target=sensorTest.PtcTask,
101  doc="The PTC analysis task.",
102  )
103  doPTC = pexConfig.Field(
104  dtype=bool,
105  doc="Measure the photon transfer curve?",
106  default=True,
107  )
108  flatPair = pexConfig.ConfigurableField(
109  target=sensorTest.FlatPairTask,
110  doc="The flat-pair analysis task.",
111  )
112  doFlatPair = pexConfig.Field(
113  dtype=bool,
114  doc="Measure the detector response vs incident flux using flat pairs?",
115  default=True,
116  )
117  eotestOutputPath = pexConfig.Field(
118  dtype=str,
119  doc="Path to which to write the eotest output results. Madatory runtime arg for running eotest.",
120  default='',
121  )
122  requireAllEOTests = pexConfig.Field(
123  dtype=bool,
124  doc="If True, all tests are required to be runnable, and will Raise if data is missing. If False, "
125  "processing will continue if a previous part failed due to the input dataset being incomplete.",
126  default=True,
127  )
128  flatPairMaxPdFracDev = pexConfig.Field(
129  dtype=float,
130  doc="Maximum allowed fractional deviation between photodiode currents for the eotest flatPair task. "
131  "This value is passed to the task's run() method at runtime rather than being stored in the task's"
132  "own pexConfig field.",
133  default=0.05,
134  )
135 
136  def setDefaults(self):
137  """Set default config options for the subTasks."""
138  # TODO: Set to proper values - DM-12939
139  self.fe55.temp_set_point = -100
140  self.fe55.temp_set_point_tol = 20
141 
142  # TODO: Set to proper values - DM-12939
143  self.readNoise.temp_set_point = -100
144  self.readNoise.temp_set_point_tol = 20
145 
146  # TODO: make this work - DM-12939
147  self.brightPixels.temp_set_point = -100
148  self.brightPixels.temp_set_point_tol = 20
149 
150  # TODO: find the proper settings for flatPairTask to work. This will mean either using the expTime,
151  # or working out what's wrong with the MONDIODE values (debug that anyway as a large difference might
152  # indicate something else going on). Note that this task doesn't really use much in its config class,
153  # but passes in things at runtime param to its run() method, hence putting in a slot for them here.
154  # DM-12939
155  self.flatPairMaxPdFracDev = 0.99
156 
157  def validate(self):
158  """Override of the valiate() method.
159 
160  The pexConfigs of the subTasks here cannot be validated in the normal way, as they are the configs
161  for eotest, which does illegal things, and this would require an upstream PR to fix. Therefore, we
162  override the validate() method here, and use it to set the output directory for each of the tasks
163  based on the legal pexConfig parameter for the main task.
164  """
165  log = lsstLog.Log.getLogger("cp.pipe.cpTaskConfig")
166  if not self.eotestOutputPath:
167  raise RuntimeError("Must supply an output path for eotest data. "
168  "Please set config.eotestOutputPath.")
169 
170  taskList = ['fe55', 'brightPixels', 'darkPixels', 'readNoise', 'traps', 'cte', 'flatPair', 'ptc']
171  for task in taskList:
172  if getattr(self, task).output_dir != '.':
173  # Being thorough here: '.' is the eotest default. If this is not the value then the user has
174  # specified something, and we're going to clobber it, so raise a warning. Unlike to happen.
175  log.warn("OVERWRITING: Found a user defined output path of %s for %sTask. "
176  "This has been overwritten with %s, as individually specified output paths for "
177  "subTasks are not supported at present" % (getattr(self, task).output_dir,
178  task, self.eotestOutputPath))
179  getattr(self, task).output_dir = self.eotestOutputPath
180 
181 
182 class CpTask(pipeBase.CmdLineTask):
183  """
184  Calibration (Products) Production (CP) task.
185 
186  This task is used to produce the calibration products required to calibrate cameras.
187  Examples of such operations are as follows:
188  * Given a set of flat-field images, find the dark pixels and columns.
189  * Given a set of darks, find the bright pixels and columns.
190  * Given a set of Fe55 exposures, calulate the gain of the readout chain, in e-/ADU
191  * Given a set of Fe55 exposures, calulate the instrinsic PSF of the silicon, and the degradation of
192  * the PSF due to CTE.
193  * Given a set of flat-pairs, measure the photon transfer curve (PTC).
194  * Given a set of bias frames, calculate the read noise of the system in e-.
195  * Given a set of pocket-pumping exposures, find charge-traps in the silicon.
196 
197  The CpTask.runEotestDirect() is only applicable to LSST sensors, and only for a specific type of dataset
198  This method takes a dafPersistance.Butler corresponding to a repository in which a full eotest run has
199  been taken and ingested, and runs each of the tasks in eotest directly, allowing for bitwise comparison
200  with results given by the camera team.
201 
202  See http://ls.st/ldm-151 Chapter 4, Calibration Products Production for further details
203  regarding the inputs and outputs.
204  """
205 
206  ConfigClass = CpTaskConfig
207  _DefaultName = "cp"
208 
209  def __init__(self, *args, **kwargs):
210  """Constructor for the CpTask."""
211  if 'lsst.eotest.sensor' not in sys.modules: # check we have eotest before going further
212  raise RuntimeError('eotest failed to import')
213 
214  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
215 
216  # Note - we can't currently call validate on the subTask configs, as they are NOT valid
217  # due to state of eotest. However, we override validate() and call it here
218  # and use it to set the output dir config parameter in the subTasks.
219  self.config.validate()
220  self.config.freeze()
221 
222  self.makeSubtask("fe55")
223  self.makeSubtask("readNoise")
224  self.makeSubtask("brightPixels")
225  self.makeSubtask("darkPixels")
226  self.makeSubtask("traps")
227  self.makeSubtask("cte")
228  self.makeSubtask("flatPair")
229  self.makeSubtask("ptc")
230 
231  def _getMaskFiles(self, path, ccd):
232  """Get all available eotest mask files for a given ccd.
233 
234  Each stage of the processing generates more mask files, so this allows each to be picked up
235  as more and more tests run, and saves having to have clever logic for if some tasks fail.
236 
237  Parameters
238  ----------
239  path : `str`
240  Path on which to find the mask files
241  ccd : `string` or `int`
242  Name/identifier of the CCD
243 
244  Returns
245  -------
246  maskFiles : iterable of `str`
247  List of mask files, or an empty tuple if none are found
248  """
249  pattern = '*' + str(ccd) + '*mask*' # the cast to str supports obs_auxTel
250  maskFiles = glob.glob(os.path.join(path, pattern))
251  return maskFiles if len(maskFiles) > 0 else () # eotest wants an empty tuple here
252 
253  def _cleanupEotest(self, path):
254  """Delete all the medianed files left behind after eotest has run.
255 
256  Running eotest generates a lot of interim medianed files, so this just cleans them up.
257 
258  Parameters
259  ----------
260  path : `str`
261  Path on which to delete all the eotest medianed files.
262  """
263  for filename in glob.glob(os.path.join(path, '*_median_*.fits')):
264  os.remove(filename)
265 
266  def makeEotestReport(self, butler):
267  """After running eotest, generate pdf(s) of the results.
268 
269  Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD.
270  The pdf file(s), along with the .tex file(s) and the individual plots are written
271  to the eotestOutputPath.
272  .pdf generation requires a TeX distro including pdflatex to be installed.
273  """
274  ccds = butler.queryMetadata('raw', self.config.ccdKey)
275  for ccd in ccds:
276  self.log.info("Starting test report generation for %s"%ccd)
277  try:
278  plotPath = os.path.join(self.config.eotestOutputPath, 'plots')
279  if not os.path.exists(plotPath):
280  os.makedirs(plotPath)
281  plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath)
282  eoTestReport = sensorTest.EOTestReport(plots, wl_dir='')
283  eoTestReport.make_figures()
284  eoTestReport.make_pdf()
285  except Exception as e:
286  self.log.warn("Failed to make eotest report for %s: %s"%(ccd, e))
287  self.log.info("Finished test report generation.")
288 
289  @pipeBase.timeMethod
290  def runEotestDirect(self, butler, run=None):
291  """
292  Generate calibration products using eotest algorithms.
293 
294  Generate all calibration products possible using the vanilla eotest implementation,
295  given a butler for a TS8 (raft-test) repo. It can contain multiple runs, but must correspond to
296  only a single raft/RTM.
297 
298  - Run all eotest tasks possible, using the butler to gather the data
299  - Write outputs in eotest format
300 
301  In order to replicate the canonical eotest analysis, the tasks should be run in a specific order.
302  This is given/defined in the "Steps" section here:
303  http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayProcess.jsp?processPath=1179
304 
305  But is replicated here for conveniece:
306  * 55Fe Analysis
307  * CCD Read Noise Analysis
308  * Bright Defects Analysis
309  * Dark Defects Analysis
310  * Traps Finding
311  * Dark Current X - will not be implemented here
312  * Charge Transfer Efficiencies
313  * Photo-response analysis X - will not be implemented here
314  * Flat Pairs Analysis
315  * Photon Transfer Curve
316  * Quantum Efficiency X - will not be implemented here
317 
318  List of tasks that exist in the eotest package but aren't mentioned on the above link:
319  * linearityTask()
320  * fe55CteTask()
321  * eperTask()
322  * crosstalkTask()
323  * persistenceTask()
324 
325  # TODO: For each eotest task, find out what the standard raft testing does for the optional params.
326  i.e. many have optional params for gains, bias-frames etc - if we want bitwise identicallity then we
327  need to know what is typically provided to these tasks when the camera team runs this code.
328  This can probably be worked out from https://github.com/lsst-camera-dh/lcatr-harness
329  but it sounds like Jim Chiang doesn't recommend trying to do that.
330  DM-12939
331 
332  Parameters
333  ----------
334  butler : `lsst.daf.persistence.butler`
335  Butler for the repo containg the eotest data to be used
336  run : `str` or `int`
337  Optional run number, to be used for repos containing multiple runs
338  """
339  self.log.info("Running eotest routines direct")
340 
341  # Input testing to check that run is in the repo
342  runs = butler.queryMetadata('raw', ['run'])
343  if run is None:
344  if len(runs) == 1:
345  run = runs[0]
346  else:
347  raise RuntimeError("Butler query found %s for runs. eotest datasets must have a run number,"
348  "and you must specify which run to use if a respoitory contains several."
349  % runs)
350  else:
351  run = str(run)
352  if run not in runs:
353  raise RuntimeError("Butler query found %s for runs, but the run specified (%s) "
354  "was not among them." % (runs, run))
355  del runs # we have run defined now, so remove this to avoid potential confusion later
356 
357  if not os.path.exists(self.config.eotestOutputPath):
358  os.makedirs(self.config.eotestOutputPath)
359 
360  ccds = butler.queryMetadata('raw', self.config.ccdKey)
361  imTypes = butler.queryMetadata('raw', ['imageType'])
362  testTypes = butler.queryMetadata('raw', ['testType'])
363 
364 
367  if self.config.doFe55:
368  fe55TaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'FE55'}
369  self.log.info("Starting Fe55 pixel task")
370  for ccd in ccds:
371  if 'FE55' not in testTypes:
372  msg = "No Fe55 tests found. Available data: %s" % testTypes
373  if self.config.requireAllEOTests:
374  raise RuntimeError(msg)
375  else:
376  self.log.warn(msg + "\nSkipping Fe55 task")
377  break
378  fe55Filenames = [butler.get('raw_filename', dataId={'visit': visit,
379  self.config.ccdKey: ccd})[0][:-3]
380  for visit in butler.queryMetadata('raw', ['visit'], dataId=fe55TaskDataId)]
381  self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(fe55Filenames)))
382  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
383  gains = self.fe55.run(sensor_id=ccd, infiles=fe55Filenames, mask_files=maskFiles)
384  # gainsPropSet = dafBase.PropertySet()
385  # for amp, gain in gains.items(): # there is no propSet.fromDict() method so make like this
386  # gainsPropSet.addDouble(str(amp), gain)
387  butler.put(gains, 'eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
388  del fe55TaskDataId
389 
390  # TODO: validate the results above, and/or change code to (be able to) always run
391  # over all files instead of stopping at the "required accuracy"
392  # This will require making changes to the eotest code.
393  # DM-12939
394 
395 
398  if self.config.doReadNoise:
399  # note that LCA-10103 defines the Fe55 bias frames as the ones to use here
400  self.log.info("Starting readNoise task")
401  noiseTaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'BIAS'}
402  for ccd in ccds:
403  if ('FE55' not in testTypes) or ('BIAS' not in imTypes):
404  msg = "Required data for readNoise unavailable. Available data:\
405  \ntestTypes: %s\nimageTypes: %s" % (testTypes, imTypes)
406  if self.config.requireAllEOTests:
407  raise RuntimeError(msg)
408  else:
409  self.log.warn(msg + "\nSkipping noise task")
410  noiseFilenames = [butler.get('raw_filename', dataId={'visit': visit,
411  self.config.ccdKey: ccd})[0][:-3]
412  for visit in butler.queryMetadata('raw', ['visit'],
413  dataId=noiseTaskDataId)]
414  self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(noiseFilenames)))
415  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
416  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
417  self.readNoise.run(sensor_id=ccd, bias_files=noiseFilenames,
418  gains=gains, mask_files=maskFiles)
419  del noiseTaskDataId
420 
421 
424  if self.config.doBrightPixels:
425  self.log.info("Starting bright pixel task")
426  brightTaskDataId = {'run': run, 'testType': 'DARK', 'imageType': 'DARK'}
427  for ccd in ccds:
428  if 'DARK' not in testTypes:
429  msg = "No dark tests found. Available data: %s" % testTypes
430  if self.config.requireAllEOTests:
431  raise RuntimeError(msg)
432  else:
433  self.log.warn(msg + "\nSkipping bright pixel task")
434  break
435  darkFilenames = [butler.get('raw_filename', dataId={'visit': visit,
436  self.config.ccdKey: ccd})[0][:-3]
437  for visit in butler.queryMetadata('raw', ['visit'],
438  dataId=brightTaskDataId)]
439  self.log.trace("BrightTask: Processing %s with %s files" % (ccd, len(darkFilenames)))
440  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
441  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
442  self.brightPixels.run(sensor_id=ccd, dark_files=darkFilenames,
443  mask_files=maskFiles, gains=gains)
444  del brightTaskDataId
445 
446 
449  if self.config.doDarkPixels:
450  self.log.info("Starting dark pixel task")
451  darkTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
452  for ccd in ccds:
453  if 'SFLAT_500' not in testTypes:
454  msg = "No superflats found. Available data: %s" % testTypes
455  if self.config.requireAllEOTests:
456  raise RuntimeError(msg)
457  else:
458  self.log.warn(msg + "\nSkipping dark pixel task")
459  break
460  sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
461  self.config.ccdKey: ccd})[0][:-3]
462  for visit in butler.queryMetadata('raw', ['visit'],
463  dataId=darkTaskDataId)]
464  self.log.trace("DarkTask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
465  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
466  self.darkPixels.run(sensor_id=ccd, sflat_files=sflatFilenames, mask_files=maskFiles)
467  del darkTaskDataId
468 
469 
472  if self.config.doTraps:
473  self.log.info("Starting trap task")
474  trapTaskDataId = {'run': run, 'testType': 'TRAP', 'imageType': 'PPUMP'}
475  for ccd in ccds:
476  if ('TRAP' not in testTypes) and ('PPUMP' not in imTypes):
477  msg = "No pocket pumping exposures found. Available data: %s" % testTypes
478  if self.config.requireAllEOTests:
479  raise RuntimeError(msg)
480  else:
481  self.log.warn(msg + "\nSkipping trap task")
482  break
483  trapFilenames = [butler.get('raw_filename', dataId={'visit': visit,
484  self.config.ccdKey: ccd})[0][:-3]
485  for visit in butler.queryMetadata('raw', ['visit'], dataId=trapTaskDataId)]
486  if len(trapFilenames) != 1: # eotest can't handle more than one
487  msg = "Trap Task: Found more than one ppump trap file: %s" % trapFilenames
488  msg += " Running using only the first one found."
489  self.log.warn(msg)
490  self.log.trace("Trap Task: Processing %s with %s files" % (ccd, len(trapFilenames)))
491  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
492  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
493  self.traps.run(sensor_id=ccd, pocket_pumped_file=trapFilenames[0],
494  mask_files=maskFiles, gains=gains)
495  del trapTaskDataId
496 
497 
500  if self.config.doCTE:
501  self.log.info("Starting CTE task")
502  cteTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'}
503  for ccd in ccds:
504  if 'SFLAT_500' not in testTypes:
505  msg = "No superflats found. Available data: %s" % testTypes
506  if self.config.requireAllEOTests:
507  raise RuntimeError(msg)
508  else:
509  self.log.warn(msg + "\nSkipping CTE task")
510  break
511  sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit,
512  self.config.ccdKey: ccd})[0][:-3]
513  for visit in butler.queryMetadata('raw', ['visit'], dataId=cteTaskDataId)]
514  self.log.trace("CTETask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
515  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
516  self.cte.run(sensor_id=ccd, superflat_files=sflatFilenames, mask_files=maskFiles)
517  del cteTaskDataId
518 
519 
522  if self.config.doFlatPair:
523  self.log.info("Starting flatPair task")
524  flatPairDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
525  for ccd in ccds:
526  if 'FLAT' not in testTypes:
527  msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
528  if self.config.requireAllEOTests:
529  raise RuntimeError(msg)
530  else:
531  self.log.warn(msg + "\nSkipping flatPair task")
532  break
533  flatPairFilenames = [butler.get('raw_filename', dataId={'visit': visit,
534  self.config.ccdKey: ccd})[0][:-3]
535  for visit in butler.queryMetadata('raw', ['visit'],
536  dataId=flatPairDataId)]
537  # Note that eotest needs the original filename as written by the test-stand data acquisition
538  # system, as that is the only place the flat pair-number is recorded, so we have to resolve
539  # sym-links and pass in the *original* paths/filenames here :(
540  # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
541  # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
542  # isn't totally thorough on rejecting the wrong types of data here)
543  # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
544  # by the butler instead of here. DM-12939
545  flatPairFilenames = [os.path.realpath(f) for f in flatPairFilenames if
546  os.path.realpath(f).find('flat1') != -1 or
547  os.path.realpath(f).find('flat2') != -1]
548  if not flatPairFilenames:
549  raise RuntimeError("No flatPair files found.")
550  self.log.trace("FlatPairTask: Processing %s with %s files" % (ccd, len(flatPairFilenames)))
551  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
552  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
553  self.flatPair.run(sensor_id=ccd, infiles=flatPairFilenames, mask_files=maskFiles,
554  gains=gains, max_pd_frac_dev=self.config.flatPairMaxPdFracDev)
555  del flatPairDataId
556 
557 
560  if self.config.doPTC:
561  self.log.info("Starting PTC task")
562  ptcDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'}
563  for ccd in ccds:
564  if 'FLAT' not in testTypes:
565  msg = "No dataset for flat_pairs found. Available data: %s" % testTypes
566  if self.config.requireAllEOTests:
567  raise RuntimeError(msg)
568  else:
569  self.log.warn(msg + "\nSkipping PTC task")
570  break
571  ptcFilenames = [butler.get('raw_filename', dataId={'visit': visit,
572  self.config.ccdKey: ccd})[0][:-3]
573  for visit in butler.queryMetadata('raw', ['visit'], dataId=ptcDataId)]
574  # Note that eotest needs the original filename as written by the test-stand data acquisition
575  # system, as that is the only place the flat pair-number is recorded, so we have to resolve
576  # sym-links and pass in the *original* paths/filenames here :(
577  # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here
578  # so we need to filter these for only the pair acquisitions (as the eotest code looks like it
579  # isn't totally thorough on rejecting the wrong types of data here)
580  # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done
581  # by the butler instead of here. DM-12939
582  ptcFilenames = [os.path.realpath(f) for f in ptcFilenames if
583  os.path.realpath(f).find('flat1') != -1 or
584  os.path.realpath(f).find('flat2') != -1]
585  if not ptcFilenames:
586  raise RuntimeError("No flatPair files found")
587  self.log.trace("PTCTask: Processing %s with %s files" % (ccd, len(ptcFilenames)))
588  maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd)
589  gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run})
590  self.ptc.run(sensor_id=ccd, infiles=ptcFilenames, mask_files=maskFiles, gains=gains)
591  del ptcDataId
592 
593  self._cleanupEotest(self.config.eotestOutputPath)
594  self.log.info("Finished running EOTest")
def _getMaskFiles(self, path, ccd)
Definition: cpTask.py:231
def runEotestDirect(self, butler, run=None)
Definition: cpTask.py:290
def _cleanupEotest(self, path)
Definition: cpTask.py:253
def __init__(self, args, kwargs)
Definition: cpTask.py:209
def makeEotestReport(self, butler)
Definition: cpTask.py:266