24 """Calibration products production task code.""" 25 from __future__
import absolute_import, division, print_function
34 import lsst.eotest.sensor
as sensorTest
38 """Config class for the calibration products production (CP) task.""" 40 ccdKey = pexConfig.Field(
42 doc=
"The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
45 fe55 = pexConfig.ConfigurableField(
46 target=sensorTest.Fe55Task,
47 doc=
"The Fe55 analysis task.",
49 doFe55 = pexConfig.Field(
51 doc=
"Measure gains using Fe55?",
54 readNoise = pexConfig.ConfigurableField(
55 target=sensorTest.ReadNoiseTask,
56 doc=
"The read noise task.",
58 doReadNoise = pexConfig.Field(
60 doc=
"Measure the read-noise?",
63 brightPixels = pexConfig.ConfigurableField(
64 target=sensorTest.BrightPixelsTask,
65 doc=
"The bright pixel/column finding task.",
67 doBrightPixels = pexConfig.Field(
69 doc=
"Find bright pixels?",
72 darkPixels = pexConfig.ConfigurableField(
73 target=sensorTest.DarkPixelsTask,
74 doc=
"The dark pixel/column finding task.",
76 doDarkPixels = pexConfig.Field(
78 doc=
"Find dark pixels?",
81 traps = pexConfig.ConfigurableField(
82 target=sensorTest.TrapTask,
83 doc=
"The trap-finding task.",
85 doTraps = pexConfig.Field(
87 doc=
"Find traps using pocket-pumping exposures?",
90 cte = pexConfig.ConfigurableField(
91 target=sensorTest.CteTask,
92 doc=
"The CTE analysis task.",
94 doCTE = pexConfig.Field(
96 doc=
"Measure the charge transfer efficiency?",
99 ptc = pexConfig.ConfigurableField(
100 target=sensorTest.PtcTask,
101 doc=
"The PTC analysis task.",
103 doPTC = pexConfig.Field(
105 doc=
"Measure the photon transfer curve?",
108 flatPair = pexConfig.ConfigurableField(
109 target=sensorTest.FlatPairTask,
110 doc=
"The flat-pair analysis task.",
112 doFlatPair = pexConfig.Field(
114 doc=
"Measure the detector response vs incident flux using flat pairs?",
117 eotestOutputPath = pexConfig.Field(
119 doc=
"Path to which to write the eotest output results. Madatory runtime arg for running eotest.",
122 requireAllEOTests = pexConfig.Field(
124 doc=
"If True, all tests are required to be runnable, and will Raise if data is missing. If False, " 125 "processing will continue if a previous part failed due to the input dataset being incomplete.",
128 flatPairMaxPdFracDev = pexConfig.Field(
130 doc=
"Maximum allowed fractional deviation between photodiode currents for the eotest flatPair task. " 131 "This value is passed to the task's run() method at runtime rather than being stored in the task's" 132 "own pexConfig field.",
137 """Set default config options for the subTasks.""" 139 self.
fe55.temp_set_point = -100
140 self.
fe55.temp_set_point_tol = 20
158 """Override of the valiate() method. 160 The pexConfigs of the subTasks here cannot be validated in the normal way, as they are the configs 161 for eotest, which does illegal things, and this would require an upstream PR to fix. Therefore, we 162 override the validate() method here, and use it to set the output directory for each of the tasks 163 based on the legal pexConfig parameter for the main task. 165 log = lsstLog.Log.getLogger(
"cp.pipe.cpTaskConfig")
167 raise RuntimeError(
"Must supply an output path for eotest data. " 168 "Please set config.eotestOutputPath.")
170 taskList = [
'fe55',
'brightPixels',
'darkPixels',
'readNoise',
'traps',
'cte',
'flatPair',
'ptc']
171 for task
in taskList:
172 if getattr(self, task).output_dir !=
'.':
175 log.warn(
"OVERWRITING: Found a user defined output path of %s for %sTask. " 176 "This has been overwritten with %s, as individually specified output paths for " 177 "subTasks are not supported at present" % (getattr(self, task).output_dir,
184 Calibration (Products) Production (CP) task. 186 This task is used to produce the calibration products required to calibrate cameras. 187 Examples of such operations are as follows: 188 * Given a set of flat-field images, find the dark pixels and columns. 189 * Given a set of darks, find the bright pixels and columns. 190 * Given a set of Fe55 exposures, calulate the gain of the readout chain, in e-/ADU 191 * Given a set of Fe55 exposures, calulate the instrinsic PSF of the silicon, and the degradation of 192 * the PSF due to CTE. 193 * Given a set of flat-pairs, measure the photon transfer curve (PTC). 194 * Given a set of bias frames, calculate the read noise of the system in e-. 195 * Given a set of pocket-pumping exposures, find charge-traps in the silicon. 197 The CpTask.runEotestDirect() is only applicable to LSST sensors, and only for a specific type of dataset 198 This method takes a dafPersistance.Butler corresponding to a repository in which a full eotest run has 199 been taken and ingested, and runs each of the tasks in eotest directly, allowing for bitwise comparison 200 with results given by the camera team. 202 See http://ls.st/ldm-151 Chapter 4, Calibration Products Production for further details 203 regarding the inputs and outputs. 206 ConfigClass = CpTaskConfig
210 """Constructor for the CpTask.""" 211 if 'lsst.eotest.sensor' not in sys.modules:
212 raise RuntimeError(
'eotest failed to import')
214 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
219 self.config.validate()
222 self.makeSubtask(
"fe55")
223 self.makeSubtask(
"readNoise")
224 self.makeSubtask(
"brightPixels")
225 self.makeSubtask(
"darkPixels")
226 self.makeSubtask(
"traps")
227 self.makeSubtask(
"cte")
228 self.makeSubtask(
"flatPair")
229 self.makeSubtask(
"ptc")
231 def _getMaskFiles(self, path, ccd):
232 """Get all available eotest mask files for a given ccd. 234 Each stage of the processing generates more mask files, so this allows each to be picked up 235 as more and more tests run, and saves having to have clever logic for if some tasks fail. 240 Path on which to find the mask files 241 ccd : `string` or `int` 242 Name/identifier of the CCD 246 maskFiles : iterable of `str` 247 List of mask files, or an empty tuple if none are found 249 pattern =
'*' + str(ccd) +
'*mask*' 250 maskFiles = glob.glob(os.path.join(path, pattern))
251 return maskFiles
if len(maskFiles) > 0
else ()
253 def _cleanupEotest(self, path):
254 """Delete all the medianed files left behind after eotest has run. 256 Running eotest generates a lot of interim medianed files, so this just cleans them up. 261 Path on which to delete all the eotest medianed files. 263 for filename
in glob.glob(os.path.join(path,
'*_median_*.fits')):
267 """After running eotest, generate pdf(s) of the results. 269 Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD. 270 The pdf file(s), along with the .tex file(s) and the individual plots are written 271 to the eotestOutputPath. 272 .pdf generation requires a TeX distro including pdflatex to be installed. 274 ccds = butler.queryMetadata(
'raw', self.config.ccdKey)
276 self.log.info(
"Starting test report generation for %s"%ccd)
278 plotPath = os.path.join(self.config.eotestOutputPath,
'plots')
279 if not os.path.exists(plotPath):
280 os.makedirs(plotPath)
281 plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath)
282 eoTestReport = sensorTest.EOTestReport(plots, wl_dir=
'')
283 eoTestReport.make_figures()
284 eoTestReport.make_pdf()
285 except Exception
as e:
286 self.log.warn(
"Failed to make eotest report for %s: %s"%(ccd, e))
287 self.log.info(
"Finished test report generation.")
292 Generate calibration products using eotest algorithms. 294 Generate all calibration products possible using the vanilla eotest implementation, 295 given a butler for a TS8 (raft-test) repo. It can contain multiple runs, but must correspond to 296 only a single raft/RTM. 298 - Run all eotest tasks possible, using the butler to gather the data 299 - Write outputs in eotest format 301 In order to replicate the canonical eotest analysis, the tasks should be run in a specific order. 302 This is given/defined in the "Steps" section here: 303 http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayProcess.jsp?processPath=1179 305 But is replicated here for conveniece: 307 * CCD Read Noise Analysis 308 * Bright Defects Analysis 309 * Dark Defects Analysis 311 * Dark Current X - will not be implemented here 312 * Charge Transfer Efficiencies 313 * Photo-response analysis X - will not be implemented here 314 * Flat Pairs Analysis 315 * Photon Transfer Curve 316 * Quantum Efficiency X - will not be implemented here 318 List of tasks that exist in the eotest package but aren't mentioned on the above link: 325 # TODO: For each eotest task, find out what the standard raft testing does for the optional params. 326 i.e. many have optional params for gains, bias-frames etc - if we want bitwise identicallity then we 327 need to know what is typically provided to these tasks when the camera team runs this code. 328 This can probably be worked out from https://github.com/lsst-camera-dh/lcatr-harness 329 but it sounds like Jim Chiang doesn't recommend trying to do that. 334 butler : `lsst.daf.persistence.butler` 335 Butler for the repo containg the eotest data to be used 337 Optional run number, to be used for repos containing multiple runs 339 self.log.info(
"Running eotest routines direct")
342 runs = butler.queryMetadata(
'raw', [
'run'])
347 raise RuntimeError(
"Butler query found %s for runs. eotest datasets must have a run number," 348 "and you must specify which run to use if a respoitory contains several." 353 raise RuntimeError(
"Butler query found %s for runs, but the run specified (%s) " 354 "was not among them." % (runs, run))
357 if not os.path.exists(self.config.eotestOutputPath):
358 os.makedirs(self.config.eotestOutputPath)
360 ccds = butler.queryMetadata(
'raw', self.config.ccdKey)
361 imTypes = butler.queryMetadata(
'raw', [
'imageType'])
362 testTypes = butler.queryMetadata(
'raw', [
'testType'])
367 if self.config.doFe55:
368 fe55TaskDataId = {
'run': run,
'testType':
'FE55',
'imageType':
'FE55'}
369 self.log.info(
"Starting Fe55 pixel task")
371 if 'FE55' not in testTypes:
372 msg =
"No Fe55 tests found. Available data: %s" % testTypes
373 if self.config.requireAllEOTests:
374 raise RuntimeError(msg)
376 self.log.warn(msg +
"\nSkipping Fe55 task")
378 fe55Filenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
379 self.config.ccdKey: ccd})[0][:-3]
380 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=fe55TaskDataId)]
381 self.log.trace(
"Fe55Task: Processing %s with %s files" % (ccd, len(fe55Filenames)))
382 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
383 gains = self.fe55.run(sensor_id=ccd, infiles=fe55Filenames, mask_files=maskFiles)
387 butler.put(gains,
'eotest_gain', dataId={self.config.ccdKey: ccd,
'run': run})
398 if self.config.doReadNoise:
400 self.log.info(
"Starting readNoise task")
401 noiseTaskDataId = {
'run': run,
'testType':
'FE55',
'imageType':
'BIAS'}
403 if (
'FE55' not in testTypes)
or (
'BIAS' not in imTypes):
404 msg =
"Required data for readNoise unavailable. Available data:\ 405 \ntestTypes: %s\nimageTypes: %s" % (testTypes, imTypes)
406 if self.config.requireAllEOTests:
407 raise RuntimeError(msg)
409 self.log.warn(msg +
"\nSkipping noise task")
410 noiseFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
411 self.config.ccdKey: ccd})[0][:-3]
412 for visit
in butler.queryMetadata(
'raw', [
'visit'],
413 dataId=noiseTaskDataId)]
414 self.log.trace(
"Fe55Task: Processing %s with %s files" % (ccd, len(noiseFilenames)))
415 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
416 gains = butler.get(
'eotest_gain', dataId={self.config.ccdKey: ccd,
'run': run})
417 self.readNoise.run(sensor_id=ccd, bias_files=noiseFilenames,
418 gains=gains, mask_files=maskFiles)
424 if self.config.doBrightPixels:
425 self.log.info(
"Starting bright pixel task")
426 brightTaskDataId = {
'run': run,
'testType':
'DARK',
'imageType':
'DARK'}
428 if 'DARK' not in testTypes:
429 msg =
"No dark tests found. Available data: %s" % testTypes
430 if self.config.requireAllEOTests:
431 raise RuntimeError(msg)
433 self.log.warn(msg +
"\nSkipping bright pixel task")
435 darkFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
436 self.config.ccdKey: ccd})[0][:-3]
437 for visit
in butler.queryMetadata(
'raw', [
'visit'],
438 dataId=brightTaskDataId)]
439 self.log.trace(
"BrightTask: Processing %s with %s files" % (ccd, len(darkFilenames)))
440 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
441 gains = butler.get(
'eotest_gain', dataId={self.config.ccdKey: ccd,
'run': run})
442 self.brightPixels.run(sensor_id=ccd, dark_files=darkFilenames,
443 mask_files=maskFiles, gains=gains)
449 if self.config.doDarkPixels:
450 self.log.info(
"Starting dark pixel task")
451 darkTaskDataId = {
'run': run,
'testType':
'SFLAT_500',
'imageType':
'FLAT'}
453 if 'SFLAT_500' not in testTypes:
454 msg =
"No superflats found. Available data: %s" % testTypes
455 if self.config.requireAllEOTests:
456 raise RuntimeError(msg)
458 self.log.warn(msg +
"\nSkipping dark pixel task")
460 sflatFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
461 self.config.ccdKey: ccd})[0][:-3]
462 for visit
in butler.queryMetadata(
'raw', [
'visit'],
463 dataId=darkTaskDataId)]
464 self.log.trace(
"DarkTask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
465 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
466 self.darkPixels.run(sensor_id=ccd, sflat_files=sflatFilenames, mask_files=maskFiles)
472 if self.config.doTraps:
473 self.log.info(
"Starting trap task")
474 trapTaskDataId = {
'run': run,
'testType':
'TRAP',
'imageType':
'PPUMP'}
476 if (
'TRAP' not in testTypes)
and (
'PPUMP' not in imTypes):
477 msg =
"No pocket pumping exposures found. Available data: %s" % testTypes
478 if self.config.requireAllEOTests:
479 raise RuntimeError(msg)
481 self.log.warn(msg +
"\nSkipping trap task")
483 trapFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
484 self.config.ccdKey: ccd})[0][:-3]
485 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=trapTaskDataId)]
486 if len(trapFilenames) != 1:
487 msg =
"Trap Task: Found more than one ppump trap file: %s" % trapFilenames
488 msg +=
" Running using only the first one found." 490 self.log.trace(
"Trap Task: Processing %s with %s files" % (ccd, len(trapFilenames)))
491 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
492 gains = butler.get(
'eotest_gain', dataId={self.config.ccdKey: ccd,
'run': run})
493 self.traps.run(sensor_id=ccd, pocket_pumped_file=trapFilenames[0],
494 mask_files=maskFiles, gains=gains)
500 if self.config.doCTE:
501 self.log.info(
"Starting CTE task")
502 cteTaskDataId = {
'run': run,
'testType':
'SFLAT_500',
'imageType':
'FLAT'}
504 if 'SFLAT_500' not in testTypes:
505 msg =
"No superflats found. Available data: %s" % testTypes
506 if self.config.requireAllEOTests:
507 raise RuntimeError(msg)
509 self.log.warn(msg +
"\nSkipping CTE task")
511 sflatFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
512 self.config.ccdKey: ccd})[0][:-3]
513 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=cteTaskDataId)]
514 self.log.trace(
"CTETask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
515 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
516 self.cte.run(sensor_id=ccd, superflat_files=sflatFilenames, mask_files=maskFiles)
522 if self.config.doFlatPair:
523 self.log.info(
"Starting flatPair task")
524 flatPairDataId = {
'run': run,
'testType':
'FLAT',
'imageType':
'FLAT'}
526 if 'FLAT' not in testTypes:
527 msg =
"No dataset for flat_pairs found. Available data: %s" % testTypes
528 if self.config.requireAllEOTests:
529 raise RuntimeError(msg)
531 self.log.warn(msg +
"\nSkipping flatPair task")
533 flatPairFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
534 self.config.ccdKey: ccd})[0][:-3]
535 for visit
in butler.queryMetadata(
'raw', [
'visit'],
536 dataId=flatPairDataId)]
545 flatPairFilenames = [os.path.realpath(f)
for f
in flatPairFilenames
if 546 os.path.realpath(f).find(
'flat1') != -1
or 547 os.path.realpath(f).find(
'flat2') != -1]
548 if not flatPairFilenames:
549 raise RuntimeError(
"No flatPair files found.")
550 self.log.trace(
"FlatPairTask: Processing %s with %s files" % (ccd, len(flatPairFilenames)))
551 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
552 gains = butler.get(
'eotest_gain', dataId={self.config.ccdKey: ccd,
'run': run})
553 self.flatPair.run(sensor_id=ccd, infiles=flatPairFilenames, mask_files=maskFiles,
554 gains=gains, max_pd_frac_dev=self.config.flatPairMaxPdFracDev)
560 if self.config.doPTC:
561 self.log.info(
"Starting PTC task")
562 ptcDataId = {
'run': run,
'testType':
'FLAT',
'imageType':
'FLAT'}
564 if 'FLAT' not in testTypes:
565 msg =
"No dataset for flat_pairs found. Available data: %s" % testTypes
566 if self.config.requireAllEOTests:
567 raise RuntimeError(msg)
569 self.log.warn(msg +
"\nSkipping PTC task")
571 ptcFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
572 self.config.ccdKey: ccd})[0][:-3]
573 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=ptcDataId)]
582 ptcFilenames = [os.path.realpath(f)
for f
in ptcFilenames
if 583 os.path.realpath(f).find(
'flat1') != -1
or 584 os.path.realpath(f).find(
'flat2') != -1]
586 raise RuntimeError(
"No flatPair files found")
587 self.log.trace(
"PTCTask: Processing %s with %s files" % (ccd, len(ptcFilenames)))
588 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
589 gains = butler.get(
'eotest_gain', dataId={self.config.ccdKey: ccd,
'run': run})
590 self.ptc.run(sensor_id=ccd, infiles=ptcFilenames, mask_files=maskFiles, gains=gains)
594 self.log.info(
"Finished running EOTest")
def _getMaskFiles(self, path, ccd)
def runEotestDirect(self, butler, run=None)
def _cleanupEotest(self, path)
def __init__(self, args, kwargs)
def makeEotestReport(self, butler)