24 """Calibration products production task code.""" 25 from __future__
import absolute_import, division, print_function
34 import lsst.eotest.sensor
as sensorTest
38 """Config class for the calibration products production (CP) task.""" 40 fe55 = pexConfig.ConfigurableField(
41 target=sensorTest.Fe55Task,
42 doc=
"The Fe55 analysis task.",
44 doFe55 = pexConfig.Field(
46 doc=
"Measure gains using Fe55?",
49 readNoise = pexConfig.ConfigurableField(
50 target=sensorTest.ReadNoiseTask,
51 doc=
"The read noise task.",
53 doReadNoise = pexConfig.Field(
55 doc=
"Measure the read-noise?",
58 brightPixels = pexConfig.ConfigurableField(
59 target=sensorTest.BrightPixelsTask,
60 doc=
"The bright pixel/column finding task.",
62 doBrightPixels = pexConfig.Field(
64 doc=
"Find bright pixels?",
67 darkPixels = pexConfig.ConfigurableField(
68 target=sensorTest.DarkPixelsTask,
69 doc=
"The dark pixel/column finding task.",
71 doDarkPixels = pexConfig.Field(
73 doc=
"Find dark pixels?",
76 traps = pexConfig.ConfigurableField(
77 target=sensorTest.TrapTask,
78 doc=
"The trap-finding task.",
80 doTraps = pexConfig.Field(
82 doc=
"Find traps using pocket-pumping exposures?",
85 cte = pexConfig.ConfigurableField(
86 target=sensorTest.CteTask,
87 doc=
"The CTE analysis task.",
89 doCTE = pexConfig.Field(
91 doc=
"Measure the charge transfer efficiency?",
94 ptc = pexConfig.ConfigurableField(
95 target=sensorTest.PtcTask,
96 doc=
"The PTC analysis task.",
98 doPTC = pexConfig.Field(
100 doc=
"Measure the photon transfer curve?",
103 flatPair = pexConfig.ConfigurableField(
104 target=sensorTest.FlatPairTask,
105 doc=
"The flat-pair analysis task.",
107 doFlatPair = pexConfig.Field(
109 doc=
"Measure the detector response vs incident flux using flat pairs?",
112 eotestOutputPath = pexConfig.Field(
114 doc=
"Path to which to write the eotest output results. Madatory runtime arg for running eotest.",
117 requireAllEOTests = pexConfig.Field(
119 doc=
"If True, all tests are required to be runnable, and will Raise if data is missing. If False, " 120 "processing will continue if a previous part failed due to the input dataset being incomplete.",
123 flatPairMaxPdFracDev = pexConfig.Field(
125 doc=
"Maximum allowed fractional deviation between photodiode currents for the eotest flatPair task. " 126 "This value is passed to the task's run() method at runtime rather than being stored in the task's" 127 "own pexConfig field.",
132 """Set default config options for the subTasks.""" 134 self.
fe55.temp_set_point = -100
135 self.
fe55.temp_set_point_tol = 20
153 """Override of the valiate() method. 155 The pexConfigs of the subTasks here cannot be validated in the normal way, as they are the configs 156 for eotest, which does illegal things, and this would require an upstream PR to fix. Therefore, we 157 override the validate() method here, and use it to set the output directory for each of the tasks 158 based on the legal pexConfig parameter for the main task. 160 log = lsstLog.Log.getLogger(
"cp.pipe.cpTaskConfig")
162 raise RuntimeError(
"Must supply an output path for eotest data. " 163 "Please set config.eotestOutputPath.")
165 taskList = [
'fe55',
'brightPixels',
'darkPixels',
'readNoise',
'traps',
'cte',
'flatPair',
'ptc']
166 for task
in taskList:
167 if getattr(self, task).output_dir !=
'.':
170 log.warn(
"OVERWRITING: Found a user defined output path of %s for %sTask. " 171 "This has been overwritten with %s, as individually specified output paths for " 172 "subTasks are not supported at present" % (getattr(self, task).output_dir,
179 Calibration (Products) Production (CP) task. 181 This task is used to produce the calibration products required to calibrate cameras. 182 Examples of such operations are as follows: 183 * Given a set of flat-field images, find the dark pixels and columns. 184 * Given a set of darks, find the bright pixels and columns. 185 * Given a set of Fe55 exposures, calulate the gain of the readout chain, in e-/ADU 186 * Given a set of Fe55 exposures, calulate the instrinsic PSF of the silicon, and the degradation of 187 * the PSF due to CTE. 188 * Given a set of flat-pairs, measure the photon transfer curve (PTC). 189 * Given a set of bias frames, calculate the read noise of the system in e-. 190 * Given a set of pocket-pumping exposures, find charge-traps in the silicon. 192 The CpTask.runEotestDirect() is only applicable to LSST sensors, and only for a specific type of dataset 193 This method takes a dafPersistance.Butler corresponding to a repository in which a full eotest run has 194 been taken and ingested, and runs each of the tasks in eotest directly, allowing for bitwise comparison 195 with results given by the camera team. 197 See http://ls.st/ldm-151 Chapter 4, Calibration Products Production for further details 198 regarding the inputs and outputs. 201 ConfigClass = CpTaskConfig
205 """Constructor for the CpTask.""" 206 if 'lsst.eotest.sensor' not in sys.modules:
207 raise RuntimeError(
'eotest failed to import')
209 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
214 self.config.validate()
217 self.makeSubtask(
"fe55")
218 self.makeSubtask(
"readNoise")
219 self.makeSubtask(
"brightPixels")
220 self.makeSubtask(
"darkPixels")
221 self.makeSubtask(
"traps")
222 self.makeSubtask(
"cte")
223 self.makeSubtask(
"flatPair")
224 self.makeSubtask(
"ptc")
226 def _getMaskFiles(self, path, ccd):
227 """Get all available eotest mask files for a given ccd. 229 Each stage of the processing generates more mask files, so this allows each to be picked up 230 as more and more tests run, and saves having to have clever logic for if some tasks fail. 235 Path on which to find the mask files 236 ccd : `string` or `int` 237 Name/identifier of the CCD 241 maskFiles : iterable of `str` 242 List of mask files, or an empty tuple if none are found 244 pattern =
'*' + str(ccd) +
'*mask*' 245 maskFiles = glob.glob(os.path.join(path, pattern))
246 return maskFiles
if len(maskFiles) > 0
else ()
248 def _cleanupEotest(self, path):
249 """Delete all the medianed files left behind after eotest has run. 251 Running eotest generates a lot of interim medianed files, so this just cleans them up. 256 Path on which to delete all the eotest medianed files. 258 for filename
in glob.glob(os.path.join(path,
'*_median_*.fits')):
262 """After running eotest, generate pdf(s) of the results. 264 Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD. 265 The pdf file(s), along with the .tex file(s) and the individual plots are written 266 to the eotestOutputPath. 267 .pdf generation requires a TeX distro including pdflatex to be installed. 269 ccds = butler.queryMetadata(
'raw', [
'ccd'])
271 self.log.info(
"Starting test report generation for %s"%ccd)
273 plotPath = os.path.join(self.config.eotestOutputPath,
'plots')
274 if not os.path.exists(plotPath):
275 os.makedirs(plotPath)
276 plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath)
277 eoTestReport = sensorTest.EOTestReport(plots, wl_dir=
'')
278 eoTestReport.make_figures()
279 eoTestReport.make_pdf()
280 except Exception
as e:
281 self.log.warn(
"Failed to make eotest report for %s: %s"%(ccd, e))
282 self.log.info(
"Finished test report generation.")
287 Generate calibration products using eotest algorithms. 289 Generate all calibration products possible using the vanilla eotest implementation, 290 given a butler for a TS8 (raft-test) repo. It can contain multiple runs, but must correspond to 291 only a single raft/RTM. 293 - Run all eotest tasks possible, using the butler to gather the data 294 - Write outputs in eotest format 296 In order to replicate the canonical eotest analysis, the tasks should be run in a specific order. 297 This is given/defined in the "Steps" section here: 298 http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayProcess.jsp?processPath=1179 300 But is replicated here for conveniece: 302 * CCD Read Noise Analysis 303 * Bright Defects Analysis 304 * Dark Defects Analysis 306 * Dark Current X - will not be implemented here 307 * Charge Transfer Efficiencies 308 * Photo-response analysis X - will not be implemented here 309 * Flat Pairs Analysis 310 * Photon Transfer Curve 311 * Quantum Efficiency X - will not be implemented here 313 List of tasks that exist in the eotest package but aren't mentioned on the above link: 320 # TODO: For each eotest task, find out what the standard raft testing does for the optional params. 321 i.e. many have optional params for gains, bias-frames etc - if we want bitwise identicallity then we 322 need to know what is typically provided to these tasks when the camera team runs this code. 323 This can probably be worked out from https://github.com/lsst-camera-dh/lcatr-harness 324 but it sounds like Jim Chiang doesn't recommend trying to do that. 329 butler : `lsst.daf.persistence.butler` 330 Butler for the repo containg the eotest data to be used 332 Optional run number, to be used for repos containing multiple runs 334 self.log.info(
"Running eotest routines direct")
337 runs = butler.queryMetadata(
'raw', [
'run'])
342 raise RuntimeError(
"Butler query found %s for runs. eotest datasets must have a run number," 343 "and you must specify which run to use if a respoitory contains several." 348 raise RuntimeError(
"Butler query found %s for runs, but the run specified (%s) " 349 "was not among them." % (runs, run))
352 if not os.path.exists(self.config.eotestOutputPath):
353 os.makedirs(self.config.eotestOutputPath)
355 ccds = butler.queryMetadata(
'raw', [
'ccd'])
356 imTypes = butler.queryMetadata(
'raw', [
'imageType'])
357 testTypes = butler.queryMetadata(
'raw', [
'testType'])
362 if self.config.doFe55:
363 fe55TaskDataId = {
'run': run,
'testType':
'FE55',
'imageType':
'FE55'}
364 self.log.info(
"Starting Fe55 pixel task")
366 if 'FE55' not in testTypes:
367 msg =
"No Fe55 tests found. Available data: %s" % testTypes
368 if self.config.requireAllEOTests:
369 raise RuntimeError(msg)
371 self.log.warn(msg +
"\nSkipping Fe55 task")
373 fe55Filenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
375 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=fe55TaskDataId)]
376 self.log.trace(
"Fe55Task: Processing %s with %s files" % (ccd, len(fe55Filenames)))
377 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
378 gains = self.fe55.run(sensor_id=ccd, infiles=fe55Filenames, mask_files=maskFiles)
382 butler.put(gains,
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
393 if self.config.doReadNoise:
395 self.log.info(
"Starting readNoise task")
396 noiseTaskDataId = {
'run': run,
'testType':
'FE55',
'imageType':
'BIAS'}
398 if (
'FE55' not in testTypes)
or (
'BIAS' not in imTypes):
399 msg =
"Required data for readNoise unavailable. Available data:\ 400 \ntestTypes: %s\nimageTypes: %s" % (testTypes, imTypes)
401 if self.config.requireAllEOTests:
402 raise RuntimeError(msg)
404 self.log.warn(msg +
"\nSkipping noise task")
405 noiseFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
407 for visit
in butler.queryMetadata(
'raw', [
'visit'],
408 dataId=noiseTaskDataId)]
409 self.log.trace(
"Fe55Task: Processing %s with %s files" % (ccd, len(noiseFilenames)))
410 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
411 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
412 self.readNoise.run(sensor_id=ccd, bias_files=noiseFilenames,
413 gains=gains, mask_files=maskFiles)
419 if self.config.doBrightPixels:
420 self.log.info(
"Starting bright pixel task")
421 brightTaskDataId = {
'run': run,
'testType':
'DARK',
'imageType':
'DARK'}
423 if 'DARK' not in testTypes:
424 msg =
"No dark tests found. Available data: %s" % testTypes
425 if self.config.requireAllEOTests:
426 raise RuntimeError(msg)
428 self.log.warn(msg +
"\nSkipping bright pixel task")
430 darkFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
432 for visit
in butler.queryMetadata(
'raw', [
'visit'],
433 dataId=brightTaskDataId)]
434 self.log.trace(
"BrightTask: Processing %s with %s files" % (ccd, len(darkFilenames)))
435 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
436 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
437 self.brightPixels.run(sensor_id=ccd, dark_files=darkFilenames,
438 mask_files=maskFiles, gains=gains)
444 if self.config.doDarkPixels:
445 self.log.info(
"Starting dark pixel task")
446 darkTaskDataId = {
'run': run,
'testType':
'SFLAT_500',
'imageType':
'FLAT'}
448 if 'SFLAT_500' not in testTypes:
449 msg =
"No superflats found. Available data: %s" % testTypes
450 if self.config.requireAllEOTests:
451 raise RuntimeError(msg)
453 self.log.warn(msg +
"\nSkipping dark pixel task")
455 sflatFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
457 for visit
in butler.queryMetadata(
'raw', [
'visit'],
458 dataId=darkTaskDataId)]
459 self.log.trace(
"DarkTask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
460 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
461 self.darkPixels.run(sensor_id=ccd, sflat_files=sflatFilenames, mask_files=maskFiles)
467 if self.config.doTraps:
468 self.log.info(
"Starting trap task")
469 trapTaskDataId = {
'run': run,
'testType':
'TRAP',
'imageType':
'PPUMP'}
471 if (
'TRAP' not in testTypes)
and (
'PPUMP' not in imTypes):
472 msg =
"No pocket pumping exposures found. Available data: %s" % testTypes
473 if self.config.requireAllEOTests:
474 raise RuntimeError(msg)
476 self.log.warn(msg +
"\nSkipping trap task")
478 trapFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
480 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=trapTaskDataId)]
481 if len(trapFilenames) != 1:
482 self.log.fatal(
"Trap Task: Found more than one ppump trap file: %s" % trapFilenames)
483 self.log.trace(
"Trap Task: Processing %s with %s files" % (ccd, len(trapFilenames)))
484 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
485 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
486 self.traps.run(sensor_id=ccd, pocket_pumped_file=trapFilenames[0],
487 mask_files=maskFiles, gains=gains)
493 if self.config.doCTE:
494 self.log.info(
"Starting CTE task")
495 cteTaskDataId = {
'run': run,
'testType':
'SFLAT_500',
'imageType':
'FLAT'}
497 if 'SFLAT_500' not in testTypes:
498 msg =
"No superflats found. Available data: %s" % testTypes
499 if self.config.requireAllEOTests:
500 raise RuntimeError(msg)
502 self.log.warn(msg +
"\nSkipping CTE task")
504 sflatFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
506 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=cteTaskDataId)]
507 self.log.trace(
"CTETask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
508 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
509 self.cte.run(sensor_id=ccd, superflat_files=sflatFilenames, mask_files=maskFiles)
515 if self.config.doFlatPair:
516 self.log.info(
"Starting flatPair task")
517 flatPairDataId = {
'run': run,
'testType':
'FLAT',
'imageType':
'FLAT'}
519 if 'FLAT' not in testTypes:
520 msg =
"No dataset for flat_pairs found. Available data: %s" % testTypes
521 if self.config.requireAllEOTests:
522 raise RuntimeError(msg)
524 self.log.warn(msg +
"\nSkipping flatPair task")
526 flatPairFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
528 for visit
in butler.queryMetadata(
'raw', [
'visit'],
529 dataId=flatPairDataId)]
538 flatPairFilenames = [os.path.realpath(f)
for f
in flatPairFilenames
if 539 os.path.realpath(f).find(
'flat1') != -1
or 540 os.path.realpath(f).find(
'flat2') != -1]
541 if not flatPairFilenames:
542 raise RuntimeError(
"No flatPair files found.")
543 self.log.trace(
"FlatPairTask: Processing %s with %s files" % (ccd, len(flatPairFilenames)))
544 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
545 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
546 self.flatPair.run(sensor_id=ccd, infiles=flatPairFilenames, mask_files=maskFiles,
547 gains=gains, max_pd_frac_dev=self.config.flatPairMaxPdFracDev)
553 if self.config.doPTC:
554 self.log.info(
"Starting PTC task")
555 ptcDataId = {
'run': run,
'testType':
'FLAT',
'imageType':
'FLAT'}
557 if 'FLAT' not in testTypes:
558 msg =
"No dataset for flat_pairs found. Available data: %s" % testTypes
559 if self.config.requireAllEOTests:
560 raise RuntimeError(msg)
562 self.log.warn(msg +
"\nSkipping PTC task")
564 ptcFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
566 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=ptcDataId)]
575 ptcFilenames = [os.path.realpath(f)
for f
in ptcFilenames
if 576 os.path.realpath(f).find(
'flat1') != -1
or 577 os.path.realpath(f).find(
'flat2') != -1]
579 raise RuntimeError(
"No flatPair files found")
580 self.log.trace(
"PTCTask: Processing %s with %s files" % (ccd, len(ptcFilenames)))
581 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
582 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
583 self.ptc.run(sensor_id=ccd, infiles=ptcFilenames, mask_files=maskFiles, gains=gains)
587 self.log.info(
"Finished running EOTest")
def _getMaskFiles(self, path, ccd)
def runEotestDirect(self, butler, run=None)
def _cleanupEotest(self, path)
def __init__(self, args, kwargs)
def makeEotestReport(self, butler)