24 """Calibration products production task code.""" 25 from __future__
import absolute_import, division, print_function
34 import lsst.eotest.sensor
as sensorTest
38 """Config class for the calibration products production (CP) task.""" 40 fe55 = pexConfig.ConfigurableField(
41 target=sensorTest.Fe55Task,
42 doc=
"The Fe55 analysis task.",
44 doFe55 = pexConfig.Field(
46 doc=
"Measure gains using Fe55?",
49 readNoise = pexConfig.ConfigurableField(
50 target=sensorTest.ReadNoiseTask,
51 doc=
"The read noise task.",
53 doReadNoise = pexConfig.Field(
55 doc=
"Measure the read-noise?",
58 brightPixels = pexConfig.ConfigurableField(
59 target=sensorTest.BrightPixelsTask,
60 doc=
"The bright pixel/column finding task.",
62 doBrightPixels = pexConfig.Field(
64 doc=
"Find bright pixels?",
67 darkPixels = pexConfig.ConfigurableField(
68 target=sensorTest.DarkPixelsTask,
69 doc=
"The dark pixel/column finding task.",
71 doDarkPixels = pexConfig.Field(
73 doc=
"Find dark pixels?",
76 traps = pexConfig.ConfigurableField(
77 target=sensorTest.TrapTask,
78 doc=
"The trap-finding task.",
80 doTraps = pexConfig.Field(
82 doc=
"Find traps using pocket-pumping exposures?",
85 cte = pexConfig.ConfigurableField(
86 target=sensorTest.CteTask,
87 doc=
"The CTE analysis task.",
89 doCTE = pexConfig.Field(
91 doc=
"Measure the charge transfer efficiency?",
94 ptc = pexConfig.ConfigurableField(
95 target=sensorTest.PtcTask,
96 doc=
"The PTC analysis task.",
98 doPTC = pexConfig.Field(
100 doc=
"Measure the photon transfer curve?",
103 flatPair = pexConfig.ConfigurableField(
104 target=sensorTest.FlatPairTask,
105 doc=
"The flat-pair analysis task.",
107 doFlatPair = pexConfig.Field(
109 doc=
"Measure the detector response vs incident flux using flat pairs?",
112 eotestOutputPath = pexConfig.Field(
114 doc=
"Path to which to write the eotest output results. Madatory runtime arg for running eotest.",
117 requireAllEOTests = pexConfig.Field(
119 doc=
"If True, all tests are required to be runnable, and will Raise if data is missing. If False, " 120 "processing will continue if a previous part failed due to the input dataset being incomplete.",
123 flatPairMaxPdFracDev = pexConfig.Field(
125 doc=
"Maximum allowed fractional deviation between photodiode currents for the eotest flatPair task. " 126 "This value is passed to the task's run() method at runtime rather than being stored in the task's" 127 "own pexConfig field.",
132 """Set default config options for the subTasks.""" 134 self.
fe55.temp_set_point = -100
135 self.
fe55.temp_set_point_tol = 20
153 """Override of the valiate() method. 155 The pexConfigs of the subTasks here cannot be validated in the normal way, as they are the configs 156 for eotest, which does illegal things, and this would require an upstream PR to fix. Therefore, we 157 override the validate() method here, and use it to set the output directory for each of the tasks 158 based on the legal pexConfig parameter for the main task. 160 log = lsstLog.Log.getLogger(
"cp.pipe.cpTaskConfig")
162 raise RuntimeError(
"Must supply an output path for eotest data. " 163 "Please set config.eotestOutputPath.")
165 taskList = [
'fe55',
'brightPixels',
'darkPixels',
'readNoise',
'traps',
'cte',
'flatPair',
'ptc']
166 for task
in taskList:
167 if getattr(self, task).output_dir !=
'.':
170 log.warn(
"OVERWRITING: Found a user defined output path of %s for %sTask. " 171 "This has been overwritten with %s, as individually specified output paths for " 172 "subTasks are not supported at present" % (getattr(self, task).output_dir,
179 Calibration (Products) Production (CP) task. 181 This task is used to produce the calibration products required to calibrate cameras. 182 Examples of such operations are as follows: 183 * Given a set of flat-field images, find the dark pixels and columns. 184 * Given a set of darks, find the bright pixels and columns. 185 * Given a set of Fe55 exposures, calulate the gain of the readout chain, in e-/ADU 186 * Given a set of Fe55 exposures, calulate the instrinsic PSF of the silicon, and the degradation of 187 * the PSF due to CTE. 188 * Given a set of flat-pairs, measure the photon transfer curve (PTC). 189 * Given a set of bias frames, calculate the read noise of the system in e-. 190 * Given a set of pocket-pumping exposures, find charge-traps in the silicon. 192 The CpTask.runEotestDirect() is only applicable to LSST sensors, and only for a specific type of dataset 193 This method takes a dafPersistance.Butler corresponding to a repository in which a full eotest run has 194 been taken and ingested, and runs each of the tasks in eotest directly, allowing for bitwise comparison 195 with results given by the camera team. 197 See http://ls.st/ldm-151 Chapter 4, Calibration Products Production for further details 198 regarding the inputs and outputs. 201 ConfigClass = CpTaskConfig
205 """Constructor for the CpTask.""" 206 if 'lsst.eotest.sensor' not in sys.modules:
207 raise RuntimeError(
'eotest failed to import')
209 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
214 self.config.validate()
217 self.makeSubtask(
"fe55")
218 self.makeSubtask(
"readNoise")
219 self.makeSubtask(
"brightPixels")
220 self.makeSubtask(
"darkPixels")
221 self.makeSubtask(
"traps")
222 self.makeSubtask(
"cte")
223 self.makeSubtask(
"flatPair")
224 self.makeSubtask(
"ptc")
226 def _getMaskFiles(self, path, ccd):
227 """Get all available eotest mask files for a given ccd. 229 Each stage of the processing generates more mask files, so this allows each to be picked up 230 as more and more tests run, and saves having to have clever logic for if some tasks fail. 235 Path on which to find the mask files 236 ccd : `string` or `int` 237 Name/identifier of the CCD 241 maskFiles : iterable of `str` 242 List of mask files, or an empty tuple if none are found 244 pattern =
'*' + str(ccd) +
'*mask*' 245 maskFiles = glob.glob(os.path.join(path, pattern))
246 return maskFiles
if len(maskFiles) > 0
else ()
248 def _cleanupEotest(self, path):
249 """Delete all the medianed files left behind after eotest has run. 251 Running eotest generates a lot of interim medianed files, so this just cleans them up. 256 Path on which to delete all the eotest medianed files. 258 for filename
in glob.glob(os.path.join(path,
'*_median_*.fits')):
262 """After running eotest, generate pdf(s) of the results. 264 Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD. 265 The pdf file(s), along with the .tex file(s) and the individual plots are written 266 to the eotestOutputPath. 267 .pdf generation requires a TeX distro including pdflatex to be installed. 269 ccds = butler.queryMetadata(
'raw', [
'ccd'])
271 self.log.info(
"Starting test report generation for %s"%ccd)
273 plotPath = os.path.join(self.config.eotestOutputPath,
'plots')
274 if not os.path.exists(plotPath):
275 os.makedirs(plotPath)
276 plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath)
277 eoTestReport = sensorTest.EOTestReport(plots, wl_dir=
'')
278 eoTestReport.make_figures()
279 eoTestReport.make_pdf()
280 except Exception
as e:
281 self.log.warn(
"Failed to make eotest report for %s: %s"%(ccd, e))
282 self.log.info(
"Finished test report generation.")
287 Generate calibration products using eotest algorithms. 289 Generate all calibration products possible using the vanilla eotest implementation, 290 given a butler for a TS8 (raft-test) repo. It can contain multiple runs, but must correspond to 291 only a single raft/RTM. 293 - Run all eotest tasks possible, using the butler to gather the data 294 - Write outputs in eotest format 296 In order to replicate the canonical eotest analysis, the tasks should be run in a specific order. 297 This is given/defined in the "Steps" section here: 298 http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayProcess.jsp?processPath=1179 300 But is replicated here for conveniece: 302 * CCD Read Noise Analysis 303 * Bright Defects Analysis 304 * Dark Defects Analysis 306 * Dark Current X - will not be implemented here 307 * Charge Transfer Efficiencies 308 * Photo-response analysis X - will not be implemented here 309 * Flat Pairs Analysis 310 * Photon Transfer Curve 311 * Quantum Efficiency X - will not be implemented here 313 List of tasks that exist in the eotest package but aren't mentioned on the above link: 321 # TODO: For each eotest task, find out what the standard raft testing does for the optional params. 322 i.e. many have optional params for gains, bias-frames etc - if we want bitwise identicallity then we 323 need to know what is typically provided to these tasks when the camera team runs this code. 324 This can probably be worked out from https://github.com/lsst-camera-dh/lcatr-harness 325 but it sounds like Jim Chiang doesn't recommend trying to do that. 330 butler : `lsst.daf.persistence.butler` 331 Butler for the repo containg the eotest data to be used 333 Optional run number, to be used for repos containing multiple runs 335 self.log.info(
"Running eotest routines direct")
338 runs = butler.queryMetadata(
'raw', [
'run'])
343 raise RuntimeError(
"Butler query found %s for runs. eotest datasets must have a run number," 344 "and you must specify which run to use if a respoitory contains several." 349 raise RuntimeError(
"Butler query found %s for runs, but the run specified (%s) " 350 "was not among them." % (runs, run))
353 if not os.path.exists(self.config.eotestOutputPath):
354 os.makedirs(self.config.eotestOutputPath)
356 ccds = butler.queryMetadata(
'raw', [
'ccd'])
357 imTypes = butler.queryMetadata(
'raw', [
'imageType'])
358 testTypes = butler.queryMetadata(
'raw', [
'testType'])
363 if self.config.doFe55:
364 fe55TaskDataId = {
'run': run,
'testType':
'FE55',
'imageType':
'FE55'}
365 self.log.info(
"Starting Fe55 pixel task")
367 if 'FE55' not in testTypes:
368 msg =
"No Fe55 tests found. Available data: %s" % testTypes
369 if self.config.requireAllEOTests:
370 raise RuntimeError(msg)
372 self.log.warn(msg +
"\nSkipping Fe55 task")
374 fe55Filenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
376 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=fe55TaskDataId)]
377 self.log.trace(
"Fe55Task: Processing %s with %s files" % (ccd, len(fe55Filenames)))
378 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
379 gains = self.fe55.run(sensor_id=ccd, infiles=fe55Filenames, mask_files=maskFiles)
383 butler.put(gains,
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
394 if self.config.doReadNoise:
396 self.log.info(
"Starting readNoise task")
397 noiseTaskDataId = {
'run': run,
'testType':
'FE55',
'imageType':
'BIAS'}
399 if (
'FE55' not in testTypes)
or (
'BIAS' not in imTypes):
400 msg =
"Required data for readNoise unavailable. Available data:\ 401 \ntestTypes: %s\nimageTypes: %s" % (testTypes, imTypes)
402 if self.config.requireAllEOTests:
403 raise RuntimeError(msg)
405 self.log.warn(msg +
"\nSkipping noise task")
406 noiseFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
408 for visit
in butler.queryMetadata(
'raw', [
'visit'],
409 dataId=noiseTaskDataId)]
410 self.log.trace(
"Fe55Task: Processing %s with %s files" % (ccd, len(noiseFilenames)))
411 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
412 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
413 self.readNoise.run(sensor_id=ccd, bias_files=noiseFilenames,
414 gains=gains, mask_files=maskFiles)
420 if self.config.doBrightPixels:
421 self.log.info(
"Starting bright pixel task")
422 brightTaskDataId = {
'run': run,
'testType':
'DARK',
'imageType':
'DARK'}
424 if 'DARK' not in testTypes:
425 msg =
"No dark tests found. Available data: %s" % testTypes
426 if self.config.requireAllEOTests:
427 raise RuntimeError(msg)
429 self.log.warn(msg +
"\nSkipping bright pixel task")
431 darkFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
433 for visit
in butler.queryMetadata(
'raw', [
'visit'],
434 dataId=brightTaskDataId)]
435 self.log.trace(
"BrightTask: Processing %s with %s files" % (ccd, len(darkFilenames)))
436 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
437 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
438 self.brightPixels.run(sensor_id=ccd, dark_files=darkFilenames,
439 mask_files=maskFiles, gains=gains)
445 if self.config.doDarkPixels:
446 self.log.info(
"Starting dark pixel task")
447 darkTaskDataId = {
'run': run,
'testType':
'SFLAT_500',
'imageType':
'FLAT'}
449 if 'SFLAT_500' not in testTypes:
450 msg =
"No superflats found. Available data: %s" % testTypes
451 if self.config.requireAllEOTests:
452 raise RuntimeError(msg)
454 self.log.warn(msg +
"\nSkipping dark pixel task")
456 sflatFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
458 for visit
in butler.queryMetadata(
'raw', [
'visit'],
459 dataId=darkTaskDataId)]
460 self.log.trace(
"DarkTask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
461 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
462 self.darkPixels.run(sensor_id=ccd, sflat_files=sflatFilenames, mask_files=maskFiles)
468 if self.config.doTraps:
469 self.log.info(
"Starting trap task")
470 trapTaskDataId = {
'run': run,
'testType':
'TRAP',
'imageType':
'PPUMP'}
472 if (
'TRAP' not in testTypes)
and (
'PPUMP' not in imTypes):
473 msg =
"No pocket pumping exposures found. Available data: %s" % testTypes
474 if self.config.requireAllEOTests:
475 raise RuntimeError(msg)
477 self.log.warn(msg +
"\nSkipping trap task")
479 trapFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
481 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=trapTaskDataId)]
482 if len(trapFilenames) != 1:
483 self.log.fatal(
"Trap Task: Found more than one ppump trap file: %s" % trapFilenames)
484 self.log.trace(
"Trap Task: Processing %s with %s files" % (ccd, len(trapFilenames)))
485 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
486 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
487 self.traps.run(sensor_id=ccd, pocket_pumped_file=trapFilenames[0],
488 mask_files=maskFiles, gains=gains)
494 if self.config.doCTE:
495 self.log.info(
"Starting CTE task")
496 cteTaskDataId = {
'run': run,
'testType':
'SFLAT_500',
'imageType':
'FLAT'}
498 if 'SFLAT_500' not in testTypes:
499 msg =
"No superflats found. Available data: %s" % testTypes
500 if self.config.requireAllEOTests:
501 raise RuntimeError(msg)
503 self.log.warn(msg +
"\nSkipping CTE task")
505 sflatFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
507 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=cteTaskDataId)]
508 self.log.trace(
"CTETask: Processing %s with %s files" % (ccd, len(sflatFilenames)))
509 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
510 self.cte.run(sensor_id=ccd, superflat_files=sflatFilenames, mask_files=maskFiles)
516 if self.config.doFlatPair:
517 self.log.info(
"Starting flatPair task")
518 flatPairDataId = {
'run': run,
'testType':
'FLAT',
'imageType':
'FLAT'}
520 if 'FLAT' not in testTypes:
521 msg =
"No dataset for flat_pairs found. Available data: %s" % testTypes
522 if self.config.requireAllEOTests:
523 raise RuntimeError(msg)
525 self.log.warn(msg +
"\nSkipping flatPair task")
527 flatPairFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
529 for visit
in butler.queryMetadata(
'raw', [
'visit'],
530 dataId=flatPairDataId)]
539 flatPairFilenames = [os.path.realpath(f)
for f
in flatPairFilenames
if 540 os.path.realpath(f).find(
'flat1') != -1
or 541 os.path.realpath(f).find(
'flat2') != -1]
542 if not flatPairFilenames:
543 raise RuntimeError(
"No flatPair files found.")
544 self.log.trace(
"FlatPairTask: Processing %s with %s files" % (ccd, len(flatPairFilenames)))
545 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
546 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
547 self.flatPair.run(sensor_id=ccd, infiles=flatPairFilenames, mask_files=maskFiles,
548 gains=gains, max_pd_frac_dev=self.config.flatPairMaxPdFracDev)
554 if self.config.doPTC:
555 self.log.info(
"Starting PTC task")
556 ptcDataId = {
'run': run,
'testType':
'FLAT',
'imageType':
'FLAT'}
558 if 'FLAT' not in testTypes:
559 msg =
"No dataset for flat_pairs found. Available data: %s" % testTypes
560 if self.config.requireAllEOTests:
561 raise RuntimeError(msg)
563 self.log.warn(msg +
"\nSkipping PTC task")
565 ptcFilenames = [butler.get(
'raw_filename', dataId={
'visit': visit,
567 for visit
in butler.queryMetadata(
'raw', [
'visit'], dataId=ptcDataId)]
576 ptcFilenames = [os.path.realpath(f)
for f
in ptcFilenames
if 577 os.path.realpath(f).find(
'flat1') != -1
or 578 os.path.realpath(f).find(
'flat2') != -1]
580 raise RuntimeError(
"No flatPair files found")
581 self.log.trace(
"PTCTask: Processing %s with %s files" % (ccd, len(ptcFilenames)))
582 maskFiles = self.
_getMaskFiles(self.config.eotestOutputPath, ccd)
583 gains = butler.get(
'eotest_gain', dataId={
'ccd': ccd,
'run': run})
584 self.ptc.run(sensor_id=ccd, infiles=ptcFilenames, mask_files=maskFiles, gains=gains)
588 self.log.info(
"Finished running EOTest")
def _getMaskFiles(self, path, ccd)
def runEotestDirect(self, butler, run=None)
def _cleanupEotest(self, path)
def __init__(self, args, kwargs)
def makeEotestReport(self, butler)