23 """Make a look-up-table (LUT) for FGCM calibration.
25 This task computes a look-up-table for the range in expected atmosphere
26 variation and variation in instrumental throughput (as tracked by the
27 transmission_filter products). By pre-computing linearized integrals,
28 the FGCM fit is orders of magnitude faster for stars with a broad range
29 of colors and observing bands, yielding precision at the 1-2 mmag level.
31 Computing a LUT requires running MODTRAN or with a pre-generated
32 atmosphere table packaged with fgcm.
40 from lsst.obs.base
import Instrument
41 import lsst.pex.config
as pexConfig
42 import lsst.pipe.base
as pipeBase
43 from lsst.pipe.base
import connectionTypes
44 import lsst.afw.table
as afwTable
45 import lsst.afw.cameraGeom
as afwCameraGeom
46 from lsst.afw.image
import Filter
47 from .utilities
import lookupStaticCalibrations
51 __all__ = [
'FgcmMakeLutParametersConfig',
'FgcmMakeLutConfig',
'FgcmMakeLutTask',
56 dimensions=(
'instrument',),
58 camera = connectionTypes.PrerequisiteInput(
59 doc=
"Camera instrument",
61 storageClass=
"Camera",
62 dimensions=(
"instrument",),
63 lookupFunction=lookupStaticCalibrations,
67 transmission_optics = connectionTypes.PrerequisiteInput(
68 doc=
"Optics transmission curve information",
69 name=
"transmission_optics",
70 storageClass=
"TransmissionCurve",
71 dimensions=(
"instrument",),
72 lookupFunction=lookupStaticCalibrations,
77 transmission_sensor = connectionTypes.PrerequisiteInput(
78 doc=
"Sensor transmission curve information",
79 name=
"transmission_sensor",
80 storageClass=
"TransmissionCurve",
81 dimensions=(
"instrument",
"detector",),
82 lookupFunction=lookupStaticCalibrations,
88 transmission_filter = connectionTypes.PrerequisiteInput(
89 doc=
"Filter transmission curve information",
90 name=
"transmission_filter",
91 storageClass=
"TransmissionCurve",
92 dimensions=(
"band",
"instrument",
"physical_filter",),
93 lookupFunction=lookupStaticCalibrations,
99 fgcmLookUpTable = connectionTypes.Output(
100 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
101 "chromatic corrections."),
102 name=
"fgcmLookUpTable",
103 storageClass=
"Catalog",
104 dimensions=(
"instrument",),
109 """Config for parameters if atmosphereTableName not available"""
112 elevation = pexConfig.Field(
113 doc=
"Telescope elevation (m)",
117 pmbRange = pexConfig.ListField(
118 doc=(
"Barometric Pressure range (millibar) "
119 "Recommended range depends on the site."),
123 pmbSteps = pexConfig.Field(
124 doc=
"Barometric Pressure number of steps",
128 pwvRange = pexConfig.ListField(
129 doc=(
"Precipitable Water Vapor range (mm) "
130 "Recommended range depends on the site."),
134 pwvSteps = pexConfig.Field(
135 doc=
"Precipitable Water Vapor number of steps",
139 o3Range = pexConfig.ListField(
140 doc=
"Ozone range (dob)",
142 default=[220.0, 310.0],
144 o3Steps = pexConfig.Field(
145 doc=
"Ozone number of steps",
149 tauRange = pexConfig.ListField(
150 doc=
"Aerosol Optical Depth range (unitless)",
152 default=[0.002, 0.35],
154 tauSteps = pexConfig.Field(
155 doc=
"Aerosol Optical Depth number of steps",
159 alphaRange = pexConfig.ListField(
160 doc=
"Aerosol alpha range (unitless)",
164 alphaSteps = pexConfig.Field(
165 doc=
"Aerosol alpha number of steps",
169 zenithRange = pexConfig.ListField(
170 doc=
"Zenith angle range (degree)",
174 zenithSteps = pexConfig.Field(
175 doc=
"Zenith angle number of steps",
181 pmbStd = pexConfig.Field(
182 doc=(
"Standard Atmosphere pressure (millibar); "
183 "Recommended default depends on the site."),
187 pwvStd = pexConfig.Field(
188 doc=(
"Standard Atmosphere PWV (mm); "
189 "Recommended default depends on the site."),
193 o3Std = pexConfig.Field(
194 doc=
"Standard Atmosphere O3 (dob)",
198 tauStd = pexConfig.Field(
199 doc=
"Standard Atmosphere aerosol optical depth",
203 alphaStd = pexConfig.Field(
204 doc=
"Standard Atmosphere aerosol alpha",
208 airmassStd = pexConfig.Field(
209 doc=(
"Standard Atmosphere airmass; "
210 "Recommended default depends on the survey strategy."),
214 lambdaNorm = pexConfig.Field(
215 doc=
"Aerosol Optical Depth normalization wavelength (Angstrom)",
219 lambdaStep = pexConfig.Field(
220 doc=
"Wavelength step for generating atmospheres (nm)",
224 lambdaRange = pexConfig.ListField(
225 doc=
"Wavelength range for LUT (Angstrom)",
227 default=[3000.0, 11000.0],
232 pipelineConnections=FgcmMakeLutConnections):
233 """Config for FgcmMakeLutTask"""
235 filterNames = pexConfig.ListField(
236 doc=
"Filter names to build LUT ('short' names)",
240 stdFilterNames = pexConfig.ListField(
241 doc=(
"Standard filterNames ('short' names). "
242 "Each filter in filterName will be calibrated to a matched "
243 "stdFilterName. In regular usage, one has g->g, r->r, ... "
244 "In the case of HSC, one would have g->g, r->r2, r2->r2, ... "
245 "which allows replacement (or time-variable) filters to be "
246 "properly cross-calibrated."),
250 atmosphereTableName = pexConfig.Field(
251 doc=
"FGCM name or filename of precomputed atmospheres",
256 parameters = pexConfig.ConfigField(
257 doc=
"Atmosphere parameters (required if no atmosphereTableName)",
258 dtype=FgcmMakeLutParametersConfig,
264 Validate the config parameters.
266 This method behaves differently from the parent validate in the case
267 that atmosphereTableName is set. In this case, the config values
268 for standard values, step sizes, and ranges are loaded
269 directly from the specified atmosphereTableName.
272 self._fields[
'filterNames'].
validate(self)
273 self._fields[
'stdFilterNames'].
validate(self)
277 self._fields[
'parameters'].
validate(self)
281 """Subclass of TaskRunner for fgcmMakeLutTask
283 fgcmMakeLutTask.run() takes one argument, the butler, and
284 does not run on any data in the repository.
285 This runner does not use any parallelization.
291 Return a list with one element, the butler.
293 return [parsedCmd.butler]
299 butler: `lsst.daf.persistence.Butler`
303 exitStatus: `list` with `pipeBase.Struct`
304 exitStatus (0: success; 1: failure)
306 task = self.TaskClass(config=self.config, log=self.log)
310 task.runDataRef(butler)
313 task.runDataRef(butler)
314 except Exception
as e:
316 task.log.fatal(
"Failed: %s" % e)
317 if not isinstance(e, pipeBase.TaskError):
318 traceback.print_exc(file=sys.stderr)
320 task.writeMetadata(butler)
323 return [pipeBase.Struct(exitStatus=exitStatus)]
327 Run the task, with no multiprocessing
331 parsedCmd: ArgumentParser parsed command line
336 if self.precall(parsedCmd):
339 resultList = self(targetList[0])
346 Make Look-Up Table for FGCM.
348 This task computes a look-up-table for the range in expected atmosphere
349 variation and variation in instrumental throughput (as tracked by the
350 transmission_filter products). By pre-computing linearized integrals,
351 the FGCM fit is orders of magnitude faster for stars with a broad range
352 of colors and observing bands, yielding precision at the 1-2 mmag level.
354 Computing a LUT requires running MODTRAN or with a pre-generated
355 atmosphere table packaged with fgcm.
358 ConfigClass = FgcmMakeLutConfig
359 RunnerClass = FgcmMakeLutRunner
360 _DefaultName =
"fgcmMakeLut"
362 def __init__(self, butler=None, initInputs=None, **kwargs):
366 def _getMetadataName(self):
372 Make a Look-Up Table for FGCM
376 butler: `lsst.daf.persistence.Butler`
380 ValueError : Raised if configured filter name does not match any of the
381 available filter transmission curves.
383 camera = butler.get(
'camera')
384 opticsDataRef = butler.dataRef(
'transmission_optics')
386 sensorDataRefDict = {}
387 for detector
in camera:
388 sensorDataRefDict[detector.getId()] = butler.dataRef(
'transmission_sensor',
389 dataId={
'ccd': detector.getId()})
391 filterDataRefDict = {}
392 for filterName
in self.config.filterNames:
393 f = Filter(filterName)
396 aliases = f.getAliases()
397 aliases.extend(filterName)
398 for alias
in f.getAliases():
399 dataRef = butler.dataRef(
'transmission_filter', filter=alias)
400 if dataRef.datasetExists():
402 filterDataRefDict[alias] = dataRef
405 raise ValueError(
"Cound not find transmission for filter %s via any alias." %
412 butler.put(lutCat,
'fgcmLookUpTable')
415 camera = butlerQC.get(inputRefs.camera)
418 _ = Instrument.fromName(inputRefs.camera.dataId[
'instrument'],
420 opticsDataRef = butlerQC.get(inputRefs.transmission_optics)
422 sensorRefs = butlerQC.get(inputRefs.transmission_sensor)
423 sensorDataRefDict = {sensorRef.dataId.byName()[
'detector']: sensorRef
for
424 sensorRef
in sensorRefs}
426 filterRefs = butlerQC.get(inputRefs.transmission_filter)
427 filterDataRefDict = {filterRef.dataId.byName()[
'physical_filter']: filterRef
for
428 filterRef
in filterRefs}
434 butlerQC.put(lutCat, outputRefs.fgcmLookUpTable)
436 def _fgcmMakeLut(self, camera, opticsDataRef, sensorDataRefDict,
439 Make a FGCM Look-up Table
443 camera : `lsst.afw.cameraGeom.Camera`
444 Camera from the butler.
445 opticsDataRef : `lsst.daf.persistence.ButlerDataRef` or
446 `lsst.daf.butler.DeferredDatasetHandle`
447 Reference to optics transmission curve.
448 sensorDataRefDict : `dict` of [`int`, `lsst.daf.persistence.ButlerDataRef` or
449 `lsst.daf.butler.DeferredDatasetHandle`]
450 Dictionary of references to sensor transmission curves. Key will
452 filterDataRefDict : `dict` of [`str`, `lsst.daf.persistence.ButlerDataRef` or
453 `lsst.daf.butler.DeferredDatasetHandle`]
454 Dictionary of references to filter transmission curves. Key will
455 be physical filter name.
459 fgcmLookUpTable : `BaseCatalog`
460 The FGCM look-up table.
464 self.log.info(
"Found %d ccds for look-up table" % (nCcd))
475 self.log.info(
"Making the LUT maker object")
483 throughputLambda = np.arange(self.
fgcmLutMakerfgcmLutMaker.lambdaRange[0],
487 self.log.info(
"Built throughput lambda, %.1f-%.1f, step %.2f" %
488 (throughputLambda[0], throughputLambda[-1],
489 throughputLambda[1] - throughputLambda[0]))
492 for i, filterName
in enumerate(self.config.filterNames):
494 tDict[
'LAMBDA'] = throughputLambda
495 for ccdIndex, detector
in enumerate(camera):
496 tDict[ccdIndex] = self.
_getThroughputDetector_getThroughputDetector(detector, filterName, throughputLambda)
497 throughputDict[filterName] = tDict
500 self.
fgcmLutMakerfgcmLutMaker.setThroughputs(throughputDict)
503 self.log.info(
"Making LUT")
510 filterNameString = comma.join(self.config.filterNames)
511 stdFilterNameString = comma.join(self.config.stdFilterNames)
513 atmosphereTableName =
'NoTableWasUsed'
514 if self.config.atmosphereTableName
is not None:
515 atmosphereTableName = self.config.atmosphereTableName
517 lutSchema = self.
_makeLutSchema_makeLutSchema(filterNameString, stdFilterNameString,
520 lutCat = self.
_makeLutCat_makeLutCat(lutSchema, filterNameString,
521 stdFilterNameString, atmosphereTableName)
524 def _createLutConfig(self, nCcd):
526 Create the fgcmLut config dictionary
531 Number of CCDs in the camera
536 lutConfig[
'logger'] = self.log
537 lutConfig[
'filterNames'] = self.config.filterNames
538 lutConfig[
'stdFilterNames'] = self.config.stdFilterNames
539 lutConfig[
'nCCD'] = nCcd
542 if self.config.atmosphereTableName
is not None:
543 lutConfig[
'atmosphereTableName'] = self.config.atmosphereTableName
546 lutConfig[
'elevation'] = self.config.parameters.elevation
547 lutConfig[
'pmbRange'] = self.config.parameters.pmbRange
548 lutConfig[
'pmbSteps'] = self.config.parameters.pmbSteps
549 lutConfig[
'pwvRange'] = self.config.parameters.pwvRange
550 lutConfig[
'pwvSteps'] = self.config.parameters.pwvSteps
551 lutConfig[
'o3Range'] = self.config.parameters.o3Range
552 lutConfig[
'o3Steps'] = self.config.parameters.o3Steps
553 lutConfig[
'tauRange'] = self.config.parameters.tauRange
554 lutConfig[
'tauSteps'] = self.config.parameters.tauSteps
555 lutConfig[
'alphaRange'] = self.config.parameters.alphaRange
556 lutConfig[
'alphaSteps'] = self.config.parameters.alphaSteps
557 lutConfig[
'zenithRange'] = self.config.parameters.zenithRange
558 lutConfig[
'zenithSteps'] = self.config.parameters.zenithSteps
559 lutConfig[
'pmbStd'] = self.config.parameters.pmbStd
560 lutConfig[
'pwvStd'] = self.config.parameters.pwvStd
561 lutConfig[
'o3Std'] = self.config.parameters.o3Std
562 lutConfig[
'tauStd'] = self.config.parameters.tauStd
563 lutConfig[
'alphaStd'] = self.config.parameters.alphaStd
564 lutConfig[
'airmassStd'] = self.config.parameters.airmassStd
565 lutConfig[
'lambdaRange'] = self.config.parameters.lambdaRange
566 lutConfig[
'lambdaStep'] = self.config.parameters.lambdaStep
567 lutConfig[
'lambdaNorm'] = self.config.parameters.lambdaNorm
571 def _loadThroughputs(self, camera, opticsDataRef, sensorDataRefDict, filterDataRefDict):
572 """Internal method to load throughput data for filters
576 camera: `lsst.afw.cameraGeom.Camera`
577 Camera from the butler
578 opticsDataRef : `lsst.daf.persistence.ButlerDataRef` or
579 `lsst.daf.butler.DeferredDatasetHandle`
580 Reference to optics transmission curve.
581 sensorDataRefDict : `dict` of [`int`, `lsst.daf.persistence.ButlerDataRef` or
582 `lsst.daf.butler.DeferredDatasetHandle`]
583 Dictionary of references to sensor transmission curves. Key will
585 filterDataRefDict : `dict` of [`str`, `lsst.daf.persistence.ButlerDataRef` or
586 `lsst.daf.butler.DeferredDatasetHandle`]
587 Dictionary of references to filter transmission curves. Key will
588 be physical filter name.
592 ValueError : Raised if configured filter name does not match any of the
593 available filter transmission curves.
598 for detector
in camera:
599 self.
_sensorsTransmission_sensorsTransmission[detector.getId()] = sensorDataRefDict[detector.getId()].get()
602 for filterName
in self.config.filterNames:
603 f = Filter(filterName)
606 aliases = f.getAliases()
607 aliases.extend(filterName)
608 for alias
in f.getAliases():
609 if alias
in filterDataRefDict:
614 raise ValueError(
"Could not find transmission for filter %s via any alias." %
617 def _getThroughputDetector(self, detector, filterName, throughputLambda):
618 """Internal method to get throughput for a detector.
620 Returns the throughput at the center of the detector for a given filter.
624 detector: `lsst.afw.cameraGeom._detector.Detector`
627 Short name for filter
628 throughputLambda: `np.array(dtype=np.float64)`
629 Wavelength steps (Angstrom)
633 throughput: `np.array(dtype=np.float64)`
634 Throughput (max 1.0) at throughputLambda
637 c = detector.getCenter(afwCameraGeom.FOCAL_PLANE)
638 c.scale(1.0/detector.getPixelSize()[0])
641 wavelengths=throughputLambda)
643 throughput *= self.
_sensorsTransmission_sensorsTransmission[detector.getId()].sampleAt(position=c,
644 wavelengths=throughputLambda)
647 wavelengths=throughputLambda)
650 throughput = np.clip(throughput, 0.0, 1.0)
654 def _makeLutSchema(self, filterNameString, stdFilterNameString,
655 atmosphereTableName):
661 filterNameString: `str`
662 Combined string of all the filterNames
663 stdFilterNameString: `str`
664 Combined string of all the standard filterNames
665 atmosphereTableName: `str`
666 Name of the atmosphere table used to generate LUT
670 lutSchema: `afwTable.schema`
673 lutSchema = afwTable.Schema()
675 lutSchema.addField(
'tablename', type=str, doc=
'Atmosphere table name',
676 size=len(atmosphereTableName))
677 lutSchema.addField(
'elevation', type=float, doc=
"Telescope elevation used for LUT")
678 lutSchema.addField(
'filterNames', type=str, doc=
'filterNames in LUT',
679 size=len(filterNameString))
680 lutSchema.addField(
'stdFilterNames', type=str, doc=
'Standard filterNames in LUT',
681 size=len(stdFilterNameString))
682 lutSchema.addField(
'pmb', type=
'ArrayD', doc=
'Barometric Pressure',
684 lutSchema.addField(
'pmbFactor', type=
'ArrayD', doc=
'PMB scaling factor',
686 lutSchema.addField(
'pmbElevation', type=np.float64, doc=
'PMB Scaling at elevation')
687 lutSchema.addField(
'pwv', type=
'ArrayD', doc=
'Preciptable Water Vapor',
689 lutSchema.addField(
'o3', type=
'ArrayD', doc=
'Ozone',
691 lutSchema.addField(
'tau', type=
'ArrayD', doc=
'Aerosol optical depth',
693 lutSchema.addField(
'lambdaNorm', type=np.float64, doc=
'AOD wavelength')
694 lutSchema.addField(
'alpha', type=
'ArrayD', doc=
'Aerosol alpha',
696 lutSchema.addField(
'zenith', type=
'ArrayD', doc=
'Zenith angle',
698 lutSchema.addField(
'nCcd', type=np.int32, doc=
'Number of CCDs')
701 lutSchema.addField(
'pmbStd', type=np.float64, doc=
'PMB Standard')
702 lutSchema.addField(
'pwvStd', type=np.float64, doc=
'PWV Standard')
703 lutSchema.addField(
'o3Std', type=np.float64, doc=
'O3 Standard')
704 lutSchema.addField(
'tauStd', type=np.float64, doc=
'Tau Standard')
705 lutSchema.addField(
'alphaStd', type=np.float64, doc=
'Alpha Standard')
706 lutSchema.addField(
'zenithStd', type=np.float64, doc=
'Zenith angle Standard')
707 lutSchema.addField(
'lambdaRange', type=
'ArrayD', doc=
'Wavelength range',
709 lutSchema.addField(
'lambdaStep', type=np.float64, doc=
'Wavelength step')
710 lutSchema.addField(
'lambdaStd', type=
'ArrayD', doc=
'Standard Wavelength',
712 lutSchema.addField(
'lambdaStdFilter', type=
'ArrayD', doc=
'Standard Wavelength (raw)',
714 lutSchema.addField(
'i0Std', type=
'ArrayD', doc=
'I0 Standard',
716 lutSchema.addField(
'i1Std', type=
'ArrayD', doc=
'I1 Standard',
718 lutSchema.addField(
'i10Std', type=
'ArrayD', doc=
'I10 Standard',
720 lutSchema.addField(
'i2Std', type=
'ArrayD', doc=
'I2 Standard',
722 lutSchema.addField(
'lambdaB', type=
'ArrayD', doc=
'Wavelength for passband (no atm)',
724 lutSchema.addField(
'atmLambda', type=
'ArrayD', doc=
'Atmosphere wavelengths (Angstrom)',
726 lutSchema.addField(
'atmStdTrans', type=
'ArrayD', doc=
'Standard Atmosphere Throughput',
730 lutSchema.addField(
'luttype', type=str, size=20, doc=
'Look-up table type')
731 lutSchema.addField(
'lut', type=
'ArrayF', doc=
'Look-up table for luttype',
736 def _makeLutCat(self, lutSchema, filterNameString, stdFilterNameString,
737 atmosphereTableName):
743 lutSchema: `afwTable.schema`
745 filterNameString: `str`
746 Combined string of all the filterNames
747 stdFilterNameString: `str`
748 Combined string of all the standard filterNames
749 atmosphereTableName: `str`
750 Name of the atmosphere table used to generate LUT
754 lutCat: `afwTable.BaseCatalog`
755 Lut catalog for persistence
762 lutCat = afwTable.BaseCatalog(lutSchema)
763 lutCat.table.preallocate(14)
766 rec = lutCat.addNew()
768 rec[
'tablename'] = atmosphereTableName
769 rec[
'elevation'] = self.
fgcmLutMakerfgcmLutMaker.atmosphereTable.elevation
770 rec[
'filterNames'] = filterNameString
771 rec[
'stdFilterNames'] = stdFilterNameString
773 rec[
'pmbFactor'][:] = self.
fgcmLutMakerfgcmLutMaker.pmbFactor
774 rec[
'pmbElevation'] = self.
fgcmLutMakerfgcmLutMaker.pmbElevation
778 rec[
'lambdaNorm'] = self.
fgcmLutMakerfgcmLutMaker.lambdaNorm
787 rec[
'alphaStd'] = self.
fgcmLutMakerfgcmLutMaker.alphaStd
788 rec[
'zenithStd'] = self.
fgcmLutMakerfgcmLutMaker.zenithStd
789 rec[
'lambdaRange'][:] = self.
fgcmLutMakerfgcmLutMaker.lambdaRange
790 rec[
'lambdaStep'] = self.
fgcmLutMakerfgcmLutMaker.lambdaStep
791 rec[
'lambdaStd'][:] = self.
fgcmLutMakerfgcmLutMaker.lambdaStd
792 rec[
'lambdaStdFilter'][:] = self.
fgcmLutMakerfgcmLutMaker.lambdaStdFilter
797 rec[
'lambdaB'][:] = self.
fgcmLutMakerfgcmLutMaker.lambdaB
798 rec[
'atmLambda'][:] = self.
fgcmLutMakerfgcmLutMaker.atmLambda
799 rec[
'atmStdTrans'][:] = self.
fgcmLutMakerfgcmLutMaker.atmStdTrans
801 rec[
'luttype'] =
'I0'
802 rec[
'lut'][:] = self.
fgcmLutMakerfgcmLutMaker.lut[
'I0'].flatten()
805 rec = lutCat.addNew()
806 rec[
'luttype'] =
'I1'
807 rec[
'lut'][:] = self.
fgcmLutMakerfgcmLutMaker.lut[
'I1'].flatten()
809 derivTypes = [
'D_PMB',
'D_LNPWV',
'D_O3',
'D_LNTAU',
'D_ALPHA',
'D_SECZENITH',
810 'D_PMB_I1',
'D_LNPWV_I1',
'D_O3_I1',
'D_LNTAU_I1',
'D_ALPHA_I1',
812 for derivType
in derivTypes:
813 rec = lutCat.addNew()
814 rec[
'luttype'] = derivType
815 rec[
'lut'][:] = self.
fgcmLutMakerfgcmLutMaker.lutDeriv[derivType].flatten()
def getTargetList(parsedCmd)
def __call__(self, butler)
def _createLutConfig(self, nCcd)
def _makeLutSchema(self, filterNameString, stdFilterNameString, atmosphereTableName)
def runDataRef(self, butler)
def _fgcmMakeLut(self, camera, opticsDataRef, sensorDataRefDict, filterDataRefDict)
def _getThroughputDetector(self, detector, filterName, throughputLambda)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def _loadThroughputs(self, camera, opticsDataRef, sensorDataRefDict, filterDataRefDict)
def _makeLutCat(self, lutSchema, filterNameString, stdFilterNameString, atmosphereTableName)
def __init__(self, butler=None, initInputs=None, **kwargs)