23"""Make a look-up-table (LUT) for FGCM calibration.
25This task computes a look-up-table for the range in expected atmosphere
26variation and variation in instrumental throughput (as tracked by the
27transmission_filter products). By pre-computing linearized integrals,
28the FGCM fit is orders of magnitude faster
for stars
with a broad range
29of colors
and observing bands, yielding precision at the 1-2 mmag level.
31Computing a LUT requires running MODTRAN
or with a pre-generated
32atmosphere table packaged
with fgcm.
37import lsst.pex.config as pexConfig
38import lsst.pipe.base as pipeBase
39from lsst.pipe.base import connectionTypes
40import lsst.afw.table as afwTable
41import lsst.afw.cameraGeom as afwCameraGeom
42from lsst.afw.image import TransmissionCurve
43from .utilities import lookupStaticCalibrations
47__all__ = ['FgcmMakeLutParametersConfig', 'FgcmMakeLutConfig', 'FgcmMakeLutTask']
50class FgcmMakeLutConnections(pipeBase.PipelineTaskConnections,
51 dimensions=('instrument',),
53 camera = connectionTypes.PrerequisiteInput(
54 doc=
"Camera instrument",
56 storageClass=
"Camera",
57 dimensions=(
"instrument",),
61 transmission_optics = connectionTypes.PrerequisiteInput(
62 doc=
"Optics transmission curve information",
63 name=
"transmission_optics",
64 storageClass=
"TransmissionCurve",
65 dimensions=(
"instrument",),
70 transmission_sensor = connectionTypes.PrerequisiteInput(
71 doc=
"Sensor transmission curve information",
72 name=
"transmission_sensor",
73 storageClass=
"TransmissionCurve",
74 dimensions=(
"instrument",
"detector",),
75 lookupFunction=lookupStaticCalibrations,
81 transmission_filter = connectionTypes.PrerequisiteInput(
82 doc=
"Filter transmission curve information",
83 name=
"transmission_filter",
84 storageClass=
"TransmissionCurve",
85 dimensions=(
"band",
"instrument",
"physical_filter",),
86 lookupFunction=lookupStaticCalibrations,
92 fgcmLookUpTable = connectionTypes.Output(
93 doc=(
"Atmosphere + instrument look-up-table for FGCM throughput and "
94 "chromatic corrections."),
95 name=
"fgcmLookUpTable",
96 storageClass=
"Catalog",
97 dimensions=(
"instrument",),
100 fgcmStandardAtmosphere = connectionTypes.Output(
101 doc=
"Standard atmosphere used for FGCM calibration.",
102 name=
"fgcm_standard_atmosphere",
103 storageClass=
"TransmissionCurve",
104 dimensions=(
"instrument",),
107 fgcmStandardPassbands = connectionTypes.Output(
108 doc=
"Standard passbands used for FGCM calibration.",
109 name=
"fgcm_standard_passband",
110 storageClass=
"TransmissionCurve",
111 dimensions=(
"instrument",
"physical_filter"),
116 if not config.doOpticsTransmission:
118 if not config.doSensorTransmission:
123 """Config for parameters if atmosphereTableName not available"""
126 elevation = pexConfig.Field(
127 doc=
"Telescope elevation (m)",
131 pmbRange = pexConfig.ListField(
132 doc=(
"Barometric Pressure range (millibar) "
133 "Recommended range depends on the site."),
137 pmbSteps = pexConfig.Field(
138 doc=
"Barometric Pressure number of steps",
142 pwvRange = pexConfig.ListField(
143 doc=(
"Precipitable Water Vapor range (mm) "
144 "Recommended range depends on the site."),
148 pwvSteps = pexConfig.Field(
149 doc=
"Precipitable Water Vapor number of steps",
153 o3Range = pexConfig.ListField(
154 doc=
"Ozone range (dob)",
156 default=[220.0, 310.0],
158 o3Steps = pexConfig.Field(
159 doc=
"Ozone number of steps",
163 tauRange = pexConfig.ListField(
164 doc=
"Aerosol Optical Depth range (unitless)",
166 default=[0.002, 0.35],
168 tauSteps = pexConfig.Field(
169 doc=
"Aerosol Optical Depth number of steps",
173 alphaRange = pexConfig.ListField(
174 doc=
"Aerosol alpha range (unitless)",
178 alphaSteps = pexConfig.Field(
179 doc=
"Aerosol alpha number of steps",
183 zenithRange = pexConfig.ListField(
184 doc=
"Zenith angle range (degree)",
188 zenithSteps = pexConfig.Field(
189 doc=
"Zenith angle number of steps",
195 pmbStd = pexConfig.Field(
196 doc=(
"Standard Atmosphere pressure (millibar); "
197 "Recommended default depends on the site."),
201 pwvStd = pexConfig.Field(
202 doc=(
"Standard Atmosphere PWV (mm); "
203 "Recommended default depends on the site."),
207 o3Std = pexConfig.Field(
208 doc=
"Standard Atmosphere O3 (dob)",
212 tauStd = pexConfig.Field(
213 doc=
"Standard Atmosphere aerosol optical depth",
217 alphaStd = pexConfig.Field(
218 doc=
"Standard Atmosphere aerosol alpha",
222 airmassStd = pexConfig.Field(
223 doc=(
"Standard Atmosphere airmass; "
224 "Recommended default depends on the survey strategy."),
228 lambdaNorm = pexConfig.Field(
229 doc=
"Aerosol Optical Depth normalization wavelength (Angstrom)",
233 lambdaStep = pexConfig.Field(
234 doc=
"Wavelength step for generating atmospheres (nm)",
238 lambdaRange = pexConfig.ListField(
239 doc=
"Wavelength range for LUT (Angstrom)",
241 default=[3000.0, 11000.0],
246 pipelineConnections=FgcmMakeLutConnections):
247 """Config for FgcmMakeLutTask"""
248 physicalFilters = pexConfig.ListField(
249 doc=
"List of physicalFilter labels to generate look-up table.",
253 stdPhysicalFilterOverrideMap = pexConfig.DictField(
254 doc=(
"Override mapping from physical filter labels to 'standard' physical "
255 "filter labels. The 'standard' physical filter defines the transmission "
256 "curve that the FGCM standard bandpass will be based on. "
257 "Any filter not listed here will be mapped to "
258 "itself (e.g. g->g or HSC-G->HSC-G). Use this override for cross-"
259 "filter calibration such as HSC-R->HSC-R2 and HSC-I->HSC-I2."),
264 atmosphereTableName = pexConfig.Field(
265 doc=
"FGCM name or filename of precomputed atmospheres",
270 doOpticsTransmission = pexConfig.Field(
271 doc=
"Include optics transmission?",
275 doSensorTransmission = pexConfig.Field(
276 doc=
"Include sensor transmission?",
280 parameters = pexConfig.ConfigField(
281 doc=
"Atmosphere parameters (required if no atmosphereTableName)",
282 dtype=FgcmMakeLutParametersConfig,
288 Validate the config parameters.
290 This method behaves differently from the parent validate
in the case
291 that atmosphereTableName
is set. In this case, the config values
292 for standard values, step sizes,
and ranges are loaded
293 directly
from the specified atmosphereTableName.
296 self._fields[
'physicalFilters'].
validate(self)
297 self._fields[
'stdPhysicalFilterOverrideMap'].
validate(self)
301 self._fields[
'parameters'].
validate(self)
306 Make Look-Up Table for FGCM.
308 This task computes a look-up-table
for the range
in expected atmosphere
309 variation
and variation
in instrumental throughput (
as tracked by the
310 transmission_filter products). By pre-computing linearized integrals,
311 the FGCM fit
is orders of magnitude faster
for stars
with a broad range
312 of colors
and observing bands, yielding precision at the 1-2 mmag level.
314 Computing a LUT requires running MODTRAN
or with a pre-generated
315 atmosphere table packaged
with fgcm.
318 ConfigClass = FgcmMakeLutConfig
319 _DefaultName = "fgcmMakeLut"
325 camera = butlerQC.get(inputRefs.camera)
327 if self.config.doOpticsTransmission:
328 opticsHandle = butlerQC.get(inputRefs.transmission_optics)
332 if self.config.doSensorTransmission:
333 sensorHandles = butlerQC.get(inputRefs.transmission_sensor)
334 sensorHandleDict = {sensorHandle.dataId.byName()[
'detector']: sensorHandle
for
335 sensorHandle
in sensorHandles}
337 sensorHandleDict = {}
339 filterHandles = butlerQC.get(inputRefs.transmission_filter)
340 filterHandleDict = {filterHandle.dataId[
'physical_filter']: filterHandle
for
341 filterHandle
in filterHandles}
348 butlerQC.put(struct.fgcmLookUpTable, outputRefs.fgcmLookUpTable)
349 butlerQC.put(struct.fgcmStandardAtmosphere, outputRefs.fgcmStandardAtmosphere)
351 refDict = {passbandRef.dataId[
'physical_filter']: passbandRef
for
352 passbandRef
in outputRefs.fgcmStandardPassbands}
353 for physical_filter, passband
in struct.fgcmStandardPassbands.items():
354 butlerQC.put(passband, refDict[physical_filter])
359 Make a FGCM Look-up Table
363 camera : `lsst.afw.cameraGeom.Camera`
364 Camera from the butler.
365 opticsHandle : `lsst.daf.butler.DeferredDatasetHandle`
366 Reference to optics transmission curve.
367 sensorHandleDict : `dict` of [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
368 Dictionary of references to sensor transmission curves. Key will
370 filterHandleDict : `dict` of [`str`, `lsst.daf.butler.DeferredDatasetHandle`]
371 Dictionary of references to filter transmission curves. Key will
372 be physical filter label.
376 retStruct : `lsst.pipe.base.Struct`
377 Output structure
with keys:
379 fgcmLookUpTable : `BaseCatalog`
380 The FGCM look-up table.
381 fgcmStandardAtmosphere : `lsst.afw.image.TransmissionCurve`
382 Transmission curve
for the FGCM standard atmosphere.
383 fgcmStandardPassbands : `dict` [`str`, `lsst.afw.image.TransmissionCurve`]
384 Dictionary of standard passbands,
with the key
as the
385 physical filter name.
389 self.log.info(
"Found %d ccds for look-up table" % (nCcd))
400 self.log.info(
"Making the LUT maker object")
408 throughputLambda = np.arange(self.
fgcmLutMaker.lambdaRange[0],
412 self.log.info(
"Built throughput lambda, %.1f-%.1f, step %.2f" %
413 (throughputLambda[0], throughputLambda[-1],
414 throughputLambda[1] - throughputLambda[0]))
417 for i, physicalFilter
in enumerate(self.config.physicalFilters):
419 tDict[
'LAMBDA'] = throughputLambda
420 for ccdIndex, detector
in enumerate(camera):
422 throughputDict[physicalFilter] = tDict
428 self.log.info(
"Making LUT")
435 physicalFilterString = comma.join(self.config.physicalFilters)
438 atmosphereTableName =
'NoTableWasUsed'
439 if self.config.atmosphereTableName
is not None:
440 atmosphereTableName = self.config.atmosphereTableName
442 lutSchema = self.
_makeLutSchema(physicalFilterString, stdPhysicalFilterString,
445 lutCat = self.
_makeLutCat(lutSchema, physicalFilterString,
446 stdPhysicalFilterString, atmosphereTableName)
448 atmStd = TransmissionCurve.makeSpatiallyConstant(
449 throughput=self.
fgcmLutMaker.atmStdTrans.astype(np.float64),
450 wavelengths=self.
fgcmLutMaker.atmLambda.astype(np.float64),
455 fgcmStandardPassbands = {}
456 for i, physical_filter
in enumerate(self.
fgcmLutMaker.filterNames):
458 fgcmStandardPassbands[physical_filter] = TransmissionCurve.makeSpatiallyConstant(
459 throughput=passband.astype(np.float64),
460 wavelengths=self.
fgcmLutMaker.atmLambda.astype(np.float64),
461 throughputAtMin=passband[0],
462 throughputAtMax=passband[-1],
465 retStruct = pipeBase.Struct(
466 fgcmLookUpTable=lutCat,
467 fgcmStandardAtmosphere=atmStd,
468 fgcmStandardPassbands=fgcmStandardPassbands,
474 """Get the standard physical filter lists from config.physicalFilters
475 and config.stdPhysicalFilterOverrideMap
479 stdPhysicalFilters : `list`
481 override = self.config.stdPhysicalFilterOverrideMap
482 return [override.get(physicalFilter, physicalFilter)
for
483 physicalFilter
in self.config.physicalFilters]
487 Create the fgcmLut config dictionary
492 Number of CCDs in the camera
497 lutConfig[
'logger'] = self.log
498 lutConfig[
'filterNames'] = self.config.physicalFilters
500 lutConfig[
'nCCD'] = nCcd
503 if self.config.atmosphereTableName
is not None:
504 lutConfig[
'atmosphereTableName'] = self.config.atmosphereTableName
507 lutConfig[
'elevation'] = self.config.parameters.elevation
508 lutConfig[
'pmbRange'] = self.config.parameters.pmbRange
509 lutConfig[
'pmbSteps'] = self.config.parameters.pmbSteps
510 lutConfig[
'pwvRange'] = self.config.parameters.pwvRange
511 lutConfig[
'pwvSteps'] = self.config.parameters.pwvSteps
512 lutConfig[
'o3Range'] = self.config.parameters.o3Range
513 lutConfig[
'o3Steps'] = self.config.parameters.o3Steps
514 lutConfig[
'tauRange'] = self.config.parameters.tauRange
515 lutConfig[
'tauSteps'] = self.config.parameters.tauSteps
516 lutConfig[
'alphaRange'] = self.config.parameters.alphaRange
517 lutConfig[
'alphaSteps'] = self.config.parameters.alphaSteps
518 lutConfig[
'zenithRange'] = self.config.parameters.zenithRange
519 lutConfig[
'zenithSteps'] = self.config.parameters.zenithSteps
520 lutConfig[
'pmbStd'] = self.config.parameters.pmbStd
521 lutConfig[
'pwvStd'] = self.config.parameters.pwvStd
522 lutConfig[
'o3Std'] = self.config.parameters.o3Std
523 lutConfig[
'tauStd'] = self.config.parameters.tauStd
524 lutConfig[
'alphaStd'] = self.config.parameters.alphaStd
525 lutConfig[
'airmassStd'] = self.config.parameters.airmassStd
526 lutConfig[
'lambdaRange'] = self.config.parameters.lambdaRange
527 lutConfig[
'lambdaStep'] = self.config.parameters.lambdaStep
528 lutConfig[
'lambdaNorm'] = self.config.parameters.lambdaNorm
533 """Internal method to load throughput data for filters
537 camera: `lsst.afw.cameraGeom.Camera`
538 Camera from the butler
539 opticsHandle : `lsst.daf.butler.DeferredDatasetHandle`
540 Reference to optics transmission curve.
541 sensorHandleDict : `dict` of [`int`, `lsst.daf.butler.DeferredDatasetHandle`]
542 Dictionary of references to sensor transmission curves. Key will
544 filterHandleDict : `dict` of [`str`, `lsst.daf.butler.DeferredDatasetHandle`]
545 Dictionary of references to filter transmission curves. Key will
546 be physical filter label.
550 ValueError : Raised
if configured filter name does
not match any of the
551 available filter transmission curves.
553 if self.config.doOpticsTransmission:
557 throughput=np.ones(100),
558 wavelengths=np.linspace(
559 self.config.parameters.lambdaRange[0],
560 self.config.parameters.lambdaRange[1],
568 for detector
in camera:
569 if self.config.doSensorTransmission:
573 throughput=np.ones(100),
574 wavelengths=np.linspace(
575 self.config.parameters.lambdaRange[0],
576 self.config.parameters.lambdaRange[1],
584 for physicalFilter
in self.config.physicalFilters:
588 """Internal method to get throughput for a detector.
590 Returns the throughput at the center of the detector for a given filter.
594 detector: `lsst.afw.cameraGeom._detector.Detector`
596 physicalFilter: `str`
597 Physical filter label
598 throughputLambda: `np.array(dtype=np.float64)`
599 Wavelength steps (Angstrom)
603 throughput: `np.array(dtype=np.float64)`
604 Throughput (max 1.0) at throughputLambda
607 c = detector.getCenter(afwCameraGeom.FOCAL_PLANE)
608 c.scale(1.0/detector.getPixelSize()[0])
611 wavelengths=throughputLambda)
614 wavelengths=throughputLambda)
617 wavelengths=throughputLambda)
620 throughput = np.clip(throughput, 0.0, 1.0)
625 atmosphereTableName):
631 physicalFilterString: `str`
632 Combined string of all the physicalFilters
633 stdPhysicalFilterString: `str`
634 Combined string of all the standard physicalFilters
635 atmosphereTableName: `str`
636 Name of the atmosphere table used to generate LUT
640 lutSchema: `afwTable.schema`
643 lutSchema = afwTable.Schema()
645 lutSchema.addField('tablename', type=str, doc=
'Atmosphere table name',
646 size=len(atmosphereTableName))
647 lutSchema.addField(
'elevation', type=float, doc=
"Telescope elevation used for LUT")
648 lutSchema.addField(
'physicalFilters', type=str, doc=
'physicalFilters in LUT',
649 size=len(physicalFilterString))
650 lutSchema.addField(
'stdPhysicalFilters', type=str, doc=
'Standard physicalFilters in LUT',
651 size=len(stdPhysicalFilterString))
652 lutSchema.addField(
'pmb', type=
'ArrayD', doc=
'Barometric Pressure',
654 lutSchema.addField(
'pmbFactor', type=
'ArrayD', doc=
'PMB scaling factor',
656 lutSchema.addField(
'pmbElevation', type=np.float64, doc=
'PMB Scaling at elevation')
657 lutSchema.addField(
'pwv', type=
'ArrayD', doc=
'Preciptable Water Vapor',
659 lutSchema.addField(
'o3', type=
'ArrayD', doc=
'Ozone',
661 lutSchema.addField(
'tau', type=
'ArrayD', doc=
'Aerosol optical depth',
663 lutSchema.addField(
'lambdaNorm', type=np.float64, doc=
'AOD wavelength')
664 lutSchema.addField(
'alpha', type=
'ArrayD', doc=
'Aerosol alpha',
666 lutSchema.addField(
'zenith', type=
'ArrayD', doc=
'Zenith angle',
668 lutSchema.addField(
'nCcd', type=np.int32, doc=
'Number of CCDs')
671 lutSchema.addField(
'pmbStd', type=np.float64, doc=
'PMB Standard')
672 lutSchema.addField(
'pwvStd', type=np.float64, doc=
'PWV Standard')
673 lutSchema.addField(
'o3Std', type=np.float64, doc=
'O3 Standard')
674 lutSchema.addField(
'tauStd', type=np.float64, doc=
'Tau Standard')
675 lutSchema.addField(
'alphaStd', type=np.float64, doc=
'Alpha Standard')
676 lutSchema.addField(
'zenithStd', type=np.float64, doc=
'Zenith angle Standard')
677 lutSchema.addField(
'lambdaRange', type=
'ArrayD', doc=
'Wavelength range',
679 lutSchema.addField(
'lambdaStep', type=np.float64, doc=
'Wavelength step')
680 lutSchema.addField(
'lambdaStd', type=
'ArrayD', doc=
'Standard Wavelength',
682 lutSchema.addField(
'lambdaStdFilter', type=
'ArrayD', doc=
'Standard Wavelength (raw)',
684 lutSchema.addField(
'i0Std', type=
'ArrayD', doc=
'I0 Standard',
686 lutSchema.addField(
'i1Std', type=
'ArrayD', doc=
'I1 Standard',
688 lutSchema.addField(
'i10Std', type=
'ArrayD', doc=
'I10 Standard',
690 lutSchema.addField(
'i2Std', type=
'ArrayD', doc=
'I2 Standard',
692 lutSchema.addField(
'lambdaB', type=
'ArrayD', doc=
'Wavelength for passband (no atm)',
694 lutSchema.addField(
'atmLambda', type=
'ArrayD', doc=
'Atmosphere wavelengths (Angstrom)',
696 lutSchema.addField(
'atmStdTrans', type=
'ArrayD', doc=
'Standard Atmosphere Throughput',
700 lutSchema.addField(
'luttype', type=str, size=20, doc=
'Look-up table type')
701 lutSchema.addField(
'lut', type=
'ArrayF', doc=
'Look-up table for luttype',
706 def _makeLutCat(self, lutSchema, physicalFilterString, stdPhysicalFilterString,
707 atmosphereTableName):
713 lutSchema: `afwTable.schema`
715 physicalFilterString: `str`
716 Combined string of all the physicalFilters
717 stdPhysicalFilterString: `str`
718 Combined string of all the standard physicalFilters
719 atmosphereTableName: `str`
720 Name of the atmosphere table used to generate LUT
724 lutCat: `afwTable.BaseCatalog`
725 Look-up table catalog for persistence.
732 lutCat = afwTable.BaseCatalog(lutSchema)
733 lutCat.table.preallocate(14)
736 rec = lutCat.addNew()
738 rec[
'tablename'] = atmosphereTableName
739 rec[
'elevation'] = self.
fgcmLutMaker.atmosphereTable.elevation
740 rec[
'physicalFilters'] = physicalFilterString
741 rec[
'stdPhysicalFilters'] = stdPhysicalFilterString
762 rec[
'lambdaStdFilter'][:] = self.
fgcmLutMaker.lambdaStdFilter
771 rec[
'luttype'] =
'I0'
775 rec = lutCat.addNew()
776 rec[
'luttype'] =
'I1'
779 derivTypes = [
'D_PMB',
'D_LNPWV',
'D_O3',
'D_LNTAU',
'D_ALPHA',
'D_SECZENITH',
780 'D_PMB_I1',
'D_LNPWV_I1',
'D_O3_I1',
'D_LNTAU_I1',
'D_ALPHA_I1',
782 for derivType
in derivTypes:
783 rec = lutCat.addNew()
784 rec[
'luttype'] = derivType
785 rec[
'lut'][:] = self.
fgcmLutMaker.lutDeriv[derivType].flatten()
__init__(self, *config=None)
_makeLutSchema(self, physicalFilterString, stdPhysicalFilterString, atmosphereTableName)
_fgcmMakeLut(self, camera, opticsHandle, sensorHandleDict, filterHandleDict)
_getStdPhysicalFilterList(self)
_getThroughputDetector(self, detector, physicalFilter, throughputLambda)
_loadThroughputs(self, camera, opticsHandle, sensorHandleDict, filterHandleDict)
runQuantum(self, butlerQC, inputRefs, outputRefs)
_createLutConfig(self, nCcd)
__init__(self, initInputs=None, **kwargs)
_makeLutCat(self, lutSchema, physicalFilterString, stdPhysicalFilterString, atmosphereTableName)