Coverage for python/lsst/obs/subaru/_instrument.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_subaru.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Gen3 Butler registry declarations for Hyper Suprime-Cam.
23"""
25__all__ = ("HyperSuprimeCam",)
27import os
28import pickle
29import logging
31import astropy.time
32from lsst.utils import getPackageDir
33from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig
34from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef,
35 TIMESPAN_MIN)
36from lsst.daf.butler.core.utils import getFullTypeName
37from lsst.obs.base import Instrument, addUnboundedCalibrationLabel
38from lsst.obs.base.gen2to3 import TranslatorFactory, PhysicalToAbstractFilterKeyHandler
40from ..hsc.hscPupil import HscPupilFactory
41from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS
42from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission,
43 getFilterTransmission, getAtmosphereTransmission)
44from .strayLight.formatter import SubaruStrayLightDataFormatter
46log = logging.getLogger(__name__)
49class HyperSuprimeCam(Instrument):
50 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam.
51 """
53 policyName = "hsc"
54 obsDataPackage = "obs_subaru_data"
55 filterDefinitions = HSC_FILTER_DEFINITIONS
57 def __init__(self, **kwargs):
58 super().__init__(**kwargs)
59 packageDir = getPackageDir("obs_subaru")
60 self.configPaths = [os.path.join(packageDir, "config"),
61 os.path.join(packageDir, "config", self.policyName)]
63 @classmethod
64 def getName(cls):
65 # Docstring inherited from Instrument.getName
66 return "HSC"
68 def register(self, registry):
69 # Docstring inherited from Instrument.register
70 camera = self.getCamera()
71 # The maximum values below make Gen3's ObservationDataIdPacker produce
72 # outputs that match Gen2's ccdExposureId.
73 obsMax = 21474800
74 registry.insertDimensionData(
75 "instrument",
76 {
77 "name": self.getName(),
78 "detector_max": 200,
79 "visit_max": obsMax,
80 "exposure_max": obsMax,
81 "class_name": getFullTypeName(self),
82 }
83 )
84 registry.insertDimensionData(
85 "detector",
86 *[
87 {
88 "instrument": self.getName(),
89 "id": detector.getId(),
90 "full_name": detector.getName(),
91 # TODO: make sure these definitions are consistent with those
92 # extracted by astro_metadata_translator, and test that they
93 # remain consistent somehow.
94 "name_in_raft": detector.getName().split("_")[1],
95 "raft": detector.getName().split("_")[0],
96 "purpose": str(detector.getType()).split(".")[-1],
97 }
98 for detector in camera
99 ]
100 )
101 self._registerFilters(registry)
103 def getRawFormatter(self, dataId):
104 # Docstring inherited from Instrument.getRawFormatter
105 # Import the formatter here to prevent a circular dependency.
106 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter
107 if dataId["detector"] in (100, 101, 102, 103):
108 return HyperSuprimeCamCornerRawFormatter
109 else:
110 return HyperSuprimeCamRawFormatter
112 def getCamera(self):
113 """Retrieve the cameraGeom representation of HSC.
115 This is a temporary API that should go away once obs_ packages have
116 a standardized approach to writing versioned cameras to a Gen3 repo.
117 """
118 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "camera")
119 config = CameraConfig()
120 config.load(os.path.join(path, "camera.py"))
121 return makeCameraFromPath(
122 cameraConfig=config,
123 ampInfoPath=path,
124 shortNameFunc=lambda name: name.replace(" ", "_"),
125 pupilFactoryClass=HscPupilFactory
126 )
128 def getBrighterFatterKernel(self):
129 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`.
131 This is a temporary API that should go away once obs_ packages have
132 a standardized approach to writing versioned kernels to a Gen3 repo.
133 """
134 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "brighter_fatter_kernel.pkl")
135 with open(path, "rb") as fd:
136 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2
137 return kernel
139 def writeCuratedCalibrations(self, butler):
140 """Write human-curated calibration Datasets to the given Butler with
141 the appropriate validity ranges.
143 This is a temporary API that should go away once obs_ packages have
144 a standardized approach to this problem.
145 """
147 # Write the generic calibrations supported by all instruments
148 super().writeCuratedCalibrations(butler)
150 # Get an unbounded calibration label
151 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
153 # Write brighter-fatter kernel, with an infinite validity range.
154 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray",
155 universe=butler.registry.dimensions)
156 butler.registry.registerDatasetType(datasetType)
157 # Load and then put instead of just moving the file in part to ensure
158 # the version in-repo is written with Python 3 and does not need
159 # `encoding='latin1'` to be read.
160 bfKernel = self.getBrighterFatterKernel()
161 butler.put(bfKernel, datasetType, unboundedDataId)
163 # The following iterate over the values of the dictionaries returned by the transmission functions
164 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
165 # which do not give the proper notion of validity. As such unbounded calibration labels are used
166 # when inserting into the database. In the future these could and probably should be updated to
167 # properly account for what ranges are considered valid.
169 # Write optical transmissions
170 opticsTransmissions = getOpticsTransmission()
171 datasetType = DatasetType("transmission_optics",
172 ("instrument", "calibration_label"),
173 "TransmissionCurve",
174 universe=butler.registry.dimensions)
175 butler.registry.registerDatasetType(datasetType)
176 for entry in opticsTransmissions.values():
177 if entry is None:
178 continue
179 butler.put(entry, datasetType, unboundedDataId)
181 # Write transmission sensor
182 sensorTransmissions = getSensorTransmission()
183 datasetType = DatasetType("transmission_sensor",
184 ("instrument", "detector", "calibration_label"),
185 "TransmissionCurve",
186 universe=butler.registry.dimensions)
187 butler.registry.registerDatasetType(datasetType)
188 for entry in sensorTransmissions.values():
189 if entry is None:
190 continue
191 for sensor, curve in entry.items():
192 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor)
193 butler.put(curve, datasetType, dataId)
195 # Write filter transmissions
196 filterTransmissions = getFilterTransmission()
197 datasetType = DatasetType("transmission_filter",
198 ("instrument", "physical_filter", "calibration_label"),
199 "TransmissionCurve",
200 universe=butler.registry.dimensions)
201 butler.registry.registerDatasetType(datasetType)
202 for entry in filterTransmissions.values():
203 if entry is None:
204 continue
205 for band, curve in entry.items():
206 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band)
207 butler.put(curve, datasetType, dataId)
209 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only
210 # look up along this dimension (ISR)
211 atmosphericTransmissions = getAtmosphereTransmission()
212 datasetType = DatasetType("transmission_atmosphere", ("instrument",),
213 "TransmissionCurve",
214 universe=butler.registry.dimensions)
215 butler.registry.registerDatasetType(datasetType)
216 for entry in atmosphericTransmissions.values():
217 if entry is None:
218 continue
219 butler.put(entry, datasetType, {"instrument": self.getName()})
221 def ingestStrayLightData(self, butler, directory, *, transfer=None):
222 """Ingest externally-produced y-band stray light data files into
223 a data repository.
225 Parameters
226 ----------
227 butler : `lsst.daf.butler.Butler`
228 Butler initialized with the collection to ingest into.
229 directory : `str`
230 Directory containing yBackground-*.fits files.
231 transfer : `str`, optional
232 If not `None`, must be one of 'move', 'copy', 'hardlink', or
233 'symlink', indicating how to transfer the files.
234 """
235 calibrationLabel = "y-LED-encoder-on"
236 # LEDs covered up around 2018-01-01, no need for correctin after that
237 # date.
238 datetime_begin = TIMESPAN_MIN
239 datetime_end = astropy.time.Time("2018-01-01", format="iso", scale="tai")
240 datasets = []
241 # TODO: should we use a more generic name for the dataset type?
242 # This is just the (rather HSC-specific) name used in Gen2, and while
243 # the instances of this dataset are camera-specific, the datasetType
244 # (which is used in the generic IsrTask) should not be.
245 datasetType = DatasetType("yBackground",
246 dimensions=("physical_filter", "detector", "calibration_label"),
247 storageClass="StrayLightData",
248 universe=butler.registry.dimensions)
249 for detector in self.getCamera():
250 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits")
251 if not os.path.exists(path):
252 log.warn(f"No stray light data found for detector {detector.getId()} @ {path}.")
253 continue
254 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(),
255 "detector": detector.getId(),
256 "physical_filter": "HSC-Y",
257 "calibration_label": calibrationLabel})
258 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter))
259 butler.registry.registerDatasetType(datasetType)
260 with butler.transaction():
261 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(),
262 "name": calibrationLabel,
263 "datetime_begin": datetime_begin,
264 "datetime_end": datetime_end})
265 butler.ingest(*datasets, transfer=transfer)
267 def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
268 # Docstring inherited from lsst.obs.base.Instrument.
269 factory = TranslatorFactory()
270 factory.addGenericInstrumentRules(self.getName())
271 # Translate Gen2 `filter` to abstract_filter if it hasn't been consumed
272 # yet and gen2keys includes tract.
273 factory.addRule(PhysicalToAbstractFilterKeyHandler(self.filterDefinitions),
274 instrument=self.getName(), gen2keys=("filter", "tract"), consume=("filter",))
275 return factory