Coverage for python/lsst/obs/subaru/_instrument.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_subaru.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Gen3 Butler registry declarations for Hyper Suprime-Cam.
23"""
25__all__ = ("HyperSuprimeCam",)
27import os
28import pickle
29import logging
31from functools import lru_cache
33import astropy.time
34from lsst.utils import getPackageDir
35from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig
36from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef,
37 CollectionType, Timespan)
38from lsst.daf.butler.core.utils import getFullTypeName
39from lsst.obs.base import Instrument, addUnboundedCalibrationLabel
40from lsst.obs.base.gen2to3 import TranslatorFactory, PhysicalToAbstractFilterKeyHandler
42from ..hsc.hscPupil import HscPupilFactory
43from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS
44from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission,
45 getFilterTransmission, getAtmosphereTransmission)
46from .strayLight.formatter import SubaruStrayLightDataFormatter
48log = logging.getLogger(__name__)
51class HyperSuprimeCam(Instrument):
52 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam.
53 """
55 policyName = "hsc"
56 obsDataPackage = "obs_subaru_data"
57 filterDefinitions = HSC_FILTER_DEFINITIONS
58 additionalCuratedDatasetTypes = ("bfKernel", "transmission_optics", "transmission_sensor",
59 "transmission_filter", "transmission_atmosphere", "yBackground")
61 def __init__(self, **kwargs):
62 super().__init__(**kwargs)
63 packageDir = getPackageDir("obs_subaru")
64 self.configPaths = [os.path.join(packageDir, "config"),
65 os.path.join(packageDir, "config", self.policyName)]
67 @classmethod
68 def getName(cls):
69 # Docstring inherited from Instrument.getName
70 return "HSC"
72 def register(self, registry):
73 # Docstring inherited from Instrument.register
74 camera = self.getCamera()
75 # The maximum values below make Gen3's ObservationDataIdPacker produce
76 # outputs that match Gen2's ccdExposureId.
77 obsMax = 21474800
78 registry.insertDimensionData(
79 "instrument",
80 {
81 "name": self.getName(),
82 "detector_max": 200,
83 "visit_max": obsMax,
84 "exposure_max": obsMax,
85 "class_name": getFullTypeName(self),
86 }
87 )
88 registry.insertDimensionData(
89 "detector",
90 *[
91 {
92 "instrument": self.getName(),
93 "id": detector.getId(),
94 "full_name": detector.getName(),
95 # TODO: make sure these definitions are consistent with those
96 # extracted by astro_metadata_translator, and test that they
97 # remain consistent somehow.
98 "name_in_raft": detector.getName().split("_")[1],
99 "raft": detector.getName().split("_")[0],
100 "purpose": str(detector.getType()).split(".")[-1],
101 }
102 for detector in camera
103 ]
104 )
105 self._registerFilters(registry)
107 def getRawFormatter(self, dataId):
108 # Docstring inherited from Instrument.getRawFormatter
109 # Import the formatter here to prevent a circular dependency.
110 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter
111 if dataId["detector"] in (100, 101, 102, 103):
112 return HyperSuprimeCamCornerRawFormatter
113 else:
114 return HyperSuprimeCamRawFormatter
116 def getCamera(self):
117 """Retrieve the cameraGeom representation of HSC.
119 This is a temporary API that should go away once obs_ packages have
120 a standardized approach to writing versioned cameras to a Gen3 repo.
121 """
122 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "camera")
123 return self._getCameraFromPath(path)
125 @staticmethod
126 @lru_cache()
127 def _getCameraFromPath(path):
128 """Return the camera geometry given solely the path to the location
129 of that definition."""
130 config = CameraConfig()
131 config.load(os.path.join(path, "camera.py"))
132 return makeCameraFromPath(
133 cameraConfig=config,
134 ampInfoPath=path,
135 shortNameFunc=lambda name: name.replace(" ", "_"),
136 pupilFactoryClass=HscPupilFactory
137 )
139 def getBrighterFatterKernel(self):
140 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`.
142 This is a temporary API that should go away once obs_ packages have
143 a standardized approach to writing versioned kernels to a Gen3 repo.
144 """
145 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "brighter_fatter_kernel.pkl")
146 with open(path, "rb") as fd:
147 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2
148 return kernel
150 def writeAdditionalCuratedCalibrations(self, butler, run=None):
151 # Get an unbounded calibration label
152 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
154 # Write brighter-fatter kernel, with an infinite validity range.
155 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray",
156 universe=butler.registry.dimensions)
157 butler.registry.registerDatasetType(datasetType)
158 # Load and then put instead of just moving the file in part to ensure
159 # the version in-repo is written with Python 3 and does not need
160 # `encoding='latin1'` to be read.
161 bfKernel = self.getBrighterFatterKernel()
162 butler.put(bfKernel, datasetType, unboundedDataId, run=run)
164 # The following iterate over the values of the dictionaries returned by the transmission functions
165 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
166 # which do not give the proper notion of validity. As such unbounded calibration labels are used
167 # when inserting into the database. In the future these could and probably should be updated to
168 # properly account for what ranges are considered valid.
170 # Write optical transmissions
171 opticsTransmissions = getOpticsTransmission()
172 datasetType = DatasetType("transmission_optics",
173 ("instrument", "calibration_label"),
174 "TransmissionCurve",
175 universe=butler.registry.dimensions)
176 butler.registry.registerDatasetType(datasetType)
177 for entry in opticsTransmissions.values():
178 if entry is None:
179 continue
180 butler.put(entry, datasetType, unboundedDataId, run=run)
182 # Write transmission sensor
183 sensorTransmissions = getSensorTransmission()
184 datasetType = DatasetType("transmission_sensor",
185 ("instrument", "detector", "calibration_label"),
186 "TransmissionCurve",
187 universe=butler.registry.dimensions)
188 butler.registry.registerDatasetType(datasetType)
189 for entry in sensorTransmissions.values():
190 if entry is None:
191 continue
192 for sensor, curve in entry.items():
193 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor)
194 butler.put(curve, datasetType, dataId, run=run)
196 # Write filter transmissions
197 filterTransmissions = getFilterTransmission()
198 datasetType = DatasetType("transmission_filter",
199 ("instrument", "physical_filter", "calibration_label"),
200 "TransmissionCurve",
201 universe=butler.registry.dimensions)
202 butler.registry.registerDatasetType(datasetType)
203 for entry in filterTransmissions.values():
204 if entry is None:
205 continue
206 for band, curve in entry.items():
207 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band)
208 butler.put(curve, datasetType, dataId, run=run)
210 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only
211 # look up along this dimension (ISR)
212 atmosphericTransmissions = getAtmosphereTransmission()
213 datasetType = DatasetType("transmission_atmosphere", ("instrument",),
214 "TransmissionCurve",
215 universe=butler.registry.dimensions)
216 butler.registry.registerDatasetType(datasetType)
217 for entry in atmosphericTransmissions.values():
218 if entry is None:
219 continue
220 butler.put(entry, datasetType, {"instrument": self.getName()}, run=run)
222 def ingestStrayLightData(self, butler, directory, *, transfer=None, run=None):
223 """Ingest externally-produced y-band stray light data files into
224 a data repository.
226 Parameters
227 ----------
228 butler : `lsst.daf.butler.Butler`
229 Butler initialized with the collection to ingest into.
230 directory : `str`
231 Directory containing yBackground-*.fits files.
232 transfer : `str`, optional
233 If not `None`, must be one of 'move', 'copy', 'hardlink', or
234 'symlink', indicating how to transfer the files.
235 run : `str`
236 Run to use for this collection of calibrations. If `None` the
237 collection name is worked out automatically from the instrument
238 name and other metadata.
239 """
240 if run is None:
241 run = self.makeCollectionName("calib")
242 butler.registry.registerCollection(run, type=CollectionType.RUN)
244 calibrationLabel = "y-LED-encoder-on"
245 # LEDs covered up around 2018-01-01, no need for correctin after that
246 # date.
247 timespan = Timespan(begin=None, end=astropy.time.Time("2018-01-01", format="iso", scale="tai"))
248 datasets = []
249 # TODO: should we use a more generic name for the dataset type?
250 # This is just the (rather HSC-specific) name used in Gen2, and while
251 # the instances of this dataset are camera-specific, the datasetType
252 # (which is used in the generic IsrTask) should not be.
253 datasetType = DatasetType("yBackground",
254 dimensions=("physical_filter", "detector", "calibration_label"),
255 storageClass="StrayLightData",
256 universe=butler.registry.dimensions)
257 for detector in self.getCamera():
258 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits")
259 if not os.path.exists(path):
260 log.warning(f"No stray light data found for detector {detector.getId()} @ {path}.")
261 continue
262 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(),
263 "detector": detector.getId(),
264 "physical_filter": "HSC-Y",
265 "calibration_label": calibrationLabel})
266 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter))
267 butler.registry.registerDatasetType(datasetType)
268 with butler.transaction():
269 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(),
270 "name": calibrationLabel,
271 "timespan": timespan})
272 butler.ingest(*datasets, transfer=transfer, run=run)
274 def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
275 # Docstring inherited from lsst.obs.base.Instrument.
276 factory = TranslatorFactory()
277 factory.addGenericInstrumentRules(self.getName())
278 # Translate Gen2 `filter` to abstract_filter if it hasn't been consumed
279 # yet and gen2keys includes tract.
280 factory.addRule(PhysicalToAbstractFilterKeyHandler(self.filterDefinitions),
281 instrument=self.getName(), gen2keys=("filter", "tract"), consume=("filter",))
282 return factory