Coverage for python/lsst/obs/subaru/_instrument.py: 72%
125 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-14 19:32 +0000
1# This file is part of obs_subaru.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Gen3 Butler registry declarations for Hyper Suprime-Cam.
23"""
25__all__ = ("HyperSuprimeCam",)
27import importlib.resources
28import os
29import pickle
30import logging
32from functools import lru_cache
34from astro_metadata_translator import HscTranslator
35import astropy.time
36from lsst.resources import ResourcePath
37from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig
38from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef,
39 CollectionType, Timespan)
40from lsst.utils.introspection import get_full_type_name
41from lsst.obs.base import Instrument, VisitSystem
43from ..hsc.hscPupil import HscPupilFactory
44from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS
45from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission,
46 getFilterTransmission, getAtmosphereTransmission)
47from .strayLight.formatter import SubaruStrayLightDataFormatter
49log = logging.getLogger(__name__)
52class HyperSuprimeCam(Instrument):
53 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam.
54 """
56 policyName = "hsc"
57 obsDataPackage = "obs_subaru_data"
58 filterDefinitions = HSC_FILTER_DEFINITIONS
59 additionalCuratedDatasetTypes = ("bfKernel", "transmission_optics", "transmission_sensor",
60 "transmission_filter", "transmission_atmosphere", "yBackground")
61 translatorClass = HscTranslator
63 def __init__(self, **kwargs):
64 super().__init__(**kwargs)
65 self.configPaths = ["resource://lsst.obs.subaru/resources/config",
66 f"resource://lsst.obs.subaru/resources/config/{self.policyName}"]
68 @classmethod
69 def getName(cls):
70 # Docstring inherited from Instrument.getName
71 return "HSC"
73 def register(self, registry, update=False):
74 # Docstring inherited from Instrument.register
75 camera = self.getCamera()
76 # The maximum values below make Gen3's ObservationDataIdPacker produce
77 # outputs that match Gen2's ccdExposureId.
78 obsMax = 21474800
79 with registry.transaction():
80 registry.syncDimensionData(
81 "instrument",
82 {
83 "name": self.getName(),
84 "detector_max": 200,
85 "visit_max": obsMax,
86 "exposure_max": obsMax,
87 "class_name": get_full_type_name(self),
88 # Some schemas support default visit_system
89 "visit_system": VisitSystem.ONE_TO_ONE.value,
90 },
91 update=update
92 )
93 for detector in camera:
94 registry.syncDimensionData(
95 "detector",
96 {
97 "instrument": self.getName(),
98 "id": detector.getId(),
99 "full_name": detector.getName(),
100 # TODO: make sure these definitions are consistent with
101 # those extracted by astro_metadata_translator, and
102 # test that they remain consistent somehow.
103 "name_in_raft": detector.getName().split("_")[1],
104 "raft": detector.getName().split("_")[0],
105 "purpose": str(detector.getType()).split(".")[-1],
106 },
107 update=update
108 )
109 self._registerFilters(registry, update=update)
111 def getRawFormatter(self, dataId):
112 # Docstring inherited from Instrument.getRawFormatter
113 # Import the formatter here to prevent a circular dependency.
114 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter
115 if dataId["detector"] in (100, 101, 102, 103):
116 return HyperSuprimeCamCornerRawFormatter
117 else:
118 return HyperSuprimeCamRawFormatter
120 def getCamera(self):
121 """Retrieve the cameraGeom representation of HSC.
123 This is a temporary API that should go away once obs_ packages have
124 a standardized approach to writing versioned cameras to a Gen3 repo.
125 """
126 # The camera geometry is a directory of files (camera.py plus per-amp
127 # FITS) consumed by makeCameraFromPath, so it must be presented as a
128 # local directory. importlib.resources.as_file yields that path
129 # without requiring an EUPS environment (unlike getPackageDir).
130 with importlib.resources.as_file(
131 importlib.resources.files("lsst.obs.subaru").joinpath(
132 f"resources/{self.policyName}/camera"
133 )
134 ) as path:
135 return self._getCameraFromPath(str(path))
137 @staticmethod
138 @lru_cache()
139 def _getCameraFromPath(path):
140 """Return the camera geometry given solely the path to the location
141 of that definition."""
142 config = CameraConfig()
143 config.load(os.path.join(path, "camera.py"))
144 return makeCameraFromPath(
145 cameraConfig=config,
146 ampInfoPath=path,
147 shortNameFunc=lambda name: name.replace(" ", "_"),
148 pupilFactoryClass=HscPupilFactory
149 )
151 def getBrighterFatterKernel(self):
152 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`.
154 This is a temporary API that should go away once obs_ packages have
155 a standardized approach to writing versioned kernels to a Gen3 repo.
156 """
157 uri = ResourcePath(f"resource://lsst.obs.subaru/resources/{self.policyName}/brighter_fatter_kernel.pkl")
158 with uri.open("rb") as fd:
159 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2
160 return kernel
162 def writeAdditionalCuratedCalibrations(self, butler, collection=None, labels=()):
163 # Register the CALIBRATION collection that adds validity ranges.
164 # This does nothing if it is already registered.
165 if collection is None: 165 ↛ 166line 165 didn't jump to line 166 because the condition on line 165 was never true
166 collection = self.makeCalibrationCollectionName(*labels)
167 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
169 # Register the RUN collection that holds these datasets directly. We
170 # only need one because all of these datasets have the same (unbounded)
171 # validity range right now.
172 run = self.makeUnboundedCalibrationRunName(*labels)
173 butler.registry.registerRun(run)
174 baseDataId = butler.registry.expandDataId(instrument=self.getName())
175 refs = []
177 # Write brighter-fatter kernel, with an infinite validity range.
178 datasetType = DatasetType("bfKernel", ("instrument",), "NumpyArray",
179 universe=butler.dimensions,
180 isCalibration=True)
181 butler.registry.registerDatasetType(datasetType)
183 # Load and then put instead of just moving the file in part to ensure
184 # the version in-repo is written with Python 3 and does not need
185 # `encoding='latin1'` to be read.
186 bfKernel = self.getBrighterFatterKernel()
187 refs.append(butler.put(bfKernel, datasetType, baseDataId, run=run))
189 # The following iterate over the values of the dictionaries returned
190 # by the transmission functions and ignore the date that is supplied.
191 # This is due to the dates not being ranges but single dates,
192 # which do not give the proper notion of validity. As such unbounded
193 # calibration labels are used when inserting into the database.
194 # In the future these could and probably should be updated to
195 # properly account for what ranges are considered valid.
197 # Write optical transmissions
198 opticsTransmissions = getOpticsTransmission()
199 datasetType = DatasetType("transmission_optics",
200 ("instrument",),
201 "TransmissionCurve",
202 universe=butler.dimensions,
203 isCalibration=True)
204 butler.registry.registerDatasetType(datasetType)
205 for entry in opticsTransmissions.values():
206 if entry is None:
207 continue
208 refs.append(butler.put(entry, datasetType, baseDataId, run=run))
210 # Write transmission sensor
211 sensorTransmissions = getSensorTransmission()
212 datasetType = DatasetType("transmission_sensor",
213 ("instrument", "detector",),
214 "TransmissionCurve",
215 universe=butler.dimensions,
216 isCalibration=True)
217 butler.registry.registerDatasetType(datasetType)
218 for entry in sensorTransmissions.values():
219 if entry is None: 219 ↛ 220line 219 didn't jump to line 220 because the condition on line 219 was never true
220 continue
221 for sensor, curve in entry.items():
222 dataId = DataCoordinate.standardize(baseDataId, detector=sensor)
223 refs.append(butler.put(curve, datasetType, dataId, run=run))
225 # Write filter transmissions
226 filterTransmissions = getFilterTransmission()
227 datasetType = DatasetType("transmission_filter",
228 ("instrument", "physical_filter",),
229 "TransmissionCurve",
230 universe=butler.dimensions,
231 isCalibration=True)
232 butler.registry.registerDatasetType(datasetType)
233 for entry in filterTransmissions.values():
234 if entry is None: 234 ↛ 235line 234 didn't jump to line 235 because the condition on line 234 was never true
235 continue
236 for band, curve in entry.items():
237 dataId = DataCoordinate.standardize(baseDataId, physical_filter=band)
238 refs.append(butler.put(curve, datasetType, dataId, run=run))
240 # Write atmospheric transmissions
241 atmosphericTransmissions = getAtmosphereTransmission()
242 datasetType = DatasetType("transmission_atmosphere", ("instrument",),
243 "TransmissionCurve",
244 universe=butler.dimensions,
245 isCalibration=True)
246 butler.registry.registerDatasetType(datasetType)
247 for entry in atmosphericTransmissions.values():
248 if entry is None: 248 ↛ 249line 248 didn't jump to line 249 because the condition on line 248 was never true
249 continue
250 refs.append(butler.put(entry, datasetType, {"instrument": self.getName()}, run=run))
252 # Associate all datasets with the unbounded validity range.
253 butler.registry.certify(collection, refs, Timespan(begin=None, end=None))
255 def ingestStrayLightData(self, butler, directory, *, transfer=None, collection=None, labels=()):
256 """Ingest externally-produced y-band stray light data files into
257 a data repository.
259 Parameters
260 ----------
261 butler : `lsst.daf.butler.Butler`
262 Butler to write with. Any collections associated with it are
263 ignored in favor of ``collection`` and/or ``labels``.
264 directory : `str`
265 Directory containing yBackground-*.fits files.
266 transfer : `str`, optional
267 If not `None`, must be one of 'move', 'copy', 'hardlink', or
268 'symlink', indicating how to transfer the files.
269 collection : `str`, optional
270 Name to use for the calibration collection that associates all
271 datasets with a validity range. If this collection already exists,
272 it must be a `~CollectionType.CALIBRATION` collection, and it must
273 not have any datasets that would conflict with those inserted by
274 this method. If `None`, a collection name is worked out
275 automatically from the instrument name and other metadata by
276 calling ``makeCuratedCalibrationCollectionName``, but this
277 default name may not work well for long-lived repositories unless
278 ``labels`` is also provided (and changed every time curated
279 calibrations are ingested).
280 labels : `Sequence` [ `str` ], optional
281 Extra strings to include in collection names, after concatenating
282 them with the standard collection name delimeter. If provided,
283 these are inserted into to the names of the `~CollectionType.RUN`
284 collections that datasets are inserted directly into, as well the
285 `~CollectionType.CALIBRATION` collection if it is generated
286 automatically (i.e. if ``collection is None``). Usually this is
287 just the name of the ticket on which the calibration collection is
288 being created.
289 """
290 # Register the CALIBRATION collection that adds validity ranges.
291 # This does nothing if it is already registered.
292 if collection is None:
293 collection = self.makeCalibrationCollectionName(*labels)
294 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
296 # Register the RUN collection that holds these datasets directly. We
297 # only need one because there is only one validity range and hence no
298 # data ID conflicts even when there are no validity ranges.
299 run = self.makeUnboundedCalibrationRunName(*labels)
300 butler.registry.registerRun(run)
302 # LEDs covered up around 2018-01-01, no need for correctin after that
303 # date.
304 timespan = Timespan(begin=None, end=astropy.time.Time("2018-01-01", format="iso", scale="tai"))
305 datasets = []
306 # TODO: should we use a more generic name for the dataset type?
307 # This is just the (rather HSC-specific) name used in Gen2, and while
308 # the instances of this dataset are camera-specific, the datasetType
309 # (which is used in the generic IsrTask) should not be.
310 datasetType = DatasetType("yBackground",
311 dimensions=("physical_filter", "detector",),
312 storageClass="StrayLightData",
313 universe=butler.dimensions,
314 isCalibration=True)
315 for detector in self.getCamera():
316 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits")
317 if not os.path.exists(path):
318 log.warning("No stray light data found for detector %s @ %s.", detector.getId(), path)
319 continue
320 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(),
321 "detector": detector.getId(),
322 "physical_filter": "HSC-Y"},
323 run=run)
324 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter))
325 butler.registry.registerDatasetType(datasetType)
326 with butler.transaction():
327 butler.ingest(*datasets, transfer=transfer)
328 refs = []
329 for dataset in datasets:
330 refs.extend(dataset.refs)
331 butler.registry.certify(collection, refs, timespan)