Coverage for python/lsst/obs/subaru/_instrument.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_subaru.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Gen3 Butler registry declarations for Hyper Suprime-Cam.
23"""
25__all__ = ("HyperSuprimeCam",)
27import os
28import pickle
29import logging
31from functools import lru_cache
33import astropy.time
34from lsst.utils import getPackageDir
35from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig
36from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef,
37 TIMESPAN_MIN, CollectionType)
38from lsst.daf.butler.core.utils import getFullTypeName
39from lsst.obs.base import Instrument, addUnboundedCalibrationLabel
40from lsst.obs.base.gen2to3 import TranslatorFactory, PhysicalToAbstractFilterKeyHandler
42from ..hsc.hscPupil import HscPupilFactory
43from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS
44from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission,
45 getFilterTransmission, getAtmosphereTransmission)
46from .strayLight.formatter import SubaruStrayLightDataFormatter
48log = logging.getLogger(__name__)
51class HyperSuprimeCam(Instrument):
52 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam.
53 """
55 policyName = "hsc"
56 obsDataPackage = "obs_subaru_data"
57 filterDefinitions = HSC_FILTER_DEFINITIONS
59 def __init__(self, **kwargs):
60 super().__init__(**kwargs)
61 packageDir = getPackageDir("obs_subaru")
62 self.configPaths = [os.path.join(packageDir, "config"),
63 os.path.join(packageDir, "config", self.policyName)]
65 @classmethod
66 def getName(cls):
67 # Docstring inherited from Instrument.getName
68 return "HSC"
70 def register(self, registry):
71 # Docstring inherited from Instrument.register
72 camera = self.getCamera()
73 # The maximum values below make Gen3's ObservationDataIdPacker produce
74 # outputs that match Gen2's ccdExposureId.
75 obsMax = 21474800
76 registry.insertDimensionData(
77 "instrument",
78 {
79 "name": self.getName(),
80 "detector_max": 200,
81 "visit_max": obsMax,
82 "exposure_max": obsMax,
83 "class_name": getFullTypeName(self),
84 }
85 )
86 registry.insertDimensionData(
87 "detector",
88 *[
89 {
90 "instrument": self.getName(),
91 "id": detector.getId(),
92 "full_name": detector.getName(),
93 # TODO: make sure these definitions are consistent with those
94 # extracted by astro_metadata_translator, and test that they
95 # remain consistent somehow.
96 "name_in_raft": detector.getName().split("_")[1],
97 "raft": detector.getName().split("_")[0],
98 "purpose": str(detector.getType()).split(".")[-1],
99 }
100 for detector in camera
101 ]
102 )
103 self._registerFilters(registry)
105 def getRawFormatter(self, dataId):
106 # Docstring inherited from Instrument.getRawFormatter
107 # Import the formatter here to prevent a circular dependency.
108 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter
109 if dataId["detector"] in (100, 101, 102, 103):
110 return HyperSuprimeCamCornerRawFormatter
111 else:
112 return HyperSuprimeCamRawFormatter
114 def getCamera(self):
115 """Retrieve the cameraGeom representation of HSC.
117 This is a temporary API that should go away once obs_ packages have
118 a standardized approach to writing versioned cameras to a Gen3 repo.
119 """
120 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "camera")
121 return self._getCameraFromPath(path)
123 @staticmethod
124 @lru_cache()
125 def _getCameraFromPath(path):
126 """Return the camera geometry given solely the path to the location
127 of that definition."""
128 config = CameraConfig()
129 config.load(os.path.join(path, "camera.py"))
130 return makeCameraFromPath(
131 cameraConfig=config,
132 ampInfoPath=path,
133 shortNameFunc=lambda name: name.replace(" ", "_"),
134 pupilFactoryClass=HscPupilFactory
135 )
137 def getBrighterFatterKernel(self):
138 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`.
140 This is a temporary API that should go away once obs_ packages have
141 a standardized approach to writing versioned kernels to a Gen3 repo.
142 """
143 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "brighter_fatter_kernel.pkl")
144 with open(path, "rb") as fd:
145 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2
146 return kernel
148 def writeAdditionalCuratedCalibrations(self, butler, run=None):
149 # Get an unbounded calibration label
150 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
152 # Write brighter-fatter kernel, with an infinite validity range.
153 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray",
154 universe=butler.registry.dimensions)
155 butler.registry.registerDatasetType(datasetType)
156 # Load and then put instead of just moving the file in part to ensure
157 # the version in-repo is written with Python 3 and does not need
158 # `encoding='latin1'` to be read.
159 bfKernel = self.getBrighterFatterKernel()
160 butler.put(bfKernel, datasetType, unboundedDataId, run=run)
162 # The following iterate over the values of the dictionaries returned by the transmission functions
163 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
164 # which do not give the proper notion of validity. As such unbounded calibration labels are used
165 # when inserting into the database. In the future these could and probably should be updated to
166 # properly account for what ranges are considered valid.
168 # Write optical transmissions
169 opticsTransmissions = getOpticsTransmission()
170 datasetType = DatasetType("transmission_optics",
171 ("instrument", "calibration_label"),
172 "TransmissionCurve",
173 universe=butler.registry.dimensions)
174 butler.registry.registerDatasetType(datasetType)
175 for entry in opticsTransmissions.values():
176 if entry is None:
177 continue
178 butler.put(entry, datasetType, unboundedDataId, run=run)
180 # Write transmission sensor
181 sensorTransmissions = getSensorTransmission()
182 datasetType = DatasetType("transmission_sensor",
183 ("instrument", "detector", "calibration_label"),
184 "TransmissionCurve",
185 universe=butler.registry.dimensions)
186 butler.registry.registerDatasetType(datasetType)
187 for entry in sensorTransmissions.values():
188 if entry is None:
189 continue
190 for sensor, curve in entry.items():
191 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor)
192 butler.put(curve, datasetType, dataId, run=run)
194 # Write filter transmissions
195 filterTransmissions = getFilterTransmission()
196 datasetType = DatasetType("transmission_filter",
197 ("instrument", "physical_filter", "calibration_label"),
198 "TransmissionCurve",
199 universe=butler.registry.dimensions)
200 butler.registry.registerDatasetType(datasetType)
201 for entry in filterTransmissions.values():
202 if entry is None:
203 continue
204 for band, curve in entry.items():
205 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band)
206 butler.put(curve, datasetType, dataId, run=run)
208 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only
209 # look up along this dimension (ISR)
210 atmosphericTransmissions = getAtmosphereTransmission()
211 datasetType = DatasetType("transmission_atmosphere", ("instrument",),
212 "TransmissionCurve",
213 universe=butler.registry.dimensions)
214 butler.registry.registerDatasetType(datasetType)
215 for entry in atmosphericTransmissions.values():
216 if entry is None:
217 continue
218 butler.put(entry, datasetType, {"instrument": self.getName()}, run=run)
220 def ingestStrayLightData(self, butler, directory, *, transfer=None, run=None):
221 """Ingest externally-produced y-band stray light data files into
222 a data repository.
224 Parameters
225 ----------
226 butler : `lsst.daf.butler.Butler`
227 Butler initialized with the collection to ingest into.
228 directory : `str`
229 Directory containing yBackground-*.fits files.
230 transfer : `str`, optional
231 If not `None`, must be one of 'move', 'copy', 'hardlink', or
232 'symlink', indicating how to transfer the files.
233 run : `str`
234 Run to use for this collection of calibrations. If `None` the
235 collection name is worked out automatically from the instrument
236 name and other metadata.
237 """
238 if run is None:
239 run = self.makeCollectionName("calib")
240 butler.registry.registerCollection(run, type=CollectionType.RUN)
242 calibrationLabel = "y-LED-encoder-on"
243 # LEDs covered up around 2018-01-01, no need for correctin after that
244 # date.
245 datetime_begin = TIMESPAN_MIN
246 datetime_end = astropy.time.Time("2018-01-01", format="iso", scale="tai")
247 datasets = []
248 # TODO: should we use a more generic name for the dataset type?
249 # This is just the (rather HSC-specific) name used in Gen2, and while
250 # the instances of this dataset are camera-specific, the datasetType
251 # (which is used in the generic IsrTask) should not be.
252 datasetType = DatasetType("yBackground",
253 dimensions=("physical_filter", "detector", "calibration_label"),
254 storageClass="StrayLightData",
255 universe=butler.registry.dimensions)
256 for detector in self.getCamera():
257 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits")
258 if not os.path.exists(path):
259 log.warning(f"No stray light data found for detector {detector.getId()} @ {path}.")
260 continue
261 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(),
262 "detector": detector.getId(),
263 "physical_filter": "HSC-Y",
264 "calibration_label": calibrationLabel})
265 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter))
266 butler.registry.registerDatasetType(datasetType)
267 with butler.transaction():
268 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(),
269 "name": calibrationLabel,
270 "datetime_begin": datetime_begin,
271 "datetime_end": datetime_end})
272 butler.ingest(*datasets, transfer=transfer, run=run)
274 def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
275 # Docstring inherited from lsst.obs.base.Instrument.
276 factory = TranslatorFactory()
277 factory.addGenericInstrumentRules(self.getName())
278 # Translate Gen2 `filter` to abstract_filter if it hasn't been consumed
279 # yet and gen2keys includes tract.
280 factory.addRule(PhysicalToAbstractFilterKeyHandler(self.filterDefinitions),
281 instrument=self.getName(), gen2keys=("filter", "tract"), consume=("filter",))
282 return factory