Coverage for python/lsst/obs/subaru/_instrument.py : 21%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_subaru.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Gen3 Butler registry declarations for Hyper Suprime-Cam.
23"""
25__all__ = ("HyperSuprimeCam",)
27import os
28import pickle
29import logging
31import astropy.time
32from lsst.utils import getPackageDir
33from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig
34from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef,
35 TIMESPAN_MIN, CollectionType)
36from lsst.daf.butler.core.utils import getFullTypeName
37from lsst.obs.base import Instrument, addUnboundedCalibrationLabel
38from lsst.obs.base.gen2to3 import TranslatorFactory, PhysicalToAbstractFilterKeyHandler
40from ..hsc.hscPupil import HscPupilFactory
41from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS
42from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission,
43 getFilterTransmission, getAtmosphereTransmission)
44from .strayLight.formatter import SubaruStrayLightDataFormatter
46log = logging.getLogger(__name__)
49class HyperSuprimeCam(Instrument):
50 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam.
51 """
53 policyName = "hsc"
54 obsDataPackage = "obs_subaru_data"
55 filterDefinitions = HSC_FILTER_DEFINITIONS
57 def __init__(self, **kwargs):
58 super().__init__(**kwargs)
59 packageDir = getPackageDir("obs_subaru")
60 self.configPaths = [os.path.join(packageDir, "config"),
61 os.path.join(packageDir, "config", self.policyName)]
63 @classmethod
64 def getName(cls):
65 # Docstring inherited from Instrument.getName
66 return "HSC"
68 def register(self, registry):
69 # Docstring inherited from Instrument.register
70 camera = self.getCamera()
71 # The maximum values below make Gen3's ObservationDataIdPacker produce
72 # outputs that match Gen2's ccdExposureId.
73 obsMax = 21474800
74 registry.insertDimensionData(
75 "instrument",
76 {
77 "name": self.getName(),
78 "detector_max": 200,
79 "visit_max": obsMax,
80 "exposure_max": obsMax,
81 "class_name": getFullTypeName(self),
82 }
83 )
84 registry.insertDimensionData(
85 "detector",
86 *[
87 {
88 "instrument": self.getName(),
89 "id": detector.getId(),
90 "full_name": detector.getName(),
91 # TODO: make sure these definitions are consistent with those
92 # extracted by astro_metadata_translator, and test that they
93 # remain consistent somehow.
94 "name_in_raft": detector.getName().split("_")[1],
95 "raft": detector.getName().split("_")[0],
96 "purpose": str(detector.getType()).split(".")[-1],
97 }
98 for detector in camera
99 ]
100 )
101 self._registerFilters(registry)
103 def getRawFormatter(self, dataId):
104 # Docstring inherited from Instrument.getRawFormatter
105 # Import the formatter here to prevent a circular dependency.
106 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter
107 if dataId["detector"] in (100, 101, 102, 103):
108 return HyperSuprimeCamCornerRawFormatter
109 else:
110 return HyperSuprimeCamRawFormatter
112 def getCamera(self):
113 """Retrieve the cameraGeom representation of HSC.
115 This is a temporary API that should go away once obs_ packages have
116 a standardized approach to writing versioned cameras to a Gen3 repo.
117 """
118 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "camera")
119 config = CameraConfig()
120 config.load(os.path.join(path, "camera.py"))
121 return makeCameraFromPath(
122 cameraConfig=config,
123 ampInfoPath=path,
124 shortNameFunc=lambda name: name.replace(" ", "_"),
125 pupilFactoryClass=HscPupilFactory
126 )
128 def getBrighterFatterKernel(self):
129 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`.
131 This is a temporary API that should go away once obs_ packages have
132 a standardized approach to writing versioned kernels to a Gen3 repo.
133 """
134 path = os.path.join(getPackageDir("obs_subaru"), self.policyName, "brighter_fatter_kernel.pkl")
135 with open(path, "rb") as fd:
136 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2
137 return kernel
139 def writeAdditionalCuratedCalibrations(self, butler, run=None):
140 # Get an unbounded calibration label
141 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
143 # Write brighter-fatter kernel, with an infinite validity range.
144 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray",
145 universe=butler.registry.dimensions)
146 butler.registry.registerDatasetType(datasetType)
147 # Load and then put instead of just moving the file in part to ensure
148 # the version in-repo is written with Python 3 and does not need
149 # `encoding='latin1'` to be read.
150 bfKernel = self.getBrighterFatterKernel()
151 butler.put(bfKernel, datasetType, unboundedDataId, run=run)
153 # The following iterate over the values of the dictionaries returned by the transmission functions
154 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
155 # which do not give the proper notion of validity. As such unbounded calibration labels are used
156 # when inserting into the database. In the future these could and probably should be updated to
157 # properly account for what ranges are considered valid.
159 # Write optical transmissions
160 opticsTransmissions = getOpticsTransmission()
161 datasetType = DatasetType("transmission_optics",
162 ("instrument", "calibration_label"),
163 "TransmissionCurve",
164 universe=butler.registry.dimensions)
165 butler.registry.registerDatasetType(datasetType)
166 for entry in opticsTransmissions.values():
167 if entry is None:
168 continue
169 butler.put(entry, datasetType, unboundedDataId, run=run)
171 # Write transmission sensor
172 sensorTransmissions = getSensorTransmission()
173 datasetType = DatasetType("transmission_sensor",
174 ("instrument", "detector", "calibration_label"),
175 "TransmissionCurve",
176 universe=butler.registry.dimensions)
177 butler.registry.registerDatasetType(datasetType)
178 for entry in sensorTransmissions.values():
179 if entry is None:
180 continue
181 for sensor, curve in entry.items():
182 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor)
183 butler.put(curve, datasetType, dataId, run=run)
185 # Write filter transmissions
186 filterTransmissions = getFilterTransmission()
187 datasetType = DatasetType("transmission_filter",
188 ("instrument", "physical_filter", "calibration_label"),
189 "TransmissionCurve",
190 universe=butler.registry.dimensions)
191 butler.registry.registerDatasetType(datasetType)
192 for entry in filterTransmissions.values():
193 if entry is None:
194 continue
195 for band, curve in entry.items():
196 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band)
197 butler.put(curve, datasetType, dataId, run=run)
199 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only
200 # look up along this dimension (ISR)
201 atmosphericTransmissions = getAtmosphereTransmission()
202 datasetType = DatasetType("transmission_atmosphere", ("instrument",),
203 "TransmissionCurve",
204 universe=butler.registry.dimensions)
205 butler.registry.registerDatasetType(datasetType)
206 for entry in atmosphericTransmissions.values():
207 if entry is None:
208 continue
209 butler.put(entry, datasetType, {"instrument": self.getName()}, run=run)
211 def ingestStrayLightData(self, butler, directory, *, transfer=None, run=None):
212 """Ingest externally-produced y-band stray light data files into
213 a data repository.
215 Parameters
216 ----------
217 butler : `lsst.daf.butler.Butler`
218 Butler initialized with the collection to ingest into.
219 directory : `str`
220 Directory containing yBackground-*.fits files.
221 transfer : `str`, optional
222 If not `None`, must be one of 'move', 'copy', 'hardlink', or
223 'symlink', indicating how to transfer the files.
224 run : `str`
225 Run to use for this collection of calibrations. If `None` the
226 collection name is worked out automatically from the instrument
227 name and other metadata.
228 """
229 if run is None:
230 run = self.makeCollectionName("calib")
231 butler.registry.registerCollection(run, type=CollectionType.RUN)
233 calibrationLabel = "y-LED-encoder-on"
234 # LEDs covered up around 2018-01-01, no need for correctin after that
235 # date.
236 datetime_begin = TIMESPAN_MIN
237 datetime_end = astropy.time.Time("2018-01-01", format="iso", scale="tai")
238 datasets = []
239 # TODO: should we use a more generic name for the dataset type?
240 # This is just the (rather HSC-specific) name used in Gen2, and while
241 # the instances of this dataset are camera-specific, the datasetType
242 # (which is used in the generic IsrTask) should not be.
243 datasetType = DatasetType("yBackground",
244 dimensions=("physical_filter", "detector", "calibration_label"),
245 storageClass="StrayLightData",
246 universe=butler.registry.dimensions)
247 for detector in self.getCamera():
248 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits")
249 if not os.path.exists(path):
250 log.warning(f"No stray light data found for detector {detector.getId()} @ {path}.")
251 continue
252 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(),
253 "detector": detector.getId(),
254 "physical_filter": "HSC-Y",
255 "calibration_label": calibrationLabel})
256 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter))
257 butler.registry.registerDatasetType(datasetType)
258 with butler.transaction():
259 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(),
260 "name": calibrationLabel,
261 "datetime_begin": datetime_begin,
262 "datetime_end": datetime_end})
263 butler.ingest(*datasets, transfer=transfer, run=run)
265 def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
266 # Docstring inherited from lsst.obs.base.Instrument.
267 factory = TranslatorFactory()
268 factory.addGenericInstrumentRules(self.getName())
269 # Translate Gen2 `filter` to abstract_filter if it hasn't been consumed
270 # yet and gen2keys includes tract.
271 factory.addRule(PhysicalToAbstractFilterKeyHandler(self.filterDefinitions),
272 instrument=self.getName(), gen2keys=("filter", "tract"), consume=("filter",))
273 return factory