Coverage for python/lsst/obs/subaru/instrument.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_subaru.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Gen3 Butler registry declarations for Hyper Suprime-Cam.
23"""
25__all__ = ("HyperSuprimeCam",)
27import os
28import pickle
29import logging
31import astropy.time
32from lsst.utils import getPackageDir
33from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig
34from lsst.daf.butler import (DatasetType, DataCoordinate, FileDataset, DatasetRef,
35 TIMESPAN_MIN, TIMESPAN_MAX)
36from lsst.obs.base import Instrument, addUnboundedCalibrationLabel
37from lsst.pipe.tasks.read_curated_calibs import read_all
39from ..hsc.hscPupil import HscPupilFactory
40from ..hsc.hscFilters import HSC_FILTER_DEFINITIONS
41from ..hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission,
42 getFilterTransmission, getAtmosphereTransmission)
43from .strayLight.formatter import SubaruStrayLightDataFormatter
45log = logging.getLogger(__name__)
48class HyperSuprimeCam(Instrument):
49 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam.
50 """
52 filterDefinitions = HSC_FILTER_DEFINITIONS
54 def __init__(self, **kwargs):
55 super().__init__(**kwargs)
56 packageDir = getPackageDir("obs_subaru")
57 self.configPaths = [os.path.join(packageDir, "config"),
58 os.path.join(packageDir, "config", "hsc")]
60 @classmethod
61 def getName(cls):
62 # Docstring inherited from Instrument.getName
63 return "HSC"
65 def register(self, registry):
66 # Docstring inherited from Instrument.register
67 camera = self.getCamera()
68 # The maximum values below make Gen3's ObservationDataIdPacker produce
69 # outputs that match Gen2's ccdExposureId.
70 obsMax = 21474800
71 registry.insertDimensionData(
72 "instrument",
73 {
74 "name": self.getName(),
75 "detector_max": 200,
76 "visit_max": obsMax,
77 "exposure_max": obsMax
78 }
79 )
80 registry.insertDimensionData(
81 "detector",
82 *[
83 {
84 "instrument": self.getName(),
85 "id": detector.getId(),
86 "full_name": detector.getName(),
87 # TODO: make sure these definitions are consistent with those
88 # extracted by astro_metadata_translator, and test that they
89 # remain consistent somehow.
90 "name_in_raft": detector.getName().split("_")[1],
91 "raft": detector.getName().split("_")[0],
92 "purpose": str(detector.getType()).split(".")[-1],
93 }
94 for detector in camera
95 ]
96 )
97 self._registerFilters(registry)
99 def getRawFormatter(self, dataId):
100 # Docstring inherited from Instrument.getRawFormatter
101 # Import the formatter here to prevent a circular dependency.
102 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter
103 if dataId["detector"] in (100, 101, 102, 103):
104 return HyperSuprimeCamCornerRawFormatter
105 else:
106 return HyperSuprimeCamRawFormatter
108 def getCamera(self):
109 """Retrieve the cameraGeom representation of HSC.
111 This is a temporary API that should go away once obs_ packages have
112 a standardized approach to writing versioned cameras to a Gen3 repo.
113 """
114 path = os.path.join(getPackageDir("obs_subaru"), "hsc", "camera")
115 config = CameraConfig()
116 config.load(os.path.join(path, "camera.py"))
117 return makeCameraFromPath(
118 cameraConfig=config,
119 ampInfoPath=path,
120 shortNameFunc=lambda name: name.replace(" ", "_"),
121 pupilFactoryClass=HscPupilFactory
122 )
124 def getBrighterFatterKernel(self):
125 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`.
127 This is a temporary API that should go away once obs_ packages have
128 a standardized approach to writing versioned kernels to a Gen3 repo.
129 """
130 path = os.path.join(getPackageDir("obs_subaru"), "hsc", "brighter_fatter_kernel.pkl")
131 with open(path, "rb") as fd:
132 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2
133 return kernel
135 def writeCuratedCalibrations(self, butler):
136 """Write human-curated calibration Datasets to the given Butler with
137 the appropriate validity ranges.
139 This is a temporary API that should go away once obs_ packages have
140 a standardized approach to this problem.
141 """
143 # Write cameraGeom.Camera, with an infinite validity range.
144 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
145 universe=butler.registry.dimensions)
146 butler.registry.registerDatasetType(datasetType)
147 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
148 camera = self.getCamera()
149 butler.put(camera, datasetType, unboundedDataId)
151 # Write brighter-fatter kernel, with an infinite validity range.
152 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray",
153 universe=butler.registry.dimensions)
154 butler.registry.registerDatasetType(datasetType)
155 # Load and then put instead of just moving the file in part to ensure
156 # the version in-repo is written with Python 3 and does not need
157 # `encoding='latin1'` to be read.
158 bfKernel = self.getBrighterFatterKernel()
159 butler.put(bfKernel, datasetType, unboundedDataId)
161 # The following iterate over the values of the dictionaries returned by the transmission functions
162 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
163 # which do not give the proper notion of validity. As such unbounded calibration labels are used
164 # when inserting into the database. In the future these could and probably should be updated to
165 # properly account for what ranges are considered valid.
167 # Write optical transmissions
168 opticsTransmissions = getOpticsTransmission()
169 datasetType = DatasetType("transmission_optics",
170 ("instrument", "calibration_label"),
171 "TransmissionCurve",
172 universe=butler.registry.dimensions)
173 butler.registry.registerDatasetType(datasetType)
174 for entry in opticsTransmissions.values():
175 if entry is None:
176 continue
177 butler.put(entry, datasetType, unboundedDataId)
179 # Write transmission sensor
180 sensorTransmissions = getSensorTransmission()
181 datasetType = DatasetType("transmission_sensor",
182 ("instrument", "detector", "calibration_label"),
183 "TransmissionCurve",
184 universe=butler.registry.dimensions)
185 butler.registry.registerDatasetType(datasetType)
186 for entry in sensorTransmissions.values():
187 if entry is None:
188 continue
189 for sensor, curve in entry.items():
190 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor)
191 butler.put(curve, datasetType, dataId)
193 # Write filter transmissions
194 filterTransmissions = getFilterTransmission()
195 datasetType = DatasetType("transmission_filter",
196 ("instrument", "physical_filter", "calibration_label"),
197 "TransmissionCurve",
198 universe=butler.registry.dimensions)
199 butler.registry.registerDatasetType(datasetType)
200 for entry in filterTransmissions.values():
201 if entry is None:
202 continue
203 for band, curve in entry.items():
204 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band)
205 butler.put(curve, datasetType, dataId)
207 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only
208 # look up along this dimension (ISR)
209 atmosphericTransmissions = getAtmosphereTransmission()
210 datasetType = DatasetType("transmission_atmosphere", ("instrument",),
211 "TransmissionCurve",
212 universe=butler.registry.dimensions)
213 butler.registry.registerDatasetType(datasetType)
214 for entry in atmosphericTransmissions.values():
215 if entry is None:
216 continue
217 butler.put(entry, datasetType, {"instrument": self.getName()})
219 # Write defects with validity ranges taken from obs_subaru_data/hsc/defects
220 # (along with the defects themselves).
221 datasetType = DatasetType("defects", ("instrument", "detector", "calibration_label"), "Defects",
222 universe=butler.registry.dimensions)
223 butler.registry.registerDatasetType(datasetType)
224 defectPath = os.path.join(getPackageDir("obs_subaru_data"), "hsc", "defects")
225 camera = self.getCamera()
226 defectsDict = read_all(defectPath, camera)[0] # This method returns a dict plus the calib type
227 endOfTime = TIMESPAN_MAX
228 dimensionRecords = []
229 datasetRecords = []
230 # First loop just gathers up the things we want to insert, so we
231 # can do some bulk inserts and minimize the time spent in transaction.
232 for det in defectsDict:
233 detector = camera[det]
234 times = sorted([k for k in defectsDict[det]])
235 defects = [defectsDict[det][time] for time in times]
236 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
237 times += [endOfTime]
238 for defect, beginTime, endTime in zip(defects, times[:-1], times[1:]):
239 md = defect.getMetadata()
240 calibrationLabel = f"defect/{md['CALIBDATE']}/{md['DETECTOR']}"
241 dataId = DataCoordinate.standardize(
242 universe=butler.registry.dimensions,
243 instrument=self.getName(),
244 calibration_label=calibrationLabel,
245 detector=detector.getId(),
246 )
247 datasetRecords.append((defect, dataId))
248 dimensionRecords.append({
249 "instrument": self.getName(),
250 "name": calibrationLabel,
251 "datetime_begin": beginTime,
252 "datetime_end": endTime,
253 })
254 # Second loop actually does the inserts and filesystem writes.
255 with butler.transaction():
256 butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
257 # TODO: vectorize these puts, once butler APIs for that become
258 # available.
259 for defect, dataId in datasetRecords:
260 butler.put(defect, datasetType, dataId)
262 def ingestStrayLightData(self, butler, directory, *, transfer=None):
263 """Ingest externally-produced y-band stray light data files into
264 a data repository.
266 Parameters
267 ----------
268 butler : `lsst.daf.butler.Butler`
269 Butler initialized with the collection to ingest into.
270 directory : `str`
271 Directory containing yBackground-*.fits files.
272 transfer : `str`, optional
273 If not `None`, must be one of 'move', 'copy', 'hardlink', or
274 'symlink', indicating how to transfer the files.
275 """
276 calibrationLabel = "y-LED-encoder-on"
277 # LEDs covered up around 2018-01-01, no need for correctin after that
278 # date.
279 datetime_begin = TIMESPAN_MIN
280 datetime_end = astropy.time.Time("2018-01-01", format="iso", scale="tai")
281 datasets = []
282 # TODO: should we use a more generic name for the dataset type?
283 # This is just the (rather HSC-specific) name used in Gen2, and while
284 # the instances of this dataset are camera-specific, the datasetType
285 # (which is used in the generic IsrTask) should not be.
286 datasetType = DatasetType("yBackground",
287 dimensions=("physical_filter", "detector", "calibration_label"),
288 storageClass="StrayLightData",
289 universe=butler.registry.dimensions)
290 for detector in self.getCamera():
291 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits")
292 if not os.path.exists(path):
293 log.warn(f"No stray light data found for detector {detector.getId()} @ {path}.")
294 continue
295 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(),
296 "detector": detector.getId(),
297 "physical_filter": "HSC-Y",
298 "calibration_label": calibrationLabel})
299 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter))
300 with butler.transaction():
301 butler.registry.registerDatasetType(datasetType)
302 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(),
303 "name": calibrationLabel,
304 "datetime_begin": datetime_begin,
305 "datetime_end": datetime_end})
306 butler.ingest(*datasets, transfer=transfer)