Coverage for python/lsst/obs/subaru/gen3/hsc/instrument.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_subaru.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Gen3 Butler registry declarations for Hyper Suprime-Cam.
23"""
25__all__ = ("HyperSuprimeCam",)
27import os
28import pickle
29import logging
30import datetime
31from dateutil import parser
33from lsst.utils import getPackageDir
34from lsst.afw.cameraGeom import makeCameraFromPath, CameraConfig
35from lsst.daf.butler import DatasetType, DataCoordinate, FileDataset, DatasetRef
36from lsst.obs.base import Instrument, addUnboundedCalibrationLabel
37from lsst.pipe.tasks.read_curated_calibs import read_all
39from lsst.obs.hsc.hscPupil import HscPupilFactory
40from lsst.obs.hsc.hscFilters import HSC_FILTER_DEFINITIONS
41from lsst.obs.hsc.makeTransmissionCurves import (getSensorTransmission, getOpticsTransmission,
42 getFilterTransmission, getAtmosphereTransmission)
43from lsst.obs.subaru.strayLight.formatter import SubaruStrayLightDataFormatter
45log = logging.getLogger(__name__)
48class HyperSuprimeCam(Instrument):
49 """Gen3 Butler specialization class for Subaru's Hyper Suprime-Cam.
50 """
52 filterDefinitions = HSC_FILTER_DEFINITIONS
54 def __init__(self, **kwargs):
55 super().__init__(**kwargs)
56 packageDir = getPackageDir("obs_subaru")
57 self.configPaths = [os.path.join(packageDir, "config"),
58 os.path.join(packageDir, "config", "hsc")]
60 @classmethod
61 def getName(cls):
62 # Docstring inherited from Instrument.getName
63 return "HSC"
65 def register(self, registry):
66 # Docstring inherited from Instrument.register
67 camera = self.getCamera()
68 # The maximum values below make Gen3's ObservationDataIdPacker produce
69 # outputs that match Gen2's ccdExposureId.
70 obsMax = 21474800
71 registry.insertDimensionData(
72 "instrument",
73 {
74 "name": self.getName(),
75 "detector_max": 200,
76 "visit_max": obsMax,
77 "exposure_max": obsMax
78 }
79 )
80 registry.insertDimensionData(
81 "detector",
82 *[
83 {
84 "instrument": self.getName(),
85 "id": detector.getId(),
86 "full_name": detector.getName(),
87 # TODO: make sure these definitions are consistent with those
88 # extracted by astro_metadata_translator, and test that they
89 # remain consistent somehow.
90 "name_in_raft": detector.getName().split("_")[1],
91 "raft": detector.getName().split("_")[0],
92 "purpose": str(detector.getType()).split(".")[-1],
93 }
94 for detector in camera
95 ]
96 )
97 self._registerFilters(registry)
99 def getRawFormatter(self, dataId):
100 # Docstring inherited from Instrument.getRawFormatter
101 # Import the formatter here to prevent a circular dependency.
102 from .rawFormatter import HyperSuprimeCamRawFormatter, HyperSuprimeCamCornerRawFormatter
103 if dataId["detector"] in (100, 101, 102, 103):
104 return HyperSuprimeCamCornerRawFormatter
105 else:
106 return HyperSuprimeCamRawFormatter
108 def getCamera(self):
109 """Retrieve the cameraGeom representation of HSC.
111 This is a temporary API that should go away once obs_ packages have
112 a standardized approach to writing versioned cameras to a Gen3 repo.
113 """
114 path = os.path.join(getPackageDir("obs_subaru"), "hsc", "camera")
115 config = CameraConfig()
116 config.load(os.path.join(path, "camera.py"))
117 return makeCameraFromPath(
118 cameraConfig=config,
119 ampInfoPath=path,
120 shortNameFunc=lambda name: name.replace(" ", "_"),
121 pupilFactoryClass=HscPupilFactory
122 )
124 def getBrighterFatterKernel(self):
125 """Return the brighter-fatter kernel for HSC as a `numpy.ndarray`.
127 This is a temporary API that should go away once obs_ packages have
128 a standardized approach to writing versioned kernels to a Gen3 repo.
129 """
130 path = os.path.join(getPackageDir("obs_subaru"), "hsc", "brighter_fatter_kernel.pkl")
131 with open(path, "rb") as fd:
132 kernel = pickle.load(fd, encoding='latin1') # encoding for pickle written with Python 2
133 return kernel
135 def writeCuratedCalibrations(self, butler):
136 """Write human-curated calibration Datasets to the given Butler with
137 the appropriate validity ranges.
139 This is a temporary API that should go away once obs_ packages have
140 a standardized approach to this problem.
141 """
143 # Write cameraGeom.Camera, with an infinite validity range.
144 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
145 universe=butler.registry.dimensions)
146 butler.registry.registerDatasetType(datasetType)
147 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
148 camera = self.getCamera()
149 butler.put(camera, datasetType, unboundedDataId)
151 # Write brighter-fatter kernel, with an infinite validity range.
152 datasetType = DatasetType("bfKernel", ("instrument", "calibration_label"), "NumpyArray",
153 universe=butler.registry.dimensions)
154 butler.registry.registerDatasetType(datasetType)
155 # Load and then put instead of just moving the file in part to ensure
156 # the version in-repo is written with Python 3 and does not need
157 # `encoding='latin1'` to be read.
158 bfKernel = self.getBrighterFatterKernel()
159 butler.put(bfKernel, datasetType, unboundedDataId)
161 # The following iterate over the values of the dictionaries returned by the transmission functions
162 # and ignore the date that is supplied. This is due to the dates not being ranges but single dates,
163 # which do not give the proper notion of validity. As such unbounded calibration labels are used
164 # when inserting into the database. In the future these could and probably should be updated to
165 # properly account for what ranges are considered valid.
167 # Write optical transmissions
168 opticsTransmissions = getOpticsTransmission()
169 datasetType = DatasetType("transmission_optics",
170 ("instrument", "calibration_label"),
171 "TransmissionCurve",
172 universe=butler.registry.dimensions)
173 butler.registry.registerDatasetType(datasetType)
174 for entry in opticsTransmissions.values():
175 if entry is None:
176 continue
177 butler.put(entry, datasetType, unboundedDataId)
179 # Write transmission sensor
180 sensorTransmissions = getSensorTransmission()
181 datasetType = DatasetType("transmission_sensor",
182 ("instrument", "detector", "calibration_label"),
183 "TransmissionCurve",
184 universe=butler.registry.dimensions)
185 butler.registry.registerDatasetType(datasetType)
186 for entry in sensorTransmissions.values():
187 if entry is None:
188 continue
189 for sensor, curve in entry.items():
190 dataId = DataCoordinate.standardize(unboundedDataId, detector=sensor)
191 butler.put(curve, datasetType, dataId)
193 # Write filter transmissions
194 filterTransmissions = getFilterTransmission()
195 datasetType = DatasetType("transmission_filter",
196 ("instrument", "physical_filter", "calibration_label"),
197 "TransmissionCurve",
198 universe=butler.registry.dimensions)
199 butler.registry.registerDatasetType(datasetType)
200 for entry in filterTransmissions.values():
201 if entry is None:
202 continue
203 for band, curve in entry.items():
204 dataId = DataCoordinate.standardize(unboundedDataId, physical_filter=band)
205 butler.put(curve, datasetType, dataId)
207 # Write atmospheric transmissions, this only as dimension of instrument as other areas will only
208 # look up along this dimension (ISR)
209 atmosphericTransmissions = getAtmosphereTransmission()
210 datasetType = DatasetType("transmission_atmosphere", ("instrument",),
211 "TransmissionCurve",
212 universe=butler.registry.dimensions)
213 butler.registry.registerDatasetType(datasetType)
214 for entry in atmosphericTransmissions.values():
215 if entry is None:
216 continue
217 butler.put(entry, datasetType, {"instrument": self.getName()})
219 # Write defects with validity ranges taken from obs_subaru_data/hsc/defects
220 # (along with the defects themselves).
221 datasetType = DatasetType("defects", ("instrument", "detector", "calibration_label"), "Defects",
222 universe=butler.registry.dimensions)
223 butler.registry.registerDatasetType(datasetType)
224 defectPath = os.path.join(getPackageDir("obs_subaru_data"), "hsc", "defects")
225 camera = self.getCamera()
226 defectsDict = read_all(defectPath, camera)[0] # This method returns a dict plus the calib type
227 endOfTime = '20380119T031407'
228 dimensionRecords = []
229 datasetRecords = []
230 # First loop just gathers up the things we want to insert, so we
231 # can do some bulk inserts and minimize the time spent in transaction.
232 for det in defectsDict:
233 detector = camera[det]
234 times = sorted([k for k in defectsDict[det]])
235 defects = [defectsDict[det][time] for time in times]
236 times = times + [parser.parse(endOfTime), ]
237 for defect, beginTime, endTime in zip(defects, times[:-1], times[1:]):
238 md = defect.getMetadata()
239 calibrationLabel = f"defect/{md['CALIBDATE']}/{md['DETECTOR']}"
240 dataId = DataCoordinate.standardize(
241 universe=butler.registry.dimensions,
242 instrument=self.getName(),
243 calibration_label=calibrationLabel,
244 detector=detector.getId(),
245 )
246 datasetRecords.append((defect, dataId))
247 dimensionRecords.append({
248 "instrument": self.getName(),
249 "name": calibrationLabel,
250 "datetime_begin": beginTime,
251 "datetime_end": endTime,
252 })
253 # Second loop actually does the inserts and filesystem writes.
254 with butler.transaction():
255 butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
256 # TODO: vectorize these puts, once butler APIs for that become
257 # available.
258 for defect, dataId in datasetRecords:
259 butler.put(defect, datasetType, dataId)
261 def ingestStrayLightData(self, butler, directory, *, transfer=None):
262 """Ingest externally-produced y-band stray light data files into
263 a data repository.
265 Parameters
266 ----------
267 butler : `lsst.daf.butler.Butler`
268 Butler initialized with the collection to ingest into.
269 directory : `str`
270 Directory containing yBackground-*.fits files.
271 transfer : `str`, optional
272 If not `None`, must be one of 'move', 'copy', 'hardlink', or
273 'symlink', indicating how to transfer the files.
274 """
275 calibrationLabel = "y-LED-encoder-on"
276 # LEDs covered up around 2018-01-01, no need for correctin after that
277 # date.
278 datetime_end = datetime.datetime(2018, 1, 1)
279 datasets = []
280 # TODO: should we use a more generic name for the dataset type?
281 # This is just the (rather HSC-specific) name used in Gen2, and while
282 # the instances of this dataset are camera-specific, the datasetType
283 # (which is used in the generic IsrTask) should not be.
284 datasetType = DatasetType("yBackground",
285 dimensions=("physical_filter", "detector", "calibration_label"),
286 storageClass="StrayLightData",
287 universe=butler.registry.dimensions)
288 for detector in self.getCamera():
289 path = os.path.join(directory, f"ybackground-{detector.getId():03d}.fits")
290 if not os.path.exists(path):
291 log.warn(f"No stray light data found for detector {detector.getId()} @ {path}.")
292 continue
293 ref = DatasetRef(datasetType, dataId={"instrument": self.getName(),
294 "detector": detector.getId(),
295 "physical_filter": "HSC-Y",
296 "calibration_label": calibrationLabel})
297 datasets.append(FileDataset(refs=ref, path=path, formatter=SubaruStrayLightDataFormatter))
298 with butler.transaction():
299 butler.registry.registerDatasetType(datasetType)
300 butler.registry.insertDimensionData("calibration_label", {"instrument": self.getName(),
301 "name": calibrationLabel,
302 "datetime_begin": datetime.date.min,
303 "datetime_end": datetime_end})
304 butler.ingest(*datasets, transfer=transfer)