Coverage for python/lsst/obs/lsst/_instrument.py: 57%
166 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-24 04:06 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-24 04:06 -0700
1# This file is part of obs_lsst.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ("LsstCam", "LsstCamImSim", "LsstCamPhoSim", "LsstTS8",
23 "Latiss", "LsstTS3", "LsstUCDCam", "LsstComCam", "LsstComCamSim",
24 "LsstCamSim")
26import datetime
27import hashlib
28import os.path
30import lsst.obs.base.yamlCamera as yamlCamera
31from lsst.utils.introspection import get_full_type_name
32from lsst.utils import getPackageDir
33from lsst.obs.base import Instrument, VisitSystem
34from .filters import (LSSTCAM_FILTER_DEFINITIONS, LATISS_FILTER_DEFINITIONS,
35 LSSTCAM_IMSIM_FILTER_DEFINITIONS, TS3_FILTER_DEFINITIONS,
36 TS8_FILTER_DEFINITIONS, COMCAM_FILTER_DEFINITIONS,
37 GENERIC_FILTER_DEFINITIONS, UCD_FILTER_DEFINITIONS,
38 )
40from .translators import LatissTranslator, LsstCamTranslator, \
41 LsstUCDCamTranslator, LsstTS3Translator, LsstComCamTranslator, \
42 LsstCamPhoSimTranslator, LsstTS8Translator, LsstCamImSimTranslator, \
43 LsstComCamSimTranslator, LsstCamSimTranslator
45from .translators.lsst import GROUP_RE, TZERO_DATETIME
47PACKAGE_DIR = getPackageDir("obs_lsst")
50class LsstCam(Instrument):
51 """Gen3 Butler specialization for the LSST Main Camera.
53 Parameters
54 ----------
55 camera : `lsst.cameraGeom.Camera`
56 Camera object from which to extract detector information.
57 filters : `list` of `FilterDefinition`
58 An ordered list of filters to define the set of PhysicalFilters
59 associated with this instrument in the registry.
61 While both the camera geometry and the set of filters associated with a
62 camera are expected to change with time in general, their Butler Registry
63 representations defined by an Instrument do not. Instead:
65 - We only extract names, IDs, and purposes from the detectors in the
66 camera, which should be static information that actually reflects
67 detector "slots" rather than the physical sensors themselves. Because
68 the distinction between physical sensors and slots is unimportant in
69 the vast majority of Butler use cases, we just use "detector" even
70 though the concept really maps better to "detector slot". Ideally in
71 the future this distinction between static and time-dependent
72 information would be encoded in cameraGeom itself (e.g. by making the
73 time-dependent Detector class inherit from a related class that only
74 carries static content).
76 - The Butler Registry is expected to contain physical_filter entries for
77 all filters an instrument has ever had, because we really only care
78 about which filters were used for particular observations, not which
79 filters were *available* at some point in the past. And changes in
80 individual filters over time will be captured as changes in their
81 TransmissionCurve datasets, not changes in the registry content (which
82 is really just a label). While at present Instrument and Registry
83 do not provide a way to add new physical_filters, they will in the
84 future.
85 """
86 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS
87 instrument = "LSSTCam"
88 policyName = "lsstCam"
89 translatorClass = LsstCamTranslator
90 obsDataPackage = "obs_lsst_data"
91 visitSystem = VisitSystem.BY_SEQ_START_END
93 @property
94 def configPaths(self):
95 return [os.path.join(PACKAGE_DIR, "config"),
96 os.path.join(PACKAGE_DIR, "config", self.policyName)]
98 @classmethod
99 def getName(cls):
100 # Docstring inherited from Instrument.getName
101 return cls.instrument
103 @classmethod
104 def getCamera(cls):
105 # Constructing a YAML camera takes a long time but we rely on
106 # yamlCamera to cache for us.
107 cameraYamlFile = os.path.join(PACKAGE_DIR, "policy", f"{cls.policyName}.yaml")
108 camera = yamlCamera.makeCamera(cameraYamlFile)
109 if camera.getName() != cls.getName():
110 raise RuntimeError(f"Expected to read camera geometry for {cls.instrument}"
111 f" but instead got geometry for {camera.getName()}")
112 return camera
114 def _make_default_dimension_packer(
115 self,
116 config_attr,
117 data_id,
118 is_exposure=None,
119 default="rubin",
120 ):
121 # Docstring inherited from Instrument._make_default_dimension_packer.
122 # Only difference is the change to default above.
123 return super()._make_default_dimension_packer(
124 config_attr,
125 data_id,
126 is_exposure=is_exposure,
127 default=default,
128 )
130 def getRawFormatter(self, dataId):
131 # Docstring inherited from Instrument.getRawFormatter
132 # local import to prevent circular dependency
133 from .rawFormatter import LsstCamRawFormatter
134 return LsstCamRawFormatter
136 def register(self, registry, update=False):
137 # Docstring inherited from Instrument.register
138 # The maximum values below make Gen3's ObservationDataIdPacker produce
139 # outputs that match Gen2's ccdExposureId.
140 obsMax = self.translatorClass.max_exposure_id()
141 # Make sure this is at least 1 to avoid non-uniqueness issues (e.g.
142 # for data ids that also get used in indexing).
143 detectorMax = max(self.translatorClass.DETECTOR_MAX, 1)
144 with registry.transaction():
145 registry.syncDimensionData(
146 "instrument",
147 {
148 "name": self.getName(),
149 "detector_max": detectorMax,
150 "visit_max": obsMax,
151 "exposure_max": obsMax,
152 "class_name": get_full_type_name(self),
153 "visit_system": None if self.visitSystem is None else self.visitSystem.value,
154 },
155 update=update
156 )
157 for detector in self.getCamera():
158 registry.syncDimensionData("detector", self.extractDetectorRecord(detector), update=update)
160 self._registerFilters(registry, update=update)
162 def extractDetectorRecord(self, camGeomDetector):
163 """Create a Gen3 Detector entry dict from a cameraGeom.Detector.
164 """
165 # All of the LSST instruments have detector names like R??_S??; we'll
166 # split them up here, and instruments with only one raft can override
167 # to change the group to something else if desired.
168 # Long-term, we should get these fields into cameraGeom separately
169 # so there's no need to specialize at this stage.
170 # They are separate in ObservationInfo
171 group, name = camGeomDetector.getName().split("_")
173 # getType() returns a pybind11-wrapped enum, which unfortunately
174 # has no way to extract the name of just the value (it's always
175 # prefixed by the enum type name).
176 purpose = str(camGeomDetector.getType()).split(".")[-1]
178 return dict(
179 instrument=self.getName(),
180 id=camGeomDetector.getId(),
181 full_name=camGeomDetector.getName(),
182 name_in_raft=name,
183 purpose=purpose,
184 raft=group,
185 )
187 @classmethod
188 def group_name_to_group_id(cls, group_name: str) -> int:
189 """Translate the exposure group name to an integer.
191 Parameters
192 ----------
193 group_name : `str`
194 The name of the exposure group.
196 Returns
197 -------
198 id : `int`
199 The exposure group name in integer form. This integer might be
200 used as an ID to uniquely identify the group in contexts where
201 a string can not be used.
203 Notes
204 -----
205 If given a group name that can be directly cast to an integer it
206 returns the integer. If the group name looks like an ISO date the ID
207 returned is seconds since an arbitrary recent epoch. Otherwise
208 the group name is hashed and the first 14 digits of the hash is
209 returned along with the length of the group name.
210 """
211 # If the group is an int we return it
212 try:
213 group_id = int(group_name)
214 return group_id
215 except ValueError:
216 pass
218 # A Group is defined as ISO date with an extension
219 # The integer must be the same for a given group so we can never
220 # use datetime_begin.
221 # Nominally a GROUPID looks like "ISODATE+N" where the +N is
222 # optional. This can be converted to seconds since epoch with
223 # N being zero-padded to 4 digits and appended (defaulting to 0).
224 # For early data lacking that form we hash the group and return
225 # the int.
226 matches_date = GROUP_RE.match(group_name)
227 if matches_date:
228 iso_str = matches_date.group(1)
229 fraction = matches_date.group(2)
230 n = matches_date.group(3)
231 if n is not None:
232 n = int(n)
233 else:
234 n = 0
235 iso = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%S")
237 tdelta = iso - TZERO_DATETIME
238 epoch = int(tdelta.total_seconds())
240 # Form the integer from EPOCH + 3 DIGIT FRAC + 0-pad N
241 group_id = int(f"{epoch}{fraction}{n:04d}")
242 else:
243 # Non-standard string so convert to numbers
244 # using a hash function. Use the first N hex digits
245 group_bytes = group_name.encode("us-ascii")
246 hasher = hashlib.blake2b(group_bytes)
247 # Need to be big enough it does not possibly clash with the
248 # date-based version above
249 digest = hasher.hexdigest()[:14]
250 group_id = int(digest, base=16)
252 # To help with hash collision, append the string length
253 group_id = int(f"{group_id}{len(group_name):02d}")
255 return group_id
258class LsstComCam(LsstCam):
259 """Gen3 Butler specialization for ComCam data.
260 """
262 filterDefinitions = COMCAM_FILTER_DEFINITIONS
263 instrument = "LSSTComCam"
264 policyName = "comCam"
265 translatorClass = LsstComCamTranslator
267 def getRawFormatter(self, dataId):
268 # local import to prevent circular dependency
269 from .rawFormatter import LsstComCamRawFormatter
270 return LsstComCamRawFormatter
273class LsstComCamSim(LsstCam):
274 """Gen3 Butler specialization for ComCamSim data.
275 """
277 filterDefinitions = COMCAM_FILTER_DEFINITIONS
278 instrument = "LSSTComCamSim"
279 policyName = "comCamSim"
280 translatorClass = LsstComCamSimTranslator
282 def getRawFormatter(self, dataId):
283 # local import to prevent circular dependency
284 from .rawFormatter import LsstComCamSimRawFormatter
285 return LsstComCamSimRawFormatter
288class LsstCamSim(LsstCam):
289 """Gen3 Butler specialization for LSSTCamSim data.
290 """
292 filterDefinitions = LSSTCAM_FILTER_DEFINITIONS
293 instrument = "LSSTCamSim"
294 policyName = "lsstCamSim"
295 translatorClass = LsstCamSimTranslator
297 def getRawFormatter(self, dataId):
298 # local import to prevent circular dependency
299 from .rawFormatter import LsstCamSimRawFormatter
300 return LsstCamSimRawFormatter
303class LsstCamImSim(LsstCam):
304 """Gen3 Butler specialization for ImSim simulations.
305 """
307 instrument = "LSSTCam-imSim"
308 policyName = "imsim"
309 translatorClass = LsstCamImSimTranslator
310 filterDefinitions = LSSTCAM_IMSIM_FILTER_DEFINITIONS
311 visitSystem = VisitSystem.ONE_TO_ONE
313 def getRawFormatter(self, dataId):
314 # local import to prevent circular dependency
315 from .rawFormatter import LsstCamImSimRawFormatter
316 return LsstCamImSimRawFormatter
318 def _make_default_dimension_packer(
319 self,
320 config_attr,
321 data_id,
322 is_exposure=None,
323 default="observation",
324 ):
325 # Docstring inherited from Instrument._make_default_dimension_packer.
326 # Only difference is the change to default above (which reverts back
327 # the default in lsst.pipe.base.Instrument).
328 return super()._make_default_dimension_packer(
329 config_attr,
330 data_id,
331 is_exposure=is_exposure,
332 default=default,
333 )
336class LsstCamPhoSim(LsstCam):
337 """Gen3 Butler specialization for Phosim simulations.
338 """
340 instrument = "LSSTCam-PhoSim"
341 policyName = "phosim"
342 translatorClass = LsstCamPhoSimTranslator
343 filterDefinitions = GENERIC_FILTER_DEFINITIONS
344 visitSystem = VisitSystem.ONE_TO_ONE
346 def getRawFormatter(self, dataId):
347 # local import to prevent circular dependency
348 from .rawFormatter import LsstCamPhoSimRawFormatter
349 return LsstCamPhoSimRawFormatter
351 def _make_default_dimension_packer(
352 self,
353 config_attr,
354 data_id,
355 is_exposure=None,
356 default="observation",
357 ):
358 # Docstring inherited from Instrument._make_default_dimension_packer.
359 # Only difference is the change to default above (which reverts back
360 # the default in lsst.pipe.base.Instrument).
361 return super()._make_default_dimension_packer(
362 config_attr,
363 data_id,
364 is_exposure=is_exposure,
365 default=default,
366 )
369class LsstTS8(LsstCam):
370 """Gen3 Butler specialization for raft test stand data.
371 """
373 filterDefinitions = TS8_FILTER_DEFINITIONS
374 instrument = "LSST-TS8"
375 policyName = "ts8"
376 translatorClass = LsstTS8Translator
377 visitSystem = VisitSystem.ONE_TO_ONE
379 def getRawFormatter(self, dataId):
380 # local import to prevent circular dependency
381 from .rawFormatter import LsstTS8RawFormatter
382 return LsstTS8RawFormatter
384 def _make_default_dimension_packer(
385 self,
386 config_attr,
387 data_id,
388 is_exposure=None,
389 default="observation",
390 ):
391 # Docstring inherited from Instrument._make_default_dimension_packer.
392 # Only difference is the change to default above (which reverts back
393 # the default in lsst.pipe.base.Instrument).
394 return super()._make_default_dimension_packer(
395 config_attr,
396 data_id,
397 is_exposure=is_exposure,
398 default=default,
399 )
402class LsstUCDCam(LsstCam):
403 """Gen3 Butler specialization for UCDCam test stand data.
404 """
405 filterDefinitions = UCD_FILTER_DEFINITIONS
406 instrument = "LSST-UCDCam"
407 policyName = "ucd"
408 translatorClass = LsstUCDCamTranslator
409 visitSystem = VisitSystem.ONE_TO_ONE
411 def getRawFormatter(self, dataId):
412 # local import to prevent circular dependency
413 from .rawFormatter import LsstUCDCamRawFormatter
414 return LsstUCDCamRawFormatter
416 def _make_default_dimension_packer(
417 self,
418 config_attr,
419 data_id,
420 is_exposure=None,
421 default="observation",
422 ):
423 # Docstring inherited from Instrument._make_default_dimension_packer.
424 # Only difference is the change to default above (which reverts back
425 # the default in lsst.pipe.base.Instrument).
426 return super()._make_default_dimension_packer(
427 config_attr,
428 data_id,
429 is_exposure=is_exposure,
430 default=default,
431 )
434class LsstTS3(LsstCam):
435 """Gen3 Butler specialization for TS3 test stand data.
436 """
438 filterDefinitions = TS3_FILTER_DEFINITIONS
439 instrument = "LSST-TS3"
440 policyName = "ts3"
441 translatorClass = LsstTS3Translator
442 visitSystem = VisitSystem.ONE_TO_ONE
444 def getRawFormatter(self, dataId):
445 # local import to prevent circular dependency
446 from .rawFormatter import LsstTS3RawFormatter
447 return LsstTS3RawFormatter
449 def _make_default_dimension_packer(
450 self,
451 config_attr,
452 data_id,
453 is_exposure=None,
454 default="observation",
455 ):
456 # Docstring inherited from Instrument._make_default_dimension_packer.
457 # Only difference is the change to default above (which reverts back
458 # the default in lsst.pipe.base.Instrument).
459 return super()._make_default_dimension_packer(
460 config_attr,
461 data_id,
462 is_exposure=is_exposure,
463 default=default,
464 )
467class Latiss(LsstCam):
468 """Gen3 Butler specialization for AuxTel LATISS data.
469 """
470 filterDefinitions = LATISS_FILTER_DEFINITIONS
471 instrument = "LATISS"
472 policyName = "latiss"
473 translatorClass = LatissTranslator
475 def extractDetectorRecord(self, camGeomDetector):
476 # Override to remove group (raft) name, because LATISS only has one
477 # detector.
478 record = super().extractDetectorRecord(camGeomDetector)
479 record["raft"] = None
480 record["name_in_raft"] = record["full_name"]
481 return record
483 def getRawFormatter(self, dataId):
484 # local import to prevent circular dependency
485 from .rawFormatter import LatissRawFormatter
486 return LatissRawFormatter