Coverage for python/lsst/obs/base/instrument.py : 24%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "makeVisitRecordFromObsInfo",
23 "addUnboundedCalibrationLabel")
25import os.path
26from abc import ABCMeta, abstractmethod
27import astropy.time
29from lsst.daf.butler import TIMESPAN_MIN, TIMESPAN_MAX, DatasetType, DataCoordinate
30from lsst.utils import getPackageDir
32# To be a standard text curated calibration means that we use a
33# standard definition for the corresponding DatasetType.
34StandardCuratedCalibrationDatasetTypes = {
35 "defects": {"dimensions": ("instrument", "detector", "calibration_label"),
36 "storageClass": "Defects"},
37 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"),
38 "storageClass": "QECurve"},
39}
42class Instrument(metaclass=ABCMeta):
43 """Base class for instrument-specific logic for the Gen3 Butler.
45 Concrete instrument subclasses should be directly constructable with no
46 arguments.
47 """
49 configPaths = ()
50 """Paths to config files to read for specific Tasks.
52 The paths in this list should contain files of the form `task.py`, for
53 each of the Tasks that requires special configuration.
54 """
56 policyName = None
57 """Instrument specific name to use when locating a policy or configuration
58 file in the file system."""
60 obsDataPackage = None
61 """Name of the package containing the text curated calibration files.
62 Usually a obs _data package. If `None` no curated calibration files
63 will be read. (`str`)"""
65 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
66 """The dataset types expected to be obtained from the obsDataPackage.
67 These dataset types are all required to have standard definitions and
68 must be known to the base class. Clearing this list will prevent
69 any of these calibrations from being stored. If a dataset type is not
70 known to a specific instrument it can still be included in this list
71 since the data package is the source of truth.
72 """
74 @property
75 @abstractmethod
76 def filterDefinitions(self):
77 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
78 for this instrument.
79 """
80 return None
82 def __init__(self, *args, **kwargs):
83 self.filterDefinitions.reset()
84 self.filterDefinitions.defineFilters()
85 self._obsDataPackageDir = None
87 @classmethod
88 @abstractmethod
89 def getName(cls):
90 raise NotImplementedError()
92 @abstractmethod
93 def getCamera(self):
94 """Retrieve the cameraGeom representation of this instrument.
96 This is a temporary API that should go away once obs_ packages have
97 a standardized approach to writing versioned cameras to a Gen3 repo.
98 """
99 raise NotImplementedError()
101 @abstractmethod
102 def register(self, registry):
103 """Insert instrument, physical_filter, and detector entries into a
104 `Registry`.
105 """
106 raise NotImplementedError()
108 @property
109 def obsDataPackageDir(self):
110 if self.obsDataPackage is None:
111 return None
112 if self._obsDataPackageDir is None:
113 # Defer any problems with locating the package until
114 # we need to find it.
115 self._obsDataPackageDir = getPackageDir(self.obsDataPackage)
116 return self._obsDataPackageDir
118 def _registerFilters(self, registry):
119 """Register the physical and abstract filter Dimension relationships.
120 This should be called in the ``register`` implementation.
122 Parameters
123 ----------
124 registry : `lsst.daf.butler.core.Registry`
125 The registry to add dimensions to.
126 """
127 for filter in self.filterDefinitions:
128 # fix for undefined abstract filters causing trouble in the registry:
129 if filter.abstract_filter is None:
130 abstract_filter = filter.physical_filter
131 else:
132 abstract_filter = filter.abstract_filter
134 registry.insertDimensionData("physical_filter",
135 {"instrument": self.getName(),
136 "name": filter.physical_filter,
137 "abstract_filter": abstract_filter
138 })
140 @abstractmethod
141 def getRawFormatter(self, dataId):
142 """Return the Formatter class that should be used to read a particular
143 raw file.
145 Parameters
146 ----------
147 dataId : `DataCoordinate`
148 Dimension-based ID for the raw file or files being ingested.
150 Returns
151 -------
152 formatter : `Formatter` class
153 Class to be used that reads the file into an
154 `lsst.afw.image.Exposure` instance.
155 """
156 raise NotImplementedError()
158 def writeCuratedCalibrations(self, butler):
159 """Write human-curated calibration Datasets to the given Butler with
160 the appropriate validity ranges.
162 Parameters
163 ----------
164 butler : `lsst.daf.butler.Butler`
165 Butler to use to store these calibrations.
167 Notes
168 -----
169 Expected to be called from subclasses. The base method calls
170 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
171 """
172 self.writeCameraGeom(butler)
173 self.writeStandardTextCuratedCalibrations(butler)
175 def applyConfigOverrides(self, name, config):
176 """Apply instrument-specific overrides for a task config.
178 Parameters
179 ----------
180 name : `str`
181 Name of the object being configured; typically the _DefaultName
182 of a Task.
183 config : `lsst.pex.config.Config`
184 Config instance to which overrides should be applied.
185 """
186 for root in self.configPaths:
187 path = os.path.join(root, f"{name}.py")
188 if os.path.exists(path):
189 config.load(path)
191 def writeCameraGeom(self, butler):
192 """Write the default camera geometry to the butler repository
193 with an infinite validity range.
195 Parameters
196 ----------
197 butler : `lsst.daf.butler.Butler`
198 Butler to receive these calibration datasets.
199 """
201 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
202 universe=butler.registry.dimensions)
203 butler.registry.registerDatasetType(datasetType)
204 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
205 camera = self.getCamera()
206 butler.put(camera, datasetType, unboundedDataId)
208 def writeStandardTextCuratedCalibrations(self, butler):
209 """Write the set of standardized curated text calibrations to
210 the repository.
212 Parameters
213 ----------
214 butler : `lsst.daf.butler.Butler`
215 Butler to receive these calibration datasets.
216 """
218 for datasetTypeName in self.standardCuratedDatasetTypes:
219 # We need to define the dataset types.
220 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
221 raise ValueError(f"DatasetType {datasetTypeName} not in understood list"
222 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
223 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
224 datasetType = DatasetType(datasetTypeName,
225 universe=butler.registry.dimensions,
226 **definition)
227 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType)
229 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType):
230 """Write standardized curated calibration datasets for this specific
231 dataset type from an obs data package.
233 Parameters
234 ----------
235 butler : `lsst.daf.butler.Butler`
236 Gen3 butler in which to put the calibrations.
237 datasetType : `lsst.daf.butler.DatasetType`
238 Dataset type to be put.
240 Notes
241 -----
242 This method scans the location defined in the ``obsDataPackageDir``
243 class attribute for curated calibrations corresponding to the
244 supplied dataset type. The directory name in the data package must
245 match the name of the dataset type. They are assumed to use the
246 standard layout and can be read by
247 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
248 metadata.
249 """
250 if self.obsDataPackageDir is None:
251 # if there is no data package then there can't be datasets
252 return
254 calibPath = os.path.join(self.obsDataPackageDir, self.policyName,
255 datasetType.name)
257 if not os.path.exists(calibPath):
258 return
260 # Register the dataset type
261 butler.registry.registerDatasetType(datasetType)
263 # obs_base can't depend on pipe_tasks but concrete obs packages
264 # can -- we therefore have to defer import
265 from lsst.pipe.tasks.read_curated_calibs import read_all
267 camera = self.getCamera()
268 calibsDict = read_all(calibPath, camera)[0] # second return is calib type
269 endOfTime = TIMESPAN_MAX
270 dimensionRecords = []
271 datasetRecords = []
272 for det in calibsDict:
273 times = sorted([k for k in calibsDict[det]])
274 calibs = [calibsDict[det][time] for time in times]
275 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
276 times += [endOfTime]
277 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
278 md = calib.getMetadata()
279 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
280 dataId = DataCoordinate.standardize(
281 universe=butler.registry.dimensions,
282 instrument=self.getName(),
283 calibration_label=calibrationLabel,
284 detector=md["DETECTOR"],
285 )
286 datasetRecords.append((calib, dataId))
287 dimensionRecords.append({
288 "instrument": self.getName(),
289 "name": calibrationLabel,
290 "datetime_begin": beginTime,
291 "datetime_end": endTime,
292 })
294 # Second loop actually does the inserts and filesystem writes.
295 with butler.transaction():
296 butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
297 # TODO: vectorize these puts, once butler APIs for that become
298 # available.
299 for calib, dataId in datasetRecords:
300 butler.put(calib, datasetType, dataId)
303def makeExposureRecordFromObsInfo(obsInfo, universe):
304 """Construct an exposure DimensionRecord from
305 `astro_metadata_translator.ObservationInfo`.
307 Parameters
308 ----------
309 obsInfo : `astro_metadata_translator.ObservationInfo`
310 A `~astro_metadata_translator.ObservationInfo` object corresponding to
311 the exposure.
312 universe : `DimensionUniverse`
313 Set of all known dimensions.
315 Returns
316 -------
317 record : `DimensionRecord`
318 A record containing exposure metadata, suitable for insertion into
319 a `Registry`.
320 """
321 dimension = universe["exposure"]
322 return dimension.RecordClass.fromDict({
323 "instrument": obsInfo.instrument,
324 "id": obsInfo.exposure_id,
325 "name": obsInfo.observation_id,
326 "group": obsInfo.exposure_group,
327 "datetime_begin": obsInfo.datetime_begin,
328 "datetime_end": obsInfo.datetime_end,
329 "exposure_time": obsInfo.exposure_time.to_value("s"),
330 "dark_time": obsInfo.dark_time.to_value("s"),
331 "observation_type": obsInfo.observation_type,
332 "physical_filter": obsInfo.physical_filter,
333 "visit": obsInfo.visit_id,
334 })
337def makeVisitRecordFromObsInfo(obsInfo, universe, *, region=None):
338 """Construct a visit `DimensionRecord` from
339 `astro_metadata_translator.ObservationInfo`.
341 Parameters
342 ----------
343 obsInfo : `astro_metadata_translator.ObservationInfo`
344 A `~astro_metadata_translator.ObservationInfo` object corresponding to
345 the exposure.
346 universe : `DimensionUniverse`
347 Set of all known dimensions.
348 region : `lsst.sphgeom.Region`, optional
349 Spatial region for the visit.
351 Returns
352 -------
353 record : `DimensionRecord`
354 A record containing visit metadata, suitable for insertion into a
355 `Registry`.
356 """
357 dimension = universe["visit"]
358 return dimension.RecordClass.fromDict({
359 "instrument": obsInfo.instrument,
360 "id": obsInfo.visit_id,
361 "name": obsInfo.observation_id,
362 "datetime_begin": obsInfo.datetime_begin,
363 "datetime_end": obsInfo.datetime_end,
364 "exposure_time": obsInfo.exposure_time.to_value("s"),
365 "physical_filter": obsInfo.physical_filter,
366 "region": region,
367 })
370def addUnboundedCalibrationLabel(registry, instrumentName):
371 """Add a special 'unbounded' calibration_label dimension entry for the
372 given camera that is valid for any exposure.
374 If such an entry already exists, this function just returns a `DataId`
375 for the existing entry.
377 Parameters
378 ----------
379 registry : `Registry`
380 Registry object in which to insert the dimension entry.
381 instrumentName : `str`
382 Name of the instrument this calibration label is associated with.
384 Returns
385 -------
386 dataId : `DataId`
387 New or existing data ID for the unbounded calibration.
388 """
389 d = dict(instrument=instrumentName, calibration_label="unbounded")
390 try:
391 return registry.expandDataId(d)
392 except LookupError:
393 pass
394 entry = d.copy()
395 entry["datetime_begin"] = TIMESPAN_MIN
396 entry["datetime_end"] = TIMESPAN_MAX
397 registry.insertDimensionData("calibration_label", entry)
398 return registry.expandDataId(d)