Coverage for python/lsst/obs/base/_read_curated_calibs.py: 17%
66 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-14 01:58 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-14 01:58 -0700
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["CuratedCalibration", "read_all"]
26import glob
27import os
28from collections.abc import Mapping
29from typing import TYPE_CHECKING, Any, Protocol, Type
31import dateutil.parser
33if TYPE_CHECKING: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true
34 import datetime
36 import lsst.afw.cameraGeom
39class CuratedCalibration(Protocol):
40 """Protocol that describes the methods needed by this class when dealing
41 with curated calibration datasets."""
43 @classmethod
44 def readText(cls, path: str) -> CuratedCalibration:
45 ...
47 def getMetadata(self) -> Mapping:
48 ...
51def read_one_chip(
52 root: str,
53 chip_name: str,
54 chip_id: int,
55 calib_class: Type[CuratedCalibration],
56) -> tuple[dict[datetime.datetime, CuratedCalibration], str]:
57 """Read data for a particular sensor from the standard format at a
58 particular root.
60 Parameters
61 ----------
62 root : `str`
63 Path to the top level of the data tree. This is expected to hold
64 directories named after the sensor names. They are expected to be
65 lower case.
66 chip_name : `str`
67 The name of the sensor for which to read data.
68 chip_id : `int`
69 The identifier for the sensor in question.
70 calib_class : `Any`
71 The class to use to read the curated calibration text file. Must
72 support the ``readText()`` method.
74 Returns
75 -------
76 `dict`
77 A dictionary of objects constructed from the appropriate factory class.
78 The key is the validity start time as a `datetime` object.
79 """
80 files = []
81 extensions = (".ecsv", ".yaml", ".json")
82 for ext in extensions:
83 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}")))
84 parts = os.path.split(root)
85 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name>
86 data_name = parts[1]
87 data_dict: dict[datetime.datetime, Any] = {}
88 for f in files:
89 date_str = os.path.splitext(os.path.basename(f))[0]
90 valid_start = dateutil.parser.parse(date_str)
91 data_dict[valid_start] = calib_class.readText(f)
92 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
93 return data_dict, data_name
96def check_metadata(
97 obj: Any, valid_start: datetime.datetime, instrument: str, chip_id: int, filepath: str, data_name: str
98) -> None:
99 """Check that the metadata is complete and self consistent
101 Parameters
102 ----------
103 obj : object of same type as the factory
104 Object to retrieve metadata from in order to compare with
105 metadata inferred from the path.
106 valid_start : `datetime`
107 Start of the validity range for data
108 instrument : `str`
109 Name of the instrument in question
110 chip_id : `int`
111 Identifier of the sensor in question
112 filepath : `str`
113 Path of the file read to construct the data
114 data_name : `str`
115 Name of the type of data being read
117 Returns
118 -------
119 None
121 Raises
122 ------
123 ValueError
124 If the metadata from the path and the metadata encoded
125 in the path do not match for any reason.
126 """
127 md = obj.getMetadata()
128 finst = md["INSTRUME"]
129 fchip_id = md["DETECTOR"]
130 fdata_name = md["OBSTYPE"]
131 if not (
132 (finst.lower(), int(fchip_id), fdata_name.lower()) == (instrument.lower(), chip_id, data_name.lower())
133 ):
134 raise ValueError(
135 f"Path and file metadata do not agree:\n"
136 f"Path metadata: {instrument} {chip_id} {data_name}\n"
137 f"File metadata: {finst} {fchip_id} {fdata_name}\n"
138 f"File read from : %s\n" % (filepath)
139 )
142def read_all(
143 root: str,
144 camera: lsst.afw.cameraGeom.Camera,
145 calib_class: Type[CuratedCalibration],
146) -> tuple[dict[str, dict[datetime.datetime, CuratedCalibration]], str]:
147 """Read all data from the standard format at a particular root.
149 Parameters
150 ----------
151 root : `str`
152 Path to the top level of the data tree. This is expected to hold
153 directories named after the sensor names. They are expected to be
154 lower case.
155 camera : `lsst.afw.cameraGeom.Camera`
156 The camera that goes with the data being read.
157 calib_class : `Any`
158 The class to use to read the curated calibration text file. Must
159 support the ``readText()`` and ``getMetadata()`` methods.
161 Returns
162 -------
163 dict
164 A dictionary of dictionaries of objects constructed with the
165 appropriate factory class. The first key is the sensor name lowered,
166 and the second is the validity start time as a `datetime` object.
168 Notes
169 -----
170 Each leaf object in the constructed dictionary has metadata associated with
171 it. The detector ID may be retrieved from the DETECTOR entry of that
172 metadata.
173 """
174 root = os.path.normpath(root)
175 dirs = os.listdir(root) # assumes all directories contain data
176 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))]
177 data_by_chip = {}
178 name_map = {
179 det.getName().lower(): det.getName() for det in camera
180 } # we assume the directories have been lowered
182 if not dirs:
183 raise RuntimeError(f"No data found on path {root}")
185 calib_types = set()
186 for d in dirs:
187 chip_name = os.path.basename(d)
188 # Give informative error message if the detector name is not known
189 # rather than a simple KeyError
190 if chip_name not in name_map:
191 detectors = [det for det in name_map.keys()]
192 max_detectors = 10
193 note_str = "knows"
194 if len(detectors) > max_detectors:
195 # report example subset
196 note_str = "examples"
197 detectors = detectors[:max_detectors]
198 raise RuntimeError(
199 f"Detector {chip_name} not known to supplied camera "
200 f"{camera.getName()} ({note_str}: {','.join(detectors)})"
201 )
202 chip_id = camera[name_map[chip_name]].getId()
203 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id, calib_class)
204 calib_types.add(calib_type)
205 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here.
206 raise ValueError(f"Error mixing calib types: {calib_types}")
208 no_data = all([v == {} for v in data_by_chip.values()])
209 if no_data:
210 raise RuntimeError("No data to ingest")
212 return data_by_chip, calib_type