4from lsst.ip.isr import (Linearizer, CrosstalkCalib, Defects, BrighterFatterKernel, PhotodiodeCalib)
9from deprecated.sphinx
import deprecated
13 """Read data for a particular sensor from the standard format at a particular root.
18 Path to the top level of the data tree. This is expected to hold directories
19 named after the sensor names. They are expected to be lower case.
21 The name of the sensor
for which to read data.
23 The identifier
for the sensor
in question.
28 A dictionary of objects constructed
from the appropriate factory
class.
29 The key
is the validity start time
as a `datetime` object.
31 factory_map = {'qe_curve': Curve,
'defects': Defects,
'linearizer': Linearizer,
32 'crosstalk': CrosstalkCalib,
'bfk': BrighterFatterKernel,
33 'photodiode': PhotodiodeCalib, }
35 extensions = (
".ecsv",
".yaml")
36 for ext
in extensions:
37 files.extend(glob.glob(os.path.join(root, chip_name, f
"*{ext}")))
38 parts = os.path.split(root)
39 instrument = os.path.split(parts[0])[1]
41 if data_name
not in factory_map:
42 raise ValueError(f
"Unknown calibration data type, '{data_name}' found. "
43 f
"Only understand {','.join(k for k in factory_map)}")
44 factory = factory_map[data_name]
47 date_str = os.path.splitext(os.path.basename(f))[0]
48 valid_start = dateutil.parser.parse(date_str)
49 data_dict[valid_start] = factory.readText(f)
50 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
51 return data_dict, data_name
54def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
55 """Check that the metadata is complete and self consistent
59 obj : object of same type as the factory
60 Object to retrieve metadata
from in order to compare
with
61 metadata inferred
from the path.
62 valid_start : `datetime`
63 Start of the validity range
for data
65 Name of the instrument
in question
67 Identifier of the sensor
in question
69 Path of the file read to construct the data
71 Name of the type of data being read
80 If the metadata
from the path
and the metadata encoded
81 in the path do
not match
for any reason.
83 md = obj.getMetadata()
84 finst = md['INSTRUME']
85 fchip_id = md[
'DETECTOR']
86 fdata_name = md[
'OBSTYPE']
87 if not ((finst.lower(), int(fchip_id), fdata_name.lower())
88 == (instrument.lower(), chip_id, data_name.lower())):
89 raise ValueError(f
"Path and file metadata do not agree:\n"
90 f
"Path metadata: {instrument} {chip_id} {data_name}\n"
91 f
"File metadata: {finst} {fchip_id} {fdata_name}\n"
92 f
"File read from : %s\n"%(filepath)
96@deprecated(reason=
"Curated calibration ingest now handled by obs_base Instrument classes."
97 " Will be removed after v25.0.",
98 version=
"v25.0", category=FutureWarning)
100 """Read all data from the standard format at a particular root.
105 Path to the top level of the data tree. This is expected to hold directories
106 named after the sensor names. They are expected to be lower case.
108 The camera that goes
with the data being read.
113 A dictionary of dictionaries of objects constructed
with the appropriate factory
class.
114 The first key
is the sensor name lowered,
and the second
is the validity
115 start time
as a `datetime` object.
119 Each leaf object
in the constructed dictionary has metadata associated
with it.
120 The detector ID may be retrieved
from the DETECTOR entry of that metadata.
122 root = os.path.normpath(root)
123 dirs = os.listdir(root)
124 dirs = [d
for d
in dirs
if os.path.isdir(os.path.join(root, d))]
126 name_map = {det.getName().lower(): det.getName()
for
130 raise RuntimeError(f
"No data found on path {root}")
134 chip_name = os.path.basename(d)
137 if chip_name
not in name_map:
138 detectors = [det
for det
in camera.getNameIter()]
141 if len(detectors) > max_detectors:
143 note_str =
"examples"
144 detectors = detectors[:max_detectors]
145 raise RuntimeError(f
"Detector {chip_name} not known to supplied camera "
146 f
"{camera.getName()} ({note_str}: {','.join(detectors)})")
147 chip_id = camera[name_map[chip_name]].getId()
148 data_by_chip[chip_name], calib_type =
read_one_chip(root, chip_name, chip_id)
149 calib_types.add(calib_type)
150 if len(calib_types) != 1:
151 raise ValueError(f
'Error mixing calib types: {calib_types}')
153 no_data = all([v == {}
for v
in data_by_chip.values()])
155 raise RuntimeError(
"No data to ingest")
157 return data_by_chip, calib_type
def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name)
def read_all(root, camera)
def read_one_chip(root, chip_name, chip_id)