2from lsst.ip.isr import (Linearizer, CrosstalkCalib, Defects, BrighterFatterKernel)
10 """Read data for a particular sensor from the standard format at a particular root.
15 Path to the top level of the data tree. This is expected to hold directories
16 named after the sensor names. They are expected to be lower case.
18 The name of the sensor
for which to read data.
20 The identifier
for the sensor
in question.
25 A dictionary of objects constructed
from the appropriate factory
class.
26 The key
is the validity start time
as a `datetime` object.
28 factory_map = {'qe_curve': Curve,
'defects': Defects,
'linearizer': Linearizer,
29 'crosstalk': CrosstalkCalib,
'bfk': BrighterFatterKernel}
31 extensions = (
".ecsv",
".yaml")
32 for ext
in extensions:
33 files.extend(glob.glob(os.path.join(root, chip_name, f
"*{ext}")))
34 parts = os.path.split(root)
35 instrument = os.path.split(parts[0])[1]
37 if data_name
not in factory_map:
38 raise ValueError(f
"Unknown calibration data type, '{data_name}' found. "
39 f
"Only understand {','.join(k for k in factory_map)}")
40 factory = factory_map[data_name]
43 date_str = os.path.splitext(os.path.basename(f))[0]
44 valid_start = dateutil.parser.parse(date_str)
45 data_dict[valid_start] = factory.readText(f)
46 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
47 return data_dict, data_name
50def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
51 """Check that the metadata is complete and self consistent
55 obj : object of same type as the factory
56 Object to retrieve metadata
from in order to compare
with
57 metadata inferred
from the path.
58 valid_start : `datetime`
59 Start of the validity range
for data
61 Name of the instrument
in question
63 Identifier of the sensor
in question
65 Path of the file read to construct the data
67 Name of the type of data being read
76 If the metadata
from the path
and the metadata encoded
77 in the path do
not match
for any reason.
79 md = obj.getMetadata()
80 finst = md['INSTRUME']
81 fchip_id = md[
'DETECTOR']
82 fdata_name = md[
'OBSTYPE']
83 if not ((finst.lower(), int(fchip_id), fdata_name.lower())
84 == (instrument.lower(), chip_id, data_name.lower())):
85 raise ValueError(f
"Path and file metadata do not agree:\n"
86 f
"Path metadata: {instrument} {chip_id} {data_name}\n"
87 f
"File metadata: {finst} {fchip_id} {fdata_name}\n"
88 f
"File read from : %s\n"%(filepath)
92def read_all(root, camera):
93 """Read all data from the standard format at a particular root.
98 Path to the top level of the data tree. This is expected to hold directories
99 named after the sensor names. They are expected to be lower case.
101 The camera that goes
with the data being read.
106 A dictionary of dictionaries of objects constructed
with the appropriate factory
class.
107 The first key
is the sensor name lowered,
and the second
is the validity
108 start time
as a `datetime` object.
112 Each leaf object
in the constructed dictionary has metadata associated
with it.
113 The detector ID may be retrieved
from the DETECTOR entry of that metadata.
115 root = os.path.normpath(root)
116 dirs = os.listdir(root)
117 dirs = [d
for d
in dirs
if os.path.isdir(os.path.join(root, d))]
119 name_map = {det.getName().lower(): det.getName()
for
123 raise RuntimeError(f
"No data found on path {root}")
127 chip_name = os.path.basename(d)
130 if chip_name
not in name_map:
131 detectors = [det
for det
in camera.getNameIter()]
134 if len(detectors) > max_detectors:
136 note_str =
"examples"
137 detectors = detectors[:max_detectors]
138 raise RuntimeError(f
"Detector {chip_name} not known to supplied camera "
139 f
"{camera.getName()} ({note_str}: {','.join(detectors)})")
140 chip_id = camera[name_map[chip_name]].getId()
141 data_by_chip[chip_name], calib_type =
read_one_chip(root, chip_name, chip_id)
142 calib_types.add(calib_type)
143 if len(calib_types) != 1:
144 raise ValueError(f
'Error mixing calib types: {calib_types}')
146 no_data = all([v == {}
for v
in data_by_chip.values()])
148 raise RuntimeError(
"No data to ingest")
150 return data_by_chip, calib_type
def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name)
def read_one_chip(root, chip_name, chip_id)