2 from lsst.ip.isr import (Linearizer, CrosstalkCalib, Defects, BrighterFatterKernel, PhotodiodeCalib)
10 """Read data for a particular sensor from the standard format at a particular root.
15 Path to the top level of the data tree. This is expected to hold directories
16 named after the sensor names. They are expected to be lower case.
18 The name of the sensor for which to read data.
20 The identifier for the sensor in question.
25 A dictionary of objects constructed from the appropriate factory class.
26 The key is the validity start time as a `datetime` object.
28 factory_map = {
'qe_curve': Curve,
'defects': Defects,
'linearizer': Linearizer,
29 'crosstalk': CrosstalkCalib,
'bfk': BrighterFatterKernel,
30 'photodiode': PhotodiodeCalib, }
32 extensions = (
".ecsv",
".yaml")
33 for ext
in extensions:
34 files.extend(glob.glob(os.path.join(root, chip_name, f
"*{ext}")))
35 parts = os.path.split(root)
36 instrument = os.path.split(parts[0])[1]
38 if data_name
not in factory_map:
39 raise ValueError(f
"Unknown calibration data type, '{data_name}' found. "
40 f
"Only understand {','.join(k for k in factory_map)}")
41 factory = factory_map[data_name]
44 date_str = os.path.splitext(os.path.basename(f))[0]
45 valid_start = dateutil.parser.parse(date_str)
46 data_dict[valid_start] = factory.readText(f)
47 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
48 return data_dict, data_name
51 def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
52 """Check that the metadata is complete and self consistent
56 obj : object of same type as the factory
57 Object to retrieve metadata from in order to compare with
58 metadata inferred from the path.
59 valid_start : `datetime`
60 Start of the validity range for data
62 Name of the instrument in question
64 Identifier of the sensor in question
66 Path of the file read to construct the data
68 Name of the type of data being read
77 If the metadata from the path and the metadata encoded
78 in the path do not match for any reason.
80 md = obj.getMetadata()
81 finst = md[
'INSTRUME']
82 fchip_id = md[
'DETECTOR']
83 fdata_name = md[
'OBSTYPE']
84 if not ((finst.lower(), int(fchip_id), fdata_name.lower())
85 == (instrument.lower(), chip_id, data_name.lower())):
86 raise ValueError(f
"Path and file metadata do not agree:\n"
87 f
"Path metadata: {instrument} {chip_id} {data_name}\n"
88 f
"File metadata: {finst} {fchip_id} {fdata_name}\n"
89 f
"File read from : %s\n"%(filepath)
94 """Read all data from the standard format at a particular root.
99 Path to the top level of the data tree. This is expected to hold directories
100 named after the sensor names. They are expected to be lower case.
101 camera : `lsst.afw.cameraGeom.Camera`
102 The camera that goes with the data being read.
107 A dictionary of dictionaries of objects constructed with the appropriate factory class.
108 The first key is the sensor name lowered, and the second is the validity
109 start time as a `datetime` object.
113 Each leaf object in the constructed dictionary has metadata associated with it.
114 The detector ID may be retrieved from the DETECTOR entry of that metadata.
116 root = os.path.normpath(root)
117 dirs = os.listdir(root)
118 dirs = [d
for d
in dirs
if os.path.isdir(os.path.join(root, d))]
120 name_map = {det.getName().lower(): det.getName()
for
124 raise RuntimeError(f
"No data found on path {root}")
128 chip_name = os.path.basename(d)
131 if chip_name
not in name_map:
132 detectors = [det
for det
in camera.getNameIter()]
135 if len(detectors) > max_detectors:
137 note_str =
"examples"
138 detectors = detectors[:max_detectors]
139 raise RuntimeError(f
"Detector {chip_name} not known to supplied camera "
140 f
"{camera.getName()} ({note_str}: {','.join(detectors)})")
141 chip_id = camera[name_map[chip_name]].getId()
142 data_by_chip[chip_name], calib_type =
read_one_chip(root, chip_name, chip_id)
143 calib_types.add(calib_type)
144 if len(calib_types) != 1:
145 raise ValueError(f
'Error mixing calib types: {calib_types}')
147 no_data = all([v == {}
for v
in data_by_chip.values()])
149 raise RuntimeError(
"No data to ingest")
151 return data_by_chip, calib_type
def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name)
def read_all(root, camera)
def read_one_chip(root, chip_name, chip_id)