10 """Read data for a particular sensor from the standard format at a particular root. 15 Path to the top level of the data tree. This is expected to hold directories 16 named after the sensor names. They are expected to be lower case. 18 The name of the sensor for which to read data. 20 The identifier for the sensor in question. 25 A dictionary of objects constructed from the appropriate factory class. 26 The key is the validity start time as a `datetime` object. 28 factory_map = {
'qe_curve': Curve,
'defects': Defects}
29 files = glob.glob(os.path.join(root, chip_name,
'*.ecsv'))
30 parts = os.path.split(root)
31 instrument = os.path.split(parts[0])[1]
33 if data_name
not in factory_map:
34 raise ValueError(f
"Unknown calibration data type, '{data_name}' found. " 35 f
"Only understand {','.join(k for k in factory_map)}")
36 factory = factory_map[data_name]
39 date_str = os.path.splitext(os.path.basename(f))[0]
40 valid_start = dateutil.parser.parse(date_str)
41 data_dict[valid_start] = factory.readText(f)
42 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
43 return data_dict, data_name
46 def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
47 """Check that the metadata is complete and self consistent 51 obj : object of same type as the factory 52 Object to retrieve metadata from in order to compare with 53 metadata inferred from the path. 54 valid_start : `datetime` 55 Start of the validity range for data 57 Name of the instrument in question 59 Identifier of the sensor in question 61 Path of the file read to construct the data 63 Name of the type of data being read 72 If the metadata from the path and the metadata encoded 73 in the path do not match for any reason. 75 md = obj.getMetadata()
76 finst = md[
'INSTRUME']
77 fchip_id = md[
'DETECTOR']
78 fdata_name = md[
'OBSTYPE']
79 if not ((finst.lower(), int(fchip_id), fdata_name) ==
80 (instrument.lower(), chip_id, data_name)):
81 raise ValueError(f
"Path and file metadata do not agree:\n" 82 f
"Path metadata: {instrument} {chip_id} {data_name}\n" 83 f
"File metadata: {finst} {fchip_id} {fdata_name}\n" 84 f
"File read from : %s\n"%(filepath)
89 """Read all data from the standard format at a particular root. 94 Path to the top level of the data tree. This is expected to hold directories 95 named after the sensor names. They are expected to be lower case. 96 camera : `lsst.afw.cameraGeom.Camera` 97 The camera that goes with the data being read. 102 A dictionary of dictionaries of objects constructed with the appropriate factory class. 103 The first key is the sensor name lowered, and the second is the validity 104 start time as a `datetime` object. 108 Each leaf object in the constructed dictionary has metadata associated with it. 109 The detector ID may be retrieved from the DETECTOR entry of that metadata. 111 root = os.path.normpath(root)
112 dirs = os.listdir(root)
113 dirs = [d
for d
in dirs
if os.path.isdir(os.path.join(root, d))]
115 name_map = {det.getName().lower(): det.getName()
for 119 raise RuntimeError(f
"No data found on path {root}")
123 chip_name = os.path.basename(d)
124 chip_id = camera[name_map[chip_name]].getId()
125 data_by_chip[chip_name], calib_type =
read_one_chip(root, chip_name, chip_id)
126 calib_types.add(calib_type)
127 if len(calib_types) != 1:
128 raise ValueError(f
'Error mixing calib types: {calib_types}')
130 no_data = all([v == {}
for v
in data_by_chip.values()])
132 raise RuntimeError(f
'No data to ingest')
134 return data_by_chip, calib_type
def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name)
def read_all(root, camera)
def read_one_chip(root, chip_name, chip_id)