Coverage for python/lsst/pipe/tasks/read_curated_calibs.py : 84%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from lsst.meas.algorithms import Defects
2from lsst.meas.algorithms.simple_curve import Curve
3from lsst.ip.isr import Linearizer
4from lsst.ip.isr import CrosstalkCalib
6import os
7import glob
8import dateutil.parser
11def read_one_chip(root, chip_name, chip_id):
12 """Read data for a particular sensor from the standard format at a particular root.
14 Parameters
15 ----------
16 root : `str`
17 Path to the top level of the data tree. This is expected to hold directories
18 named after the sensor names. They are expected to be lower case.
19 chip_name : `str`
20 The name of the sensor for which to read data.
21 chip_id : `int`
22 The identifier for the sensor in question.
24 Returns
25 -------
26 `dict`
27 A dictionary of objects constructed from the appropriate factory class.
28 The key is the validity start time as a `datetime` object.
29 """
30 factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer,
31 'crosstalk': CrosstalkCalib}
32 files = []
33 extensions = (".ecsv", ".yaml")
34 for ext in extensions:
35 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}")))
36 parts = os.path.split(root)
37 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name>
38 data_name = parts[1]
39 if data_name not in factory_map: 39 ↛ 40line 39 didn't jump to line 40, because the condition on line 39 was never true
40 raise ValueError(f"Unknown calibration data type, '{data_name}' found. "
41 f"Only understand {','.join(k for k in factory_map)}")
42 factory = factory_map[data_name]
43 data_dict = {}
44 for f in files:
45 date_str = os.path.splitext(os.path.basename(f))[0]
46 valid_start = dateutil.parser.parse(date_str)
47 data_dict[valid_start] = factory.readText(f)
48 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
49 return data_dict, data_name
52def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
53 """Check that the metadata is complete and self consistent
55 Parameters
56 ----------
57 obj : object of same type as the factory
58 Object to retrieve metadata from in order to compare with
59 metadata inferred from the path.
60 valid_start : `datetime`
61 Start of the validity range for data
62 instrument : `str`
63 Name of the instrument in question
64 chip_id : `int`
65 Identifier of the sensor in question
66 filepath : `str`
67 Path of the file read to construct the data
68 data_name : `str`
69 Name of the type of data being read
71 Returns
72 -------
73 None
75 Raises
76 ------
77 ValueError
78 If the metadata from the path and the metadata encoded
79 in the path do not match for any reason.
80 """
81 md = obj.getMetadata()
82 finst = md['INSTRUME']
83 fchip_id = md['DETECTOR']
84 fdata_name = md['OBSTYPE']
85 if not ((finst.lower(), int(fchip_id), fdata_name.lower()) 85 ↛ 87line 85 didn't jump to line 87, because the condition on line 85 was never true
86 == (instrument.lower(), chip_id, data_name.lower())):
87 raise ValueError(f"Path and file metadata do not agree:\n"
88 f"Path metadata: {instrument} {chip_id} {data_name}\n"
89 f"File metadata: {finst} {fchip_id} {fdata_name}\n"
90 f"File read from : %s\n"%(filepath)
91 )
94def read_all(root, camera):
95 """Read all data from the standard format at a particular root.
97 Parameters
98 ----------
99 root : `str`
100 Path to the top level of the data tree. This is expected to hold directories
101 named after the sensor names. They are expected to be lower case.
102 camera : `lsst.afw.cameraGeom.Camera`
103 The camera that goes with the data being read.
105 Returns
106 -------
107 dict
108 A dictionary of dictionaries of objects constructed with the appropriate factory class.
109 The first key is the sensor name lowered, and the second is the validity
110 start time as a `datetime` object.
112 Notes
113 -----
114 Each leaf object in the constructed dictionary has metadata associated with it.
115 The detector ID may be retrieved from the DETECTOR entry of that metadata.
116 """
117 root = os.path.normpath(root)
118 dirs = os.listdir(root) # assumes all directories contain data
119 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))]
120 data_by_chip = {}
121 name_map = {det.getName().lower(): det.getName() for
122 det in camera} # we assume the directories have been lowered
124 if not dirs: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true
125 raise RuntimeError(f"No data found on path {root}")
127 calib_types = set()
128 for d in dirs:
129 chip_name = os.path.basename(d)
130 chip_id = camera[name_map[chip_name]].getId()
131 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id)
132 calib_types.add(calib_type)
133 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here. 133 ↛ 134line 133 didn't jump to line 134, because the condition on line 133 was never true
134 raise ValueError(f'Error mixing calib types: {calib_types}')
136 no_data = all([v == {} for v in data_by_chip.values()])
137 if no_data: 137 ↛ 138line 137 didn't jump to line 138, because the condition on line 137 was never true
138 raise RuntimeError(f'No data to ingest')
140 return data_by_chip, calib_type