Coverage for python/lsst/pipe/tasks/read_curated_calibs.py : 84%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from lsst.meas.algorithms import Defects
2from lsst.meas.algorithms.simple_curve import Curve
3from lsst.ip.isr import Linearizer
5import os
6import glob
7import dateutil.parser
10def read_one_chip(root, chip_name, chip_id):
11 """Read data for a particular sensor from the standard format at a particular root.
13 Parameters
14 ----------
15 root : `str`
16 Path to the top level of the data tree. This is expected to hold directories
17 named after the sensor names. They are expected to be lower case.
18 chip_name : `str`
19 The name of the sensor for which to read data.
20 chip_id : `int`
21 The identifier for the sensor in question.
23 Returns
24 -------
25 `dict`
26 A dictionary of objects constructed from the appropriate factory class.
27 The key is the validity start time as a `datetime` object.
28 """
29 factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer}
30 files = []
31 extensions = (".ecsv", ".yaml")
32 for ext in extensions:
33 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}")))
34 parts = os.path.split(root)
35 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name>
36 data_name = parts[1]
37 if data_name not in factory_map: 37 ↛ 38line 37 didn't jump to line 38, because the condition on line 37 was never true
38 raise ValueError(f"Unknown calibration data type, '{data_name}' found. "
39 f"Only understand {','.join(k for k in factory_map)}")
40 factory = factory_map[data_name]
41 data_dict = {}
42 for f in files:
43 date_str = os.path.splitext(os.path.basename(f))[0]
44 valid_start = dateutil.parser.parse(date_str)
45 data_dict[valid_start] = factory.readText(f)
46 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
47 return data_dict, data_name
50def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
51 """Check that the metadata is complete and self consistent
53 Parameters
54 ----------
55 obj : object of same type as the factory
56 Object to retrieve metadata from in order to compare with
57 metadata inferred from the path.
58 valid_start : `datetime`
59 Start of the validity range for data
60 instrument : `str`
61 Name of the instrument in question
62 chip_id : `int`
63 Identifier of the sensor in question
64 filepath : `str`
65 Path of the file read to construct the data
66 data_name : `str`
67 Name of the type of data being read
69 Returns
70 -------
71 None
73 Raises
74 ------
75 ValueError
76 If the metadata from the path and the metadata encoded
77 in the path do not match for any reason.
78 """
79 md = obj.getMetadata()
80 finst = md['INSTRUME']
81 fchip_id = md['DETECTOR']
82 fdata_name = md['OBSTYPE']
83 if not ((finst.lower(), int(fchip_id), fdata_name) 83 ↛ 85line 83 didn't jump to line 85, because the condition on line 83 was never true
84 == (instrument.lower(), chip_id, data_name)):
85 raise ValueError(f"Path and file metadata do not agree:\n"
86 f"Path metadata: {instrument} {chip_id} {data_name}\n"
87 f"File metadata: {finst} {fchip_id} {fdata_name}\n"
88 f"File read from : %s\n"%(filepath)
89 )
92def read_all(root, camera):
93 """Read all data from the standard format at a particular root.
95 Parameters
96 ----------
97 root : `str`
98 Path to the top level of the data tree. This is expected to hold directories
99 named after the sensor names. They are expected to be lower case.
100 camera : `lsst.afw.cameraGeom.Camera`
101 The camera that goes with the data being read.
103 Returns
104 -------
105 dict
106 A dictionary of dictionaries of objects constructed with the appropriate factory class.
107 The first key is the sensor name lowered, and the second is the validity
108 start time as a `datetime` object.
110 Notes
111 -----
112 Each leaf object in the constructed dictionary has metadata associated with it.
113 The detector ID may be retrieved from the DETECTOR entry of that metadata.
114 """
115 root = os.path.normpath(root)
116 dirs = os.listdir(root) # assumes all directories contain data
117 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))]
118 data_by_chip = {}
119 name_map = {det.getName().lower(): det.getName() for
120 det in camera} # we assume the directories have been lowered
122 if not dirs: 122 ↛ 123line 122 didn't jump to line 123, because the condition on line 122 was never true
123 raise RuntimeError(f"No data found on path {root}")
125 calib_types = set()
126 for d in dirs:
127 chip_name = os.path.basename(d)
128 chip_id = camera[name_map[chip_name]].getId()
129 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id)
130 calib_types.add(calib_type)
131 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here. 131 ↛ 132line 131 didn't jump to line 132, because the condition on line 131 was never true
132 raise ValueError(f'Error mixing calib types: {calib_types}')
134 no_data = all([v == {} for v in data_by_chip.values()])
135 if no_data: 135 ↛ 136line 135 didn't jump to line 136, because the condition on line 135 was never true
136 raise RuntimeError(f'No data to ingest')
138 return data_by_chip, calib_type