Coverage for python/lsst/pipe/tasks/read_curated_calibs.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from lsst.meas.algorithms import Defects
2from lsst.meas.algorithms.simple_curve import Curve
4import os
5import glob
6import dateutil.parser
9def read_one_chip(root, chip_name, chip_id):
10 """Read data for a particular sensor from the standard format at a particular root.
12 Parameters
13 ----------
14 root : `str`
15 Path to the top level of the data tree. This is expected to hold directories
16 named after the sensor names. They are expected to be lower case.
17 chip_name : `str`
18 The name of the sensor for which to read data.
19 chip_id : `int`
20 The identifier for the sensor in question.
22 Returns
23 -------
24 `dict`
25 A dictionary of objects constructed from the appropriate factory class.
26 The key is the validity start time as a `datetime` object.
27 """
28 factory_map = {'qe_curve': Curve, 'defects': Defects}
29 files = glob.glob(os.path.join(root, chip_name, '*.ecsv'))
30 parts = os.path.split(root)
31 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name>
32 data_name = parts[1]
33 if data_name not in factory_map:
34 raise ValueError(f"Unknown calibration data type, '{data_name}' found. "
35 f"Only understand {','.join(k for k in factory_map)}")
36 factory = factory_map[data_name]
37 data_dict = {}
38 for f in files:
39 date_str = os.path.splitext(os.path.basename(f))[0]
40 valid_start = dateutil.parser.parse(date_str)
41 data_dict[valid_start] = factory.readText(f)
42 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
43 return data_dict, data_name
46def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
47 """Check that the metadata is complete and self consistent
49 Parameters
50 ----------
51 obj : object of same type as the factory
52 Object to retrieve metadata from in order to compare with
53 metadata inferred from the path.
54 valid_start : `datetime`
55 Start of the validity range for data
56 instrument : `str`
57 Name of the instrument in question
58 chip_id : `int`
59 Identifier of the sensor in question
60 filepath : `str`
61 Path of the file read to construct the data
62 data_name : `str`
63 Name of the type of data being read
65 Returns
66 -------
67 None
69 Raises
70 ------
71 ValueError
72 If the metadata from the path and the metadata encoded
73 in the path do not match for any reason.
74 """
75 md = obj.getMetadata()
76 finst = md['INSTRUME']
77 fchip_id = md['DETECTOR']
78 fdata_name = md['OBSTYPE']
79 if not ((finst.lower(), int(fchip_id), fdata_name) ==
80 (instrument.lower(), chip_id, data_name)):
81 raise ValueError(f"Path and file metadata do not agree:\n"
82 f"Path metadata: {instrument} {chip_id} {data_name}\n"
83 f"File metadata: {finst} {fchip_id} {fdata_name}\n"
84 f"File read from : %s\n"%(filepath)
85 )
88def read_all(root, camera):
89 """Read all data from the standard format at a particular root.
91 Parameters
92 ----------
93 root : `str`
94 Path to the top level of the data tree. This is expected to hold directories
95 named after the sensor names. They are expected to be lower case.
96 camera : `lsst.afw.cameraGeom.Camera`
97 The camera that goes with the data being read.
99 Returns
100 -------
101 dict
102 A dictionary of dictionaries of objects constructed with the appropriate factory class.
103 The first key is the sensor name lowered, and the second is the validity
104 start time as a `datetime` object.
106 Notes
107 -----
108 Each leaf object in the constructed dictionary has metadata associated with it.
109 The detector ID may be retrieved from the DETECTOR entry of that metadata.
110 """
111 root = os.path.normpath(root)
112 dirs = os.listdir(root) # assumes all directories contain data
113 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))]
114 data_by_chip = {}
115 name_map = {det.getName().lower(): det.getName() for
116 det in camera} # we assume the directories have been lowered
118 if not dirs:
119 raise RuntimeError(f"No data found on path {root}")
121 calib_types = set()
122 for d in dirs:
123 chip_name = os.path.basename(d)
124 chip_id = camera[name_map[chip_name]].getId()
125 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id)
126 calib_types.add(calib_type)
127 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here.
128 raise ValueError(f'Error mixing calib types: {calib_types}')
130 no_data = all([v == {} for v in data_by_chip.values()])
131 if no_data:
132 raise RuntimeError(f'No data to ingest')
134 return data_by_chip, calib_type