lsst.pipe.tasks  19.0.0-39-g09c7e8d9+4
read_curated_calibs.py
Go to the documentation of this file.
1 from lsst.meas.algorithms import Defects
2 from lsst.meas.algorithms.simple_curve import Curve
3 
4 import os
5 import glob
6 import dateutil.parser
7 
8 
9 def read_one_chip(root, chip_name, chip_id):
10  """Read data for a particular sensor from the standard format at a particular root.
11 
12  Parameters
13  ----------
14  root : `str`
15  Path to the top level of the data tree. This is expected to hold directories
16  named after the sensor names. They are expected to be lower case.
17  chip_name : `str`
18  The name of the sensor for which to read data.
19  chip_id : `int`
20  The identifier for the sensor in question.
21 
22  Returns
23  -------
24  `dict`
25  A dictionary of objects constructed from the appropriate factory class.
26  The key is the validity start time as a `datetime` object.
27  """
28  factory_map = {'qe_curve': Curve, 'defects': Defects}
29  files = glob.glob(os.path.join(root, chip_name, '*.ecsv'))
30  parts = os.path.split(root)
31  instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name>
32  data_name = parts[1]
33  if data_name not in factory_map:
34  raise ValueError(f"Unknown calibration data type, '{data_name}' found. "
35  f"Only understand {','.join(k for k in factory_map)}")
36  factory = factory_map[data_name]
37  data_dict = {}
38  for f in files:
39  date_str = os.path.splitext(os.path.basename(f))[0]
40  valid_start = dateutil.parser.parse(date_str)
41  data_dict[valid_start] = factory.readText(f)
42  check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
43  return data_dict, data_name
44 
45 
46 def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
47  """Check that the metadata is complete and self consistent
48 
49  Parameters
50  ----------
51  obj : object of same type as the factory
52  Object to retrieve metadata from in order to compare with
53  metadata inferred from the path.
54  valid_start : `datetime`
55  Start of the validity range for data
56  instrument : `str`
57  Name of the instrument in question
58  chip_id : `int`
59  Identifier of the sensor in question
60  filepath : `str`
61  Path of the file read to construct the data
62  data_name : `str`
63  Name of the type of data being read
64 
65  Returns
66  -------
67  None
68 
69  Raises
70  ------
71  ValueError
72  If the metadata from the path and the metadata encoded
73  in the path do not match for any reason.
74  """
75  md = obj.getMetadata()
76  finst = md['INSTRUME']
77  fchip_id = md['DETECTOR']
78  fdata_name = md['OBSTYPE']
79  if not ((finst.lower(), int(fchip_id), fdata_name) ==
80  (instrument.lower(), chip_id, data_name)):
81  raise ValueError(f"Path and file metadata do not agree:\n"
82  f"Path metadata: {instrument} {chip_id} {data_name}\n"
83  f"File metadata: {finst} {fchip_id} {fdata_name}\n"
84  f"File read from : %s\n"%(filepath)
85  )
86 
87 
88 def read_all(root, camera):
89  """Read all data from the standard format at a particular root.
90 
91  Parameters
92  ----------
93  root : `str`
94  Path to the top level of the data tree. This is expected to hold directories
95  named after the sensor names. They are expected to be lower case.
96  camera : `lsst.afw.cameraGeom.Camera`
97  The camera that goes with the data being read.
98 
99  Returns
100  -------
101  dict
102  A dictionary of dictionaries of objects constructed with the appropriate factory class.
103  The first key is the sensor name lowered, and the second is the validity
104  start time as a `datetime` object.
105 
106  Notes
107  -----
108  Each leaf object in the constructed dictionary has metadata associated with it.
109  The detector ID may be retrieved from the DETECTOR entry of that metadata.
110  """
111  root = os.path.normpath(root)
112  dirs = os.listdir(root) # assumes all directories contain data
113  dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))]
114  data_by_chip = {}
115  name_map = {det.getName().lower(): det.getName() for
116  det in camera} # we assume the directories have been lowered
117 
118  if not dirs:
119  raise RuntimeError(f"No data found on path {root}")
120 
121  calib_types = set()
122  for d in dirs:
123  chip_name = os.path.basename(d)
124  chip_id = camera[name_map[chip_name]].getId()
125  data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id)
126  calib_types.add(calib_type)
127  if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here.
128  raise ValueError(f'Error mixing calib types: {calib_types}')
129 
130  no_data = all([v == {} for v in data_by_chip.values()])
131  if no_data:
132  raise RuntimeError(f'No data to ingest')
133 
134  return data_by_chip, calib_type
def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name)
def read_one_chip(root, chip_name, chip_id)