lsst.pipe.tasks  20.0.0-32-g15a0e07c+fb20712e7c
read_curated_calibs.py
Go to the documentation of this file.
1 from lsst.meas.algorithms import Defects
2 from lsst.meas.algorithms.simple_curve import Curve
3 from lsst.ip.isr import Linearizer
4 from lsst.ip.isr import CrosstalkCalib
5 
6 import os
7 import glob
8 import dateutil.parser
9 
10 
11 def read_one_chip(root, chip_name, chip_id):
12  """Read data for a particular sensor from the standard format at a particular root.
13 
14  Parameters
15  ----------
16  root : `str`
17  Path to the top level of the data tree. This is expected to hold directories
18  named after the sensor names. They are expected to be lower case.
19  chip_name : `str`
20  The name of the sensor for which to read data.
21  chip_id : `int`
22  The identifier for the sensor in question.
23 
24  Returns
25  -------
26  `dict`
27  A dictionary of objects constructed from the appropriate factory class.
28  The key is the validity start time as a `datetime` object.
29  """
30  factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer,
31  'crosstalk': CrosstalkCalib}
32  files = []
33  extensions = (".ecsv", ".yaml")
34  for ext in extensions:
35  files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}")))
36  parts = os.path.split(root)
37  instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name>
38  data_name = parts[1]
39  if data_name not in factory_map:
40  raise ValueError(f"Unknown calibration data type, '{data_name}' found. "
41  f"Only understand {','.join(k for k in factory_map)}")
42  factory = factory_map[data_name]
43  data_dict = {}
44  for f in files:
45  date_str = os.path.splitext(os.path.basename(f))[0]
46  valid_start = dateutil.parser.parse(date_str)
47  data_dict[valid_start] = factory.readText(f)
48  check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
49  return data_dict, data_name
50 
51 
52 def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
53  """Check that the metadata is complete and self consistent
54 
55  Parameters
56  ----------
57  obj : object of same type as the factory
58  Object to retrieve metadata from in order to compare with
59  metadata inferred from the path.
60  valid_start : `datetime`
61  Start of the validity range for data
62  instrument : `str`
63  Name of the instrument in question
64  chip_id : `int`
65  Identifier of the sensor in question
66  filepath : `str`
67  Path of the file read to construct the data
68  data_name : `str`
69  Name of the type of data being read
70 
71  Returns
72  -------
73  None
74 
75  Raises
76  ------
77  ValueError
78  If the metadata from the path and the metadata encoded
79  in the path do not match for any reason.
80  """
81  md = obj.getMetadata()
82  finst = md['INSTRUME']
83  fchip_id = md['DETECTOR']
84  fdata_name = md['OBSTYPE']
85  if not ((finst.lower(), int(fchip_id), fdata_name.lower())
86  == (instrument.lower(), chip_id, data_name.lower())):
87  raise ValueError(f"Path and file metadata do not agree:\n"
88  f"Path metadata: {instrument} {chip_id} {data_name}\n"
89  f"File metadata: {finst} {fchip_id} {fdata_name}\n"
90  f"File read from : %s\n"%(filepath)
91  )
92 
93 
94 def read_all(root, camera):
95  """Read all data from the standard format at a particular root.
96 
97  Parameters
98  ----------
99  root : `str`
100  Path to the top level of the data tree. This is expected to hold directories
101  named after the sensor names. They are expected to be lower case.
102  camera : `lsst.afw.cameraGeom.Camera`
103  The camera that goes with the data being read.
104 
105  Returns
106  -------
107  dict
108  A dictionary of dictionaries of objects constructed with the appropriate factory class.
109  The first key is the sensor name lowered, and the second is the validity
110  start time as a `datetime` object.
111 
112  Notes
113  -----
114  Each leaf object in the constructed dictionary has metadata associated with it.
115  The detector ID may be retrieved from the DETECTOR entry of that metadata.
116  """
117  root = os.path.normpath(root)
118  dirs = os.listdir(root) # assumes all directories contain data
119  dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))]
120  data_by_chip = {}
121  name_map = {det.getName().lower(): det.getName() for
122  det in camera} # we assume the directories have been lowered
123 
124  if not dirs:
125  raise RuntimeError(f"No data found on path {root}")
126 
127  calib_types = set()
128  for d in dirs:
129  chip_name = os.path.basename(d)
130  # Give informative error message if the detector name is not known
131  # rather than a simple KeyError
132  if chip_name not in name_map:
133  detectors = [det for det in camera.getNameIter()]
134  max_detectors = 10
135  note_str = "knows"
136  if len(detectors) > max_detectors:
137  # report example subset
138  note_str = "examples"
139  detectors = detectors[:max_detectors]
140  raise RuntimeError(f"Detector {chip_name} not known to supplied camera "
141  f"{camera.getName()} ({note_str}: {','.join(detectors)})")
142  chip_id = camera[name_map[chip_name]].getId()
143  data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id)
144  calib_types.add(calib_type)
145  if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here.
146  raise ValueError(f'Error mixing calib types: {calib_types}')
147 
148  no_data = all([v == {} for v in data_by_chip.values()])
149  if no_data:
150  raise RuntimeError("No data to ingest")
151 
152  return data_by_chip, calib_type
lsst::meas::algorithms::simple_curve
lsst.pipe.tasks.read_curated_calibs.read_all
def read_all(root, camera)
Definition: read_curated_calibs.py:94
lsst::ip::isr
lsst::meas::algorithms
lsst.pipe.tasks.read_curated_calibs.check_metadata
def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name)
Definition: read_curated_calibs.py:52
lsst.pipe.tasks.read_curated_calibs.read_one_chip
def read_one_chip(root, chip_name, chip_id)
Definition: read_curated_calibs.py:11