Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from lsst.meas.algorithms.simple_curve import Curve 

2from lsst.ip.isr import (Linearizer, CrosstalkCalib, Defects) 

3 

4import os 

5import glob 

6import dateutil.parser 

7 

8 

9def read_one_chip(root, chip_name, chip_id): 

10 """Read data for a particular sensor from the standard format at a particular root. 

11 

12 Parameters 

13 ---------- 

14 root : `str` 

15 Path to the top level of the data tree. This is expected to hold directories 

16 named after the sensor names. They are expected to be lower case. 

17 chip_name : `str` 

18 The name of the sensor for which to read data. 

19 chip_id : `int` 

20 The identifier for the sensor in question. 

21 

22 Returns 

23 ------- 

24 `dict` 

25 A dictionary of objects constructed from the appropriate factory class. 

26 The key is the validity start time as a `datetime` object. 

27 """ 

28 factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer, 

29 'crosstalk': CrosstalkCalib} 

30 files = [] 

31 extensions = (".ecsv", ".yaml") 

32 for ext in extensions: 

33 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}"))) 

34 parts = os.path.split(root) 

35 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name> 

36 data_name = parts[1] 

37 if data_name not in factory_map: 

38 raise ValueError(f"Unknown calibration data type, '{data_name}' found. " 

39 f"Only understand {','.join(k for k in factory_map)}") 

40 factory = factory_map[data_name] 

41 data_dict = {} 

42 for f in files: 

43 date_str = os.path.splitext(os.path.basename(f))[0] 

44 valid_start = dateutil.parser.parse(date_str) 

45 data_dict[valid_start] = factory.readText(f) 

46 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name) 

47 return data_dict, data_name 

48 

49 

50def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name): 

51 """Check that the metadata is complete and self consistent 

52 

53 Parameters 

54 ---------- 

55 obj : object of same type as the factory 

56 Object to retrieve metadata from in order to compare with 

57 metadata inferred from the path. 

58 valid_start : `datetime` 

59 Start of the validity range for data 

60 instrument : `str` 

61 Name of the instrument in question 

62 chip_id : `int` 

63 Identifier of the sensor in question 

64 filepath : `str` 

65 Path of the file read to construct the data 

66 data_name : `str` 

67 Name of the type of data being read 

68 

69 Returns 

70 ------- 

71 None 

72 

73 Raises 

74 ------ 

75 ValueError 

76 If the metadata from the path and the metadata encoded 

77 in the path do not match for any reason. 

78 """ 

79 md = obj.getMetadata() 

80 finst = md['INSTRUME'] 

81 fchip_id = md['DETECTOR'] 

82 fdata_name = md['OBSTYPE'] 

83 if not ((finst.lower(), int(fchip_id), fdata_name.lower()) 

84 == (instrument.lower(), chip_id, data_name.lower())): 

85 raise ValueError(f"Path and file metadata do not agree:\n" 

86 f"Path metadata: {instrument} {chip_id} {data_name}\n" 

87 f"File metadata: {finst} {fchip_id} {fdata_name}\n" 

88 f"File read from : %s\n"%(filepath) 

89 ) 

90 

91 

92def read_all(root, camera): 

93 """Read all data from the standard format at a particular root. 

94 

95 Parameters 

96 ---------- 

97 root : `str` 

98 Path to the top level of the data tree. This is expected to hold directories 

99 named after the sensor names. They are expected to be lower case. 

100 camera : `lsst.afw.cameraGeom.Camera` 

101 The camera that goes with the data being read. 

102 

103 Returns 

104 ------- 

105 dict 

106 A dictionary of dictionaries of objects constructed with the appropriate factory class. 

107 The first key is the sensor name lowered, and the second is the validity 

108 start time as a `datetime` object. 

109 

110 Notes 

111 ----- 

112 Each leaf object in the constructed dictionary has metadata associated with it. 

113 The detector ID may be retrieved from the DETECTOR entry of that metadata. 

114 """ 

115 root = os.path.normpath(root) 

116 dirs = os.listdir(root) # assumes all directories contain data 

117 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))] 

118 data_by_chip = {} 

119 name_map = {det.getName().lower(): det.getName() for 

120 det in camera} # we assume the directories have been lowered 

121 

122 if not dirs: 

123 raise RuntimeError(f"No data found on path {root}") 

124 

125 calib_types = set() 

126 for d in dirs: 

127 chip_name = os.path.basename(d) 

128 # Give informative error message if the detector name is not known 

129 # rather than a simple KeyError 

130 if chip_name not in name_map: 

131 detectors = [det for det in camera.getNameIter()] 

132 max_detectors = 10 

133 note_str = "knows" 

134 if len(detectors) > max_detectors: 

135 # report example subset 

136 note_str = "examples" 

137 detectors = detectors[:max_detectors] 

138 raise RuntimeError(f"Detector {chip_name} not known to supplied camera " 

139 f"{camera.getName()} ({note_str}: {','.join(detectors)})") 

140 chip_id = camera[name_map[chip_name]].getId() 

141 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id) 

142 calib_types.add(calib_type) 

143 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here. 

144 raise ValueError(f'Error mixing calib types: {calib_types}') 

145 

146 no_data = all([v == {} for v in data_by_chip.values()]) 

147 if no_data: 

148 raise RuntimeError("No data to ingest") 

149 

150 return data_by_chip, calib_type