Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from lsst.meas.algorithms import Defects 

2from lsst.meas.algorithms.simple_curve import Curve 

3from lsst.ip.isr import Linearizer 

4from lsst.ip.isr import CrosstalkCalib 

5 

6import os 

7import glob 

8import dateutil.parser 

9 

10 

11def read_one_chip(root, chip_name, chip_id): 

12 """Read data for a particular sensor from the standard format at a particular root. 

13 

14 Parameters 

15 ---------- 

16 root : `str` 

17 Path to the top level of the data tree. This is expected to hold directories 

18 named after the sensor names. They are expected to be lower case. 

19 chip_name : `str` 

20 The name of the sensor for which to read data. 

21 chip_id : `int` 

22 The identifier for the sensor in question. 

23 

24 Returns 

25 ------- 

26 `dict` 

27 A dictionary of objects constructed from the appropriate factory class. 

28 The key is the validity start time as a `datetime` object. 

29 """ 

30 factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer, 

31 'crosstalk': CrosstalkCalib} 

32 files = [] 

33 extensions = (".ecsv", ".yaml") 

34 for ext in extensions: 

35 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}"))) 

36 parts = os.path.split(root) 

37 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name> 

38 data_name = parts[1] 

39 if data_name not in factory_map: 39 ↛ 40line 39 didn't jump to line 40, because the condition on line 39 was never true

40 raise ValueError(f"Unknown calibration data type, '{data_name}' found. " 

41 f"Only understand {','.join(k for k in factory_map)}") 

42 factory = factory_map[data_name] 

43 data_dict = {} 

44 for f in files: 

45 date_str = os.path.splitext(os.path.basename(f))[0] 

46 valid_start = dateutil.parser.parse(date_str) 

47 data_dict[valid_start] = factory.readText(f) 

48 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name) 

49 return data_dict, data_name 

50 

51 

52def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name): 

53 """Check that the metadata is complete and self consistent 

54 

55 Parameters 

56 ---------- 

57 obj : object of same type as the factory 

58 Object to retrieve metadata from in order to compare with 

59 metadata inferred from the path. 

60 valid_start : `datetime` 

61 Start of the validity range for data 

62 instrument : `str` 

63 Name of the instrument in question 

64 chip_id : `int` 

65 Identifier of the sensor in question 

66 filepath : `str` 

67 Path of the file read to construct the data 

68 data_name : `str` 

69 Name of the type of data being read 

70 

71 Returns 

72 ------- 

73 None 

74 

75 Raises 

76 ------ 

77 ValueError 

78 If the metadata from the path and the metadata encoded 

79 in the path do not match for any reason. 

80 """ 

81 md = obj.getMetadata() 

82 finst = md['INSTRUME'] 

83 fchip_id = md['DETECTOR'] 

84 fdata_name = md['OBSTYPE'] 

85 if not ((finst.lower(), int(fchip_id), fdata_name.lower()) 85 ↛ 87line 85 didn't jump to line 87, because the condition on line 85 was never true

86 == (instrument.lower(), chip_id, data_name.lower())): 

87 raise ValueError(f"Path and file metadata do not agree:\n" 

88 f"Path metadata: {instrument} {chip_id} {data_name}\n" 

89 f"File metadata: {finst} {fchip_id} {fdata_name}\n" 

90 f"File read from : %s\n"%(filepath) 

91 ) 

92 

93 

94def read_all(root, camera): 

95 """Read all data from the standard format at a particular root. 

96 

97 Parameters 

98 ---------- 

99 root : `str` 

100 Path to the top level of the data tree. This is expected to hold directories 

101 named after the sensor names. They are expected to be lower case. 

102 camera : `lsst.afw.cameraGeom.Camera` 

103 The camera that goes with the data being read. 

104 

105 Returns 

106 ------- 

107 dict 

108 A dictionary of dictionaries of objects constructed with the appropriate factory class. 

109 The first key is the sensor name lowered, and the second is the validity 

110 start time as a `datetime` object. 

111 

112 Notes 

113 ----- 

114 Each leaf object in the constructed dictionary has metadata associated with it. 

115 The detector ID may be retrieved from the DETECTOR entry of that metadata. 

116 """ 

117 root = os.path.normpath(root) 

118 dirs = os.listdir(root) # assumes all directories contain data 

119 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))] 

120 data_by_chip = {} 

121 name_map = {det.getName().lower(): det.getName() for 

122 det in camera} # we assume the directories have been lowered 

123 

124 if not dirs: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true

125 raise RuntimeError(f"No data found on path {root}") 

126 

127 calib_types = set() 

128 for d in dirs: 

129 chip_name = os.path.basename(d) 

130 # Give informative error message if the detector name is not known 

131 # rather than a simple KeyError 

132 if chip_name not in name_map: 132 ↛ 133line 132 didn't jump to line 133, because the condition on line 132 was never true

133 detectors = [det for det in camera.getNameIter()] 

134 max_detectors = 10 

135 note_str = "knows" 

136 if len(detectors) > max_detectors: 

137 # report example subset 

138 note_str = "examples" 

139 detectors = detectors[:max_detectors] 

140 raise RuntimeError(f"Detector {chip_name} not known to supplied camera " 

141 f"{camera.getName()} ({note_str}: {','.join(detectors)})") 

142 chip_id = camera[name_map[chip_name]].getId() 

143 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id) 

144 calib_types.add(calib_type) 

145 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here. 145 ↛ 146line 145 didn't jump to line 146, because the condition on line 145 was never true

146 raise ValueError(f'Error mixing calib types: {calib_types}') 

147 

148 no_data = all([v == {} for v in data_by_chip.values()]) 

149 if no_data: 149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true

150 raise RuntimeError("No data to ingest") 

151 

152 return data_by_chip, calib_type