Coverage for python/lsst/pipe/tasks/read_curated_calibs.py: 12%

62 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-18 20:09 +0000

1__all__ = ["read_all"] 

2 

3from lsst.meas.algorithms.simple_curve import Curve 

4from lsst.ip.isr import (Linearizer, CrosstalkCalib, Defects, BrighterFatterKernel, PhotodiodeCalib) 

5 

6import os 

7import glob 

8import dateutil.parser 

9from deprecated.sphinx import deprecated 

10 

11 

12def read_one_chip(root, chip_name, chip_id): 

13 """Read data for a particular sensor from the standard format at a particular root. 

14 

15 Parameters 

16 ---------- 

17 root : `str` 

18 Path to the top level of the data tree. This is expected to hold directories 

19 named after the sensor names. They are expected to be lower case. 

20 chip_name : `str` 

21 The name of the sensor for which to read data. 

22 chip_id : `int` 

23 The identifier for the sensor in question. 

24 

25 Returns 

26 ------- 

27 `dict` 

28 A dictionary of objects constructed from the appropriate factory class. 

29 The key is the validity start time as a `datetime` object. 

30 """ 

31 factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer, 

32 'crosstalk': CrosstalkCalib, 'bfk': BrighterFatterKernel, 

33 'photodiode': PhotodiodeCalib, } 

34 files = [] 

35 extensions = (".ecsv", ".yaml") 

36 for ext in extensions: 

37 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}"))) 

38 parts = os.path.split(root) 

39 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name> 

40 data_name = parts[1] 

41 if data_name not in factory_map: 

42 raise ValueError(f"Unknown calibration data type, '{data_name}' found. " 

43 f"Only understand {','.join(k for k in factory_map)}") 

44 factory = factory_map[data_name] 

45 data_dict = {} 

46 for f in files: 

47 date_str = os.path.splitext(os.path.basename(f))[0] 

48 valid_start = dateutil.parser.parse(date_str) 

49 data_dict[valid_start] = factory.readText(f) 

50 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name) 

51 return data_dict, data_name 

52 

53 

54def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name): 

55 """Check that the metadata is complete and self consistent 

56 

57 Parameters 

58 ---------- 

59 obj : object of same type as the factory 

60 Object to retrieve metadata from in order to compare with 

61 metadata inferred from the path. 

62 valid_start : `datetime` 

63 Start of the validity range for data 

64 instrument : `str` 

65 Name of the instrument in question 

66 chip_id : `int` 

67 Identifier of the sensor in question 

68 filepath : `str` 

69 Path of the file read to construct the data 

70 data_name : `str` 

71 Name of the type of data being read 

72 

73 Returns 

74 ------- 

75 None 

76 

77 Raises 

78 ------ 

79 ValueError 

80 If the metadata from the path and the metadata encoded 

81 in the path do not match for any reason. 

82 """ 

83 md = obj.getMetadata() 

84 finst = md['INSTRUME'] 

85 fchip_id = md['DETECTOR'] 

86 fdata_name = md['OBSTYPE'] 

87 if not ((finst.lower(), int(fchip_id), fdata_name.lower()) 

88 == (instrument.lower(), chip_id, data_name.lower())): 

89 raise ValueError(f"Path and file metadata do not agree:\n" 

90 f"Path metadata: {instrument} {chip_id} {data_name}\n" 

91 f"File metadata: {finst} {fchip_id} {fdata_name}\n" 

92 f"File read from : %s\n"%(filepath) 

93 ) 

94 

95 

96@deprecated(reason="Curated calibration ingest now handled by obs_base Instrument classes." 

97 " Will be removed after v25.0.", 

98 version="v25.0", category=FutureWarning) 

99def read_all(root, camera): 

100 """Read all data from the standard format at a particular root. 

101 

102 Parameters 

103 ---------- 

104 root : `str` 

105 Path to the top level of the data tree. This is expected to hold directories 

106 named after the sensor names. They are expected to be lower case. 

107 camera : `lsst.afw.cameraGeom.Camera` 

108 The camera that goes with the data being read. 

109 

110 Returns 

111 ------- 

112 dict 

113 A dictionary of dictionaries of objects constructed with the appropriate factory class. 

114 The first key is the sensor name lowered, and the second is the validity 

115 start time as a `datetime` object. 

116 

117 Notes 

118 ----- 

119 Each leaf object in the constructed dictionary has metadata associated with it. 

120 The detector ID may be retrieved from the DETECTOR entry of that metadata. 

121 """ 

122 root = os.path.normpath(root) 

123 dirs = os.listdir(root) # assumes all directories contain data 

124 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))] 

125 data_by_chip = {} 

126 name_map = {det.getName().lower(): det.getName() for 

127 det in camera} # we assume the directories have been lowered 

128 

129 if not dirs: 

130 raise RuntimeError(f"No data found on path {root}") 

131 

132 calib_types = set() 

133 for d in dirs: 

134 chip_name = os.path.basename(d) 

135 # Give informative error message if the detector name is not known 

136 # rather than a simple KeyError 

137 if chip_name not in name_map: 

138 detectors = [det for det in camera.getNameIter()] 

139 max_detectors = 10 

140 note_str = "knows" 

141 if len(detectors) > max_detectors: 

142 # report example subset 

143 note_str = "examples" 

144 detectors = detectors[:max_detectors] 

145 raise RuntimeError(f"Detector {chip_name} not known to supplied camera " 

146 f"{camera.getName()} ({note_str}: {','.join(detectors)})") 

147 chip_id = camera[name_map[chip_name]].getId() 

148 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id) 

149 calib_types.add(calib_type) 

150 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here. 

151 raise ValueError(f'Error mixing calib types: {calib_types}') 

152 

153 no_data = all([v == {} for v in data_by_chip.values()]) 

154 if no_data: 

155 raise RuntimeError("No data to ingest") 

156 

157 return data_by_chip, calib_type