Coverage for python/lsst/pipe/tasks/read_curated_calibs.py: 9%

59 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-03 01:38 -0700

1from lsst.meas.algorithms.simple_curve import Curve 

2from lsst.ip.isr import (Linearizer, CrosstalkCalib, Defects, BrighterFatterKernel, PhotodiodeCalib) 

3 

4import os 

5import glob 

6import dateutil.parser 

7 

8 

9def read_one_chip(root, chip_name, chip_id): 

10 """Read data for a particular sensor from the standard format at a particular root. 

11 

12 Parameters 

13 ---------- 

14 root : `str` 

15 Path to the top level of the data tree. This is expected to hold directories 

16 named after the sensor names. They are expected to be lower case. 

17 chip_name : `str` 

18 The name of the sensor for which to read data. 

19 chip_id : `int` 

20 The identifier for the sensor in question. 

21 

22 Returns 

23 ------- 

24 `dict` 

25 A dictionary of objects constructed from the appropriate factory class. 

26 The key is the validity start time as a `datetime` object. 

27 """ 

28 factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer, 

29 'crosstalk': CrosstalkCalib, 'bfk': BrighterFatterKernel, 

30 'photodiode': PhotodiodeCalib, } 

31 files = [] 

32 extensions = (".ecsv", ".yaml") 

33 for ext in extensions: 

34 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}"))) 

35 parts = os.path.split(root) 

36 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name> 

37 data_name = parts[1] 

38 if data_name not in factory_map: 

39 raise ValueError(f"Unknown calibration data type, '{data_name}' found. " 

40 f"Only understand {','.join(k for k in factory_map)}") 

41 factory = factory_map[data_name] 

42 data_dict = {} 

43 for f in files: 

44 date_str = os.path.splitext(os.path.basename(f))[0] 

45 valid_start = dateutil.parser.parse(date_str) 

46 data_dict[valid_start] = factory.readText(f) 

47 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name) 

48 return data_dict, data_name 

49 

50 

51def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name): 

52 """Check that the metadata is complete and self consistent 

53 

54 Parameters 

55 ---------- 

56 obj : object of same type as the factory 

57 Object to retrieve metadata from in order to compare with 

58 metadata inferred from the path. 

59 valid_start : `datetime` 

60 Start of the validity range for data 

61 instrument : `str` 

62 Name of the instrument in question 

63 chip_id : `int` 

64 Identifier of the sensor in question 

65 filepath : `str` 

66 Path of the file read to construct the data 

67 data_name : `str` 

68 Name of the type of data being read 

69 

70 Returns 

71 ------- 

72 None 

73 

74 Raises 

75 ------ 

76 ValueError 

77 If the metadata from the path and the metadata encoded 

78 in the path do not match for any reason. 

79 """ 

80 md = obj.getMetadata() 

81 finst = md['INSTRUME'] 

82 fchip_id = md['DETECTOR'] 

83 fdata_name = md['OBSTYPE'] 

84 if not ((finst.lower(), int(fchip_id), fdata_name.lower()) 

85 == (instrument.lower(), chip_id, data_name.lower())): 

86 raise ValueError(f"Path and file metadata do not agree:\n" 

87 f"Path metadata: {instrument} {chip_id} {data_name}\n" 

88 f"File metadata: {finst} {fchip_id} {fdata_name}\n" 

89 f"File read from : %s\n"%(filepath) 

90 ) 

91 

92 

93def read_all(root, camera): 

94 """Read all data from the standard format at a particular root. 

95 

96 Parameters 

97 ---------- 

98 root : `str` 

99 Path to the top level of the data tree. This is expected to hold directories 

100 named after the sensor names. They are expected to be lower case. 

101 camera : `lsst.afw.cameraGeom.Camera` 

102 The camera that goes with the data being read. 

103 

104 Returns 

105 ------- 

106 dict 

107 A dictionary of dictionaries of objects constructed with the appropriate factory class. 

108 The first key is the sensor name lowered, and the second is the validity 

109 start time as a `datetime` object. 

110 

111 Notes 

112 ----- 

113 Each leaf object in the constructed dictionary has metadata associated with it. 

114 The detector ID may be retrieved from the DETECTOR entry of that metadata. 

115 """ 

116 root = os.path.normpath(root) 

117 dirs = os.listdir(root) # assumes all directories contain data 

118 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))] 

119 data_by_chip = {} 

120 name_map = {det.getName().lower(): det.getName() for 

121 det in camera} # we assume the directories have been lowered 

122 

123 if not dirs: 

124 raise RuntimeError(f"No data found on path {root}") 

125 

126 calib_types = set() 

127 for d in dirs: 

128 chip_name = os.path.basename(d) 

129 # Give informative error message if the detector name is not known 

130 # rather than a simple KeyError 

131 if chip_name not in name_map: 

132 detectors = [det for det in camera.getNameIter()] 

133 max_detectors = 10 

134 note_str = "knows" 

135 if len(detectors) > max_detectors: 

136 # report example subset 

137 note_str = "examples" 

138 detectors = detectors[:max_detectors] 

139 raise RuntimeError(f"Detector {chip_name} not known to supplied camera " 

140 f"{camera.getName()} ({note_str}: {','.join(detectors)})") 

141 chip_id = camera[name_map[chip_name]].getId() 

142 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id) 

143 calib_types.add(calib_type) 

144 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here. 

145 raise ValueError(f'Error mixing calib types: {calib_types}') 

146 

147 no_data = all([v == {} for v in data_by_chip.values()]) 

148 if no_data: 

149 raise RuntimeError("No data to ingest") 

150 

151 return data_by_chip, calib_type