Coverage for python/lsst/pipe/tasks/read_curated_calibs.py: 14%

62 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-20 12:27 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["read_all"] 

23 

24from lsst.meas.algorithms.simple_curve import Curve 

25from lsst.ip.isr import (Linearizer, CrosstalkCalib, Defects, BrighterFatterKernel, PhotodiodeCalib) 

26 

27import os 

28import glob 

29import dateutil.parser 

30from deprecated.sphinx import deprecated 

31 

32 

33def read_one_chip(root, chip_name, chip_id): 

34 """Read data for a particular sensor from the standard format at a particular root. 

35 

36 Parameters 

37 ---------- 

38 root : `str` 

39 Path to the top level of the data tree. This is expected to hold directories 

40 named after the sensor names. They are expected to be lower case. 

41 chip_name : `str` 

42 The name of the sensor for which to read data. 

43 chip_id : `int` 

44 The identifier for the sensor in question. 

45 

46 Returns 

47 ------- 

48 `dict` 

49 A dictionary of objects constructed from the appropriate factory class. 

50 The key is the validity start time as a `datetime` object. 

51 """ 

52 factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer, 

53 'crosstalk': CrosstalkCalib, 'bfk': BrighterFatterKernel, 

54 'photodiode': PhotodiodeCalib, } 

55 files = [] 

56 extensions = (".ecsv", ".yaml") 

57 for ext in extensions: 

58 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}"))) 

59 parts = os.path.split(root) 

60 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name> 

61 data_name = parts[1] 

62 if data_name not in factory_map: 

63 raise ValueError(f"Unknown calibration data type, '{data_name}' found. " 

64 f"Only understand {','.join(k for k in factory_map)}") 

65 factory = factory_map[data_name] 

66 data_dict = {} 

67 for f in files: 

68 date_str = os.path.splitext(os.path.basename(f))[0] 

69 valid_start = dateutil.parser.parse(date_str) 

70 data_dict[valid_start] = factory.readText(f) 

71 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name) 

72 return data_dict, data_name 

73 

74 

75def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name): 

76 """Check that the metadata is complete and self consistent 

77 

78 Parameters 

79 ---------- 

80 obj : object of same type as the factory 

81 Object to retrieve metadata from in order to compare with 

82 metadata inferred from the path. 

83 valid_start : `datetime` 

84 Start of the validity range for data 

85 instrument : `str` 

86 Name of the instrument in question 

87 chip_id : `int` 

88 Identifier of the sensor in question 

89 filepath : `str` 

90 Path of the file read to construct the data 

91 data_name : `str` 

92 Name of the type of data being read 

93 

94 Returns 

95 ------- 

96 None 

97 

98 Raises 

99 ------ 

100 ValueError 

101 If the metadata from the path and the metadata encoded 

102 in the path do not match for any reason. 

103 """ 

104 md = obj.getMetadata() 

105 finst = md['INSTRUME'] 

106 fchip_id = md['DETECTOR'] 

107 fdata_name = md['OBSTYPE'] 

108 if not ((finst.lower(), int(fchip_id), fdata_name.lower()) 

109 == (instrument.lower(), chip_id, data_name.lower())): 

110 raise ValueError(f"Path and file metadata do not agree:\n" 

111 f"Path metadata: {instrument} {chip_id} {data_name}\n" 

112 f"File metadata: {finst} {fchip_id} {fdata_name}\n" 

113 f"File read from : %s\n"%(filepath) 

114 ) 

115 

116 

117@deprecated(reason="Curated calibration ingest now handled by obs_base Instrument classes." 

118 " Will be removed after v25.0.", 

119 version="v25.0", category=FutureWarning) 

120def read_all(root, camera): 

121 """Read all data from the standard format at a particular root. 

122 

123 Parameters 

124 ---------- 

125 root : `str` 

126 Path to the top level of the data tree. This is expected to hold directories 

127 named after the sensor names. They are expected to be lower case. 

128 camera : `lsst.afw.cameraGeom.Camera` 

129 The camera that goes with the data being read. 

130 

131 Returns 

132 ------- 

133 dict 

134 A dictionary of dictionaries of objects constructed with the appropriate factory class. 

135 The first key is the sensor name lowered, and the second is the validity 

136 start time as a `datetime` object. 

137 

138 Notes 

139 ----- 

140 Each leaf object in the constructed dictionary has metadata associated with it. 

141 The detector ID may be retrieved from the DETECTOR entry of that metadata. 

142 """ 

143 root = os.path.normpath(root) 

144 dirs = os.listdir(root) # assumes all directories contain data 

145 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))] 

146 data_by_chip = {} 

147 name_map = {det.getName().lower(): det.getName() for 

148 det in camera} # we assume the directories have been lowered 

149 

150 if not dirs: 

151 raise RuntimeError(f"No data found on path {root}") 

152 

153 calib_types = set() 

154 for d in dirs: 

155 chip_name = os.path.basename(d) 

156 # Give informative error message if the detector name is not known 

157 # rather than a simple KeyError 

158 if chip_name not in name_map: 

159 detectors = [det for det in camera.getNameIter()] 

160 max_detectors = 10 

161 note_str = "knows" 

162 if len(detectors) > max_detectors: 

163 # report example subset 

164 note_str = "examples" 

165 detectors = detectors[:max_detectors] 

166 raise RuntimeError(f"Detector {chip_name} not known to supplied camera " 

167 f"{camera.getName()} ({note_str}: {','.join(detectors)})") 

168 chip_id = camera[name_map[chip_name]].getId() 

169 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id) 

170 calib_types.add(calib_type) 

171 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here. 

172 raise ValueError(f'Error mixing calib types: {calib_types}') 

173 

174 no_data = all([v == {} for v in data_by_chip.values()]) 

175 if no_data: 

176 raise RuntimeError("No data to ingest") 

177 

178 return data_by_chip, calib_type