Coverage for python/lsst/obs/base/_read_curated_calibs.py: 17%

66 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-01 01:59 -0700

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["CuratedCalibration", "read_all"] 

25 

26import glob 

27import os 

28from collections.abc import Mapping 

29from typing import TYPE_CHECKING, Any, Protocol, Type 

30 

31import dateutil.parser 

32 

33if TYPE_CHECKING: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true

34 import datetime 

35 

36 import lsst.afw.cameraGeom 

37 

38 

39class CuratedCalibration(Protocol): 

40 """Protocol that describes the methods needed by this class when dealing 

41 with curated calibration datasets.""" 

42 

43 @classmethod 

44 def readText(cls, path: str) -> CuratedCalibration: 

45 ... 

46 

47 def getMetadata(self) -> Mapping: 

48 ... 

49 

50 

51def read_one_chip( 

52 root: str, 

53 chip_name: str, 

54 chip_id: int, 

55 calib_class: Type[CuratedCalibration], 

56) -> tuple[dict[datetime.datetime, CuratedCalibration], str]: 

57 """Read data for a particular sensor from the standard format at a 

58 particular root. 

59 

60 Parameters 

61 ---------- 

62 root : `str` 

63 Path to the top level of the data tree. This is expected to hold 

64 directories named after the sensor names. They are expected to be 

65 lower case. 

66 chip_name : `str` 

67 The name of the sensor for which to read data. 

68 chip_id : `int` 

69 The identifier for the sensor in question. 

70 calib_class : `Any` 

71 The class to use to read the curated calibration text file. Must 

72 support the ``readText()`` method. 

73 

74 Returns 

75 ------- 

76 `dict` 

77 A dictionary of objects constructed from the appropriate factory class. 

78 The key is the validity start time as a `datetime` object. 

79 """ 

80 files = [] 

81 extensions = (".ecsv", ".yaml", ".json") 

82 for ext in extensions: 

83 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}"))) 

84 parts = os.path.split(root) 

85 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name> 

86 data_name = parts[1] 

87 data_dict: dict[datetime.datetime, Any] = {} 

88 for f in files: 

89 date_str = os.path.splitext(os.path.basename(f))[0] 

90 valid_start = dateutil.parser.parse(date_str) 

91 data_dict[valid_start] = calib_class.readText(f) 

92 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name) 

93 return data_dict, data_name 

94 

95 

96def check_metadata( 

97 obj: Any, valid_start: datetime.datetime, instrument: str, chip_id: int, filepath: str, data_name: str 

98) -> None: 

99 """Check that the metadata is complete and self consistent 

100 

101 Parameters 

102 ---------- 

103 obj : object of same type as the factory 

104 Object to retrieve metadata from in order to compare with 

105 metadata inferred from the path. 

106 valid_start : `datetime` 

107 Start of the validity range for data 

108 instrument : `str` 

109 Name of the instrument in question 

110 chip_id : `int` 

111 Identifier of the sensor in question 

112 filepath : `str` 

113 Path of the file read to construct the data 

114 data_name : `str` 

115 Name of the type of data being read 

116 

117 Returns 

118 ------- 

119 None 

120 

121 Raises 

122 ------ 

123 ValueError 

124 If the metadata from the path and the metadata encoded 

125 in the path do not match for any reason. 

126 """ 

127 md = obj.getMetadata() 

128 finst = md["INSTRUME"] 

129 fchip_id = md["DETECTOR"] 

130 fdata_name = md["OBSTYPE"] 

131 if not ( 

132 (finst.lower(), int(fchip_id), fdata_name.lower()) == (instrument.lower(), chip_id, data_name.lower()) 

133 ): 

134 raise ValueError( 

135 f"Path and file metadata do not agree:\n" 

136 f"Path metadata: {instrument} {chip_id} {data_name}\n" 

137 f"File metadata: {finst} {fchip_id} {fdata_name}\n" 

138 f"File read from : %s\n" % (filepath) 

139 ) 

140 

141 

142def read_all( 

143 root: str, 

144 camera: lsst.afw.cameraGeom.Camera, 

145 calib_class: Type[CuratedCalibration], 

146) -> tuple[dict[str, dict[datetime.datetime, CuratedCalibration]], str]: 

147 """Read all data from the standard format at a particular root. 

148 

149 Parameters 

150 ---------- 

151 root : `str` 

152 Path to the top level of the data tree. This is expected to hold 

153 directories named after the sensor names. They are expected to be 

154 lower case. 

155 camera : `lsst.afw.cameraGeom.Camera` 

156 The camera that goes with the data being read. 

157 calib_class : `Any` 

158 The class to use to read the curated calibration text file. Must 

159 support the ``readText()`` and ``getMetadata()`` methods. 

160 

161 Returns 

162 ------- 

163 dict 

164 A dictionary of dictionaries of objects constructed with the 

165 appropriate factory class. The first key is the sensor name lowered, 

166 and the second is the validity start time as a `datetime` object. 

167 

168 Notes 

169 ----- 

170 Each leaf object in the constructed dictionary has metadata associated with 

171 it. The detector ID may be retrieved from the DETECTOR entry of that 

172 metadata. 

173 """ 

174 root = os.path.normpath(root) 

175 dirs = os.listdir(root) # assumes all directories contain data 

176 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))] 

177 data_by_chip = {} 

178 name_map = { 

179 det.getName().lower(): det.getName() for det in camera 

180 } # we assume the directories have been lowered 

181 

182 if not dirs: 

183 raise RuntimeError(f"No data found on path {root}") 

184 

185 calib_types = set() 

186 for d in dirs: 

187 chip_name = os.path.basename(d) 

188 # Give informative error message if the detector name is not known 

189 # rather than a simple KeyError 

190 if chip_name not in name_map: 

191 detectors = [det for det in name_map.keys()] 

192 max_detectors = 10 

193 note_str = "knows" 

194 if len(detectors) > max_detectors: 

195 # report example subset 

196 note_str = "examples" 

197 detectors = detectors[:max_detectors] 

198 raise RuntimeError( 

199 f"Detector {chip_name} not known to supplied camera " 

200 f"{camera.getName()} ({note_str}: {','.join(detectors)})" 

201 ) 

202 chip_id = camera[name_map[chip_name]].getId() 

203 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id, calib_class) 

204 calib_types.add(calib_type) 

205 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here. 

206 raise ValueError(f"Error mixing calib types: {calib_types}") 

207 

208 no_data = all([v == {} for v in data_by_chip.values()]) 

209 if no_data: 

210 raise RuntimeError("No data to ingest") 

211 

212 return data_by_chip, calib_type