Coverage for python/lsst/obs/lsst/ingest.py: 35%

113 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:52 -0800

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import logging 

23import re 

24from lsst.pipe.tasks.ingest import ParseTask 

25from lsst.pipe.tasks.ingestCalibs import CalibsParseTask 

26from astro_metadata_translator import ObservationInfo 

27from .translators import LsstCamTranslator 

28from .lsstCamMapper import LsstCamMapper 

29from ._fitsHeader import readRawFitsHeader 

30 

31EXTENSIONS = ["fits", "gz", "fz"] # Filename extensions to strip off 

32 

33__all__ = ["LsstCamParseTask", "LsstCamEimgParseTask"] 

34 

35 

36class LsstCamParseTask(ParseTask): 

37 """Parser suitable for lsstCam data. 

38 

39 See `LCA-13501 <https://ls.st/LCA-13501>`_ and 

40 `LSE-400 <https://ls.st/LSE-400>`_. 

41 """ 

42 

43 _mapperClass = LsstCamMapper 

44 _translatorClass = LsstCamTranslator 

45 

46 def __init__(self, config, *args, **kwargs): 

47 super().__init__(config, *args, **kwargs) 

48 

49 self.observationInfo = None 

50 

51 def getInfo(self, filename): 

52 """Get information about the image from the filename and its contents 

53 

54 Here, we open the image and parse the header. 

55 

56 Parameters 

57 ---------- 

58 filename : `str` 

59 Name of file to inspect 

60 

61 Returns 

62 ------- 

63 info : `dict` 

64 File properties 

65 linfo : `list` of `dict` 

66 List of file properties. Always contains the same as ``info`` 

67 because no extensions are read. 

68 """ 

69 md = readRawFitsHeader(filename, translator_class=self._translatorClass) 

70 phuInfo = self.getInfoFromMetadata(md) 

71 # No extensions to worry about 

72 return phuInfo, [phuInfo] 

73 

74 def getInfoFromMetadata(self, md, info=None): 

75 """Attempt to pull the desired information out of the header. 

76 

77 Parameters 

78 ---------- 

79 md : `lsst.daf.base.PropertyList` 

80 FITS header. 

81 info : `dict`, optional 

82 File properties, to be updated by this routine. If `None` 

83 it will be created. 

84 

85 Returns 

86 ------- 

87 info : `dict` 

88 Translated information from the metadata. Updated form of the 

89 input parameter. 

90 

91 Notes 

92 ----- 

93 

94 This is done through two mechanisms: 

95 

96 * translation: a property is set directly from the relevant header 

97 keyword. 

98 * translator: a property is set with the result of calling a method. 

99 

100 The translator methods receive the header metadata and should return 

101 the appropriate value, or None if the value cannot be determined. 

102 

103 This implementation constructs an 

104 `~astro_metadata_translator.ObservationInfo` object prior to calling 

105 each translator method, making the translated information available 

106 through the ``observationInfo`` attribute. 

107 

108 """ 

109 # Always calculate a new ObservationInfo since getInfo calls 

110 # this method repeatedly for each header. 

111 self.observationInfo = ObservationInfo(md, translator_class=self._translatorClass, 

112 pedantic=False) 

113 

114 info = super().getInfoFromMetadata(md, info) 

115 

116 # Ensure that the translated ObservationInfo is cleared. 

117 # This avoids possible confusion. 

118 self.observationInfo = None 

119 return info 

120 

121 def translate_wavelength(self, md): 

122 """Translate wavelength provided by teststand readout. 

123 

124 The teststand driving script asks for a wavelength, and then reads the 

125 value back to ensure that the correct position was moved to. This 

126 number is therefore read back with sub-nm precision. Typically the 

127 position is within 0.005nm of the desired position, so we warn if it's 

128 not very close to an integer value. 

129 

130 Future users should be aware that the ``HIERARCH MONOCH-WAVELENG`` key 

131 is NOT the requested value, and therefore cannot be used as a 

132 cross-check that the wavelength was close to the one requested. 

133 The only record of the wavelength that was set is in the original 

134 filename. 

135 

136 Parameters 

137 ---------- 

138 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

139 Image metadata. 

140 

141 Returns 

142 ------- 

143 wavelength : `int` 

144 The recorded wavelength in nanometers as an `int`. 

145 """ 

146 bad_wl = -666 # Bad value for wavelength 

147 if "MONOWL" not in md: 

148 return bad_wl 

149 

150 raw_wl = float(md.getScalar("MONOWL")) 

151 

152 # Negative wavelengths are bad so normalize the bad value 

153 if raw_wl < 0: 

154 return bad_wl 

155 

156 wl = int(round(raw_wl)) 

157 if abs(raw_wl-wl) >= 0.1: 

158 logger = logging.getLogger('obs.lsst.ingest') 

159 logger.warning( 

160 'Translated significantly non-integer wavelength; ' 

161 '%s is more than 0.1nm from an integer value', raw_wl) 

162 return wl 

163 

164 def translate_dateObs(self, md): 

165 """Retrieve the date of observation as an ISO format string. 

166 

167 Parameters 

168 ---------- 

169 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

170 Image metadata. 

171 

172 Returns 

173 ------- 

174 dateObs : `str` 

175 The date that the data was taken in FITS ISO format, 

176 e.g. ``2018-08-20T21:56:24.608``. 

177 """ 

178 dateObs = self.observationInfo.datetime_begin 

179 dateObs.format = "isot" 

180 return str(dateObs) 

181 

182 translate_date = translate_dateObs 

183 

184 def translate_dayObs(self, md): 

185 """Generate the day that the observation was taken. 

186 

187 Parameters 

188 ---------- 

189 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

190 image metadata 

191 

192 Returns 

193 ------- 

194 dayObs : `str` 

195 The day that the data was taken, e.g. ``1958-02-05``. 

196 """ 

197 dayObs = str(self.observationInfo.observing_day) 

198 return "-".join([dayObs[:4], dayObs[4:6], dayObs[6:]]) 

199 

200 def translate_snap(self, md): 

201 """Extract snap number from metadata. 

202 

203 Parameters 

204 ---------- 

205 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

206 Image metadata. 

207 

208 Returns 

209 ------- 

210 snap : `int` 

211 Snap number (default: 0). 

212 """ 

213 try: 

214 return int(md.getScalar("SNAP")) 

215 except KeyError: 

216 return 0 

217 

218 def translate_detectorName(self, md): 

219 """Extract ccd ID from CHIPID. 

220 

221 Parameters 

222 ---------- 

223 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

224 Image metadata. 

225 

226 Returns 

227 ------- 

228 ccdID : `str` 

229 Name of ccd, e.g. ``S01``. 

230 """ 

231 return self.observationInfo.detector_name 

232 

233 def translate_raftName(self, md): 

234 """Extract raft ID from CHIPID. 

235 

236 Parameters 

237 ---------- 

238 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

239 Image metadata. 

240 

241 Returns 

242 ------- 

243 raftID : `str` 

244 Name of raft, e.g. ``R21``. 

245 """ 

246 return self.observationInfo.detector_group 

247 

248 def translate_detector(self, md): 

249 """Extract detector ID from metadata 

250 

251 Parameters 

252 ---------- 

253 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

254 Image metadata. 

255 

256 Returns 

257 ------- 

258 detID : `int` 

259 Detector ID, e.g. ``4``. 

260 """ 

261 return self.observationInfo.detector_num 

262 

263 def translate_expTime(self, md): 

264 return self.observationInfo.exposure_time.value 

265 

266 def translate_object(self, md): 

267 return self.observationInfo.object 

268 

269 def translate_imageType(self, md): 

270 obstype = self.observationInfo.observation_type.upper() 

271 # Dictionary for obstype values is not yet clear 

272 if obstype == "SCIENCE": 

273 obstype = "SKYEXP" 

274 return obstype 

275 

276 def translate_filter(self, md): 

277 return self.observationInfo.physical_filter 

278 

279 def translate_lsstSerial(self, md): 

280 return self.observationInfo.detector_serial 

281 

282 def translate_run(self, md): 

283 return self.observationInfo.science_program 

284 

285 def translate_visit(self, md): 

286 return self.observationInfo.visit_id 

287 

288 def translate_obsid(self, md): 

289 return self.observationInfo.observation_id 

290 

291 def translate_testType(self, md): 

292 # Gen2 prefers upper case 

293 return self.observationInfo.observation_reason.upper() 

294 

295 def translate_expGroup(self, md): 

296 return self.observationInfo.exposure_group 

297 

298 def translate_expId(self, md): 

299 return self.observationInfo.exposure_id 

300 

301 def translate_controller(self, md): 

302 if "CONTRLLR" in md: 

303 if md["CONTRLLR"]: 

304 return md["CONTRLLR"] 

305 else: 

306 # Was undefined, sometimes it is in fact in the OBSID 

307 obsid = self.translate_obsid(md) 

308 components = obsid.split("_") 

309 if len(components) >= 2 and len(components[1]) == 1: 

310 # AT_C_20190319_00001 

311 return components[1] 

312 # Assume OCS control 

313 return "O" 

314 else: 

315 # Assume it is under camera control 

316 return "C" 

317 

318 

319class LsstCamCalibsParseTask(CalibsParseTask): 

320 """Parser for calibs.""" 

321 

322 def _translateFromCalibId(self, field, md): 

323 """Get a value from the CALIB_ID written by ``constructCalibs``.""" 

324 data = md.getScalar("CALIB_ID") 

325 match = re.search(r".*%s=(\S+)" % field, data) 

326 return match.groups()[0] 

327 

328 def translate_raftName(self, md): 

329 return self._translateFromCalibId("raftName", md) 

330 

331 def translate_detectorName(self, md): 

332 return self._translateFromCalibId("detectorName", md) 

333 

334 def translate_detector(self, md): 

335 # this is not a _great_ fix, but this obs_package is enforcing that 

336 # detectors be integers and there's not an elegant way of ensuring 

337 # this is the right type really 

338 return int(self._translateFromCalibId("detector", md)) 

339 

340 def translate_filter(self, md): 

341 return self._translateFromCalibId("filter", md) 

342 

343 def translate_calibDate(self, md): 

344 return self._translateFromCalibId("calibDate", md) 

345 

346 

347class LsstCamEimgParseTask(LsstCamParseTask): 

348 """Parser suitable for phosim LsstCam eimage data. 

349 """ 

350 

351 def getDestination(self, butler, info, filename): 

352 """Get destination for the file. 

353 

354 Parameters 

355 ---------- 

356 butler : `lsst.daf.persistence.Butler` 

357 Data butler 

358 info : data ID 

359 File properties, used as dataId for the butler. 

360 filename : `str` 

361 Input filename. 

362 

363 Returns 

364 ------- 

365 `str` 

366 Destination filename. 

367 """ 

368 

369 eimage = butler.get("eimage_filename", info)[0] 

370 # Ensure filename is devoid of cfitsio directions about HDUs 

371 c = eimage.find("[") 

372 if c > 0: 

373 eimage = eimage[:c] 

374 return eimage