Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import re 

23from lsst.pipe.tasks.ingest import ParseTask 

24from lsst.pipe.tasks.ingestCalibs import CalibsParseTask 

25from astro_metadata_translator import ObservationInfo 

26import lsst.log as lsstLog 

27from .translators.lsst import ROLLOVERTIME 

28from .translators import LsstCamTranslator 

29from .lsstCamMapper import LsstCamMapper 

30 

31EXTENSIONS = ["fits", "gz", "fz"] # Filename extensions to strip off 

32 

33__all__ = ["LsstCamParseTask"] 

34 

35 

36class LsstCamParseTask(ParseTask): 

37 """Parser suitable for lsstCam data. 

38 

39 See `LCA-13501 <https://ls.st/LCA-13501>`_ and 

40 `LSE-400 <https://ls.st/LSE-400>`_. 

41 """ 

42 

43 _mapperClass = LsstCamMapper 

44 _translatorClass = LsstCamTranslator 

45 

46 def __init__(self, config, *args, **kwargs): 

47 super().__init__(config, *args, **kwargs) 

48 

49 self.observationInfo = None 

50 

51 def getInfoFromMetadata(self, md, info=None): 

52 """Attempt to pull the desired information out of the header. 

53 

54 Parameters 

55 ---------- 

56 md : `lsst.daf.base.PropertyList` 

57 FITS header. 

58 info : `dict`, optional 

59 File properties, to be updated by this routine. If `None` 

60 it will be created. 

61 

62 Returns 

63 ------- 

64 info : `dict` 

65 Translated information from the metadata. Updated form of the 

66 input parameter. 

67 

68 Notes 

69 ----- 

70 

71 This is done through two mechanisms: 

72 

73 * translation: a property is set directly from the relevant header 

74 keyword. 

75 * translator: a property is set with the result of calling a method. 

76 

77 The translator methods receive the header metadata and should return 

78 the appropriate value, or None if the value cannot be determined. 

79 

80 This implementation constructs an 

81 `~astro_metadata_translator.ObservationInfo` object prior to calling 

82 each translator method, making the translated information available 

83 through the ``observationInfo`` attribute. 

84 

85 """ 

86 # Always calculate a new ObservationInfo since getInfo calls 

87 # this method repeatedly for each header. 

88 self.observationInfo = ObservationInfo(md, translator_class=self._translatorClass, 

89 pedantic=False) 

90 

91 info = super().getInfoFromMetadata(md, info) 

92 

93 # Ensure that the translated ObservationInfo is cleared. 

94 # This avoids possible confusion. 

95 self.observationInfo = None 

96 return info 

97 

98 def translate_wavelength(self, md): 

99 """Translate wavelength provided by teststand readout. 

100 

101 The teststand driving script asks for a wavelength, and then reads the 

102 value back to ensure that the correct position was moved to. This 

103 number is therefore read back with sub-nm precision. Typically the 

104 position is within 0.005nm of the desired position, so we warn if it's 

105 not very close to an integer value. 

106 

107 Future users should be aware that the ``HIERARCH MONOCH-WAVELENG`` key 

108 is NOT the requested value, and therefore cannot be used as a 

109 cross-check that the wavelength was close to the one requested. 

110 The only record of the wavelength that was set is in the original 

111 filename. 

112 

113 Parameters 

114 ---------- 

115 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

116 Image metadata. 

117 

118 Returns 

119 ------- 

120 wavelength : `int` 

121 The recorded wavelength in nanometers as an `int`. 

122 """ 

123 bad_wl = -666 # Bad value for wavelength 

124 if "MONOWL" not in md: 

125 return bad_wl 

126 

127 raw_wl = float(md.getScalar("MONOWL")) 

128 

129 # Negative wavelengths are bad so normalize the bad value 

130 if raw_wl < 0: 

131 return bad_wl 

132 

133 wl = int(round(raw_wl)) 

134 if abs(raw_wl-wl) >= 0.1: 

135 logger = lsstLog.Log.getLogger('obs.lsst.ingest') 

136 logger.warn( 

137 'Translated significantly non-integer wavelength; ' 

138 '%s is more than 0.1nm from an integer value', raw_wl) 

139 return wl 

140 

141 def translate_dateObs(self, md): 

142 """Retrieve the date of observation as an ISO format string. 

143 

144 Parameters 

145 ---------- 

146 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

147 Image metadata. 

148 

149 Returns 

150 ------- 

151 dateObs : `str` 

152 The date that the data was taken in FITS ISO format, 

153 e.g. ``2018-08-20T21:56:24.608``. 

154 """ 

155 dateObs = self.observationInfo.datetime_begin 

156 dateObs.format = "isot" 

157 return str(dateObs) 

158 

159 translate_date = translate_dateObs 

160 

161 def translate_dayObs(self, md): 

162 """Generate the day that the observation was taken. 

163 

164 Parameters 

165 ---------- 

166 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

167 image metadata 

168 

169 Returns 

170 ------- 

171 dayObs : `str` 

172 The day that the data was taken, e.g. ``1958-02-05``. 

173 """ 

174 # Trust DAYOBS if it is there 

175 if "DAYOBS" in md: 

176 dayObs = str(md.getScalar("DAYOBS")) 

177 

178 if re.match(r"^\d{8}$", dayObs): 

179 dateObs = f"{dayObs[:4]}-{dayObs[4:6]}-{dayObs[6:8]}" 

180 return dateObs 

181 

182 # Try to work it out from date of observation 

183 dateObs = self.observationInfo.datetime_begin 

184 dateObs -= ROLLOVERTIME 

185 dateObs.format = "iso" 

186 dateObs.out_subfmt = "date" # YYYY-MM-DD format 

187 return str(dateObs) 

188 

189 def translate_snap(self, md): 

190 """Extract snap number from metadata. 

191 

192 Parameters 

193 ---------- 

194 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

195 Image metadata. 

196 

197 Returns 

198 ------- 

199 snap : `int` 

200 Snap number (default: 0). 

201 """ 

202 try: 

203 return int(md.getScalar("SNAP")) 

204 except KeyError: 

205 return 0 

206 

207 def translate_detectorName(self, md): 

208 """Extract ccd ID from CHIPID. 

209 

210 Parameters 

211 ---------- 

212 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

213 Image metadata. 

214 

215 Returns 

216 ------- 

217 ccdID : `str` 

218 Name of ccd, e.g. ``S01``. 

219 """ 

220 return self.observationInfo.detector_name 

221 

222 def translate_raftName(self, md): 

223 """Extract raft ID from CHIPID. 

224 

225 Parameters 

226 ---------- 

227 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

228 Image metadata. 

229 

230 Returns 

231 ------- 

232 raftID : `str` 

233 Name of raft, e.g. ``R21``. 

234 """ 

235 return self.observationInfo.detector_group 

236 

237 def translate_detector(self, md): 

238 """Extract detector ID from metadata 

239 

240 Parameters 

241 ---------- 

242 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet` 

243 Image metadata. 

244 

245 Returns 

246 ------- 

247 detID : `int` 

248 Detector ID, e.g. ``4``. 

249 """ 

250 return self.observationInfo.detector_num 

251 

252 def translate_expTime(self, md): 

253 return self.observationInfo.exposure_time.value 

254 

255 def translate_object(self, md): 

256 return self.observationInfo.object 

257 

258 def translate_imageType(self, md): 

259 obstype = self.observationInfo.observation_type.upper() 

260 # Dictionary for obstype values is not yet clear 

261 if obstype == "SCIENCE": 

262 obstype = "SKYEXP" 

263 return obstype 

264 

265 def translate_filter(self, md): 

266 return self.observationInfo.physical_filter 

267 

268 def translate_lsstSerial(self, md): 

269 return self.observationInfo.detector_serial 

270 

271 def translate_run(self, md): 

272 return self.observationInfo.science_program 

273 

274 def translate_visit(self, md): 

275 return self.observationInfo.visit_id 

276 

277 def translate_obsid(self, md): 

278 return self.observationInfo.observation_id 

279 

280 def translate_expGroup(self, md): 

281 return self.observationInfo.exposure_group 

282 

283 def translate_expId(self, md): 

284 return self.observationInfo.exposure_id 

285 

286 def translate_controller(self, md): 

287 if "CONTRLLR" in md: 

288 if md["CONTRLLR"]: 

289 return md["CONTRLLR"] 

290 else: 

291 # Was undefined, sometimes it is in fact in the OBSID 

292 obsid = self.translate_obsid(md) 

293 components = obsid.split("_") 

294 if len(components) >= 2 and len(components[1]) == 1: 

295 # AT_C_20190319_00001 

296 return components[1] 

297 # Assume OCS control 

298 return "O" 

299 else: 

300 # Assume it is under camera control 

301 return "C" 

302 

303 

304class LsstCamCalibsParseTask(CalibsParseTask): 

305 """Parser for calibs.""" 

306 

307 def _translateFromCalibId(self, field, md): 

308 """Get a value from the CALIB_ID written by ``constructCalibs``.""" 

309 data = md.getScalar("CALIB_ID") 

310 match = re.search(r".*%s=(\S+)" % field, data) 

311 return match.groups()[0] 

312 

313 def translate_raftName(self, md): 

314 return self._translateFromCalibId("raftName", md) 

315 

316 def translate_detectorName(self, md): 

317 return self._translateFromCalibId("detectorName", md) 

318 

319 def translate_detector(self, md): 

320 # this is not a _great_ fix, but this obs_package is enforcing that 

321 # detectors be integers and there's not an elegant way of ensuring 

322 # this is the right type really 

323 return int(self._translateFromCalibId("detector", md)) 

324 

325 def translate_filter(self, md): 

326 return self._translateFromCalibId("filter", md) 

327 

328 def translate_calibDate(self, md): 

329 return self._translateFromCalibId("calibDate", md)