Coverage for python/lsst/obs/lsst/ingest.py : 33%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_lsst.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import re
23from lsst.pipe.tasks.ingest import ParseTask
24from lsst.pipe.tasks.ingestCalibs import CalibsParseTask
25from astro_metadata_translator import ObservationInfo
26import lsst.log as lsstLog
27from .translators.lsst import ROLLOVERTIME
28from .translators import LsstCamTranslator
29from .lsstCamMapper import LsstCamMapper
31EXTENSIONS = ["fits", "gz", "fz"] # Filename extensions to strip off
33__all__ = ["LsstCamParseTask"]
36class LsstCamParseTask(ParseTask):
37 """Parser suitable for lsstCam data.
39 See `LCA-13501 <https://ls.st/LCA-13501>`_ and
40 `LSE-400 <https://ls.st/LSE-400>`_.
41 """
43 _mapperClass = LsstCamMapper
44 _translatorClass = LsstCamTranslator
46 def __init__(self, config, *args, **kwargs):
47 super().__init__(config, *args, **kwargs)
49 self.observationInfo = None
51 def getInfoFromMetadata(self, md, info=None):
52 """Attempt to pull the desired information out of the header.
54 Parameters
55 ----------
56 md : `lsst.daf.base.PropertyList`
57 FITS header.
58 info : `dict`, optional
59 File properties, to be updated by this routine. If `None`
60 it will be created.
62 Returns
63 -------
64 info : `dict`
65 Translated information from the metadata. Updated form of the
66 input parameter.
68 Notes
69 -----
71 This is done through two mechanisms:
73 * translation: a property is set directly from the relevant header
74 keyword.
75 * translator: a property is set with the result of calling a method.
77 The translator methods receive the header metadata and should return
78 the appropriate value, or None if the value cannot be determined.
80 This implementation constructs an
81 `~astro_metadata_translator.ObservationInfo` object prior to calling
82 each translator method, making the translated information available
83 through the ``observationInfo`` attribute.
85 """
86 # Always calculate a new ObservationInfo since getInfo calls
87 # this method repeatedly for each header.
88 self.observationInfo = ObservationInfo(md, translator_class=self._translatorClass,
89 pedantic=False)
91 info = super().getInfoFromMetadata(md, info)
93 # Ensure that the translated ObservationInfo is cleared.
94 # This avoids possible confusion.
95 self.observationInfo = None
96 return info
98 def translate_wavelength(self, md):
99 """Translate wavelength provided by teststand readout.
101 The teststand driving script asks for a wavelength, and then reads the
102 value back to ensure that the correct position was moved to. This
103 number is therefore read back with sub-nm precision. Typically the
104 position is within 0.005nm of the desired position, so we warn if it's
105 not very close to an integer value.
107 Future users should be aware that the ``HIERARCH MONOCH-WAVELENG`` key
108 is NOT the requested value, and therefore cannot be used as a
109 cross-check that the wavelength was close to the one requested.
110 The only record of the wavelength that was set is in the original
111 filename.
113 Parameters
114 ----------
115 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
116 Image metadata.
118 Returns
119 -------
120 wavelength : `int`
121 The recorded wavelength in nanometers as an `int`.
122 """
123 bad_wl = -666 # Bad value for wavelength
124 if "MONOWL" not in md:
125 return bad_wl
127 raw_wl = float(md.getScalar("MONOWL"))
129 # Negative wavelengths are bad so normalize the bad value
130 if raw_wl < 0:
131 return bad_wl
133 wl = int(round(raw_wl))
134 if abs(raw_wl-wl) >= 0.1:
135 logger = lsstLog.Log.getLogger('obs.lsst.ingest')
136 logger.warn(
137 'Translated significantly non-integer wavelength; '
138 '%s is more than 0.1nm from an integer value', raw_wl)
139 return wl
141 def translate_dateObs(self, md):
142 """Retrieve the date of observation as an ISO format string.
144 Parameters
145 ----------
146 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
147 Image metadata.
149 Returns
150 -------
151 dateObs : `str`
152 The date that the data was taken in FITS ISO format,
153 e.g. ``2018-08-20T21:56:24.608``.
154 """
155 dateObs = self.observationInfo.datetime_begin
156 dateObs.format = "isot"
157 return str(dateObs)
159 translate_date = translate_dateObs
161 def translate_dayObs(self, md):
162 """Generate the day that the observation was taken.
164 Parameters
165 ----------
166 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
167 image metadata
169 Returns
170 -------
171 dayObs : `str`
172 The day that the data was taken, e.g. ``1958-02-05``.
173 """
174 # Trust DAYOBS if it is there
175 if "DAYOBS" in md:
176 dayObs = str(md.getScalar("DAYOBS"))
178 if re.match(r"^\d{8}$", dayObs):
179 dateObs = f"{dayObs[:4]}-{dayObs[4:6]}-{dayObs[6:8]}"
180 return dateObs
182 # Try to work it out from date of observation
183 dateObs = self.observationInfo.datetime_begin
184 dateObs -= ROLLOVERTIME
185 dateObs.format = "iso"
186 dateObs.out_subfmt = "date" # YYYY-MM-DD format
187 return str(dateObs)
189 def translate_snap(self, md):
190 """Extract snap number from metadata.
192 Parameters
193 ----------
194 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
195 Image metadata.
197 Returns
198 -------
199 snap : `int`
200 Snap number (default: 0).
201 """
202 try:
203 return int(md.getScalar("SNAP"))
204 except KeyError:
205 return 0
207 def translate_detectorName(self, md):
208 """Extract ccd ID from CHIPID.
210 Parameters
211 ----------
212 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
213 Image metadata.
215 Returns
216 -------
217 ccdID : `str`
218 Name of ccd, e.g. ``S01``.
219 """
220 return self.observationInfo.detector_name
222 def translate_raftName(self, md):
223 """Extract raft ID from CHIPID.
225 Parameters
226 ----------
227 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
228 Image metadata.
230 Returns
231 -------
232 raftID : `str`
233 Name of raft, e.g. ``R21``.
234 """
235 return self.observationInfo.detector_group
237 def translate_detector(self, md):
238 """Extract detector ID from metadata
240 Parameters
241 ----------
242 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
243 Image metadata.
245 Returns
246 -------
247 detID : `int`
248 Detector ID, e.g. ``4``.
249 """
250 return self.observationInfo.detector_num
252 def translate_expTime(self, md):
253 return self.observationInfo.exposure_time.value
255 def translate_object(self, md):
256 return self.observationInfo.object
258 def translate_imageType(self, md):
259 obstype = self.observationInfo.observation_type.upper()
260 # Dictionary for obstype values is not yet clear
261 if obstype == "SCIENCE":
262 obstype = "SKYEXP"
263 return obstype
265 def translate_filter(self, md):
266 return self.observationInfo.physical_filter
268 def translate_lsstSerial(self, md):
269 return self.observationInfo.detector_serial
271 def translate_run(self, md):
272 return self.observationInfo.science_program
274 def translate_visit(self, md):
275 return self.observationInfo.visit_id
277 def translate_obsid(self, md):
278 return self.observationInfo.observation_id
280 def translate_expGroup(self, md):
281 return self.observationInfo.exposure_group
283 def translate_expId(self, md):
284 return self.observationInfo.exposure_id
286 def translate_controller(self, md):
287 if "CONTRLLR" in md:
288 if md["CONTRLLR"]:
289 return md["CONTRLLR"]
290 else:
291 # Was undefined, sometimes it is in fact in the OBSID
292 obsid = self.translate_obsid(md)
293 components = obsid.split("_")
294 if len(components) >= 2 and len(components[1]) == 1:
295 # AT_C_20190319_00001
296 return components[1]
297 # Assume OCS control
298 return "O"
299 else:
300 # Assume it is under camera control
301 return "C"
304class LsstCamCalibsParseTask(CalibsParseTask):
305 """Parser for calibs."""
307 def _translateFromCalibId(self, field, md):
308 """Get a value from the CALIB_ID written by ``constructCalibs``."""
309 data = md.getScalar("CALIB_ID")
310 match = re.search(r".*%s=(\S+)" % field, data)
311 return match.groups()[0]
313 def translate_raftName(self, md):
314 return self._translateFromCalibId("raftName", md)
316 def translate_detectorName(self, md):
317 return self._translateFromCalibId("detectorName", md)
319 def translate_detector(self, md):
320 # this is not a _great_ fix, but this obs_package is enforcing that
321 # detectors be integers and there's not an elegant way of ensuring
322 # this is the right type really
323 return int(self._translateFromCalibId("detector", md))
325 def translate_filter(self, md):
326 return self._translateFromCalibId("filter", md)
328 def translate_calibDate(self, md):
329 return self._translateFromCalibId("calibDate", md)