Coverage for python/lsst/obs/lsst/ingest.py: 38%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_lsst.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23import re
24from lsst.pipe.tasks.ingest import ParseTask
25from lsst.pipe.tasks.ingestCalibs import CalibsParseTask
26from astro_metadata_translator import ObservationInfo
27from .translators import LsstCamTranslator
28from .lsstCamMapper import LsstCamMapper
29from ._fitsHeader import readRawFitsHeader
31EXTENSIONS = ["fits", "gz", "fz"] # Filename extensions to strip off
33__all__ = ["LsstCamParseTask", "LsstCamEimgParseTask"]
36class LsstCamParseTask(ParseTask):
37 """Parser suitable for lsstCam data.
39 See `LCA-13501 <https://ls.st/LCA-13501>`_ and
40 `LSE-400 <https://ls.st/LSE-400>`_.
41 """
43 _mapperClass = LsstCamMapper
44 _translatorClass = LsstCamTranslator
46 def __init__(self, config, *args, **kwargs):
47 super().__init__(config, *args, **kwargs)
49 self.observationInfo = None
51 def getInfo(self, filename):
52 """Get information about the image from the filename and its contents
54 Here, we open the image and parse the header.
56 Parameters
57 ----------
58 filename : `str`
59 Name of file to inspect
61 Returns
62 -------
63 info : `dict`
64 File properties
65 linfo : `list` of `dict`
66 List of file properties. Always contains the same as ``info``
67 because no extensions are read.
68 """
69 md = readRawFitsHeader(filename, translator_class=self._translatorClass)
70 phuInfo = self.getInfoFromMetadata(md)
71 # No extensions to worry about
72 return phuInfo, [phuInfo]
74 def getInfoFromMetadata(self, md, info=None):
75 """Attempt to pull the desired information out of the header.
77 Parameters
78 ----------
79 md : `lsst.daf.base.PropertyList`
80 FITS header.
81 info : `dict`, optional
82 File properties, to be updated by this routine. If `None`
83 it will be created.
85 Returns
86 -------
87 info : `dict`
88 Translated information from the metadata. Updated form of the
89 input parameter.
91 Notes
92 -----
94 This is done through two mechanisms:
96 * translation: a property is set directly from the relevant header
97 keyword.
98 * translator: a property is set with the result of calling a method.
100 The translator methods receive the header metadata and should return
101 the appropriate value, or None if the value cannot be determined.
103 This implementation constructs an
104 `~astro_metadata_translator.ObservationInfo` object prior to calling
105 each translator method, making the translated information available
106 through the ``observationInfo`` attribute.
108 """
109 # Always calculate a new ObservationInfo since getInfo calls
110 # this method repeatedly for each header.
111 self.observationInfo = ObservationInfo(md, translator_class=self._translatorClass,
112 pedantic=False)
114 info = super().getInfoFromMetadata(md, info)
116 # Ensure that the translated ObservationInfo is cleared.
117 # This avoids possible confusion.
118 self.observationInfo = None
119 return info
121 def translate_wavelength(self, md):
122 """Translate wavelength provided by teststand readout.
124 The teststand driving script asks for a wavelength, and then reads the
125 value back to ensure that the correct position was moved to. This
126 number is therefore read back with sub-nm precision. Typically the
127 position is within 0.005nm of the desired position, so we warn if it's
128 not very close to an integer value.
130 Future users should be aware that the ``HIERARCH MONOCH-WAVELENG`` key
131 is NOT the requested value, and therefore cannot be used as a
132 cross-check that the wavelength was close to the one requested.
133 The only record of the wavelength that was set is in the original
134 filename.
136 Parameters
137 ----------
138 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
139 Image metadata.
141 Returns
142 -------
143 wavelength : `int`
144 The recorded wavelength in nanometers as an `int`.
145 """
146 bad_wl = -666 # Bad value for wavelength
147 if "MONOWL" not in md:
148 return bad_wl
150 raw_wl = float(md.getScalar("MONOWL"))
152 # Negative wavelengths are bad so normalize the bad value
153 if raw_wl < 0:
154 return bad_wl
156 wl = int(round(raw_wl))
157 if abs(raw_wl-wl) >= 0.1:
158 logger = logging.getLogger(__name__)
159 logger.warning(
160 'Translated significantly non-integer wavelength; '
161 '%s is more than 0.1nm from an integer value', raw_wl)
162 return wl
164 def translate_dateObs(self, md):
165 """Retrieve the date of observation as an ISO format string.
167 Parameters
168 ----------
169 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
170 Image metadata.
172 Returns
173 -------
174 dateObs : `str`
175 The date that the data was taken in FITS ISO format,
176 e.g. ``2018-08-20T21:56:24.608``.
177 """
178 dateObs = self.observationInfo.datetime_begin
179 dateObs.format = "isot"
180 return str(dateObs)
182 translate_date = translate_dateObs
184 def translate_dayObs(self, md):
185 """Generate the day that the observation was taken.
187 Parameters
188 ----------
189 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
190 image metadata
192 Returns
193 -------
194 dayObs : `str`
195 The day that the data was taken, e.g. ``1958-02-05``.
196 """
197 dayObs = str(self.observationInfo.observing_day)
198 return "-".join([dayObs[:4], dayObs[4:6], dayObs[6:]])
200 def translate_snap(self, md):
201 """Extract snap number from metadata.
203 Parameters
204 ----------
205 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
206 Image metadata.
208 Returns
209 -------
210 snap : `int`
211 Snap number (default: 0).
212 """
213 try:
214 return int(md.getScalar("SNAP"))
215 except KeyError:
216 return 0
218 def translate_detectorName(self, md):
219 """Extract ccd ID from CHIPID.
221 Parameters
222 ----------
223 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
224 Image metadata.
226 Returns
227 -------
228 ccdID : `str`
229 Name of ccd, e.g. ``S01``.
230 """
231 return self.observationInfo.detector_name
233 def translate_raftName(self, md):
234 """Extract raft ID from CHIPID.
236 Parameters
237 ----------
238 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
239 Image metadata.
241 Returns
242 -------
243 raftID : `str`
244 Name of raft, e.g. ``R21``.
245 """
246 return self.observationInfo.detector_group
248 def translate_detector(self, md):
249 """Extract detector ID from metadata
251 Parameters
252 ----------
253 md : `~lsst.daf.base.PropertyList` or `~lsst.daf.base.PropertySet`
254 Image metadata.
256 Returns
257 -------
258 detID : `int`
259 Detector ID, e.g. ``4``.
260 """
261 return self.observationInfo.detector_num
263 def translate_expTime(self, md):
264 return self.observationInfo.exposure_time.value
266 def translate_object(self, md):
267 return self.observationInfo.object
269 def translate_imageType(self, md):
270 obstype = self.observationInfo.observation_type.upper()
271 # Dictionary for obstype values is not yet clear
272 if obstype == "SCIENCE":
273 obstype = "SKYEXP"
274 return obstype
276 def translate_filter(self, md):
277 return self.observationInfo.physical_filter
279 def translate_lsstSerial(self, md):
280 return self.observationInfo.detector_serial
282 def translate_run(self, md):
283 return self.observationInfo.science_program
285 def translate_visit(self, md):
286 return self.observationInfo.visit_id
288 def translate_obsid(self, md):
289 return self.observationInfo.observation_id
291 def translate_testType(self, md):
292 # Gen2 prefers upper case
293 return self.observationInfo.observation_reason.upper()
295 def translate_expGroup(self, md):
296 return self.observationInfo.exposure_group
298 def translate_expId(self, md):
299 return self.observationInfo.exposure_id
301 def translate_controller(self, md):
302 if "CONTRLLR" in md:
303 if md["CONTRLLR"]:
304 return md["CONTRLLR"]
305 else:
306 # Was undefined, sometimes it is in fact in the OBSID
307 obsid = self.translate_obsid(md)
308 components = obsid.split("_")
309 if len(components) >= 2 and len(components[1]) == 1:
310 # AT_C_20190319_00001
311 return components[1]
312 # Assume OCS control
313 return "O"
314 else:
315 # Assume it is under camera control
316 return "C"
319class LsstCamCalibsParseTask(CalibsParseTask):
320 """Parser for calibs."""
322 def _translateFromCalibId(self, field, md):
323 """Get a value from the CALIB_ID written by ``constructCalibs``."""
324 data = md.getScalar("CALIB_ID")
325 match = re.search(r".*%s=(\S+)" % field, data)
326 return match.groups()[0]
328 def translate_raftName(self, md):
329 return self._translateFromCalibId("raftName", md)
331 def translate_detectorName(self, md):
332 return self._translateFromCalibId("detectorName", md)
334 def translate_detector(self, md):
335 # this is not a _great_ fix, but this obs_package is enforcing that
336 # detectors be integers and there's not an elegant way of ensuring
337 # this is the right type really
338 return int(self._translateFromCalibId("detector", md))
340 def translate_filter(self, md):
341 return self._translateFromCalibId("filter", md)
343 def translate_calibDate(self, md):
344 return self._translateFromCalibId("calibDate", md)
347class LsstCamEimgParseTask(LsstCamParseTask):
348 """Parser suitable for phosim LsstCam eimage data.
349 """
351 def getDestination(self, butler, info, filename):
352 """Get destination for the file.
354 Parameters
355 ----------
356 butler : `lsst.daf.persistence.Butler`
357 Data butler
358 info : data ID
359 File properties, used as dataId for the butler.
360 filename : `str`
361 Input filename.
363 Returns
364 -------
365 `str`
366 Destination filename.
367 """
369 eimage = butler.get("eimage_filename", info)[0]
370 # Ensure filename is devoid of cfitsio directions about HDUs
371 c = eimage.find("[")
372 if c > 0:
373 eimage = eimage[:c]
374 return eimage