Coverage for python/lsst/obs/cfht/ingest.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23__all__ = ["MegacamParseTask", "MegaPrimeRawIngestTask"]
25import re
27from lsst.daf.butler import ButlerURI, Formatter
28import lsst.obs.base
29from lsst.obs.base.ingest import RawFileData
30from astro_metadata_translator import fix_header
32from lsst.pipe.tasks.ingest import ParseTask
33import lsst.pex.exceptions
35filters = {'u.MP9301': 'u',
36 'u.MP9302': 'u2',
37 'g.MP9401': 'g',
38 'g.MP9402': 'g2',
39 'r.MP9601': 'r',
40 'r.MP9602': 'r2',
41 'i.MP9701': 'i',
42 'i.MP9702': 'i2',
43 'i.MP9703': 'i3',
44 'z.MP9801': 'z',
45 'z.MP9901': 'z2',
46 }
49class MegaPrimeRawIngestTask(lsst.obs.base.RawIngestTask):
50 """Task for ingesting raw MegaPrime multi-extension FITS data into Gen3.
51 """
52 def extractMetadata(self, filename: ButlerURI) -> RawFileData:
53 datasets = []
55 try:
56 with filename.as_local() as local_file:
57 fitsData = lsst.afw.fits.Fits(local_file.ospath, "r")
59 # NOTE: The primary header (HDU=0) does not contain detector
60 # data.
61 for i in range(1, fitsData.countHdus()):
62 fitsData.setHdu(i)
63 header = fitsData.readMetadata()
64 if not header["EXTNAME"].startswith("ccd"):
65 continue
66 fix_header(header)
67 datasets.append(self._calculate_dataset_info(header, filename))
68 except Exception as e:
69 self.log.debug("Problem extracting metadata from %s: %s", filename, e)
70 # Indicate to the caller that we failed to read.
71 # Do not try to ingest partial contents of file.
72 datasets = []
73 formatterClass = Formatter
74 instrument = None
75 self._on_metadata_failure(filename, e)
76 if self.config.failFast:
77 raise RuntimeError(f"Problem extracting metadata for file {filename}") from e
78 else:
79 # The data model currently assumes that whilst multiple datasets
80 # can be associated with a single file, they must all share the
81 # same formatter.
82 instrument, formatterClass = self._determine_instrument_formatter(datasets[0].dataId, filename)
83 if instrument is None:
84 datasets = []
86 self.log.info(f"Found images for {len(datasets)} detectors in {filename}")
87 return RawFileData(datasets=datasets, filename=filename,
88 FormatterClass=formatterClass,
89 instrumentClass=type(instrument))
92class MegacamParseTask(ParseTask):
94 def translate_ccd(self, md):
95 try:
96 extname = self.getExtensionName(md)
97 return int(extname[3:]) # chop off "ccd"
98 except LookupError:
99 # Dummy value, intended for PHU (need something to get filename)
100 return 99
102 def translate_filter(self, md):
103 filtName = md.getScalar("FILTER").strip()
104 if filtName not in filters:
105 return "UNKNOWN"
106 return filters[filtName]
108 def translate_taiObs(self, md):
109 # Field name is "taiObs" but we're giving it UTC; shouldn't matter so
110 # long as we're consistent
111 (yr, month, day) = (md.getScalar("DATE-OBS").strip()).split("-")
112 (hr, min, sec) = (md.getScalar("UTC-OBS").strip()).split(":")
113 (sec1, sec2) = sec.split('.')
114 return "%04d-%02d-%02dT%02d:%02d:%02d.%02d"%(int(yr), int(month), int(day),
115 int(hr), int(min), int(sec1), int(sec2))
117 def translate_defects(self, md):
118 maskName = md.getScalar("IMRED_MK").strip()
119 maskName, ccd = maskName.split(".fits")
120 filter = md.getScalar("FILTER").strip().split('.')[0]
121 if filter in ["i", "i2", "i3", "z", "z2"]:
122 maskName = maskName+"_enlarged"
123 maskFile = maskName+".nn/"+ccd[1:6]+".fits"
124 return maskFile
126 def getInfo(self, filename):
127 phuInfo, infoList = super(MegacamParseTask, self).getInfo(filename)
128 match = re.search(r"\d+(?P<state>o|p)\.fits.*", filename)
129 if not match:
130 raise RuntimeError("Unable to parse filename: %s" % filename)
131 phuInfo['state'] = match.group('state')
132 phuInfo['extension'] = 0
133 for num, info in enumerate(infoList):
134 info['state'] = match.group('state')
135 info['extension'] = num + 1
136 return phuInfo, infoList
138 def getExtensionName(self, md):
139 """Get the name of an extension.
141 Parameters
142 ----------
143 md : `PropertySet`
144 Metadata to get the name from.
146 Returns
147 -------
148 name : `str` or None
149 Name of the extension if it exists. None otherwise.
150 """
151 # We have to overwrite this method because some (mostly recent) Megacam
152 # images have a different header where the keword "EXTNAME" appears one
153 # time instead of two. In the later case ext is a tuple while in the
154 # other case it is a single value
155 try:
156 # This returns a tuple
157 ext = md.getScalar("EXTNAME")
158 # Most of the time the EXTNAME keyword appears 2 times in the
159 # header (1st time to specify that the image is compressed) but
160 # sometimes it appears only once even if the image is compressed
161 if type(ext) == tuple or type(ext) == list:
162 return ext[1]
163 else:
164 return ext
165 except lsst.pex.exceptions.Exception:
166 return None