Coverage for python/lsst/obs/decam/ingestCalibs.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
2import collections.abc
3import re
4from lsst.afw.fits import readMetadata
5from lsst.pipe.tasks.ingestCalibs import CalibsParseTask
7__all__ = ["DecamCalibsParseTask"]
10class DecamCalibsParseTask(CalibsParseTask):
11 """Parse calibration products for ingestion.
13 Handle either DECam Community Pipeline calibration products or
14 calibration products produced with the LSST Science Pipelines
15 (i.e., pipe_drivers' constructBias.py and constructFlat.py).
16 """
18 def getInfo(self, filename):
19 """Retrieve path, calib_hdu, and possibly calibDate.
21 Parameters
22 ----------
23 filename: `str`
24 Calibration file to inspect.
26 Returns
27 -------
28 phuInfo : `dict`
29 Primary header unit info.
30 infoList : `list` of `dict`
31 List of file properties to use for each extension.
32 """
33 phuInfo, infoList = CalibsParseTask.getInfo(self, filename)
34 # In practice, this returns an empty dict
35 # and a list containing an empty dict.
36 for item in infoList:
37 item['path'] = filename
38 try:
39 item['calib_hdu'] = item['hdu']
40 except KeyError: # calib_hdu is required for the calib registry
41 item['calib_hdu'] = 1
42 # Try to fetch a date from filename
43 # and use as the calibration dates if not already set
44 found = re.search(r'(\d\d\d\d-\d\d-\d\d)', filename)
45 if found:
46 date = found.group(1)
47 for info in infoList:
48 if 'calibDate' not in info or info['calibDate'] == "unknown":
49 info['calibDate'] = date
50 return phuInfo, infoList
52 def _translateFromCalibId(self, field, md):
53 """Fetch the ID from the CALIB_ID header.
55 Calibration products made with constructCalibs have some metadata
56 saved in its FITS header CALIB_ID.
57 """
58 data = md["CALIB_ID"]
59 match = re.search(r".*%s=(\S+)" % field, data)
60 return match.groups()[0]
62 def translate_ccdnum(self, md):
63 """Return CCDNUM as a integer.
65 Parameters
66 ----------
67 md : `lsst.daf.base.PropertySet`
68 FITS header metadata.
69 """
70 if "CCDNUM" in md:
71 ccdnum = md["CCDNUM"]
72 else:
73 return self._translateFromCalibId("ccdnum", md)
74 # Some MasterCal from NOAO Archive has 2 CCDNUM keys in each HDU
75 # Make sure only one integer is returned.
76 if isinstance(ccdnum, collections.abc.Sequence):
77 try:
78 ccdnum = ccdnum[0]
79 except IndexError:
80 ccdnum = None
81 return ccdnum
83 def translate_date(self, md):
84 """Extract the date as a strong in format YYYY-MM-DD from the FITS header DATE-OBS.
85 Return "unknown" if the value cannot be found or converted.
87 Parameters
88 ----------
89 md : `lsst.daf.base.PropertySet`
90 FITS header metadata.
91 """
92 if "DATE-OBS" in md:
93 date = md["DATE-OBS"]
94 found = re.search(r'(\d\d\d\d-\d\d-\d\d)', date)
95 if found:
96 date = found.group(1)
97 else:
98 self.log.warn("DATE-OBS does not match format YYYY-MM-DD")
99 date = "unknown"
100 elif "CALIB_ID" in md:
101 date = self._translateFromCalibId("calibDate", md)
102 else:
103 date = "unknown"
104 return date
106 def translate_filter(self, md):
107 """Extract the filter name.
109 Translate a full filter description into a mere filter name.
110 Return "unknown" if the keyword FILTER does not exist in the header,
111 which can happen for some valid Community Pipeline products.
113 Parameters
114 ----------
115 md : `lsst.daf.base.PropertySet`
116 FITS header metadata.
118 Returns
119 -------
120 filter : `str`
121 The name of the filter to use in the calib registry.
122 """
123 if "FILTER" in md:
124 if "OBSTYPE" in md:
125 obstype = md["OBSTYPE"].strip().lower()
126 if "zero" in obstype or "bias" in obstype:
127 return "NONE"
128 filterName = CalibsParseTask.translate_filter(self, md)
129 # TODO (DM-24514): remove workaround if/else
130 if filterName == '_unknown_' and "CALIB_ID" in md:
131 return self._translateFromCalibId("filter", md)
132 else:
133 return CalibsParseTask.translate_filter(self, md)
134 elif "CALIB_ID" in md:
135 return self._translateFromCalibId("filter", md)
136 else:
137 return "unknown"
139 def getDestination(self, butler, info, filename):
140 """Get destination for the file.
142 Parameters
143 ----------
144 butler : `lsst.daf.persistence.Butler`
145 Data butler.
146 info : data ID
147 File properties, used as dataId for the butler.
148 filename : `str`
149 Input filename.
151 Returns
152 -------
153 raw : `str`
154 Destination filename.
155 """
156 calibType = self.getCalibType(filename)
157 md = readMetadata(filename, self.config.hdu)
158 if "PROCTYPE" not in md:
159 raise RuntimeError(f"Unable to find the calib header keyword PROCTYPE \
160 in {filename}, hdu {self.config.hdu}")
161 proctype = md["PROCTYPE"].strip()
162 if "MasterCal" in proctype: # DECam Community Pipeline calibration product case
163 # Arbitrarily set ccdnum and calib_hdu to 1 to make the mapper template happy
164 info["ccdnum"] = 1
165 info["calib_hdu"] = 1
166 if "flat" in calibType.lower():
167 raw = butler.get("cpFlat_filename", info)[0]
168 elif ("bias" or "zero") in calibType.lower():
169 raw = butler.get("cpBias_filename", info)[0]
170 elif ("illumcor") in calibType.lower():
171 raw = butler.get("cpIllumcor_filename", info)[0]
172 else:
173 raise RuntimeError(f"Invalid DECam Community Pipeline calibType {calibType}")
174 else: # LSST-built calibration product case
175 raw = butler.get(calibType + "_filename", info)[0]
176 # Remove HDU extension (ccdnum) since we want to refer to the whole file
177 c = raw.find("[")
178 if c > 0:
179 raw = raw[:c]
180 return raw