Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1 

2import collections.abc 

3import re 

4from lsst.afw.fits import readMetadata 

5from lsst.pipe.tasks.ingestCalibs import CalibsParseTask 

6 

7__all__ = ["DecamCalibsParseTask"] 

8 

9 

10class DecamCalibsParseTask(CalibsParseTask): 

11 """Parse calibration products for ingestion. 

12 

13 Handle either DECam Community Pipeline calibration products or 

14 calibration products produced with the LSST Science Pipelines 

15 (i.e., pipe_drivers' constructBias.py and constructFlat.py). 

16 """ 

17 

18 def getInfo(self, filename): 

19 """Retrieve path, calib_hdu, and possibly calibDate. 

20 

21 Parameters 

22 ---------- 

23 filename: `str` 

24 Calibration file to inspect. 

25 

26 Returns 

27 ------- 

28 phuInfo : `dict` 

29 Primary header unit info. 

30 infoList : `list` of `dict` 

31 List of file properties to use for each extension. 

32 """ 

33 phuInfo, infoList = CalibsParseTask.getInfo(self, filename) 

34 # In practice, this returns an empty dict 

35 # and a list containing an empty dict. 

36 for item in infoList: 

37 item['path'] = filename 

38 try: 

39 item['calib_hdu'] = item['hdu'] 

40 except KeyError: # calib_hdu is required for the calib registry 

41 item['calib_hdu'] = 1 

42 # Try to fetch a date from filename 

43 # and use as the calibration dates if not already set 

44 found = re.search(r'(\d\d\d\d-\d\d-\d\d)', filename) 

45 if found: 

46 date = found.group(1) 

47 for info in infoList: 

48 if 'calibDate' not in info or info['calibDate'] == "unknown": 

49 info['calibDate'] = date 

50 return phuInfo, infoList 

51 

52 def _translateFromCalibId(self, field, md): 

53 """Fetch the ID from the CALIB_ID header. 

54 

55 Calibration products made with constructCalibs have some metadata 

56 saved in its FITS header CALIB_ID. 

57 """ 

58 data = md["CALIB_ID"] 

59 match = re.search(r".*%s=(\S+)" % field, data) 

60 return match.groups()[0] 

61 

62 def translate_ccdnum(self, md): 

63 """Return CCDNUM as a integer. 

64 

65 Parameters 

66 ---------- 

67 md : `lsst.daf.base.PropertySet` 

68 FITS header metadata. 

69 """ 

70 if "CCDNUM" in md: 

71 ccdnum = md["CCDNUM"] 

72 else: 

73 return self._translateFromCalibId("ccdnum", md) 

74 # Some MasterCal from NOAO Archive has 2 CCDNUM keys in each HDU 

75 # Make sure only one integer is returned. 

76 if isinstance(ccdnum, collections.abc.Sequence): 

77 try: 

78 ccdnum = ccdnum[0] 

79 except IndexError: 

80 ccdnum = None 

81 return ccdnum 

82 

83 def translate_date(self, md): 

84 """Extract the date as a strong in format YYYY-MM-DD from the FITS header DATE-OBS. 

85 Return "unknown" if the value cannot be found or converted. 

86 

87 Parameters 

88 ---------- 

89 md : `lsst.daf.base.PropertySet` 

90 FITS header metadata. 

91 """ 

92 if "DATE-OBS" in md: 

93 date = md["DATE-OBS"] 

94 found = re.search(r'(\d\d\d\d-\d\d-\d\d)', date) 

95 if found: 

96 date = found.group(1) 

97 else: 

98 self.log.warn("DATE-OBS does not match format YYYY-MM-DD") 

99 date = "unknown" 

100 elif "CALIB_ID" in md: 

101 date = self._translateFromCalibId("calibDate", md) 

102 else: 

103 date = "unknown" 

104 return date 

105 

106 def translate_filter(self, md): 

107 """Extract the filter name. 

108 

109 Translate a full filter description into a mere filter name. 

110 Return "unknown" if the keyword FILTER does not exist in the header, 

111 which can happen for some valid Community Pipeline products. 

112 

113 Parameters 

114 ---------- 

115 md : `lsst.daf.base.PropertySet` 

116 FITS header metadata. 

117 

118 Returns 

119 ------- 

120 filter : `str` 

121 The name of the filter to use in the calib registry. 

122 """ 

123 if "FILTER" in md: 

124 if "OBSTYPE" in md: 

125 obstype = md["OBSTYPE"].strip().lower() 

126 if "zero" in obstype or "bias" in obstype: 

127 return "NONE" 

128 filterName = CalibsParseTask.translate_filter(self, md) 

129 # TODO (DM-24514): remove workaround if/else 

130 if filterName == '_unknown_' and "CALIB_ID" in md: 

131 return self._translateFromCalibId("filter", md) 

132 else: 

133 return CalibsParseTask.translate_filter(self, md) 

134 elif "CALIB_ID" in md: 

135 return self._translateFromCalibId("filter", md) 

136 else: 

137 return "unknown" 

138 

139 def getDestination(self, butler, info, filename): 

140 """Get destination for the file. 

141 

142 Parameters 

143 ---------- 

144 butler : `lsst.daf.persistence.Butler` 

145 Data butler. 

146 info : data ID 

147 File properties, used as dataId for the butler. 

148 filename : `str` 

149 Input filename. 

150 

151 Returns 

152 ------- 

153 raw : `str` 

154 Destination filename. 

155 """ 

156 calibType = self.getCalibType(filename) 

157 md = readMetadata(filename, self.config.hdu) 

158 if "PROCTYPE" not in md: 

159 raise RuntimeError(f"Unable to find the calib header keyword PROCTYPE \ 

160 in {filename}, hdu {self.config.hdu}") 

161 proctype = md["PROCTYPE"].strip() 

162 if "MasterCal" in proctype: # DECam Community Pipeline calibration product case 

163 # Arbitrarily set ccdnum and calib_hdu to 1 to make the mapper template happy 

164 info["ccdnum"] = 1 

165 info["calib_hdu"] = 1 

166 if "flat" in calibType.lower(): 

167 raw = butler.get("cpFlat_filename", info)[0] 

168 elif ("bias" or "zero") in calibType.lower(): 

169 raw = butler.get("cpBias_filename", info)[0] 

170 elif ("illumcor") in calibType.lower(): 

171 raw = butler.get("cpIllumcor_filename", info)[0] 

172 else: 

173 raise RuntimeError(f"Invalid DECam Community Pipeline calibType {calibType}") 

174 else: # LSST-built calibration product case 

175 raw = butler.get(calibType + "_filename", info)[0] 

176 # Remove HDU extension (ccdnum) since we want to refer to the whole file 

177 c = raw.find("[") 

178 if c > 0: 

179 raw = raw[:c] 

180 return raw