Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23__all__ = ["MegacamParseTask", "MegaPrimeRawIngestTask"] 

24 

25import re 

26 

27from lsst.daf.butler import ButlerURI, Formatter 

28import lsst.obs.base 

29from lsst.obs.base.ingest import RawFileData 

30from astro_metadata_translator import fix_header 

31 

32from lsst.pipe.tasks.ingest import ParseTask 

33import lsst.pex.exceptions 

34 

35filters = {'u.MP9301': 'u', 

36 'u.MP9302': 'u2', 

37 'g.MP9401': 'g', 

38 'g.MP9402': 'g2', 

39 'r.MP9601': 'r', 

40 'r.MP9602': 'r2', 

41 'i.MP9701': 'i', 

42 'i.MP9702': 'i2', 

43 'i.MP9703': 'i3', 

44 'z.MP9801': 'z', 

45 'z.MP9901': 'z2', 

46 } 

47 

48 

49class MegaPrimeRawIngestTask(lsst.obs.base.RawIngestTask): 

50 """Task for ingesting raw MegaPrime multi-extension FITS data into Gen3. 

51 """ 

52 def extractMetadata(self, filename: ButlerURI) -> RawFileData: 

53 datasets = [] 

54 

55 try: 

56 with filename.as_local() as local_file: 

57 fitsData = lsst.afw.fits.Fits(local_file.ospath, "r") 

58 

59 # NOTE: The primary header (HDU=0) does not contain detector 

60 # data. 

61 for i in range(1, fitsData.countHdus()): 

62 fitsData.setHdu(i) 

63 header = fitsData.readMetadata() 

64 if not header["EXTNAME"].startswith("ccd"): 

65 continue 

66 fix_header(header) 

67 datasets.append(self._calculate_dataset_info(header, filename)) 

68 except Exception as e: 

69 self.log.debug("Problem extracting metadata from %s: %s", filename, e) 

70 # Indicate to the caller that we failed to read. 

71 # Do not try to ingest partial contents of file. 

72 datasets = [] 

73 formatterClass = Formatter 

74 instrument = None 

75 self._on_metadata_failure(filename, e) 

76 if self.config.failFast: 

77 raise RuntimeError(f"Problem extracting metadata for file {filename}") from e 

78 else: 

79 # The data model currently assumes that whilst multiple datasets 

80 # can be associated with a single file, they must all share the 

81 # same formatter. 

82 instrument, formatterClass = self._determine_instrument_formatter(datasets[0].dataId, filename) 

83 if instrument is None: 

84 datasets = [] 

85 

86 self.log.info(f"Found images for {len(datasets)} detectors in {filename}") 

87 return RawFileData(datasets=datasets, filename=filename, 

88 FormatterClass=formatterClass, 

89 instrumentClass=type(instrument)) 

90 

91 

92class MegacamParseTask(ParseTask): 

93 

94 def translate_ccd(self, md): 

95 try: 

96 extname = self.getExtensionName(md) 

97 return int(extname[3:]) # chop off "ccd" 

98 except LookupError: 

99 # Dummy value, intended for PHU (need something to get filename) 

100 return 99 

101 

102 def translate_filter(self, md): 

103 filtName = md.getScalar("FILTER").strip() 

104 if filtName not in filters: 

105 return "UNKNOWN" 

106 return filters[filtName] 

107 

108 def translate_taiObs(self, md): 

109 # Field name is "taiObs" but we're giving it UTC; shouldn't matter so 

110 # long as we're consistent 

111 (yr, month, day) = (md.getScalar("DATE-OBS").strip()).split("-") 

112 (hr, min, sec) = (md.getScalar("UTC-OBS").strip()).split(":") 

113 (sec1, sec2) = sec.split('.') 

114 return "%04d-%02d-%02dT%02d:%02d:%02d.%02d"%(int(yr), int(month), int(day), 

115 int(hr), int(min), int(sec1), int(sec2)) 

116 

117 def translate_defects(self, md): 

118 maskName = md.getScalar("IMRED_MK").strip() 

119 maskName, ccd = maskName.split(".fits") 

120 filter = md.getScalar("FILTER").strip().split('.')[0] 

121 if filter in ["i", "i2", "i3", "z", "z2"]: 

122 maskName = maskName+"_enlarged" 

123 maskFile = maskName+".nn/"+ccd[1:6]+".fits" 

124 return maskFile 

125 

126 def getInfo(self, filename): 

127 phuInfo, infoList = super(MegacamParseTask, self).getInfo(filename) 

128 match = re.search(r"\d+(?P<state>o|p)\.fits.*", filename) 

129 if not match: 

130 raise RuntimeError("Unable to parse filename: %s" % filename) 

131 phuInfo['state'] = match.group('state') 

132 phuInfo['extension'] = 0 

133 for num, info in enumerate(infoList): 

134 info['state'] = match.group('state') 

135 info['extension'] = num + 1 

136 return phuInfo, infoList 

137 

138 def getExtensionName(self, md): 

139 """Get the name of an extension. 

140 

141 Parameters 

142 ---------- 

143 md : `PropertySet` 

144 Metadata to get the name from. 

145 

146 Returns 

147 ------- 

148 name : `str` or None 

149 Name of the extension if it exists. None otherwise. 

150 """ 

151 # We have to overwrite this method because some (mostly recent) Megacam 

152 # images have a different header where the keword "EXTNAME" appears one 

153 # time instead of two. In the later case ext is a tuple while in the 

154 # other case it is a single value 

155 try: 

156 # This returns a tuple 

157 ext = md.getScalar("EXTNAME") 

158 # Most of the time the EXTNAME keyword appears 2 times in the 

159 # header (1st time to specify that the image is compressed) but 

160 # sometimes it appears only once even if the image is compressed 

161 if type(ext) == tuple or type(ext) == list: 

162 return ext[1] 

163 else: 

164 return ext 

165 except lsst.pex.exceptions.Exception: 

166 return None