Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2012,2015 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import re 

24 

25 

26from astro_metadata_translator import fix_header, DecamTranslator 

27from lsst.afw.fits import readMetadata 

28from lsst.pipe.tasks.ingest import ParseTask, IngestTask, IngestArgumentParser 

29from lsst.obs.base.ingest import RawFileData 

30import lsst.obs.base 

31from ._instrument import DarkEnergyCamera 

32 

33__all__ = ["DecamRawIngestTask", "DecamIngestArgumentParser", "DecamIngestTask", "DecamParseTask"] 

34 

35 

36class DecamRawIngestTask(lsst.obs.base.RawIngestTask): 

37 """Task for ingesting raw DECam data into a Gen3 Butler repository. 

38 """ 

39 def extractMetadata(self, filename: str) -> RawFileData: 

40 datasets = [] 

41 fitsData = lsst.afw.fits.Fits(filename, 'r') 

42 # NOTE: The primary header (HDU=0) does not contain detector data. 

43 for i in range(1, fitsData.countHdus()): 

44 fitsData.setHdu(i) 

45 header = fitsData.readMetadata() 

46 if header['CCDNUM'] > 62: # ignore the guide CCDs 

47 continue 

48 datasets.append(self._calculate_dataset_info(header, filename)) 

49 

50 # The data model currently assumes that whilst multiple datasets 

51 # can be associated with a single file, they must all share the 

52 # same formatter. 

53 instrument = DarkEnergyCamera() 

54 FormatterClass = instrument.getRawFormatter(datasets[0].dataId) 

55 

56 self.log.debug(f"Found images for {len(datasets)} detectors in {filename}") 

57 return RawFileData(datasets=datasets, filename=filename, 

58 FormatterClass=FormatterClass, 

59 instrumentClass=type(instrument)) 

60 

61 

62class DecamIngestArgumentParser(IngestArgumentParser): 

63 """Gen2 DECam ingest additional arguments. 

64 """ 

65 

66 def __init__(self, *args, **kwargs): 

67 super(DecamIngestArgumentParser, self).__init__(*args, **kwargs) 

68 self.add_argument("--filetype", default="raw", choices=["instcal", "raw"], 

69 help="Data processing level of the files to be ingested") 

70 

71 

72class DecamIngestTask(IngestTask): 

73 """Gen2 DECam file ingest task. 

74 """ 

75 ArgumentParser = DecamIngestArgumentParser 

76 

77 def __init__(self, *args, **kwargs): 

78 super(DecamIngestTask, self).__init__(*args, **kwargs) 

79 

80 def run(self, args): 

81 """Ingest all specified files and add them to the registry 

82 """ 

83 if args.filetype == "instcal": 

84 root = args.input 

85 with self.register.openRegistry(root, create=args.create, dryrun=args.dryrun) as registry: 

86 for infile in args.files: 

87 fileInfo, hduInfoList = self.parse.getInfo(infile, args.filetype) 

88 if len(hduInfoList) > 0: 

89 outfileInstcal = os.path.join(root, self.parse.getDestination(args.butler, 

90 hduInfoList[0], 

91 infile, "instcal")) 

92 outfileDqmask = os.path.join(root, self.parse.getDestination(args.butler, 

93 hduInfoList[0], infile, 

94 "dqmask")) 

95 outfileWtmap = os.path.join(root, self.parse.getDestination(args.butler, 

96 hduInfoList[0], infile, 

97 "wtmap")) 

98 

99 ingestedInstcal = self.ingest(fileInfo["instcal"], outfileInstcal, 

100 mode=args.mode, dryrun=args.dryrun) 

101 ingestedDqmask = self.ingest(fileInfo["dqmask"], outfileDqmask, 

102 mode=args.mode, dryrun=args.dryrun) 

103 ingestedWtmap = self.ingest(fileInfo["wtmap"], outfileWtmap, 

104 mode=args.mode, dryrun=args.dryrun) 

105 

106 if not (ingestedInstcal or ingestedDqmask or ingestedWtmap): 

107 continue 

108 

109 for info in hduInfoList: 

110 self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create) 

111 

112 elif args.filetype == "raw": 

113 IngestTask.run(self, args) 

114 

115 

116class DecamParseTask(ParseTask): 

117 """Parse an image filename to get the required information to 

118 put the file in the correct location and populate the registry. 

119 """ 

120 

121 _translatorClass = DecamTranslator 

122 

123 def __init__(self, *args, **kwargs): 

124 super(ParseTask, self).__init__(*args, **kwargs) 

125 

126 self.expnumMapper = None 

127 

128 # Note that these should be syncronized with the fields in 

129 # root.register.columns defined in config/ingest.py 

130 self.instcalPrefix = "instcal" 

131 self.dqmaskPrefix = "dqmask" 

132 self.wtmapPrefix = "wtmap" 

133 

134 def _listdir(self, path, prefix): 

135 for file in os.listdir(path): 

136 fileName = os.path.join(path, file) 

137 md = readMetadata(fileName) 

138 fix_header(md, translator_class=self._translatorClass) 

139 if "EXPNUM" not in md: 

140 return 

141 expnum = md["EXPNUM"] 

142 if expnum not in self.expnumMapper: 

143 self.expnumMapper[expnum] = {self.instcalPrefix: None, 

144 self.wtmapPrefix: None, 

145 self.dqmaskPrefix: None} 

146 self.expnumMapper[expnum][prefix] = fileName 

147 

148 def buildExpnumMapper(self, basepath): 

149 """Extract exposure numbers from filenames to set self.expnumMapper 

150 

151 Parameters 

152 ---------- 

153 basepath : `str` 

154 Location on disk of instcal, dqmask, and wtmap subdirectories. 

155 """ 

156 self.expnumMapper = {} 

157 

158 instcalPath = basepath 

159 dqmaskPath = re.sub(self.instcalPrefix, self.dqmaskPrefix, instcalPath) 

160 wtmapPath = re.sub(self.instcalPrefix, self.wtmapPrefix, instcalPath) 

161 if instcalPath == dqmaskPath: 

162 raise RuntimeError("instcal and mask directories are the same") 

163 if instcalPath == wtmapPath: 

164 raise RuntimeError("instcal and weight map directories are the same") 

165 

166 if not os.path.isdir(dqmaskPath): 

167 raise OSError("Directory %s does not exist" % (dqmaskPath)) 

168 if not os.path.isdir(wtmapPath): 

169 raise OSError("Directory %s does not exist" % (wtmapPath)) 

170 

171 # Traverse each directory and extract the expnums 

172 for path, prefix in zip((instcalPath, dqmaskPath, wtmapPath), 

173 (self.instcalPrefix, self.dqmaskPrefix, self.wtmapPrefix)): 

174 self._listdir(path, prefix) 

175 

176 def getInfo(self, filename, filetype="raw"): 

177 """Get metadata header info from multi-extension FITS decam image file. 

178 

179 The science pixels, mask, and weight (inverse variance) are 

180 stored in separate files each with a unique name but with a 

181 common unique identifier EXPNUM in the FITS header. We have 

182 to aggregate the 3 filenames for a given EXPNUM and return 

183 this information along with that returned by the base class. 

184 

185 Parameters 

186 ---------- 

187 filename : `str` 

188 Image file to retrieve info from. 

189 filetype : `str` 

190 One of "raw" or "instcal". 

191 

192 Returns 

193 ------- 

194 phuInfo : `dict` 

195 Primary header unit info. 

196 infoList : `list` of `dict` 

197 Info for the other HDUs. 

198 

199 Notes 

200 ----- 

201 For filetype="instcal", we expect a directory structure that looks 

202 like the following: 

203 

204 .. code-block:: none 

205 

206 dqmask/ 

207 instcal/ 

208 wtmap/ 

209 

210 The user creates the registry by running: 

211 

212 .. code-block:: none 

213 

214 ingestImagesDecam.py outputRepository --filetype=instcal --mode=link instcal/*fits 

215 """ 

216 if filetype == "instcal": 

217 if self.expnumMapper is None: 

218 self.buildExpnumMapper(os.path.dirname(os.path.abspath(filename))) 

219 

220 # Note that phuInfo will have 

221 # 'side': 'X', 'ccd': 0 

222 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename) 

223 expnum = phuInfo["visit"] 

224 phuInfo[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] 

225 phuInfo[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] 

226 phuInfo[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] 

227 for info in infoList: 

228 expnum = info["visit"] 

229 info[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] 

230 info[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] 

231 info[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] 

232 

233 elif filetype == "raw": 

234 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename) 

235 for info in infoList: 

236 info[self.instcalPrefix] = "" 

237 info[self.dqmaskPrefix] = "" 

238 info[self.wtmapPrefix] = "" 

239 

240 # Some data IDs can not be extracted from the zeroth extension 

241 # of the MEF. Add them so Butler does not try to find them 

242 # in the registry which may still yet to be created. 

243 for key in ("ccdnum", "hdu", "ccd", "calib_hdu"): 

244 if key not in phuInfo: 

245 phuInfo[key] = 0 

246 

247 return phuInfo, infoList 

248 

249 def getDestination(self, butler, info, filename, filetype="raw"): 

250 """Get destination for the file 

251 

252 Parameters 

253 ---------- 

254 butler : `lsst.daf.persistence.Butler` 

255 Data butler. 

256 info : data ID 

257 File properties, used as dataId for the butler. 

258 filename : `str` 

259 Input filename. 

260 

261 Returns 

262 ------- 

263 raw : `str` 

264 Destination filename. 

265 """ 

266 raw = butler.get("%s_filename"%(filetype), info)[0] 

267 # Ensure filename is devoid of cfitsio directions about HDUs 

268 c = raw.find("[") 

269 if c > 0: 

270 raw = raw[:c] 

271 return raw