Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2012,2015 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import re 

24 

25 

26from astro_metadata_translator import fix_header, DecamTranslator 

27from lsst.afw.fits import readMetadata 

28from lsst.pipe.tasks.ingest import ParseTask, IngestTask, IngestArgumentParser 

29from lsst.obs.base.ingest import RawFileData 

30import lsst.obs.base 

31from ._instrument import DarkEnergyCamera 

32 

33__all__ = ["DecamRawIngestTask", "DecamIngestArgumentParser", "DecamIngestTask", "DecamParseTask"] 

34 

35 

36class DecamRawIngestTask(lsst.obs.base.RawIngestTask): 

37 """Task for ingesting raw DECam data into a Gen3 Butler repository. 

38 """ 

39 def extractMetadata(self, filename: str) -> RawFileData: 

40 datasets = [] 

41 fitsData = lsst.afw.fits.Fits(filename, 'r') 

42 # NOTE: The primary header (HDU=0) does not contain detector data. 

43 for i in range(1, fitsData.countHdus()): 

44 fitsData.setHdu(i) 

45 header = fitsData.readMetadata() 

46 if header['CCDNUM'] > 62: # ignore the guide CCDs 

47 continue 

48 fix_header(header) 

49 datasets.append(self._calculate_dataset_info(header, filename)) 

50 

51 # The data model currently assumes that whilst multiple datasets 

52 # can be associated with a single file, they must all share the 

53 # same formatter. 

54 instrument = DarkEnergyCamera() 

55 FormatterClass = instrument.getRawFormatter(datasets[0].dataId) 

56 

57 self.log.debug(f"Found images for {len(datasets)} detectors in {filename}") 

58 return RawFileData(datasets=datasets, filename=filename, 

59 FormatterClass=FormatterClass) 

60 

61 

62class DecamIngestArgumentParser(IngestArgumentParser): 

63 """Gen2 DECam ingest additional arguments. 

64 """ 

65 

66 def __init__(self, *args, **kwargs): 

67 super(DecamIngestArgumentParser, self).__init__(*args, **kwargs) 

68 self.add_argument("--filetype", default="raw", choices=["instcal", "raw"], 

69 help="Data processing level of the files to be ingested") 

70 

71 

72class DecamIngestTask(IngestTask): 

73 """Gen2 DECam file ingest task. 

74 """ 

75 ArgumentParser = DecamIngestArgumentParser 

76 

77 def __init__(self, *args, **kwargs): 

78 super(DecamIngestTask, self).__init__(*args, **kwargs) 

79 

80 def run(self, args): 

81 """Ingest all specified files and add them to the registry 

82 """ 

83 if args.filetype == "instcal": 

84 root = args.input 

85 with self.register.openRegistry(root, create=args.create, dryrun=args.dryrun) as registry: 

86 for infile in args.files: 

87 fileInfo, hduInfoList = self.parse.getInfo(infile, args.filetype) 

88 if len(hduInfoList) > 0: 

89 outfileInstcal = os.path.join(root, self.parse.getDestination(args.butler, 

90 hduInfoList[0], 

91 infile, "instcal")) 

92 outfileDqmask = os.path.join(root, self.parse.getDestination(args.butler, 

93 hduInfoList[0], infile, 

94 "dqmask")) 

95 outfileWtmap = os.path.join(root, self.parse.getDestination(args.butler, 

96 hduInfoList[0], infile, 

97 "wtmap")) 

98 

99 ingestedInstcal = self.ingest(fileInfo["instcal"], outfileInstcal, 

100 mode=args.mode, dryrun=args.dryrun) 

101 ingestedDqmask = self.ingest(fileInfo["dqmask"], outfileDqmask, 

102 mode=args.mode, dryrun=args.dryrun) 

103 ingestedWtmap = self.ingest(fileInfo["wtmap"], outfileWtmap, 

104 mode=args.mode, dryrun=args.dryrun) 

105 

106 if not (ingestedInstcal or ingestedDqmask or ingestedWtmap): 

107 continue 

108 

109 for info in hduInfoList: 

110 self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create) 

111 

112 elif args.filetype == "raw": 

113 IngestTask.run(self, args) 

114 

115 

116class DecamParseTask(ParseTask): 

117 """Parse an image filename to get the required information to 

118 put the file in the correct location and populate the registry. 

119 """ 

120 

121 def __init__(self, *args, **kwargs): 

122 super(ParseTask, self).__init__(*args, **kwargs) 

123 

124 self.expnumMapper = None 

125 

126 # Note that these should be syncronized with the fields in 

127 # root.register.columns defined in config/ingest.py 

128 self.instcalPrefix = "instcal" 

129 self.dqmaskPrefix = "dqmask" 

130 self.wtmapPrefix = "wtmap" 

131 

132 def _listdir(self, path, prefix): 

133 for file in os.listdir(path): 

134 fileName = os.path.join(path, file) 

135 md = readMetadata(fileName) 

136 fix_header(md, translator_class=DecamTranslator) 

137 if "EXPNUM" not in md.names(): 

138 return 

139 expnum = md.getScalar("EXPNUM") 

140 if expnum not in self.expnumMapper: 

141 self.expnumMapper[expnum] = {self.instcalPrefix: None, 

142 self.wtmapPrefix: None, 

143 self.dqmaskPrefix: None} 

144 self.expnumMapper[expnum][prefix] = fileName 

145 

146 def buildExpnumMapper(self, basepath): 

147 """Extract exposure numbers from filenames to set self.expnumMapper 

148 

149 Parameters 

150 ---------- 

151 basepath : `str` 

152 Location on disk of instcal, dqmask, and wtmap subdirectories. 

153 """ 

154 self.expnumMapper = {} 

155 

156 instcalPath = basepath 

157 dqmaskPath = re.sub(self.instcalPrefix, self.dqmaskPrefix, instcalPath) 

158 wtmapPath = re.sub(self.instcalPrefix, self.wtmapPrefix, instcalPath) 

159 if instcalPath == dqmaskPath: 

160 raise RuntimeError("instcal and mask directories are the same") 

161 if instcalPath == wtmapPath: 

162 raise RuntimeError("instcal and weight map directories are the same") 

163 

164 if not os.path.isdir(dqmaskPath): 

165 raise OSError("Directory %s does not exist" % (dqmaskPath)) 

166 if not os.path.isdir(wtmapPath): 

167 raise OSError("Directory %s does not exist" % (wtmapPath)) 

168 

169 # Traverse each directory and extract the expnums 

170 for path, prefix in zip((instcalPath, dqmaskPath, wtmapPath), 

171 (self.instcalPrefix, self.dqmaskPrefix, self.wtmapPrefix)): 

172 self._listdir(path, prefix) 

173 

174 def getInfo(self, filename, filetype="raw"): 

175 """Get metadata header info from multi-extension FITS decam image file. 

176 

177 The science pixels, mask, and weight (inverse variance) are 

178 stored in separate files each with a unique name but with a 

179 common unique identifier EXPNUM in the FITS header. We have 

180 to aggregate the 3 filenames for a given EXPNUM and return 

181 this information along with that returned by the base class. 

182 

183 Parameters 

184 ---------- 

185 filename : `str` 

186 Image file to retrieve info from. 

187 filetype : `str` 

188 One of "raw" or "instcal". 

189 

190 Returns 

191 ------- 

192 phuInfo : `dict` 

193 Primary header unit info. 

194 infoList : `list` of `dict` 

195 Info for the other HDUs. 

196 

197 Notes 

198 ----- 

199 For filetype="instcal", we expect a directory structure that looks 

200 like the following: 

201 

202 .. code-block:: none 

203 

204 dqmask/ 

205 instcal/ 

206 wtmap/ 

207 

208 The user creates the registry by running: 

209 

210 .. code-block:: none 

211 

212 ingestImagesDecam.py outputRepository --filetype=instcal --mode=link instcal/*fits 

213 """ 

214 if filetype == "instcal": 

215 if self.expnumMapper is None: 

216 self.buildExpnumMapper(os.path.dirname(os.path.abspath(filename))) 

217 

218 # Note that phuInfo will have 

219 # 'side': 'X', 'ccd': 0 

220 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename) 

221 expnum = phuInfo["visit"] 

222 phuInfo[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] 

223 phuInfo[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] 

224 phuInfo[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] 

225 for info in infoList: 

226 expnum = info["visit"] 

227 info[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] 

228 info[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] 

229 info[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] 

230 

231 elif filetype == "raw": 

232 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename) 

233 for info in infoList: 

234 info[self.instcalPrefix] = "" 

235 info[self.dqmaskPrefix] = "" 

236 info[self.wtmapPrefix] = "" 

237 

238 # Some data IDs can not be extracted from the zeroth extension 

239 # of the MEF. Add them so Butler does not try to find them 

240 # in the registry which may still yet to be created. 

241 for key in ("ccdnum", "hdu", "ccd", "calib_hdu"): 

242 if key not in phuInfo: 

243 phuInfo[key] = 0 

244 

245 return phuInfo, infoList 

246 

247 @staticmethod 

248 def getExtensionName(md): 

249 return md.getScalar('EXTNAME') 

250 

251 def getDestination(self, butler, info, filename, filetype="raw"): 

252 """Get destination for the file 

253 

254 Parameters 

255 ---------- 

256 butler : `lsst.daf.persistence.Butler` 

257 Data butler. 

258 info : data ID 

259 File properties, used as dataId for the butler. 

260 filename : `str` 

261 Input filename. 

262 

263 Returns 

264 ------- 

265 raw : `str` 

266 Destination filename. 

267 """ 

268 raw = butler.get("%s_filename"%(filetype), info)[0] 

269 # Ensure filename is devoid of cfitsio directions about HDUs 

270 c = raw.find("[") 

271 if c > 0: 

272 raw = raw[:c] 

273 return raw