Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2012,2015 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import re 

24 

25 

26from astro_metadata_translator import fix_header, DecamTranslator 

27from lsst.afw.fits import readMetadata 

28from lsst.pipe.tasks.ingest import ParseTask, IngestTask, IngestArgumentParser 

29from lsst.obs.base.ingest import RawFileData 

30import lsst.obs.base 

31from ._instrument import DarkEnergyCamera 

32 

33__all__ = ["DecamRawIngestTask", "DecamIngestArgumentParser", "DecamIngestTask", "DecamParseTask"] 

34 

35 

36class DecamRawIngestTask(lsst.obs.base.RawIngestTask): 

37 """Task for ingesting raw DECam data into a Gen3 Butler repository. 

38 """ 

39 def extractMetadata(self, filename: str) -> RawFileData: 

40 datasets = [] 

41 fitsData = lsst.afw.fits.Fits(filename, 'r') 

42 # NOTE: The primary header (HDU=0) does not contain detector data. 

43 for i in range(1, fitsData.countHdus()): 

44 fitsData.setHdu(i) 

45 header = fitsData.readMetadata() 

46 if header['CCDNUM'] > 62: # ignore the guide CCDs 

47 continue 

48 fix_header(header) 

49 datasets.append(self._calculate_dataset_info(header, filename)) 

50 

51 # The data model currently assumes that whilst multiple datasets 

52 # can be associated with a single file, they must all share the 

53 # same formatter. 

54 instrument = DarkEnergyCamera() 

55 FormatterClass = instrument.getRawFormatter(datasets[0].dataId) 

56 

57 self.log.debug(f"Found images for {len(datasets)} detectors in {filename}") 

58 return RawFileData(datasets=datasets, filename=filename, 

59 FormatterClass=FormatterClass, 

60 instrumentClass=type(instrument)) 

61 

62 

63class DecamIngestArgumentParser(IngestArgumentParser): 

64 """Gen2 DECam ingest additional arguments. 

65 """ 

66 

67 def __init__(self, *args, **kwargs): 

68 super(DecamIngestArgumentParser, self).__init__(*args, **kwargs) 

69 self.add_argument("--filetype", default="raw", choices=["instcal", "raw"], 

70 help="Data processing level of the files to be ingested") 

71 

72 

73class DecamIngestTask(IngestTask): 

74 """Gen2 DECam file ingest task. 

75 """ 

76 ArgumentParser = DecamIngestArgumentParser 

77 

78 def __init__(self, *args, **kwargs): 

79 super(DecamIngestTask, self).__init__(*args, **kwargs) 

80 

81 def run(self, args): 

82 """Ingest all specified files and add them to the registry 

83 """ 

84 if args.filetype == "instcal": 

85 root = args.input 

86 with self.register.openRegistry(root, create=args.create, dryrun=args.dryrun) as registry: 

87 for infile in args.files: 

88 fileInfo, hduInfoList = self.parse.getInfo(infile, args.filetype) 

89 if len(hduInfoList) > 0: 

90 outfileInstcal = os.path.join(root, self.parse.getDestination(args.butler, 

91 hduInfoList[0], 

92 infile, "instcal")) 

93 outfileDqmask = os.path.join(root, self.parse.getDestination(args.butler, 

94 hduInfoList[0], infile, 

95 "dqmask")) 

96 outfileWtmap = os.path.join(root, self.parse.getDestination(args.butler, 

97 hduInfoList[0], infile, 

98 "wtmap")) 

99 

100 ingestedInstcal = self.ingest(fileInfo["instcal"], outfileInstcal, 

101 mode=args.mode, dryrun=args.dryrun) 

102 ingestedDqmask = self.ingest(fileInfo["dqmask"], outfileDqmask, 

103 mode=args.mode, dryrun=args.dryrun) 

104 ingestedWtmap = self.ingest(fileInfo["wtmap"], outfileWtmap, 

105 mode=args.mode, dryrun=args.dryrun) 

106 

107 if not (ingestedInstcal or ingestedDqmask or ingestedWtmap): 

108 continue 

109 

110 for info in hduInfoList: 

111 self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create) 

112 

113 elif args.filetype == "raw": 

114 IngestTask.run(self, args) 

115 

116 

117class DecamParseTask(ParseTask): 

118 """Parse an image filename to get the required information to 

119 put the file in the correct location and populate the registry. 

120 """ 

121 

122 def __init__(self, *args, **kwargs): 

123 super(ParseTask, self).__init__(*args, **kwargs) 

124 

125 self.expnumMapper = None 

126 

127 # Note that these should be syncronized with the fields in 

128 # root.register.columns defined in config/ingest.py 

129 self.instcalPrefix = "instcal" 

130 self.dqmaskPrefix = "dqmask" 

131 self.wtmapPrefix = "wtmap" 

132 

133 def _listdir(self, path, prefix): 

134 for file in os.listdir(path): 

135 fileName = os.path.join(path, file) 

136 md = readMetadata(fileName) 

137 fix_header(md, translator_class=DecamTranslator) 

138 if "EXPNUM" not in md.names(): 

139 return 

140 expnum = md.getScalar("EXPNUM") 

141 if expnum not in self.expnumMapper: 

142 self.expnumMapper[expnum] = {self.instcalPrefix: None, 

143 self.wtmapPrefix: None, 

144 self.dqmaskPrefix: None} 

145 self.expnumMapper[expnum][prefix] = fileName 

146 

147 def buildExpnumMapper(self, basepath): 

148 """Extract exposure numbers from filenames to set self.expnumMapper 

149 

150 Parameters 

151 ---------- 

152 basepath : `str` 

153 Location on disk of instcal, dqmask, and wtmap subdirectories. 

154 """ 

155 self.expnumMapper = {} 

156 

157 instcalPath = basepath 

158 dqmaskPath = re.sub(self.instcalPrefix, self.dqmaskPrefix, instcalPath) 

159 wtmapPath = re.sub(self.instcalPrefix, self.wtmapPrefix, instcalPath) 

160 if instcalPath == dqmaskPath: 

161 raise RuntimeError("instcal and mask directories are the same") 

162 if instcalPath == wtmapPath: 

163 raise RuntimeError("instcal and weight map directories are the same") 

164 

165 if not os.path.isdir(dqmaskPath): 

166 raise OSError("Directory %s does not exist" % (dqmaskPath)) 

167 if not os.path.isdir(wtmapPath): 

168 raise OSError("Directory %s does not exist" % (wtmapPath)) 

169 

170 # Traverse each directory and extract the expnums 

171 for path, prefix in zip((instcalPath, dqmaskPath, wtmapPath), 

172 (self.instcalPrefix, self.dqmaskPrefix, self.wtmapPrefix)): 

173 self._listdir(path, prefix) 

174 

175 def getInfo(self, filename, filetype="raw"): 

176 """Get metadata header info from multi-extension FITS decam image file. 

177 

178 The science pixels, mask, and weight (inverse variance) are 

179 stored in separate files each with a unique name but with a 

180 common unique identifier EXPNUM in the FITS header. We have 

181 to aggregate the 3 filenames for a given EXPNUM and return 

182 this information along with that returned by the base class. 

183 

184 Parameters 

185 ---------- 

186 filename : `str` 

187 Image file to retrieve info from. 

188 filetype : `str` 

189 One of "raw" or "instcal". 

190 

191 Returns 

192 ------- 

193 phuInfo : `dict` 

194 Primary header unit info. 

195 infoList : `list` of `dict` 

196 Info for the other HDUs. 

197 

198 Notes 

199 ----- 

200 For filetype="instcal", we expect a directory structure that looks 

201 like the following: 

202 

203 .. code-block:: none 

204 

205 dqmask/ 

206 instcal/ 

207 wtmap/ 

208 

209 The user creates the registry by running: 

210 

211 .. code-block:: none 

212 

213 ingestImagesDecam.py outputRepository --filetype=instcal --mode=link instcal/*fits 

214 """ 

215 if filetype == "instcal": 

216 if self.expnumMapper is None: 

217 self.buildExpnumMapper(os.path.dirname(os.path.abspath(filename))) 

218 

219 # Note that phuInfo will have 

220 # 'side': 'X', 'ccd': 0 

221 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename) 

222 expnum = phuInfo["visit"] 

223 phuInfo[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] 

224 phuInfo[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] 

225 phuInfo[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] 

226 for info in infoList: 

227 expnum = info["visit"] 

228 info[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] 

229 info[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] 

230 info[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] 

231 

232 elif filetype == "raw": 

233 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename) 

234 for info in infoList: 

235 info[self.instcalPrefix] = "" 

236 info[self.dqmaskPrefix] = "" 

237 info[self.wtmapPrefix] = "" 

238 

239 # Some data IDs can not be extracted from the zeroth extension 

240 # of the MEF. Add them so Butler does not try to find them 

241 # in the registry which may still yet to be created. 

242 for key in ("ccdnum", "hdu", "ccd", "calib_hdu"): 

243 if key not in phuInfo: 

244 phuInfo[key] = 0 

245 

246 return phuInfo, infoList 

247 

248 def getDestination(self, butler, info, filename, filetype="raw"): 

249 """Get destination for the file 

250 

251 Parameters 

252 ---------- 

253 butler : `lsst.daf.persistence.Butler` 

254 Data butler. 

255 info : data ID 

256 File properties, used as dataId for the butler. 

257 filename : `str` 

258 Input filename. 

259 

260 Returns 

261 ------- 

262 raw : `str` 

263 Destination filename. 

264 """ 

265 raw = butler.get("%s_filename"%(filetype), info)[0] 

266 # Ensure filename is devoid of cfitsio directions about HDUs 

267 c = raw.find("[") 

268 if c > 0: 

269 raw = raw[:c] 

270 return raw