Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

# 

# LSST Data Management System 

# Copyright 2012,2015 LSST Corporation. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <http://www.lsstcorp.org/LegalNotices/>. 

# 

import os 

import re 

 

 

from astro_metadata_translator import fix_header, DecamTranslator 

from lsst.afw.fits import readMetadata 

from lsst.pipe.tasks.ingest import ParseTask, IngestTask, IngestArgumentParser 

from lsst.obs.base.ingest import RawFileData 

import lsst.obs.base 

 

__all__ = ["DecamRawIngestTask", "DecamIngestArgumentParser", "DecamIngestTask", "DecamParseTask"] 

 

 

class DecamRawIngestTask(lsst.obs.base.RawIngestTask): 

"""Task for ingesting raw DECam data into a Gen3 Butler repository. 

""" 

def extractMetadata(self, filename: str) -> RawFileData: 

datasets = [] 

fitsData = lsst.afw.fits.Fits(filename, 'r') 

# NOTE: The primary header (HDU=0) does not contain detector data. 

for i in range(1, fitsData.countHdus()): 

fitsData.setHdu(i) 

header = fitsData.readMetadata() 

if header['CCDNUM'] > 62: # ignore the guide CCDs 

continue 

fix_header(header) 

datasets.append(self._calculate_dataset_info(header, filename)) 

 

# The data model currently assumes that whilst multiple datasets 

# can be associated with a single file, they must all share the 

# same formatter. 

FormatterClass = self.instrument.getRawFormatter(datasets[0].dataId) 

 

self.log.debug(f"Found images for {len(datasets)} detectors in {filename}") 

return RawFileData(datasets=datasets, filename=filename, 

FormatterClass=FormatterClass) 

 

 

class DecamIngestArgumentParser(IngestArgumentParser): 

"""Gen2 DECam ingest additional arguments. 

""" 

 

def __init__(self, *args, **kwargs): 

super(DecamIngestArgumentParser, self).__init__(*args, **kwargs) 

self.add_argument("--filetype", default="raw", choices=["instcal", "raw"], 

help="Data processing level of the files to be ingested") 

 

 

class DecamIngestTask(IngestTask): 

"""Gen2 DECam file ingest task. 

""" 

ArgumentParser = DecamIngestArgumentParser 

 

def __init__(self, *args, **kwargs): 

super(DecamIngestTask, self).__init__(*args, **kwargs) 

 

def run(self, args): 

"""Ingest all specified files and add them to the registry 

""" 

if args.filetype == "instcal": 

root = args.input 

with self.register.openRegistry(root, create=args.create, dryrun=args.dryrun) as registry: 

for infile in args.files: 

fileInfo, hduInfoList = self.parse.getInfo(infile, args.filetype) 

if len(hduInfoList) > 0: 

outfileInstcal = os.path.join(root, self.parse.getDestination(args.butler, 

hduInfoList[0], 

infile, "instcal")) 

outfileDqmask = os.path.join(root, self.parse.getDestination(args.butler, 

hduInfoList[0], infile, 

"dqmask")) 

outfileWtmap = os.path.join(root, self.parse.getDestination(args.butler, 

hduInfoList[0], infile, 

"wtmap")) 

 

ingestedInstcal = self.ingest(fileInfo["instcal"], outfileInstcal, 

mode=args.mode, dryrun=args.dryrun) 

ingestedDqmask = self.ingest(fileInfo["dqmask"], outfileDqmask, 

mode=args.mode, dryrun=args.dryrun) 

ingestedWtmap = self.ingest(fileInfo["wtmap"], outfileWtmap, 

mode=args.mode, dryrun=args.dryrun) 

 

if not (ingestedInstcal or ingestedDqmask or ingestedWtmap): 

continue 

 

for info in hduInfoList: 

self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create) 

 

elif args.filetype == "raw": 

IngestTask.run(self, args) 

 

 

class DecamParseTask(ParseTask): 

"""Parse an image filename to get the required information to 

put the file in the correct location and populate the registry. 

""" 

 

def __init__(self, *args, **kwargs): 

super(ParseTask, self).__init__(*args, **kwargs) 

 

self.expnumMapper = None 

 

# Note that these should be syncronized with the fields in 

# root.register.columns defined in config/ingest.py 

self.instcalPrefix = "instcal" 

self.dqmaskPrefix = "dqmask" 

self.wtmapPrefix = "wtmap" 

 

def _listdir(self, path, prefix): 

for file in os.listdir(path): 

fileName = os.path.join(path, file) 

md = readMetadata(fileName) 

fix_header(md, translator_class=DecamTranslator) 

if "EXPNUM" not in md.names(): 

return 

expnum = md.getScalar("EXPNUM") 

if expnum not in self.expnumMapper: 

self.expnumMapper[expnum] = {self.instcalPrefix: None, 

self.wtmapPrefix: None, 

self.dqmaskPrefix: None} 

self.expnumMapper[expnum][prefix] = fileName 

 

def buildExpnumMapper(self, basepath): 

"""Extract exposure numbers from filenames to set self.expnumMapper 

 

Parameters 

---------- 

basepath : `str` 

Location on disk of instcal, dqmask, and wtmap subdirectories. 

""" 

self.expnumMapper = {} 

 

instcalPath = basepath 

dqmaskPath = re.sub(self.instcalPrefix, self.dqmaskPrefix, instcalPath) 

wtmapPath = re.sub(self.instcalPrefix, self.wtmapPrefix, instcalPath) 

if instcalPath == dqmaskPath: 

raise RuntimeError("instcal and mask directories are the same") 

if instcalPath == wtmapPath: 

raise RuntimeError("instcal and weight map directories are the same") 

 

if not os.path.isdir(dqmaskPath): 

raise OSError("Directory %s does not exist" % (dqmaskPath)) 

if not os.path.isdir(wtmapPath): 

raise OSError("Directory %s does not exist" % (wtmapPath)) 

 

# Traverse each directory and extract the expnums 

for path, prefix in zip((instcalPath, dqmaskPath, wtmapPath), 

(self.instcalPrefix, self.dqmaskPrefix, self.wtmapPrefix)): 

self._listdir(path, prefix) 

 

def getInfo(self, filename, filetype="raw"): 

"""Get metadata header info from multi-extension FITS decam image file. 

 

The science pixels, mask, and weight (inverse variance) are 

stored in separate files each with a unique name but with a 

common unique identifier EXPNUM in the FITS header. We have 

to aggregate the 3 filenames for a given EXPNUM and return 

this information along with that returned by the base class. 

 

Parameters 

---------- 

filename : `str` 

Image file to retrieve info from. 

filetype : `str` 

One of "raw" or "instcal". 

 

Returns 

------- 

phuInfo : `dict` 

Primary header unit info. 

infoList : `list` of `dict` 

Info for the other HDUs. 

 

Notes 

----- 

For filetype="instcal", we expect a directory structure that looks 

like the following: 

 

.. code-block:: none 

 

dqmask/ 

instcal/ 

wtmap/ 

 

The user creates the registry by running: 

 

.. code-block:: none 

 

ingestImagesDecam.py outputRepository --filetype=instcal --mode=link instcal/*fits 

""" 

if filetype == "instcal": 

if self.expnumMapper is None: 

self.buildExpnumMapper(os.path.dirname(os.path.abspath(filename))) 

 

# Note that phuInfo will have 

# 'side': 'X', 'ccd': 0 

phuInfo, infoList = super(DecamParseTask, self).getInfo(filename) 

expnum = phuInfo["visit"] 

phuInfo[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] 

phuInfo[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] 

phuInfo[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] 

for info in infoList: 

expnum = info["visit"] 

info[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix] 

info[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix] 

info[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix] 

 

elif filetype == "raw": 

phuInfo, infoList = super(DecamParseTask, self).getInfo(filename) 

for info in infoList: 

info[self.instcalPrefix] = "" 

info[self.dqmaskPrefix] = "" 

info[self.wtmapPrefix] = "" 

 

# Some data IDs can not be extracted from the zeroth extension 

# of the MEF. Add them so Butler does not try to find them 

# in the registry which may still yet to be created. 

for key in ("ccdnum", "hdu", "ccd", "calib_hdu"): 

if key not in phuInfo: 

phuInfo[key] = 0 

 

return phuInfo, infoList 

 

@staticmethod 

def getExtensionName(md): 

return md.getScalar('EXTNAME') 

 

def getDestination(self, butler, info, filename, filetype="raw"): 

"""Get destination for the file 

 

Parameters 

---------- 

butler : `lsst.daf.persistence.Butler` 

Data butler. 

info : data ID 

File properties, used as dataId for the butler. 

filename : `str` 

Input filename. 

 

Returns 

------- 

raw : `str` 

Destination filename. 

""" 

raw = butler.get("%s_filename"%(filetype), info)[0] 

# Ensure filename is devoid of cfitsio directions about HDUs 

c = raw.find("[") 

if c > 0: 

raw = raw[:c] 

return raw