Coverage for python/lsst/obs/decam/ingest.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2012,2015 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import re
26from astro_metadata_translator import fix_header, DecamTranslator
27from lsst.afw.fits import readMetadata
28from lsst.pipe.tasks.ingest import ParseTask, IngestTask, IngestArgumentParser
29from lsst.obs.base.ingest import RawFileData
30import lsst.obs.base
31from ._instrument import DarkEnergyCamera
33__all__ = ["DecamRawIngestTask", "DecamIngestArgumentParser", "DecamIngestTask", "DecamParseTask"]
36class DecamRawIngestTask(lsst.obs.base.RawIngestTask):
37 """Task for ingesting raw DECam data into a Gen3 Butler repository.
38 """
39 def extractMetadata(self, filename: str) -> RawFileData:
40 datasets = []
41 fitsData = lsst.afw.fits.Fits(filename, 'r')
42 # NOTE: The primary header (HDU=0) does not contain detector data.
43 for i in range(1, fitsData.countHdus()):
44 fitsData.setHdu(i)
45 header = fitsData.readMetadata()
46 if header['CCDNUM'] > 62: # ignore the guide CCDs
47 continue
48 fix_header(header)
49 datasets.append(self._calculate_dataset_info(header, filename))
51 # The data model currently assumes that whilst multiple datasets
52 # can be associated with a single file, they must all share the
53 # same formatter.
54 instrument = DarkEnergyCamera()
55 FormatterClass = instrument.getRawFormatter(datasets[0].dataId)
57 self.log.debug(f"Found images for {len(datasets)} detectors in {filename}")
58 return RawFileData(datasets=datasets, filename=filename,
59 FormatterClass=FormatterClass,
60 instrumentClass=type(instrument))
63class DecamIngestArgumentParser(IngestArgumentParser):
64 """Gen2 DECam ingest additional arguments.
65 """
67 def __init__(self, *args, **kwargs):
68 super(DecamIngestArgumentParser, self).__init__(*args, **kwargs)
69 self.add_argument("--filetype", default="raw", choices=["instcal", "raw"],
70 help="Data processing level of the files to be ingested")
73class DecamIngestTask(IngestTask):
74 """Gen2 DECam file ingest task.
75 """
76 ArgumentParser = DecamIngestArgumentParser
78 def __init__(self, *args, **kwargs):
79 super(DecamIngestTask, self).__init__(*args, **kwargs)
81 def run(self, args):
82 """Ingest all specified files and add them to the registry
83 """
84 if args.filetype == "instcal":
85 root = args.input
86 with self.register.openRegistry(root, create=args.create, dryrun=args.dryrun) as registry:
87 for infile in args.files:
88 fileInfo, hduInfoList = self.parse.getInfo(infile, args.filetype)
89 if len(hduInfoList) > 0:
90 outfileInstcal = os.path.join(root, self.parse.getDestination(args.butler,
91 hduInfoList[0],
92 infile, "instcal"))
93 outfileDqmask = os.path.join(root, self.parse.getDestination(args.butler,
94 hduInfoList[0], infile,
95 "dqmask"))
96 outfileWtmap = os.path.join(root, self.parse.getDestination(args.butler,
97 hduInfoList[0], infile,
98 "wtmap"))
100 ingestedInstcal = self.ingest(fileInfo["instcal"], outfileInstcal,
101 mode=args.mode, dryrun=args.dryrun)
102 ingestedDqmask = self.ingest(fileInfo["dqmask"], outfileDqmask,
103 mode=args.mode, dryrun=args.dryrun)
104 ingestedWtmap = self.ingest(fileInfo["wtmap"], outfileWtmap,
105 mode=args.mode, dryrun=args.dryrun)
107 if not (ingestedInstcal or ingestedDqmask or ingestedWtmap):
108 continue
110 for info in hduInfoList:
111 self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
113 elif args.filetype == "raw":
114 IngestTask.run(self, args)
117class DecamParseTask(ParseTask):
118 """Parse an image filename to get the required information to
119 put the file in the correct location and populate the registry.
120 """
122 def __init__(self, *args, **kwargs):
123 super(ParseTask, self).__init__(*args, **kwargs)
125 self.expnumMapper = None
127 # Note that these should be syncronized with the fields in
128 # root.register.columns defined in config/ingest.py
129 self.instcalPrefix = "instcal"
130 self.dqmaskPrefix = "dqmask"
131 self.wtmapPrefix = "wtmap"
133 def _listdir(self, path, prefix):
134 for file in os.listdir(path):
135 fileName = os.path.join(path, file)
136 md = readMetadata(fileName)
137 fix_header(md, translator_class=DecamTranslator)
138 if "EXPNUM" not in md.names():
139 return
140 expnum = md.getScalar("EXPNUM")
141 if expnum not in self.expnumMapper:
142 self.expnumMapper[expnum] = {self.instcalPrefix: None,
143 self.wtmapPrefix: None,
144 self.dqmaskPrefix: None}
145 self.expnumMapper[expnum][prefix] = fileName
147 def buildExpnumMapper(self, basepath):
148 """Extract exposure numbers from filenames to set self.expnumMapper
150 Parameters
151 ----------
152 basepath : `str`
153 Location on disk of instcal, dqmask, and wtmap subdirectories.
154 """
155 self.expnumMapper = {}
157 instcalPath = basepath
158 dqmaskPath = re.sub(self.instcalPrefix, self.dqmaskPrefix, instcalPath)
159 wtmapPath = re.sub(self.instcalPrefix, self.wtmapPrefix, instcalPath)
160 if instcalPath == dqmaskPath:
161 raise RuntimeError("instcal and mask directories are the same")
162 if instcalPath == wtmapPath:
163 raise RuntimeError("instcal and weight map directories are the same")
165 if not os.path.isdir(dqmaskPath):
166 raise OSError("Directory %s does not exist" % (dqmaskPath))
167 if not os.path.isdir(wtmapPath):
168 raise OSError("Directory %s does not exist" % (wtmapPath))
170 # Traverse each directory and extract the expnums
171 for path, prefix in zip((instcalPath, dqmaskPath, wtmapPath),
172 (self.instcalPrefix, self.dqmaskPrefix, self.wtmapPrefix)):
173 self._listdir(path, prefix)
175 def getInfo(self, filename, filetype="raw"):
176 """Get metadata header info from multi-extension FITS decam image file.
178 The science pixels, mask, and weight (inverse variance) are
179 stored in separate files each with a unique name but with a
180 common unique identifier EXPNUM in the FITS header. We have
181 to aggregate the 3 filenames for a given EXPNUM and return
182 this information along with that returned by the base class.
184 Parameters
185 ----------
186 filename : `str`
187 Image file to retrieve info from.
188 filetype : `str`
189 One of "raw" or "instcal".
191 Returns
192 -------
193 phuInfo : `dict`
194 Primary header unit info.
195 infoList : `list` of `dict`
196 Info for the other HDUs.
198 Notes
199 -----
200 For filetype="instcal", we expect a directory structure that looks
201 like the following:
203 .. code-block:: none
205 dqmask/
206 instcal/
207 wtmap/
209 The user creates the registry by running:
211 .. code-block:: none
213 ingestImagesDecam.py outputRepository --filetype=instcal --mode=link instcal/*fits
214 """
215 if filetype == "instcal":
216 if self.expnumMapper is None:
217 self.buildExpnumMapper(os.path.dirname(os.path.abspath(filename)))
219 # Note that phuInfo will have
220 # 'side': 'X', 'ccd': 0
221 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename)
222 expnum = phuInfo["visit"]
223 phuInfo[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix]
224 phuInfo[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix]
225 phuInfo[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix]
226 for info in infoList:
227 expnum = info["visit"]
228 info[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix]
229 info[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix]
230 info[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix]
232 elif filetype == "raw":
233 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename)
234 for info in infoList:
235 info[self.instcalPrefix] = ""
236 info[self.dqmaskPrefix] = ""
237 info[self.wtmapPrefix] = ""
239 # Some data IDs can not be extracted from the zeroth extension
240 # of the MEF. Add them so Butler does not try to find them
241 # in the registry which may still yet to be created.
242 for key in ("ccdnum", "hdu", "ccd", "calib_hdu"):
243 if key not in phuInfo:
244 phuInfo[key] = 0
246 return phuInfo, infoList
248 def getDestination(self, butler, info, filename, filetype="raw"):
249 """Get destination for the file
251 Parameters
252 ----------
253 butler : `lsst.daf.persistence.Butler`
254 Data butler.
255 info : data ID
256 File properties, used as dataId for the butler.
257 filename : `str`
258 Input filename.
260 Returns
261 -------
262 raw : `str`
263 Destination filename.
264 """
265 raw = butler.get("%s_filename"%(filetype), info)[0]
266 # Ensure filename is devoid of cfitsio directions about HDUs
267 c = raw.find("[")
268 if c > 0:
269 raw = raw[:c]
270 return raw