Coverage for python/lsst/obs/decam/ingest.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2012,2015 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import re
26from astro_metadata_translator import fix_header, DecamTranslator
27from lsst.afw.fits import readMetadata
28from lsst.pipe.tasks.ingest import ParseTask, IngestTask, IngestArgumentParser
29from lsst.obs.base.ingest import RawFileData
30import lsst.obs.base
32__all__ = ["DecamRawIngestTask", "DecamIngestArgumentParser", "DecamIngestTask", "DecamParseTask"]
35class DecamRawIngestTask(lsst.obs.base.RawIngestTask):
36 """Task for ingesting raw DECam data into a Gen3 Butler repository.
37 """
38 def extractMetadata(self, filename: str) -> RawFileData:
39 datasets = []
40 fitsData = lsst.afw.fits.Fits(filename, 'r')
41 # NOTE: The primary header (HDU=0) does not contain detector data.
42 for i in range(1, fitsData.countHdus()):
43 fitsData.setHdu(i)
44 header = fitsData.readMetadata()
45 if header['CCDNUM'] > 62: # ignore the guide CCDs
46 continue
47 fix_header(header)
48 datasets.append(self._calculate_dataset_info(header, filename))
50 # The data model currently assumes that whilst multiple datasets
51 # can be associated with a single file, they must all share the
52 # same formatter.
53 FormatterClass = self.instrument.getRawFormatter(datasets[0].dataId)
55 self.log.debug(f"Found images for {len(datasets)} detectors in {filename}")
56 return RawFileData(datasets=datasets, filename=filename,
57 FormatterClass=FormatterClass)
60class DecamIngestArgumentParser(IngestArgumentParser):
61 """Gen2 DECam ingest additional arguments.
62 """
64 def __init__(self, *args, **kwargs):
65 super(DecamIngestArgumentParser, self).__init__(*args, **kwargs)
66 self.add_argument("--filetype", default="raw", choices=["instcal", "raw"],
67 help="Data processing level of the files to be ingested")
70class DecamIngestTask(IngestTask):
71 """Gen2 DECam file ingest task.
72 """
73 ArgumentParser = DecamIngestArgumentParser
75 def __init__(self, *args, **kwargs):
76 super(DecamIngestTask, self).__init__(*args, **kwargs)
78 def run(self, args):
79 """Ingest all specified files and add them to the registry
80 """
81 if args.filetype == "instcal":
82 root = args.input
83 with self.register.openRegistry(root, create=args.create, dryrun=args.dryrun) as registry:
84 for infile in args.files:
85 fileInfo, hduInfoList = self.parse.getInfo(infile, args.filetype)
86 if len(hduInfoList) > 0:
87 outfileInstcal = os.path.join(root, self.parse.getDestination(args.butler,
88 hduInfoList[0],
89 infile, "instcal"))
90 outfileDqmask = os.path.join(root, self.parse.getDestination(args.butler,
91 hduInfoList[0], infile,
92 "dqmask"))
93 outfileWtmap = os.path.join(root, self.parse.getDestination(args.butler,
94 hduInfoList[0], infile,
95 "wtmap"))
97 ingestedInstcal = self.ingest(fileInfo["instcal"], outfileInstcal,
98 mode=args.mode, dryrun=args.dryrun)
99 ingestedDqmask = self.ingest(fileInfo["dqmask"], outfileDqmask,
100 mode=args.mode, dryrun=args.dryrun)
101 ingestedWtmap = self.ingest(fileInfo["wtmap"], outfileWtmap,
102 mode=args.mode, dryrun=args.dryrun)
104 if not (ingestedInstcal or ingestedDqmask or ingestedWtmap):
105 continue
107 for info in hduInfoList:
108 self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
110 elif args.filetype == "raw":
111 IngestTask.run(self, args)
114class DecamParseTask(ParseTask):
115 """Parse an image filename to get the required information to
116 put the file in the correct location and populate the registry.
117 """
119 def __init__(self, *args, **kwargs):
120 super(ParseTask, self).__init__(*args, **kwargs)
122 self.expnumMapper = None
124 # Note that these should be syncronized with the fields in
125 # root.register.columns defined in config/ingest.py
126 self.instcalPrefix = "instcal"
127 self.dqmaskPrefix = "dqmask"
128 self.wtmapPrefix = "wtmap"
130 def _listdir(self, path, prefix):
131 for file in os.listdir(path):
132 fileName = os.path.join(path, file)
133 md = readMetadata(fileName)
134 fix_header(md, translator_class=DecamTranslator)
135 if "EXPNUM" not in md.names():
136 return
137 expnum = md.getScalar("EXPNUM")
138 if expnum not in self.expnumMapper:
139 self.expnumMapper[expnum] = {self.instcalPrefix: None,
140 self.wtmapPrefix: None,
141 self.dqmaskPrefix: None}
142 self.expnumMapper[expnum][prefix] = fileName
144 def buildExpnumMapper(self, basepath):
145 """Extract exposure numbers from filenames to set self.expnumMapper
147 Parameters
148 ----------
149 basepath : `str`
150 Location on disk of instcal, dqmask, and wtmap subdirectories.
151 """
152 self.expnumMapper = {}
154 instcalPath = basepath
155 dqmaskPath = re.sub(self.instcalPrefix, self.dqmaskPrefix, instcalPath)
156 wtmapPath = re.sub(self.instcalPrefix, self.wtmapPrefix, instcalPath)
157 if instcalPath == dqmaskPath:
158 raise RuntimeError("instcal and mask directories are the same")
159 if instcalPath == wtmapPath:
160 raise RuntimeError("instcal and weight map directories are the same")
162 if not os.path.isdir(dqmaskPath):
163 raise OSError("Directory %s does not exist" % (dqmaskPath))
164 if not os.path.isdir(wtmapPath):
165 raise OSError("Directory %s does not exist" % (wtmapPath))
167 # Traverse each directory and extract the expnums
168 for path, prefix in zip((instcalPath, dqmaskPath, wtmapPath),
169 (self.instcalPrefix, self.dqmaskPrefix, self.wtmapPrefix)):
170 self._listdir(path, prefix)
172 def getInfo(self, filename, filetype="raw"):
173 """Get metadata header info from multi-extension FITS decam image file.
175 The science pixels, mask, and weight (inverse variance) are
176 stored in separate files each with a unique name but with a
177 common unique identifier EXPNUM in the FITS header. We have
178 to aggregate the 3 filenames for a given EXPNUM and return
179 this information along with that returned by the base class.
181 Parameters
182 ----------
183 filename : `str`
184 Image file to retrieve info from.
185 filetype : `str`
186 One of "raw" or "instcal".
188 Returns
189 -------
190 phuInfo : `dict`
191 Primary header unit info.
192 infoList : `list` of `dict`
193 Info for the other HDUs.
195 Notes
196 -----
197 For filetype="instcal", we expect a directory structure that looks
198 like the following:
200 .. code-block:: none
202 dqmask/
203 instcal/
204 wtmap/
206 The user creates the registry by running:
208 .. code-block:: none
210 ingestImagesDecam.py outputRepository --filetype=instcal --mode=link instcal/*fits
211 """
212 if filetype == "instcal":
213 if self.expnumMapper is None:
214 self.buildExpnumMapper(os.path.dirname(os.path.abspath(filename)))
216 # Note that phuInfo will have
217 # 'side': 'X', 'ccd': 0
218 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename)
219 expnum = phuInfo["visit"]
220 phuInfo[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix]
221 phuInfo[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix]
222 phuInfo[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix]
223 for info in infoList:
224 expnum = info["visit"]
225 info[self.instcalPrefix] = self.expnumMapper[expnum][self.instcalPrefix]
226 info[self.dqmaskPrefix] = self.expnumMapper[expnum][self.dqmaskPrefix]
227 info[self.wtmapPrefix] = self.expnumMapper[expnum][self.wtmapPrefix]
229 elif filetype == "raw":
230 phuInfo, infoList = super(DecamParseTask, self).getInfo(filename)
231 for info in infoList:
232 info[self.instcalPrefix] = ""
233 info[self.dqmaskPrefix] = ""
234 info[self.wtmapPrefix] = ""
236 # Some data IDs can not be extracted from the zeroth extension
237 # of the MEF. Add them so Butler does not try to find them
238 # in the registry which may still yet to be created.
239 for key in ("ccdnum", "hdu", "ccd", "calib_hdu"):
240 if key not in phuInfo:
241 phuInfo[key] = 0
243 return phuInfo, infoList
245 @staticmethod
246 def getExtensionName(md):
247 return md.getScalar('EXTNAME')
249 def getDestination(self, butler, info, filename, filetype="raw"):
250 """Get destination for the file
252 Parameters
253 ----------
254 butler : `lsst.daf.persistence.Butler`
255 Data butler.
256 info : data ID
257 File properties, used as dataId for the butler.
258 filename : `str`
259 Input filename.
261 Returns
262 -------
263 raw : `str`
264 Destination filename.
265 """
266 raw = butler.get("%s_filename"%(filetype), info)[0]
267 # Ensure filename is devoid of cfitsio directions about HDUs
268 c = raw.find("[")
269 if c > 0:
270 raw = raw[:c]
271 return raw