Coverage for python/lsst/ap/pipe/ap_pipe.py : 28%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_pipe.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24__all__ = ["ApPipeConfig", "ApPipeTask"]
26import warnings
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
31from lsst.pipe.tasks.processCcd import ProcessCcdTask
32from lsst.pipe.tasks.imageDifference import ImageDifferenceTask
33from lsst.ap.association import DiaPipelineTask
34from lsst.ap.association.transformDiaSourceCatalog import TransformDiaSourceCatalogTask
35from lsst.ap.pipe.apPipeParser import ApPipeParser
36from lsst.ap.pipe.apPipeTaskRunner import ApPipeTaskRunner
39class ApPipeConfig(pexConfig.Config):
40 """Settings and defaults for ApPipeTask.
41 """
43 ccdProcessor = pexConfig.ConfigurableField(
44 target=ProcessCcdTask,
45 doc="Task used to perform basic image reduction and characterization.",
46 )
47 differencer = pexConfig.ConfigurableField(
48 target=ImageDifferenceTask,
49 doc="Task used to do image subtraction and DiaSource detection.",
50 )
51 transformDiaSrcCat = pexConfig.ConfigurableField(
52 target=TransformDiaSourceCatalogTask,
53 doc="Task for converting and calibrating the afw SourceCatalog of "
54 "DiaSources to Pandas DataFrame for use in Association."
55 )
56 diaPipe = pexConfig.ConfigurableField(
57 target=DiaPipelineTask,
58 doc="Pipeline task for loading/store DiaSources and DiaObjects and "
59 "spatially associating them.",
60 )
62 def setDefaults(self):
63 """Settings appropriate for most or all ap_pipe runs.
64 """
65 # Always prefer decorrelation; may eventually become ImageDifferenceTask default
66 self.differencer.doDecorrelation = True
67 self.differencer.detection.thresholdValue = 5.0 # needed with doDecorrelation
69 # Don't have source catalogs for templates
70 self.differencer.doSelectSources = False
72 # Write the WarpedExposure to disk for use in Alert Packet creation.
73 self.differencer.doWriteWarpedExp = True
75 def validate(self):
76 pexConfig.Config.validate(self)
77 if not self.ccdProcessor.calibrate.doWrite or not self.ccdProcessor.calibrate.doWriteExposure:
78 raise ValueError("Differencing needs calexps [ccdProcessor.calibrate.doWrite, doWriteExposure]")
79 if not self.differencer.doMeasurement:
80 raise ValueError("Source association needs diaSource fluxes [differencer.doMeasurement].")
81 if not self.differencer.doWriteSources:
82 raise ValueError("Source association needs diaSource catalogs [differencer.doWriteSources].")
83 if not self.differencer.doWriteSubtractedExp:
84 raise ValueError("Source association needs difference exposures "
85 "[differencer.doWriteSubtractedExp].")
88class ApPipeTask(pipeBase.CmdLineTask):
89 """Command-line task representing the entire AP pipeline.
91 ``ApPipeTask`` processes raw DECam images from basic processing through
92 source association. Other observatories will be supported in the future.
94 ``ApPipeTask`` can be run from the command line, but it can also be called
95 from other pipeline code. It provides public methods for executing each
96 major step of the pipeline by itself.
98 Parameters
99 ----------
100 butler : `lsst.daf.persistence.Butler`
101 A Butler providing access to the science, calibration, and (unless
102 ``config.differencer.getTemplate`` is overridden) template data to
103 be processed. Its output repository must be both readable
104 and writable.
105 """
107 ConfigClass = ApPipeConfig
108 RunnerClass = ApPipeTaskRunner
109 _DefaultName = "apPipe"
111 def __init__(self, butler, *args, **kwargs):
112 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
114 self.makeSubtask("ccdProcessor", butler=butler)
115 self.makeSubtask("differencer", butler=butler)
116 self.makeSubtask("transformDiaSrcCat", initInputs={"diaSourceSchema": self.differencer.outputSchema})
117 self.makeSubtask("diaPipe")
119 @pipeBase.timeMethod
120 def runDataRef(self, rawRef, templateIds=None, reuse=None):
121 """Execute the ap_pipe pipeline on a single image.
123 Parameters
124 ----------
125 rawRef : `lsst.daf.persistence.ButlerDataRef`
126 A reference to the raw data to process.
127 templateIds : `list` of `dict`, optional
128 A list of parsed data IDs for templates to use. Only used if
129 ``config.differencer`` is configured to do so. ``differencer`` or
130 its subtasks may restrict the allowed IDs.
131 reuse : `list` of `str`, optional
132 The names of all subtasks that may be skipped if their output is
133 present. Defaults to skipping nothing.
135 Returns
136 -------
137 result : `lsst.pipe.base.Struct`
138 Result struct with components:
140 - l1Database : handle for accessing the final association database, conforming to
141 `ap_association`'s DB access API
142 - ccdProcessor : output of `config.ccdProcessor.runDataRef` (`lsst.pipe.base.Struct` or `None`).
143 - differencer : output of `config.differencer.runDataRef` (`lsst.pipe.base.Struct` or `None`).
144 - diaPipe : output of `config.diaPipe.run` (`lsst.pipe.base.Struct` or `None`).
145 """
146 if reuse is None:
147 reuse = []
148 # Work around mismatched HDU lists for raw and processed data
149 calexpId = rawRef.dataId.copy()
150 if 'hdu' in calexpId:
151 del calexpId['hdu']
152 calexpRef = rawRef.getButler().dataRef("calexp", dataId=calexpId)
154 # Ensure that templateIds make it through basic data reduction
155 # TODO: treat as independent jobs (may need SuperTask framework?)
156 if templateIds is not None:
157 for templateId in templateIds:
158 # templateId is typically visit-only; consider only the same raft/CCD/etc. as rawRef
159 rawTemplateRef = _siblingRef(rawRef, "raw", templateId)
160 calexpTemplateRef = _siblingRef(calexpRef, "calexp", templateId)
161 if "ccdProcessor" not in reuse or not calexpTemplateRef.datasetExists("calexp"):
162 self.runProcessCcd(rawTemplateRef)
164 if "ccdProcessor" in reuse and calexpRef.datasetExists("calexp"):
165 self.log.info("ProcessCcd has already been run for {0}, skipping...".format(rawRef.dataId))
166 processResults = None
167 else:
168 processResults = self.runProcessCcd(rawRef)
170 diffType = self.config.differencer.coaddName
171 if "differencer" in reuse and calexpRef.datasetExists(diffType + "Diff_diaSrc"):
172 self.log.info("DiffIm has already been run for {0}, skipping...".format(calexpRef.dataId))
173 diffImResults = None
174 else:
175 diffImResults = self.runDiffIm(calexpRef, templateIds)
177 if "diaPipe" in reuse:
178 warnings.warn(
179 "Reusing association results for some images while rerunning "
180 "others may change the associations. If exact reproducibility "
181 "matters, please clear the association database and run "
182 "ap_pipe.py with --reuse-output-from=differencer to redo all "
183 "association results consistently.")
184 if "diaPipe" in reuse and calexpRef.datasetExists("apdb_marker"):
185 message = "DiaPipeline has already been run for {0}, skipping...".format(calexpRef.dataId)
186 self.log.info(message)
187 diaPipeResults = None
188 else:
189 diaPipeResults = self.runAssociation(calexpRef)
191 return pipeBase.Struct(
192 l1Database=self.diaPipe.apdb,
193 ccdProcessor=processResults if processResults else None,
194 differencer=diffImResults if diffImResults else None,
195 diaPipe=diaPipeResults.taskResults if diaPipeResults else None
196 )
198 @pipeBase.timeMethod
199 def runProcessCcd(self, sensorRef):
200 """Perform ISR with ingested images and calibrations via processCcd.
202 The output repository associated with ``sensorRef`` will be populated with the
203 usual post-ISR data (bkgd, calexp, icExp, icSrc, postISR).
205 Parameters
206 ----------
207 sensorRef : `lsst.daf.persistence.ButlerDataRef`
208 Data reference for raw data.
210 Returns
211 -------
212 result : `lsst.pipe.base.Struct`
213 Output of `config.ccdProcessor.runDataRef`.
215 Notes
216 -----
217 The input repository corresponding to ``sensorRef`` must already contain the refcats.
218 """
219 self.log.info("Running ProcessCcd...")
220 return self.ccdProcessor.runDataRef(sensorRef)
222 @pipeBase.timeMethod
223 def runDiffIm(self, sensorRef, templateIds=None):
224 """Do difference imaging with a template and a science image
226 The output repository associated with ``sensorRef`` will be populated with difference images
227 and catalogs of detected sources (diaSrc, diffexp, and metadata files)
229 Parameters
230 ----------
231 sensorRef : `lsst.daf.persistence.ButlerDataRef`
232 Data reference for multiple dataset types, both input and output.
233 templateIds : `list` of `dict`, optional
234 A list of parsed data IDs for templates to use. Only used if
235 ``config.differencer`` is configured to do so. ``differencer`` or
236 its subtasks may restrict the allowed IDs.
238 Returns
239 -------
240 result : `lsst.pipe.base.Struct`
241 Output of `config.differencer.runDataRef`.
242 """
243 self.log.info("Running ImageDifference...")
244 return self.differencer.runDataRef(sensorRef, templateIdList=templateIds)
246 @pipeBase.timeMethod
247 def runAssociation(self, sensorRef):
248 """Do source association.
250 This method writes an ``apdb_marker`` dataset once all changes related
251 to the current exposure have been committed.
253 Parameters
254 ----------
255 sensorRef : `lsst.daf.persistence.ButlerDataRef`
256 Data reference for multiple input dataset types.
258 Returns
259 -------
260 result : `lsst.pipe.base.Struct`
261 Result struct with components:
263 - apdb : `lsst.dax.apdb.Apdb` Initialized association database containing final association
264 results.
265 - taskResults : output of `config.diaPipe.run` (`lsst.pipe.base.Struct`).
266 """
267 diffType = self.config.differencer.coaddName
268 diffIm = sensorRef.get(diffType + "Diff_differenceExp")
270 transformResult = self.transformDiaSrcCat.run(
271 diaSourceCat=sensorRef.get(diffType + "Diff_diaSrc"),
272 diffIm=diffIm,
273 band=diffIm.getFilterLabel().bandLabel,
274 ccdVisitId=diffIm.getInfo().getVisitInfo().getExposureId())
276 results = self.diaPipe.run(
277 diaSourceTable=transformResult.diaSourceTable,
278 diffIm=sensorRef.get(diffType + "Diff_differenceExp"),
279 exposure=sensorRef.get("calexp"),
280 warpedExposure=sensorRef.get(diffType + "Diff_warpedExp"),
281 ccdExposureIdBits=sensorRef.get("ccdExposureId_bits"))
283 # apdb_marker triggers metrics processing; let them try to read
284 # something even if association failed
285 sensorRef.put(results.apdbMarker, "apdb_marker")
287 return pipeBase.Struct(
288 l1Database=self.diaPipe.apdb,
289 taskResults=results
290 )
292 @classmethod
293 def _makeArgumentParser(cls):
294 """A parser that can handle extra arguments for ap_pipe.
295 """
296 return ApPipeParser(name=cls._DefaultName)
299def _siblingRef(original, datasetType, dataId):
300 """Construct a new dataRef using an existing dataRef as a template.
302 The typical application is to construct a data ID that differs from an
303 existing ID in one or two keys, but is more specific than expanding a
304 partial data ID would be.
306 Parameters
307 ----------
308 original : `lsst.daf.persistence.ButlerDataRef`
309 A dataRef related to the desired one. Assumed to represent a unique dataset.
310 datasetType : `str`
311 The desired type of the new dataRef. Must be compatible
312 with ``original``.
313 dataId : `dict` from `str` to any
314 A possibly partial data ID for the new dataRef. Any properties left
315 unspecified shall be copied from ``original``.
317 Returns
318 -------
319 dataRef : `lsst.daf.persistence.ButlerDataRef`
320 A dataRef to the same butler as ``original``, but of type
321 ``datasetType`` and with data ID equivalent to
322 ``original.dataId.update(dataId)``.
323 """
324 butler = original.getButler()
325 return butler.dataRef(datasetType, dataId=original.dataId, **dataId)