Coverage for python/lsst/ap/pipe/ap_pipe.py : 25%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_pipe.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24__all__ = ["ApPipeConfig", "ApPipeTask"]
26import warnings
28from sqlalchemy.exc import OperationalError, ProgrammingError
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
33from lsst.pipe.tasks.processCcd import ProcessCcdTask
34from lsst.pipe.tasks.imageDifference import ImageDifferenceTask
35from lsst.ap.association import DiaPipelineTask
36from lsst.ap.pipe.apPipeParser import ApPipeParser
37from lsst.ap.pipe.apPipeTaskRunner import ApPipeTaskRunner
40class ApPipeConfig(pexConfig.Config):
41 """Settings and defaults for ApPipeTask.
42 """
44 ccdProcessor = pexConfig.ConfigurableField(
45 target=ProcessCcdTask,
46 doc="Task used to perform basic image reduction and characterization.",
47 )
48 differencer = pexConfig.ConfigurableField(
49 target=ImageDifferenceTask,
50 doc="Task used to do image subtraction and DiaSource detection.",
51 )
52 diaPipe = pexConfig.ConfigurableField(
53 target=DiaPipelineTask,
54 doc="Pipeline task for loading/store DiaSources and DiaObjects and "
55 "spatially associating them.",
56 )
58 def setDefaults(self):
59 """Settings appropriate for most or all ap_pipe runs.
60 """
61 # Always prefer decorrelation; may eventually become ImageDifferenceTask default
62 self.differencer.doDecorrelation = True
63 self.differencer.detection.thresholdValue = 5.0 # needed with doDecorrelation
65 # Don't have source catalogs for templates
66 self.differencer.doSelectSources = False
68 # Write the WarpedExposure to disk for use in Alert Packet creation.
69 self.differencer.doWriteWarpedExp = True
71 def validate(self):
72 pexConfig.Config.validate(self)
73 if not self.ccdProcessor.calibrate.doWrite or not self.ccdProcessor.calibrate.doWriteExposure:
74 raise ValueError("Differencing needs calexps [ccdProcessor.calibrate.doWrite, doWriteExposure]")
75 if not self.differencer.doMeasurement:
76 raise ValueError("Source association needs diaSource fluxes [differencer.doMeasurement].")
77 if not self.differencer.doWriteSources:
78 raise ValueError("Source association needs diaSource catalogs [differencer.doWriteSources].")
79 if not self.differencer.doWriteSubtractedExp:
80 raise ValueError("Source association needs difference exposures "
81 "[differencer.doWriteSubtractedExp].")
84class ApPipeTask(pipeBase.CmdLineTask):
85 """Command-line task representing the entire AP pipeline.
87 ``ApPipeTask`` processes raw DECam images from basic processing through
88 source association. Other observatories will be supported in the future.
90 ``ApPipeTask`` can be run from the command line, but it can also be called
91 from other pipeline code. It provides public methods for executing each
92 major step of the pipeline by itself.
94 Parameters
95 ----------
96 butler : `lsst.daf.persistence.Butler`
97 A Butler providing access to the science, calibration, and (unless
98 ``config.differencer.getTemplate`` is overridden) template data to
99 be processed. Its output repository must be both readable
100 and writable.
101 """
103 ConfigClass = ApPipeConfig
104 RunnerClass = ApPipeTaskRunner
105 _DefaultName = "apPipe"
107 def __init__(self, butler, *args, **kwargs):
108 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
110 self.makeSubtask("ccdProcessor", butler=butler)
111 self.makeSubtask("differencer", butler=butler)
112 self.makeSubtask("diaPipe", initInputs={"diaSourceSchema": self.differencer.outputSchema})
114 @pipeBase.timeMethod
115 def runDataRef(self, rawRef, templateIds=None, reuse=None):
116 """Execute the ap_pipe pipeline on a single image.
118 Parameters
119 ----------
120 rawRef : `lsst.daf.persistence.ButlerDataRef`
121 A reference to the raw data to process.
122 templateIds : `list` of `dict`, optional
123 A list of parsed data IDs for templates to use. Only used if
124 ``config.differencer`` is configured to do so. ``differencer`` or
125 its subtasks may restrict the allowed IDs.
126 reuse : `list` of `str`, optional
127 The names of all subtasks that may be skipped if their output is
128 present. Defaults to skipping nothing.
130 Returns
131 -------
132 result : `lsst.pipe.base.Struct`
133 Result struct with components:
135 - l1Database : handle for accessing the final association database, conforming to
136 `ap_association`'s DB access API
137 - ccdProcessor : output of `config.ccdProcessor.runDataRef` (`lsst.pipe.base.Struct` or `None`).
138 - differencer : output of `config.differencer.runDataRef` (`lsst.pipe.base.Struct` or `None`).
139 - diaPipe : output of `config.diaPipe.run` (`lsst.pipe.base.Struct` or `None`).
140 """
141 if reuse is None:
142 reuse = []
143 # Work around mismatched HDU lists for raw and processed data
144 calexpId = rawRef.dataId.copy()
145 if 'hdu' in calexpId:
146 del calexpId['hdu']
147 calexpRef = rawRef.getButler().dataRef("calexp", dataId=calexpId)
149 # Ensure that templateIds make it through basic data reduction
150 # TODO: treat as independent jobs (may need SuperTask framework?)
151 if templateIds is not None:
152 for templateId in templateIds:
153 # templateId is typically visit-only; consider only the same raft/CCD/etc. as rawRef
154 rawTemplateRef = _siblingRef(rawRef, "raw", templateId)
155 calexpTemplateRef = _siblingRef(calexpRef, "calexp", templateId)
156 if "ccdProcessor" not in reuse or not calexpTemplateRef.datasetExists("calexp", write=True):
157 self.runProcessCcd(rawTemplateRef)
159 if "ccdProcessor" in reuse and calexpRef.datasetExists("calexp", write=True):
160 self.log.info("ProcessCcd has already been run for {0}, skipping...".format(rawRef.dataId))
161 processResults = None
162 else:
163 processResults = self.runProcessCcd(rawRef)
165 diffType = self.config.differencer.coaddName
166 if "differencer" in reuse and calexpRef.datasetExists(diffType + "Diff_diaSrc", write=True):
167 self.log.info("DiffIm has already been run for {0}, skipping...".format(calexpRef.dataId))
168 diffImResults = None
169 else:
170 diffImResults = self.runDiffIm(calexpRef, templateIds)
172 try:
173 if "diaPipe" in reuse:
174 warnings.warn(
175 "Reusing association results for some images while rerunning "
176 "others may change the associations. If exact reproducibility "
177 "matters, please clear the association database and run "
178 "ap_pipe.py with --reuse-output-from=differencer to redo all "
179 "association results consistently.")
180 if "diaPipe" in reuse and calexpRef.datasetExists("apdb_marker", write=True):
181 message = "DiaPipeline has already been run for {0}, skipping...".format(calexpRef.dataId)
182 self.log.info(message)
183 diaPipeResults = None
184 else:
185 diaPipeResults = self.runAssociation(calexpRef)
186 except (OperationalError, ProgrammingError) as e:
187 # Don't use lsst.pipe.base.TaskError because it mixes poorly with exception chaining
188 raise RuntimeError("Database query failed; did you call make_apdb.py first?") from e
190 return pipeBase.Struct(
191 l1Database=self.diaPipe.apdb,
192 ccdProcessor=processResults if processResults else None,
193 differencer=diffImResults if diffImResults else None,
194 diaPipe=diaPipeResults.taskResults if diaPipeResults else None
195 )
197 @pipeBase.timeMethod
198 def runProcessCcd(self, sensorRef):
199 """Perform ISR with ingested images and calibrations via processCcd.
201 The output repository associated with ``sensorRef`` will be populated with the
202 usual post-ISR data (bkgd, calexp, icExp, icSrc, postISR).
204 Parameters
205 ----------
206 sensorRef : `lsst.daf.persistence.ButlerDataRef`
207 Data reference for raw data.
209 Returns
210 -------
211 result : `lsst.pipe.base.Struct`
212 Output of `config.ccdProcessor.runDataRef`.
214 Notes
215 -----
216 The input repository corresponding to ``sensorRef`` must already contain the refcats.
217 """
218 self.log.info("Running ProcessCcd...")
219 return self.ccdProcessor.runDataRef(sensorRef)
221 @pipeBase.timeMethod
222 def runDiffIm(self, sensorRef, templateIds=None):
223 """Do difference imaging with a template and a science image
225 The output repository associated with ``sensorRef`` will be populated with difference images
226 and catalogs of detected sources (diaSrc, diffexp, and metadata files)
228 Parameters
229 ----------
230 sensorRef : `lsst.daf.persistence.ButlerDataRef`
231 Data reference for multiple dataset types, both input and output.
232 templateIds : `list` of `dict`, optional
233 A list of parsed data IDs for templates to use. Only used if
234 ``config.differencer`` is configured to do so. ``differencer`` or
235 its subtasks may restrict the allowed IDs.
237 Returns
238 -------
239 result : `lsst.pipe.base.Struct`
240 Output of `config.differencer.runDataRef`.
241 """
242 self.log.info("Running ImageDifference...")
243 return self.differencer.runDataRef(sensorRef, templateIdList=templateIds)
245 @pipeBase.timeMethod
246 def runAssociation(self, sensorRef):
247 """Do source association.
249 This method writes an ``apdb_marker`` dataset once all changes related
250 to the current exposure have been committed.
252 Parameters
253 ----------
254 sensorRef : `lsst.daf.persistence.ButlerDataRef`
255 Data reference for multiple input dataset types.
257 Returns
258 -------
259 result : `lsst.pipe.base.Struct`
260 Result struct with components:
262 - apdb : `lsst.dax.apdb.Apdb` Initialized association database containing final association
263 results.
264 - taskResults : output of `config.diaPipe.run` (`lsst.pipe.base.Struct`).
265 """
266 diffType = self.config.differencer.coaddName
268 results = self.diaPipe.run(
269 diaSourceCat=sensorRef.get(diffType + "Diff_diaSrc"),
270 diffIm=sensorRef.get(diffType + "Diff_differenceExp"),
271 exposure=sensorRef.get("calexp"),
272 template=sensorRef.get(diffType + "Diff_warpedExp"),
273 ccdExposureIdBits=sensorRef.get("ccdExposureId_bits"))
275 # apdb_marker triggers metrics processing; let them try to read
276 # something even if association failed
277 sensorRef.put(results.apdbMarker, "apdb_marker")
279 return pipeBase.Struct(
280 l1Database=self.diaPipe.apdb,
281 taskResults=results
282 )
284 @classmethod
285 def _makeArgumentParser(cls):
286 """A parser that can handle extra arguments for ap_pipe.
287 """
288 return ApPipeParser(name=cls._DefaultName)
291def _siblingRef(original, datasetType, dataId):
292 """Construct a new dataRef using an existing dataRef as a template.
294 The typical application is to construct a data ID that differs from an
295 existing ID in one or two keys, but is more specific than expanding a
296 partial data ID would be.
298 Parameters
299 ----------
300 original : `lsst.daf.persistence.ButlerDataRef`
301 A dataRef related to the desired one. Assumed to represent a unique dataset.
302 datasetType : `str`
303 The desired type of the new dataRef. Must be compatible
304 with ``original``.
305 dataId : `dict` from `str` to any
306 A possibly partial data ID for the new dataRef. Any properties left
307 unspecified shall be copied from ``original``.
309 Returns
310 -------
311 dataRef : `lsst.daf.persistence.ButlerDataRef`
312 A dataRef to the same butler as ``original``, but of type
313 ``datasetType`` and with data ID equivalent to
314 ``original.dataId.update(dataId)``.
315 """
316 butler = original.getButler()
317 return butler.dataRef(datasetType, dataId=original.dataId, **dataId)