Coverage for python/lsst/ap/association/diaPipe.py : 39%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23"""PipelineTask for associating DiaSources with previous DiaObjects.
25Additionally performs forced photometry on the calibrated and difference
26images at the updated locations of DiaObjects.
28Currently loads directly from the Apdb rather than pre-loading.
29"""
31import os
33import lsst.dax.apdb as daxApdb
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as connTypes
37from lsst.utils import getPackageDir
39from lsst.ap.association import (
40 AssociationTask,
41 DiaForcedSourceTask,
42 LoadDiaCatalogsTask,
43 make_dia_object_schema,
44 make_dia_source_schema,
45 PackageAlertsTask)
47__all__ = ("DiaPipelineConfig",
48 "DiaPipelineTask",
49 "DiaPipelineConnections")
52class DiaPipelineConnections(
53 pipeBase.PipelineTaskConnections,
54 dimensions=("instrument", "visit", "detector"),
55 defaultTemplates={"coaddName": "deep", "fakesType": ""}):
56 """Butler connections for DiaPipelineTask.
57 """
58 diaSourceTable = connTypes.Input(
59 doc="Catalog of calibrated DiaSources.",
60 name="{fakesType}{coaddName}Diff_diaSrcTable",
61 storageClass="DataFrame",
62 dimensions=("instrument", "visit", "detector"),
63 )
64 diffIm = connTypes.Input(
65 doc="Difference image on which the DiaSources were detected.",
66 name="{fakesType}{coaddName}Diff_differenceExp",
67 storageClass="ExposureF",
68 dimensions=("instrument", "visit", "detector"),
69 )
70 exposure = connTypes.Input(
71 doc="Calibrated exposure differenced with a template image during "
72 "image differencing.",
73 name="calexp",
74 storageClass="ExposureF",
75 dimensions=("instrument", "visit", "detector"),
76 )
77 warpedExposure = connTypes.Input(
78 doc="Warped template used to create `subtractedExposure`. Not PSF "
79 "matched.",
80 dimensions=("instrument", "visit", "detector"),
81 storageClass="ExposureF",
82 name="{fakesType}{coaddName}Diff_warpedExp",
83 )
84 apdbMarker = connTypes.Output(
85 doc="Marker dataset storing the configuration of the Apdb for each "
86 "visit/detector. Used to signal the completion of the pipeline.",
87 name="apdb_marker",
88 storageClass="Config",
89 dimensions=("instrument", "visit", "detector"),
90 )
91 associatedDiaSources = connTypes.Output(
92 doc="Optional output storing the DiaSource catalog after matching, "
93 "calibration, and standardization for insertation into the Apdb.",
94 name="{fakesType}{coaddName}Diff_assocDiaSrc",
95 storageClass="DataFrame",
96 dimensions=("instrument", "visit", "detector"),
97 )
99 def __init__(self, *, config=None):
100 super().__init__(config=config)
102 if not config.doWriteAssociatedSources:
103 self.outputs.remove("associatedDiaSources")
105 def adjustQuantum(self, inputs, outputs, label, dataId):
106 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects
107 in the `lsst.daf.butler.core.Quantum` during the graph generation stage
108 of the activator.
110 This implementation checks to make sure that the filters in the dataset
111 are compatible with AP processing as set by the Apdb/DPDD schema.
113 Parameters
114 ----------
115 inputs : `dict`
116 Dictionary whose keys are an input (regular or prerequisite)
117 connection name and whose values are a tuple of the connection
118 instance and a collection of associated `DatasetRef` objects.
119 The exact type of the nested collections is unspecified; it can be
120 assumed to be multi-pass iterable and support `len` and ``in``, but
121 it should not be mutated in place. In contrast, the outer
122 dictionaries are guaranteed to be temporary copies that are true
123 `dict` instances, and hence may be modified and even returned; this
124 is especially useful for delegating to `super` (see notes below).
125 outputs : `dict`
126 Dict of output datasets, with the same structure as ``inputs``.
127 label : `str`
128 Label for this task in the pipeline (should be used in all
129 diagnostic messages).
130 data_id : `lsst.daf.butler.DataCoordinate`
131 Data ID for this quantum in the pipeline (should be used in all
132 diagnostic messages).
134 Returns
135 -------
136 adjusted_inputs : `dict`
137 Dict of the same form as ``inputs`` with updated containers of
138 input `DatasetRef` objects. Connections that are not changed
139 should not be returned at all. Datasets may only be removed, not
140 added. Nested collections may be of any multi-pass iterable type,
141 and the order of iteration will set the order of iteration within
142 `PipelineTask.runQuantum`.
143 adjusted_outputs : `dict`
144 Dict of updated output datasets, with the same structure and
145 interpretation as ``adjusted_inputs``.
147 Raises
148 ------
149 ScalarError
150 Raised if any `Input` or `PrerequisiteInput` connection has
151 ``multiple`` set to `False`, but multiple datasets.
152 NoWorkFound
153 Raised to indicate that this quantum should not be run; not enough
154 datasets were found for a regular `Input` connection, and the
155 quantum should be pruned or skipped.
156 FileNotFoundError
157 Raised to cause QuantumGraph generation to fail (with the message
158 included in this exception); not enough datasets were found for a
159 `PrerequisiteInput` connection.
160 """
161 _, refs = inputs["diffIm"]
162 for ref in refs:
163 if ref.dataId["band"] not in self.config.validBands:
164 raise ValueError(
165 f"Requested '{ref.dataId['band']}' not in "
166 "DiaPipelineConfig.validBands. To process bands not in "
167 "the standard Rubin set (ugrizy) you must add the band to "
168 "the validBands list in DiaPipelineConfig and add the "
169 "appropriate columns to the Apdb schema.")
170 return super().adjustQuantum(inputs, outputs, label, dataId)
173class DiaPipelineConfig(pipeBase.PipelineTaskConfig,
174 pipelineConnections=DiaPipelineConnections):
175 """Config for DiaPipelineTask.
176 """
177 coaddName = pexConfig.Field(
178 doc="coadd name: typically one of deep, goodSeeing, or dcr",
179 dtype=str,
180 default="deep",
181 )
182 apdb = pexConfig.ConfigurableField(
183 target=daxApdb.Apdb,
184 ConfigClass=daxApdb.ApdbConfig,
185 doc="Database connection for storing associated DiaSources and "
186 "DiaObjects. Must already be initialized.",
187 )
188 validBands = pexConfig.ListField(
189 dtype=str,
190 default=["u", "g", "r", "i", "z", "y"],
191 doc="List of bands that are valid for AP processing. To process a "
192 "band not on this list, the appropriate band specific columns "
193 "must be added to the Apdb schema in dax_apdb.",
194 )
195 diaCatalogLoader = pexConfig.ConfigurableField(
196 target=LoadDiaCatalogsTask,
197 doc="Task to load DiaObjects and DiaSources from the Apdb.",
198 )
199 associator = pexConfig.ConfigurableField(
200 target=AssociationTask,
201 doc="Task used to associate DiaSources with DiaObjects.",
202 )
203 diaForcedSource = pexConfig.ConfigurableField(
204 target=DiaForcedSourceTask,
205 doc="Task used for force photometer DiaObject locations in direct and "
206 "difference images.",
207 )
208 alertPackager = pexConfig.ConfigurableField(
209 target=PackageAlertsTask,
210 doc="Subtask for packaging Ap data into alerts.",
211 )
212 doPackageAlerts = pexConfig.Field(
213 dtype=bool,
214 default=False,
215 doc="Package Dia-data into serialized alerts for distribution and "
216 "write them to disk.",
217 )
218 doWriteAssociatedSources = pexConfig.Field(
219 dtype=bool,
220 default=False,
221 doc="Write out associated and SDMed DiaSources.",
222 )
224 def setDefaults(self):
225 self.apdb.dia_object_index = "baseline"
226 self.apdb.dia_object_columns = []
227 self.apdb.extra_schema_file = os.path.join(
228 getPackageDir("ap_association"),
229 "data",
230 "apdb-ap-pipe-schema-extra.yaml")
232 def validate(self):
233 pexConfig.Config.validate(self)
234 if self.diaCatalogLoader.htmLevel != \
235 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel:
236 raise ValueError("HTM index level in LoadDiaCatalogsTask must be "
237 "equal to HTMIndexDiaCalculationPlugin index "
238 "level.")
241class DiaPipelineTask(pipeBase.PipelineTask):
242 """Task for loading, associating and storing Difference Image Analysis
243 (DIA) Objects and Sources.
244 """
245 ConfigClass = DiaPipelineConfig
246 _DefaultName = "diaPipe"
247 RunnerClass = pipeBase.ButlerInitializedTaskRunner
249 def __init__(self, initInputs=None, **kwargs):
250 super().__init__(**kwargs)
251 self.apdb = self.config.apdb.apply(
252 afw_schemas=dict(DiaObject=make_dia_object_schema(),
253 DiaSource=make_dia_source_schema()))
254 self.makeSubtask("diaCatalogLoader")
255 self.makeSubtask("associator")
256 self.makeSubtask("diaForcedSource")
257 if self.config.doPackageAlerts:
258 self.makeSubtask("alertPackager")
260 def runQuantum(self, butlerQC, inputRefs, outputRefs):
261 inputs = butlerQC.get(inputRefs)
262 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
263 returnMaxBits=True)
264 inputs["ccdExposureIdBits"] = expBits
266 outputs = self.run(**inputs)
268 butlerQC.put(outputs, outputRefs)
270 @pipeBase.timeMethod
271 def run(self,
272 diaSourceTable,
273 diffIm,
274 exposure,
275 warpedExposure,
276 ccdExposureIdBits):
277 """Process DiaSources and DiaObjects.
279 Load previous DiaObjects and their DiaSource history. Calibrate the
280 values in the diaSourceCat. Associate new DiaSources with previous
281 DiaObjects. Run forced photometry at the updated DiaObject locations.
282 Store the results in the Alert Production Database (Apdb).
284 Parameters
285 ----------
286 diaSourceTable : `pandas.DataFrame`
287 Newly detected DiaSources.
288 diffIm : `lsst.afw.image.ExposureF`
289 Difference image exposure in which the sources in ``diaSourceCat``
290 were detected.
291 exposure : `lsst.afw.image.ExposureF`
292 Calibrated exposure differenced with a template to create
293 ``diffIm``.
294 warpedExposure : `lsst.afw.image.ExposureF`
295 Template exposure used to create diffIm.
296 ccdExposureIdBits : `int`
297 Number of bits used for a unique ``ccdVisitId``.
299 Returns
300 -------
301 results : `lsst.pipe.base.Struct`
302 Results struct with components.
304 - ``apdb_maker`` : Marker dataset to store in the Butler indicating
305 that this ccdVisit has completed successfully.
306 (`lsst.dax.apdb.ApdbConfig`)
307 """
308 self.log.info("Running DiaPipeline...")
309 # Put the SciencePipelines through a SDMification step and return
310 # calibrated columns with the expect output database names.
312 # Load the DiaObjects and DiaSource history.
313 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb)
315 # Associate new DiaSources with existing DiaObjects and update
316 # DiaObject summary statistics using the full DiaSource history.
317 assocResults = self.associator.run(diaSourceTable,
318 loaderResult.diaObjects,
319 loaderResult.diaSources)
321 # Force photometer on the Difference and Calibrated exposures using
322 # the new and updated DiaObject locations.
323 diaForcedSources = self.diaForcedSource.run(
324 assocResults.diaObjects,
325 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(),
326 ccdExposureIdBits,
327 exposure,
328 diffIm)
330 # Store DiaSources and updated DiaObjects in the Apdb.
331 self.apdb.storeDiaSources(assocResults.diaSources)
332 self.apdb.storeDiaObjects(
333 assocResults.updatedDiaObjects,
334 exposure.getInfo().getVisitInfo().getDate().toPython())
335 self.apdb.storeDiaForcedSources(diaForcedSources)
337 if self.config.doPackageAlerts:
338 if len(loaderResult.diaForcedSources) > 1:
339 diaForcedSources = diaForcedSources.append(
340 loaderResult.diaForcedSources,
341 sort=True)
342 if diaForcedSources.index.has_duplicates:
343 self.log.warn(
344 "Duplicate DiaForcedSources created after merge with "
345 "history and new sources. This may cause downstream "
346 "problems. Dropping duplicates.")
347 # Drop duplicates via index and keep the first appearance.
348 # Reset due to the index shape being slight different than
349 # expected.
350 diaForcedSources = diaForcedSources.groupby(
351 diaForcedSources.index).first()
352 diaForcedSources.reset_index(drop=True, inplace=True)
353 diaForcedSources.set_index(
354 ["diaObjectId", "diaForcedSourceId"],
355 drop=False,
356 inplace=True)
357 self.alertPackager.run(assocResults.diaSources,
358 assocResults.diaObjects,
359 loaderResult.diaSources,
360 diaForcedSources,
361 diffIm,
362 warpedExposure,
363 ccdExposureIdBits)
365 return pipeBase.Struct(apdbMarker=self.config.apdb.value,
366 associatedDiaSources=assocResults.diaSources)