Coverage for python/lsst/ap/association/diaPipe.py : 39%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23"""PipelineTask for associating DiaSources with previous DiaObjects.
25Additionally performs forced photometry on the calibrated and difference
26images at the updated locations of DiaObjects.
28Currently loads directly from the Apdb rather than pre-loading.
29"""
31import os
33import lsst.dax.apdb as daxApdb
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as connTypes
37from lsst.utils import getPackageDir
39from lsst.ap.association import (
40 AssociationTask,
41 DiaForcedSourceTask,
42 LoadDiaCatalogsTask,
43 make_dia_object_schema,
44 make_dia_source_schema,
45 PackageAlertsTask)
47__all__ = ("DiaPipelineConfig",
48 "DiaPipelineTask",
49 "DiaPipelineConnections")
52class DiaPipelineConnections(
53 pipeBase.PipelineTaskConnections,
54 dimensions=("instrument", "visit", "detector"),
55 defaultTemplates={"coaddName": "deep", "fakesType": ""}):
56 """Butler connections for DiaPipelineTask.
57 """
58 diaSourceTable = connTypes.Input(
59 doc="Catalog of calibrated DiaSources.",
60 name="{fakesType}{coaddName}Diff_diaSrcTable",
61 storageClass="DataFrame",
62 dimensions=("instrument", "visit", "detector"),
63 )
64 diffIm = connTypes.Input(
65 doc="Difference image on which the DiaSources were detected.",
66 name="{fakesType}{coaddName}Diff_differenceExp",
67 storageClass="ExposureF",
68 dimensions=("instrument", "visit", "detector"),
69 )
70 exposure = connTypes.Input(
71 doc="Calibrated exposure differenced with a template image during "
72 "image differencing.",
73 name="calexp",
74 storageClass="ExposureF",
75 dimensions=("instrument", "visit", "detector"),
76 )
77 warpedExposure = connTypes.Input(
78 doc="Warped template used to create `subtractedExposure`. Not PSF "
79 "matched.",
80 dimensions=("instrument", "visit", "detector"),
81 storageClass="ExposureF",
82 name="{fakesType}{coaddName}Diff_warpedExp",
83 )
84 apdbMarker = connTypes.Output(
85 doc="Marker dataset storing the configuration of the Apdb for each "
86 "visit/detector. Used to signal the completion of the pipeline.",
87 name="apdb_marker",
88 storageClass="Config",
89 dimensions=("instrument", "visit", "detector"),
90 )
91 associatedDiaSources = connTypes.Output(
92 doc="Optional output storing the DiaSource catalog after matching, "
93 "calibration, and standardization for insertation into the Apdb.",
94 name="{fakesType}{coaddName}Diff_assocDiaSrc",
95 storageClass="DataFrame",
96 dimensions=("instrument", "visit", "detector"),
97 )
99 def __init__(self, *, config=None):
100 super().__init__(config=config)
102 if not config.doWriteAssociatedSources:
103 self.outputs.remove("associatedDiaSources")
105 def adjustQuantum(self, datasetRefMap: pipeBase.InputQuantizedConnection):
106 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects
107 in the `lsst.daf.butler.core.Quantum` during the graph generation stage
108 of the activator.
110 This implementation checks to make sure that the filters in the dataset
111 are compatible with AP processing as set by the Apdb/DPDD schema.
113 Parameters
114 ----------
115 datasetRefMap : `NamedKeyDict`
116 Mapping from dataset type to a `set` of
117 `lsst.daf.butler.DatasetRef` objects
119 Returns
120 -------
121 datasetRefMap : `NamedKeyDict`
122 Mapping of input with assurances that bands incompatible with the
123 Apdb are present.
125 Raises
126 ------
127 ValueError
128 Raises if a data ref in the quantum has a band not available in the
129 Apdb.
130 """
131 refs = datasetRefMap[self.diffIm.name]
132 for ref in refs:
133 if ref.dataId["band"] not in self.config.validBands:
134 raise ValueError(
135 f"Requested '{ref.dataId['band']}' not in "
136 "DiaPipelineConfig.validBands. To process bands not in "
137 "the standard Rubin set (ugrizy) you must add the band to "
138 "the validBands list in DiaPipelineConfig and add the "
139 "appropriate columns to the Apdb schema.")
140 return super().adjustQuantum(datasetRefMap)
143class DiaPipelineConfig(pipeBase.PipelineTaskConfig,
144 pipelineConnections=DiaPipelineConnections):
145 """Config for DiaPipelineTask.
146 """
147 coaddName = pexConfig.Field(
148 doc="coadd name: typically one of deep, goodSeeing, or dcr",
149 dtype=str,
150 default="deep",
151 )
152 apdb = pexConfig.ConfigurableField(
153 target=daxApdb.Apdb,
154 ConfigClass=daxApdb.ApdbConfig,
155 doc="Database connection for storing associated DiaSources and "
156 "DiaObjects. Must already be initialized.",
157 )
158 validBands = pexConfig.ListField(
159 dtype=str,
160 default=["u", "g", "r", "i", "z", "y"],
161 doc="List of bands that are valid for AP processing. To process a "
162 "band not on this list, the appropriate band specific columns "
163 "must be added to the Apdb schema in dax_apdb.",
164 )
165 diaCatalogLoader = pexConfig.ConfigurableField(
166 target=LoadDiaCatalogsTask,
167 doc="Task to load DiaObjects and DiaSources from the Apdb.",
168 )
169 associator = pexConfig.ConfigurableField(
170 target=AssociationTask,
171 doc="Task used to associate DiaSources with DiaObjects.",
172 )
173 diaForcedSource = pexConfig.ConfigurableField(
174 target=DiaForcedSourceTask,
175 doc="Task used for force photometer DiaObject locations in direct and "
176 "difference images.",
177 )
178 alertPackager = pexConfig.ConfigurableField(
179 target=PackageAlertsTask,
180 doc="Subtask for packaging Ap data into alerts.",
181 )
182 doPackageAlerts = pexConfig.Field(
183 dtype=bool,
184 default=False,
185 doc="Package Dia-data into serialized alerts for distribution and "
186 "write them to disk.",
187 )
188 doWriteAssociatedSources = pexConfig.Field(
189 dtype=bool,
190 default=False,
191 doc="Write out associated and SDMed DiaSources.",
192 )
194 def setDefaults(self):
195 self.apdb.dia_object_index = "baseline"
196 self.apdb.dia_object_columns = []
197 self.apdb.extra_schema_file = os.path.join(
198 getPackageDir("ap_association"),
199 "data",
200 "apdb-ap-pipe-schema-extra.yaml")
202 def validate(self):
203 pexConfig.Config.validate(self)
204 if self.diaCatalogLoader.htmLevel != \
205 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel:
206 raise ValueError("HTM index level in LoadDiaCatalogsTask must be "
207 "equal to HTMIndexDiaCalculationPlugin index "
208 "level.")
211class DiaPipelineTask(pipeBase.PipelineTask):
212 """Task for loading, associating and storing Difference Image Analysis
213 (DIA) Objects and Sources.
214 """
215 ConfigClass = DiaPipelineConfig
216 _DefaultName = "diaPipe"
217 RunnerClass = pipeBase.ButlerInitializedTaskRunner
219 def __init__(self, initInputs=None, **kwargs):
220 super().__init__(**kwargs)
221 self.apdb = self.config.apdb.apply(
222 afw_schemas=dict(DiaObject=make_dia_object_schema(),
223 DiaSource=make_dia_source_schema()))
224 self.makeSubtask("diaCatalogLoader")
225 self.makeSubtask("associator")
226 self.makeSubtask("diaForcedSource")
227 if self.config.doPackageAlerts:
228 self.makeSubtask("alertPackager")
230 def runQuantum(self, butlerQC, inputRefs, outputRefs):
231 inputs = butlerQC.get(inputRefs)
232 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
233 returnMaxBits=True)
234 inputs["ccdExposureIdBits"] = expBits
236 outputs = self.run(**inputs)
238 butlerQC.put(outputs, outputRefs)
240 @pipeBase.timeMethod
241 def run(self,
242 diaSourceTable,
243 diffIm,
244 exposure,
245 warpedExposure,
246 ccdExposureIdBits):
247 """Process DiaSources and DiaObjects.
249 Load previous DiaObjects and their DiaSource history. Calibrate the
250 values in the diaSourceCat. Associate new DiaSources with previous
251 DiaObjects. Run forced photometry at the updated DiaObject locations.
252 Store the results in the Alert Production Database (Apdb).
254 Parameters
255 ----------
256 diaSourceTable : `pandas.DataFrame`
257 Newly detected DiaSources.
258 diffIm : `lsst.afw.image.ExposureF`
259 Difference image exposure in which the sources in ``diaSourceCat``
260 were detected.
261 exposure : `lsst.afw.image.ExposureF`
262 Calibrated exposure differenced with a template to create
263 ``diffIm``.
264 warpedExposure : `lsst.afw.image.ExposureF`
265 Template exposure used to create diffIm.
266 ccdExposureIdBits : `int`
267 Number of bits used for a unique ``ccdVisitId``.
269 Returns
270 -------
271 results : `lsst.pipe.base.Struct`
272 Results struct with components.
274 - ``apdb_maker`` : Marker dataset to store in the Butler indicating
275 that this ccdVisit has completed successfully.
276 (`lsst.dax.apdb.ApdbConfig`)
277 """
278 self.log.info("Running DiaPipeline...")
279 # Put the SciencePipelines through a SDMification step and return
280 # calibrated columns with the expect output database names.
282 # Load the DiaObjects and DiaSource history.
283 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb)
285 # Associate new DiaSources with existing DiaObjects and update
286 # DiaObject summary statistics using the full DiaSource history.
287 assocResults = self.associator.run(diaSourceTable,
288 loaderResult.diaObjects,
289 loaderResult.diaSources)
291 # Force photometer on the Difference and Calibrated exposures using
292 # the new and updated DiaObject locations.
293 diaForcedSources = self.diaForcedSource.run(
294 assocResults.diaObjects,
295 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(),
296 ccdExposureIdBits,
297 exposure,
298 diffIm)
300 # Store DiaSources and updated DiaObjects in the Apdb.
301 self.apdb.storeDiaSources(assocResults.diaSources)
302 self.apdb.storeDiaObjects(
303 assocResults.updatedDiaObjects,
304 exposure.getInfo().getVisitInfo().getDate().toPython())
305 self.apdb.storeDiaForcedSources(diaForcedSources)
307 if self.config.doPackageAlerts:
308 if len(loaderResult.diaForcedSources) > 1:
309 diaForcedSources = diaForcedSources.append(
310 loaderResult.diaForcedSources,
311 sort=True)
312 if diaForcedSources.index.has_duplicates:
313 self.log.warn(
314 "Duplicate DiaForcedSources created after merge with "
315 "history and new sources. This may cause downstream "
316 "problems. Dropping duplicates.")
317 # Drop duplicates via index and keep the first appearance.
318 # Reset due to the index shape being slight different than
319 # expected.
320 diaForcedSources = diaForcedSources.groupby(
321 diaForcedSources.index).first()
322 diaForcedSources.reset_index(drop=True, inplace=True)
323 diaForcedSources.set_index(
324 ["diaObjectId", "diaForcedSourceId"],
325 drop=False,
326 inplace=True)
327 self.alertPackager.run(assocResults.diaSources,
328 assocResults.diaObjects,
329 loaderResult.diaSources,
330 diaForcedSources,
331 diffIm,
332 warpedExposure,
333 ccdExposureIdBits)
335 return pipeBase.Struct(apdbMarker=self.config.apdb.value,
336 associatedDiaSources=assocResults.diaSources)