Coverage for python/lsst/ap/association/diaPipe.py : 39%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23"""PipelineTask for associating DiaSources with previous DiaObjects.
25Additionally performs forced photometry on the calibrated and difference
26images at the updated locations of DiaObjects.
28Currently loads directly from the Apdb rather than pre-loading.
29"""
31import os
33import lsst.dax.apdb as daxApdb
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as connTypes
38from lsst.ap.association import (
39 AssociationTask,
40 DiaForcedSourceTask,
41 LoadDiaCatalogsTask,
42 make_dia_object_schema,
43 make_dia_source_schema,
44 PackageAlertsTask)
46__all__ = ("DiaPipelineConfig",
47 "DiaPipelineTask",
48 "DiaPipelineConnections")
51class DiaPipelineConnections(
52 pipeBase.PipelineTaskConnections,
53 dimensions=("instrument", "visit", "detector"),
54 defaultTemplates={"coaddName": "deep", "fakesType": ""}):
55 """Butler connections for DiaPipelineTask.
56 """
57 diaSourceTable = connTypes.Input(
58 doc="Catalog of calibrated DiaSources.",
59 name="{fakesType}{coaddName}Diff_diaSrcTable",
60 storageClass="DataFrame",
61 dimensions=("instrument", "visit", "detector"),
62 )
63 diffIm = connTypes.Input(
64 doc="Difference image on which the DiaSources were detected.",
65 name="{fakesType}{coaddName}Diff_differenceExp",
66 storageClass="ExposureF",
67 dimensions=("instrument", "visit", "detector"),
68 )
69 exposure = connTypes.Input(
70 doc="Calibrated exposure differenced with a template image during "
71 "image differencing.",
72 name="calexp",
73 storageClass="ExposureF",
74 dimensions=("instrument", "visit", "detector"),
75 )
76 warpedExposure = connTypes.Input(
77 doc="Warped template used to create `subtractedExposure`. Not PSF "
78 "matched.",
79 dimensions=("instrument", "visit", "detector"),
80 storageClass="ExposureF",
81 name="{fakesType}{coaddName}Diff_warpedExp",
82 )
83 apdbMarker = connTypes.Output(
84 doc="Marker dataset storing the configuration of the Apdb for each "
85 "visit/detector. Used to signal the completion of the pipeline.",
86 name="apdb_marker",
87 storageClass="Config",
88 dimensions=("instrument", "visit", "detector"),
89 )
90 associatedDiaSources = connTypes.Output(
91 doc="Optional output storing the DiaSource catalog after matching, "
92 "calibration, and standardization for insertation into the Apdb.",
93 name="{fakesType}{coaddName}Diff_assocDiaSrc",
94 storageClass="DataFrame",
95 dimensions=("instrument", "visit", "detector"),
96 )
98 def __init__(self, *, config=None):
99 super().__init__(config=config)
101 if not config.doWriteAssociatedSources:
102 self.outputs.remove("associatedDiaSources")
104 def adjustQuantum(self, inputs, outputs, label, dataId):
105 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects
106 in the `lsst.daf.butler.core.Quantum` during the graph generation stage
107 of the activator.
109 This implementation checks to make sure that the filters in the dataset
110 are compatible with AP processing as set by the Apdb/DPDD schema.
112 Parameters
113 ----------
114 inputs : `dict`
115 Dictionary whose keys are an input (regular or prerequisite)
116 connection name and whose values are a tuple of the connection
117 instance and a collection of associated `DatasetRef` objects.
118 The exact type of the nested collections is unspecified; it can be
119 assumed to be multi-pass iterable and support `len` and ``in``, but
120 it should not be mutated in place. In contrast, the outer
121 dictionaries are guaranteed to be temporary copies that are true
122 `dict` instances, and hence may be modified and even returned; this
123 is especially useful for delegating to `super` (see notes below).
124 outputs : `dict`
125 Dict of output datasets, with the same structure as ``inputs``.
126 label : `str`
127 Label for this task in the pipeline (should be used in all
128 diagnostic messages).
129 data_id : `lsst.daf.butler.DataCoordinate`
130 Data ID for this quantum in the pipeline (should be used in all
131 diagnostic messages).
133 Returns
134 -------
135 adjusted_inputs : `dict`
136 Dict of the same form as ``inputs`` with updated containers of
137 input `DatasetRef` objects. Connections that are not changed
138 should not be returned at all. Datasets may only be removed, not
139 added. Nested collections may be of any multi-pass iterable type,
140 and the order of iteration will set the order of iteration within
141 `PipelineTask.runQuantum`.
142 adjusted_outputs : `dict`
143 Dict of updated output datasets, with the same structure and
144 interpretation as ``adjusted_inputs``.
146 Raises
147 ------
148 ScalarError
149 Raised if any `Input` or `PrerequisiteInput` connection has
150 ``multiple`` set to `False`, but multiple datasets.
151 NoWorkFound
152 Raised to indicate that this quantum should not be run; not enough
153 datasets were found for a regular `Input` connection, and the
154 quantum should be pruned or skipped.
155 FileNotFoundError
156 Raised to cause QuantumGraph generation to fail (with the message
157 included in this exception); not enough datasets were found for a
158 `PrerequisiteInput` connection.
159 """
160 _, refs = inputs["diffIm"]
161 for ref in refs:
162 if ref.dataId["band"] not in self.config.validBands:
163 raise ValueError(
164 f"Requested '{ref.dataId['band']}' not in "
165 "DiaPipelineConfig.validBands. To process bands not in "
166 "the standard Rubin set (ugrizy) you must add the band to "
167 "the validBands list in DiaPipelineConfig and add the "
168 "appropriate columns to the Apdb schema.")
169 return super().adjustQuantum(inputs, outputs, label, dataId)
172class DiaPipelineConfig(pipeBase.PipelineTaskConfig,
173 pipelineConnections=DiaPipelineConnections):
174 """Config for DiaPipelineTask.
175 """
176 coaddName = pexConfig.Field(
177 doc="coadd name: typically one of deep, goodSeeing, or dcr",
178 dtype=str,
179 default="deep",
180 )
181 apdb = pexConfig.ConfigurableField(
182 target=daxApdb.Apdb,
183 ConfigClass=daxApdb.ApdbConfig,
184 doc="Database connection for storing associated DiaSources and "
185 "DiaObjects. Must already be initialized.",
186 )
187 validBands = pexConfig.ListField(
188 dtype=str,
189 default=["u", "g", "r", "i", "z", "y"],
190 doc="List of bands that are valid for AP processing. To process a "
191 "band not on this list, the appropriate band specific columns "
192 "must be added to the Apdb schema in dax_apdb.",
193 )
194 diaCatalogLoader = pexConfig.ConfigurableField(
195 target=LoadDiaCatalogsTask,
196 doc="Task to load DiaObjects and DiaSources from the Apdb.",
197 )
198 associator = pexConfig.ConfigurableField(
199 target=AssociationTask,
200 doc="Task used to associate DiaSources with DiaObjects.",
201 )
202 diaForcedSource = pexConfig.ConfigurableField(
203 target=DiaForcedSourceTask,
204 doc="Task used for force photometer DiaObject locations in direct and "
205 "difference images.",
206 )
207 alertPackager = pexConfig.ConfigurableField(
208 target=PackageAlertsTask,
209 doc="Subtask for packaging Ap data into alerts.",
210 )
211 doPackageAlerts = pexConfig.Field(
212 dtype=bool,
213 default=False,
214 doc="Package Dia-data into serialized alerts for distribution and "
215 "write them to disk.",
216 )
217 doWriteAssociatedSources = pexConfig.Field(
218 dtype=bool,
219 default=False,
220 doc="Write out associated and SDMed DiaSources.",
221 )
223 def setDefaults(self):
224 self.apdb.dia_object_index = "baseline"
225 self.apdb.dia_object_columns = []
226 self.apdb.extra_schema_file = os.path.join(
227 "${AP_ASSOCIATION_DIR}",
228 "data",
229 "apdb-ap-pipe-schema-extra.yaml")
231 def validate(self):
232 pexConfig.Config.validate(self)
233 if self.diaCatalogLoader.htmLevel != \
234 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel:
235 raise ValueError("HTM index level in LoadDiaCatalogsTask must be "
236 "equal to HTMIndexDiaCalculationPlugin index "
237 "level.")
240class DiaPipelineTask(pipeBase.PipelineTask):
241 """Task for loading, associating and storing Difference Image Analysis
242 (DIA) Objects and Sources.
243 """
244 ConfigClass = DiaPipelineConfig
245 _DefaultName = "diaPipe"
246 RunnerClass = pipeBase.ButlerInitializedTaskRunner
248 def __init__(self, initInputs=None, **kwargs):
249 super().__init__(**kwargs)
250 self.apdb = self.config.apdb.apply(
251 afw_schemas=dict(DiaObject=make_dia_object_schema(),
252 DiaSource=make_dia_source_schema()))
253 self.makeSubtask("diaCatalogLoader")
254 self.makeSubtask("associator")
255 self.makeSubtask("diaForcedSource")
256 if self.config.doPackageAlerts:
257 self.makeSubtask("alertPackager")
259 def runQuantum(self, butlerQC, inputRefs, outputRefs):
260 inputs = butlerQC.get(inputRefs)
261 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
262 returnMaxBits=True)
263 inputs["ccdExposureIdBits"] = expBits
265 outputs = self.run(**inputs)
267 butlerQC.put(outputs, outputRefs)
269 @pipeBase.timeMethod
270 def run(self,
271 diaSourceTable,
272 diffIm,
273 exposure,
274 warpedExposure,
275 ccdExposureIdBits):
276 """Process DiaSources and DiaObjects.
278 Load previous DiaObjects and their DiaSource history. Calibrate the
279 values in the diaSourceCat. Associate new DiaSources with previous
280 DiaObjects. Run forced photometry at the updated DiaObject locations.
281 Store the results in the Alert Production Database (Apdb).
283 Parameters
284 ----------
285 diaSourceTable : `pandas.DataFrame`
286 Newly detected DiaSources.
287 diffIm : `lsst.afw.image.ExposureF`
288 Difference image exposure in which the sources in ``diaSourceCat``
289 were detected.
290 exposure : `lsst.afw.image.ExposureF`
291 Calibrated exposure differenced with a template to create
292 ``diffIm``.
293 warpedExposure : `lsst.afw.image.ExposureF`
294 Template exposure used to create diffIm.
295 ccdExposureIdBits : `int`
296 Number of bits used for a unique ``ccdVisitId``.
298 Returns
299 -------
300 results : `lsst.pipe.base.Struct`
301 Results struct with components.
303 - ``apdb_maker`` : Marker dataset to store in the Butler indicating
304 that this ccdVisit has completed successfully.
305 (`lsst.dax.apdb.ApdbConfig`)
306 """
307 self.log.info("Running DiaPipeline...")
308 # Put the SciencePipelines through a SDMification step and return
309 # calibrated columns with the expect output database names.
311 # Load the DiaObjects and DiaSource history.
312 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb)
314 # Associate new DiaSources with existing DiaObjects and update
315 # DiaObject summary statistics using the full DiaSource history.
316 assocResults = self.associator.run(diaSourceTable,
317 loaderResult.diaObjects,
318 loaderResult.diaSources)
320 # Force photometer on the Difference and Calibrated exposures using
321 # the new and updated DiaObject locations.
322 diaForcedSources = self.diaForcedSource.run(
323 assocResults.diaObjects,
324 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(),
325 ccdExposureIdBits,
326 exposure,
327 diffIm)
329 # Store DiaSources and updated DiaObjects in the Apdb.
330 self.apdb.storeDiaSources(assocResults.diaSources)
331 self.apdb.storeDiaObjects(
332 assocResults.updatedDiaObjects,
333 exposure.getInfo().getVisitInfo().getDate().toPython())
334 self.apdb.storeDiaForcedSources(diaForcedSources)
336 if self.config.doPackageAlerts:
337 if len(loaderResult.diaForcedSources) > 1:
338 diaForcedSources = diaForcedSources.append(
339 loaderResult.diaForcedSources,
340 sort=True)
341 if diaForcedSources.index.has_duplicates:
342 self.log.warn(
343 "Duplicate DiaForcedSources created after merge with "
344 "history and new sources. This may cause downstream "
345 "problems. Dropping duplicates.")
346 # Drop duplicates via index and keep the first appearance.
347 # Reset due to the index shape being slight different than
348 # expected.
349 diaForcedSources = diaForcedSources.groupby(
350 diaForcedSources.index).first()
351 diaForcedSources.reset_index(drop=True, inplace=True)
352 diaForcedSources.set_index(
353 ["diaObjectId", "diaForcedSourceId"],
354 drop=False,
355 inplace=True)
356 self.alertPackager.run(assocResults.diaSources,
357 assocResults.diaObjects,
358 loaderResult.diaSources,
359 diaForcedSources,
360 diffIm,
361 warpedExposure,
362 ccdExposureIdBits)
364 return pipeBase.Struct(apdbMarker=self.config.apdb.value,
365 associatedDiaSources=assocResults.diaSources)