Coverage for python/lsst/ap/association/diaPipe.py : 40%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23"""PipelineTask for associating DiaSources with previous DiaObjects.
25Additionally performs forced photometry on the calibrated and difference
26images at the updated locations of DiaObjects.
28Currently loads directly from the Apdb rather than pre-loading.
29"""
31import os
33import lsst.dax.apdb as daxApdb
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as connTypes
37from lsst.utils import getPackageDir
39from lsst.ap.association import (
40 AssociationTask,
41 DiaForcedSourceTask,
42 LoadDiaCatalogsTask,
43 MapDiaSourceTask,
44 make_dia_object_schema,
45 make_dia_source_schema,
46 PackageAlertsTask)
48__all__ = ("DiaPipelineConfig",
49 "DiaPipelineTask",
50 "DiaPipelineConnections")
53class DiaPipelineConnections(pipeBase.PipelineTaskConnections,
54 dimensions=("instrument", "visit", "detector"),
55 defaultTemplates={"coaddName": "deep", "fakesType": ""}):
56 """Butler connections for DiaPipelineTask.
57 """
58 diaSourceSchema = connTypes.InitInput(
59 doc="Schema of the DiaSource catalog produced during image "
60 "differencing",
61 name="{fakesType}{coaddName}Diff_diaSrc_schema",
62 storageClass="SourceCatalog",
63 multiple=True
64 )
65 diaSourceCat = connTypes.Input(
66 doc="Catalog of DiaSources produced during image differencing.",
67 name="{fakesType}{coaddName}Diff_diaSrc",
68 storageClass="SourceCatalog",
69 dimensions=("instrument", "visit", "detector"),
70 )
71 diffIm = connTypes.Input(
72 doc="Difference image on which the DiaSources were detected.",
73 name="{fakesType}{coaddName}Diff_differenceExp",
74 storageClass="ExposureF",
75 dimensions=("instrument", "visit", "detector"),
76 )
77 exposure = connTypes.Input(
78 doc="Calibrated exposure differenced with a template image during "
79 "image differencing.",
80 name="calexp",
81 storageClass="ExposureF",
82 dimensions=("instrument", "visit", "detector"),
83 )
84 warpedExposure = connTypes.Input(
85 doc="Warped template used to create `subtractedExposure`. Not PSF "
86 "matched.",
87 dimensions=("instrument", "visit", "detector"),
88 storageClass="ExposureF",
89 name="{fakesType}{coaddName}Diff_warpedExp",
90 )
91 apdbMarker = connTypes.Output(
92 doc="Marker dataset storing the configuration of the Apdb for each "
93 "visit/detector. Used to signal the completion of the pipeline.",
94 name="apdb_marker",
95 storageClass="Config",
96 dimensions=("instrument", "visit", "detector"),
97 )
98 associatedDiaSources = connTypes.Output(
99 doc="Optional output storing the DiaSource catalog after matching, "
100 "calibration, and standardization for insertation into the Apdb.",
101 name="{fakesType}{coaddName}Diff_assocDiaSrc",
102 storageClass="DataFrame",
103 dimensions=("instrument", "visit", "detector"),
104 )
106 def __init__(self, *, config=None):
107 super().__init__(config=config)
109 if not config.doWriteAssociatedSources:
110 self.outputs.remove("associatedDiaSources")
112 def adjustQuantum(self, datasetRefMap: pipeBase.InputQuantizedConnection):
113 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects
114 in the `lsst.daf.butler.core.Quantum` during the graph generation stage
115 of the activator.
117 This implementation checks to make sure that the filters in the dataset
118 are compatible with AP processing as set by the Apdb/DPDD schema.
120 Parameters
121 ----------
122 datasetRefMap : `NamedKeyDict`
123 Mapping from dataset type to a `set` of
124 `lsst.daf.butler.DatasetRef` objects
126 Returns
127 -------
128 datasetRefMap : `NamedKeyDict`
129 Mapping of input with assurances that bands incompatible with the
130 Apdb are present.
132 Raises
133 ------
134 ValueError
135 Raises if a data ref in the quantum has a band not available in the
136 Apdb.
137 """
138 refs = datasetRefMap[self.diffIm.name]
139 for ref in refs:
140 if ref.dataId["band"] not in self.config.validBands:
141 raise ValueError(
142 f"Requested '{ref.dataId['band']}' not in "
143 "DiaPipelineConfig.validBands. To process bands not in the "
144 "standard Rubin set (ugrizy) you must add the band to the "
145 "validBands list in DiaPipelineConfig and add the "
146 "appropriate columns to the Apdb schema.")
147 return super().adjustQuantum(datasetRefMap)
150class DiaPipelineConfig(pipeBase.PipelineTaskConfig,
151 pipelineConnections=DiaPipelineConnections):
152 """Config for DiaPipelineTask.
153 """
154 coaddName = pexConfig.Field(
155 doc="coadd name: typically one of deep, goodSeeing, or dcr",
156 dtype=str,
157 default="deep",
158 )
159 apdb = pexConfig.ConfigurableField(
160 target=daxApdb.Apdb,
161 ConfigClass=daxApdb.ApdbConfig,
162 doc="Database connection for storing associated DiaSources and "
163 "DiaObjects. Must already be initialized.",
164 )
165 validBands = pexConfig.ListField(
166 dtype=str,
167 default=["u", "g", "r", "i", "z", "y"],
168 doc="List of bands that are valid for AP processing. To process a "
169 "band not on this list, the appropriate band specific columns "
170 "must be added to the Apdb schema in dax_apdb.",
171 )
172 diaSourceDpddifier = pexConfig.ConfigurableField(
173 target=MapDiaSourceTask,
174 doc="Task for assigning columns from the raw output of ip_diffim into "
175 "a schema that more closely resembles the DPDD.",
176 )
177 diaCatalogLoader = pexConfig.ConfigurableField(
178 target=LoadDiaCatalogsTask,
179 doc="Task to load DiaObjects and DiaSources from the Apdb.",
180 )
181 associator = pexConfig.ConfigurableField(
182 target=AssociationTask,
183 doc="Task used to associate DiaSources with DiaObjects.",
184 )
185 diaForcedSource = pexConfig.ConfigurableField(
186 target=DiaForcedSourceTask,
187 doc="Task used for force photometer DiaObject locations in direct and "
188 "difference images.",
189 )
190 alertPackager = pexConfig.ConfigurableField(
191 target=PackageAlertsTask,
192 doc="Subtask for packaging Ap data into alerts.",
193 )
194 doPackageAlerts = pexConfig.Field(
195 dtype=bool,
196 default=False,
197 doc="Package Dia-data into serialized alerts for distribution and "
198 "write them to disk.",
199 )
200 doWriteAssociatedSources = pexConfig.Field(
201 dtype=bool,
202 default=False,
203 doc="Write out associated and SDMed DiaSources.",
204 )
206 def setDefaults(self):
207 self.apdb.dia_object_index = "baseline"
208 self.apdb.dia_object_columns = []
209 self.apdb.extra_schema_file = os.path.join(
210 getPackageDir("ap_association"),
211 "data",
212 "apdb-ap-pipe-schema-extra.yaml")
214 def validate(self):
215 pexConfig.Config.validate(self)
216 if self.diaCatalogLoader.htmLevel != \
217 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel:
218 raise ValueError("HTM index level in LoadDiaCatalogsTask must be "
219 "equal to HTMIndexDiaCalculationPlugin index "
220 "level.")
223class DiaPipelineTask(pipeBase.PipelineTask):
224 """Task for loading, associating and storing Difference Image Analysis
225 (DIA) Objects and Sources.
226 """
227 ConfigClass = DiaPipelineConfig
228 _DefaultName = "diaPipe"
229 RunnerClass = pipeBase.ButlerInitializedTaskRunner
231 def __init__(self, initInputs=None, **kwargs):
232 super().__init__(**kwargs)
233 self.apdb = self.config.apdb.apply(
234 afw_schemas=dict(DiaObject=make_dia_object_schema(),
235 DiaSource=make_dia_source_schema()))
236 self.makeSubtask("diaSourceDpddifier",
237 inputSchema=initInputs["diaSourceSchema"].schema)
238 self.makeSubtask("diaCatalogLoader")
239 self.makeSubtask("associator")
240 self.makeSubtask("diaForcedSource")
241 if self.config.doPackageAlerts:
242 self.makeSubtask("alertPackager")
244 def runQuantum(self, butlerQC, inputRefs, outputRefs):
245 inputs = butlerQC.get(inputRefs)
246 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
247 returnMaxBits=True)
248 inputs["ccdExposureIdBits"] = expBits
250 outputs = self.run(**inputs)
252 butlerQC.put(outputs, outputRefs)
254 @pipeBase.timeMethod
255 def run(self, diaSourceCat, diffIm, exposure, warpedExposure, ccdExposureIdBits):
256 """Process DiaSources and DiaObjects.
258 Load previous DiaObjects and their DiaSource history. Calibrate the
259 values in the diaSourceCat. Associate new DiaSources with previous
260 DiaObjects. Run forced photometry at the updated DiaObject locations.
261 Store the results in the Alert Production Database (Apdb).
263 Parameters
264 ----------
265 diaSourceCat : `lsst.afw.table.SourceCatalog`
266 Newly detected DiaSources.
267 diffIm : `lsst.afw.image.ExposureF`
268 Difference image exposure in which the sources in ``diaSourceCat``
269 were detected.
270 exposure : `lsst.afw.image.ExposureF`
271 Calibrated exposure differenced with a template to create
272 ``diffIm``.
273 warpedExposure : `lsst.afw.image.ExposureF`
274 Template exposure used to create diffIm.
275 ccdExposureIdBits : `int`
276 Number of bits used for a unique ``ccdVisitId``.
278 Returns
279 -------
280 results : `lsst.pipe.base.Struct`
281 Results struct with components.
283 - ``apdb_maker`` : Marker dataset to store in the Butler indicating
284 that this ccdVisit has completed successfully.
285 (`lsst.dax.apdb.ApdbConfig`)
286 """
287 self.log.info("Running DiaPipeline...")
288 # Put the SciencePipelines through a SDMification step and return
289 # calibrated columns with the expect output database names.
290 diaSources = self.diaSourceDpddifier.run(diaSourceCat,
291 diffIm,
292 return_pandas=True)
294 # Load the DiaObjects and DiaSource history.
295 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb)
297 # Associate new DiaSources with existing DiaObjects and update
298 # DiaObject summary statistics using the full DiaSource history.
299 assocResults = self.associator.run(diaSources,
300 loaderResult.diaObjects,
301 loaderResult.diaSources)
303 # Force photometer on the Difference and Calibrated exposures using
304 # the new and updated DiaObject locations.
305 diaForcedSources = self.diaForcedSource.run(
306 assocResults.diaObjects,
307 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(),
308 ccdExposureIdBits,
309 exposure,
310 diffIm)
312 # Store DiaSources and updated DiaObjects in the Apdb.
313 self.apdb.storeDiaSources(assocResults.diaSources)
314 self.apdb.storeDiaObjects(
315 assocResults.updatedDiaObjects,
316 exposure.getInfo().getVisitInfo().getDate().toPython())
317 self.apdb.storeDiaForcedSources(diaForcedSources)
319 if self.config.doPackageAlerts:
320 if len(loaderResult.diaForcedSources) > 1:
321 diaForcedSources = diaForcedSources.append(
322 loaderResult.diaForcedSources,
323 sort=True)
324 if diaForcedSources.index.has_duplicates:
325 self.log.warn(
326 "Duplicate DiaForcedSources created after merge with "
327 "history and new sources. This may cause downstream "
328 "problems. Dropping duplicates.")
329 # Drop duplicates via index and keep the first appearance.
330 # Reset due to the index shape being slight different than
331 # expected.
332 diaForcedSources = diaForcedSources.groupby(
333 diaForcedSources.index).first()
334 diaForcedSources.reset_index(drop=True, inplace=True)
335 diaForcedSources.set_index(
336 ["diaObjectId", "diaForcedSourceId"],
337 drop=False,
338 inplace=True)
339 self.alertPackager.run(assocResults.diaSources,
340 assocResults.diaObjects,
341 loaderResult.diaSources,
342 diaForcedSources,
343 diffIm,
344 warpedExposure,
345 ccdExposureIdBits)
347 return pipeBase.Struct(apdbMarker=self.config.apdb.value,
348 associatedDiaSources=assocResults.diaSources)