Coverage for python/lsst/ap/association/diaPipe.py : 46%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23"""PipelineTask for associating DiaSources with previous DiaObjects.
25Additionally performs forced photometry on the calibrated and difference
26images at the updated locations of DiaObjects.
28Currently loads directly from the Apdb rather than pre-loading.
29"""
31import os
33import lsst.dax.apdb as daxApdb
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as connTypes
37from lsst.utils import getPackageDir
39from lsst.ap.association import (
40 AssociationTask,
41 DiaForcedSourceTask,
42 LoadDiaCatalogsTask,
43 MapDiaSourceTask,
44 make_dia_object_schema,
45 make_dia_source_schema,
46 PackageAlertsTask)
48__all__ = ("DiaPipelineConfig",
49 "DiaPipelineTask",
50 "DiaPipelineConnections")
53class DiaPipelineConnections(pipeBase.PipelineTaskConnections,
54 dimensions=("instrument", "visit", "detector"),
55 defaultTemplates={"coaddName": "deep", "fakesType": ""}):
56 """Butler connections for DiaPipelineTask.
57 """
58 diaSourceSchema = connTypes.InitInput(
59 doc="Schema of the DiaSource catalog produced during image "
60 "differencing",
61 name="{fakesType}{coaddName}Diff_diaSrc_schema",
62 storageClass="SourceCatalog",
63 multiple=True
64 )
65 diaSourceCat = connTypes.Input(
66 doc="Catalog of DiaSources produced during image differencing.",
67 name="{fakesType}{coaddName}Diff_diaSrc",
68 storageClass="SourceCatalog",
69 dimensions=("instrument", "visit", "detector"),
70 )
71 diffIm = connTypes.Input(
72 doc="Difference image on which the DiaSources were detected.",
73 name="{fakesType}{coaddName}Diff_differenceExp",
74 storageClass="ExposureF",
75 dimensions=("instrument", "visit", "detector"),
76 )
77 exposure = connTypes.Input(
78 doc="Calibrated exposure differenced with a template image during "
79 "image differencing.",
80 name="calexp",
81 storageClass="ExposureF",
82 dimensions=("instrument", "visit", "detector"),
83 )
84 warpedExposure = connTypes.Input(
85 doc="Warped template used to create `subtractedExposure`. Not PSF "
86 "matched.",
87 dimensions=("instrument", "visit", "detector"),
88 storageClass="ExposureF",
89 name="{fakesType}{coaddName}Diff_warpedExp",
90 )
91 apdbMarker = connTypes.Output(
92 doc="Marker dataset storing the configuration of the Apdb for each "
93 "visit/detector. Used to signal the completion of the pipeline.",
94 name="apdb_marker",
95 storageClass="Config",
96 dimensions=("instrument", "visit", "detector"),
97 )
98 associatedDiaSources = connTypes.Output(
99 doc="Optional output storing the DiaSource catalog after matching, "
100 "calibration, and standardization for insertation into the Apdb.",
101 name="{fakesType}{coaddName}Diff_assocDiaSrc",
102 storageClass="DataFrame",
103 dimensions=("instrument", "visit", "detector"),
104 )
106 def __init__(self, *, config=None):
107 super().__init__(config=config)
109 if not config.doWriteAssociatedSources:
110 self.outputs.remove("associatedDiaSources")
113class DiaPipelineConfig(pipeBase.PipelineTaskConfig,
114 pipelineConnections=DiaPipelineConnections):
115 """Config for DiaPipelineTask.
116 """
117 coaddName = pexConfig.Field(
118 doc="coadd name: typically one of deep, goodSeeing, or dcr",
119 dtype=str,
120 default="deep",
121 )
122 apdb = pexConfig.ConfigurableField(
123 target=daxApdb.Apdb,
124 ConfigClass=daxApdb.ApdbConfig,
125 doc="Database connection for storing associated DiaSources and "
126 "DiaObjects. Must already be initialized.",
127 )
128 diaSourceDpddifier = pexConfig.ConfigurableField(
129 target=MapDiaSourceTask,
130 doc="Task for assigning columns from the raw output of ip_diffim into "
131 "a schema that more closely resembles the DPDD.",
132 )
133 diaCatalogLoader = pexConfig.ConfigurableField(
134 target=LoadDiaCatalogsTask,
135 doc="Task to load DiaObjects and DiaSources from the Apdb.",
136 )
137 associator = pexConfig.ConfigurableField(
138 target=AssociationTask,
139 doc="Task used to associate DiaSources with DiaObjects.",
140 )
141 diaForcedSource = pexConfig.ConfigurableField(
142 target=DiaForcedSourceTask,
143 doc="Task used for force photometer DiaObject locations in direct and "
144 "difference images.",
145 )
146 alertPackager = pexConfig.ConfigurableField(
147 target=PackageAlertsTask,
148 doc="Subtask for packaging Ap data into alerts.",
149 )
150 doPackageAlerts = pexConfig.Field(
151 dtype=bool,
152 default=False,
153 doc="Package Dia-data into serialized alerts for distribution and "
154 "write them to disk.",
155 )
156 doWriteAssociatedSources = pexConfig.Field(
157 dtype=bool,
158 default=False,
159 doc="Write out associated and SDMed DiaSources.",
160 )
162 def setDefaults(self):
163 self.apdb.dia_object_index = "baseline"
164 self.apdb.dia_object_columns = []
165 self.apdb.extra_schema_file = os.path.join(
166 getPackageDir("ap_association"),
167 "data",
168 "apdb-ap-pipe-schema-extra.yaml")
170 def validate(self):
171 pexConfig.Config.validate(self)
172 if self.diaCatalogLoader.htmLevel != \
173 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel:
174 raise ValueError("HTM index level in LoadDiaCatalogsTask must be "
175 "equal to HTMIndexDiaCalculationPlugin index "
176 "level.")
179class DiaPipelineTask(pipeBase.PipelineTask):
180 """Task for loading, associating and storing Difference Image Analysis
181 (DIA) Objects and Sources.
182 """
183 ConfigClass = DiaPipelineConfig
184 _DefaultName = "diaPipe"
185 RunnerClass = pipeBase.ButlerInitializedTaskRunner
187 def __init__(self, initInputs=None, **kwargs):
188 super().__init__(**kwargs)
189 self.apdb = self.config.apdb.apply(
190 afw_schemas=dict(DiaObject=make_dia_object_schema(),
191 DiaSource=make_dia_source_schema()))
192 self.makeSubtask("diaSourceDpddifier",
193 inputSchema=initInputs["diaSourceSchema"].schema)
194 self.makeSubtask("diaCatalogLoader")
195 self.makeSubtask("associator")
196 self.makeSubtask("diaForcedSource")
197 if self.config.doPackageAlerts:
198 self.makeSubtask("alertPackager")
200 def runQuantum(self, butlerQC, inputRefs, outputRefs):
201 inputs = butlerQC.get(inputRefs)
202 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
203 returnMaxBits=True)
204 inputs["ccdExposureIdBits"] = expBits
206 outputs = self.run(**inputs)
208 butlerQC.put(outputs, outputRefs)
210 @pipeBase.timeMethod
211 def run(self, diaSourceCat, diffIm, exposure, warpedExposure, ccdExposureIdBits):
212 """Process DiaSources and DiaObjects.
214 Load previous DiaObjects and their DiaSource history. Calibrate the
215 values in the diaSourceCat. Associate new DiaSources with previous
216 DiaObjects. Run forced photometry at the updated DiaObject locations.
217 Store the results in the Alert Production Database (Apdb).
219 Parameters
220 ----------
221 diaSourceCat : `lsst.afw.table.SourceCatalog`
222 Newly detected DiaSources.
223 diffIm : `lsst.afw.image.ExposureF`
224 Difference image exposure in which the sources in ``diaSourceCat``
225 were detected.
226 exposure : `lsst.afw.image.ExposureF`
227 Calibrated exposure differenced with a template to create
228 ``diffIm``.
229 warpedExposure : `lsst.afw.image.ExposureF`
230 Template exposure used to create diffIm.
231 ccdExposureIdBits : `int`
232 Number of bits used for a unique ``ccdVisitId``.
234 Returns
235 -------
236 results : `lsst.pipe.base.Struct`
237 Results struct with components.
239 - ``apdb_maker`` : Marker dataset to store in the Butler indicating
240 that this ccdVisit has completed successfully.
241 (`lsst.dax.apdb.ApdbConfig`)
242 """
243 self.log.info("Running DiaPipeline...")
244 # Put the SciencePipelines through a SDMification step and return
245 # calibrated columns with the expect output database names.
246 diaSources = self.diaSourceDpddifier.run(diaSourceCat,
247 diffIm,
248 return_pandas=True)
250 # Load the DiaObjects and DiaSource history.
251 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb)
253 # Associate new DiaSources with existing DiaObjects and update
254 # DiaObject summary statistics using the full DiaSource history.
255 assocResults = self.associator.run(diaSources,
256 loaderResult.diaObjects,
257 loaderResult.diaSources)
259 # Force photometer on the Difference and Calibrated exposures using
260 # the new and updated DiaObject locations.
261 diaForcedSources = self.diaForcedSource.run(
262 assocResults.diaObjects,
263 assocResults.updatedDiaObjects.loc[:, "diaObjectId"].to_numpy(),
264 ccdExposureIdBits,
265 exposure,
266 diffIm)
268 # Store DiaSources and updated DiaObjects in the Apdb.
269 self.apdb.storeDiaSources(assocResults.diaSources)
270 self.apdb.storeDiaObjects(
271 assocResults.updatedDiaObjects,
272 exposure.getInfo().getVisitInfo().getDate().toPython())
273 self.apdb.storeDiaForcedSources(diaForcedSources)
274 if self.config.doPackageAlerts:
275 if len(loaderResult.diaForcedSources) > 1:
276 diaForcedSources = diaForcedSources.append(
277 loaderResult.diaForcedSources,
278 sort=True)
279 self.alertPackager.run(assocResults.diaSources,
280 assocResults.diaObjects,
281 loaderResult.diaSources,
282 diaForcedSources,
283 diffIm,
284 warpedExposure,
285 ccdExposureIdBits)
287 return pipeBase.Struct(apdbMarker=self.config.apdb.value,
288 associatedDiaSources=assocResults.diaSources)