Coverage for python/lsst/ap/association/diaPipe.py : 52%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23"""PipelineTask for associating DiaSources with previous DiaObjects.
25Additionally performs forced photometry on the calibrated and difference
26images at the updated locations of DiaObjects.
28Currently loads directly from the Apdb rather than pre-loading.
29"""
31import os
33import lsst.dax.apdb as daxApdb
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as connTypes
37from lsst.utils import getPackageDir
39from lsst.ap.association import (
40 AssociationTask,
41 DiaForcedSourceTask,
42 LoadDiaCatalogsTask,
43 MapDiaSourceTask,
44 make_dia_object_schema,
45 make_dia_source_schema)
47__all__ = ("DiaPipelineConfig",
48 "DiaPipelineTask",
49 "DiaPipelineConnections")
52class DiaPipelineConnections(pipeBase.PipelineTaskConnections,
53 dimensions=("instrument", "visit", "detector"),
54 defaultTemplates={"coaddName": "deep"}):
55 """Butler connections for DiaPipelineTask.
56 """
57 diaSourceSchema = connTypes.InitInput(
58 doc="Schema of the DiaSource catalog produced during image "
59 "differencing",
60 name="{coaddName}Diff_diaSrc_schema",
61 storageClass="SourceCatalog",
62 multiple=True
63 )
64 diaSourceCat = connTypes.Input(
65 doc="Catalog of DiaSources produced during image differencing.",
66 name="{coaddName}Diff_diaSrc",
67 storageClass="SourceCatalog",
68 dimensions=("instrument", "visit", "detector"),
69 )
70 diffIm = connTypes.Input(
71 doc="Difference image on which the DiaSources were detected.",
72 name="{coaddName}Diff_differenceExp",
73 storageClass="ExposureF",
74 dimensions=("instrument", "visit", "detector"),
75 )
76 exposure = connTypes.Input(
77 doc="Calibrated exposure differenced with a template image during "
78 "image differencing.",
79 name="calexp",
80 storageClass="ExposureF",
81 dimensions=("instrument", "visit", "detector"),
82 )
83 apdbMarker = connTypes.Output(
84 doc="Marker dataset storing the configuration of the Apdb for each "
85 "visit/detector. Used to signal the completion of the pipeline.",
86 name="apdb_marker",
87 storageClass="",
88 dimensions=("instrument", "visit", "detector"),
89 )
92class DiaPipelineConfig(pipeBase.PipelineTaskConfig,
93 pipelineConnections=DiaPipelineConnections):
94 """Config for DiaPipelineTask.
95 """
96 apdb = pexConfig.ConfigurableField(
97 target=daxApdb.Apdb,
98 ConfigClass=daxApdb.ApdbConfig,
99 doc="Database connection for storing associated DiaSources and "
100 "DiaObjects. Must already be initialized.",
101 )
102 diaSourceDpddifier = pexConfig.ConfigurableField(
103 target=MapDiaSourceTask,
104 doc="Task for assigning columns from the raw output of ip_diffim into "
105 "a schema that more closely resembles the DPDD.",
106 )
107 diaCatalogLoader = pexConfig.ConfigurableField(
108 target=LoadDiaCatalogsTask,
109 doc="Task to load DiaObjects and DiaSources from the Apdb.",
110 )
111 associator = pexConfig.ConfigurableField(
112 target=AssociationTask,
113 doc="Task used to associate DiaSources with DiaObjects.",
114 )
115 diaForcedSource = pexConfig.ConfigurableField(
116 target=DiaForcedSourceTask,
117 doc="Task used for force photometer DiaObject locations in direct and "
118 "difference images.",
119 )
121 def setDefaults(self):
122 self.apdb.dia_object_index = "baseline"
123 self.apdb.dia_object_columns = []
124 self.apdb.extra_schema_file = os.path.join(
125 getPackageDir("ap_association"),
126 "data",
127 "apdb-ap-pipe-schema-extra.yaml")
129 def validate(self):
130 pexConfig.Config.validate(self)
131 if self.diaCatalogLoader.htmLevel != \
132 self.associator.diaCalculation.plugins["ap_HTMIndex"].htmLevel:
133 raise ValueError("HTM index level in LoadDiaCatalogsTask must be "
134 "equal to HTMIndexDiaCalculationPlugin index "
135 "level.")
138class DiaPipelineTask(pipeBase.PipelineTask):
139 """Task for loading, associating and storing Difference Image Analysis
140 (DIA) Objects and Sources.
141 """
142 ConfigClass = DiaPipelineConfig
143 _DefaultName = "diaPipe"
144 RunnerClass = pipeBase.ButlerInitializedTaskRunner
146 def __init__(self, initInputs=None, **kwargs):
147 super().__init__(**kwargs)
148 self.apdb = self.config.apdb.apply(
149 afw_schemas=dict(DiaObject=make_dia_object_schema(),
150 DiaSource=make_dia_source_schema()))
151 self.makeSubtask("diaSourceDpddifier",
152 inputSchema=initInputs["diaSourceSchema"])
153 self.makeSubtask("diaCatalogLoader")
154 self.makeSubtask("associator")
155 self.makeSubtask("diaForcedSource")
157 def runQuantum(self, butlerQC, inputRefs, outputRefs):
158 inputs = butlerQC.get(inputRefs)
159 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
160 returnMaxBits=True)
161 inputs["ccdExposureIdBits"] = expBits
163 outputs = self.run(**inputs)
165 butlerQC.put(outputs, outputRefs)
167 @pipeBase.timeMethod
168 def run(self, diaSourceCat, diffIm, exposure, ccdExposureIdBits):
169 """Process DiaSources and DiaObjects.
171 Load previous DiaObjects and their DiaSource history. Calibrate the
172 values in the diaSourceCat. Associate new DiaSources with previous
173 DiaObjects. Run forced photometry at the updated DiaObject locations.
174 Store the results in the Alert Production Database (Apdb).
176 Parameters
177 ----------
178 diaSourceCat : `lsst.afw.table.SourceCatalog`
179 Newly detected DiaSources.
180 diffIm : `lsst.afw.image.Exposure`
181 Difference image exposure in which the sources in ``diaSourceCat``
182 were detected.
183 exposure : `lsst.afw.image.Exposure`
184 Calibrated exposure differenced with a template to create
185 ``diffIm``.
186 ccdExposureIdBits : `int`
187 Number of bits used for a unique ``ccdVisitId``.
189 Returns
190 -------
191 results : `lsst.pipe.base.Struct`
192 Results struct with components.
194 - ``apdb_maker`` : Marker dataset to store in the Butler indicating
195 that this ccdVisit has completed successfully.
196 (`lsst.dax.apdb.ApdbConfig`)
197 """
198 self.log.info("Running DiaPipeline...")
199 # Put the SciencePipelines through a SDMification step and return
200 # calibrated columns with the expect output database names.
201 diaSources = self.diaSourceDpddifier.run(diaSourceCat,
202 diffIm,
203 return_pandas=True)
205 # Load the DiaObjects and DiaSource history.
206 loaderResult = self.diaCatalogLoader.run(diffIm, self.apdb)
208 # Associate new DiaSources with existing DiaObjects and update
209 # DiaObject summary statistics using the full DiaSource history.
210 assocResults = self.associator.run(diaSources,
211 loaderResult.diaObjects,
212 loaderResult.diaSources)
214 # Force photometer on the Difference and Calibrated exposures using
215 # the new and updated DiaObject locations.
216 diaForcedSources = self.diaForcedSource.run(
217 assocResults.diaObjects,
218 ccdExposureIdBits,
219 exposure,
220 diffIm)
222 # Store DiaSources and updated DiaObjects in the Apdb.
223 self.apdb.storeDiaSources(assocResults.diaSources)
224 self.apdb.storeDiaObjects(
225 assocResults.updatedDiaObjects,
226 exposure.getInfo().getVisitInfo().getDate().toPython())
227 self.apdb.storeDiaForcedSources(diaForcedSources)
229 return pipeBase.Struct(apdb_marker=self.config.apdb.value)