Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 32%
124 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-02 07:21 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-02 07:21 -0800
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import lsst.afw.table as afwTable
23import lsst.daf.base as dafBase
24from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
25from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
26import lsst.meas.extensions.trailedSources # noqa: F401
27import lsst.meas.extensions.shapeHSM
28from lsst.obs.base import ExposureIdInfo
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31import lsst.utils
32from lsst.utils.timer import timeMethod
34from . import DipoleFitTask
36__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask"]
39class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
40 dimensions=("instrument", "visit", "detector"),
41 defaultTemplates={"coaddName": "deep",
42 "warpTypeSuffix": "",
43 "fakesType": ""}):
44 science = pipeBase.connectionTypes.Input(
45 doc="Input science exposure.",
46 dimensions=("instrument", "visit", "detector"),
47 storageClass="ExposureF",
48 name="{fakesType}calexp"
49 )
50 matchedTemplate = pipeBase.connectionTypes.Input(
51 doc="Warped and PSF-matched template used to create the difference image.",
52 dimensions=("instrument", "visit", "detector"),
53 storageClass="ExposureF",
54 name="{fakesType}{coaddName}Diff_matchedExp",
55 )
56 difference = pipeBase.connectionTypes.Input(
57 doc="Result of subtracting template from science.",
58 dimensions=("instrument", "visit", "detector"),
59 storageClass="ExposureF",
60 name="{fakesType}{coaddName}Diff_differenceTempExp",
61 )
62 outputSchema = pipeBase.connectionTypes.InitOutput(
63 doc="Schema (as an example catalog) for output DIASource catalog.",
64 storageClass="SourceCatalog",
65 name="{fakesType}{coaddName}Diff_diaSrc_schema",
66 )
67 diaSources = pipeBase.connectionTypes.Output(
68 doc="Detected diaSources on the difference image.",
69 dimensions=("instrument", "visit", "detector"),
70 storageClass="SourceCatalog",
71 name="{fakesType}{coaddName}Diff_diaSrc",
72 )
73 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
74 doc="Difference image with detection mask plane filled in.",
75 dimensions=("instrument", "visit", "detector"),
76 storageClass="ExposureF",
77 name="{fakesType}{coaddName}Diff_differenceExp",
78 )
81class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
82 pipelineConnections=DetectAndMeasureConnections):
83 """Config for DetectAndMeasureTask
84 """
85 doMerge = pexConfig.Field(
86 dtype=bool,
87 default=True,
88 doc="Merge positive and negative diaSources with grow radius "
89 "set by growFootprint"
90 )
91 doForcedMeasurement = pexConfig.Field(
92 dtype=bool,
93 default=True,
94 doc="Force photometer diaSource locations on PVI?")
95 doAddMetrics = pexConfig.Field(
96 dtype=bool,
97 default=False,
98 doc="Add columns to the source table to hold analysis metrics?"
99 )
100 detection = pexConfig.ConfigurableField(
101 target=SourceDetectionTask,
102 doc="Final source detection for diaSource measurement",
103 )
104 measurement = pexConfig.ConfigurableField(
105 target=DipoleFitTask,
106 doc="Task to measure sources on the difference image.",
107 )
108 doApCorr = lsst.pex.config.Field(
109 dtype=bool,
110 default=True,
111 doc="Run subtask to apply aperture corrections"
112 )
113 applyApCorr = lsst.pex.config.ConfigurableField(
114 target=ApplyApCorrTask,
115 doc="Task to apply aperture corrections"
116 )
117 forcedMeasurement = pexConfig.ConfigurableField(
118 target=ForcedMeasurementTask,
119 doc="Task to force photometer science image at diaSource locations.",
120 )
121 growFootprint = pexConfig.Field(
122 dtype=int,
123 default=2,
124 doc="Grow positive and negative footprints by this many pixels before merging"
125 )
126 diaSourceMatchRadius = pexConfig.Field(
127 dtype=float,
128 default=0.5,
129 doc="Match radius (in arcseconds) for DiaSource to Source association"
130 )
131 doSkySources = pexConfig.Field(
132 dtype=bool,
133 default=False,
134 doc="Generate sky sources?",
135 )
136 skySources = pexConfig.ConfigurableField(
137 target=SkyObjectsTask,
138 doc="Generate sky sources",
139 )
141 def setDefaults(self):
142 # DiaSource Detection
143 self.detection.thresholdPolarity = "both"
144 self.detection.thresholdValue = 5.0
145 self.detection.reEstimateBackground = False
146 self.detection.thresholdType = "pixel_stdev"
148 # Add filtered flux measurement, the correct measurement for pre-convolved images.
149 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
150 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
151 'base_LocalPhotoCalib',
152 'base_LocalWcs',
153 'ext_shapeHSM_HsmSourceMoments',
154 'ext_shapeHSM_HsmPsfMoments',
155 ]
156 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
157 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
159 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
160 self.forcedMeasurement.copyColumns = {
161 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
162 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
163 self.forcedMeasurement.slots.shape = None
166class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
167 """Detect and measure sources on a difference image.
168 """
169 ConfigClass = DetectAndMeasureConfig
170 _DefaultName = "detectAndMeasure"
172 def __init__(self, **kwargs):
173 super().__init__(**kwargs)
174 self.schema = afwTable.SourceTable.makeMinimalSchema()
176 self.algMetadata = dafBase.PropertyList()
177 self.makeSubtask("detection", schema=self.schema)
178 self.makeSubtask("measurement", schema=self.schema,
179 algMetadata=self.algMetadata)
180 if self.config.doApCorr:
181 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
182 if self.config.doForcedMeasurement:
183 self.schema.addField(
184 "ip_diffim_forced_PsfFlux_instFlux", "D",
185 "Forced PSF flux measured on the direct image.",
186 units="count")
187 self.schema.addField(
188 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
189 "Forced PSF flux error measured on the direct image.",
190 units="count")
191 self.schema.addField(
192 "ip_diffim_forced_PsfFlux_area", "F",
193 "Forced PSF flux effective area of PSF.",
194 units="pixel")
195 self.schema.addField(
196 "ip_diffim_forced_PsfFlux_flag", "Flag",
197 "Forced PSF flux general failure flag.")
198 self.schema.addField(
199 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
200 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
201 self.schema.addField(
202 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
203 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
204 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
206 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
207 self.schema.addField("srcMatchId", "L", "unique id of source match")
208 if self.config.doSkySources:
209 self.makeSubtask("skySources")
210 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
212 # initialize InitOutputs
213 self.outputSchema = afwTable.SourceCatalog(self.schema)
214 self.outputSchema.getTable().setMetadata(self.algMetadata)
216 @staticmethod
217 def makeIdFactory(expId, expBits):
218 """Create IdFactory instance for unique 64 bit diaSource id-s.
220 Parameters
221 ----------
222 expId : `int`
223 Exposure id.
225 expBits: `int`
226 Number of used bits in ``expId``.
228 Notes
229 -----
230 The diasource id-s consists of the ``expId`` stored fixed in the highest value
231 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
232 low value end of the integer.
234 Returns
235 -------
236 idFactory: `lsst.afw.table.IdFactory`
237 """
238 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
240 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
241 inputRefs: pipeBase.InputQuantizedConnection,
242 outputRefs: pipeBase.OutputQuantizedConnection):
243 inputs = butlerQC.get(inputRefs)
244 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
245 returnMaxBits=True)
246 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
248 outputs = self.run(inputs['science'],
249 inputs['matchedTemplate'],
250 inputs['difference'],
251 idFactory=idFactory)
252 butlerQC.put(outputs, outputRefs)
254 @timeMethod
255 def run(self, science, matchedTemplate, difference,
256 idFactory=None):
257 """Detect and measure sources on a difference image.
259 Parameters
260 ----------
261 science : `lsst.afw.image.ExposureF`
262 Science exposure that the template was subtracted from.
263 matchedTemplate : `lsst.afw.image.ExposureF`
264 Warped and PSF-matched template that was used produce the
265 difference image.
266 difference : `lsst.afw.image.ExposureF`
267 Result of subtracting template from the science image.
268 idFactory : `lsst.afw.table.IdFactory`, optional
269 Generator object to assign ids to detected sources in the difference image.
271 Returns
272 -------
273 results : `lsst.pipe.base.Struct`
275 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
276 Subtracted exposure with detection mask applied.
277 ``diaSources`` : `lsst.afw.table.SourceCatalog`
278 The catalog of detected sources.
279 """
280 # Ensure that we start with an empty detection mask.
281 mask = difference.mask
282 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
284 table = afwTable.SourceTable.make(self.schema, idFactory)
285 table.setMetadata(self.algMetadata)
286 results = self.detection.run(
287 table=table,
288 exposure=difference,
289 doSmooth=True,
290 )
292 if self.config.doMerge:
293 fpSet = results.fpSets.positive
294 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
295 self.config.growFootprint, False)
296 diaSources = afwTable.SourceCatalog(table)
297 fpSet.makeSources(diaSources)
298 self.log.info("Merging detections into %d sources", len(diaSources))
299 else:
300 diaSources = results.sources
302 if self.config.doSkySources:
303 self.addSkySources(diaSources, difference.mask, difference.info.id)
305 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
307 if self.config.doForcedMeasurement:
308 self.measureForcedSources(diaSources, science, difference.getWcs())
310 return pipeBase.Struct(
311 subtractedMeasuredExposure=difference,
312 diaSources=diaSources,
313 )
315 def addSkySources(self, diaSources, mask, seed):
316 """Add sources in empty regions of the difference image
317 for measuring the background.
319 Parameters
320 ----------
321 diaSources : `lsst.afw.table.SourceCatalog`
322 The catalog of detected sources.
323 mask : `lsst.afw.image.Mask`
324 Mask plane for determining regions where Sky sources can be added.
325 seed : `int`
326 Seed value to initialize the random number generator.
327 """
328 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
329 if skySourceFootprints:
330 for foot in skySourceFootprints:
331 s = diaSources.addNew()
332 s.setFootprint(foot)
333 s.set(self.skySourceKey, True)
335 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
336 """Use (matched) template and science image to constrain dipole fitting.
338 Parameters
339 ----------
340 diaSources : `lsst.afw.table.SourceCatalog`
341 The catalog of detected sources.
342 science : `lsst.afw.image.ExposureF`
343 Science exposure that the template was subtracted from.
344 difference : `lsst.afw.image.ExposureF`
345 Result of subtracting template from the science image.
346 matchedTemplate : `lsst.afw.image.ExposureF`
347 Warped and PSF-matched template that was used produce the
348 difference image.
349 """
350 # Note that this may not be correct if we convolved the science image.
351 # In the future we may wish to persist the matchedScience image.
352 self.measurement.run(diaSources, difference, science, matchedTemplate)
353 if self.config.doApCorr:
354 self.applyApCorr.run(
355 catalog=diaSources,
356 apCorrMap=difference.getInfo().getApCorrMap()
357 )
359 def measureForcedSources(self, diaSources, science, wcs):
360 """Perform forced measurement of the diaSources on the science image.
362 Parameters
363 ----------
364 diaSources : `lsst.afw.table.SourceCatalog`
365 The catalog of detected sources.
366 science : `lsst.afw.image.ExposureF`
367 Science exposure that the template was subtracted from.
368 wcs : `lsst.afw.geom.SkyWcs`
369 Coordinate system definition (wcs) for the exposure.
370 """
371 # Run forced psf photometry on the PVI at the diaSource locations.
372 # Copy the measured flux and error into the diaSource.
373 forcedSources = self.forcedMeasurement.generateMeasCat(
374 science, diaSources, wcs)
375 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
376 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
377 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
378 "ip_diffim_forced_PsfFlux_instFlux", True)
379 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
380 "ip_diffim_forced_PsfFlux_instFluxErr", True)
381 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
382 "ip_diffim_forced_PsfFlux_area", True)
383 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
384 "ip_diffim_forced_PsfFlux_flag", True)
385 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
386 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
387 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
388 "ip_diffim_forced_PsfFlux_flag_edge", True)
389 for diaSource, forcedSource in zip(diaSources, forcedSources):
390 diaSource.assign(forcedSource, mapper)