Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 38%
152 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-25 17:01 +0000
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from deprecated.sphinx import deprecated
24import lsst.afw.table as afwTable
25import lsst.daf.base as dafBase
26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources # noqa: F401
29import lsst.meas.extensions.shapeHSM
30from lsst.obs.base import ExposureIdInfo
31import lsst.pex.config as pexConfig
32import lsst.pipe.base as pipeBase
33import lsst.utils
34from lsst.utils.timer import timeMethod
36from . import DipoleFitTask
38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
43 dimensions=("instrument", "visit", "detector"),
44 defaultTemplates={"coaddName": "deep",
45 "warpTypeSuffix": "",
46 "fakesType": ""}):
47 science = pipeBase.connectionTypes.Input(
48 doc="Input science exposure.",
49 dimensions=("instrument", "visit", "detector"),
50 storageClass="ExposureF",
51 name="{fakesType}calexp"
52 )
53 matchedTemplate = pipeBase.connectionTypes.Input(
54 doc="Warped and PSF-matched template used to create the difference image.",
55 dimensions=("instrument", "visit", "detector"),
56 storageClass="ExposureF",
57 name="{fakesType}{coaddName}Diff_matchedExp",
58 )
59 difference = pipeBase.connectionTypes.Input(
60 doc="Result of subtracting template from science.",
61 dimensions=("instrument", "visit", "detector"),
62 storageClass="ExposureF",
63 name="{fakesType}{coaddName}Diff_differenceTempExp",
64 )
65 outputSchema = pipeBase.connectionTypes.InitOutput(
66 doc="Schema (as an example catalog) for output DIASource catalog.",
67 storageClass="SourceCatalog",
68 name="{fakesType}{coaddName}Diff_diaSrc_schema",
69 )
70 diaSources = pipeBase.connectionTypes.Output(
71 doc="Detected diaSources on the difference image.",
72 dimensions=("instrument", "visit", "detector"),
73 storageClass="SourceCatalog",
74 name="{fakesType}{coaddName}Diff_diaSrc",
75 )
76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
77 doc="Difference image with detection mask plane filled in.",
78 dimensions=("instrument", "visit", "detector"),
79 storageClass="ExposureF",
80 name="{fakesType}{coaddName}Diff_differenceExp",
81 )
84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
85 pipelineConnections=DetectAndMeasureConnections):
86 """Config for DetectAndMeasureTask
87 """
88 doMerge = pexConfig.Field(
89 dtype=bool,
90 default=True,
91 doc="Merge positive and negative diaSources with grow radius "
92 "set by growFootprint"
93 )
94 doForcedMeasurement = pexConfig.Field(
95 dtype=bool,
96 default=True,
97 doc="Force photometer diaSource locations on PVI?")
98 doAddMetrics = pexConfig.Field(
99 dtype=bool,
100 default=False,
101 doc="Add columns to the source table to hold analysis metrics?"
102 )
103 detection = pexConfig.ConfigurableField(
104 target=SourceDetectionTask,
105 doc="Final source detection for diaSource measurement",
106 )
107 measurement = pexConfig.ConfigurableField(
108 target=DipoleFitTask,
109 doc="Task to measure sources on the difference image.",
110 )
111 doApCorr = lsst.pex.config.Field(
112 dtype=bool,
113 default=True,
114 doc="Run subtask to apply aperture corrections"
115 )
116 applyApCorr = lsst.pex.config.ConfigurableField(
117 target=ApplyApCorrTask,
118 doc="Task to apply aperture corrections"
119 )
120 forcedMeasurement = pexConfig.ConfigurableField(
121 target=ForcedMeasurementTask,
122 doc="Task to force photometer science image at diaSource locations.",
123 )
124 growFootprint = pexConfig.Field(
125 dtype=int,
126 default=2,
127 doc="Grow positive and negative footprints by this many pixels before merging"
128 )
129 diaSourceMatchRadius = pexConfig.Field(
130 dtype=float,
131 default=0.5,
132 doc="Match radius (in arcseconds) for DiaSource to Source association"
133 )
134 doSkySources = pexConfig.Field(
135 dtype=bool,
136 default=False,
137 doc="Generate sky sources?",
138 )
139 skySources = pexConfig.ConfigurableField(
140 target=SkyObjectsTask,
141 doc="Generate sky sources",
142 )
143 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
145 def setDefaults(self):
146 # DiaSource Detection
147 self.detection.thresholdPolarity = "both"
148 self.detection.thresholdValue = 5.0
149 self.detection.reEstimateBackground = False
150 self.detection.thresholdType = "pixel_stdev"
151 self.detection.excludeMaskPlanes = ["EDGE"]
153 # Add filtered flux measurement, the correct measurement for pre-convolved images.
154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
155 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
156 'base_LocalPhotoCalib',
157 'base_LocalWcs',
158 'ext_shapeHSM_HsmSourceMoments',
159 'ext_shapeHSM_HsmPsfMoments',
160 ]
161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
163 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0
164 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
165 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
166 self.forcedMeasurement.copyColumns = {
167 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
168 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
169 self.forcedMeasurement.slots.shape = None
172class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
173 """Detect and measure sources on a difference image.
174 """
175 ConfigClass = DetectAndMeasureConfig
176 _DefaultName = "detectAndMeasure"
178 def __init__(self, **kwargs):
179 super().__init__(**kwargs)
180 self.schema = afwTable.SourceTable.makeMinimalSchema()
182 self.algMetadata = dafBase.PropertyList()
183 self.makeSubtask("detection", schema=self.schema)
184 self.makeSubtask("measurement", schema=self.schema,
185 algMetadata=self.algMetadata)
186 if self.config.doApCorr:
187 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
188 if self.config.doForcedMeasurement:
189 self.schema.addField(
190 "ip_diffim_forced_PsfFlux_instFlux", "D",
191 "Forced PSF flux measured on the direct image.",
192 units="count")
193 self.schema.addField(
194 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
195 "Forced PSF flux error measured on the direct image.",
196 units="count")
197 self.schema.addField(
198 "ip_diffim_forced_PsfFlux_area", "F",
199 "Forced PSF flux effective area of PSF.",
200 units="pixel")
201 self.schema.addField(
202 "ip_diffim_forced_PsfFlux_flag", "Flag",
203 "Forced PSF flux general failure flag.")
204 self.schema.addField(
205 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
206 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
207 self.schema.addField(
208 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
209 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
210 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
212 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
213 self.schema.addField("srcMatchId", "L", "unique id of source match")
214 if self.config.doSkySources:
215 self.makeSubtask("skySources")
216 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
218 # initialize InitOutputs
219 self.outputSchema = afwTable.SourceCatalog(self.schema)
220 self.outputSchema.getTable().setMetadata(self.algMetadata)
222 # TODO: remove on DM-38687.
223 @staticmethod
224 @deprecated(
225 reason=(
226 "ID factory construction now depends on configuration; use the "
227 "idGenerator config field. Will be removed after v26."
228 ),
229 version="v26.0",
230 category=FutureWarning,
231 )
232 def makeIdFactory(expId, expBits):
233 """Create IdFactory instance for unique 64 bit diaSource id-s.
235 Parameters
236 ----------
237 expId : `int`
238 Exposure id.
240 expBits: `int`
241 Number of used bits in ``expId``.
243 Notes
244 -----
245 The diasource id-s consists of the ``expId`` stored fixed in the highest value
246 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
247 low value end of the integer.
249 Returns
250 -------
251 idFactory: `lsst.afw.table.IdFactory`
252 """
253 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
255 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
256 inputRefs: pipeBase.InputQuantizedConnection,
257 outputRefs: pipeBase.OutputQuantizedConnection):
258 inputs = butlerQC.get(inputRefs)
259 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
260 idFactory = idGenerator.make_table_id_factory()
261 outputs = self.run(**inputs, idFactory=idFactory)
262 butlerQC.put(outputs, outputRefs)
264 @timeMethod
265 def run(self, science, matchedTemplate, difference,
266 idFactory=None):
267 """Detect and measure sources on a difference image.
269 The difference image will be convolved with a gaussian approximation of
270 the PSF to form a maximum likelihood image for detection.
271 Close positive and negative detections will optionally be merged into
272 dipole diaSources.
273 Sky sources, or forced detections in background regions, will optionally
274 be added, and the configured measurement algorithm will be run on all
275 detections.
277 Parameters
278 ----------
279 science : `lsst.afw.image.ExposureF`
280 Science exposure that the template was subtracted from.
281 matchedTemplate : `lsst.afw.image.ExposureF`
282 Warped and PSF-matched template that was used produce the
283 difference image.
284 difference : `lsst.afw.image.ExposureF`
285 Result of subtracting template from the science image.
286 idFactory : `lsst.afw.table.IdFactory`, optional
287 Generator object to assign ids to detected sources in the difference image.
289 Returns
290 -------
291 measurementResults : `lsst.pipe.base.Struct`
293 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
294 Subtracted exposure with detection mask applied.
295 ``diaSources`` : `lsst.afw.table.SourceCatalog`
296 The catalog of detected sources.
297 """
298 # Ensure that we start with an empty detection mask.
299 mask = difference.mask
300 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
302 table = afwTable.SourceTable.make(self.schema, idFactory)
303 table.setMetadata(self.algMetadata)
304 results = self.detection.run(
305 table=table,
306 exposure=difference,
307 doSmooth=True,
308 )
310 return self.processResults(science, matchedTemplate, difference, results.sources, table,
311 positiveFootprints=results.positive, negativeFootprints=results.negative)
313 def processResults(self, science, matchedTemplate, difference, sources, table,
314 positiveFootprints=None, negativeFootprints=None,):
315 """Measure and process the results of source detection.
317 Parameters
318 ----------
319 sources : `lsst.afw.table.SourceCatalog`
320 Detected sources on the difference exposure.
321 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
322 Positive polarity footprints.
323 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
324 Negative polarity footprints.
325 table : `lsst.afw.table.SourceTable`
326 Table object that will be used to create the SourceCatalog.
327 science : `lsst.afw.image.ExposureF`
328 Science exposure that the template was subtracted from.
329 matchedTemplate : `lsst.afw.image.ExposureF`
330 Warped and PSF-matched template that was used produce the
331 difference image.
332 difference : `lsst.afw.image.ExposureF`
333 Result of subtracting template from the science image.
335 Returns
336 -------
337 measurementResults : `lsst.pipe.base.Struct`
339 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
340 Subtracted exposure with detection mask applied.
341 ``diaSources`` : `lsst.afw.table.SourceCatalog`
342 The catalog of detected sources.
343 """
344 if self.config.doMerge:
345 fpSet = positiveFootprints
346 fpSet.merge(negativeFootprints, self.config.growFootprint,
347 self.config.growFootprint, False)
348 diaSources = afwTable.SourceCatalog(table)
349 fpSet.makeSources(diaSources)
350 self.log.info("Merging detections into %d sources", len(diaSources))
351 else:
352 diaSources = sources
354 if self.config.doSkySources:
355 self.addSkySources(diaSources, difference.mask, difference.info.id)
357 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
359 if self.config.doForcedMeasurement:
360 self.measureForcedSources(diaSources, science, difference.getWcs())
362 measurementResults = pipeBase.Struct(
363 subtractedMeasuredExposure=difference,
364 diaSources=diaSources,
365 )
367 return measurementResults
369 def addSkySources(self, diaSources, mask, seed):
370 """Add sources in empty regions of the difference image
371 for measuring the background.
373 Parameters
374 ----------
375 diaSources : `lsst.afw.table.SourceCatalog`
376 The catalog of detected sources.
377 mask : `lsst.afw.image.Mask`
378 Mask plane for determining regions where Sky sources can be added.
379 seed : `int`
380 Seed value to initialize the random number generator.
381 """
382 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
383 if skySourceFootprints:
384 for foot in skySourceFootprints:
385 s = diaSources.addNew()
386 s.setFootprint(foot)
387 s.set(self.skySourceKey, True)
389 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
390 """Use (matched) template and science image to constrain dipole fitting.
392 Parameters
393 ----------
394 diaSources : `lsst.afw.table.SourceCatalog`
395 The catalog of detected sources.
396 science : `lsst.afw.image.ExposureF`
397 Science exposure that the template was subtracted from.
398 difference : `lsst.afw.image.ExposureF`
399 Result of subtracting template from the science image.
400 matchedTemplate : `lsst.afw.image.ExposureF`
401 Warped and PSF-matched template that was used produce the
402 difference image.
403 """
404 # Note that this may not be correct if we convolved the science image.
405 # In the future we may wish to persist the matchedScience image.
406 self.measurement.run(diaSources, difference, science, matchedTemplate)
407 if self.config.doApCorr:
408 apCorrMap = difference.getInfo().getApCorrMap()
409 if apCorrMap is None:
410 self.log.warning("Difference image does not have valid aperture correction; skipping.")
411 else:
412 self.applyApCorr.run(
413 catalog=diaSources,
414 apCorrMap=apCorrMap,
415 )
417 def measureForcedSources(self, diaSources, science, wcs):
418 """Perform forced measurement of the diaSources on the science image.
420 Parameters
421 ----------
422 diaSources : `lsst.afw.table.SourceCatalog`
423 The catalog of detected sources.
424 science : `lsst.afw.image.ExposureF`
425 Science exposure that the template was subtracted from.
426 wcs : `lsst.afw.geom.SkyWcs`
427 Coordinate system definition (wcs) for the exposure.
428 """
429 # Run forced psf photometry on the PVI at the diaSource locations.
430 # Copy the measured flux and error into the diaSource.
431 forcedSources = self.forcedMeasurement.generateMeasCat(
432 science, diaSources, wcs)
433 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
434 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
435 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
436 "ip_diffim_forced_PsfFlux_instFlux", True)
437 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
438 "ip_diffim_forced_PsfFlux_instFluxErr", True)
439 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
440 "ip_diffim_forced_PsfFlux_area", True)
441 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
442 "ip_diffim_forced_PsfFlux_flag", True)
443 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
444 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
445 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
446 "ip_diffim_forced_PsfFlux_flag_edge", True)
447 for diaSource, forcedSource in zip(diaSources, forcedSources):
448 diaSource.assign(forcedSource, mapper)
451class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
452 scoreExposure = pipeBase.connectionTypes.Input(
453 doc="Maximum likelihood image for detection.",
454 dimensions=("instrument", "visit", "detector"),
455 storageClass="ExposureF",
456 name="{fakesType}{coaddName}Diff_scoreExp",
457 )
460class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
461 pipelineConnections=DetectAndMeasureScoreConnections):
462 pass
465class DetectAndMeasureScoreTask(DetectAndMeasureTask):
466 """Detect DIA sources using a score image,
467 and measure the detections on the difference image.
469 Source detection is run on the supplied score, or maximum likelihood,
470 image. Note that no additional convolution will be done in this case.
471 Close positive and negative detections will optionally be merged into
472 dipole diaSources.
473 Sky sources, or forced detections in background regions, will optionally
474 be added, and the configured measurement algorithm will be run on all
475 detections.
476 """
477 ConfigClass = DetectAndMeasureScoreConfig
478 _DefaultName = "detectAndMeasureScore"
480 @timeMethod
481 def run(self, science, matchedTemplate, difference, scoreExposure,
482 idFactory=None):
483 """Detect and measure sources on a score image.
485 Parameters
486 ----------
487 science : `lsst.afw.image.ExposureF`
488 Science exposure that the template was subtracted from.
489 matchedTemplate : `lsst.afw.image.ExposureF`
490 Warped and PSF-matched template that was used produce the
491 difference image.
492 difference : `lsst.afw.image.ExposureF`
493 Result of subtracting template from the science image.
494 scoreExposure : `lsst.afw.image.ExposureF`
495 Score or maximum likelihood difference image
496 idFactory : `lsst.afw.table.IdFactory`, optional
497 Generator object to assign ids to detected sources in the difference image.
499 Returns
500 -------
501 measurementResults : `lsst.pipe.base.Struct`
503 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
504 Subtracted exposure with detection mask applied.
505 ``diaSources`` : `lsst.afw.table.SourceCatalog`
506 The catalog of detected sources.
507 """
508 # Ensure that we start with an empty detection mask.
509 mask = scoreExposure.mask
510 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
512 table = afwTable.SourceTable.make(self.schema, idFactory)
513 table.setMetadata(self.algMetadata)
514 results = self.detection.run(
515 table=table,
516 exposure=scoreExposure,
517 doSmooth=False,
518 )
519 # Copy the detection mask from the Score image to the difference image
520 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
522 return self.processResults(science, matchedTemplate, difference, results.sources, table,
523 positiveFootprints=results.positive, negativeFootprints=results.negative)