Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 35%
172 statements
« prev ^ index » next coverage.py v7.3.3, created at 2023-12-16 13:38 +0000
« prev ^ index » next coverage.py v7.3.3, created at 2023-12-16 13:38 +0000
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from deprecated.sphinx import deprecated
23import numpy as np
25import lsst.afw.table as afwTable
26import lsst.daf.base as dafBase
27from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
28from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
29import lsst.meas.extensions.trailedSources # noqa: F401
30import lsst.meas.extensions.shapeHSM
31from lsst.obs.base import ExposureIdInfo
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34import lsst.utils
35from lsst.utils.timer import timeMethod
37from . import DipoleFitTask
39__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
40 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
43class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
44 dimensions=("instrument", "visit", "detector"),
45 defaultTemplates={"coaddName": "deep",
46 "warpTypeSuffix": "",
47 "fakesType": ""}):
48 science = pipeBase.connectionTypes.Input(
49 doc="Input science exposure.",
50 dimensions=("instrument", "visit", "detector"),
51 storageClass="ExposureF",
52 name="{fakesType}calexp"
53 )
54 matchedTemplate = pipeBase.connectionTypes.Input(
55 doc="Warped and PSF-matched template used to create the difference image.",
56 dimensions=("instrument", "visit", "detector"),
57 storageClass="ExposureF",
58 name="{fakesType}{coaddName}Diff_matchedExp",
59 )
60 difference = pipeBase.connectionTypes.Input(
61 doc="Result of subtracting template from science.",
62 dimensions=("instrument", "visit", "detector"),
63 storageClass="ExposureF",
64 name="{fakesType}{coaddName}Diff_differenceTempExp",
65 )
66 outputSchema = pipeBase.connectionTypes.InitOutput(
67 doc="Schema (as an example catalog) for output DIASource catalog.",
68 storageClass="SourceCatalog",
69 name="{fakesType}{coaddName}Diff_diaSrc_schema",
70 )
71 diaSources = pipeBase.connectionTypes.Output(
72 doc="Detected diaSources on the difference image.",
73 dimensions=("instrument", "visit", "detector"),
74 storageClass="SourceCatalog",
75 name="{fakesType}{coaddName}Diff_diaSrc",
76 )
77 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
78 doc="Difference image with detection mask plane filled in.",
79 dimensions=("instrument", "visit", "detector"),
80 storageClass="ExposureF",
81 name="{fakesType}{coaddName}Diff_differenceExp",
82 )
85class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
86 pipelineConnections=DetectAndMeasureConnections):
87 """Config for DetectAndMeasureTask
88 """
89 doMerge = pexConfig.Field(
90 dtype=bool,
91 default=True,
92 doc="Merge positive and negative diaSources with grow radius "
93 "set by growFootprint"
94 )
95 doForcedMeasurement = pexConfig.Field(
96 dtype=bool,
97 default=True,
98 doc="Force photometer diaSource locations on PVI?")
99 doAddMetrics = pexConfig.Field(
100 dtype=bool,
101 default=False,
102 doc="Add columns to the source table to hold analysis metrics?"
103 )
104 detection = pexConfig.ConfigurableField(
105 target=SourceDetectionTask,
106 doc="Final source detection for diaSource measurement",
107 )
108 measurement = pexConfig.ConfigurableField(
109 target=DipoleFitTask,
110 doc="Task to measure sources on the difference image.",
111 )
112 doApCorr = lsst.pex.config.Field(
113 dtype=bool,
114 default=True,
115 doc="Run subtask to apply aperture corrections"
116 )
117 applyApCorr = lsst.pex.config.ConfigurableField(
118 target=ApplyApCorrTask,
119 doc="Task to apply aperture corrections"
120 )
121 forcedMeasurement = pexConfig.ConfigurableField(
122 target=ForcedMeasurementTask,
123 doc="Task to force photometer science image at diaSource locations.",
124 )
125 growFootprint = pexConfig.Field(
126 dtype=int,
127 default=2,
128 doc="Grow positive and negative footprints by this many pixels before merging"
129 )
130 diaSourceMatchRadius = pexConfig.Field(
131 dtype=float,
132 default=0.5,
133 doc="Match radius (in arcseconds) for DiaSource to Source association"
134 )
135 doSkySources = pexConfig.Field(
136 dtype=bool,
137 default=False,
138 doc="Generate sky sources?",
139 )
140 skySources = pexConfig.ConfigurableField(
141 target=SkyObjectsTask,
142 doc="Generate sky sources",
143 )
144 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
146 def setDefaults(self):
147 # DiaSource Detection
148 self.detection.thresholdPolarity = "both"
149 self.detection.thresholdValue = 5.0
150 self.detection.reEstimateBackground = False
151 self.detection.thresholdType = "pixel_stdev"
152 self.detection.excludeMaskPlanes = ["EDGE"]
154 # Add filtered flux measurement, the correct measurement for pre-convolved images.
155 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
156 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
157 'base_LocalPhotoCalib',
158 'base_LocalWcs',
159 'ext_shapeHSM_HsmSourceMoments',
160 'ext_shapeHSM_HsmPsfMoments',
161 ]
162 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
163 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
164 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0
165 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
166 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
167 self.forcedMeasurement.copyColumns = {
168 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
169 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
170 self.forcedMeasurement.slots.shape = None
172 # Keep track of which footprints contain streaks
173 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK']
174 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK']
177class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
178 """Detect and measure sources on a difference image.
179 """
180 ConfigClass = DetectAndMeasureConfig
181 _DefaultName = "detectAndMeasure"
183 def __init__(self, **kwargs):
184 super().__init__(**kwargs)
185 self.schema = afwTable.SourceTable.makeMinimalSchema()
186 # Add coordinate error fields:
187 afwTable.CoordKey.addErrorFields(self.schema)
189 self.algMetadata = dafBase.PropertyList()
190 self.makeSubtask("detection", schema=self.schema)
191 self.makeSubtask("measurement", schema=self.schema,
192 algMetadata=self.algMetadata)
193 if self.config.doApCorr:
194 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
195 if self.config.doForcedMeasurement:
196 self.schema.addField(
197 "ip_diffim_forced_PsfFlux_instFlux", "D",
198 "Forced PSF flux measured on the direct image.",
199 units="count")
200 self.schema.addField(
201 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
202 "Forced PSF flux error measured on the direct image.",
203 units="count")
204 self.schema.addField(
205 "ip_diffim_forced_PsfFlux_area", "F",
206 "Forced PSF flux effective area of PSF.",
207 units="pixel")
208 self.schema.addField(
209 "ip_diffim_forced_PsfFlux_flag", "Flag",
210 "Forced PSF flux general failure flag.")
211 self.schema.addField(
212 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
213 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
214 self.schema.addField(
215 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
216 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
217 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
219 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
220 self.schema.addField("srcMatchId", "L", "unique id of source match")
221 if self.config.doSkySources:
222 self.makeSubtask("skySources")
223 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
225 # initialize InitOutputs
226 self.outputSchema = afwTable.SourceCatalog(self.schema)
227 self.outputSchema.getTable().setMetadata(self.algMetadata)
229 # TODO: remove on DM-38687.
230 @staticmethod
231 @deprecated(
232 reason=(
233 "ID factory construction now depends on configuration; use the "
234 "idGenerator config field. Will be removed after v26."
235 ),
236 version="v26.0",
237 category=FutureWarning,
238 )
239 def makeIdFactory(expId, expBits):
240 """Create IdFactory instance for unique 64 bit diaSource id-s.
242 Parameters
243 ----------
244 expId : `int`
245 Exposure id.
247 expBits: `int`
248 Number of used bits in ``expId``.
250 Notes
251 -----
252 The diasource id-s consists of the ``expId`` stored fixed in the highest value
253 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
254 low value end of the integer.
256 Returns
257 -------
258 idFactory: `lsst.afw.table.IdFactory`
259 """
260 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
262 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
263 inputRefs: pipeBase.InputQuantizedConnection,
264 outputRefs: pipeBase.OutputQuantizedConnection):
265 inputs = butlerQC.get(inputRefs)
266 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
267 idFactory = idGenerator.make_table_id_factory()
268 outputs = self.run(**inputs, idFactory=idFactory)
269 butlerQC.put(outputs, outputRefs)
271 @timeMethod
272 def run(self, science, matchedTemplate, difference,
273 idFactory=None):
274 """Detect and measure sources on a difference image.
276 The difference image will be convolved with a gaussian approximation of
277 the PSF to form a maximum likelihood image for detection.
278 Close positive and negative detections will optionally be merged into
279 dipole diaSources.
280 Sky sources, or forced detections in background regions, will optionally
281 be added, and the configured measurement algorithm will be run on all
282 detections.
284 Parameters
285 ----------
286 science : `lsst.afw.image.ExposureF`
287 Science exposure that the template was subtracted from.
288 matchedTemplate : `lsst.afw.image.ExposureF`
289 Warped and PSF-matched template that was used produce the
290 difference image.
291 difference : `lsst.afw.image.ExposureF`
292 Result of subtracting template from the science image.
293 idFactory : `lsst.afw.table.IdFactory`, optional
294 Generator object to assign ids to detected sources in the difference image.
296 Returns
297 -------
298 measurementResults : `lsst.pipe.base.Struct`
300 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
301 Subtracted exposure with detection mask applied.
302 ``diaSources`` : `lsst.afw.table.SourceCatalog`
303 The catalog of detected sources.
304 """
305 # Ensure that we start with an empty detection mask.
306 mask = difference.mask
307 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
309 table = afwTable.SourceTable.make(self.schema, idFactory)
310 table.setMetadata(self.algMetadata)
311 results = self.detection.run(
312 table=table,
313 exposure=difference,
314 doSmooth=True,
315 )
317 return self.processResults(science, matchedTemplate, difference, results.sources, table,
318 positiveFootprints=results.positive, negativeFootprints=results.negative)
320 def processResults(self, science, matchedTemplate, difference, sources, table,
321 positiveFootprints=None, negativeFootprints=None,):
322 """Measure and process the results of source detection.
324 Parameters
325 ----------
326 sources : `lsst.afw.table.SourceCatalog`
327 Detected sources on the difference exposure.
328 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
329 Positive polarity footprints.
330 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
331 Negative polarity footprints.
332 table : `lsst.afw.table.SourceTable`
333 Table object that will be used to create the SourceCatalog.
334 science : `lsst.afw.image.ExposureF`
335 Science exposure that the template was subtracted from.
336 matchedTemplate : `lsst.afw.image.ExposureF`
337 Warped and PSF-matched template that was used produce the
338 difference image.
339 difference : `lsst.afw.image.ExposureF`
340 Result of subtracting template from the science image.
342 Returns
343 -------
344 measurementResults : `lsst.pipe.base.Struct`
346 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
347 Subtracted exposure with detection mask applied.
348 ``diaSources`` : `lsst.afw.table.SourceCatalog`
349 The catalog of detected sources.
350 """
351 self.metadata.add("nUnmergedDiaSources", len(sources))
352 if self.config.doMerge:
353 fpSet = positiveFootprints
354 fpSet.merge(negativeFootprints, self.config.growFootprint,
355 self.config.growFootprint, False)
356 diaSources = afwTable.SourceCatalog(table)
357 fpSet.makeSources(diaSources)
358 self.log.info("Merging detections into %d sources", len(diaSources))
359 else:
360 diaSources = sources
361 self.metadata.add("nMergedDiaSources", len(diaSources))
363 if self.config.doSkySources:
364 self.addSkySources(diaSources, difference.mask, difference.info.id)
366 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
368 if self.config.doForcedMeasurement:
369 self.measureForcedSources(diaSources, science, difference.getWcs())
371 measurementResults = pipeBase.Struct(
372 subtractedMeasuredExposure=difference,
373 diaSources=diaSources,
374 )
375 self.calculateMetrics(difference)
377 return measurementResults
379 def addSkySources(self, diaSources, mask, seed):
380 """Add sources in empty regions of the difference image
381 for measuring the background.
383 Parameters
384 ----------
385 diaSources : `lsst.afw.table.SourceCatalog`
386 The catalog of detected sources.
387 mask : `lsst.afw.image.Mask`
388 Mask plane for determining regions where Sky sources can be added.
389 seed : `int`
390 Seed value to initialize the random number generator.
391 """
392 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
393 if skySourceFootprints:
394 for foot in skySourceFootprints:
395 s = diaSources.addNew()
396 s.setFootprint(foot)
397 s.set(self.skySourceKey, True)
399 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
400 """Use (matched) template and science image to constrain dipole fitting.
402 Parameters
403 ----------
404 diaSources : `lsst.afw.table.SourceCatalog`
405 The catalog of detected sources.
406 science : `lsst.afw.image.ExposureF`
407 Science exposure that the template was subtracted from.
408 difference : `lsst.afw.image.ExposureF`
409 Result of subtracting template from the science image.
410 matchedTemplate : `lsst.afw.image.ExposureF`
411 Warped and PSF-matched template that was used produce the
412 difference image.
413 """
414 # Note that this may not be correct if we convolved the science image.
415 # In the future we may wish to persist the matchedScience image.
416 self.measurement.run(diaSources, difference, science, matchedTemplate)
417 if self.config.doApCorr:
418 apCorrMap = difference.getInfo().getApCorrMap()
419 if apCorrMap is None:
420 self.log.warning("Difference image does not have valid aperture correction; skipping.")
421 else:
422 self.applyApCorr.run(
423 catalog=diaSources,
424 apCorrMap=apCorrMap,
425 )
427 def measureForcedSources(self, diaSources, science, wcs):
428 """Perform forced measurement of the diaSources on the science image.
430 Parameters
431 ----------
432 diaSources : `lsst.afw.table.SourceCatalog`
433 The catalog of detected sources.
434 science : `lsst.afw.image.ExposureF`
435 Science exposure that the template was subtracted from.
436 wcs : `lsst.afw.geom.SkyWcs`
437 Coordinate system definition (wcs) for the exposure.
438 """
439 # Run forced psf photometry on the PVI at the diaSource locations.
440 # Copy the measured flux and error into the diaSource.
441 forcedSources = self.forcedMeasurement.generateMeasCat(
442 science, diaSources, wcs)
443 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
444 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
445 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
446 "ip_diffim_forced_PsfFlux_instFlux", True)
447 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
448 "ip_diffim_forced_PsfFlux_instFluxErr", True)
449 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
450 "ip_diffim_forced_PsfFlux_area", True)
451 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
452 "ip_diffim_forced_PsfFlux_flag", True)
453 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
454 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
455 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
456 "ip_diffim_forced_PsfFlux_flag_edge", True)
457 for diaSource, forcedSource in zip(diaSources, forcedSources):
458 diaSource.assign(forcedSource, mapper)
460 def calculateMetrics(self, difference):
461 """Add image QA metrics to the Task metadata.
463 Parameters
464 ----------
465 difference : `lsst.afw.image.Exposure`
466 The target image to calculate metrics for.
467 """
468 mask = difference.mask
469 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
470 self.metadata.add("nGoodPixels", np.sum(~badPix))
471 self.metadata.add("nBadPixels", np.sum(badPix))
472 detPosPix = (mask.array & mask.getPlaneBitMask("DETECTED")) > 0
473 detNegPix = (mask.array & mask.getPlaneBitMask("DETECTED_NEGATIVE")) > 0
474 self.metadata.add("nPixelsDetectedPositive", np.sum(detPosPix))
475 self.metadata.add("nPixelsDetectedNegative", np.sum(detNegPix))
476 detPosPix &= badPix
477 detNegPix &= badPix
478 self.metadata.add("nBadPixelsDetectedPositive", np.sum(detPosPix))
479 self.metadata.add("nBadPixelsDetectedNegative", np.sum(detNegPix))
482class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
483 scoreExposure = pipeBase.connectionTypes.Input(
484 doc="Maximum likelihood image for detection.",
485 dimensions=("instrument", "visit", "detector"),
486 storageClass="ExposureF",
487 name="{fakesType}{coaddName}Diff_scoreExp",
488 )
491class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
492 pipelineConnections=DetectAndMeasureScoreConnections):
493 pass
496class DetectAndMeasureScoreTask(DetectAndMeasureTask):
497 """Detect DIA sources using a score image,
498 and measure the detections on the difference image.
500 Source detection is run on the supplied score, or maximum likelihood,
501 image. Note that no additional convolution will be done in this case.
502 Close positive and negative detections will optionally be merged into
503 dipole diaSources.
504 Sky sources, or forced detections in background regions, will optionally
505 be added, and the configured measurement algorithm will be run on all
506 detections.
507 """
508 ConfigClass = DetectAndMeasureScoreConfig
509 _DefaultName = "detectAndMeasureScore"
511 @timeMethod
512 def run(self, science, matchedTemplate, difference, scoreExposure,
513 idFactory=None):
514 """Detect and measure sources on a score image.
516 Parameters
517 ----------
518 science : `lsst.afw.image.ExposureF`
519 Science exposure that the template was subtracted from.
520 matchedTemplate : `lsst.afw.image.ExposureF`
521 Warped and PSF-matched template that was used produce the
522 difference image.
523 difference : `lsst.afw.image.ExposureF`
524 Result of subtracting template from the science image.
525 scoreExposure : `lsst.afw.image.ExposureF`
526 Score or maximum likelihood difference image
527 idFactory : `lsst.afw.table.IdFactory`, optional
528 Generator object to assign ids to detected sources in the difference image.
530 Returns
531 -------
532 measurementResults : `lsst.pipe.base.Struct`
534 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
535 Subtracted exposure with detection mask applied.
536 ``diaSources`` : `lsst.afw.table.SourceCatalog`
537 The catalog of detected sources.
538 """
539 # Ensure that we start with an empty detection mask.
540 mask = scoreExposure.mask
541 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
543 table = afwTable.SourceTable.make(self.schema, idFactory)
544 table.setMetadata(self.algMetadata)
545 results = self.detection.run(
546 table=table,
547 exposure=scoreExposure,
548 doSmooth=False,
549 )
550 # Copy the detection mask from the Score image to the difference image
551 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
553 return self.processResults(science, matchedTemplate, difference, results.sources, table,
554 positiveFootprints=results.positive, negativeFootprints=results.negative)