Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 28%
185 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-05 13:22 +0000
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-05 13:22 +0000
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
24import lsst.afw.table as afwTable
25import lsst.daf.base as dafBase
26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources # noqa: F401
29import lsst.meas.extensions.shapeHSM
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.utils
33from lsst.utils.timer import timeMethod
35from . import DipoleFitTask
37__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
38 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
41class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument", "visit", "detector"),
43 defaultTemplates={"coaddName": "deep",
44 "warpTypeSuffix": "",
45 "fakesType": ""}):
46 science = pipeBase.connectionTypes.Input(
47 doc="Input science exposure.",
48 dimensions=("instrument", "visit", "detector"),
49 storageClass="ExposureF",
50 name="{fakesType}calexp"
51 )
52 matchedTemplate = pipeBase.connectionTypes.Input(
53 doc="Warped and PSF-matched template used to create the difference image.",
54 dimensions=("instrument", "visit", "detector"),
55 storageClass="ExposureF",
56 name="{fakesType}{coaddName}Diff_matchedExp",
57 )
58 difference = pipeBase.connectionTypes.Input(
59 doc="Result of subtracting template from science.",
60 dimensions=("instrument", "visit", "detector"),
61 storageClass="ExposureF",
62 name="{fakesType}{coaddName}Diff_differenceTempExp",
63 )
64 outputSchema = pipeBase.connectionTypes.InitOutput(
65 doc="Schema (as an example catalog) for output DIASource catalog.",
66 storageClass="SourceCatalog",
67 name="{fakesType}{coaddName}Diff_diaSrc_schema",
68 )
69 diaSources = pipeBase.connectionTypes.Output(
70 doc="Detected diaSources on the difference image.",
71 dimensions=("instrument", "visit", "detector"),
72 storageClass="SourceCatalog",
73 name="{fakesType}{coaddName}Diff_diaSrc",
74 )
75 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
76 doc="Difference image with detection mask plane filled in.",
77 dimensions=("instrument", "visit", "detector"),
78 storageClass="ExposureF",
79 name="{fakesType}{coaddName}Diff_differenceExp",
80 )
83class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
84 pipelineConnections=DetectAndMeasureConnections):
85 """Config for DetectAndMeasureTask
86 """
87 doMerge = pexConfig.Field(
88 dtype=bool,
89 default=True,
90 doc="Merge positive and negative diaSources with grow radius "
91 "set by growFootprint"
92 )
93 doForcedMeasurement = pexConfig.Field(
94 dtype=bool,
95 default=True,
96 doc="Force photometer diaSource locations on PVI?")
97 doAddMetrics = pexConfig.Field(
98 dtype=bool,
99 default=False,
100 doc="Add columns to the source table to hold analysis metrics?"
101 )
102 detection = pexConfig.ConfigurableField(
103 target=SourceDetectionTask,
104 doc="Final source detection for diaSource measurement",
105 )
106 measurement = pexConfig.ConfigurableField(
107 target=DipoleFitTask,
108 doc="Task to measure sources on the difference image.",
109 )
110 doApCorr = lsst.pex.config.Field(
111 dtype=bool,
112 default=True,
113 doc="Run subtask to apply aperture corrections"
114 )
115 applyApCorr = lsst.pex.config.ConfigurableField(
116 target=ApplyApCorrTask,
117 doc="Task to apply aperture corrections"
118 )
119 forcedMeasurement = pexConfig.ConfigurableField(
120 target=ForcedMeasurementTask,
121 doc="Task to force photometer science image at diaSource locations.",
122 )
123 growFootprint = pexConfig.Field(
124 dtype=int,
125 default=2,
126 doc="Grow positive and negative footprints by this many pixels before merging"
127 )
128 diaSourceMatchRadius = pexConfig.Field(
129 dtype=float,
130 default=0.5,
131 doc="Match radius (in arcseconds) for DiaSource to Source association"
132 )
133 doSkySources = pexConfig.Field(
134 dtype=bool,
135 default=False,
136 doc="Generate sky sources?",
137 )
138 skySources = pexConfig.ConfigurableField(
139 target=SkyObjectsTask,
140 doc="Generate sky sources",
141 )
142 badSourceFlags = lsst.pex.config.ListField(
143 dtype=str,
144 doc="Sources with any of these flags set are removed before writing the output catalog.",
145 default=("base_PixelFlags_flag_offimage",
146 "base_PixelFlags_flag_interpolatedCenterAll",
147 "base_PixelFlags_flag_badCenterAll",
148 "base_PixelFlags_flag_edgeCenterAll",
149 ),
150 )
151 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
153 def setDefaults(self):
154 # DiaSource Detection
155 self.detection.thresholdPolarity = "both"
156 self.detection.thresholdValue = 5.0
157 self.detection.reEstimateBackground = False
158 self.detection.thresholdType = "pixel_stdev"
159 self.detection.excludeMaskPlanes = ["EDGE"]
161 # Add filtered flux measurement, the correct measurement for pre-convolved images.
162 self.measurement.algorithms.names.add("base_PeakLikelihoodFlux")
163 self.measurement.plugins.names |= ["ext_trailedSources_Naive",
164 "base_LocalPhotoCalib",
165 "base_LocalWcs",
166 "ext_shapeHSM_HsmSourceMoments",
167 "ext_shapeHSM_HsmPsfMoments",
168 ]
169 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
170 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
171 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0
172 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
173 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
174 self.forcedMeasurement.copyColumns = {
175 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
176 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
177 self.forcedMeasurement.slots.shape = None
179 # Keep track of which footprints contain streaks
180 self.measurement.plugins["base_PixelFlags"].masksFpAnywhere = [
181 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
182 self.measurement.plugins["base_PixelFlags"].masksFpCenter = [
183 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
184 self.skySources.avoidMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "NO_DATA", "EDGE"]
187class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
188 """Detect and measure sources on a difference image.
189 """
190 ConfigClass = DetectAndMeasureConfig
191 _DefaultName = "detectAndMeasure"
193 def __init__(self, **kwargs):
194 super().__init__(**kwargs)
195 self.schema = afwTable.SourceTable.makeMinimalSchema()
196 # Add coordinate error fields:
197 afwTable.CoordKey.addErrorFields(self.schema)
199 self.algMetadata = dafBase.PropertyList()
200 self.makeSubtask("detection", schema=self.schema)
201 self.makeSubtask("measurement", schema=self.schema,
202 algMetadata=self.algMetadata)
203 if self.config.doApCorr:
204 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
205 if self.config.doForcedMeasurement:
206 self.schema.addField(
207 "ip_diffim_forced_PsfFlux_instFlux", "D",
208 "Forced PSF flux measured on the direct image.",
209 units="count")
210 self.schema.addField(
211 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
212 "Forced PSF flux error measured on the direct image.",
213 units="count")
214 self.schema.addField(
215 "ip_diffim_forced_PsfFlux_area", "F",
216 "Forced PSF flux effective area of PSF.",
217 units="pixel")
218 self.schema.addField(
219 "ip_diffim_forced_PsfFlux_flag", "Flag",
220 "Forced PSF flux general failure flag.")
221 self.schema.addField(
222 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
223 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
224 self.schema.addField(
225 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
226 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
227 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
229 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
230 self.schema.addField("srcMatchId", "L", "unique id of source match")
231 if self.config.doSkySources:
232 self.makeSubtask("skySources")
233 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
235 # Check that the schema and config are consistent
236 for flag in self.config.badSourceFlags:
237 if flag not in self.schema:
238 raise pipeBase.InvalidQuantumError("Field %s not in schema" % flag)
239 # initialize InitOutputs
240 self.outputSchema = afwTable.SourceCatalog(self.schema)
241 self.outputSchema.getTable().setMetadata(self.algMetadata)
243 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
244 inputRefs: pipeBase.InputQuantizedConnection,
245 outputRefs: pipeBase.OutputQuantizedConnection):
246 inputs = butlerQC.get(inputRefs)
247 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
248 idFactory = idGenerator.make_table_id_factory()
249 outputs = self.run(**inputs, idFactory=idFactory)
250 butlerQC.put(outputs, outputRefs)
252 @timeMethod
253 def run(self, science, matchedTemplate, difference,
254 idFactory=None):
255 """Detect and measure sources on a difference image.
257 The difference image will be convolved with a gaussian approximation of
258 the PSF to form a maximum likelihood image for detection.
259 Close positive and negative detections will optionally be merged into
260 dipole diaSources.
261 Sky sources, or forced detections in background regions, will optionally
262 be added, and the configured measurement algorithm will be run on all
263 detections.
265 Parameters
266 ----------
267 science : `lsst.afw.image.ExposureF`
268 Science exposure that the template was subtracted from.
269 matchedTemplate : `lsst.afw.image.ExposureF`
270 Warped and PSF-matched template that was used produce the
271 difference image.
272 difference : `lsst.afw.image.ExposureF`
273 Result of subtracting template from the science image.
274 idFactory : `lsst.afw.table.IdFactory`, optional
275 Generator object to assign ids to detected sources in the difference image.
277 Returns
278 -------
279 measurementResults : `lsst.pipe.base.Struct`
281 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
282 Subtracted exposure with detection mask applied.
283 ``diaSources`` : `lsst.afw.table.SourceCatalog`
284 The catalog of detected sources.
285 """
286 # Ensure that we start with an empty detection mask.
287 mask = difference.mask
288 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
290 table = afwTable.SourceTable.make(self.schema, idFactory)
291 table.setMetadata(self.algMetadata)
292 results = self.detection.run(
293 table=table,
294 exposure=difference,
295 doSmooth=True,
296 )
298 return self.processResults(science, matchedTemplate, difference, results.sources, table,
299 positiveFootprints=results.positive, negativeFootprints=results.negative)
301 def processResults(self, science, matchedTemplate, difference, sources, table,
302 positiveFootprints=None, negativeFootprints=None,):
303 """Measure and process the results of source detection.
305 Parameters
306 ----------
307 sources : `lsst.afw.table.SourceCatalog`
308 Detected sources on the difference exposure.
309 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
310 Positive polarity footprints.
311 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
312 Negative polarity footprints.
313 table : `lsst.afw.table.SourceTable`
314 Table object that will be used to create the SourceCatalog.
315 science : `lsst.afw.image.ExposureF`
316 Science exposure that the template was subtracted from.
317 matchedTemplate : `lsst.afw.image.ExposureF`
318 Warped and PSF-matched template that was used produce the
319 difference image.
320 difference : `lsst.afw.image.ExposureF`
321 Result of subtracting template from the science image.
323 Returns
324 -------
325 measurementResults : `lsst.pipe.base.Struct`
327 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
328 Subtracted exposure with detection mask applied.
329 ``diaSources`` : `lsst.afw.table.SourceCatalog`
330 The catalog of detected sources.
331 """
332 self.metadata.add("nUnmergedDiaSources", len(sources))
333 if self.config.doMerge:
334 fpSet = positiveFootprints
335 fpSet.merge(negativeFootprints, self.config.growFootprint,
336 self.config.growFootprint, False)
337 initialDiaSources = afwTable.SourceCatalog(table)
338 fpSet.makeSources(initialDiaSources)
339 self.log.info("Merging detections into %d sources", len(initialDiaSources))
340 else:
341 initialDiaSources = sources
342 self.metadata.add("nMergedDiaSources", len(initialDiaSources))
344 if self.config.doSkySources:
345 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
347 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
348 diaSources = self._removeBadSources(initialDiaSources)
350 if self.config.doForcedMeasurement:
351 self.measureForcedSources(diaSources, science, difference.getWcs())
353 measurementResults = pipeBase.Struct(
354 subtractedMeasuredExposure=difference,
355 diaSources=diaSources,
356 )
357 self.calculateMetrics(difference)
359 return measurementResults
361 def _removeBadSources(self, diaSources):
362 """Remove bad diaSources from the catalog.
364 Parameters
365 ----------
366 diaSources : `lsst.afw.table.SourceCatalog`
367 The catalog of detected sources.
369 Returns
370 -------
371 diaSources : `lsst.afw.table.SourceCatalog`
372 The updated catalog of detected sources, with any source that has a
373 flag in ``config.badSourceFlags`` set removed.
374 """
375 nBadTotal = 0
376 selector = np.ones(len(diaSources), dtype=bool)
377 for flag in self.config.badSourceFlags:
378 flags = diaSources[flag]
379 nBad = np.count_nonzero(flags)
380 if nBad > 0:
381 self.log.info("Found and removed %d unphysical sources with flag %s.", nBad, flag)
382 selector &= ~flags
383 nBadTotal += nBad
384 self.metadata.add("nRemovedBadFlaggedSources", nBadTotal)
385 return diaSources[selector].copy(deep=True)
387 def addSkySources(self, diaSources, mask, seed):
388 """Add sources in empty regions of the difference image
389 for measuring the background.
391 Parameters
392 ----------
393 diaSources : `lsst.afw.table.SourceCatalog`
394 The catalog of detected sources.
395 mask : `lsst.afw.image.Mask`
396 Mask plane for determining regions where Sky sources can be added.
397 seed : `int`
398 Seed value to initialize the random number generator.
399 """
400 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
401 self.metadata.add("nSkySources", len(skySourceFootprints))
402 if skySourceFootprints:
403 for foot in skySourceFootprints:
404 s = diaSources.addNew()
405 s.setFootprint(foot)
406 s.set(self.skySourceKey, True)
408 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
409 """Use (matched) template and science image to constrain dipole fitting.
411 Parameters
412 ----------
413 diaSources : `lsst.afw.table.SourceCatalog`
414 The catalog of detected sources.
415 science : `lsst.afw.image.ExposureF`
416 Science exposure that the template was subtracted from.
417 difference : `lsst.afw.image.ExposureF`
418 Result of subtracting template from the science image.
419 matchedTemplate : `lsst.afw.image.ExposureF`
420 Warped and PSF-matched template that was used produce the
421 difference image.
422 """
423 # Note that this may not be correct if we convolved the science image.
424 # In the future we may wish to persist the matchedScience image.
425 self.measurement.run(diaSources, difference, science, matchedTemplate)
426 if self.config.doApCorr:
427 apCorrMap = difference.getInfo().getApCorrMap()
428 if apCorrMap is None:
429 self.log.warning("Difference image does not have valid aperture correction; skipping.")
430 else:
431 self.applyApCorr.run(
432 catalog=diaSources,
433 apCorrMap=apCorrMap,
434 )
436 def measureForcedSources(self, diaSources, science, wcs):
437 """Perform forced measurement of the diaSources on the science image.
439 Parameters
440 ----------
441 diaSources : `lsst.afw.table.SourceCatalog`
442 The catalog of detected sources.
443 science : `lsst.afw.image.ExposureF`
444 Science exposure that the template was subtracted from.
445 wcs : `lsst.afw.geom.SkyWcs`
446 Coordinate system definition (wcs) for the exposure.
447 """
448 # Run forced psf photometry on the PVI at the diaSource locations.
449 # Copy the measured flux and error into the diaSource.
450 forcedSources = self.forcedMeasurement.generateMeasCat(
451 science, diaSources, wcs)
452 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
453 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
454 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
455 "ip_diffim_forced_PsfFlux_instFlux", True)
456 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
457 "ip_diffim_forced_PsfFlux_instFluxErr", True)
458 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
459 "ip_diffim_forced_PsfFlux_area", True)
460 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
461 "ip_diffim_forced_PsfFlux_flag", True)
462 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
463 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
464 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
465 "ip_diffim_forced_PsfFlux_flag_edge", True)
466 for diaSource, forcedSource in zip(diaSources, forcedSources):
467 diaSource.assign(forcedSource, mapper)
469 def calculateMetrics(self, difference):
470 """Add image QA metrics to the Task metadata.
472 Parameters
473 ----------
474 difference : `lsst.afw.image.Exposure`
475 The target image to calculate metrics for.
476 """
477 mask = difference.mask
478 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
479 self.metadata.add("nGoodPixels", np.sum(~badPix))
480 self.metadata.add("nBadPixels", np.sum(badPix))
481 detPosPix = (mask.array & mask.getPlaneBitMask("DETECTED")) > 0
482 detNegPix = (mask.array & mask.getPlaneBitMask("DETECTED_NEGATIVE")) > 0
483 self.metadata.add("nPixelsDetectedPositive", np.sum(detPosPix))
484 self.metadata.add("nPixelsDetectedNegative", np.sum(detNegPix))
485 detPosPix &= badPix
486 detNegPix &= badPix
487 self.metadata.add("nBadPixelsDetectedPositive", np.sum(detPosPix))
488 self.metadata.add("nBadPixelsDetectedNegative", np.sum(detNegPix))
491class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
492 scoreExposure = pipeBase.connectionTypes.Input(
493 doc="Maximum likelihood image for detection.",
494 dimensions=("instrument", "visit", "detector"),
495 storageClass="ExposureF",
496 name="{fakesType}{coaddName}Diff_scoreExp",
497 )
500class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
501 pipelineConnections=DetectAndMeasureScoreConnections):
502 pass
505class DetectAndMeasureScoreTask(DetectAndMeasureTask):
506 """Detect DIA sources using a score image,
507 and measure the detections on the difference image.
509 Source detection is run on the supplied score, or maximum likelihood,
510 image. Note that no additional convolution will be done in this case.
511 Close positive and negative detections will optionally be merged into
512 dipole diaSources.
513 Sky sources, or forced detections in background regions, will optionally
514 be added, and the configured measurement algorithm will be run on all
515 detections.
516 """
517 ConfigClass = DetectAndMeasureScoreConfig
518 _DefaultName = "detectAndMeasureScore"
520 @timeMethod
521 def run(self, science, matchedTemplate, difference, scoreExposure,
522 idFactory=None):
523 """Detect and measure sources on a score image.
525 Parameters
526 ----------
527 science : `lsst.afw.image.ExposureF`
528 Science exposure that the template was subtracted from.
529 matchedTemplate : `lsst.afw.image.ExposureF`
530 Warped and PSF-matched template that was used produce the
531 difference image.
532 difference : `lsst.afw.image.ExposureF`
533 Result of subtracting template from the science image.
534 scoreExposure : `lsst.afw.image.ExposureF`
535 Score or maximum likelihood difference image
536 idFactory : `lsst.afw.table.IdFactory`, optional
537 Generator object to assign ids to detected sources in the difference image.
539 Returns
540 -------
541 measurementResults : `lsst.pipe.base.Struct`
543 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
544 Subtracted exposure with detection mask applied.
545 ``diaSources`` : `lsst.afw.table.SourceCatalog`
546 The catalog of detected sources.
547 """
548 # Ensure that we start with an empty detection mask.
549 mask = scoreExposure.mask
550 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
552 table = afwTable.SourceTable.make(self.schema, idFactory)
553 table.setMetadata(self.algMetadata)
554 results = self.detection.run(
555 table=table,
556 exposure=scoreExposure,
557 doSmooth=False,
558 )
559 # Copy the detection mask from the Score image to the difference image
560 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
562 return self.processResults(science, matchedTemplate, difference, results.sources, table,
563 positiveFootprints=results.positive, negativeFootprints=results.negative)