lsst.ip.diffim gd0838329fc+f3ac227b3c
Loading...
Searching...
No Matches
detectAndMeasure.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import numpy as np
23
24import lsst.afw.table as afwTable
25import lsst.daf.base as dafBase
26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources # noqa: F401
29import lsst.meas.extensions.shapeHSM
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.utils
33from lsst.utils.timer import timeMethod
34
35from . import DipoleFitTask
36
37__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
38 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
39
40
41class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument", "visit", "detector"),
43 defaultTemplates={"coaddName": "deep",
44 "warpTypeSuffix": "",
45 "fakesType": ""}):
46 science = pipeBase.connectionTypes.Input(
47 doc="Input science exposure.",
48 dimensions=("instrument", "visit", "detector"),
49 storageClass="ExposureF",
50 name="{fakesType}calexp"
51 )
52 matchedTemplate = pipeBase.connectionTypes.Input(
53 doc="Warped and PSF-matched template used to create the difference image.",
54 dimensions=("instrument", "visit", "detector"),
55 storageClass="ExposureF",
56 name="{fakesType}{coaddName}Diff_matchedExp",
57 )
58 difference = pipeBase.connectionTypes.Input(
59 doc="Result of subtracting template from science.",
60 dimensions=("instrument", "visit", "detector"),
61 storageClass="ExposureF",
62 name="{fakesType}{coaddName}Diff_differenceTempExp",
63 )
64 outputSchema = pipeBase.connectionTypes.InitOutput(
65 doc="Schema (as an example catalog) for output DIASource catalog.",
66 storageClass="SourceCatalog",
67 name="{fakesType}{coaddName}Diff_diaSrc_schema",
68 )
69 diaSources = pipeBase.connectionTypes.Output(
70 doc="Detected diaSources on the difference image.",
71 dimensions=("instrument", "visit", "detector"),
72 storageClass="SourceCatalog",
73 name="{fakesType}{coaddName}Diff_diaSrc",
74 )
75 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
76 doc="Difference image with detection mask plane filled in.",
77 dimensions=("instrument", "visit", "detector"),
78 storageClass="ExposureF",
79 name="{fakesType}{coaddName}Diff_differenceExp",
80 )
81
82
83class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
84 pipelineConnections=DetectAndMeasureConnections):
85 """Config for DetectAndMeasureTask
86 """
87 doMerge = pexConfig.Field(
88 dtype=bool,
89 default=True,
90 doc="Merge positive and negative diaSources with grow radius "
91 "set by growFootprint"
92 )
93 doForcedMeasurement = pexConfig.Field(
94 dtype=bool,
95 default=True,
96 doc="Force photometer diaSource locations on PVI?")
97 doAddMetrics = pexConfig.Field(
98 dtype=bool,
99 default=False,
100 doc="Add columns to the source table to hold analysis metrics?"
101 )
102 detection = pexConfig.ConfigurableField(
103 target=SourceDetectionTask,
104 doc="Final source detection for diaSource measurement",
105 )
106 measurement = pexConfig.ConfigurableField(
107 target=DipoleFitTask,
108 doc="Task to measure sources on the difference image.",
109 )
110 doApCorr = lsst.pex.config.Field(
111 dtype=bool,
112 default=True,
113 doc="Run subtask to apply aperture corrections"
114 )
115 applyApCorr = lsst.pex.config.ConfigurableField(
116 target=ApplyApCorrTask,
117 doc="Task to apply aperture corrections"
118 )
119 forcedMeasurement = pexConfig.ConfigurableField(
120 target=ForcedMeasurementTask,
121 doc="Task to force photometer science image at diaSource locations.",
122 )
123 growFootprint = pexConfig.Field(
124 dtype=int,
125 default=2,
126 doc="Grow positive and negative footprints by this many pixels before merging"
127 )
128 diaSourceMatchRadius = pexConfig.Field(
129 dtype=float,
130 default=0.5,
131 doc="Match radius (in arcseconds) for DiaSource to Source association"
132 )
133 doSkySources = pexConfig.Field(
134 dtype=bool,
135 default=False,
136 doc="Generate sky sources?",
137 )
138 skySources = pexConfig.ConfigurableField(
139 target=SkyObjectsTask,
140 doc="Generate sky sources",
141 )
142 badSourceFlags = lsst.pex.config.ListField(
143 dtype=str,
144 doc="Sources with any of these flags set are removed before writing the output catalog.",
145 default=("base_PixelFlags_flag_offimage",
146 ),
147 )
148 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
149
150 def setDefaults(self):
151 # DiaSource Detection
152 self.detection.thresholdPolarity = "both"
153 self.detection.thresholdValue = 5.0
154 self.detection.reEstimateBackground = False
155 self.detection.thresholdType = "pixel_stdev"
156 self.detection.excludeMaskPlanes = ["EDGE"]
157
158 # Add filtered flux measurement, the correct measurement for pre-convolved images.
159 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
160 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
161 'base_LocalPhotoCalib',
162 'base_LocalWcs',
163 'ext_shapeHSM_HsmSourceMoments',
164 'ext_shapeHSM_HsmPsfMoments',
165 ]
166 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
167 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
168 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0
169 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
170 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
171 self.forcedMeasurement.copyColumns = {
172 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
173 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
174 self.forcedMeasurement.slots.shape = None
175
176 # Keep track of which footprints contain streaks
177 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK']
178 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK']
179
180
181class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
182 """Detect and measure sources on a difference image.
183 """
184 ConfigClass = DetectAndMeasureConfig
185 _DefaultName = "detectAndMeasure"
186
187 def __init__(self, **kwargs):
188 super().__init__(**kwargs)
189 self.schema = afwTable.SourceTable.makeMinimalSchema()
190 # Add coordinate error fields:
191 afwTable.CoordKey.addErrorFields(self.schema)
192
193 self.algMetadata = dafBase.PropertyList()
194 self.makeSubtask("detection", schema=self.schema)
195 self.makeSubtask("measurement", schema=self.schema,
196 algMetadata=self.algMetadata)
197 if self.config.doApCorr:
198 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
199 if self.config.doForcedMeasurement:
200 self.schema.addField(
201 "ip_diffim_forced_PsfFlux_instFlux", "D",
202 "Forced PSF flux measured on the direct image.",
203 units="count")
204 self.schema.addField(
205 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
206 "Forced PSF flux error measured on the direct image.",
207 units="count")
208 self.schema.addField(
209 "ip_diffim_forced_PsfFlux_area", "F",
210 "Forced PSF flux effective area of PSF.",
211 units="pixel")
212 self.schema.addField(
213 "ip_diffim_forced_PsfFlux_flag", "Flag",
214 "Forced PSF flux general failure flag.")
215 self.schema.addField(
216 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
217 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
218 self.schema.addField(
219 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
220 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
221 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
222
223 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
224 self.schema.addField("srcMatchId", "L", "unique id of source match")
225 if self.config.doSkySources:
226 self.makeSubtask("skySources")
227 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
228
229 # Check that the schema and config are consistent
230 for flag in self.config.badSourceFlags:
231 if flag not in self.schema:
232 raise pipeBase.InvalidQuantumError("Field %s not in schema" % flag)
233 # initialize InitOutputs
234 self.outputSchema = afwTable.SourceCatalog(self.schema)
235 self.outputSchema.getTable().setMetadata(self.algMetadata)
236
237 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
238 inputRefs: pipeBase.InputQuantizedConnection,
239 outputRefs: pipeBase.OutputQuantizedConnection):
240 inputs = butlerQC.get(inputRefs)
241 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
242 idFactory = idGenerator.make_table_id_factory()
243 outputs = self.run(**inputs, idFactory=idFactory)
244 butlerQC.put(outputs, outputRefs)
245
246 @timeMethod
247 def run(self, science, matchedTemplate, difference,
248 idFactory=None):
249 """Detect and measure sources on a difference image.
250
251 The difference image will be convolved with a gaussian approximation of
252 the PSF to form a maximum likelihood image for detection.
253 Close positive and negative detections will optionally be merged into
254 dipole diaSources.
255 Sky sources, or forced detections in background regions, will optionally
256 be added, and the configured measurement algorithm will be run on all
257 detections.
258
259 Parameters
260 ----------
261 science : `lsst.afw.image.ExposureF`
262 Science exposure that the template was subtracted from.
263 matchedTemplate : `lsst.afw.image.ExposureF`
264 Warped and PSF-matched template that was used produce the
265 difference image.
266 difference : `lsst.afw.image.ExposureF`
267 Result of subtracting template from the science image.
268 idFactory : `lsst.afw.table.IdFactory`, optional
269 Generator object to assign ids to detected sources in the difference image.
270
271 Returns
272 -------
273 measurementResults : `lsst.pipe.base.Struct`
274
275 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
276 Subtracted exposure with detection mask applied.
277 ``diaSources`` : `lsst.afw.table.SourceCatalog`
278 The catalog of detected sources.
279 """
280 # Ensure that we start with an empty detection mask.
281 mask = difference.mask
282 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
283
284 table = afwTable.SourceTable.make(self.schema, idFactory)
285 table.setMetadata(self.algMetadata)
286 results = self.detection.run(
287 table=table,
288 exposure=difference,
289 doSmooth=True,
290 )
291
292 return self.processResults(science, matchedTemplate, difference, results.sources, table,
293 positiveFootprints=results.positive, negativeFootprints=results.negative)
294
295 def processResults(self, science, matchedTemplate, difference, sources, table,
296 positiveFootprints=None, negativeFootprints=None,):
297 """Measure and process the results of source detection.
298
299 Parameters
300 ----------
301 sources : `lsst.afw.table.SourceCatalog`
302 Detected sources on the difference exposure.
303 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
304 Positive polarity footprints.
305 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
306 Negative polarity footprints.
307 table : `lsst.afw.table.SourceTable`
308 Table object that will be used to create the SourceCatalog.
309 science : `lsst.afw.image.ExposureF`
310 Science exposure that the template was subtracted from.
311 matchedTemplate : `lsst.afw.image.ExposureF`
312 Warped and PSF-matched template that was used produce the
313 difference image.
314 difference : `lsst.afw.image.ExposureF`
315 Result of subtracting template from the science image.
316
317 Returns
318 -------
319 measurementResults : `lsst.pipe.base.Struct`
320
321 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
322 Subtracted exposure with detection mask applied.
323 ``diaSources`` : `lsst.afw.table.SourceCatalog`
324 The catalog of detected sources.
325 """
326 self.metadata.add("nUnmergedDiaSources", len(sources))
327 if self.config.doMerge:
328 fpSet = positiveFootprints
329 fpSet.merge(negativeFootprints, self.config.growFootprint,
330 self.config.growFootprint, False)
331 initialDiaSources = afwTable.SourceCatalog(table)
332 fpSet.makeSources(initialDiaSources)
333 self.log.info("Merging detections into %d sources", len(initialDiaSources))
334 else:
335 initialDiaSources = sources
336 self.metadata.add("nMergedDiaSources", len(initialDiaSources))
337
338 if self.config.doSkySources:
339 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
340
341 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
342 diaSources = self._removeBadSources(initialDiaSources)
343
344 if self.config.doForcedMeasurement:
345 self.measureForcedSources(diaSources, science, difference.getWcs())
346
347 measurementResults = pipeBase.Struct(
348 subtractedMeasuredExposure=difference,
349 diaSources=diaSources,
350 )
351 self.calculateMetrics(difference)
352
353 return measurementResults
354
355 def _removeBadSources(self, diaSources):
356 """Remove bad diaSources from the catalog.
357
358 Parameters
359 ----------
360 diaSources : `lsst.afw.table.SourceCatalog`
361 The catalog of detected sources.
362
363 Returns
364 -------
365 diaSources : `lsst.afw.table.SourceCatalog`
366 The updated catalog of detected sources, with any source that has a
367 flag in ``config.badSourceFlags`` set removed.
368 """
369 nBadTotal = 0
370 selector = np.ones(len(diaSources), dtype=bool)
371 for flag in self.config.badSourceFlags:
372 flags = diaSources[flag]
373 nBad = np.count_nonzero(flags)
374 if nBad > 0:
375 self.log.info("Found and removed %d unphysical sources with flag %s.", nBad, flag)
376 selector &= ~flags
377 nBadTotal += nBad
378 self.metadata.add("nRemovedBadFlaggedSources", nBadTotal)
379 return diaSources[selector].copy(deep=True)
380
381 def addSkySources(self, diaSources, mask, seed):
382 """Add sources in empty regions of the difference image
383 for measuring the background.
384
385 Parameters
386 ----------
387 diaSources : `lsst.afw.table.SourceCatalog`
388 The catalog of detected sources.
389 mask : `lsst.afw.image.Mask`
390 Mask plane for determining regions where Sky sources can be added.
391 seed : `int`
392 Seed value to initialize the random number generator.
393 """
394 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
395 if skySourceFootprints:
396 for foot in skySourceFootprints:
397 s = diaSources.addNew()
398 s.setFootprint(foot)
399 s.set(self.skySourceKey, True)
400
401 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
402 """Use (matched) template and science image to constrain dipole fitting.
403
404 Parameters
405 ----------
406 diaSources : `lsst.afw.table.SourceCatalog`
407 The catalog of detected sources.
408 science : `lsst.afw.image.ExposureF`
409 Science exposure that the template was subtracted from.
410 difference : `lsst.afw.image.ExposureF`
411 Result of subtracting template from the science image.
412 matchedTemplate : `lsst.afw.image.ExposureF`
413 Warped and PSF-matched template that was used produce the
414 difference image.
415 """
416 # Note that this may not be correct if we convolved the science image.
417 # In the future we may wish to persist the matchedScience image.
418 self.measurement.run(diaSources, difference, science, matchedTemplate)
419 if self.config.doApCorr:
420 apCorrMap = difference.getInfo().getApCorrMap()
421 if apCorrMap is None:
422 self.log.warning("Difference image does not have valid aperture correction; skipping.")
423 else:
424 self.applyApCorr.run(
425 catalog=diaSources,
426 apCorrMap=apCorrMap,
427 )
428
429 def measureForcedSources(self, diaSources, science, wcs):
430 """Perform forced measurement of the diaSources on the science image.
431
432 Parameters
433 ----------
434 diaSources : `lsst.afw.table.SourceCatalog`
435 The catalog of detected sources.
436 science : `lsst.afw.image.ExposureF`
437 Science exposure that the template was subtracted from.
438 wcs : `lsst.afw.geom.SkyWcs`
439 Coordinate system definition (wcs) for the exposure.
440 """
441 # Run forced psf photometry on the PVI at the diaSource locations.
442 # Copy the measured flux and error into the diaSource.
443 forcedSources = self.forcedMeasurement.generateMeasCat(
444 science, diaSources, wcs)
445 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
446 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
447 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
448 "ip_diffim_forced_PsfFlux_instFlux", True)
449 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
450 "ip_diffim_forced_PsfFlux_instFluxErr", True)
451 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
452 "ip_diffim_forced_PsfFlux_area", True)
453 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
454 "ip_diffim_forced_PsfFlux_flag", True)
455 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
456 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
457 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
458 "ip_diffim_forced_PsfFlux_flag_edge", True)
459 for diaSource, forcedSource in zip(diaSources, forcedSources):
460 diaSource.assign(forcedSource, mapper)
461
462 def calculateMetrics(self, difference):
463 """Add image QA metrics to the Task metadata.
464
465 Parameters
466 ----------
467 difference : `lsst.afw.image.Exposure`
468 The target image to calculate metrics for.
469 """
470 mask = difference.mask
471 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
472 self.metadata.add("nGoodPixels", np.sum(~badPix))
473 self.metadata.add("nBadPixels", np.sum(badPix))
474 detPosPix = (mask.array & mask.getPlaneBitMask("DETECTED")) > 0
475 detNegPix = (mask.array & mask.getPlaneBitMask("DETECTED_NEGATIVE")) > 0
476 self.metadata.add("nPixelsDetectedPositive", np.sum(detPosPix))
477 self.metadata.add("nPixelsDetectedNegative", np.sum(detNegPix))
478 detPosPix &= badPix
479 detNegPix &= badPix
480 self.metadata.add("nBadPixelsDetectedPositive", np.sum(detPosPix))
481 self.metadata.add("nBadPixelsDetectedNegative", np.sum(detNegPix))
482
483
485 scoreExposure = pipeBase.connectionTypes.Input(
486 doc="Maximum likelihood image for detection.",
487 dimensions=("instrument", "visit", "detector"),
488 storageClass="ExposureF",
489 name="{fakesType}{coaddName}Diff_scoreExp",
490 )
491
492
494 pipelineConnections=DetectAndMeasureScoreConnections):
495 pass
496
497
499 """Detect DIA sources using a score image,
500 and measure the detections on the difference image.
501
502 Source detection is run on the supplied score, or maximum likelihood,
503 image. Note that no additional convolution will be done in this case.
504 Close positive and negative detections will optionally be merged into
505 dipole diaSources.
506 Sky sources, or forced detections in background regions, will optionally
507 be added, and the configured measurement algorithm will be run on all
508 detections.
509 """
510 ConfigClass = DetectAndMeasureScoreConfig
511 _DefaultName = "detectAndMeasureScore"
512
513 @timeMethod
514 def run(self, science, matchedTemplate, difference, scoreExposure,
515 idFactory=None):
516 """Detect and measure sources on a score image.
517
518 Parameters
519 ----------
520 science : `lsst.afw.image.ExposureF`
521 Science exposure that the template was subtracted from.
522 matchedTemplate : `lsst.afw.image.ExposureF`
523 Warped and PSF-matched template that was used produce the
524 difference image.
525 difference : `lsst.afw.image.ExposureF`
526 Result of subtracting template from the science image.
527 scoreExposure : `lsst.afw.image.ExposureF`
528 Score or maximum likelihood difference image
529 idFactory : `lsst.afw.table.IdFactory`, optional
530 Generator object to assign ids to detected sources in the difference image.
531
532 Returns
533 -------
534 measurementResults : `lsst.pipe.base.Struct`
535
536 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
537 Subtracted exposure with detection mask applied.
538 ``diaSources`` : `lsst.afw.table.SourceCatalog`
539 The catalog of detected sources.
540 """
541 # Ensure that we start with an empty detection mask.
542 mask = scoreExposure.mask
543 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
544
545 table = afwTable.SourceTable.make(self.schema, idFactory)
546 table.setMetadata(self.algMetadata)
547 results = self.detection.run(
548 table=table,
549 exposure=scoreExposure,
550 doSmooth=False,
551 )
552 # Copy the detection mask from the Score image to the difference image
553 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
554
555 return self.processResults(science, matchedTemplate, difference, results.sources, table,
556 positiveFootprints=results.positive, negativeFootprints=results.negative)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)