lsst.ip.diffim g88964a4962+36e368cbb6
Loading...
Searching...
No Matches
detectAndMeasure.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import numpy as np
23
24import lsst.afw.table as afwTable
25import lsst.daf.base as dafBase
26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources # noqa: F401
29import lsst.meas.extensions.shapeHSM
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.utils
33from lsst.utils.timer import timeMethod
34
35from . import DipoleFitTask
36
37__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
38 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
39
40
41class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument", "visit", "detector"),
43 defaultTemplates={"coaddName": "deep",
44 "warpTypeSuffix": "",
45 "fakesType": ""}):
46 science = pipeBase.connectionTypes.Input(
47 doc="Input science exposure.",
48 dimensions=("instrument", "visit", "detector"),
49 storageClass="ExposureF",
50 name="{fakesType}calexp"
51 )
52 matchedTemplate = pipeBase.connectionTypes.Input(
53 doc="Warped and PSF-matched template used to create the difference image.",
54 dimensions=("instrument", "visit", "detector"),
55 storageClass="ExposureF",
56 name="{fakesType}{coaddName}Diff_matchedExp",
57 )
58 difference = pipeBase.connectionTypes.Input(
59 doc="Result of subtracting template from science.",
60 dimensions=("instrument", "visit", "detector"),
61 storageClass="ExposureF",
62 name="{fakesType}{coaddName}Diff_differenceTempExp",
63 )
64 outputSchema = pipeBase.connectionTypes.InitOutput(
65 doc="Schema (as an example catalog) for output DIASource catalog.",
66 storageClass="SourceCatalog",
67 name="{fakesType}{coaddName}Diff_diaSrc_schema",
68 )
69 diaSources = pipeBase.connectionTypes.Output(
70 doc="Detected diaSources on the difference image.",
71 dimensions=("instrument", "visit", "detector"),
72 storageClass="SourceCatalog",
73 name="{fakesType}{coaddName}Diff_diaSrc",
74 )
75 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
76 doc="Difference image with detection mask plane filled in.",
77 dimensions=("instrument", "visit", "detector"),
78 storageClass="ExposureF",
79 name="{fakesType}{coaddName}Diff_differenceExp",
80 )
81
82
83class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
84 pipelineConnections=DetectAndMeasureConnections):
85 """Config for DetectAndMeasureTask
86 """
87 doMerge = pexConfig.Field(
88 dtype=bool,
89 default=True,
90 doc="Merge positive and negative diaSources with grow radius "
91 "set by growFootprint"
92 )
93 doForcedMeasurement = pexConfig.Field(
94 dtype=bool,
95 default=True,
96 doc="Force photometer diaSource locations on PVI?")
97 doAddMetrics = pexConfig.Field(
98 dtype=bool,
99 default=False,
100 doc="Add columns to the source table to hold analysis metrics?"
101 )
102 detection = pexConfig.ConfigurableField(
103 target=SourceDetectionTask,
104 doc="Final source detection for diaSource measurement",
105 )
106 measurement = pexConfig.ConfigurableField(
107 target=DipoleFitTask,
108 doc="Task to measure sources on the difference image.",
109 )
110 doApCorr = lsst.pex.config.Field(
111 dtype=bool,
112 default=True,
113 doc="Run subtask to apply aperture corrections"
114 )
115 applyApCorr = lsst.pex.config.ConfigurableField(
116 target=ApplyApCorrTask,
117 doc="Task to apply aperture corrections"
118 )
119 forcedMeasurement = pexConfig.ConfigurableField(
120 target=ForcedMeasurementTask,
121 doc="Task to force photometer science image at diaSource locations.",
122 )
123 growFootprint = pexConfig.Field(
124 dtype=int,
125 default=2,
126 doc="Grow positive and negative footprints by this many pixels before merging"
127 )
128 diaSourceMatchRadius = pexConfig.Field(
129 dtype=float,
130 default=0.5,
131 doc="Match radius (in arcseconds) for DiaSource to Source association"
132 )
133 doSkySources = pexConfig.Field(
134 dtype=bool,
135 default=False,
136 doc="Generate sky sources?",
137 )
138 skySources = pexConfig.ConfigurableField(
139 target=SkyObjectsTask,
140 doc="Generate sky sources",
141 )
142 badSourceFlags = lsst.pex.config.ListField(
143 dtype=str,
144 doc="Sources with any of these flags set are removed before writing the output catalog.",
145 default=("base_PixelFlags_flag_offimage",
146 ),
147 )
148 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
149
150 def setDefaults(self):
151 # DiaSource Detection
152 self.detection.thresholdPolarity = "both"
153 self.detection.thresholdValue = 5.0
154 self.detection.reEstimateBackground = False
155 self.detection.thresholdType = "pixel_stdev"
156 self.detection.excludeMaskPlanes = ["EDGE"]
157
158 # Add filtered flux measurement, the correct measurement for pre-convolved images.
159 self.measurement.algorithms.names.add("base_PeakLikelihoodFlux")
160 self.measurement.plugins.names |= ["ext_trailedSources_Naive",
161 "base_LocalPhotoCalib",
162 "base_LocalWcs",
163 "ext_shapeHSM_HsmSourceMoments",
164 "ext_shapeHSM_HsmPsfMoments",
165 ]
166 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
167 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
168 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0
169 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
170 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
171 self.forcedMeasurement.copyColumns = {
172 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
173 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
174 self.forcedMeasurement.slots.shape = None
175
176 # Keep track of which footprints contain streaks
177 self.measurement.plugins["base_PixelFlags"].masksFpAnywhere = [
178 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
179 self.measurement.plugins["base_PixelFlags"].masksFpCenter = [
180 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
181
182
183class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
184 """Detect and measure sources on a difference image.
185 """
186 ConfigClass = DetectAndMeasureConfig
187 _DefaultName = "detectAndMeasure"
188
189 def __init__(self, **kwargs):
190 super().__init__(**kwargs)
191 self.schema = afwTable.SourceTable.makeMinimalSchema()
192 # Add coordinate error fields:
193 afwTable.CoordKey.addErrorFields(self.schema)
194
195 self.algMetadata = dafBase.PropertyList()
196 self.makeSubtask("detection", schema=self.schema)
197 self.makeSubtask("measurement", schema=self.schema,
198 algMetadata=self.algMetadata)
199 if self.config.doApCorr:
200 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
201 if self.config.doForcedMeasurement:
202 self.schema.addField(
203 "ip_diffim_forced_PsfFlux_instFlux", "D",
204 "Forced PSF flux measured on the direct image.",
205 units="count")
206 self.schema.addField(
207 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
208 "Forced PSF flux error measured on the direct image.",
209 units="count")
210 self.schema.addField(
211 "ip_diffim_forced_PsfFlux_area", "F",
212 "Forced PSF flux effective area of PSF.",
213 units="pixel")
214 self.schema.addField(
215 "ip_diffim_forced_PsfFlux_flag", "Flag",
216 "Forced PSF flux general failure flag.")
217 self.schema.addField(
218 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
219 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
220 self.schema.addField(
221 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
222 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
223 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
224
225 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
226 self.schema.addField("srcMatchId", "L", "unique id of source match")
227 if self.config.doSkySources:
228 self.makeSubtask("skySources")
229 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
230
231 # Check that the schema and config are consistent
232 for flag in self.config.badSourceFlags:
233 if flag not in self.schema:
234 raise pipeBase.InvalidQuantumError("Field %s not in schema" % flag)
235 # initialize InitOutputs
236 self.outputSchema = afwTable.SourceCatalog(self.schema)
237 self.outputSchema.getTable().setMetadata(self.algMetadata)
238
239 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
240 inputRefs: pipeBase.InputQuantizedConnection,
241 outputRefs: pipeBase.OutputQuantizedConnection):
242 inputs = butlerQC.get(inputRefs)
243 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
244 idFactory = idGenerator.make_table_id_factory()
245 outputs = self.run(**inputs, idFactory=idFactory)
246 butlerQC.put(outputs, outputRefs)
247
248 @timeMethod
249 def run(self, science, matchedTemplate, difference,
250 idFactory=None):
251 """Detect and measure sources on a difference image.
252
253 The difference image will be convolved with a gaussian approximation of
254 the PSF to form a maximum likelihood image for detection.
255 Close positive and negative detections will optionally be merged into
256 dipole diaSources.
257 Sky sources, or forced detections in background regions, will optionally
258 be added, and the configured measurement algorithm will be run on all
259 detections.
260
261 Parameters
262 ----------
263 science : `lsst.afw.image.ExposureF`
264 Science exposure that the template was subtracted from.
265 matchedTemplate : `lsst.afw.image.ExposureF`
266 Warped and PSF-matched template that was used produce the
267 difference image.
268 difference : `lsst.afw.image.ExposureF`
269 Result of subtracting template from the science image.
270 idFactory : `lsst.afw.table.IdFactory`, optional
271 Generator object to assign ids to detected sources in the difference image.
272
273 Returns
274 -------
275 measurementResults : `lsst.pipe.base.Struct`
276
277 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
278 Subtracted exposure with detection mask applied.
279 ``diaSources`` : `lsst.afw.table.SourceCatalog`
280 The catalog of detected sources.
281 """
282 # Ensure that we start with an empty detection mask.
283 mask = difference.mask
284 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
285
286 table = afwTable.SourceTable.make(self.schema, idFactory)
287 table.setMetadata(self.algMetadata)
288 results = self.detection.run(
289 table=table,
290 exposure=difference,
291 doSmooth=True,
292 )
293
294 return self.processResults(science, matchedTemplate, difference, results.sources, table,
295 positiveFootprints=results.positive, negativeFootprints=results.negative)
296
297 def processResults(self, science, matchedTemplate, difference, sources, table,
298 positiveFootprints=None, negativeFootprints=None,):
299 """Measure and process the results of source detection.
300
301 Parameters
302 ----------
303 sources : `lsst.afw.table.SourceCatalog`
304 Detected sources on the difference exposure.
305 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
306 Positive polarity footprints.
307 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
308 Negative polarity footprints.
309 table : `lsst.afw.table.SourceTable`
310 Table object that will be used to create the SourceCatalog.
311 science : `lsst.afw.image.ExposureF`
312 Science exposure that the template was subtracted from.
313 matchedTemplate : `lsst.afw.image.ExposureF`
314 Warped and PSF-matched template that was used produce the
315 difference image.
316 difference : `lsst.afw.image.ExposureF`
317 Result of subtracting template from the science image.
318
319 Returns
320 -------
321 measurementResults : `lsst.pipe.base.Struct`
322
323 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
324 Subtracted exposure with detection mask applied.
325 ``diaSources`` : `lsst.afw.table.SourceCatalog`
326 The catalog of detected sources.
327 """
328 self.metadata.add("nUnmergedDiaSources", len(sources))
329 if self.config.doMerge:
330 fpSet = positiveFootprints
331 fpSet.merge(negativeFootprints, self.config.growFootprint,
332 self.config.growFootprint, False)
333 initialDiaSources = afwTable.SourceCatalog(table)
334 fpSet.makeSources(initialDiaSources)
335 self.log.info("Merging detections into %d sources", len(initialDiaSources))
336 else:
337 initialDiaSources = sources
338 self.metadata.add("nMergedDiaSources", len(initialDiaSources))
339
340 if self.config.doSkySources:
341 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
342
343 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
344 diaSources = self._removeBadSources(initialDiaSources)
345
346 if self.config.doForcedMeasurement:
347 self.measureForcedSources(diaSources, science, difference.getWcs())
348
349 measurementResults = pipeBase.Struct(
350 subtractedMeasuredExposure=difference,
351 diaSources=diaSources,
352 )
353 self.calculateMetrics(difference)
354
355 return measurementResults
356
357 def _removeBadSources(self, diaSources):
358 """Remove bad diaSources from the catalog.
359
360 Parameters
361 ----------
362 diaSources : `lsst.afw.table.SourceCatalog`
363 The catalog of detected sources.
364
365 Returns
366 -------
367 diaSources : `lsst.afw.table.SourceCatalog`
368 The updated catalog of detected sources, with any source that has a
369 flag in ``config.badSourceFlags`` set removed.
370 """
371 nBadTotal = 0
372 selector = np.ones(len(diaSources), dtype=bool)
373 for flag in self.config.badSourceFlags:
374 flags = diaSources[flag]
375 nBad = np.count_nonzero(flags)
376 if nBad > 0:
377 self.log.info("Found and removed %d unphysical sources with flag %s.", nBad, flag)
378 selector &= ~flags
379 nBadTotal += nBad
380 self.metadata.add("nRemovedBadFlaggedSources", nBadTotal)
381 return diaSources[selector].copy(deep=True)
382
383 def addSkySources(self, diaSources, mask, seed):
384 """Add sources in empty regions of the difference image
385 for measuring the background.
386
387 Parameters
388 ----------
389 diaSources : `lsst.afw.table.SourceCatalog`
390 The catalog of detected sources.
391 mask : `lsst.afw.image.Mask`
392 Mask plane for determining regions where Sky sources can be added.
393 seed : `int`
394 Seed value to initialize the random number generator.
395 """
396 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
397 if skySourceFootprints:
398 for foot in skySourceFootprints:
399 s = diaSources.addNew()
400 s.setFootprint(foot)
401 s.set(self.skySourceKey, True)
402
403 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
404 """Use (matched) template and science image to constrain dipole fitting.
405
406 Parameters
407 ----------
408 diaSources : `lsst.afw.table.SourceCatalog`
409 The catalog of detected sources.
410 science : `lsst.afw.image.ExposureF`
411 Science exposure that the template was subtracted from.
412 difference : `lsst.afw.image.ExposureF`
413 Result of subtracting template from the science image.
414 matchedTemplate : `lsst.afw.image.ExposureF`
415 Warped and PSF-matched template that was used produce the
416 difference image.
417 """
418 # Note that this may not be correct if we convolved the science image.
419 # In the future we may wish to persist the matchedScience image.
420 self.measurement.run(diaSources, difference, science, matchedTemplate)
421 if self.config.doApCorr:
422 apCorrMap = difference.getInfo().getApCorrMap()
423 if apCorrMap is None:
424 self.log.warning("Difference image does not have valid aperture correction; skipping.")
425 else:
426 self.applyApCorr.run(
427 catalog=diaSources,
428 apCorrMap=apCorrMap,
429 )
430
431 def measureForcedSources(self, diaSources, science, wcs):
432 """Perform forced measurement of the diaSources on the science image.
433
434 Parameters
435 ----------
436 diaSources : `lsst.afw.table.SourceCatalog`
437 The catalog of detected sources.
438 science : `lsst.afw.image.ExposureF`
439 Science exposure that the template was subtracted from.
440 wcs : `lsst.afw.geom.SkyWcs`
441 Coordinate system definition (wcs) for the exposure.
442 """
443 # Run forced psf photometry on the PVI at the diaSource locations.
444 # Copy the measured flux and error into the diaSource.
445 forcedSources = self.forcedMeasurement.generateMeasCat(
446 science, diaSources, wcs)
447 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
448 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
449 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
450 "ip_diffim_forced_PsfFlux_instFlux", True)
451 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
452 "ip_diffim_forced_PsfFlux_instFluxErr", True)
453 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
454 "ip_diffim_forced_PsfFlux_area", True)
455 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
456 "ip_diffim_forced_PsfFlux_flag", True)
457 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
458 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
459 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
460 "ip_diffim_forced_PsfFlux_flag_edge", True)
461 for diaSource, forcedSource in zip(diaSources, forcedSources):
462 diaSource.assign(forcedSource, mapper)
463
464 def calculateMetrics(self, difference):
465 """Add image QA metrics to the Task metadata.
466
467 Parameters
468 ----------
469 difference : `lsst.afw.image.Exposure`
470 The target image to calculate metrics for.
471 """
472 mask = difference.mask
473 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
474 self.metadata.add("nGoodPixels", np.sum(~badPix))
475 self.metadata.add("nBadPixels", np.sum(badPix))
476 detPosPix = (mask.array & mask.getPlaneBitMask("DETECTED")) > 0
477 detNegPix = (mask.array & mask.getPlaneBitMask("DETECTED_NEGATIVE")) > 0
478 self.metadata.add("nPixelsDetectedPositive", np.sum(detPosPix))
479 self.metadata.add("nPixelsDetectedNegative", np.sum(detNegPix))
480 detPosPix &= badPix
481 detNegPix &= badPix
482 self.metadata.add("nBadPixelsDetectedPositive", np.sum(detPosPix))
483 self.metadata.add("nBadPixelsDetectedNegative", np.sum(detNegPix))
484
485
487 scoreExposure = pipeBase.connectionTypes.Input(
488 doc="Maximum likelihood image for detection.",
489 dimensions=("instrument", "visit", "detector"),
490 storageClass="ExposureF",
491 name="{fakesType}{coaddName}Diff_scoreExp",
492 )
493
494
496 pipelineConnections=DetectAndMeasureScoreConnections):
497 pass
498
499
501 """Detect DIA sources using a score image,
502 and measure the detections on the difference image.
503
504 Source detection is run on the supplied score, or maximum likelihood,
505 image. Note that no additional convolution will be done in this case.
506 Close positive and negative detections will optionally be merged into
507 dipole diaSources.
508 Sky sources, or forced detections in background regions, will optionally
509 be added, and the configured measurement algorithm will be run on all
510 detections.
511 """
512 ConfigClass = DetectAndMeasureScoreConfig
513 _DefaultName = "detectAndMeasureScore"
514
515 @timeMethod
516 def run(self, science, matchedTemplate, difference, scoreExposure,
517 idFactory=None):
518 """Detect and measure sources on a score image.
519
520 Parameters
521 ----------
522 science : `lsst.afw.image.ExposureF`
523 Science exposure that the template was subtracted from.
524 matchedTemplate : `lsst.afw.image.ExposureF`
525 Warped and PSF-matched template that was used produce the
526 difference image.
527 difference : `lsst.afw.image.ExposureF`
528 Result of subtracting template from the science image.
529 scoreExposure : `lsst.afw.image.ExposureF`
530 Score or maximum likelihood difference image
531 idFactory : `lsst.afw.table.IdFactory`, optional
532 Generator object to assign ids to detected sources in the difference image.
533
534 Returns
535 -------
536 measurementResults : `lsst.pipe.base.Struct`
537
538 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
539 Subtracted exposure with detection mask applied.
540 ``diaSources`` : `lsst.afw.table.SourceCatalog`
541 The catalog of detected sources.
542 """
543 # Ensure that we start with an empty detection mask.
544 mask = scoreExposure.mask
545 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
546
547 table = afwTable.SourceTable.make(self.schema, idFactory)
548 table.setMetadata(self.algMetadata)
549 results = self.detection.run(
550 table=table,
551 exposure=scoreExposure,
552 doSmooth=False,
553 )
554 # Copy the detection mask from the Score image to the difference image
555 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
556
557 return self.processResults(science, matchedTemplate, difference, results.sources, table,
558 positiveFootprints=results.positive, negativeFootprints=results.negative)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)