lsst.ip.diffim g9c4eec70e9+5425abcb23
Loading...
Searching...
No Matches
detectAndMeasure.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import numpy as np
23
24import lsst.afw.table as afwTable
25import lsst.daf.base as dafBase
26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources # noqa: F401
29import lsst.meas.extensions.shapeHSM
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.utils
33from lsst.utils.timer import timeMethod
34
35from . import DipoleFitTask
36
37__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
38 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
39
40
41class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
42 dimensions=("instrument", "visit", "detector"),
43 defaultTemplates={"coaddName": "deep",
44 "warpTypeSuffix": "",
45 "fakesType": ""}):
46 science = pipeBase.connectionTypes.Input(
47 doc="Input science exposure.",
48 dimensions=("instrument", "visit", "detector"),
49 storageClass="ExposureF",
50 name="{fakesType}calexp"
51 )
52 matchedTemplate = pipeBase.connectionTypes.Input(
53 doc="Warped and PSF-matched template used to create the difference image.",
54 dimensions=("instrument", "visit", "detector"),
55 storageClass="ExposureF",
56 name="{fakesType}{coaddName}Diff_matchedExp",
57 )
58 difference = pipeBase.connectionTypes.Input(
59 doc="Result of subtracting template from science.",
60 dimensions=("instrument", "visit", "detector"),
61 storageClass="ExposureF",
62 name="{fakesType}{coaddName}Diff_differenceTempExp",
63 )
64 outputSchema = pipeBase.connectionTypes.InitOutput(
65 doc="Schema (as an example catalog) for output DIASource catalog.",
66 storageClass="SourceCatalog",
67 name="{fakesType}{coaddName}Diff_diaSrc_schema",
68 )
69 diaSources = pipeBase.connectionTypes.Output(
70 doc="Detected diaSources on the difference image.",
71 dimensions=("instrument", "visit", "detector"),
72 storageClass="SourceCatalog",
73 name="{fakesType}{coaddName}Diff_diaSrc",
74 )
75 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
76 doc="Difference image with detection mask plane filled in.",
77 dimensions=("instrument", "visit", "detector"),
78 storageClass="ExposureF",
79 name="{fakesType}{coaddName}Diff_differenceExp",
80 )
81
82
83class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
84 pipelineConnections=DetectAndMeasureConnections):
85 """Config for DetectAndMeasureTask
86 """
87 doMerge = pexConfig.Field(
88 dtype=bool,
89 default=True,
90 doc="Merge positive and negative diaSources with grow radius "
91 "set by growFootprint"
92 )
93 doForcedMeasurement = pexConfig.Field(
94 dtype=bool,
95 default=True,
96 doc="Force photometer diaSource locations on PVI?")
97 doAddMetrics = pexConfig.Field(
98 dtype=bool,
99 default=False,
100 doc="Add columns to the source table to hold analysis metrics?"
101 )
102 detection = pexConfig.ConfigurableField(
103 target=SourceDetectionTask,
104 doc="Final source detection for diaSource measurement",
105 )
106 measurement = pexConfig.ConfigurableField(
107 target=DipoleFitTask,
108 doc="Task to measure sources on the difference image.",
109 )
110 doApCorr = lsst.pex.config.Field(
111 dtype=bool,
112 default=True,
113 doc="Run subtask to apply aperture corrections"
114 )
115 applyApCorr = lsst.pex.config.ConfigurableField(
116 target=ApplyApCorrTask,
117 doc="Task to apply aperture corrections"
118 )
119 forcedMeasurement = pexConfig.ConfigurableField(
120 target=ForcedMeasurementTask,
121 doc="Task to force photometer science image at diaSource locations.",
122 )
123 growFootprint = pexConfig.Field(
124 dtype=int,
125 default=2,
126 doc="Grow positive and negative footprints by this many pixels before merging"
127 )
128 diaSourceMatchRadius = pexConfig.Field(
129 dtype=float,
130 default=0.5,
131 doc="Match radius (in arcseconds) for DiaSource to Source association"
132 )
133 doSkySources = pexConfig.Field(
134 dtype=bool,
135 default=False,
136 doc="Generate sky sources?",
137 )
138 skySources = pexConfig.ConfigurableField(
139 target=SkyObjectsTask,
140 doc="Generate sky sources",
141 )
142 badSourceFlags = lsst.pex.config.ListField(
143 dtype=str,
144 doc="Sources with any of these flags set are removed before writing the output catalog.",
145 default=("base_PixelFlags_flag_offimage",
146 "base_PixelFlags_flag_interpolatedCenterAll",
147 "base_PixelFlags_flag_badCenterAll",
148 "base_PixelFlags_flag_edgeCenterAll",
149 ),
150 )
151 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
152
153 def setDefaults(self):
154 # DiaSource Detection
155 self.detection.thresholdPolarity = "both"
156 self.detection.thresholdValue = 5.0
157 self.detection.reEstimateBackground = False
158 self.detection.thresholdType = "pixel_stdev"
159 self.detection.excludeMaskPlanes = ["EDGE"]
160
161 # Add filtered flux measurement, the correct measurement for pre-convolved images.
162 self.measurement.algorithms.names.add("base_PeakLikelihoodFlux")
163 self.measurement.plugins.names |= ["ext_trailedSources_Naive",
164 "base_LocalPhotoCalib",
165 "base_LocalWcs",
166 "ext_shapeHSM_HsmSourceMoments",
167 "ext_shapeHSM_HsmPsfMoments",
168 ]
169 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
170 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
171 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0
172 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
173 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
174 self.forcedMeasurement.copyColumns = {
175 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
176 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
177 self.forcedMeasurement.slots.shape = None
178
179 # Keep track of which footprints contain streaks
180 self.measurement.plugins["base_PixelFlags"].masksFpAnywhere = [
181 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
182 self.measurement.plugins["base_PixelFlags"].masksFpCenter = [
183 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
184 self.skySources.avoidMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "NO_DATA", "EDGE"]
185
186
187class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
188 """Detect and measure sources on a difference image.
189 """
190 ConfigClass = DetectAndMeasureConfig
191 _DefaultName = "detectAndMeasure"
192
193 def __init__(self, **kwargs):
194 super().__init__(**kwargs)
195 self.schema = afwTable.SourceTable.makeMinimalSchema()
196 # Add coordinate error fields:
197 afwTable.CoordKey.addErrorFields(self.schema)
198
199 self.algMetadata = dafBase.PropertyList()
200 self.makeSubtask("detection", schema=self.schema)
201 self.makeSubtask("measurement", schema=self.schema,
202 algMetadata=self.algMetadata)
203 if self.config.doApCorr:
204 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
205 if self.config.doForcedMeasurement:
206 self.schema.addField(
207 "ip_diffim_forced_PsfFlux_instFlux", "D",
208 "Forced PSF flux measured on the direct image.",
209 units="count")
210 self.schema.addField(
211 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
212 "Forced PSF flux error measured on the direct image.",
213 units="count")
214 self.schema.addField(
215 "ip_diffim_forced_PsfFlux_area", "F",
216 "Forced PSF flux effective area of PSF.",
217 units="pixel")
218 self.schema.addField(
219 "ip_diffim_forced_PsfFlux_flag", "Flag",
220 "Forced PSF flux general failure flag.")
221 self.schema.addField(
222 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
223 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
224 self.schema.addField(
225 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
226 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
227 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
228
229 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
230 self.schema.addField("srcMatchId", "L", "unique id of source match")
231 if self.config.doSkySources:
232 self.makeSubtask("skySources")
233 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
234
235 # Check that the schema and config are consistent
236 for flag in self.config.badSourceFlags:
237 if flag not in self.schema:
238 raise pipeBase.InvalidQuantumError("Field %s not in schema" % flag)
239 # initialize InitOutputs
240 self.outputSchema = afwTable.SourceCatalog(self.schema)
241 self.outputSchema.getTable().setMetadata(self.algMetadata)
242
243 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
244 inputRefs: pipeBase.InputQuantizedConnection,
245 outputRefs: pipeBase.OutputQuantizedConnection):
246 inputs = butlerQC.get(inputRefs)
247 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
248 idFactory = idGenerator.make_table_id_factory()
249 outputs = self.run(**inputs, idFactory=idFactory)
250 butlerQC.put(outputs, outputRefs)
251
252 @timeMethod
253 def run(self, science, matchedTemplate, difference,
254 idFactory=None):
255 """Detect and measure sources on a difference image.
256
257 The difference image will be convolved with a gaussian approximation of
258 the PSF to form a maximum likelihood image for detection.
259 Close positive and negative detections will optionally be merged into
260 dipole diaSources.
261 Sky sources, or forced detections in background regions, will optionally
262 be added, and the configured measurement algorithm will be run on all
263 detections.
264
265 Parameters
266 ----------
267 science : `lsst.afw.image.ExposureF`
268 Science exposure that the template was subtracted from.
269 matchedTemplate : `lsst.afw.image.ExposureF`
270 Warped and PSF-matched template that was used produce the
271 difference image.
272 difference : `lsst.afw.image.ExposureF`
273 Result of subtracting template from the science image.
274 idFactory : `lsst.afw.table.IdFactory`, optional
275 Generator object to assign ids to detected sources in the difference image.
276
277 Returns
278 -------
279 measurementResults : `lsst.pipe.base.Struct`
280
281 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
282 Subtracted exposure with detection mask applied.
283 ``diaSources`` : `lsst.afw.table.SourceCatalog`
284 The catalog of detected sources.
285 """
286 # Ensure that we start with an empty detection mask.
287 mask = difference.mask
288 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
289
290 table = afwTable.SourceTable.make(self.schema, idFactory)
291 table.setMetadata(self.algMetadata)
292 results = self.detection.run(
293 table=table,
294 exposure=difference,
295 doSmooth=True,
296 )
297
298 return self.processResults(science, matchedTemplate, difference, results.sources, table,
299 positiveFootprints=results.positive, negativeFootprints=results.negative)
300
301 def processResults(self, science, matchedTemplate, difference, sources, table,
302 positiveFootprints=None, negativeFootprints=None,):
303 """Measure and process the results of source detection.
304
305 Parameters
306 ----------
307 science : `lsst.afw.image.ExposureF`
308 Science exposure that the template was subtracted from.
309 matchedTemplate : `lsst.afw.image.ExposureF`
310 Warped and PSF-matched template that was used produce the
311 difference image.
312 difference : `lsst.afw.image.ExposureF`
313 Result of subtracting template from the science image.
314 sources : `lsst.afw.table.SourceCatalog`
315 Detected sources on the difference exposure.
316 table : `lsst.afw.table.SourceTable`
317 Table object that will be used to create the SourceCatalog.
318 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
319 Positive polarity footprints.
320 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
321 Negative polarity footprints.
322
323 Returns
324 -------
325 measurementResults : `lsst.pipe.base.Struct`
326
327 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
328 Subtracted exposure with detection mask applied.
329 ``diaSources`` : `lsst.afw.table.SourceCatalog`
330 The catalog of detected sources.
331 """
332 self.metadata.add("nUnmergedDiaSources", len(sources))
333 if self.config.doMerge:
334 fpSet = positiveFootprints
335 fpSet.merge(negativeFootprints, self.config.growFootprint,
336 self.config.growFootprint, False)
337 initialDiaSources = afwTable.SourceCatalog(table)
338 fpSet.makeSources(initialDiaSources)
339 self.log.info("Merging detections into %d sources", len(initialDiaSources))
340 else:
341 initialDiaSources = sources
342 self.metadata.add("nMergedDiaSources", len(initialDiaSources))
343
344 if self.config.doSkySources:
345 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
346
347 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
348 diaSources = self._removeBadSources(initialDiaSources)
349
350 if self.config.doForcedMeasurement:
351 self.measureForcedSources(diaSources, science, difference.getWcs())
352
353 measurementResults = pipeBase.Struct(
354 subtractedMeasuredExposure=difference,
355 diaSources=diaSources,
356 )
357 self.calculateMetrics(difference)
358
359 return measurementResults
360
361 def _removeBadSources(self, diaSources):
362 """Remove bad diaSources from the catalog.
363
364 Parameters
365 ----------
366 diaSources : `lsst.afw.table.SourceCatalog`
367 The catalog of detected sources.
368
369 Returns
370 -------
371 diaSources : `lsst.afw.table.SourceCatalog`
372 The updated catalog of detected sources, with any source that has a
373 flag in ``config.badSourceFlags`` set removed.
374 """
375 nBadTotal = 0
376 selector = np.ones(len(diaSources), dtype=bool)
377 for flag in self.config.badSourceFlags:
378 flags = diaSources[flag]
379 nBad = np.count_nonzero(flags)
380 if nBad > 0:
381 self.log.info("Found and removed %d unphysical sources with flag %s.", nBad, flag)
382 selector &= ~flags
383 nBadTotal += nBad
384 self.metadata.add("nRemovedBadFlaggedSources", nBadTotal)
385 return diaSources[selector].copy(deep=True)
386
387 def addSkySources(self, diaSources, mask, seed):
388 """Add sources in empty regions of the difference image
389 for measuring the background.
390
391 Parameters
392 ----------
393 diaSources : `lsst.afw.table.SourceCatalog`
394 The catalog of detected sources.
395 mask : `lsst.afw.image.Mask`
396 Mask plane for determining regions where Sky sources can be added.
397 seed : `int`
398 Seed value to initialize the random number generator.
399 """
400 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
401 self.metadata.add("nSkySources", len(skySourceFootprints))
402 if skySourceFootprints:
403 for foot in skySourceFootprints:
404 s = diaSources.addNew()
405 s.setFootprint(foot)
406 s.set(self.skySourceKey, True)
407
408 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
409 """Use (matched) template and science image to constrain dipole fitting.
410
411 Parameters
412 ----------
413 diaSources : `lsst.afw.table.SourceCatalog`
414 The catalog of detected sources.
415 science : `lsst.afw.image.ExposureF`
416 Science exposure that the template was subtracted from.
417 difference : `lsst.afw.image.ExposureF`
418 Result of subtracting template from the science image.
419 matchedTemplate : `lsst.afw.image.ExposureF`
420 Warped and PSF-matched template that was used produce the
421 difference image.
422 """
423 # Ensure that the required mask planes are present
424 for mp in self.config.measurement.plugins["base_PixelFlags"].masksFpAnywhere:
425 difference.mask.addMaskPlane(mp)
426 # Note that this may not be correct if we convolved the science image.
427 # In the future we may wish to persist the matchedScience image.
428 self.measurement.run(diaSources, difference, science, matchedTemplate)
429 if self.config.doApCorr:
430 apCorrMap = difference.getInfo().getApCorrMap()
431 if apCorrMap is None:
432 self.log.warning("Difference image does not have valid aperture correction; skipping.")
433 else:
434 self.applyApCorr.run(
435 catalog=diaSources,
436 apCorrMap=apCorrMap,
437 )
438
439 def measureForcedSources(self, diaSources, science, wcs):
440 """Perform forced measurement of the diaSources on the science image.
441
442 Parameters
443 ----------
444 diaSources : `lsst.afw.table.SourceCatalog`
445 The catalog of detected sources.
446 science : `lsst.afw.image.ExposureF`
447 Science exposure that the template was subtracted from.
448 wcs : `lsst.afw.geom.SkyWcs`
449 Coordinate system definition (wcs) for the exposure.
450 """
451 # Run forced psf photometry on the PVI at the diaSource locations.
452 # Copy the measured flux and error into the diaSource.
453 forcedSources = self.forcedMeasurement.generateMeasCat(
454 science, diaSources, wcs)
455 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
456 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
457 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
458 "ip_diffim_forced_PsfFlux_instFlux", True)
459 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
460 "ip_diffim_forced_PsfFlux_instFluxErr", True)
461 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
462 "ip_diffim_forced_PsfFlux_area", True)
463 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
464 "ip_diffim_forced_PsfFlux_flag", True)
465 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
466 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
467 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
468 "ip_diffim_forced_PsfFlux_flag_edge", True)
469 for diaSource, forcedSource in zip(diaSources, forcedSources):
470 diaSource.assign(forcedSource, mapper)
471
472 def calculateMetrics(self, difference):
473 """Add image QA metrics to the Task metadata.
474
475 Parameters
476 ----------
477 difference : `lsst.afw.image.Exposure`
478 The target image to calculate metrics for.
479 """
480 mask = difference.mask
481 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
482 self.metadata.add("nGoodPixels", np.sum(~badPix))
483 self.metadata.add("nBadPixels", np.sum(badPix))
484 detPosPix = (mask.array & mask.getPlaneBitMask("DETECTED")) > 0
485 detNegPix = (mask.array & mask.getPlaneBitMask("DETECTED_NEGATIVE")) > 0
486 self.metadata.add("nPixelsDetectedPositive", np.sum(detPosPix))
487 self.metadata.add("nPixelsDetectedNegative", np.sum(detNegPix))
488 detPosPix &= badPix
489 detNegPix &= badPix
490 self.metadata.add("nBadPixelsDetectedPositive", np.sum(detPosPix))
491 self.metadata.add("nBadPixelsDetectedNegative", np.sum(detNegPix))
492
493
495 scoreExposure = pipeBase.connectionTypes.Input(
496 doc="Maximum likelihood image for detection.",
497 dimensions=("instrument", "visit", "detector"),
498 storageClass="ExposureF",
499 name="{fakesType}{coaddName}Diff_scoreExp",
500 )
501
502
504 pipelineConnections=DetectAndMeasureScoreConnections):
505 pass
506
507
509 """Detect DIA sources using a score image,
510 and measure the detections on the difference image.
511
512 Source detection is run on the supplied score, or maximum likelihood,
513 image. Note that no additional convolution will be done in this case.
514 Close positive and negative detections will optionally be merged into
515 dipole diaSources.
516 Sky sources, or forced detections in background regions, will optionally
517 be added, and the configured measurement algorithm will be run on all
518 detections.
519 """
520 ConfigClass = DetectAndMeasureScoreConfig
521 _DefaultName = "detectAndMeasureScore"
522
523 @timeMethod
524 def run(self, science, matchedTemplate, difference, scoreExposure,
525 idFactory=None):
526 """Detect and measure sources on a score image.
527
528 Parameters
529 ----------
530 science : `lsst.afw.image.ExposureF`
531 Science exposure that the template was subtracted from.
532 matchedTemplate : `lsst.afw.image.ExposureF`
533 Warped and PSF-matched template that was used produce the
534 difference image.
535 difference : `lsst.afw.image.ExposureF`
536 Result of subtracting template from the science image.
537 scoreExposure : `lsst.afw.image.ExposureF`
538 Score or maximum likelihood difference image
539 idFactory : `lsst.afw.table.IdFactory`, optional
540 Generator object to assign ids to detected sources in the difference image.
541
542 Returns
543 -------
544 measurementResults : `lsst.pipe.base.Struct`
545
546 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
547 Subtracted exposure with detection mask applied.
548 ``diaSources`` : `lsst.afw.table.SourceCatalog`
549 The catalog of detected sources.
550 """
551 # Ensure that we start with an empty detection mask.
552 mask = scoreExposure.mask
553 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
554
555 table = afwTable.SourceTable.make(self.schema, idFactory)
556 table.setMetadata(self.algMetadata)
557 results = self.detection.run(
558 table=table,
559 exposure=scoreExposure,
560 doSmooth=False,
561 )
562 # Copy the detection mask from the Score image to the difference image
563 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
564
565 return self.processResults(science, matchedTemplate, difference, results.sources, table,
566 positiveFootprints=results.positive, negativeFootprints=results.negative)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)