lsst.ip.diffim gc0f3af6251+10a3fd39cd
Loading...
Searching...
No Matches
detectAndMeasure.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import numpy as np
23
24import lsst.afw.detection as afwDetection
25import lsst.afw.table as afwTable
26import lsst.daf.base as dafBase
27import lsst.geom
28from lsst.ip.diffim.utils import getPsfFwhm, angleMean
29from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask, SetPrimaryFlagsTask
30from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
31import lsst.meas.deblender
32import lsst.meas.extensions.trailedSources # noqa: F401
33import lsst.meas.extensions.shapeHSM
34import lsst.pex.config as pexConfig
35from lsst.pex.exceptions import InvalidParameterError
36import lsst.pipe.base as pipeBase
37import lsst.utils
38from lsst.utils.timer import timeMethod
39
40from . import DipoleFitTask
41
42__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
43 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
44
45
46class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("instrument", "visit", "detector"),
48 defaultTemplates={"coaddName": "deep",
49 "warpTypeSuffix": "",
50 "fakesType": ""}):
51 science = pipeBase.connectionTypes.Input(
52 doc="Input science exposure.",
53 dimensions=("instrument", "visit", "detector"),
54 storageClass="ExposureF",
55 name="{fakesType}calexp"
56 )
57 matchedTemplate = pipeBase.connectionTypes.Input(
58 doc="Warped and PSF-matched template used to create the difference image.",
59 dimensions=("instrument", "visit", "detector"),
60 storageClass="ExposureF",
61 name="{fakesType}{coaddName}Diff_matchedExp",
62 )
63 difference = pipeBase.connectionTypes.Input(
64 doc="Result of subtracting template from science.",
65 dimensions=("instrument", "visit", "detector"),
66 storageClass="ExposureF",
67 name="{fakesType}{coaddName}Diff_differenceTempExp",
68 )
69 outputSchema = pipeBase.connectionTypes.InitOutput(
70 doc="Schema (as an example catalog) for output DIASource catalog.",
71 storageClass="SourceCatalog",
72 name="{fakesType}{coaddName}Diff_diaSrc_schema",
73 )
74 diaSources = pipeBase.connectionTypes.Output(
75 doc="Detected diaSources on the difference image.",
76 dimensions=("instrument", "visit", "detector"),
77 storageClass="SourceCatalog",
78 name="{fakesType}{coaddName}Diff_diaSrc",
79 )
80 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
81 doc="Difference image with detection mask plane filled in.",
82 dimensions=("instrument", "visit", "detector"),
83 storageClass="ExposureF",
84 name="{fakesType}{coaddName}Diff_differenceExp",
85 )
86 spatiallySampledMetrics = pipeBase.connectionTypes.Output(
87 doc="Summary metrics computed at randomized locations.",
88 dimensions=("instrument", "visit", "detector"),
89 storageClass="ArrowAstropy",
90 name="{fakesType}{coaddName}Diff_spatiallySampledMetrics",
91 )
92
93 def __init__(self, *, config=None):
94 super().__init__(config=config)
95 if not config.doWriteMetrics:
96 self.outputs.remove("spatiallySampledMetrics")
97
98
99class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
100 pipelineConnections=DetectAndMeasureConnections):
101 """Config for DetectAndMeasureTask
102 """
103 doMerge = pexConfig.Field(
104 dtype=bool,
105 default=True,
106 doc="Merge positive and negative diaSources with grow radius "
107 "set by growFootprint"
108 )
109 doForcedMeasurement = pexConfig.Field(
110 dtype=bool,
111 default=True,
112 doc="Force photometer diaSource locations on PVI?")
113 doAddMetrics = pexConfig.Field(
114 dtype=bool,
115 default=False,
116 doc="Add columns to the source table to hold analysis metrics?"
117 )
118 detection = pexConfig.ConfigurableField(
119 target=SourceDetectionTask,
120 doc="Final source detection for diaSource measurement",
121 )
122 deblend = pexConfig.ConfigurableField(
123 target=lsst.meas.deblender.SourceDeblendTask,
124 doc="Task to split blended sources into their components."
125 )
126 measurement = pexConfig.ConfigurableField(
127 target=DipoleFitTask,
128 doc="Task to measure sources on the difference image.",
129 )
130 doApCorr = lsst.pex.config.Field(
131 dtype=bool,
132 default=True,
133 doc="Run subtask to apply aperture corrections"
134 )
135 applyApCorr = lsst.pex.config.ConfigurableField(
136 target=ApplyApCorrTask,
137 doc="Task to apply aperture corrections"
138 )
139 forcedMeasurement = pexConfig.ConfigurableField(
140 target=ForcedMeasurementTask,
141 doc="Task to force photometer science image at diaSource locations.",
142 )
143 growFootprint = pexConfig.Field(
144 dtype=int,
145 default=2,
146 doc="Grow positive and negative footprints by this many pixels before merging"
147 )
148 diaSourceMatchRadius = pexConfig.Field(
149 dtype=float,
150 default=0.5,
151 doc="Match radius (in arcseconds) for DiaSource to Source association"
152 )
153 doSkySources = pexConfig.Field(
154 dtype=bool,
155 default=False,
156 doc="Generate sky sources?",
157 )
158 skySources = pexConfig.ConfigurableField(
159 target=SkyObjectsTask,
160 doc="Generate sky sources",
161 )
162 setPrimaryFlags = pexConfig.ConfigurableField(
163 target=SetPrimaryFlagsTask,
164 doc="Task to add isPrimary and deblending-related flags to the catalog."
165 )
166 badSourceFlags = lsst.pex.config.ListField(
167 dtype=str,
168 doc="Sources with any of these flags set are removed before writing the output catalog.",
169 default=("base_PixelFlags_flag_offimage",
170 "base_PixelFlags_flag_interpolatedCenterAll",
171 "base_PixelFlags_flag_badCenterAll",
172 "base_PixelFlags_flag_edgeCenterAll",
173 "base_PixelFlags_flag_saturatedCenterAll",
174 ),
175 )
176 metricsMaskPlanes = lsst.pex.config.ListField(
177 dtype=str,
178 doc="List of mask planes to include in metrics",
179 default=('BAD', 'CLIPPED', 'CR', 'DETECTED', 'DETECTED_NEGATIVE', 'EDGE',
180 'INEXACT_PSF', 'INJECTED', 'INJECTED_TEMPLATE', 'INTRP', 'NOT_DEBLENDED',
181 'NO_DATA', 'REJECTED', 'SAT', 'SAT_TEMPLATE', 'SENSOR_EDGE', 'STREAK', 'SUSPECT',
182 'UNMASKEDNAN',
183 ),
184 )
185 metricSources = pexConfig.ConfigurableField(
186 target=SkyObjectsTask,
187 doc="Generate QA metric sources",
188 )
189 doWriteMetrics = lsst.pex.config.Field(
190 dtype=bool,
191 default=True,
192 doc="Compute and write summary metrics."
193 )
194 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
195
196 def setDefaults(self):
197 # DiaSource Detection
198 self.detection.thresholdPolarity = "both"
199 self.detection.thresholdValue = 5.0
200 self.detection.reEstimateBackground = False
201 self.detection.thresholdType = "pixel_stdev"
202 self.detection.excludeMaskPlanes = ["EDGE"]
203
204 # Add filtered flux measurement, the correct measurement for pre-convolved images.
205 self.measurement.algorithms.names.add("base_PeakLikelihoodFlux")
206 self.measurement.plugins.names |= ["ext_trailedSources_Naive",
207 "base_LocalPhotoCalib",
208 "base_LocalWcs",
209 "ext_shapeHSM_HsmSourceMoments",
210 "ext_shapeHSM_HsmPsfMoments",
211 ]
212 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
213 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
214 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0
215 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
216 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
217 self.forcedMeasurement.copyColumns = {
218 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
219 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
220 self.forcedMeasurement.slots.shape = None
221
222 # Keep track of which footprints contain streaks
223 self.measurement.plugins["base_PixelFlags"].masksFpAnywhere = [
224 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
225 self.measurement.plugins["base_PixelFlags"].masksFpCenter = [
226 "STREAK", "INJECTED", "INJECTED_TEMPLATE"]
227 self.skySources.avoidMask = ["DETECTED", "DETECTED_NEGATIVE", "BAD", "NO_DATA", "EDGE"]
228 self.metricSources.avoidMask = ["NO_DATA", "EDGE"]
229
230
231class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
232 """Detect and measure sources on a difference image.
233 """
234 ConfigClass = DetectAndMeasureConfig
235 _DefaultName = "detectAndMeasure"
236
237 def __init__(self, **kwargs):
238 super().__init__(**kwargs)
239 self.schema = afwTable.SourceTable.makeMinimalSchema()
240 # Add coordinate error fields:
241 afwTable.CoordKey.addErrorFields(self.schema)
242
243 self.algMetadata = dafBase.PropertyList()
244 self.makeSubtask("detection", schema=self.schema)
245 self.makeSubtask("deblend", schema=self.schema)
246 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True)
247 self.makeSubtask("measurement", schema=self.schema,
248 algMetadata=self.algMetadata)
249 if self.config.doApCorr:
250 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
251 if self.config.doForcedMeasurement:
252 self.schema.addField(
253 "ip_diffim_forced_PsfFlux_instFlux", "D",
254 "Forced PSF flux measured on the direct image.",
255 units="count")
256 self.schema.addField(
257 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
258 "Forced PSF flux error measured on the direct image.",
259 units="count")
260 self.schema.addField(
261 "ip_diffim_forced_PsfFlux_area", "F",
262 "Forced PSF flux effective area of PSF.",
263 units="pixel")
264 self.schema.addField(
265 "ip_diffim_forced_PsfFlux_flag", "Flag",
266 "Forced PSF flux general failure flag.")
267 self.schema.addField(
268 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
269 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
270 self.schema.addField(
271 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
272 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
273 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
274
275 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
276 self.schema.addField("srcMatchId", "L", "unique id of source match")
277 if self.config.doSkySources:
278 self.makeSubtask("skySources", schema=self.schema)
279
280 # Check that the schema and config are consistent
281 for flag in self.config.badSourceFlags:
282 if flag not in self.schema:
283 raise pipeBase.InvalidQuantumError("Field %s not in schema" % flag)
284
285 if self.config.doWriteMetrics:
286 self.makeSubtask("metricSources")
287 self.metricSchema = afwTable.SourceTable.makeMinimalSchema()
288 self.metricSchema.addField(
289 "x", "F",
290 "X location of the metric evaluation.",
291 units="pixel")
292 self.metricSchema.addField(
293 "y", "F",
294 "Y location of the metric evaluation.",
295 units="pixel")
296 self.metricSources.skySourceKey = self.metricSchema.addField("sky_source", type="Flag",
297 doc="Metric evaluation objects.")
298 self.metricSchema.addField(
299 "source_density", "F",
300 "Density of diaSources at location.",
301 units="count/degree^2")
302 self.metricSchema.addField(
303 "dipole_density", "F",
304 "Density of dipoles at location.",
305 units="count/degree^2")
306 self.metricSchema.addField(
307 "dipole_direction", "F",
308 "Mean dipole orientation.",
309 units="radian")
310 self.metricSchema.addField(
311 "dipole_separation", "F",
312 "Mean dipole separation.",
313 units="pixel")
314 self.metricSchema.addField(
315 "template_value", "F",
316 "Median of template at location.",
317 units="nJy")
318 self.metricSchema.addField(
319 "science_value", "F",
320 "Median of science at location.",
321 units="nJy")
322 self.metricSchema.addField(
323 "diffim_value", "F",
324 "Median of diffim at location.",
325 units="nJy")
326 self.metricSchema.addField(
327 "science_psfSize", "F",
328 "Width of the science image PSF at location.",
329 units="pixel")
330 self.metricSchema.addField(
331 "template_psfSize", "F",
332 "Width of the template image PSF at location.",
333 units="pixel")
334 for maskPlane in self.config.metricsMaskPlanes:
335 self.metricSchema.addField(
336 "%s_mask_fraction"%maskPlane.lower(), "F",
337 "Fraction of pixels with %s mask"%maskPlane
338 )
339
340 # initialize InitOutputs
341 self.outputSchema = afwTable.SourceCatalog(self.schema)
342 self.outputSchema.getTable().setMetadata(self.algMetadata)
343
344 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
345 inputRefs: pipeBase.InputQuantizedConnection,
346 outputRefs: pipeBase.OutputQuantizedConnection):
347 inputs = butlerQC.get(inputRefs)
348 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
349 idFactory = idGenerator.make_table_id_factory()
350 outputs = self.run(**inputs, idFactory=idFactory)
351 butlerQC.put(outputs, outputRefs)
352
353 @timeMethod
354 def run(self, science, matchedTemplate, difference,
355 idFactory=None):
356 """Detect and measure sources on a difference image.
357
358 The difference image will be convolved with a gaussian approximation of
359 the PSF to form a maximum likelihood image for detection.
360 Close positive and negative detections will optionally be merged into
361 dipole diaSources.
362 Sky sources, or forced detections in background regions, will optionally
363 be added, and the configured measurement algorithm will be run on all
364 detections.
365
366 Parameters
367 ----------
368 science : `lsst.afw.image.ExposureF`
369 Science exposure that the template was subtracted from.
370 matchedTemplate : `lsst.afw.image.ExposureF`
371 Warped and PSF-matched template that was used produce the
372 difference image.
373 difference : `lsst.afw.image.ExposureF`
374 Result of subtracting template from the science image.
375 idFactory : `lsst.afw.table.IdFactory`, optional
376 Generator object used to assign ids to detected sources in the
377 difference image. Ids from this generator are not set until after
378 deblending and merging positive/negative peaks.
379
380 Returns
381 -------
382 measurementResults : `lsst.pipe.base.Struct`
383
384 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
385 Subtracted exposure with detection mask applied.
386 ``diaSources`` : `lsst.afw.table.SourceCatalog`
387 The catalog of detected sources.
388 """
389 if idFactory is None:
390 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory()
391
392 # Ensure that we start with an empty detection mask.
393 mask = difference.mask
394 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
395
396 # Don't use the idFactory until after deblend+merge, so that we aren't
397 # generating ids that just get thrown away (footprint merge doesn't
398 # know about past ids).
399 table = afwTable.SourceTable.make(self.schema)
400 results = self.detection.run(
401 table=table,
402 exposure=difference,
403 doSmooth=True,
404 )
405
406 sources, positives, negatives = self._deblend(difference,
407 results.positive,
408 results.negative)
409
410 return self.processResults(science, matchedTemplate, difference, sources, idFactory,
411 positiveFootprints=positives,
412 negativeFootprints=negatives)
413
414 def processResults(self, science, matchedTemplate, difference, sources, idFactory,
415 positiveFootprints=None, negativeFootprints=None,):
416 """Measure and process the results of source detection.
417
418 Parameters
419 ----------
420 science : `lsst.afw.image.ExposureF`
421 Science exposure that the template was subtracted from.
422 matchedTemplate : `lsst.afw.image.ExposureF`
423 Warped and PSF-matched template that was used produce the
424 difference image.
425 difference : `lsst.afw.image.ExposureF`
426 Result of subtracting template from the science image.
427 sources : `lsst.afw.table.SourceCatalog`
428 Detected sources on the difference exposure.
429 idFactory : `lsst.afw.table.IdFactory`
430 Generator object used to assign ids to detected sources in the
431 difference image.
432 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
433 Positive polarity footprints.
434 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
435 Negative polarity footprints.
436
437 Returns
438 -------
439 measurementResults : `lsst.pipe.base.Struct`
440
441 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
442 Subtracted exposure with detection mask applied.
443 ``diaSources`` : `lsst.afw.table.SourceCatalog`
444 The catalog of detected sources.
445 """
446 self.metadata.add("nUnmergedDiaSources", len(sources))
447 if self.config.doMerge:
448 fpSet = positiveFootprints
449 fpSet.merge(negativeFootprints, self.config.growFootprint,
450 self.config.growFootprint, False)
451 initialDiaSources = afwTable.SourceCatalog(self.schema)
452 fpSet.makeSources(initialDiaSources)
453 self.log.info("Merging detections into %d sources", len(initialDiaSources))
454 else:
455 initialDiaSources = sources
456
457 # Assign source ids at the end: deblend/merge mean that we don't keep
458 # track of parents and children, we only care about the final ids.
459 for source in initialDiaSources:
460 source.setId(idFactory())
461 # Ensure sources added after this get correct ids.
462 initialDiaSources.getTable().setIdFactory(idFactory)
463 initialDiaSources.setMetadata(self.algMetadata)
464
465 self.metadata.add("nMergedDiaSources", len(initialDiaSources))
466
467 if self.config.doSkySources:
468 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
469
470 if not initialDiaSources.isContiguous():
471 initialDiaSources = initialDiaSources.copy(deep=True)
472
473 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
474 diaSources = self._removeBadSources(initialDiaSources)
475
476 if self.config.doForcedMeasurement:
477 self.measureForcedSources(diaSources, science, difference.getWcs())
478
479 spatiallySampledMetrics = self.calculateMetrics(difference, diaSources, science, matchedTemplate,
480 idFactory)
481
482 measurementResults = pipeBase.Struct(
483 subtractedMeasuredExposure=difference,
484 diaSources=diaSources,
485 spatiallySampledMetrics=spatiallySampledMetrics,
486 )
487
488 return measurementResults
489
490 def _deblend(self, difference, positiveFootprints, negativeFootprints):
491 """Deblend the positive and negative footprints and return a catalog
492 containing just the children, and the deblended footprints.
493
494 Parameters
495 ----------
496 difference : `lsst.afw.image.Exposure`
497 Result of subtracting template from the science image.
498 positiveFootprints, negativeFootprints : `lsst.afw.detection.FootprintSet`
499 Positive and negative polarity footprints measured on
500 ``difference`` to be deblended separately.
501
502 Returns
503 -------
504 sources : `lsst.afw.table.SourceCatalog`
505 Positive and negative deblended children.
506 positives, negatives : `lsst.afw.detection.FootprintSet`
507 Deblended positive and negative polarity footprints measured on
508 ``difference``.
509 """
510 def makeFootprints(sources):
511 footprints = afwDetection.FootprintSet(difference.getBBox())
512 footprints.setFootprints([src.getFootprint() for src in sources])
513 return footprints
514
515 def deblend(footprints):
516 """Deblend a positive or negative footprint set,
517 and return the deblended children.
518 """
519 sources = afwTable.SourceCatalog(self.schema)
520 footprints.makeSources(sources)
521 self.deblend.run(exposure=difference, sources=sources)
522 self.setPrimaryFlags.run(sources)
523 children = sources["detect_isDeblendedSource"] == 1
524 sources = sources[children].copy(deep=True)
525 # Clear parents, so that measurement plugins behave correctly.
526 sources['parent'] = 0
527 return sources.copy(deep=True)
528
529 positives = deblend(positiveFootprints)
530 negatives = deblend(negativeFootprints)
531
532 sources = afwTable.SourceCatalog(self.schema)
533 sources.reserve(len(positives) + len(negatives))
534 sources.extend(positives, deep=True)
535 sources.extend(negatives, deep=True)
536 return sources, makeFootprints(positives), makeFootprints(negatives)
537
538 def _removeBadSources(self, diaSources):
539 """Remove unphysical diaSources from the catalog.
540
541 Parameters
542 ----------
543 diaSources : `lsst.afw.table.SourceCatalog`
544 The catalog of detected sources.
545
546 Returns
547 -------
548 diaSources : `lsst.afw.table.SourceCatalog`
549 The updated catalog of detected sources, with any source that has a
550 flag in ``config.badSourceFlags`` set removed.
551 """
552 selector = np.ones(len(diaSources), dtype=bool)
553 for flag in self.config.badSourceFlags:
554 flags = diaSources[flag]
555 nBad = np.count_nonzero(flags)
556 if nBad > 0:
557 self.log.debug("Found %d unphysical sources with flag %s.", nBad, flag)
558 selector &= ~flags
559 nBadTotal = np.count_nonzero(~selector)
560 self.metadata.add("nRemovedBadFlaggedSources", nBadTotal)
561 self.log.info("Removed %d unphysical sources.", nBadTotal)
562 return diaSources[selector].copy(deep=True)
563
564 def addSkySources(self, diaSources, mask, seed,
565 subtask=None):
566 """Add sources in empty regions of the difference image
567 for measuring the background.
568
569 Parameters
570 ----------
571 diaSources : `lsst.afw.table.SourceCatalog`
572 The catalog of detected sources.
573 mask : `lsst.afw.image.Mask`
574 Mask plane for determining regions where Sky sources can be added.
575 seed : `int`
576 Seed value to initialize the random number generator.
577 """
578 if subtask is None:
579 subtask = self.skySources
580 skySourceFootprints = subtask.run(mask=mask, seed=seed, catalog=diaSources)
581 self.metadata.add(f"n_{subtask.getName()}", len(skySourceFootprints))
582
583 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
584 """Use (matched) template and science image to constrain dipole fitting.
585
586 Parameters
587 ----------
588 diaSources : `lsst.afw.table.SourceCatalog`
589 The catalog of detected sources.
590 science : `lsst.afw.image.ExposureF`
591 Science exposure that the template was subtracted from.
592 difference : `lsst.afw.image.ExposureF`
593 Result of subtracting template from the science image.
594 matchedTemplate : `lsst.afw.image.ExposureF`
595 Warped and PSF-matched template that was used produce the
596 difference image.
597 """
598 # Ensure that the required mask planes are present
599 for mp in self.config.measurement.plugins["base_PixelFlags"].masksFpAnywhere:
600 difference.mask.addMaskPlane(mp)
601 # Note that this may not be correct if we convolved the science image.
602 # In the future we may wish to persist the matchedScience image.
603 self.measurement.run(diaSources, difference, science, matchedTemplate)
604 if self.config.doApCorr:
605 apCorrMap = difference.getInfo().getApCorrMap()
606 if apCorrMap is None:
607 self.log.warning("Difference image does not have valid aperture correction; skipping.")
608 else:
609 self.applyApCorr.run(
610 catalog=diaSources,
611 apCorrMap=apCorrMap,
612 )
613
614 def measureForcedSources(self, diaSources, science, wcs):
615 """Perform forced measurement of the diaSources on the science image.
616
617 Parameters
618 ----------
619 diaSources : `lsst.afw.table.SourceCatalog`
620 The catalog of detected sources.
621 science : `lsst.afw.image.ExposureF`
622 Science exposure that the template was subtracted from.
623 wcs : `lsst.afw.geom.SkyWcs`
624 Coordinate system definition (wcs) for the exposure.
625 """
626 # Run forced psf photometry on the PVI at the diaSource locations.
627 # Copy the measured flux and error into the diaSource.
628 forcedSources = self.forcedMeasurement.generateMeasCat(science, diaSources, wcs)
629 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
630 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
631 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
632 "ip_diffim_forced_PsfFlux_instFlux", True)
633 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
634 "ip_diffim_forced_PsfFlux_instFluxErr", True)
635 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
636 "ip_diffim_forced_PsfFlux_area", True)
637 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
638 "ip_diffim_forced_PsfFlux_flag", True)
639 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
640 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
641 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
642 "ip_diffim_forced_PsfFlux_flag_edge", True)
643 for diaSource, forcedSource in zip(diaSources, forcedSources):
644 diaSource.assign(forcedSource, mapper)
645
646 def calculateMetrics(self, difference, diaSources, science, matchedTemplate, idFactory):
647 """Add image QA metrics to the Task metadata.
648
649 Parameters
650 ----------
651 difference : `lsst.afw.image.Exposure`
652 The target image to calculate metrics for.
653 diaSources : `lsst.afw.table.SourceCatalog`
654 The catalog of detected sources.
655 science : `lsst.afw.image.Exposure`
656 The science image.
657 matchedTemplate : `lsst.afw.image.Exposure`
658 The reference image, warped and psf-matched to the science image.
659 idFactory : `lsst.afw.table.IdFactory`, optional
660 Generator object used to assign ids to detected sources in the
661 difference image.
662
663 Returns
664 -------
665 spatiallySampledMetrics : `lsst.afw.table.SourceCatalog`, or `None`
666 A catalog of randomized locations containing locally evaluated
667 metric results
668 """
669 mask = difference.mask
670 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
671 self.metadata.add("nGoodPixels", np.sum(~badPix))
672 self.metadata.add("nBadPixels", np.sum(badPix))
673 detPosPix = (mask.array & mask.getPlaneBitMask("DETECTED")) > 0
674 detNegPix = (mask.array & mask.getPlaneBitMask("DETECTED_NEGATIVE")) > 0
675 self.metadata.add("nPixelsDetectedPositive", np.sum(detPosPix))
676 self.metadata.add("nPixelsDetectedNegative", np.sum(detNegPix))
677 detPosPix &= badPix
678 detNegPix &= badPix
679 self.metadata.add("nBadPixelsDetectedPositive", np.sum(detPosPix))
680 self.metadata.add("nBadPixelsDetectedNegative", np.sum(detNegPix))
681 metricsMaskPlanes = []
682 for maskPlane in self.config.metricsMaskPlanes:
683 try:
684 self.metadata.add("%s_mask_fraction"%maskPlane.lower(), evaluateMaskFraction(mask, maskPlane))
685 metricsMaskPlanes.append(maskPlane)
686 except InvalidParameterError:
687 self.metadata.add("%s_mask_fraction"%maskPlane.lower(), -1)
688 self.log.info("Unable to calculate metrics for mask plane %s: not in image"%maskPlane)
689
690 if self.config.doWriteMetrics:
691 spatiallySampledMetrics = afwTable.SourceCatalog(self.metricSchema)
692 spatiallySampledMetrics.getTable().setIdFactory(idFactory)
693 self.addSkySources(spatiallySampledMetrics, science.mask, difference.info.id,
694 subtask=self.metricSources)
695 for src in spatiallySampledMetrics:
696 self._evaluateLocalMetric(src, diaSources, science, matchedTemplate, difference,
697 metricsMaskPlanes=metricsMaskPlanes)
698
699 return spatiallySampledMetrics.asAstropy()
700
701 def _evaluateLocalMetric(self, src, diaSources, science, matchedTemplate, difference,
702 metricsMaskPlanes):
703 """Calculate image quality metrics at spatially sampled locations.
704
705 Parameters
706 ----------
707 src : `lsst.afw.table.SourceRecord`
708 The source record to be updated with metric calculations.
709 diaSources : `lsst.afw.table.SourceCatalog`
710 The catalog of detected sources.
711 science : `lsst.afw.image.Exposure`
712 The science image.
713 matchedTemplate : `lsst.afw.image.Exposure`
714 The reference image, warped and psf-matched to the science image.
715 difference : `lsst.afw.image.Exposure`
716 Result of subtracting template from the science image.
717 metricsMaskPlanes : `list` of `str`
718 Mask planes to calculate metrics from.
719 """
720 bbox = src.getFootprint().getBBox()
721 pix = bbox.getCenter()
722 src.set('science_psfSize', getPsfFwhm(science.psf, position=pix))
723 src.set('template_psfSize', getPsfFwhm(matchedTemplate.psf, position=pix))
724
725 metricRegionSize = 100
726 bbox.grow(metricRegionSize)
727 bbox = bbox.clippedTo(science.getBBox())
728 nPix = bbox.getArea()
729 pixScale = science.wcs.getPixelScale()
730 area = nPix*pixScale.asDegrees()**2
731 peak = src.getFootprint().getPeaks()[0]
732 src.set('x', peak['i_x'])
733 src.set('y', peak['i_y'])
734 src.setCoord(science.wcs.pixelToSky(peak['i_x'], peak['i_y']))
735 selectSources = diaSources[bbox.contains(diaSources.getX(), diaSources.getY())]
736 if self.config.doSkySources:
737 selectSources = selectSources[~selectSources["sky_source"]]
738 sourceDensity = len(selectSources)/area
739 dipoleSources = selectSources[selectSources["ip_diffim_DipoleFit_flag_classification"]]
740 dipoleDensity = len(dipoleSources)/area
741 if dipoleSources:
742 meanDipoleOrientation = angleMean(dipoleSources["ip_diffim_DipoleFit_orientation"])
743 src.set('dipole_direction', meanDipoleOrientation)
744 meanDipoleSeparation = np.mean(dipoleSources["ip_diffim_DipoleFit_separation"])
745 src.set('dipole_separation', meanDipoleSeparation)
746 templateVal = np.median(matchedTemplate[bbox].image.array)
747 scienceVal = np.median(science[bbox].image.array)
748 diffimVal = np.median(difference[bbox].image.array)
749 src.set('source_density', sourceDensity)
750 src.set('dipole_density', dipoleDensity)
751 src.set('template_value', templateVal)
752 src.set('science_value', scienceVal)
753 src.set('diffim_value', diffimVal)
754 for maskPlane in metricsMaskPlanes:
755 src.set("%s_mask_fraction"%maskPlane.lower(),
756 evaluateMaskFraction(difference.mask[bbox], maskPlane)
757 )
758
759
761 scoreExposure = pipeBase.connectionTypes.Input(
762 doc="Maximum likelihood image for detection.",
763 dimensions=("instrument", "visit", "detector"),
764 storageClass="ExposureF",
765 name="{fakesType}{coaddName}Diff_scoreExp",
766 )
767
768
770 pipelineConnections=DetectAndMeasureScoreConnections):
771 pass
772
773
775 """Detect DIA sources using a score image,
776 and measure the detections on the difference image.
777
778 Source detection is run on the supplied score, or maximum likelihood,
779 image. Note that no additional convolution will be done in this case.
780 Close positive and negative detections will optionally be merged into
781 dipole diaSources.
782 Sky sources, or forced detections in background regions, will optionally
783 be added, and the configured measurement algorithm will be run on all
784 detections.
785 """
786 ConfigClass = DetectAndMeasureScoreConfig
787 _DefaultName = "detectAndMeasureScore"
788
789 @timeMethod
790 def run(self, science, matchedTemplate, difference, scoreExposure,
791 idFactory=None):
792 """Detect and measure sources on a score image.
793
794 Parameters
795 ----------
796 science : `lsst.afw.image.ExposureF`
797 Science exposure that the template was subtracted from.
798 matchedTemplate : `lsst.afw.image.ExposureF`
799 Warped and PSF-matched template that was used produce the
800 difference image.
801 difference : `lsst.afw.image.ExposureF`
802 Result of subtracting template from the science image.
803 scoreExposure : `lsst.afw.image.ExposureF`
804 Score or maximum likelihood difference image
805 idFactory : `lsst.afw.table.IdFactory`, optional
806 Generator object used to assign ids to detected sources in the
807 difference image. Ids from this generator are not set until after
808 deblending and merging positive/negative peaks.
809
810 Returns
811 -------
812 measurementResults : `lsst.pipe.base.Struct`
813
814 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
815 Subtracted exposure with detection mask applied.
816 ``diaSources`` : `lsst.afw.table.SourceCatalog`
817 The catalog of detected sources.
818 """
819 if idFactory is None:
820 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory()
821
822 # Ensure that we start with an empty detection mask.
823 mask = scoreExposure.mask
824 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
825
826 # Don't use the idFactory until after deblend+merge, so that we aren't
827 # generating ids that just get thrown away (footprint merge doesn't
828 # know about past ids).
829 table = afwTable.SourceTable.make(self.schema)
830 results = self.detection.run(
831 table=table,
832 exposure=scoreExposure,
833 doSmooth=False,
834 )
835 # Copy the detection mask from the Score image to the difference image
836 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
837
838 sources, positives, negatives = self._deblend(difference,
839 results.positive,
840 results.negative)
841
842 return self.processResults(science, matchedTemplate, difference, sources, idFactory,
843 positiveFootprints=positives, negativeFootprints=negatives)
844
845
846def evaluateMaskFraction(mask, maskPlane):
847 nMaskSet = np.count_nonzero((mask.array & mask.getPlaneBitMask(maskPlane)))
848 return nMaskSet/mask.array.size
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)