27from lsst.meas.algorithms
import SkyObjectsTask, SourceDetectionTask, SetPrimaryFlagsTask
28from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
29import lsst.meas.deblender
30import lsst.meas.extensions.trailedSources
31import lsst.meas.extensions.shapeHSM
35from lsst.utils.timer
import timeMethod
37from .
import DipoleFitTask
39__all__ = [
"DetectAndMeasureConfig",
"DetectAndMeasureTask",
40 "DetectAndMeasureScoreConfig",
"DetectAndMeasureScoreTask"]
44 dimensions=(
"instrument",
"visit",
"detector"),
45 defaultTemplates={
"coaddName":
"deep",
48 science = pipeBase.connectionTypes.Input(
49 doc=
"Input science exposure.",
50 dimensions=(
"instrument",
"visit",
"detector"),
51 storageClass=
"ExposureF",
52 name=
"{fakesType}calexp"
54 matchedTemplate = pipeBase.connectionTypes.Input(
55 doc=
"Warped and PSF-matched template used to create the difference image.",
56 dimensions=(
"instrument",
"visit",
"detector"),
57 storageClass=
"ExposureF",
58 name=
"{fakesType}{coaddName}Diff_matchedExp",
60 difference = pipeBase.connectionTypes.Input(
61 doc=
"Result of subtracting template from science.",
62 dimensions=(
"instrument",
"visit",
"detector"),
63 storageClass=
"ExposureF",
64 name=
"{fakesType}{coaddName}Diff_differenceTempExp",
66 outputSchema = pipeBase.connectionTypes.InitOutput(
67 doc=
"Schema (as an example catalog) for output DIASource catalog.",
68 storageClass=
"SourceCatalog",
69 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
71 diaSources = pipeBase.connectionTypes.Output(
72 doc=
"Detected diaSources on the difference image.",
73 dimensions=(
"instrument",
"visit",
"detector"),
74 storageClass=
"SourceCatalog",
75 name=
"{fakesType}{coaddName}Diff_diaSrc",
77 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
78 doc=
"Difference image with detection mask plane filled in.",
79 dimensions=(
"instrument",
"visit",
"detector"),
80 storageClass=
"ExposureF",
81 name=
"{fakesType}{coaddName}Diff_differenceExp",
86 pipelineConnections=DetectAndMeasureConnections):
87 """Config for DetectAndMeasureTask
89 doMerge = pexConfig.Field(
92 doc=
"Merge positive and negative diaSources with grow radius "
93 "set by growFootprint"
95 doForcedMeasurement = pexConfig.Field(
98 doc=
"Force photometer diaSource locations on PVI?")
99 doAddMetrics = pexConfig.Field(
102 doc=
"Add columns to the source table to hold analysis metrics?"
104 detection = pexConfig.ConfigurableField(
105 target=SourceDetectionTask,
106 doc=
"Final source detection for diaSource measurement",
108 deblend = pexConfig.ConfigurableField(
109 target=lsst.meas.deblender.SourceDeblendTask,
110 doc=
"Task to split blended sources into their components."
112 measurement = pexConfig.ConfigurableField(
113 target=DipoleFitTask,
114 doc=
"Task to measure sources on the difference image.",
116 doApCorr = lsst.pex.config.Field(
119 doc=
"Run subtask to apply aperture corrections"
121 applyApCorr = lsst.pex.config.ConfigurableField(
122 target=ApplyApCorrTask,
123 doc=
"Task to apply aperture corrections"
125 forcedMeasurement = pexConfig.ConfigurableField(
126 target=ForcedMeasurementTask,
127 doc=
"Task to force photometer science image at diaSource locations.",
129 growFootprint = pexConfig.Field(
132 doc=
"Grow positive and negative footprints by this many pixels before merging"
134 diaSourceMatchRadius = pexConfig.Field(
137 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
139 doSkySources = pexConfig.Field(
142 doc=
"Generate sky sources?",
144 skySources = pexConfig.ConfigurableField(
145 target=SkyObjectsTask,
146 doc=
"Generate sky sources",
148 setPrimaryFlags = pexConfig.ConfigurableField(
149 target=SetPrimaryFlagsTask,
150 doc=
"Task to add isPrimary and deblending-related flags to the catalog."
152 badSourceFlags = lsst.pex.config.ListField(
154 doc=
"Sources with any of these flags set are removed before writing the output catalog.",
155 default=(
"base_PixelFlags_flag_offimage",
156 "base_PixelFlags_flag_interpolatedCenterAll",
157 "base_PixelFlags_flag_badCenterAll",
158 "base_PixelFlags_flag_edgeCenterAll",
161 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
163 def setDefaults(self):
165 self.detection.thresholdPolarity =
"both"
166 self.detection.thresholdValue = 5.0
167 self.detection.reEstimateBackground =
False
168 self.detection.thresholdType =
"pixel_stdev"
169 self.detection.excludeMaskPlanes = [
"EDGE"]
172 self.measurement.algorithms.names.add(
"base_PeakLikelihoodFlux")
173 self.measurement.plugins.names |= [
"ext_trailedSources_Naive",
174 "base_LocalPhotoCalib",
176 "ext_shapeHSM_HsmSourceMoments",
177 "ext_shapeHSM_HsmPsfMoments",
179 self.measurement.slots.psfShape =
"ext_shapeHSM_HsmPsfMoments"
180 self.measurement.slots.shape =
"ext_shapeHSM_HsmSourceMoments"
181 self.measurement.plugins[
"base_NaiveCentroid"].maxDistToPeak = 5.0
182 self.measurement.plugins[
"base_SdssCentroid"].maxDistToPeak = 5.0
183 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
184 self.forcedMeasurement.copyColumns = {
185 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
186 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
187 self.forcedMeasurement.slots.shape =
None
190 self.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere = [
191 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
192 self.measurement.plugins[
"base_PixelFlags"].masksFpCenter = [
193 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
194 self.skySources.avoidMask = [
"DETECTED",
"DETECTED_NEGATIVE",
"BAD",
"NO_DATA",
"EDGE"]
198 """Detect and measure sources on a difference image.
200 ConfigClass = DetectAndMeasureConfig
201 _DefaultName =
"detectAndMeasure"
203 def __init__(self, **kwargs):
204 super().__init__(**kwargs)
205 self.schema = afwTable.SourceTable.makeMinimalSchema()
207 afwTable.CoordKey.addErrorFields(self.schema)
210 self.makeSubtask(
"detection", schema=self.schema)
211 self.makeSubtask(
"deblend", schema=self.schema)
212 self.makeSubtask(
"setPrimaryFlags", schema=self.schema, isSingleFrame=
True)
213 self.makeSubtask(
"measurement", schema=self.schema,
214 algMetadata=self.algMetadata)
215 if self.config.doApCorr:
216 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
217 if self.config.doForcedMeasurement:
218 self.schema.addField(
219 "ip_diffim_forced_PsfFlux_instFlux",
"D",
220 "Forced PSF flux measured on the direct image.",
222 self.schema.addField(
223 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
224 "Forced PSF flux error measured on the direct image.",
226 self.schema.addField(
227 "ip_diffim_forced_PsfFlux_area",
"F",
228 "Forced PSF flux effective area of PSF.",
230 self.schema.addField(
231 "ip_diffim_forced_PsfFlux_flag",
"Flag",
232 "Forced PSF flux general failure flag.")
233 self.schema.addField(
234 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
235 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
236 self.schema.addField(
237 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
238 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
239 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
241 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
242 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
243 if self.config.doSkySources:
244 self.makeSubtask(
"skySources", schema=self.schema)
247 for flag
in self.config.badSourceFlags:
248 if flag
not in self.schema:
249 raise pipeBase.InvalidQuantumError(
"Field %s not in schema" % flag)
251 self.outputSchema = afwTable.SourceCatalog(self.schema)
252 self.outputSchema.getTable().setMetadata(self.algMetadata)
254 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
255 inputRefs: pipeBase.InputQuantizedConnection,
256 outputRefs: pipeBase.OutputQuantizedConnection):
257 inputs = butlerQC.get(inputRefs)
258 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
259 idFactory = idGenerator.make_table_id_factory()
260 outputs = self.run(**inputs, idFactory=idFactory)
261 butlerQC.put(outputs, outputRefs)
264 def run(self, science, matchedTemplate, difference,
266 """Detect and measure sources on a difference image.
268 The difference image will be convolved with a gaussian approximation of
269 the PSF to form a maximum likelihood image for detection.
270 Close positive and negative detections will optionally be merged into
272 Sky sources, or forced detections in background regions, will optionally
273 be added, and the configured measurement algorithm will be run on all
278 science : `lsst.afw.image.ExposureF`
279 Science exposure that the template was subtracted from.
280 matchedTemplate : `lsst.afw.image.ExposureF`
281 Warped and PSF-matched template that was used produce the
283 difference : `lsst.afw.image.ExposureF`
284 Result of subtracting template from the science image.
285 idFactory : `lsst.afw.table.IdFactory`, optional
286 Generator object used to assign ids to detected sources in the
287 difference image. Ids from this generator are not set until after
288 deblending and merging positive/negative peaks.
292 measurementResults : `lsst.pipe.base.Struct`
294 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
295 Subtracted exposure with detection mask applied.
296 ``diaSources`` : `lsst.afw.table.SourceCatalog`
297 The catalog of detected sources.
299 if idFactory
is None:
300 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory()
303 mask = difference.mask
304 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
309 table = afwTable.SourceTable.make(self.schema)
310 results = self.detection.
run(
316 sources, positives, negatives = self._deblend(difference,
320 return self.processResults(science, matchedTemplate, difference, sources, idFactory,
321 positiveFootprints=positives,
322 negativeFootprints=negatives)
324 def processResults(self, science, matchedTemplate, difference, sources, idFactory,
325 positiveFootprints=None, negativeFootprints=None,):
326 """Measure and process the results of source detection.
330 science : `lsst.afw.image.ExposureF`
331 Science exposure that the template was subtracted from.
332 matchedTemplate : `lsst.afw.image.ExposureF`
333 Warped and PSF-matched template that was used produce the
335 difference : `lsst.afw.image.ExposureF`
336 Result of subtracting template from the science image.
337 sources : `lsst.afw.table.SourceCatalog`
338 Detected sources on the difference exposure.
339 idFactory : `lsst.afw.table.IdFactory`
340 Generator object used to assign ids to detected sources in the
342 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
343 Positive polarity footprints.
344 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
345 Negative polarity footprints.
349 measurementResults : `lsst.pipe.base.Struct`
351 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
352 Subtracted exposure with detection mask applied.
353 ``diaSources`` : `lsst.afw.table.SourceCatalog`
354 The catalog of detected sources.
356 self.metadata.add(
"nUnmergedDiaSources",
len(sources))
357 if self.config.doMerge:
358 fpSet = positiveFootprints
359 fpSet.merge(negativeFootprints, self.config.growFootprint,
360 self.config.growFootprint,
False)
361 initialDiaSources = afwTable.SourceCatalog(self.schema)
362 fpSet.makeSources(initialDiaSources)
363 self.log.
info(
"Merging detections into %d sources",
len(initialDiaSources))
365 initialDiaSources = sources
369 for source
in initialDiaSources:
372 initialDiaSources.getTable().setIdFactory(idFactory)
373 initialDiaSources.setMetadata(self.algMetadata)
375 self.metadata.add(
"nMergedDiaSources",
len(initialDiaSources))
377 if self.config.doSkySources:
378 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
380 if not initialDiaSources.isContiguous():
381 initialDiaSources = initialDiaSources.copy(deep=
True)
383 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
384 diaSources = self._removeBadSources(initialDiaSources)
386 if self.config.doForcedMeasurement:
387 self.measureForcedSources(diaSources, science, difference.getWcs())
389 measurementResults = pipeBase.Struct(
390 subtractedMeasuredExposure=difference,
391 diaSources=diaSources,
393 self.calculateMetrics(difference)
395 return measurementResults
397 def _deblend(self, difference, positiveFootprints, negativeFootprints):
398 """Deblend the positive and negative footprints and return a catalog
399 containing just the children, and the deblended footprints.
403 difference : `lsst.afw.image.Exposure`
404 Result of subtracting template from the science image.
405 positiveFootprints, negativeFootprints : `lsst.afw.detection.FootprintSet`
406 Positive and negative polarity footprints measured on
407 ``difference`` to be deblended separately.
411 sources : `lsst.afw.table.SourceCatalog`
412 Positive and negative deblended children.
413 positives, negatives : `lsst.afw.detection.FootprintSet`
414 Deblended positive and negative polarity footprints measured on
418 footprints = afwDetection.FootprintSet(difference.getBBox())
419 footprints.setFootprints([src.getFootprint()
for src
in sources])
423 """Deblend a positive or negative footprint set,
424 and return the deblended children.
426 sources = afwTable.SourceCatalog(self.schema)
427 footprints.makeSources(sources)
428 self.deblend.
run(exposure=difference, sources=sources)
429 self.setPrimaryFlags.
run(sources)
430 children = sources[
"detect_isDeblendedSource"] == 1
431 sources = sources[children].copy(deep=
True)
433 sources[
'parent'] = 0
434 return sources.copy(deep=
True)
436 positives =
deblend(positiveFootprints)
437 negatives =
deblend(negativeFootprints)
439 sources = afwTable.SourceCatalog(self.schema)
440 sources.reserve(
len(positives) +
len(negatives))
441 sources.extend(positives, deep=
True)
442 sources.extend(negatives, deep=
True)
446 """Remove bad diaSources from the catalog.
450 diaSources : `lsst.afw.table.SourceCatalog`
451 The catalog of detected sources.
455 diaSources : `lsst.afw.table.SourceCatalog`
456 The updated catalog of detected sources, with any source that has a
457 flag in ``config.badSourceFlags`` set removed.
460 selector = np.ones(
len(diaSources), dtype=bool)
461 for flag
in self.config.badSourceFlags:
462 flags = diaSources[flag]
463 nBad = np.count_nonzero(flags)
465 self.log.
info(
"Found and removed %d unphysical sources with flag %s.", nBad, flag)
468 self.metadata.add(
"nRemovedBadFlaggedSources", nBadTotal)
469 return diaSources[selector].copy(deep=
True)
472 """Add sources in empty regions of the difference image
473 for measuring the background.
477 diaSources : `lsst.afw.table.SourceCatalog`
478 The catalog of detected sources.
479 mask : `lsst.afw.image.Mask`
480 Mask plane for determining regions where Sky sources can be added.
482 Seed value to initialize the random number generator.
484 skySourceFootprints = self.skySources.
run(mask=mask, seed=seed, catalog=diaSources)
485 self.metadata.add(
"nSkySources",
len(skySourceFootprints))
488 """Use (matched) template and science image to constrain dipole fitting.
492 diaSources : `lsst.afw.table.SourceCatalog`
493 The catalog of detected sources.
494 science : `lsst.afw.image.ExposureF`
495 Science exposure that the template was subtracted from.
496 difference : `lsst.afw.image.ExposureF`
497 Result of subtracting template from the science image.
498 matchedTemplate : `lsst.afw.image.ExposureF`
499 Warped and PSF-matched template that was used produce the
503 for mp
in self.config.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere:
504 difference.mask.addMaskPlane(mp)
507 self.measurement.
run(diaSources, difference, science, matchedTemplate)
508 if self.config.doApCorr:
509 apCorrMap = difference.getInfo().getApCorrMap()
510 if apCorrMap
is None:
511 self.log.
warning(
"Difference image does not have valid aperture correction; skipping.")
513 self.applyApCorr.
run(
519 """Perform forced measurement of the diaSources on the science image.
523 diaSources : `lsst.afw.table.SourceCatalog`
524 The catalog of detected sources.
525 science : `lsst.afw.image.ExposureF`
526 Science exposure that the template was subtracted from.
527 wcs : `lsst.afw.geom.SkyWcs`
528 Coordinate system definition (wcs) for the exposure.
532 forcedSources = self.forcedMeasurement.generateMeasCat(science, diaSources, wcs)
533 self.forcedMeasurement.
run(forcedSources, science, diaSources, wcs)
534 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
535 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
536 "ip_diffim_forced_PsfFlux_instFlux",
True)
537 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
538 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
539 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
540 "ip_diffim_forced_PsfFlux_area",
True)
541 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
542 "ip_diffim_forced_PsfFlux_flag",
True)
543 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
544 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
545 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
546 "ip_diffim_forced_PsfFlux_flag_edge",
True)
547 for diaSource, forcedSource
in zip(diaSources, forcedSources):
548 diaSource.assign(forcedSource, mapper)
551 """Add image QA metrics to the Task metadata.
555 difference : `lsst.afw.image.Exposure`
556 The target image to calculate metrics for.
558 mask = difference.mask
559 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
560 self.metadata.add(
"nGoodPixels", np.sum(~badPix))
561 self.metadata.add(
"nBadPixels", np.sum(badPix))
562 detPosPix = (mask.array & mask.getPlaneBitMask(
"DETECTED")) > 0
563 detNegPix = (mask.array & mask.getPlaneBitMask(
"DETECTED_NEGATIVE")) > 0
564 self.metadata.add(
"nPixelsDetectedPositive", np.sum(detPosPix))
565 self.metadata.add(
"nPixelsDetectedNegative", np.sum(detNegPix))
568 self.metadata.add(
"nBadPixelsDetectedPositive", np.sum(detPosPix))
569 self.metadata.add(
"nBadPixelsDetectedNegative", np.sum(detNegPix))
573 scoreExposure = pipeBase.connectionTypes.Input(
574 doc=
"Maximum likelihood image for detection.",
575 dimensions=(
"instrument",
"visit",
"detector"),
576 storageClass=
"ExposureF",
577 name=
"{fakesType}{coaddName}Diff_scoreExp",
582 pipelineConnections=DetectAndMeasureScoreConnections):
587 """Detect DIA sources using a score image,
588 and measure the detections on the difference image.
590 Source detection is run on the supplied score, or maximum likelihood,
591 image. Note that no additional convolution will be done in this case.
592 Close positive and negative detections will optionally be merged into
594 Sky sources, or forced detections in background regions, will optionally
595 be added, and the configured measurement algorithm will be run on all
598 ConfigClass = DetectAndMeasureScoreConfig
599 _DefaultName =
"detectAndMeasureScore"
602 def run(self, science, matchedTemplate, difference, scoreExposure,
604 """Detect and measure sources on a score image.
608 science : `lsst.afw.image.ExposureF`
609 Science exposure that the template was subtracted from.
610 matchedTemplate : `lsst.afw.image.ExposureF`
611 Warped and PSF-matched template that was used produce the
613 difference : `lsst.afw.image.ExposureF`
614 Result of subtracting template from the science image.
615 scoreExposure : `lsst.afw.image.ExposureF`
616 Score or maximum likelihood difference image
617 idFactory : `lsst.afw.table.IdFactory`, optional
618 Generator object used to assign ids to detected sources in the
619 difference image. Ids from this generator are not set until after
620 deblending and merging positive/negative peaks.
624 measurementResults : `lsst.pipe.base.Struct`
626 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
627 Subtracted exposure with detection mask applied.
628 ``diaSources`` : `lsst.afw.table.SourceCatalog`
629 The catalog of detected sources.
631 if idFactory
is None:
632 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory()
635 mask = scoreExposure.mask
636 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
641 table = afwTable.SourceTable.make(self.schema)
642 results = self.detection.
run(
644 exposure=scoreExposure,
648 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
650 sources, positives, negatives = self._deblend(difference,
654 return self.processResults(science, matchedTemplate, difference, sources, idFactory,
655 positiveFootprints=positives, negativeFootprints=negatives)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)