27from lsst.meas.algorithms
import SkyObjectsTask, SourceDetectionTask, SetPrimaryFlagsTask
28from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
29import lsst.meas.deblender
30import lsst.meas.extensions.trailedSources
31import lsst.meas.extensions.shapeHSM
35from lsst.utils.timer
import timeMethod
37from .
import DipoleFitTask
39__all__ = [
"DetectAndMeasureConfig",
"DetectAndMeasureTask",
40 "DetectAndMeasureScoreConfig",
"DetectAndMeasureScoreTask"]
44 dimensions=(
"instrument",
"visit",
"detector"),
45 defaultTemplates={
"coaddName":
"deep",
48 science = pipeBase.connectionTypes.Input(
49 doc=
"Input science exposure.",
50 dimensions=(
"instrument",
"visit",
"detector"),
51 storageClass=
"ExposureF",
52 name=
"{fakesType}calexp"
54 matchedTemplate = pipeBase.connectionTypes.Input(
55 doc=
"Warped and PSF-matched template used to create the difference image.",
56 dimensions=(
"instrument",
"visit",
"detector"),
57 storageClass=
"ExposureF",
58 name=
"{fakesType}{coaddName}Diff_matchedExp",
60 difference = pipeBase.connectionTypes.Input(
61 doc=
"Result of subtracting template from science.",
62 dimensions=(
"instrument",
"visit",
"detector"),
63 storageClass=
"ExposureF",
64 name=
"{fakesType}{coaddName}Diff_differenceTempExp",
66 outputSchema = pipeBase.connectionTypes.InitOutput(
67 doc=
"Schema (as an example catalog) for output DIASource catalog.",
68 storageClass=
"SourceCatalog",
69 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
71 diaSources = pipeBase.connectionTypes.Output(
72 doc=
"Detected diaSources on the difference image.",
73 dimensions=(
"instrument",
"visit",
"detector"),
74 storageClass=
"SourceCatalog",
75 name=
"{fakesType}{coaddName}Diff_diaSrc",
77 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
78 doc=
"Difference image with detection mask plane filled in.",
79 dimensions=(
"instrument",
"visit",
"detector"),
80 storageClass=
"ExposureF",
81 name=
"{fakesType}{coaddName}Diff_differenceExp",
86 pipelineConnections=DetectAndMeasureConnections):
87 """Config for DetectAndMeasureTask
89 doMerge = pexConfig.Field(
92 doc=
"Merge positive and negative diaSources with grow radius "
93 "set by growFootprint"
95 doForcedMeasurement = pexConfig.Field(
98 doc=
"Force photometer diaSource locations on PVI?")
99 doAddMetrics = pexConfig.Field(
102 doc=
"Add columns to the source table to hold analysis metrics?"
104 detection = pexConfig.ConfigurableField(
105 target=SourceDetectionTask,
106 doc=
"Final source detection for diaSource measurement",
108 deblend = pexConfig.ConfigurableField(
109 target=lsst.meas.deblender.SourceDeblendTask,
110 doc=
"Task to split blended sources into their components."
112 measurement = pexConfig.ConfigurableField(
113 target=DipoleFitTask,
114 doc=
"Task to measure sources on the difference image.",
116 doApCorr = lsst.pex.config.Field(
119 doc=
"Run subtask to apply aperture corrections"
121 applyApCorr = lsst.pex.config.ConfigurableField(
122 target=ApplyApCorrTask,
123 doc=
"Task to apply aperture corrections"
125 forcedMeasurement = pexConfig.ConfigurableField(
126 target=ForcedMeasurementTask,
127 doc=
"Task to force photometer science image at diaSource locations.",
129 growFootprint = pexConfig.Field(
132 doc=
"Grow positive and negative footprints by this many pixels before merging"
134 diaSourceMatchRadius = pexConfig.Field(
137 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
139 doSkySources = pexConfig.Field(
142 doc=
"Generate sky sources?",
144 skySources = pexConfig.ConfigurableField(
145 target=SkyObjectsTask,
146 doc=
"Generate sky sources",
148 setPrimaryFlags = pexConfig.ConfigurableField(
149 target=SetPrimaryFlagsTask,
150 doc=
"Task to add isPrimary and deblending-related flags to the catalog."
152 badSourceFlags = lsst.pex.config.ListField(
154 doc=
"Sources with any of these flags set are removed before writing the output catalog.",
155 default=(
"base_PixelFlags_flag_offimage",
156 "base_PixelFlags_flag_interpolatedCenterAll",
157 "base_PixelFlags_flag_badCenterAll",
158 "base_PixelFlags_flag_edgeCenterAll",
161 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
163 def setDefaults(self):
165 self.detection.thresholdPolarity =
"both"
166 self.detection.thresholdValue = 5.0
167 self.detection.reEstimateBackground =
False
168 self.detection.thresholdType =
"pixel_stdev"
169 self.detection.excludeMaskPlanes = [
"EDGE"]
172 self.measurement.algorithms.names.add(
"base_PeakLikelihoodFlux")
173 self.measurement.plugins.names |= [
"ext_trailedSources_Naive",
174 "base_LocalPhotoCalib",
176 "ext_shapeHSM_HsmSourceMoments",
177 "ext_shapeHSM_HsmPsfMoments",
179 self.measurement.slots.psfShape =
"ext_shapeHSM_HsmPsfMoments"
180 self.measurement.slots.shape =
"ext_shapeHSM_HsmSourceMoments"
181 self.measurement.plugins[
"base_NaiveCentroid"].maxDistToPeak = 5.0
182 self.measurement.plugins[
"base_SdssCentroid"].maxDistToPeak = 5.0
183 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
184 self.forcedMeasurement.copyColumns = {
185 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
186 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
187 self.forcedMeasurement.slots.shape =
None
190 self.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere = [
191 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
192 self.measurement.plugins[
"base_PixelFlags"].masksFpCenter = [
193 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
194 self.skySources.avoidMask = [
"DETECTED",
"DETECTED_NEGATIVE",
"BAD",
"NO_DATA",
"EDGE"]
198 """Detect and measure sources on a difference image.
200 ConfigClass = DetectAndMeasureConfig
201 _DefaultName =
"detectAndMeasure"
203 def __init__(self, **kwargs):
204 super().__init__(**kwargs)
205 self.schema = afwTable.SourceTable.makeMinimalSchema()
207 afwTable.CoordKey.addErrorFields(self.schema)
210 self.makeSubtask(
"detection", schema=self.schema)
211 self.makeSubtask(
"deblend", schema=self.schema)
212 self.makeSubtask(
"setPrimaryFlags", schema=self.schema, isSingleFrame=
True)
213 self.makeSubtask(
"measurement", schema=self.schema,
214 algMetadata=self.algMetadata)
215 if self.config.doApCorr:
216 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
217 if self.config.doForcedMeasurement:
218 self.schema.addField(
219 "ip_diffim_forced_PsfFlux_instFlux",
"D",
220 "Forced PSF flux measured on the direct image.",
222 self.schema.addField(
223 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
224 "Forced PSF flux error measured on the direct image.",
226 self.schema.addField(
227 "ip_diffim_forced_PsfFlux_area",
"F",
228 "Forced PSF flux effective area of PSF.",
230 self.schema.addField(
231 "ip_diffim_forced_PsfFlux_flag",
"Flag",
232 "Forced PSF flux general failure flag.")
233 self.schema.addField(
234 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
235 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
236 self.schema.addField(
237 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
238 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
239 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
241 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
242 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
243 if self.config.doSkySources:
244 self.makeSubtask(
"skySources", schema=self.schema)
247 for flag
in self.config.badSourceFlags:
248 if flag
not in self.schema:
249 raise pipeBase.InvalidQuantumError(
"Field %s not in schema" % flag)
251 self.outputSchema = afwTable.SourceCatalog(self.schema)
252 self.outputSchema.getTable().setMetadata(self.algMetadata)
254 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
255 inputRefs: pipeBase.InputQuantizedConnection,
256 outputRefs: pipeBase.OutputQuantizedConnection):
257 inputs = butlerQC.get(inputRefs)
258 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
259 idFactory = idGenerator.make_table_id_factory()
260 outputs = self.run(**inputs, idFactory=idFactory)
261 butlerQC.put(outputs, outputRefs)
264 def run(self, science, matchedTemplate, difference,
266 """Detect and measure sources on a difference image.
268 The difference image will be convolved with a gaussian approximation of
269 the PSF to form a maximum likelihood image for detection.
270 Close positive and negative detections will optionally be merged into
272 Sky sources, or forced detections in background regions, will optionally
273 be added, and the configured measurement algorithm will be run on all
278 science : `lsst.afw.image.ExposureF`
279 Science exposure that the template was subtracted from.
280 matchedTemplate : `lsst.afw.image.ExposureF`
281 Warped and PSF-matched template that was used produce the
283 difference : `lsst.afw.image.ExposureF`
284 Result of subtracting template from the science image.
285 idFactory : `lsst.afw.table.IdFactory`, optional
286 Generator object used to assign ids to detected sources in the
287 difference image. Ids from this generator are not set until after
288 deblending and merging positive/negative peaks.
292 measurementResults : `lsst.pipe.base.Struct`
294 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
295 Subtracted exposure with detection mask applied.
296 ``diaSources`` : `lsst.afw.table.SourceCatalog`
297 The catalog of detected sources.
299 if idFactory
is None:
300 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory()
303 mask = difference.mask
304 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
309 table = afwTable.SourceTable.make(self.schema)
310 results = self.detection.
run(
316 sources, positives, negatives = self._deblend(difference,
320 return self.processResults(science, matchedTemplate, difference, sources, idFactory,
321 positiveFootprints=positives,
322 negativeFootprints=negatives)
324 def processResults(self, science, matchedTemplate, difference, sources, idFactory,
325 positiveFootprints=None, negativeFootprints=None,):
326 """Measure and process the results of source detection.
330 science : `lsst.afw.image.ExposureF`
331 Science exposure that the template was subtracted from.
332 matchedTemplate : `lsst.afw.image.ExposureF`
333 Warped and PSF-matched template that was used produce the
335 difference : `lsst.afw.image.ExposureF`
336 Result of subtracting template from the science image.
337 sources : `lsst.afw.table.SourceCatalog`
338 Detected sources on the difference exposure.
339 idFactory : `lsst.afw.table.IdFactory`
340 Generator object used to assign ids to detected sources in the
342 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
343 Positive polarity footprints.
344 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
345 Negative polarity footprints.
349 measurementResults : `lsst.pipe.base.Struct`
351 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
352 Subtracted exposure with detection mask applied.
353 ``diaSources`` : `lsst.afw.table.SourceCatalog`
354 The catalog of detected sources.
356 self.metadata.add(
"nUnmergedDiaSources",
len(sources))
357 if self.config.doMerge:
358 fpSet = positiveFootprints
359 fpSet.merge(negativeFootprints, self.config.growFootprint,
360 self.config.growFootprint,
False)
361 initialDiaSources = afwTable.SourceCatalog(self.schema)
362 fpSet.makeSources(initialDiaSources)
363 self.log.
info(
"Merging detections into %d sources",
len(initialDiaSources))
365 initialDiaSources = sources
369 for source
in initialDiaSources:
372 initialDiaSources.getTable().setIdFactory(idFactory)
373 initialDiaSources.setMetadata(self.algMetadata)
375 self.metadata.add(
"nMergedDiaSources",
len(initialDiaSources))
377 if self.config.doSkySources:
378 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
380 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
381 diaSources = self._removeBadSources(initialDiaSources)
383 if self.config.doForcedMeasurement:
384 self.measureForcedSources(diaSources, science, difference.getWcs())
386 measurementResults = pipeBase.Struct(
387 subtractedMeasuredExposure=difference,
388 diaSources=diaSources,
390 self.calculateMetrics(difference)
392 return measurementResults
394 def _deblend(self, difference, positiveFootprints, negativeFootprints):
395 """Deblend the positive and negative footprints and return a catalog
396 containing just the children, and the deblended footprints.
400 difference : `lsst.afw.image.Exposure`
401 Result of subtracting template from the science image.
402 positiveFootprints, negativeFootprints : `lsst.afw.detection.FootprintSet`
403 Positive and negative polarity footprints measured on
404 ``difference`` to be deblended separately.
408 sources : `lsst.afw.table.SourceCatalog`
409 Positive and negative deblended children.
410 positives, negatives : `lsst.afw.detection.FootprintSet`
411 Deblended positive and negative polarity footprints measured on
415 footprints = afwDetection.FootprintSet(difference.getBBox())
416 footprints.setFootprints([src.getFootprint()
for src
in sources])
420 """Deblend a positive or negative footprint set,
421 and return the deblended children.
423 sources = afwTable.SourceCatalog(self.schema)
424 footprints.makeSources(sources)
425 self.deblend.
run(exposure=difference, sources=sources)
426 self.setPrimaryFlags.
run(sources)
427 children = sources[
"detect_isDeblendedSource"] == 1
428 sources = sources[children].copy(deep=
True)
430 sources[
'parent'] = 0
431 return sources.copy(deep=
True)
433 positives =
deblend(positiveFootprints)
434 negatives =
deblend(negativeFootprints)
436 sources = afwTable.SourceCatalog(self.schema)
437 sources.reserve(
len(positives) +
len(negatives))
438 sources.extend(positives, deep=
True)
439 sources.extend(negatives, deep=
True)
443 """Remove bad diaSources from the catalog.
447 diaSources : `lsst.afw.table.SourceCatalog`
448 The catalog of detected sources.
452 diaSources : `lsst.afw.table.SourceCatalog`
453 The updated catalog of detected sources, with any source that has a
454 flag in ``config.badSourceFlags`` set removed.
457 selector = np.ones(
len(diaSources), dtype=bool)
458 for flag
in self.config.badSourceFlags:
459 flags = diaSources[flag]
460 nBad = np.count_nonzero(flags)
462 self.log.
info(
"Found and removed %d unphysical sources with flag %s.", nBad, flag)
465 self.metadata.add(
"nRemovedBadFlaggedSources", nBadTotal)
466 return diaSources[selector].copy(deep=
True)
469 """Add sources in empty regions of the difference image
470 for measuring the background.
474 diaSources : `lsst.afw.table.SourceCatalog`
475 The catalog of detected sources.
476 mask : `lsst.afw.image.Mask`
477 Mask plane for determining regions where Sky sources can be added.
479 Seed value to initialize the random number generator.
481 skySourceFootprints = self.skySources.
run(mask=mask, seed=seed, catalog=diaSources)
482 self.metadata.add(
"nSkySources",
len(skySourceFootprints))
485 """Use (matched) template and science image to constrain dipole fitting.
489 diaSources : `lsst.afw.table.SourceCatalog`
490 The catalog of detected sources.
491 science : `lsst.afw.image.ExposureF`
492 Science exposure that the template was subtracted from.
493 difference : `lsst.afw.image.ExposureF`
494 Result of subtracting template from the science image.
495 matchedTemplate : `lsst.afw.image.ExposureF`
496 Warped and PSF-matched template that was used produce the
500 for mp
in self.config.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere:
501 difference.mask.addMaskPlane(mp)
504 self.measurement.
run(diaSources, difference, science, matchedTemplate)
505 if self.config.doApCorr:
506 apCorrMap = difference.getInfo().getApCorrMap()
507 if apCorrMap
is None:
508 self.log.
warning(
"Difference image does not have valid aperture correction; skipping.")
510 self.applyApCorr.
run(
516 """Perform forced measurement of the diaSources on the science image.
520 diaSources : `lsst.afw.table.SourceCatalog`
521 The catalog of detected sources.
522 science : `lsst.afw.image.ExposureF`
523 Science exposure that the template was subtracted from.
524 wcs : `lsst.afw.geom.SkyWcs`
525 Coordinate system definition (wcs) for the exposure.
529 forcedSources = self.forcedMeasurement.generateMeasCat(science, diaSources, wcs)
530 self.forcedMeasurement.
run(forcedSources, science, diaSources, wcs)
531 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
532 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
533 "ip_diffim_forced_PsfFlux_instFlux",
True)
534 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
535 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
536 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
537 "ip_diffim_forced_PsfFlux_area",
True)
538 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
539 "ip_diffim_forced_PsfFlux_flag",
True)
540 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
541 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
542 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
543 "ip_diffim_forced_PsfFlux_flag_edge",
True)
544 for diaSource, forcedSource
in zip(diaSources, forcedSources):
545 diaSource.assign(forcedSource, mapper)
548 """Add image QA metrics to the Task metadata.
552 difference : `lsst.afw.image.Exposure`
553 The target image to calculate metrics for.
555 mask = difference.mask
556 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
557 self.metadata.add(
"nGoodPixels", np.sum(~badPix))
558 self.metadata.add(
"nBadPixels", np.sum(badPix))
559 detPosPix = (mask.array & mask.getPlaneBitMask(
"DETECTED")) > 0
560 detNegPix = (mask.array & mask.getPlaneBitMask(
"DETECTED_NEGATIVE")) > 0
561 self.metadata.add(
"nPixelsDetectedPositive", np.sum(detPosPix))
562 self.metadata.add(
"nPixelsDetectedNegative", np.sum(detNegPix))
565 self.metadata.add(
"nBadPixelsDetectedPositive", np.sum(detPosPix))
566 self.metadata.add(
"nBadPixelsDetectedNegative", np.sum(detNegPix))
570 scoreExposure = pipeBase.connectionTypes.Input(
571 doc=
"Maximum likelihood image for detection.",
572 dimensions=(
"instrument",
"visit",
"detector"),
573 storageClass=
"ExposureF",
574 name=
"{fakesType}{coaddName}Diff_scoreExp",
579 pipelineConnections=DetectAndMeasureScoreConnections):
584 """Detect DIA sources using a score image,
585 and measure the detections on the difference image.
587 Source detection is run on the supplied score, or maximum likelihood,
588 image. Note that no additional convolution will be done in this case.
589 Close positive and negative detections will optionally be merged into
591 Sky sources, or forced detections in background regions, will optionally
592 be added, and the configured measurement algorithm will be run on all
595 ConfigClass = DetectAndMeasureScoreConfig
596 _DefaultName =
"detectAndMeasureScore"
599 def run(self, science, matchedTemplate, difference, scoreExposure,
601 """Detect and measure sources on a score image.
605 science : `lsst.afw.image.ExposureF`
606 Science exposure that the template was subtracted from.
607 matchedTemplate : `lsst.afw.image.ExposureF`
608 Warped and PSF-matched template that was used produce the
610 difference : `lsst.afw.image.ExposureF`
611 Result of subtracting template from the science image.
612 scoreExposure : `lsst.afw.image.ExposureF`
613 Score or maximum likelihood difference image
614 idFactory : `lsst.afw.table.IdFactory`, optional
615 Generator object used to assign ids to detected sources in the
616 difference image. Ids from this generator are not set until after
617 deblending and merging positive/negative peaks.
621 measurementResults : `lsst.pipe.base.Struct`
623 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
624 Subtracted exposure with detection mask applied.
625 ``diaSources`` : `lsst.afw.table.SourceCatalog`
626 The catalog of detected sources.
628 if idFactory
is None:
629 idFactory = lsst.meas.base.IdGenerator().make_table_id_factory()
632 mask = scoreExposure.mask
633 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
638 table = afwTable.SourceTable.make(self.schema)
639 results = self.detection.
run(
641 exposure=scoreExposure,
645 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
647 sources, positives, negatives = self._deblend(difference,
651 return self.processResults(science, matchedTemplate, difference, sources, idFactory,
652 positiveFootprints=positives, negativeFootprints=negatives)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)