26from lsst.meas.algorithms
import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources
29import lsst.meas.extensions.shapeHSM
33from lsst.utils.timer
import timeMethod
35from .
import DipoleFitTask
37__all__ = [
"DetectAndMeasureConfig",
"DetectAndMeasureTask",
38 "DetectAndMeasureScoreConfig",
"DetectAndMeasureScoreTask"]
42 dimensions=(
"instrument",
"visit",
"detector"),
43 defaultTemplates={
"coaddName":
"deep",
46 science = pipeBase.connectionTypes.Input(
47 doc=
"Input science exposure.",
48 dimensions=(
"instrument",
"visit",
"detector"),
49 storageClass=
"ExposureF",
50 name=
"{fakesType}calexp"
52 matchedTemplate = pipeBase.connectionTypes.Input(
53 doc=
"Warped and PSF-matched template used to create the difference image.",
54 dimensions=(
"instrument",
"visit",
"detector"),
55 storageClass=
"ExposureF",
56 name=
"{fakesType}{coaddName}Diff_matchedExp",
58 difference = pipeBase.connectionTypes.Input(
59 doc=
"Result of subtracting template from science.",
60 dimensions=(
"instrument",
"visit",
"detector"),
61 storageClass=
"ExposureF",
62 name=
"{fakesType}{coaddName}Diff_differenceTempExp",
64 outputSchema = pipeBase.connectionTypes.InitOutput(
65 doc=
"Schema (as an example catalog) for output DIASource catalog.",
66 storageClass=
"SourceCatalog",
67 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
69 diaSources = pipeBase.connectionTypes.Output(
70 doc=
"Detected diaSources on the difference image.",
71 dimensions=(
"instrument",
"visit",
"detector"),
72 storageClass=
"SourceCatalog",
73 name=
"{fakesType}{coaddName}Diff_diaSrc",
75 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
76 doc=
"Difference image with detection mask plane filled in.",
77 dimensions=(
"instrument",
"visit",
"detector"),
78 storageClass=
"ExposureF",
79 name=
"{fakesType}{coaddName}Diff_differenceExp",
84 pipelineConnections=DetectAndMeasureConnections):
85 """Config for DetectAndMeasureTask
87 doMerge = pexConfig.Field(
90 doc=
"Merge positive and negative diaSources with grow radius "
91 "set by growFootprint"
93 doForcedMeasurement = pexConfig.Field(
96 doc=
"Force photometer diaSource locations on PVI?")
97 doAddMetrics = pexConfig.Field(
100 doc=
"Add columns to the source table to hold analysis metrics?"
102 detection = pexConfig.ConfigurableField(
103 target=SourceDetectionTask,
104 doc=
"Final source detection for diaSource measurement",
106 measurement = pexConfig.ConfigurableField(
107 target=DipoleFitTask,
108 doc=
"Task to measure sources on the difference image.",
110 doApCorr = lsst.pex.config.Field(
113 doc=
"Run subtask to apply aperture corrections"
115 applyApCorr = lsst.pex.config.ConfigurableField(
116 target=ApplyApCorrTask,
117 doc=
"Task to apply aperture corrections"
119 forcedMeasurement = pexConfig.ConfigurableField(
120 target=ForcedMeasurementTask,
121 doc=
"Task to force photometer science image at diaSource locations.",
123 growFootprint = pexConfig.Field(
126 doc=
"Grow positive and negative footprints by this many pixels before merging"
128 diaSourceMatchRadius = pexConfig.Field(
131 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
133 doSkySources = pexConfig.Field(
136 doc=
"Generate sky sources?",
138 skySources = pexConfig.ConfigurableField(
139 target=SkyObjectsTask,
140 doc=
"Generate sky sources",
142 badSourceFlags = lsst.pex.config.ListField(
144 doc=
"Sources with any of these flags set are removed before writing the output catalog.",
145 default=(
"base_PixelFlags_flag_offimage",
146 "base_PixelFlags_flag_interpolatedCenterAll",
147 "base_PixelFlags_flag_badCenterAll",
148 "base_PixelFlags_flag_edgeCenterAll",
151 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
153 def setDefaults(self):
155 self.detection.thresholdPolarity =
"both"
156 self.detection.thresholdValue = 5.0
157 self.detection.reEstimateBackground =
False
158 self.detection.thresholdType =
"pixel_stdev"
159 self.detection.excludeMaskPlanes = [
"EDGE"]
162 self.measurement.algorithms.names.add(
"base_PeakLikelihoodFlux")
163 self.measurement.plugins.names |= [
"ext_trailedSources_Naive",
164 "base_LocalPhotoCalib",
166 "ext_shapeHSM_HsmSourceMoments",
167 "ext_shapeHSM_HsmPsfMoments",
169 self.measurement.slots.psfShape =
"ext_shapeHSM_HsmPsfMoments"
170 self.measurement.slots.shape =
"ext_shapeHSM_HsmSourceMoments"
171 self.measurement.plugins[
"base_NaiveCentroid"].maxDistToPeak = 5.0
172 self.measurement.plugins[
"base_SdssCentroid"].maxDistToPeak = 5.0
173 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
174 self.forcedMeasurement.copyColumns = {
175 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
176 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
177 self.forcedMeasurement.slots.shape =
None
180 self.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere = [
181 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
182 self.measurement.plugins[
"base_PixelFlags"].masksFpCenter = [
183 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
184 self.skySources.avoidMask = [
"DETECTED",
"DETECTED_NEGATIVE",
"BAD",
"NO_DATA",
"EDGE"]
188 """Detect and measure sources on a difference image.
190 ConfigClass = DetectAndMeasureConfig
191 _DefaultName =
"detectAndMeasure"
193 def __init__(self, **kwargs):
194 super().__init__(**kwargs)
195 self.schema = afwTable.SourceTable.makeMinimalSchema()
197 afwTable.CoordKey.addErrorFields(self.schema)
200 self.makeSubtask(
"detection", schema=self.schema)
201 self.makeSubtask(
"measurement", schema=self.schema,
202 algMetadata=self.algMetadata)
203 if self.config.doApCorr:
204 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
205 if self.config.doForcedMeasurement:
206 self.schema.addField(
207 "ip_diffim_forced_PsfFlux_instFlux",
"D",
208 "Forced PSF flux measured on the direct image.",
210 self.schema.addField(
211 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
212 "Forced PSF flux error measured on the direct image.",
214 self.schema.addField(
215 "ip_diffim_forced_PsfFlux_area",
"F",
216 "Forced PSF flux effective area of PSF.",
218 self.schema.addField(
219 "ip_diffim_forced_PsfFlux_flag",
"Flag",
220 "Forced PSF flux general failure flag.")
221 self.schema.addField(
222 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
223 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
224 self.schema.addField(
225 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
226 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
227 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
229 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
230 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
231 if self.config.doSkySources:
232 self.makeSubtask(
"skySources")
233 self.skySourceKey = self.schema.addField(
"sky_source", type=
"Flag", doc=
"Sky objects.")
236 for flag
in self.config.badSourceFlags:
237 if flag
not in self.schema:
238 raise pipeBase.InvalidQuantumError(
"Field %s not in schema" % flag)
240 self.outputSchema = afwTable.SourceCatalog(self.schema)
241 self.outputSchema.getTable().setMetadata(self.algMetadata)
243 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
244 inputRefs: pipeBase.InputQuantizedConnection,
245 outputRefs: pipeBase.OutputQuantizedConnection):
246 inputs = butlerQC.get(inputRefs)
247 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
248 idFactory = idGenerator.make_table_id_factory()
249 outputs = self.run(**inputs, idFactory=idFactory)
250 butlerQC.put(outputs, outputRefs)
253 def run(self, science, matchedTemplate, difference,
255 """Detect and measure sources on a difference image.
257 The difference image will be convolved with a gaussian approximation of
258 the PSF to form a maximum likelihood image for detection.
259 Close positive and negative detections will optionally be merged into
261 Sky sources, or forced detections in background regions, will optionally
262 be added, and the configured measurement algorithm will be run on all
267 science : `lsst.afw.image.ExposureF`
268 Science exposure that the template was subtracted from.
269 matchedTemplate : `lsst.afw.image.ExposureF`
270 Warped and PSF-matched template that was used produce the
272 difference : `lsst.afw.image.ExposureF`
273 Result of subtracting template from the science image.
274 idFactory : `lsst.afw.table.IdFactory`, optional
275 Generator object to assign ids to detected sources in the difference image.
279 measurementResults : `lsst.pipe.base.Struct`
281 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
282 Subtracted exposure with detection mask applied.
283 ``diaSources`` : `lsst.afw.table.SourceCatalog`
284 The catalog of detected sources.
287 mask = difference.mask
288 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
290 table = afwTable.SourceTable.make(self.schema, idFactory)
291 table.setMetadata(self.algMetadata)
292 results = self.detection.
run(
298 return self.processResults(science, matchedTemplate, difference, results.sources, table,
299 positiveFootprints=results.positive, negativeFootprints=results.negative)
301 def processResults(self, science, matchedTemplate, difference, sources, table,
302 positiveFootprints=None, negativeFootprints=None,):
303 """Measure and process the results of source detection.
307 science : `lsst.afw.image.ExposureF`
308 Science exposure that the template was subtracted from.
309 matchedTemplate : `lsst.afw.image.ExposureF`
310 Warped and PSF-matched template that was used produce the
312 difference : `lsst.afw.image.ExposureF`
313 Result of subtracting template from the science image.
314 sources : `lsst.afw.table.SourceCatalog`
315 Detected sources on the difference exposure.
316 table : `lsst.afw.table.SourceTable`
317 Table object that will be used to create the SourceCatalog.
318 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
319 Positive polarity footprints.
320 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
321 Negative polarity footprints.
325 measurementResults : `lsst.pipe.base.Struct`
327 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
328 Subtracted exposure with detection mask applied.
329 ``diaSources`` : `lsst.afw.table.SourceCatalog`
330 The catalog of detected sources.
332 self.metadata.add(
"nUnmergedDiaSources",
len(sources))
333 if self.config.doMerge:
334 fpSet = positiveFootprints
335 fpSet.merge(negativeFootprints, self.config.growFootprint,
336 self.config.growFootprint,
False)
337 initialDiaSources = afwTable.SourceCatalog(table)
338 fpSet.makeSources(initialDiaSources)
339 self.log.
info(
"Merging detections into %d sources",
len(initialDiaSources))
341 initialDiaSources = sources
342 self.metadata.add(
"nMergedDiaSources",
len(initialDiaSources))
344 if self.config.doSkySources:
345 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
347 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
348 diaSources = self._removeBadSources(initialDiaSources)
350 if self.config.doForcedMeasurement:
351 self.measureForcedSources(diaSources, science, difference.getWcs())
353 measurementResults = pipeBase.Struct(
354 subtractedMeasuredExposure=difference,
355 diaSources=diaSources,
357 self.calculateMetrics(difference)
359 return measurementResults
362 """Remove bad diaSources from the catalog.
366 diaSources : `lsst.afw.table.SourceCatalog`
367 The catalog of detected sources.
371 diaSources : `lsst.afw.table.SourceCatalog`
372 The updated catalog of detected sources, with any source that has a
373 flag in ``config.badSourceFlags`` set removed.
376 selector = np.ones(
len(diaSources), dtype=bool)
377 for flag
in self.config.badSourceFlags:
378 flags = diaSources[flag]
379 nBad = np.count_nonzero(flags)
381 self.log.
info(
"Found and removed %d unphysical sources with flag %s.", nBad, flag)
384 self.metadata.add(
"nRemovedBadFlaggedSources", nBadTotal)
385 return diaSources[selector].copy(deep=
True)
388 """Add sources in empty regions of the difference image
389 for measuring the background.
393 diaSources : `lsst.afw.table.SourceCatalog`
394 The catalog of detected sources.
395 mask : `lsst.afw.image.Mask`
396 Mask plane for determining regions where Sky sources can be added.
398 Seed value to initialize the random number generator.
400 skySourceFootprints = self.skySources.
run(mask=mask, seed=seed)
401 self.metadata.add(
"nSkySources",
len(skySourceFootprints))
402 if skySourceFootprints:
403 for foot
in skySourceFootprints:
404 s = diaSources.addNew()
406 s.set(self.skySourceKey,
True)
409 """Use (matched) template and science image to constrain dipole fitting.
413 diaSources : `lsst.afw.table.SourceCatalog`
414 The catalog of detected sources.
415 science : `lsst.afw.image.ExposureF`
416 Science exposure that the template was subtracted from.
417 difference : `lsst.afw.image.ExposureF`
418 Result of subtracting template from the science image.
419 matchedTemplate : `lsst.afw.image.ExposureF`
420 Warped and PSF-matched template that was used produce the
424 for mp
in self.config.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere:
425 difference.mask.addMaskPlane(mp)
428 self.measurement.
run(diaSources, difference, science, matchedTemplate)
429 if self.config.doApCorr:
430 apCorrMap = difference.getInfo().getApCorrMap()
431 if apCorrMap
is None:
432 self.log.
warning(
"Difference image does not have valid aperture correction; skipping.")
434 self.applyApCorr.
run(
440 """Perform forced measurement of the diaSources on the science image.
444 diaSources : `lsst.afw.table.SourceCatalog`
445 The catalog of detected sources.
446 science : `lsst.afw.image.ExposureF`
447 Science exposure that the template was subtracted from.
448 wcs : `lsst.afw.geom.SkyWcs`
449 Coordinate system definition (wcs) for the exposure.
453 forcedSources = self.forcedMeasurement.generateMeasCat(
454 science, diaSources, wcs)
455 self.forcedMeasurement.
run(forcedSources, science, diaSources, wcs)
456 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
457 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
458 "ip_diffim_forced_PsfFlux_instFlux",
True)
459 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
460 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
461 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
462 "ip_diffim_forced_PsfFlux_area",
True)
463 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
464 "ip_diffim_forced_PsfFlux_flag",
True)
465 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
466 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
467 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
468 "ip_diffim_forced_PsfFlux_flag_edge",
True)
469 for diaSource, forcedSource
in zip(diaSources, forcedSources):
470 diaSource.assign(forcedSource, mapper)
473 """Add image QA metrics to the Task metadata.
477 difference : `lsst.afw.image.Exposure`
478 The target image to calculate metrics for.
480 mask = difference.mask
481 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
482 self.metadata.add(
"nGoodPixels", np.sum(~badPix))
483 self.metadata.add(
"nBadPixels", np.sum(badPix))
484 detPosPix = (mask.array & mask.getPlaneBitMask(
"DETECTED")) > 0
485 detNegPix = (mask.array & mask.getPlaneBitMask(
"DETECTED_NEGATIVE")) > 0
486 self.metadata.add(
"nPixelsDetectedPositive", np.sum(detPosPix))
487 self.metadata.add(
"nPixelsDetectedNegative", np.sum(detNegPix))
490 self.metadata.add(
"nBadPixelsDetectedPositive", np.sum(detPosPix))
491 self.metadata.add(
"nBadPixelsDetectedNegative", np.sum(detNegPix))
495 scoreExposure = pipeBase.connectionTypes.Input(
496 doc=
"Maximum likelihood image for detection.",
497 dimensions=(
"instrument",
"visit",
"detector"),
498 storageClass=
"ExposureF",
499 name=
"{fakesType}{coaddName}Diff_scoreExp",
504 pipelineConnections=DetectAndMeasureScoreConnections):
509 """Detect DIA sources using a score image,
510 and measure the detections on the difference image.
512 Source detection is run on the supplied score, or maximum likelihood,
513 image. Note that no additional convolution will be done in this case.
514 Close positive and negative detections will optionally be merged into
516 Sky sources, or forced detections in background regions, will optionally
517 be added, and the configured measurement algorithm will be run on all
520 ConfigClass = DetectAndMeasureScoreConfig
521 _DefaultName =
"detectAndMeasureScore"
524 def run(self, science, matchedTemplate, difference, scoreExposure,
526 """Detect and measure sources on a score image.
530 science : `lsst.afw.image.ExposureF`
531 Science exposure that the template was subtracted from.
532 matchedTemplate : `lsst.afw.image.ExposureF`
533 Warped and PSF-matched template that was used produce the
535 difference : `lsst.afw.image.ExposureF`
536 Result of subtracting template from the science image.
537 scoreExposure : `lsst.afw.image.ExposureF`
538 Score or maximum likelihood difference image
539 idFactory : `lsst.afw.table.IdFactory`, optional
540 Generator object to assign ids to detected sources in the difference image.
544 measurementResults : `lsst.pipe.base.Struct`
546 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
547 Subtracted exposure with detection mask applied.
548 ``diaSources`` : `lsst.afw.table.SourceCatalog`
549 The catalog of detected sources.
552 mask = scoreExposure.mask
553 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
555 table = afwTable.SourceTable.make(self.schema, idFactory)
556 table.setMetadata(self.algMetadata)
557 results = self.detection.
run(
559 exposure=scoreExposure,
563 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
565 return self.processResults(science, matchedTemplate, difference, results.sources, table,
566 positiveFootprints=results.positive, negativeFootprints=results.negative)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)