26from lsst.meas.algorithms
import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources
29import lsst.meas.extensions.shapeHSM
33from lsst.utils.timer
import timeMethod
35from .
import DipoleFitTask
37__all__ = [
"DetectAndMeasureConfig",
"DetectAndMeasureTask",
38 "DetectAndMeasureScoreConfig",
"DetectAndMeasureScoreTask"]
42 dimensions=(
"instrument",
"visit",
"detector"),
43 defaultTemplates={
"coaddName":
"deep",
46 science = pipeBase.connectionTypes.Input(
47 doc=
"Input science exposure.",
48 dimensions=(
"instrument",
"visit",
"detector"),
49 storageClass=
"ExposureF",
50 name=
"{fakesType}calexp"
52 matchedTemplate = pipeBase.connectionTypes.Input(
53 doc=
"Warped and PSF-matched template used to create the difference image.",
54 dimensions=(
"instrument",
"visit",
"detector"),
55 storageClass=
"ExposureF",
56 name=
"{fakesType}{coaddName}Diff_matchedExp",
58 difference = pipeBase.connectionTypes.Input(
59 doc=
"Result of subtracting template from science.",
60 dimensions=(
"instrument",
"visit",
"detector"),
61 storageClass=
"ExposureF",
62 name=
"{fakesType}{coaddName}Diff_differenceTempExp",
64 outputSchema = pipeBase.connectionTypes.InitOutput(
65 doc=
"Schema (as an example catalog) for output DIASource catalog.",
66 storageClass=
"SourceCatalog",
67 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
69 diaSources = pipeBase.connectionTypes.Output(
70 doc=
"Detected diaSources on the difference image.",
71 dimensions=(
"instrument",
"visit",
"detector"),
72 storageClass=
"SourceCatalog",
73 name=
"{fakesType}{coaddName}Diff_diaSrc",
75 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
76 doc=
"Difference image with detection mask plane filled in.",
77 dimensions=(
"instrument",
"visit",
"detector"),
78 storageClass=
"ExposureF",
79 name=
"{fakesType}{coaddName}Diff_differenceExp",
84 pipelineConnections=DetectAndMeasureConnections):
85 """Config for DetectAndMeasureTask
87 doMerge = pexConfig.Field(
90 doc=
"Merge positive and negative diaSources with grow radius "
91 "set by growFootprint"
93 doForcedMeasurement = pexConfig.Field(
96 doc=
"Force photometer diaSource locations on PVI?")
97 doAddMetrics = pexConfig.Field(
100 doc=
"Add columns to the source table to hold analysis metrics?"
102 detection = pexConfig.ConfigurableField(
103 target=SourceDetectionTask,
104 doc=
"Final source detection for diaSource measurement",
106 measurement = pexConfig.ConfigurableField(
107 target=DipoleFitTask,
108 doc=
"Task to measure sources on the difference image.",
110 doApCorr = lsst.pex.config.Field(
113 doc=
"Run subtask to apply aperture corrections"
115 applyApCorr = lsst.pex.config.ConfigurableField(
116 target=ApplyApCorrTask,
117 doc=
"Task to apply aperture corrections"
119 forcedMeasurement = pexConfig.ConfigurableField(
120 target=ForcedMeasurementTask,
121 doc=
"Task to force photometer science image at diaSource locations.",
123 growFootprint = pexConfig.Field(
126 doc=
"Grow positive and negative footprints by this many pixels before merging"
128 diaSourceMatchRadius = pexConfig.Field(
131 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
133 doSkySources = pexConfig.Field(
136 doc=
"Generate sky sources?",
138 skySources = pexConfig.ConfigurableField(
139 target=SkyObjectsTask,
140 doc=
"Generate sky sources",
142 badSourceFlags = lsst.pex.config.ListField(
144 doc=
"Sources with any of these flags set are removed before writing the output catalog.",
145 default=(
"base_PixelFlags_flag_offimage",
146 "base_PixelFlags_flag_interpolatedCenterAll",
147 "base_PixelFlags_flag_badCenterAll",
148 "base_PixelFlags_flag_edgeCenterAll",
151 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
153 def setDefaults(self):
155 self.detection.thresholdPolarity =
"both"
156 self.detection.thresholdValue = 5.0
157 self.detection.reEstimateBackground =
False
158 self.detection.thresholdType =
"pixel_stdev"
159 self.detection.excludeMaskPlanes = [
"EDGE"]
162 self.measurement.algorithms.names.add(
"base_PeakLikelihoodFlux")
163 self.measurement.plugins.names |= [
"ext_trailedSources_Naive",
164 "base_LocalPhotoCalib",
166 "ext_shapeHSM_HsmSourceMoments",
167 "ext_shapeHSM_HsmPsfMoments",
169 self.measurement.slots.psfShape =
"ext_shapeHSM_HsmPsfMoments"
170 self.measurement.slots.shape =
"ext_shapeHSM_HsmSourceMoments"
171 self.measurement.plugins[
"base_NaiveCentroid"].maxDistToPeak = 5.0
172 self.measurement.plugins[
"base_SdssCentroid"].maxDistToPeak = 5.0
173 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
174 self.forcedMeasurement.copyColumns = {
175 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
176 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
177 self.forcedMeasurement.slots.shape =
None
180 self.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere = [
181 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
182 self.measurement.plugins[
"base_PixelFlags"].masksFpCenter = [
183 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
184 self.skySources.avoidMask = [
"DETECTED",
"DETECTED_NEGATIVE",
"BAD",
"NO_DATA",
"EDGE"]
188 """Detect and measure sources on a difference image.
190 ConfigClass = DetectAndMeasureConfig
191 _DefaultName =
"detectAndMeasure"
193 def __init__(self, **kwargs):
194 super().__init__(**kwargs)
195 self.schema = afwTable.SourceTable.makeMinimalSchema()
197 afwTable.CoordKey.addErrorFields(self.schema)
200 self.makeSubtask(
"detection", schema=self.schema)
201 self.makeSubtask(
"measurement", schema=self.schema,
202 algMetadata=self.algMetadata)
203 if self.config.doApCorr:
204 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
205 if self.config.doForcedMeasurement:
206 self.schema.addField(
207 "ip_diffim_forced_PsfFlux_instFlux",
"D",
208 "Forced PSF flux measured on the direct image.",
210 self.schema.addField(
211 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
212 "Forced PSF flux error measured on the direct image.",
214 self.schema.addField(
215 "ip_diffim_forced_PsfFlux_area",
"F",
216 "Forced PSF flux effective area of PSF.",
218 self.schema.addField(
219 "ip_diffim_forced_PsfFlux_flag",
"Flag",
220 "Forced PSF flux general failure flag.")
221 self.schema.addField(
222 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
223 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
224 self.schema.addField(
225 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
226 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
227 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
229 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
230 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
231 if self.config.doSkySources:
232 self.makeSubtask(
"skySources")
233 self.skySourceKey = self.schema.addField(
"sky_source", type=
"Flag", doc=
"Sky objects.")
236 for flag
in self.config.badSourceFlags:
237 if flag
not in self.schema:
238 raise pipeBase.InvalidQuantumError(
"Field %s not in schema" % flag)
240 self.outputSchema = afwTable.SourceCatalog(self.schema)
241 self.outputSchema.getTable().setMetadata(self.algMetadata)
243 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
244 inputRefs: pipeBase.InputQuantizedConnection,
245 outputRefs: pipeBase.OutputQuantizedConnection):
246 inputs = butlerQC.get(inputRefs)
247 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
248 idFactory = idGenerator.make_table_id_factory()
249 outputs = self.run(**inputs, idFactory=idFactory)
250 butlerQC.put(outputs, outputRefs)
253 def run(self, science, matchedTemplate, difference,
255 """Detect and measure sources on a difference image.
257 The difference image will be convolved with a gaussian approximation of
258 the PSF to form a maximum likelihood image for detection.
259 Close positive and negative detections will optionally be merged into
261 Sky sources, or forced detections in background regions, will optionally
262 be added, and the configured measurement algorithm will be run on all
267 science : `lsst.afw.image.ExposureF`
268 Science exposure that the template was subtracted from.
269 matchedTemplate : `lsst.afw.image.ExposureF`
270 Warped and PSF-matched template that was used produce the
272 difference : `lsst.afw.image.ExposureF`
273 Result of subtracting template from the science image.
274 idFactory : `lsst.afw.table.IdFactory`, optional
275 Generator object to assign ids to detected sources in the difference image.
279 measurementResults : `lsst.pipe.base.Struct`
281 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
282 Subtracted exposure with detection mask applied.
283 ``diaSources`` : `lsst.afw.table.SourceCatalog`
284 The catalog of detected sources.
287 mask = difference.mask
288 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
290 table = afwTable.SourceTable.make(self.schema, idFactory)
291 table.setMetadata(self.algMetadata)
292 results = self.detection.
run(
298 return self.processResults(science, matchedTemplate, difference, results.sources, table,
299 positiveFootprints=results.positive, negativeFootprints=results.negative)
301 def processResults(self, science, matchedTemplate, difference, sources, table,
302 positiveFootprints=None, negativeFootprints=None,):
303 """Measure and process the results of source detection.
307 sources : `lsst.afw.table.SourceCatalog`
308 Detected sources on the difference exposure.
309 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
310 Positive polarity footprints.
311 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
312 Negative polarity footprints.
313 table : `lsst.afw.table.SourceTable`
314 Table object that will be used to create the SourceCatalog.
315 science : `lsst.afw.image.ExposureF`
316 Science exposure that the template was subtracted from.
317 matchedTemplate : `lsst.afw.image.ExposureF`
318 Warped and PSF-matched template that was used produce the
320 difference : `lsst.afw.image.ExposureF`
321 Result of subtracting template from the science image.
325 measurementResults : `lsst.pipe.base.Struct`
327 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
328 Subtracted exposure with detection mask applied.
329 ``diaSources`` : `lsst.afw.table.SourceCatalog`
330 The catalog of detected sources.
332 self.metadata.add(
"nUnmergedDiaSources",
len(sources))
333 if self.config.doMerge:
334 fpSet = positiveFootprints
335 fpSet.merge(negativeFootprints, self.config.growFootprint,
336 self.config.growFootprint,
False)
337 initialDiaSources = afwTable.SourceCatalog(table)
338 fpSet.makeSources(initialDiaSources)
339 self.log.
info(
"Merging detections into %d sources",
len(initialDiaSources))
341 initialDiaSources = sources
342 self.metadata.add(
"nMergedDiaSources",
len(initialDiaSources))
344 if self.config.doSkySources:
345 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
347 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
348 diaSources = self._removeBadSources(initialDiaSources)
350 if self.config.doForcedMeasurement:
351 self.measureForcedSources(diaSources, science, difference.getWcs())
353 measurementResults = pipeBase.Struct(
354 subtractedMeasuredExposure=difference,
355 diaSources=diaSources,
357 self.calculateMetrics(difference)
359 return measurementResults
362 """Remove bad diaSources from the catalog.
366 diaSources : `lsst.afw.table.SourceCatalog`
367 The catalog of detected sources.
371 diaSources : `lsst.afw.table.SourceCatalog`
372 The updated catalog of detected sources, with any source that has a
373 flag in ``config.badSourceFlags`` set removed.
376 selector = np.ones(
len(diaSources), dtype=bool)
377 for flag
in self.config.badSourceFlags:
378 flags = diaSources[flag]
379 nBad = np.count_nonzero(flags)
381 self.log.
info(
"Found and removed %d unphysical sources with flag %s.", nBad, flag)
384 self.metadata.add(
"nRemovedBadFlaggedSources", nBadTotal)
385 return diaSources[selector].copy(deep=
True)
388 """Add sources in empty regions of the difference image
389 for measuring the background.
393 diaSources : `lsst.afw.table.SourceCatalog`
394 The catalog of detected sources.
395 mask : `lsst.afw.image.Mask`
396 Mask plane for determining regions where Sky sources can be added.
398 Seed value to initialize the random number generator.
400 skySourceFootprints = self.skySources.
run(mask=mask, seed=seed)
401 self.metadata.add(
"nSkySources",
len(skySourceFootprints))
402 if skySourceFootprints:
403 for foot
in skySourceFootprints:
404 s = diaSources.addNew()
406 s.set(self.skySourceKey,
True)
409 """Use (matched) template and science image to constrain dipole fitting.
413 diaSources : `lsst.afw.table.SourceCatalog`
414 The catalog of detected sources.
415 science : `lsst.afw.image.ExposureF`
416 Science exposure that the template was subtracted from.
417 difference : `lsst.afw.image.ExposureF`
418 Result of subtracting template from the science image.
419 matchedTemplate : `lsst.afw.image.ExposureF`
420 Warped and PSF-matched template that was used produce the
425 self.measurement.
run(diaSources, difference, science, matchedTemplate)
426 if self.config.doApCorr:
427 apCorrMap = difference.getInfo().getApCorrMap()
428 if apCorrMap
is None:
429 self.log.
warning(
"Difference image does not have valid aperture correction; skipping.")
431 self.applyApCorr.
run(
437 """Perform forced measurement of the diaSources on the science image.
441 diaSources : `lsst.afw.table.SourceCatalog`
442 The catalog of detected sources.
443 science : `lsst.afw.image.ExposureF`
444 Science exposure that the template was subtracted from.
445 wcs : `lsst.afw.geom.SkyWcs`
446 Coordinate system definition (wcs) for the exposure.
450 forcedSources = self.forcedMeasurement.generateMeasCat(
451 science, diaSources, wcs)
452 self.forcedMeasurement.
run(forcedSources, science, diaSources, wcs)
453 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
454 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
455 "ip_diffim_forced_PsfFlux_instFlux",
True)
456 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
457 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
458 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
459 "ip_diffim_forced_PsfFlux_area",
True)
460 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
461 "ip_diffim_forced_PsfFlux_flag",
True)
462 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
463 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
464 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
465 "ip_diffim_forced_PsfFlux_flag_edge",
True)
466 for diaSource, forcedSource
in zip(diaSources, forcedSources):
467 diaSource.assign(forcedSource, mapper)
470 """Add image QA metrics to the Task metadata.
474 difference : `lsst.afw.image.Exposure`
475 The target image to calculate metrics for.
477 mask = difference.mask
478 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
479 self.metadata.add(
"nGoodPixels", np.sum(~badPix))
480 self.metadata.add(
"nBadPixels", np.sum(badPix))
481 detPosPix = (mask.array & mask.getPlaneBitMask(
"DETECTED")) > 0
482 detNegPix = (mask.array & mask.getPlaneBitMask(
"DETECTED_NEGATIVE")) > 0
483 self.metadata.add(
"nPixelsDetectedPositive", np.sum(detPosPix))
484 self.metadata.add(
"nPixelsDetectedNegative", np.sum(detNegPix))
487 self.metadata.add(
"nBadPixelsDetectedPositive", np.sum(detPosPix))
488 self.metadata.add(
"nBadPixelsDetectedNegative", np.sum(detNegPix))
492 scoreExposure = pipeBase.connectionTypes.Input(
493 doc=
"Maximum likelihood image for detection.",
494 dimensions=(
"instrument",
"visit",
"detector"),
495 storageClass=
"ExposureF",
496 name=
"{fakesType}{coaddName}Diff_scoreExp",
501 pipelineConnections=DetectAndMeasureScoreConnections):
506 """Detect DIA sources using a score image,
507 and measure the detections on the difference image.
509 Source detection is run on the supplied score, or maximum likelihood,
510 image. Note that no additional convolution will be done in this case.
511 Close positive and negative detections will optionally be merged into
513 Sky sources, or forced detections in background regions, will optionally
514 be added, and the configured measurement algorithm will be run on all
517 ConfigClass = DetectAndMeasureScoreConfig
518 _DefaultName =
"detectAndMeasureScore"
521 def run(self, science, matchedTemplate, difference, scoreExposure,
523 """Detect and measure sources on a score image.
527 science : `lsst.afw.image.ExposureF`
528 Science exposure that the template was subtracted from.
529 matchedTemplate : `lsst.afw.image.ExposureF`
530 Warped and PSF-matched template that was used produce the
532 difference : `lsst.afw.image.ExposureF`
533 Result of subtracting template from the science image.
534 scoreExposure : `lsst.afw.image.ExposureF`
535 Score or maximum likelihood difference image
536 idFactory : `lsst.afw.table.IdFactory`, optional
537 Generator object to assign ids to detected sources in the difference image.
541 measurementResults : `lsst.pipe.base.Struct`
543 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
544 Subtracted exposure with detection mask applied.
545 ``diaSources`` : `lsst.afw.table.SourceCatalog`
546 The catalog of detected sources.
549 mask = scoreExposure.mask
550 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
552 table = afwTable.SourceTable.make(self.schema, idFactory)
553 table.setMetadata(self.algMetadata)
554 results = self.detection.
run(
556 exposure=scoreExposure,
560 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
562 return self.processResults(science, matchedTemplate, difference, results.sources, table,
563 positiveFootprints=results.positive, negativeFootprints=results.negative)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)