26from lsst.meas.algorithms
import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources
29import lsst.meas.extensions.shapeHSM
33from lsst.utils.timer
import timeMethod
35from .
import DipoleFitTask
37__all__ = [
"DetectAndMeasureConfig",
"DetectAndMeasureTask",
38 "DetectAndMeasureScoreConfig",
"DetectAndMeasureScoreTask"]
42 dimensions=(
"instrument",
"visit",
"detector"),
43 defaultTemplates={
"coaddName":
"deep",
46 science = pipeBase.connectionTypes.Input(
47 doc=
"Input science exposure.",
48 dimensions=(
"instrument",
"visit",
"detector"),
49 storageClass=
"ExposureF",
50 name=
"{fakesType}calexp"
52 matchedTemplate = pipeBase.connectionTypes.Input(
53 doc=
"Warped and PSF-matched template used to create the difference image.",
54 dimensions=(
"instrument",
"visit",
"detector"),
55 storageClass=
"ExposureF",
56 name=
"{fakesType}{coaddName}Diff_matchedExp",
58 difference = pipeBase.connectionTypes.Input(
59 doc=
"Result of subtracting template from science.",
60 dimensions=(
"instrument",
"visit",
"detector"),
61 storageClass=
"ExposureF",
62 name=
"{fakesType}{coaddName}Diff_differenceTempExp",
64 outputSchema = pipeBase.connectionTypes.InitOutput(
65 doc=
"Schema (as an example catalog) for output DIASource catalog.",
66 storageClass=
"SourceCatalog",
67 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
69 diaSources = pipeBase.connectionTypes.Output(
70 doc=
"Detected diaSources on the difference image.",
71 dimensions=(
"instrument",
"visit",
"detector"),
72 storageClass=
"SourceCatalog",
73 name=
"{fakesType}{coaddName}Diff_diaSrc",
75 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
76 doc=
"Difference image with detection mask plane filled in.",
77 dimensions=(
"instrument",
"visit",
"detector"),
78 storageClass=
"ExposureF",
79 name=
"{fakesType}{coaddName}Diff_differenceExp",
84 pipelineConnections=DetectAndMeasureConnections):
85 """Config for DetectAndMeasureTask
87 doMerge = pexConfig.Field(
90 doc=
"Merge positive and negative diaSources with grow radius "
91 "set by growFootprint"
93 doForcedMeasurement = pexConfig.Field(
96 doc=
"Force photometer diaSource locations on PVI?")
97 doAddMetrics = pexConfig.Field(
100 doc=
"Add columns to the source table to hold analysis metrics?"
102 detection = pexConfig.ConfigurableField(
103 target=SourceDetectionTask,
104 doc=
"Final source detection for diaSource measurement",
106 measurement = pexConfig.ConfigurableField(
107 target=DipoleFitTask,
108 doc=
"Task to measure sources on the difference image.",
110 doApCorr = lsst.pex.config.Field(
113 doc=
"Run subtask to apply aperture corrections"
115 applyApCorr = lsst.pex.config.ConfigurableField(
116 target=ApplyApCorrTask,
117 doc=
"Task to apply aperture corrections"
119 forcedMeasurement = pexConfig.ConfigurableField(
120 target=ForcedMeasurementTask,
121 doc=
"Task to force photometer science image at diaSource locations.",
123 growFootprint = pexConfig.Field(
126 doc=
"Grow positive and negative footprints by this many pixels before merging"
128 diaSourceMatchRadius = pexConfig.Field(
131 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
133 doSkySources = pexConfig.Field(
136 doc=
"Generate sky sources?",
138 skySources = pexConfig.ConfigurableField(
139 target=SkyObjectsTask,
140 doc=
"Generate sky sources",
142 badSourceFlags = lsst.pex.config.ListField(
144 doc=
"Sources with any of these flags set are removed before writing the output catalog.",
145 default=(
"base_PixelFlags_flag_offimage",
148 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
150 def setDefaults(self):
152 self.detection.thresholdPolarity =
"both"
153 self.detection.thresholdValue = 5.0
154 self.detection.reEstimateBackground =
False
155 self.detection.thresholdType =
"pixel_stdev"
156 self.detection.excludeMaskPlanes = [
"EDGE"]
159 self.measurement.algorithms.names.add(
"base_PeakLikelihoodFlux")
160 self.measurement.plugins.names |= [
"ext_trailedSources_Naive",
161 "base_LocalPhotoCalib",
163 "ext_shapeHSM_HsmSourceMoments",
164 "ext_shapeHSM_HsmPsfMoments",
166 self.measurement.slots.psfShape =
"ext_shapeHSM_HsmPsfMoments"
167 self.measurement.slots.shape =
"ext_shapeHSM_HsmSourceMoments"
168 self.measurement.plugins[
"base_NaiveCentroid"].maxDistToPeak = 5.0
169 self.measurement.plugins[
"base_SdssCentroid"].maxDistToPeak = 5.0
170 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
171 self.forcedMeasurement.copyColumns = {
172 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
173 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
174 self.forcedMeasurement.slots.shape =
None
177 self.measurement.plugins[
"base_PixelFlags"].masksFpAnywhere = [
178 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
179 self.measurement.plugins[
"base_PixelFlags"].masksFpCenter = [
180 "STREAK",
"INJECTED",
"INJECTED_TEMPLATE"]
184 """Detect and measure sources on a difference image.
186 ConfigClass = DetectAndMeasureConfig
187 _DefaultName =
"detectAndMeasure"
189 def __init__(self, **kwargs):
190 super().__init__(**kwargs)
191 self.schema = afwTable.SourceTable.makeMinimalSchema()
193 afwTable.CoordKey.addErrorFields(self.schema)
196 self.makeSubtask(
"detection", schema=self.schema)
197 self.makeSubtask(
"measurement", schema=self.schema,
198 algMetadata=self.algMetadata)
199 if self.config.doApCorr:
200 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
201 if self.config.doForcedMeasurement:
202 self.schema.addField(
203 "ip_diffim_forced_PsfFlux_instFlux",
"D",
204 "Forced PSF flux measured on the direct image.",
206 self.schema.addField(
207 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
208 "Forced PSF flux error measured on the direct image.",
210 self.schema.addField(
211 "ip_diffim_forced_PsfFlux_area",
"F",
212 "Forced PSF flux effective area of PSF.",
214 self.schema.addField(
215 "ip_diffim_forced_PsfFlux_flag",
"Flag",
216 "Forced PSF flux general failure flag.")
217 self.schema.addField(
218 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
219 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
220 self.schema.addField(
221 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
222 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
223 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
225 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
226 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
227 if self.config.doSkySources:
228 self.makeSubtask(
"skySources")
229 self.skySourceKey = self.schema.addField(
"sky_source", type=
"Flag", doc=
"Sky objects.")
232 for flag
in self.config.badSourceFlags:
233 if flag
not in self.schema:
234 raise pipeBase.InvalidQuantumError(
"Field %s not in schema" % flag)
236 self.outputSchema = afwTable.SourceCatalog(self.schema)
237 self.outputSchema.getTable().setMetadata(self.algMetadata)
239 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
240 inputRefs: pipeBase.InputQuantizedConnection,
241 outputRefs: pipeBase.OutputQuantizedConnection):
242 inputs = butlerQC.get(inputRefs)
243 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
244 idFactory = idGenerator.make_table_id_factory()
245 outputs = self.run(**inputs, idFactory=idFactory)
246 butlerQC.put(outputs, outputRefs)
249 def run(self, science, matchedTemplate, difference,
251 """Detect and measure sources on a difference image.
253 The difference image will be convolved with a gaussian approximation of
254 the PSF to form a maximum likelihood image for detection.
255 Close positive and negative detections will optionally be merged into
257 Sky sources, or forced detections in background regions, will optionally
258 be added, and the configured measurement algorithm will be run on all
263 science : `lsst.afw.image.ExposureF`
264 Science exposure that the template was subtracted from.
265 matchedTemplate : `lsst.afw.image.ExposureF`
266 Warped and PSF-matched template that was used produce the
268 difference : `lsst.afw.image.ExposureF`
269 Result of subtracting template from the science image.
270 idFactory : `lsst.afw.table.IdFactory`, optional
271 Generator object to assign ids to detected sources in the difference image.
275 measurementResults : `lsst.pipe.base.Struct`
277 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
278 Subtracted exposure with detection mask applied.
279 ``diaSources`` : `lsst.afw.table.SourceCatalog`
280 The catalog of detected sources.
283 mask = difference.mask
284 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
286 table = afwTable.SourceTable.make(self.schema, idFactory)
287 table.setMetadata(self.algMetadata)
288 results = self.detection.
run(
294 return self.processResults(science, matchedTemplate, difference, results.sources, table,
295 positiveFootprints=results.positive, negativeFootprints=results.negative)
297 def processResults(self, science, matchedTemplate, difference, sources, table,
298 positiveFootprints=None, negativeFootprints=None,):
299 """Measure and process the results of source detection.
303 sources : `lsst.afw.table.SourceCatalog`
304 Detected sources on the difference exposure.
305 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
306 Positive polarity footprints.
307 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
308 Negative polarity footprints.
309 table : `lsst.afw.table.SourceTable`
310 Table object that will be used to create the SourceCatalog.
311 science : `lsst.afw.image.ExposureF`
312 Science exposure that the template was subtracted from.
313 matchedTemplate : `lsst.afw.image.ExposureF`
314 Warped and PSF-matched template that was used produce the
316 difference : `lsst.afw.image.ExposureF`
317 Result of subtracting template from the science image.
321 measurementResults : `lsst.pipe.base.Struct`
323 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
324 Subtracted exposure with detection mask applied.
325 ``diaSources`` : `lsst.afw.table.SourceCatalog`
326 The catalog of detected sources.
328 self.metadata.add(
"nUnmergedDiaSources",
len(sources))
329 if self.config.doMerge:
330 fpSet = positiveFootprints
331 fpSet.merge(negativeFootprints, self.config.growFootprint,
332 self.config.growFootprint,
False)
333 initialDiaSources = afwTable.SourceCatalog(table)
334 fpSet.makeSources(initialDiaSources)
335 self.log.
info(
"Merging detections into %d sources",
len(initialDiaSources))
337 initialDiaSources = sources
338 self.metadata.add(
"nMergedDiaSources",
len(initialDiaSources))
340 if self.config.doSkySources:
341 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
343 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
344 diaSources = self._removeBadSources(initialDiaSources)
346 if self.config.doForcedMeasurement:
347 self.measureForcedSources(diaSources, science, difference.getWcs())
349 measurementResults = pipeBase.Struct(
350 subtractedMeasuredExposure=difference,
351 diaSources=diaSources,
353 self.calculateMetrics(difference)
355 return measurementResults
358 """Remove bad diaSources from the catalog.
362 diaSources : `lsst.afw.table.SourceCatalog`
363 The catalog of detected sources.
367 diaSources : `lsst.afw.table.SourceCatalog`
368 The updated catalog of detected sources, with any source that has a
369 flag in ``config.badSourceFlags`` set removed.
372 selector = np.ones(
len(diaSources), dtype=bool)
373 for flag
in self.config.badSourceFlags:
374 flags = diaSources[flag]
375 nBad = np.count_nonzero(flags)
377 self.log.
info(
"Found and removed %d unphysical sources with flag %s.", nBad, flag)
380 self.metadata.add(
"nRemovedBadFlaggedSources", nBadTotal)
381 return diaSources[selector].copy(deep=
True)
384 """Add sources in empty regions of the difference image
385 for measuring the background.
389 diaSources : `lsst.afw.table.SourceCatalog`
390 The catalog of detected sources.
391 mask : `lsst.afw.image.Mask`
392 Mask plane for determining regions where Sky sources can be added.
394 Seed value to initialize the random number generator.
396 skySourceFootprints = self.skySources.
run(mask=mask, seed=seed)
397 if skySourceFootprints:
398 for foot
in skySourceFootprints:
399 s = diaSources.addNew()
401 s.set(self.skySourceKey,
True)
404 """Use (matched) template and science image to constrain dipole fitting.
408 diaSources : `lsst.afw.table.SourceCatalog`
409 The catalog of detected sources.
410 science : `lsst.afw.image.ExposureF`
411 Science exposure that the template was subtracted from.
412 difference : `lsst.afw.image.ExposureF`
413 Result of subtracting template from the science image.
414 matchedTemplate : `lsst.afw.image.ExposureF`
415 Warped and PSF-matched template that was used produce the
420 self.measurement.
run(diaSources, difference, science, matchedTemplate)
421 if self.config.doApCorr:
422 apCorrMap = difference.getInfo().getApCorrMap()
423 if apCorrMap
is None:
424 self.log.
warning(
"Difference image does not have valid aperture correction; skipping.")
426 self.applyApCorr.
run(
432 """Perform forced measurement of the diaSources on the science image.
436 diaSources : `lsst.afw.table.SourceCatalog`
437 The catalog of detected sources.
438 science : `lsst.afw.image.ExposureF`
439 Science exposure that the template was subtracted from.
440 wcs : `lsst.afw.geom.SkyWcs`
441 Coordinate system definition (wcs) for the exposure.
445 forcedSources = self.forcedMeasurement.generateMeasCat(
446 science, diaSources, wcs)
447 self.forcedMeasurement.
run(forcedSources, science, diaSources, wcs)
448 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
449 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
450 "ip_diffim_forced_PsfFlux_instFlux",
True)
451 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
452 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
453 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
454 "ip_diffim_forced_PsfFlux_area",
True)
455 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
456 "ip_diffim_forced_PsfFlux_flag",
True)
457 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
458 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
459 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
460 "ip_diffim_forced_PsfFlux_flag_edge",
True)
461 for diaSource, forcedSource
in zip(diaSources, forcedSources):
462 diaSource.assign(forcedSource, mapper)
465 """Add image QA metrics to the Task metadata.
469 difference : `lsst.afw.image.Exposure`
470 The target image to calculate metrics for.
472 mask = difference.mask
473 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
474 self.metadata.add(
"nGoodPixels", np.sum(~badPix))
475 self.metadata.add(
"nBadPixels", np.sum(badPix))
476 detPosPix = (mask.array & mask.getPlaneBitMask(
"DETECTED")) > 0
477 detNegPix = (mask.array & mask.getPlaneBitMask(
"DETECTED_NEGATIVE")) > 0
478 self.metadata.add(
"nPixelsDetectedPositive", np.sum(detPosPix))
479 self.metadata.add(
"nPixelsDetectedNegative", np.sum(detNegPix))
482 self.metadata.add(
"nBadPixelsDetectedPositive", np.sum(detPosPix))
483 self.metadata.add(
"nBadPixelsDetectedNegative", np.sum(detNegPix))
487 scoreExposure = pipeBase.connectionTypes.Input(
488 doc=
"Maximum likelihood image for detection.",
489 dimensions=(
"instrument",
"visit",
"detector"),
490 storageClass=
"ExposureF",
491 name=
"{fakesType}{coaddName}Diff_scoreExp",
496 pipelineConnections=DetectAndMeasureScoreConnections):
501 """Detect DIA sources using a score image,
502 and measure the detections on the difference image.
504 Source detection is run on the supplied score, or maximum likelihood,
505 image. Note that no additional convolution will be done in this case.
506 Close positive and negative detections will optionally be merged into
508 Sky sources, or forced detections in background regions, will optionally
509 be added, and the configured measurement algorithm will be run on all
512 ConfigClass = DetectAndMeasureScoreConfig
513 _DefaultName =
"detectAndMeasureScore"
516 def run(self, science, matchedTemplate, difference, scoreExposure,
518 """Detect and measure sources on a score image.
522 science : `lsst.afw.image.ExposureF`
523 Science exposure that the template was subtracted from.
524 matchedTemplate : `lsst.afw.image.ExposureF`
525 Warped and PSF-matched template that was used produce the
527 difference : `lsst.afw.image.ExposureF`
528 Result of subtracting template from the science image.
529 scoreExposure : `lsst.afw.image.ExposureF`
530 Score or maximum likelihood difference image
531 idFactory : `lsst.afw.table.IdFactory`, optional
532 Generator object to assign ids to detected sources in the difference image.
536 measurementResults : `lsst.pipe.base.Struct`
538 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
539 Subtracted exposure with detection mask applied.
540 ``diaSources`` : `lsst.afw.table.SourceCatalog`
541 The catalog of detected sources.
544 mask = scoreExposure.mask
545 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
547 table = afwTable.SourceTable.make(self.schema, idFactory)
548 table.setMetadata(self.algMetadata)
549 results = self.detection.
run(
551 exposure=scoreExposure,
555 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
557 return self.processResults(science, matchedTemplate, difference, results.sources, table,
558 positiveFootprints=results.positive, negativeFootprints=results.negative)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)