26from lsst.meas.algorithms
import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources
29import lsst.meas.extensions.shapeHSM
33from lsst.utils.timer
import timeMethod
35from .
import DipoleFitTask
37__all__ = [
"DetectAndMeasureConfig",
"DetectAndMeasureTask",
38 "DetectAndMeasureScoreConfig",
"DetectAndMeasureScoreTask"]
42 dimensions=(
"instrument",
"visit",
"detector"),
43 defaultTemplates={
"coaddName":
"deep",
46 science = pipeBase.connectionTypes.Input(
47 doc=
"Input science exposure.",
48 dimensions=(
"instrument",
"visit",
"detector"),
49 storageClass=
"ExposureF",
50 name=
"{fakesType}calexp"
52 matchedTemplate = pipeBase.connectionTypes.Input(
53 doc=
"Warped and PSF-matched template used to create the difference image.",
54 dimensions=(
"instrument",
"visit",
"detector"),
55 storageClass=
"ExposureF",
56 name=
"{fakesType}{coaddName}Diff_matchedExp",
58 difference = pipeBase.connectionTypes.Input(
59 doc=
"Result of subtracting template from science.",
60 dimensions=(
"instrument",
"visit",
"detector"),
61 storageClass=
"ExposureF",
62 name=
"{fakesType}{coaddName}Diff_differenceTempExp",
64 outputSchema = pipeBase.connectionTypes.InitOutput(
65 doc=
"Schema (as an example catalog) for output DIASource catalog.",
66 storageClass=
"SourceCatalog",
67 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
69 diaSources = pipeBase.connectionTypes.Output(
70 doc=
"Detected diaSources on the difference image.",
71 dimensions=(
"instrument",
"visit",
"detector"),
72 storageClass=
"SourceCatalog",
73 name=
"{fakesType}{coaddName}Diff_diaSrc",
75 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
76 doc=
"Difference image with detection mask plane filled in.",
77 dimensions=(
"instrument",
"visit",
"detector"),
78 storageClass=
"ExposureF",
79 name=
"{fakesType}{coaddName}Diff_differenceExp",
84 pipelineConnections=DetectAndMeasureConnections):
85 """Config for DetectAndMeasureTask
87 doMerge = pexConfig.Field(
90 doc=
"Merge positive and negative diaSources with grow radius "
91 "set by growFootprint"
93 doForcedMeasurement = pexConfig.Field(
96 doc=
"Force photometer diaSource locations on PVI?")
97 doAddMetrics = pexConfig.Field(
100 doc=
"Add columns to the source table to hold analysis metrics?"
102 detection = pexConfig.ConfigurableField(
103 target=SourceDetectionTask,
104 doc=
"Final source detection for diaSource measurement",
106 measurement = pexConfig.ConfigurableField(
107 target=DipoleFitTask,
108 doc=
"Task to measure sources on the difference image.",
110 doApCorr = lsst.pex.config.Field(
113 doc=
"Run subtask to apply aperture corrections"
115 applyApCorr = lsst.pex.config.ConfigurableField(
116 target=ApplyApCorrTask,
117 doc=
"Task to apply aperture corrections"
119 forcedMeasurement = pexConfig.ConfigurableField(
120 target=ForcedMeasurementTask,
121 doc=
"Task to force photometer science image at diaSource locations.",
123 growFootprint = pexConfig.Field(
126 doc=
"Grow positive and negative footprints by this many pixels before merging"
128 diaSourceMatchRadius = pexConfig.Field(
131 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
133 doSkySources = pexConfig.Field(
136 doc=
"Generate sky sources?",
138 skySources = pexConfig.ConfigurableField(
139 target=SkyObjectsTask,
140 doc=
"Generate sky sources",
142 badSourceFlags = lsst.pex.config.ListField(
144 doc=
"Sources with any of these flags set are removed before writing the output catalog.",
145 default=(
"base_PixelFlags_flag_offimage",
148 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
150 def setDefaults(self):
152 self.detection.thresholdPolarity =
"both"
153 self.detection.thresholdValue = 5.0
154 self.detection.reEstimateBackground =
False
155 self.detection.thresholdType =
"pixel_stdev"
156 self.detection.excludeMaskPlanes = [
"EDGE"]
159 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
160 self.measurement.plugins.names |= [
'ext_trailedSources_Naive',
161 'base_LocalPhotoCalib',
163 'ext_shapeHSM_HsmSourceMoments',
164 'ext_shapeHSM_HsmPsfMoments',
166 self.measurement.slots.psfShape =
"ext_shapeHSM_HsmPsfMoments"
167 self.measurement.slots.shape =
"ext_shapeHSM_HsmSourceMoments"
168 self.measurement.plugins[
"base_NaiveCentroid"].maxDistToPeak = 5.0
169 self.measurement.plugins[
"base_SdssCentroid"].maxDistToPeak = 5.0
170 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
171 self.forcedMeasurement.copyColumns = {
172 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
173 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
174 self.forcedMeasurement.slots.shape =
None
177 self.measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'STREAK']
178 self.measurement.plugins[
'base_PixelFlags'].masksFpCenter = [
'STREAK']
182 """Detect and measure sources on a difference image.
184 ConfigClass = DetectAndMeasureConfig
185 _DefaultName =
"detectAndMeasure"
187 def __init__(self, **kwargs):
188 super().__init__(**kwargs)
189 self.schema = afwTable.SourceTable.makeMinimalSchema()
191 afwTable.CoordKey.addErrorFields(self.schema)
194 self.makeSubtask(
"detection", schema=self.schema)
195 self.makeSubtask(
"measurement", schema=self.schema,
196 algMetadata=self.algMetadata)
197 if self.config.doApCorr:
198 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
199 if self.config.doForcedMeasurement:
200 self.schema.addField(
201 "ip_diffim_forced_PsfFlux_instFlux",
"D",
202 "Forced PSF flux measured on the direct image.",
204 self.schema.addField(
205 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
206 "Forced PSF flux error measured on the direct image.",
208 self.schema.addField(
209 "ip_diffim_forced_PsfFlux_area",
"F",
210 "Forced PSF flux effective area of PSF.",
212 self.schema.addField(
213 "ip_diffim_forced_PsfFlux_flag",
"Flag",
214 "Forced PSF flux general failure flag.")
215 self.schema.addField(
216 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
217 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
218 self.schema.addField(
219 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
220 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
221 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
223 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
224 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
225 if self.config.doSkySources:
226 self.makeSubtask(
"skySources")
227 self.skySourceKey = self.schema.addField(
"sky_source", type=
"Flag", doc=
"Sky objects.")
230 for flag
in self.config.badSourceFlags:
231 if flag
not in self.schema:
232 raise pipeBase.InvalidQuantumError(
"Field %s not in schema" % flag)
234 self.outputSchema = afwTable.SourceCatalog(self.schema)
235 self.outputSchema.getTable().setMetadata(self.algMetadata)
237 def runQuantum(self, butlerQC: pipeBase.QuantumContext,
238 inputRefs: pipeBase.InputQuantizedConnection,
239 outputRefs: pipeBase.OutputQuantizedConnection):
240 inputs = butlerQC.get(inputRefs)
241 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
242 idFactory = idGenerator.make_table_id_factory()
243 outputs = self.run(**inputs, idFactory=idFactory)
244 butlerQC.put(outputs, outputRefs)
247 def run(self, science, matchedTemplate, difference,
249 """Detect and measure sources on a difference image.
251 The difference image will be convolved with a gaussian approximation of
252 the PSF to form a maximum likelihood image for detection.
253 Close positive and negative detections will optionally be merged into
255 Sky sources, or forced detections in background regions, will optionally
256 be added, and the configured measurement algorithm will be run on all
261 science : `lsst.afw.image.ExposureF`
262 Science exposure that the template was subtracted from.
263 matchedTemplate : `lsst.afw.image.ExposureF`
264 Warped and PSF-matched template that was used produce the
266 difference : `lsst.afw.image.ExposureF`
267 Result of subtracting template from the science image.
268 idFactory : `lsst.afw.table.IdFactory`, optional
269 Generator object to assign ids to detected sources in the difference image.
273 measurementResults : `lsst.pipe.base.Struct`
275 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
276 Subtracted exposure with detection mask applied.
277 ``diaSources`` : `lsst.afw.table.SourceCatalog`
278 The catalog of detected sources.
281 mask = difference.mask
282 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
284 table = afwTable.SourceTable.make(self.schema, idFactory)
285 table.setMetadata(self.algMetadata)
286 results = self.detection.
run(
292 return self.processResults(science, matchedTemplate, difference, results.sources, table,
293 positiveFootprints=results.positive, negativeFootprints=results.negative)
295 def processResults(self, science, matchedTemplate, difference, sources, table,
296 positiveFootprints=None, negativeFootprints=None,):
297 """Measure and process the results of source detection.
301 sources : `lsst.afw.table.SourceCatalog`
302 Detected sources on the difference exposure.
303 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
304 Positive polarity footprints.
305 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
306 Negative polarity footprints.
307 table : `lsst.afw.table.SourceTable`
308 Table object that will be used to create the SourceCatalog.
309 science : `lsst.afw.image.ExposureF`
310 Science exposure that the template was subtracted from.
311 matchedTemplate : `lsst.afw.image.ExposureF`
312 Warped and PSF-matched template that was used produce the
314 difference : `lsst.afw.image.ExposureF`
315 Result of subtracting template from the science image.
319 measurementResults : `lsst.pipe.base.Struct`
321 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
322 Subtracted exposure with detection mask applied.
323 ``diaSources`` : `lsst.afw.table.SourceCatalog`
324 The catalog of detected sources.
326 self.metadata.add(
"nUnmergedDiaSources",
len(sources))
327 if self.config.doMerge:
328 fpSet = positiveFootprints
329 fpSet.merge(negativeFootprints, self.config.growFootprint,
330 self.config.growFootprint,
False)
331 initialDiaSources = afwTable.SourceCatalog(table)
332 fpSet.makeSources(initialDiaSources)
333 self.log.
info(
"Merging detections into %d sources",
len(initialDiaSources))
335 initialDiaSources = sources
336 self.metadata.add(
"nMergedDiaSources",
len(initialDiaSources))
338 if self.config.doSkySources:
339 self.addSkySources(initialDiaSources, difference.mask, difference.info.id)
341 self.measureDiaSources(initialDiaSources, science, difference, matchedTemplate)
342 diaSources = self._removeBadSources(initialDiaSources)
344 if self.config.doForcedMeasurement:
345 self.measureForcedSources(diaSources, science, difference.getWcs())
347 measurementResults = pipeBase.Struct(
348 subtractedMeasuredExposure=difference,
349 diaSources=diaSources,
351 self.calculateMetrics(difference)
353 return measurementResults
356 """Remove bad diaSources from the catalog.
360 diaSources : `lsst.afw.table.SourceCatalog`
361 The catalog of detected sources.
365 diaSources : `lsst.afw.table.SourceCatalog`
366 The updated catalog of detected sources, with any source that has a
367 flag in ``config.badSourceFlags`` set removed.
370 selector = np.ones(
len(diaSources), dtype=bool)
371 for flag
in self.config.badSourceFlags:
372 flags = diaSources[flag]
373 nBad = np.count_nonzero(flags)
375 self.log.
info(
"Found and removed %d unphysical sources with flag %s.", nBad, flag)
378 self.metadata.add(
"nRemovedBadFlaggedSources", nBadTotal)
379 return diaSources[selector].copy(deep=
True)
382 """Add sources in empty regions of the difference image
383 for measuring the background.
387 diaSources : `lsst.afw.table.SourceCatalog`
388 The catalog of detected sources.
389 mask : `lsst.afw.image.Mask`
390 Mask plane for determining regions where Sky sources can be added.
392 Seed value to initialize the random number generator.
394 skySourceFootprints = self.skySources.
run(mask=mask, seed=seed)
395 if skySourceFootprints:
396 for foot
in skySourceFootprints:
397 s = diaSources.addNew()
399 s.set(self.skySourceKey,
True)
402 """Use (matched) template and science image to constrain dipole fitting.
406 diaSources : `lsst.afw.table.SourceCatalog`
407 The catalog of detected sources.
408 science : `lsst.afw.image.ExposureF`
409 Science exposure that the template was subtracted from.
410 difference : `lsst.afw.image.ExposureF`
411 Result of subtracting template from the science image.
412 matchedTemplate : `lsst.afw.image.ExposureF`
413 Warped and PSF-matched template that was used produce the
418 self.measurement.
run(diaSources, difference, science, matchedTemplate)
419 if self.config.doApCorr:
420 apCorrMap = difference.getInfo().getApCorrMap()
421 if apCorrMap
is None:
422 self.log.
warning(
"Difference image does not have valid aperture correction; skipping.")
424 self.applyApCorr.
run(
430 """Perform forced measurement of the diaSources on the science image.
434 diaSources : `lsst.afw.table.SourceCatalog`
435 The catalog of detected sources.
436 science : `lsst.afw.image.ExposureF`
437 Science exposure that the template was subtracted from.
438 wcs : `lsst.afw.geom.SkyWcs`
439 Coordinate system definition (wcs) for the exposure.
443 forcedSources = self.forcedMeasurement.generateMeasCat(
444 science, diaSources, wcs)
445 self.forcedMeasurement.
run(forcedSources, science, diaSources, wcs)
446 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
447 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
448 "ip_diffim_forced_PsfFlux_instFlux",
True)
449 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
450 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
451 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
452 "ip_diffim_forced_PsfFlux_area",
True)
453 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
454 "ip_diffim_forced_PsfFlux_flag",
True)
455 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
456 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
457 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
458 "ip_diffim_forced_PsfFlux_flag_edge",
True)
459 for diaSource, forcedSource
in zip(diaSources, forcedSources):
460 diaSource.assign(forcedSource, mapper)
463 """Add image QA metrics to the Task metadata.
467 difference : `lsst.afw.image.Exposure`
468 The target image to calculate metrics for.
470 mask = difference.mask
471 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
472 self.metadata.add(
"nGoodPixels", np.sum(~badPix))
473 self.metadata.add(
"nBadPixels", np.sum(badPix))
474 detPosPix = (mask.array & mask.getPlaneBitMask(
"DETECTED")) > 0
475 detNegPix = (mask.array & mask.getPlaneBitMask(
"DETECTED_NEGATIVE")) > 0
476 self.metadata.add(
"nPixelsDetectedPositive", np.sum(detPosPix))
477 self.metadata.add(
"nPixelsDetectedNegative", np.sum(detNegPix))
480 self.metadata.add(
"nBadPixelsDetectedPositive", np.sum(detPosPix))
481 self.metadata.add(
"nBadPixelsDetectedNegative", np.sum(detNegPix))
485 scoreExposure = pipeBase.connectionTypes.Input(
486 doc=
"Maximum likelihood image for detection.",
487 dimensions=(
"instrument",
"visit",
"detector"),
488 storageClass=
"ExposureF",
489 name=
"{fakesType}{coaddName}Diff_scoreExp",
494 pipelineConnections=DetectAndMeasureScoreConnections):
499 """Detect DIA sources using a score image,
500 and measure the detections on the difference image.
502 Source detection is run on the supplied score, or maximum likelihood,
503 image. Note that no additional convolution will be done in this case.
504 Close positive and negative detections will optionally be merged into
506 Sky sources, or forced detections in background regions, will optionally
507 be added, and the configured measurement algorithm will be run on all
510 ConfigClass = DetectAndMeasureScoreConfig
511 _DefaultName =
"detectAndMeasureScore"
514 def run(self, science, matchedTemplate, difference, scoreExposure,
516 """Detect and measure sources on a score image.
520 science : `lsst.afw.image.ExposureF`
521 Science exposure that the template was subtracted from.
522 matchedTemplate : `lsst.afw.image.ExposureF`
523 Warped and PSF-matched template that was used produce the
525 difference : `lsst.afw.image.ExposureF`
526 Result of subtracting template from the science image.
527 scoreExposure : `lsst.afw.image.ExposureF`
528 Score or maximum likelihood difference image
529 idFactory : `lsst.afw.table.IdFactory`, optional
530 Generator object to assign ids to detected sources in the difference image.
534 measurementResults : `lsst.pipe.base.Struct`
536 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
537 Subtracted exposure with detection mask applied.
538 ``diaSources`` : `lsst.afw.table.SourceCatalog`
539 The catalog of detected sources.
542 mask = scoreExposure.mask
543 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
545 table = afwTable.SourceTable.make(self.schema, idFactory)
546 table.setMetadata(self.algMetadata)
547 results = self.detection.
run(
549 exposure=scoreExposure,
553 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
555 return self.processResults(science, matchedTemplate, difference, results.sources, table,
556 positiveFootprints=results.positive, negativeFootprints=results.negative)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)