22from deprecated.sphinx
import deprecated
27from lsst.meas.algorithms
import SkyObjectsTask, SourceDetectionTask
28from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
29import lsst.meas.extensions.trailedSources
30import lsst.meas.extensions.shapeHSM
31from lsst.obs.base
import ExposureIdInfo
35from lsst.utils.timer
import timeMethod
37from .
import DipoleFitTask
39__all__ = [
"DetectAndMeasureConfig",
"DetectAndMeasureTask",
40 "DetectAndMeasureScoreConfig",
"DetectAndMeasureScoreTask"]
44 dimensions=(
"instrument",
"visit",
"detector"),
45 defaultTemplates={
"coaddName":
"deep",
48 science = pipeBase.connectionTypes.Input(
49 doc=
"Input science exposure.",
50 dimensions=(
"instrument",
"visit",
"detector"),
51 storageClass=
"ExposureF",
52 name=
"{fakesType}calexp"
54 matchedTemplate = pipeBase.connectionTypes.Input(
55 doc=
"Warped and PSF-matched template used to create the difference image.",
56 dimensions=(
"instrument",
"visit",
"detector"),
57 storageClass=
"ExposureF",
58 name=
"{fakesType}{coaddName}Diff_matchedExp",
60 difference = pipeBase.connectionTypes.Input(
61 doc=
"Result of subtracting template from science.",
62 dimensions=(
"instrument",
"visit",
"detector"),
63 storageClass=
"ExposureF",
64 name=
"{fakesType}{coaddName}Diff_differenceTempExp",
66 outputSchema = pipeBase.connectionTypes.InitOutput(
67 doc=
"Schema (as an example catalog) for output DIASource catalog.",
68 storageClass=
"SourceCatalog",
69 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
71 diaSources = pipeBase.connectionTypes.Output(
72 doc=
"Detected diaSources on the difference image.",
73 dimensions=(
"instrument",
"visit",
"detector"),
74 storageClass=
"SourceCatalog",
75 name=
"{fakesType}{coaddName}Diff_diaSrc",
77 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
78 doc=
"Difference image with detection mask plane filled in.",
79 dimensions=(
"instrument",
"visit",
"detector"),
80 storageClass=
"ExposureF",
81 name=
"{fakesType}{coaddName}Diff_differenceExp",
86 pipelineConnections=DetectAndMeasureConnections):
87 """Config for DetectAndMeasureTask
89 doMerge = pexConfig.Field(
92 doc=
"Merge positive and negative diaSources with grow radius "
93 "set by growFootprint"
95 doForcedMeasurement = pexConfig.Field(
98 doc=
"Force photometer diaSource locations on PVI?")
99 doAddMetrics = pexConfig.Field(
102 doc=
"Add columns to the source table to hold analysis metrics?"
104 detection = pexConfig.ConfigurableField(
105 target=SourceDetectionTask,
106 doc=
"Final source detection for diaSource measurement",
108 measurement = pexConfig.ConfigurableField(
109 target=DipoleFitTask,
110 doc=
"Task to measure sources on the difference image.",
112 doApCorr = lsst.pex.config.Field(
115 doc=
"Run subtask to apply aperture corrections"
117 applyApCorr = lsst.pex.config.ConfigurableField(
118 target=ApplyApCorrTask,
119 doc=
"Task to apply aperture corrections"
121 forcedMeasurement = pexConfig.ConfigurableField(
122 target=ForcedMeasurementTask,
123 doc=
"Task to force photometer science image at diaSource locations.",
125 growFootprint = pexConfig.Field(
128 doc=
"Grow positive and negative footprints by this many pixels before merging"
130 diaSourceMatchRadius = pexConfig.Field(
133 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
135 doSkySources = pexConfig.Field(
138 doc=
"Generate sky sources?",
140 skySources = pexConfig.ConfigurableField(
141 target=SkyObjectsTask,
142 doc=
"Generate sky sources",
144 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
146 def setDefaults(self):
148 self.detection.thresholdPolarity =
"both"
149 self.detection.thresholdValue = 5.0
150 self.detection.reEstimateBackground =
False
151 self.detection.thresholdType =
"pixel_stdev"
152 self.detection.excludeMaskPlanes = [
"EDGE"]
155 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
156 self.measurement.plugins.names |= [
'ext_trailedSources_Naive',
157 'base_LocalPhotoCalib',
159 'ext_shapeHSM_HsmSourceMoments',
160 'ext_shapeHSM_HsmPsfMoments',
162 self.measurement.slots.psfShape =
"ext_shapeHSM_HsmPsfMoments"
163 self.measurement.slots.shape =
"ext_shapeHSM_HsmSourceMoments"
164 self.measurement.plugins[
"base_NaiveCentroid"].maxDistToPeak = 5.0
165 self.measurement.plugins[
"base_SdssCentroid"].maxDistToPeak = 5.0
166 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
167 self.forcedMeasurement.copyColumns = {
168 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
169 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
170 self.forcedMeasurement.slots.shape =
None
173 self.measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'STREAK']
174 self.measurement.plugins[
'base_PixelFlags'].masksFpCenter = [
'STREAK']
178 """Detect and measure sources on a difference image.
180 ConfigClass = DetectAndMeasureConfig
181 _DefaultName =
"detectAndMeasure"
183 def __init__(self, **kwargs):
184 super().__init__(**kwargs)
185 self.schema = afwTable.SourceTable.makeMinimalSchema()
187 afwTable.CoordKey.addErrorFields(self.schema)
190 self.makeSubtask(
"detection", schema=self.schema)
191 self.makeSubtask(
"measurement", schema=self.schema,
192 algMetadata=self.algMetadata)
193 if self.config.doApCorr:
194 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
195 if self.config.doForcedMeasurement:
196 self.schema.addField(
197 "ip_diffim_forced_PsfFlux_instFlux",
"D",
198 "Forced PSF flux measured on the direct image.",
200 self.schema.addField(
201 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
202 "Forced PSF flux error measured on the direct image.",
204 self.schema.addField(
205 "ip_diffim_forced_PsfFlux_area",
"F",
206 "Forced PSF flux effective area of PSF.",
208 self.schema.addField(
209 "ip_diffim_forced_PsfFlux_flag",
"Flag",
210 "Forced PSF flux general failure flag.")
211 self.schema.addField(
212 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
213 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
214 self.schema.addField(
215 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
216 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
217 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
219 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
220 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
221 if self.config.doSkySources:
222 self.makeSubtask(
"skySources")
223 self.skySourceKey = self.schema.addField(
"sky_source", type=
"Flag", doc=
"Sky objects.")
226 self.outputSchema = afwTable.SourceCatalog(self.schema)
227 self.outputSchema.getTable().setMetadata(self.algMetadata)
233 "ID factory construction now depends on configuration; use the "
234 "idGenerator config field. Will be removed after v26."
237 category=FutureWarning,
240 """Create IdFactory instance for unique 64 bit diaSource id-s.
248 Number of used bits in ``expId``.
252 The diasource id-s consists of the ``expId`` stored fixed in the highest value
253 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
254 low value end of the integer.
258 idFactory: `lsst.afw.table.IdFactory`
262 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
263 inputRefs: pipeBase.InputQuantizedConnection,
264 outputRefs: pipeBase.OutputQuantizedConnection):
265 inputs = butlerQC.get(inputRefs)
266 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
267 idFactory = idGenerator.make_table_id_factory()
268 outputs = self.run(**inputs, idFactory=idFactory)
269 butlerQC.put(outputs, outputRefs)
272 def run(self, science, matchedTemplate, difference,
274 """Detect and measure sources on a difference image.
276 The difference image will be convolved with a gaussian approximation of
277 the PSF to form a maximum likelihood image for detection.
278 Close positive and negative detections will optionally be merged into
280 Sky sources, or forced detections in background regions, will optionally
281 be added, and the configured measurement algorithm will be run on all
286 science : `lsst.afw.image.ExposureF`
287 Science exposure that the template was subtracted from.
288 matchedTemplate : `lsst.afw.image.ExposureF`
289 Warped and PSF-matched template that was used produce the
291 difference : `lsst.afw.image.ExposureF`
292 Result of subtracting template from the science image.
293 idFactory : `lsst.afw.table.IdFactory`, optional
294 Generator object to assign ids to detected sources in the difference image.
298 measurementResults : `lsst.pipe.base.Struct`
300 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
301 Subtracted exposure with detection mask applied.
302 ``diaSources`` : `lsst.afw.table.SourceCatalog`
303 The catalog of detected sources.
306 mask = difference.mask
307 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
309 table = afwTable.SourceTable.make(self.schema, idFactory)
310 table.setMetadata(self.algMetadata)
311 results = self.detection.
run(
317 return self.processResults(science, matchedTemplate, difference, results.sources, table,
318 positiveFootprints=results.positive, negativeFootprints=results.negative)
320 def processResults(self, science, matchedTemplate, difference, sources, table,
321 positiveFootprints=None, negativeFootprints=None,):
322 """Measure and process the results of source detection.
326 sources : `lsst.afw.table.SourceCatalog`
327 Detected sources on the difference exposure.
328 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
329 Positive polarity footprints.
330 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
331 Negative polarity footprints.
332 table : `lsst.afw.table.SourceTable`
333 Table object that will be used to create the SourceCatalog.
334 science : `lsst.afw.image.ExposureF`
335 Science exposure that the template was subtracted from.
336 matchedTemplate : `lsst.afw.image.ExposureF`
337 Warped and PSF-matched template that was used produce the
339 difference : `lsst.afw.image.ExposureF`
340 Result of subtracting template from the science image.
344 measurementResults : `lsst.pipe.base.Struct`
346 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
347 Subtracted exposure with detection mask applied.
348 ``diaSources`` : `lsst.afw.table.SourceCatalog`
349 The catalog of detected sources.
351 self.metadata.add(
"nUnmergedDiaSources",
len(sources))
352 if self.config.doMerge:
353 fpSet = positiveFootprints
354 fpSet.merge(negativeFootprints, self.config.growFootprint,
355 self.config.growFootprint,
False)
356 diaSources = afwTable.SourceCatalog(table)
357 fpSet.makeSources(diaSources)
358 self.log.
info(
"Merging detections into %d sources",
len(diaSources))
361 self.metadata.add(
"nMergedDiaSources",
len(diaSources))
363 if self.config.doSkySources:
364 self.addSkySources(diaSources, difference.mask, difference.info.id)
366 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
368 if self.config.doForcedMeasurement:
369 self.measureForcedSources(diaSources, science, difference.getWcs())
371 measurementResults = pipeBase.Struct(
372 subtractedMeasuredExposure=difference,
373 diaSources=diaSources,
375 self.calculateMetrics(difference)
377 return measurementResults
380 """Add sources in empty regions of the difference image
381 for measuring the background.
385 diaSources : `lsst.afw.table.SourceCatalog`
386 The catalog of detected sources.
387 mask : `lsst.afw.image.Mask`
388 Mask plane for determining regions where Sky sources can be added.
390 Seed value to initialize the random number generator.
392 skySourceFootprints = self.skySources.
run(mask=mask, seed=seed)
393 if skySourceFootprints:
394 for foot
in skySourceFootprints:
395 s = diaSources.addNew()
397 s.set(self.skySourceKey,
True)
400 """Use (matched) template and science image to constrain dipole fitting.
404 diaSources : `lsst.afw.table.SourceCatalog`
405 The catalog of detected sources.
406 science : `lsst.afw.image.ExposureF`
407 Science exposure that the template was subtracted from.
408 difference : `lsst.afw.image.ExposureF`
409 Result of subtracting template from the science image.
410 matchedTemplate : `lsst.afw.image.ExposureF`
411 Warped and PSF-matched template that was used produce the
416 self.measurement.
run(diaSources, difference, science, matchedTemplate)
417 if self.config.doApCorr:
418 apCorrMap = difference.getInfo().getApCorrMap()
419 if apCorrMap
is None:
420 self.log.
warning(
"Difference image does not have valid aperture correction; skipping.")
422 self.applyApCorr.
run(
428 """Perform forced measurement of the diaSources on the science image.
432 diaSources : `lsst.afw.table.SourceCatalog`
433 The catalog of detected sources.
434 science : `lsst.afw.image.ExposureF`
435 Science exposure that the template was subtracted from.
436 wcs : `lsst.afw.geom.SkyWcs`
437 Coordinate system definition (wcs) for the exposure.
441 forcedSources = self.forcedMeasurement.generateMeasCat(
442 science, diaSources, wcs)
443 self.forcedMeasurement.
run(forcedSources, science, diaSources, wcs)
444 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
445 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
446 "ip_diffim_forced_PsfFlux_instFlux",
True)
447 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
448 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
449 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
450 "ip_diffim_forced_PsfFlux_area",
True)
451 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
452 "ip_diffim_forced_PsfFlux_flag",
True)
453 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
454 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
455 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
456 "ip_diffim_forced_PsfFlux_flag_edge",
True)
457 for diaSource, forcedSource
in zip(diaSources, forcedSources):
458 diaSource.assign(forcedSource, mapper)
461 """Add image QA metrics to the Task metadata.
465 difference : `lsst.afw.image.Exposure`
466 The target image to calculate metrics for.
468 mask = difference.mask
469 badPix = (mask.array & mask.getPlaneBitMask(self.config.detection.excludeMaskPlanes)) > 0
470 self.metadata.add(
"nGoodPixels", np.sum(~badPix))
471 self.metadata.add(
"nBadPixels", np.sum(badPix))
472 detPosPix = (mask.array & mask.getPlaneBitMask(
"DETECTED")) > 0
473 detNegPix = (mask.array & mask.getPlaneBitMask(
"DETECTED_NEGATIVE")) > 0
474 self.metadata.add(
"nPixelsDetectedPositive", np.sum(detPosPix))
475 self.metadata.add(
"nPixelsDetectedNegative", np.sum(detNegPix))
478 self.metadata.add(
"nBadPixelsDetectedPositive", np.sum(detPosPix))
479 self.metadata.add(
"nBadPixelsDetectedNegative", np.sum(detNegPix))
483 scoreExposure = pipeBase.connectionTypes.Input(
484 doc=
"Maximum likelihood image for detection.",
485 dimensions=(
"instrument",
"visit",
"detector"),
486 storageClass=
"ExposureF",
487 name=
"{fakesType}{coaddName}Diff_scoreExp",
492 pipelineConnections=DetectAndMeasureScoreConnections):
497 """Detect DIA sources using a score image,
498 and measure the detections on the difference image.
500 Source detection is run on the supplied score, or maximum likelihood,
501 image. Note that no additional convolution will be done in this case.
502 Close positive and negative detections will optionally be merged into
504 Sky sources, or forced detections in background regions, will optionally
505 be added, and the configured measurement algorithm will be run on all
508 ConfigClass = DetectAndMeasureScoreConfig
509 _DefaultName =
"detectAndMeasureScore"
512 def run(self, science, matchedTemplate, difference, scoreExposure,
514 """Detect and measure sources on a score image.
518 science : `lsst.afw.image.ExposureF`
519 Science exposure that the template was subtracted from.
520 matchedTemplate : `lsst.afw.image.ExposureF`
521 Warped and PSF-matched template that was used produce the
523 difference : `lsst.afw.image.ExposureF`
524 Result of subtracting template from the science image.
525 scoreExposure : `lsst.afw.image.ExposureF`
526 Score or maximum likelihood difference image
527 idFactory : `lsst.afw.table.IdFactory`, optional
528 Generator object to assign ids to detected sources in the difference image.
532 measurementResults : `lsst.pipe.base.Struct`
534 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
535 Subtracted exposure with detection mask applied.
536 ``diaSources`` : `lsst.afw.table.SourceCatalog`
537 The catalog of detected sources.
540 mask = scoreExposure.mask
541 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
543 table = afwTable.SourceTable.make(self.schema, idFactory)
544 table.setMetadata(self.algMetadata)
545 results = self.detection.
run(
547 exposure=scoreExposure,
551 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
553 return self.processResults(science, matchedTemplate, difference, results.sources, table,
554 positiveFootprints=results.positive, negativeFootprints=results.negative)
Asseses the quality of a candidate given a spatial kernel and background model.
run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)