Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 37%
153 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-12 10:57 +0000
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from deprecated.sphinx import deprecated
24import lsst.afw.table as afwTable
25import lsst.daf.base as dafBase
26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources # noqa: F401
29import lsst.meas.extensions.shapeHSM
30from lsst.obs.base import ExposureIdInfo
31import lsst.pex.config as pexConfig
32import lsst.pipe.base as pipeBase
33import lsst.utils
34from lsst.utils.timer import timeMethod
36from . import DipoleFitTask
38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
43 dimensions=("instrument", "visit", "detector"),
44 defaultTemplates={"coaddName": "deep",
45 "warpTypeSuffix": "",
46 "fakesType": ""}):
47 science = pipeBase.connectionTypes.Input(
48 doc="Input science exposure.",
49 dimensions=("instrument", "visit", "detector"),
50 storageClass="ExposureF",
51 name="{fakesType}calexp"
52 )
53 matchedTemplate = pipeBase.connectionTypes.Input(
54 doc="Warped and PSF-matched template used to create the difference image.",
55 dimensions=("instrument", "visit", "detector"),
56 storageClass="ExposureF",
57 name="{fakesType}{coaddName}Diff_matchedExp",
58 )
59 difference = pipeBase.connectionTypes.Input(
60 doc="Result of subtracting template from science.",
61 dimensions=("instrument", "visit", "detector"),
62 storageClass="ExposureF",
63 name="{fakesType}{coaddName}Diff_differenceTempExp",
64 )
65 outputSchema = pipeBase.connectionTypes.InitOutput(
66 doc="Schema (as an example catalog) for output DIASource catalog.",
67 storageClass="SourceCatalog",
68 name="{fakesType}{coaddName}Diff_diaSrc_schema",
69 )
70 diaSources = pipeBase.connectionTypes.Output(
71 doc="Detected diaSources on the difference image.",
72 dimensions=("instrument", "visit", "detector"),
73 storageClass="SourceCatalog",
74 name="{fakesType}{coaddName}Diff_diaSrc",
75 )
76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
77 doc="Difference image with detection mask plane filled in.",
78 dimensions=("instrument", "visit", "detector"),
79 storageClass="ExposureF",
80 name="{fakesType}{coaddName}Diff_differenceExp",
81 )
84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
85 pipelineConnections=DetectAndMeasureConnections):
86 """Config for DetectAndMeasureTask
87 """
88 doMerge = pexConfig.Field(
89 dtype=bool,
90 default=True,
91 doc="Merge positive and negative diaSources with grow radius "
92 "set by growFootprint"
93 )
94 doForcedMeasurement = pexConfig.Field(
95 dtype=bool,
96 default=True,
97 doc="Force photometer diaSource locations on PVI?")
98 doAddMetrics = pexConfig.Field(
99 dtype=bool,
100 default=False,
101 doc="Add columns to the source table to hold analysis metrics?"
102 )
103 detection = pexConfig.ConfigurableField(
104 target=SourceDetectionTask,
105 doc="Final source detection for diaSource measurement",
106 )
107 measurement = pexConfig.ConfigurableField(
108 target=DipoleFitTask,
109 doc="Task to measure sources on the difference image.",
110 )
111 doApCorr = lsst.pex.config.Field(
112 dtype=bool,
113 default=True,
114 doc="Run subtask to apply aperture corrections"
115 )
116 applyApCorr = lsst.pex.config.ConfigurableField(
117 target=ApplyApCorrTask,
118 doc="Task to apply aperture corrections"
119 )
120 forcedMeasurement = pexConfig.ConfigurableField(
121 target=ForcedMeasurementTask,
122 doc="Task to force photometer science image at diaSource locations.",
123 )
124 growFootprint = pexConfig.Field(
125 dtype=int,
126 default=2,
127 doc="Grow positive and negative footprints by this many pixels before merging"
128 )
129 diaSourceMatchRadius = pexConfig.Field(
130 dtype=float,
131 default=0.5,
132 doc="Match radius (in arcseconds) for DiaSource to Source association"
133 )
134 doSkySources = pexConfig.Field(
135 dtype=bool,
136 default=False,
137 doc="Generate sky sources?",
138 )
139 skySources = pexConfig.ConfigurableField(
140 target=SkyObjectsTask,
141 doc="Generate sky sources",
142 )
143 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
145 def setDefaults(self):
146 # DiaSource Detection
147 self.detection.thresholdPolarity = "both"
148 self.detection.thresholdValue = 5.0
149 self.detection.reEstimateBackground = False
150 self.detection.thresholdType = "pixel_stdev"
151 self.detection.excludeMaskPlanes = ["EDGE"]
153 # Add filtered flux measurement, the correct measurement for pre-convolved images.
154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
155 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
156 'base_LocalPhotoCalib',
157 'base_LocalWcs',
158 'ext_shapeHSM_HsmSourceMoments',
159 'ext_shapeHSM_HsmPsfMoments',
160 ]
161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
163 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0
164 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
165 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
166 self.forcedMeasurement.copyColumns = {
167 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
168 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
169 self.forcedMeasurement.slots.shape = None
172class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
173 """Detect and measure sources on a difference image.
174 """
175 ConfigClass = DetectAndMeasureConfig
176 _DefaultName = "detectAndMeasure"
178 def __init__(self, **kwargs):
179 super().__init__(**kwargs)
180 self.schema = afwTable.SourceTable.makeMinimalSchema()
181 # Add coordinate error fields:
182 afwTable.CoordKey.addErrorFields(self.schema)
184 self.algMetadata = dafBase.PropertyList()
185 self.makeSubtask("detection", schema=self.schema)
186 self.makeSubtask("measurement", schema=self.schema,
187 algMetadata=self.algMetadata)
188 if self.config.doApCorr:
189 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
190 if self.config.doForcedMeasurement:
191 self.schema.addField(
192 "ip_diffim_forced_PsfFlux_instFlux", "D",
193 "Forced PSF flux measured on the direct image.",
194 units="count")
195 self.schema.addField(
196 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
197 "Forced PSF flux error measured on the direct image.",
198 units="count")
199 self.schema.addField(
200 "ip_diffim_forced_PsfFlux_area", "F",
201 "Forced PSF flux effective area of PSF.",
202 units="pixel")
203 self.schema.addField(
204 "ip_diffim_forced_PsfFlux_flag", "Flag",
205 "Forced PSF flux general failure flag.")
206 self.schema.addField(
207 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
208 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
209 self.schema.addField(
210 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
211 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
212 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
214 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
215 self.schema.addField("srcMatchId", "L", "unique id of source match")
216 if self.config.doSkySources:
217 self.makeSubtask("skySources")
218 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
220 # initialize InitOutputs
221 self.outputSchema = afwTable.SourceCatalog(self.schema)
222 self.outputSchema.getTable().setMetadata(self.algMetadata)
224 # TODO: remove on DM-38687.
225 @staticmethod
226 @deprecated(
227 reason=(
228 "ID factory construction now depends on configuration; use the "
229 "idGenerator config field. Will be removed after v26."
230 ),
231 version="v26.0",
232 category=FutureWarning,
233 )
234 def makeIdFactory(expId, expBits):
235 """Create IdFactory instance for unique 64 bit diaSource id-s.
237 Parameters
238 ----------
239 expId : `int`
240 Exposure id.
242 expBits: `int`
243 Number of used bits in ``expId``.
245 Notes
246 -----
247 The diasource id-s consists of the ``expId`` stored fixed in the highest value
248 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
249 low value end of the integer.
251 Returns
252 -------
253 idFactory: `lsst.afw.table.IdFactory`
254 """
255 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
257 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
258 inputRefs: pipeBase.InputQuantizedConnection,
259 outputRefs: pipeBase.OutputQuantizedConnection):
260 inputs = butlerQC.get(inputRefs)
261 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
262 idFactory = idGenerator.make_table_id_factory()
263 outputs = self.run(**inputs, idFactory=idFactory)
264 butlerQC.put(outputs, outputRefs)
266 @timeMethod
267 def run(self, science, matchedTemplate, difference,
268 idFactory=None):
269 """Detect and measure sources on a difference image.
271 The difference image will be convolved with a gaussian approximation of
272 the PSF to form a maximum likelihood image for detection.
273 Close positive and negative detections will optionally be merged into
274 dipole diaSources.
275 Sky sources, or forced detections in background regions, will optionally
276 be added, and the configured measurement algorithm will be run on all
277 detections.
279 Parameters
280 ----------
281 science : `lsst.afw.image.ExposureF`
282 Science exposure that the template was subtracted from.
283 matchedTemplate : `lsst.afw.image.ExposureF`
284 Warped and PSF-matched template that was used produce the
285 difference image.
286 difference : `lsst.afw.image.ExposureF`
287 Result of subtracting template from the science image.
288 idFactory : `lsst.afw.table.IdFactory`, optional
289 Generator object to assign ids to detected sources in the difference image.
291 Returns
292 -------
293 measurementResults : `lsst.pipe.base.Struct`
295 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
296 Subtracted exposure with detection mask applied.
297 ``diaSources`` : `lsst.afw.table.SourceCatalog`
298 The catalog of detected sources.
299 """
300 # Ensure that we start with an empty detection mask.
301 mask = difference.mask
302 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
304 table = afwTable.SourceTable.make(self.schema, idFactory)
305 table.setMetadata(self.algMetadata)
306 results = self.detection.run(
307 table=table,
308 exposure=difference,
309 doSmooth=True,
310 )
312 return self.processResults(science, matchedTemplate, difference, results.sources, table,
313 positiveFootprints=results.positive, negativeFootprints=results.negative)
315 def processResults(self, science, matchedTemplate, difference, sources, table,
316 positiveFootprints=None, negativeFootprints=None,):
317 """Measure and process the results of source detection.
319 Parameters
320 ----------
321 sources : `lsst.afw.table.SourceCatalog`
322 Detected sources on the difference exposure.
323 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
324 Positive polarity footprints.
325 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
326 Negative polarity footprints.
327 table : `lsst.afw.table.SourceTable`
328 Table object that will be used to create the SourceCatalog.
329 science : `lsst.afw.image.ExposureF`
330 Science exposure that the template was subtracted from.
331 matchedTemplate : `lsst.afw.image.ExposureF`
332 Warped and PSF-matched template that was used produce the
333 difference image.
334 difference : `lsst.afw.image.ExposureF`
335 Result of subtracting template from the science image.
337 Returns
338 -------
339 measurementResults : `lsst.pipe.base.Struct`
341 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
342 Subtracted exposure with detection mask applied.
343 ``diaSources`` : `lsst.afw.table.SourceCatalog`
344 The catalog of detected sources.
345 """
346 if self.config.doMerge:
347 fpSet = positiveFootprints
348 fpSet.merge(negativeFootprints, self.config.growFootprint,
349 self.config.growFootprint, False)
350 diaSources = afwTable.SourceCatalog(table)
351 fpSet.makeSources(diaSources)
352 self.log.info("Merging detections into %d sources", len(diaSources))
353 else:
354 diaSources = sources
356 if self.config.doSkySources:
357 self.addSkySources(diaSources, difference.mask, difference.info.id)
359 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
361 if self.config.doForcedMeasurement:
362 self.measureForcedSources(diaSources, science, difference.getWcs())
364 measurementResults = pipeBase.Struct(
365 subtractedMeasuredExposure=difference,
366 diaSources=diaSources,
367 )
369 return measurementResults
371 def addSkySources(self, diaSources, mask, seed):
372 """Add sources in empty regions of the difference image
373 for measuring the background.
375 Parameters
376 ----------
377 diaSources : `lsst.afw.table.SourceCatalog`
378 The catalog of detected sources.
379 mask : `lsst.afw.image.Mask`
380 Mask plane for determining regions where Sky sources can be added.
381 seed : `int`
382 Seed value to initialize the random number generator.
383 """
384 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
385 if skySourceFootprints:
386 for foot in skySourceFootprints:
387 s = diaSources.addNew()
388 s.setFootprint(foot)
389 s.set(self.skySourceKey, True)
391 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
392 """Use (matched) template and science image to constrain dipole fitting.
394 Parameters
395 ----------
396 diaSources : `lsst.afw.table.SourceCatalog`
397 The catalog of detected sources.
398 science : `lsst.afw.image.ExposureF`
399 Science exposure that the template was subtracted from.
400 difference : `lsst.afw.image.ExposureF`
401 Result of subtracting template from the science image.
402 matchedTemplate : `lsst.afw.image.ExposureF`
403 Warped and PSF-matched template that was used produce the
404 difference image.
405 """
406 # Note that this may not be correct if we convolved the science image.
407 # In the future we may wish to persist the matchedScience image.
408 self.measurement.run(diaSources, difference, science, matchedTemplate)
409 if self.config.doApCorr:
410 apCorrMap = difference.getInfo().getApCorrMap()
411 if apCorrMap is None:
412 self.log.warning("Difference image does not have valid aperture correction; skipping.")
413 else:
414 self.applyApCorr.run(
415 catalog=diaSources,
416 apCorrMap=apCorrMap,
417 )
419 def measureForcedSources(self, diaSources, science, wcs):
420 """Perform forced measurement of the diaSources on the science image.
422 Parameters
423 ----------
424 diaSources : `lsst.afw.table.SourceCatalog`
425 The catalog of detected sources.
426 science : `lsst.afw.image.ExposureF`
427 Science exposure that the template was subtracted from.
428 wcs : `lsst.afw.geom.SkyWcs`
429 Coordinate system definition (wcs) for the exposure.
430 """
431 # Run forced psf photometry on the PVI at the diaSource locations.
432 # Copy the measured flux and error into the diaSource.
433 forcedSources = self.forcedMeasurement.generateMeasCat(
434 science, diaSources, wcs)
435 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
436 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
437 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
438 "ip_diffim_forced_PsfFlux_instFlux", True)
439 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
440 "ip_diffim_forced_PsfFlux_instFluxErr", True)
441 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
442 "ip_diffim_forced_PsfFlux_area", True)
443 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
444 "ip_diffim_forced_PsfFlux_flag", True)
445 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
446 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
447 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
448 "ip_diffim_forced_PsfFlux_flag_edge", True)
449 for diaSource, forcedSource in zip(diaSources, forcedSources):
450 diaSource.assign(forcedSource, mapper)
453class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
454 scoreExposure = pipeBase.connectionTypes.Input(
455 doc="Maximum likelihood image for detection.",
456 dimensions=("instrument", "visit", "detector"),
457 storageClass="ExposureF",
458 name="{fakesType}{coaddName}Diff_scoreExp",
459 )
462class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
463 pipelineConnections=DetectAndMeasureScoreConnections):
464 pass
467class DetectAndMeasureScoreTask(DetectAndMeasureTask):
468 """Detect DIA sources using a score image,
469 and measure the detections on the difference image.
471 Source detection is run on the supplied score, or maximum likelihood,
472 image. Note that no additional convolution will be done in this case.
473 Close positive and negative detections will optionally be merged into
474 dipole diaSources.
475 Sky sources, or forced detections in background regions, will optionally
476 be added, and the configured measurement algorithm will be run on all
477 detections.
478 """
479 ConfigClass = DetectAndMeasureScoreConfig
480 _DefaultName = "detectAndMeasureScore"
482 @timeMethod
483 def run(self, science, matchedTemplate, difference, scoreExposure,
484 idFactory=None):
485 """Detect and measure sources on a score image.
487 Parameters
488 ----------
489 science : `lsst.afw.image.ExposureF`
490 Science exposure that the template was subtracted from.
491 matchedTemplate : `lsst.afw.image.ExposureF`
492 Warped and PSF-matched template that was used produce the
493 difference image.
494 difference : `lsst.afw.image.ExposureF`
495 Result of subtracting template from the science image.
496 scoreExposure : `lsst.afw.image.ExposureF`
497 Score or maximum likelihood difference image
498 idFactory : `lsst.afw.table.IdFactory`, optional
499 Generator object to assign ids to detected sources in the difference image.
501 Returns
502 -------
503 measurementResults : `lsst.pipe.base.Struct`
505 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
506 Subtracted exposure with detection mask applied.
507 ``diaSources`` : `lsst.afw.table.SourceCatalog`
508 The catalog of detected sources.
509 """
510 # Ensure that we start with an empty detection mask.
511 mask = scoreExposure.mask
512 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
514 table = afwTable.SourceTable.make(self.schema, idFactory)
515 table.setMetadata(self.algMetadata)
516 results = self.detection.run(
517 table=table,
518 exposure=scoreExposure,
519 doSmooth=False,
520 )
521 # Copy the detection mask from the Score image to the difference image
522 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
524 return self.processResults(science, matchedTemplate, difference, results.sources, table,
525 positiveFootprints=results.positive, negativeFootprints=results.negative)