Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 34%
144 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-10 11:28 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-10 11:28 +0000
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import lsst.afw.table as afwTable
23import lsst.daf.base as dafBase
24from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
25from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
26import lsst.meas.extensions.trailedSources # noqa: F401
27import lsst.meas.extensions.shapeHSM
28from lsst.obs.base import ExposureIdInfo
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31import lsst.utils
32from lsst.utils.timer import timeMethod
34from . import DipoleFitTask
36__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
37 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
40class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
41 dimensions=("instrument", "visit", "detector"),
42 defaultTemplates={"coaddName": "deep",
43 "warpTypeSuffix": "",
44 "fakesType": ""}):
45 science = pipeBase.connectionTypes.Input(
46 doc="Input science exposure.",
47 dimensions=("instrument", "visit", "detector"),
48 storageClass="ExposureF",
49 name="{fakesType}calexp"
50 )
51 matchedTemplate = pipeBase.connectionTypes.Input(
52 doc="Warped and PSF-matched template used to create the difference image.",
53 dimensions=("instrument", "visit", "detector"),
54 storageClass="ExposureF",
55 name="{fakesType}{coaddName}Diff_matchedExp",
56 )
57 difference = pipeBase.connectionTypes.Input(
58 doc="Result of subtracting template from science.",
59 dimensions=("instrument", "visit", "detector"),
60 storageClass="ExposureF",
61 name="{fakesType}{coaddName}Diff_differenceTempExp",
62 )
63 outputSchema = pipeBase.connectionTypes.InitOutput(
64 doc="Schema (as an example catalog) for output DIASource catalog.",
65 storageClass="SourceCatalog",
66 name="{fakesType}{coaddName}Diff_diaSrc_schema",
67 )
68 diaSources = pipeBase.connectionTypes.Output(
69 doc="Detected diaSources on the difference image.",
70 dimensions=("instrument", "visit", "detector"),
71 storageClass="SourceCatalog",
72 name="{fakesType}{coaddName}Diff_diaSrc",
73 )
74 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
75 doc="Difference image with detection mask plane filled in.",
76 dimensions=("instrument", "visit", "detector"),
77 storageClass="ExposureF",
78 name="{fakesType}{coaddName}Diff_differenceExp",
79 )
82class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
83 pipelineConnections=DetectAndMeasureConnections):
84 """Config for DetectAndMeasureTask
85 """
86 doMerge = pexConfig.Field(
87 dtype=bool,
88 default=True,
89 doc="Merge positive and negative diaSources with grow radius "
90 "set by growFootprint"
91 )
92 doForcedMeasurement = pexConfig.Field(
93 dtype=bool,
94 default=True,
95 doc="Force photometer diaSource locations on PVI?")
96 doAddMetrics = pexConfig.Field(
97 dtype=bool,
98 default=False,
99 doc="Add columns to the source table to hold analysis metrics?"
100 )
101 detection = pexConfig.ConfigurableField(
102 target=SourceDetectionTask,
103 doc="Final source detection for diaSource measurement",
104 )
105 measurement = pexConfig.ConfigurableField(
106 target=DipoleFitTask,
107 doc="Task to measure sources on the difference image.",
108 )
109 doApCorr = lsst.pex.config.Field(
110 dtype=bool,
111 default=True,
112 doc="Run subtask to apply aperture corrections"
113 )
114 applyApCorr = lsst.pex.config.ConfigurableField(
115 target=ApplyApCorrTask,
116 doc="Task to apply aperture corrections"
117 )
118 forcedMeasurement = pexConfig.ConfigurableField(
119 target=ForcedMeasurementTask,
120 doc="Task to force photometer science image at diaSource locations.",
121 )
122 growFootprint = pexConfig.Field(
123 dtype=int,
124 default=2,
125 doc="Grow positive and negative footprints by this many pixels before merging"
126 )
127 diaSourceMatchRadius = pexConfig.Field(
128 dtype=float,
129 default=0.5,
130 doc="Match radius (in arcseconds) for DiaSource to Source association"
131 )
132 doSkySources = pexConfig.Field(
133 dtype=bool,
134 default=False,
135 doc="Generate sky sources?",
136 )
137 skySources = pexConfig.ConfigurableField(
138 target=SkyObjectsTask,
139 doc="Generate sky sources",
140 )
142 def setDefaults(self):
143 # DiaSource Detection
144 self.detection.thresholdPolarity = "both"
145 self.detection.thresholdValue = 5.0
146 self.detection.reEstimateBackground = False
147 self.detection.thresholdType = "pixel_stdev"
149 # Add filtered flux measurement, the correct measurement for pre-convolved images.
150 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
151 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
152 'base_LocalPhotoCalib',
153 'base_LocalWcs',
154 'ext_shapeHSM_HsmSourceMoments',
155 'ext_shapeHSM_HsmPsfMoments',
156 ]
157 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
158 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
160 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
161 self.forcedMeasurement.copyColumns = {
162 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
163 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
164 self.forcedMeasurement.slots.shape = None
167class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
168 """Detect and measure sources on a difference image.
169 """
170 ConfigClass = DetectAndMeasureConfig
171 _DefaultName = "detectAndMeasure"
173 def __init__(self, **kwargs):
174 super().__init__(**kwargs)
175 self.schema = afwTable.SourceTable.makeMinimalSchema()
177 self.algMetadata = dafBase.PropertyList()
178 self.makeSubtask("detection", schema=self.schema)
179 self.makeSubtask("measurement", schema=self.schema,
180 algMetadata=self.algMetadata)
181 if self.config.doApCorr:
182 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
183 if self.config.doForcedMeasurement:
184 self.schema.addField(
185 "ip_diffim_forced_PsfFlux_instFlux", "D",
186 "Forced PSF flux measured on the direct image.",
187 units="count")
188 self.schema.addField(
189 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
190 "Forced PSF flux error measured on the direct image.",
191 units="count")
192 self.schema.addField(
193 "ip_diffim_forced_PsfFlux_area", "F",
194 "Forced PSF flux effective area of PSF.",
195 units="pixel")
196 self.schema.addField(
197 "ip_diffim_forced_PsfFlux_flag", "Flag",
198 "Forced PSF flux general failure flag.")
199 self.schema.addField(
200 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
201 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
202 self.schema.addField(
203 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
204 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
205 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
207 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
208 self.schema.addField("srcMatchId", "L", "unique id of source match")
209 if self.config.doSkySources:
210 self.makeSubtask("skySources")
211 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
213 # initialize InitOutputs
214 self.outputSchema = afwTable.SourceCatalog(self.schema)
215 self.outputSchema.getTable().setMetadata(self.algMetadata)
217 @staticmethod
218 def makeIdFactory(expId, expBits):
219 """Create IdFactory instance for unique 64 bit diaSource id-s.
221 Parameters
222 ----------
223 expId : `int`
224 Exposure id.
226 expBits: `int`
227 Number of used bits in ``expId``.
229 Notes
230 -----
231 The diasource id-s consists of the ``expId`` stored fixed in the highest value
232 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
233 low value end of the integer.
235 Returns
236 -------
237 idFactory: `lsst.afw.table.IdFactory`
238 """
239 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
241 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
242 inputRefs: pipeBase.InputQuantizedConnection,
243 outputRefs: pipeBase.OutputQuantizedConnection):
244 inputs = butlerQC.get(inputRefs)
245 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
246 returnMaxBits=True)
247 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
249 outputs = self.run(**inputs, idFactory=idFactory)
250 butlerQC.put(outputs, outputRefs)
252 @timeMethod
253 def run(self, science, matchedTemplate, difference,
254 idFactory=None):
255 """Detect and measure sources on a difference image.
257 The difference image will be convolved with a gaussian approximation of
258 the PSF to form a maximum likelihood image for detection.
259 Close positive and negative detections will optionally be merged into
260 dipole diaSources.
261 Sky sources, or forced detections in background regions, will optionally
262 be added, and the configured measurement algorithm will be run on all
263 detections.
265 Parameters
266 ----------
267 science : `lsst.afw.image.ExposureF`
268 Science exposure that the template was subtracted from.
269 matchedTemplate : `lsst.afw.image.ExposureF`
270 Warped and PSF-matched template that was used produce the
271 difference image.
272 difference : `lsst.afw.image.ExposureF`
273 Result of subtracting template from the science image.
274 idFactory : `lsst.afw.table.IdFactory`, optional
275 Generator object to assign ids to detected sources in the difference image.
277 Returns
278 -------
279 measurementResults : `lsst.pipe.base.Struct`
281 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
282 Subtracted exposure with detection mask applied.
283 ``diaSources`` : `lsst.afw.table.SourceCatalog`
284 The catalog of detected sources.
285 """
286 # Ensure that we start with an empty detection mask.
287 mask = difference.mask
288 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
290 table = afwTable.SourceTable.make(self.schema, idFactory)
291 table.setMetadata(self.algMetadata)
292 results = self.detection.run(
293 table=table,
294 exposure=difference,
295 doSmooth=True,
296 )
298 return self.processResults(science, matchedTemplate, difference, results.sources, table,
299 positiveFootprints=results.positive, negativeFootprints=results.negative)
301 def processResults(self, science, matchedTemplate, difference, sources, table,
302 positiveFootprints=None, negativeFootprints=None,):
303 """Measure and process the results of source detection.
305 Parameters
306 ----------
307 sources : `lsst.afw.table.SourceCatalog`
308 Detected sources on the difference exposure.
309 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
310 Positive polarity footprints.
311 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
312 Negative polarity footprints.
313 table : `lsst.afw.table.SourceTable`
314 Table object that will be used to create the SourceCatalog.
315 science : `lsst.afw.image.ExposureF`
316 Science exposure that the template was subtracted from.
317 matchedTemplate : `lsst.afw.image.ExposureF`
318 Warped and PSF-matched template that was used produce the
319 difference image.
320 difference : `lsst.afw.image.ExposureF`
321 Result of subtracting template from the science image.
323 Returns
324 -------
325 measurementResults : `lsst.pipe.base.Struct`
327 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
328 Subtracted exposure with detection mask applied.
329 ``diaSources`` : `lsst.afw.table.SourceCatalog`
330 The catalog of detected sources.
331 """
332 if self.config.doMerge:
333 fpSet = positiveFootprints
334 fpSet.merge(negativeFootprints, self.config.growFootprint,
335 self.config.growFootprint, False)
336 diaSources = afwTable.SourceCatalog(table)
337 fpSet.makeSources(diaSources)
338 self.log.info("Merging detections into %d sources", len(diaSources))
339 else:
340 diaSources = sources
342 if self.config.doSkySources:
343 self.addSkySources(diaSources, difference.mask, difference.info.id)
345 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
347 if self.config.doForcedMeasurement:
348 self.measureForcedSources(diaSources, science, difference.getWcs())
350 measurementResults = pipeBase.Struct(
351 subtractedMeasuredExposure=difference,
352 diaSources=diaSources,
353 )
355 return measurementResults
357 def addSkySources(self, diaSources, mask, seed):
358 """Add sources in empty regions of the difference image
359 for measuring the background.
361 Parameters
362 ----------
363 diaSources : `lsst.afw.table.SourceCatalog`
364 The catalog of detected sources.
365 mask : `lsst.afw.image.Mask`
366 Mask plane for determining regions where Sky sources can be added.
367 seed : `int`
368 Seed value to initialize the random number generator.
369 """
370 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
371 if skySourceFootprints:
372 for foot in skySourceFootprints:
373 s = diaSources.addNew()
374 s.setFootprint(foot)
375 s.set(self.skySourceKey, True)
377 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
378 """Use (matched) template and science image to constrain dipole fitting.
380 Parameters
381 ----------
382 diaSources : `lsst.afw.table.SourceCatalog`
383 The catalog of detected sources.
384 science : `lsst.afw.image.ExposureF`
385 Science exposure that the template was subtracted from.
386 difference : `lsst.afw.image.ExposureF`
387 Result of subtracting template from the science image.
388 matchedTemplate : `lsst.afw.image.ExposureF`
389 Warped and PSF-matched template that was used produce the
390 difference image.
391 """
392 # Note that this may not be correct if we convolved the science image.
393 # In the future we may wish to persist the matchedScience image.
394 self.measurement.run(diaSources, difference, science, matchedTemplate)
395 if self.config.doApCorr:
396 self.applyApCorr.run(
397 catalog=diaSources,
398 apCorrMap=difference.getInfo().getApCorrMap()
399 )
401 def measureForcedSources(self, diaSources, science, wcs):
402 """Perform forced measurement of the diaSources on the science image.
404 Parameters
405 ----------
406 diaSources : `lsst.afw.table.SourceCatalog`
407 The catalog of detected sources.
408 science : `lsst.afw.image.ExposureF`
409 Science exposure that the template was subtracted from.
410 wcs : `lsst.afw.geom.SkyWcs`
411 Coordinate system definition (wcs) for the exposure.
412 """
413 # Run forced psf photometry on the PVI at the diaSource locations.
414 # Copy the measured flux and error into the diaSource.
415 forcedSources = self.forcedMeasurement.generateMeasCat(
416 science, diaSources, wcs)
417 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
418 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
419 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
420 "ip_diffim_forced_PsfFlux_instFlux", True)
421 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
422 "ip_diffim_forced_PsfFlux_instFluxErr", True)
423 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
424 "ip_diffim_forced_PsfFlux_area", True)
425 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
426 "ip_diffim_forced_PsfFlux_flag", True)
427 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
428 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
429 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
430 "ip_diffim_forced_PsfFlux_flag_edge", True)
431 for diaSource, forcedSource in zip(diaSources, forcedSources):
432 diaSource.assign(forcedSource, mapper)
435class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
436 scoreExposure = pipeBase.connectionTypes.Input(
437 doc="Maximum likelihood image for detection.",
438 dimensions=("instrument", "visit", "detector"),
439 storageClass="ExposureF",
440 name="{fakesType}{coaddName}Diff_scoreExp",
441 )
444class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
445 pipelineConnections=DetectAndMeasureScoreConnections):
446 pass
449class DetectAndMeasureScoreTask(DetectAndMeasureTask):
450 """Detect DIA sources using a score image,
451 and measure the detections on the difference image.
453 Source detection is run on the supplied score, or maximum likelihood,
454 image. Note that no additional convolution will be done in this case.
455 Close positive and negative detections will optionally be merged into
456 dipole diaSources.
457 Sky sources, or forced detections in background regions, will optionally
458 be added, and the configured measurement algorithm will be run on all
459 detections.
460 """
461 ConfigClass = DetectAndMeasureScoreConfig
462 _DefaultName = "detectAndMeasureScore"
464 @timeMethod
465 def run(self, science, matchedTemplate, difference, scoreExposure,
466 idFactory=None):
467 """Detect and measure sources on a score image.
469 Parameters
470 ----------
471 science : `lsst.afw.image.ExposureF`
472 Science exposure that the template was subtracted from.
473 matchedTemplate : `lsst.afw.image.ExposureF`
474 Warped and PSF-matched template that was used produce the
475 difference image.
476 difference : `lsst.afw.image.ExposureF`
477 Result of subtracting template from the science image.
478 scoreExposure : `lsst.afw.image.ExposureF`
479 Score or maximum likelihood difference image
480 idFactory : `lsst.afw.table.IdFactory`, optional
481 Generator object to assign ids to detected sources in the difference image.
483 Returns
484 -------
485 measurementResults : `lsst.pipe.base.Struct`
487 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
488 Subtracted exposure with detection mask applied.
489 ``diaSources`` : `lsst.afw.table.SourceCatalog`
490 The catalog of detected sources.
491 """
492 # Ensure that we start with an empty detection mask.
493 mask = scoreExposure.mask
494 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
496 table = afwTable.SourceTable.make(self.schema, idFactory)
497 table.setMetadata(self.algMetadata)
498 # Exclude the edge of the CCD from detection.
499 # This operation would be performed in the detection subtask if doSmooth=True
500 # but we need to apply the cut here since we are using a preconvolved image.
501 goodBBox = scoreExposure.getPsf().getKernel().shrinkBBox(scoreExposure.getBBox())
502 results = self.detection.run(
503 table=table,
504 exposure=scoreExposure[goodBBox],
505 doSmooth=False,
506 )
507 # Copy the detection mask from the Score image to the difference image
508 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
510 return self.processResults(science, matchedTemplate, difference, results.sources, table,
511 positiveFootprints=results.positive, negativeFootprints=results.negative)