lsst.ip.diffim gfaa7bcd731+3e3b9e0885
Loading...
Searching...
No Matches
detectAndMeasure.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22from deprecated.sphinx import deprecated
23
24import lsst.afw.table as afwTable
25import lsst.daf.base as dafBase
26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources # noqa: F401
29import lsst.meas.extensions.shapeHSM
30from lsst.obs.base import ExposureIdInfo
31import lsst.pex.config as pexConfig
32import lsst.pipe.base as pipeBase
33import lsst.utils
34from lsst.utils.timer import timeMethod
35
36from . import DipoleFitTask
37
38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
40
41
42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
43 dimensions=("instrument", "visit", "detector"),
44 defaultTemplates={"coaddName": "deep",
45 "warpTypeSuffix": "",
46 "fakesType": ""}):
47 science = pipeBase.connectionTypes.Input(
48 doc="Input science exposure.",
49 dimensions=("instrument", "visit", "detector"),
50 storageClass="ExposureF",
51 name="{fakesType}calexp"
52 )
53 matchedTemplate = pipeBase.connectionTypes.Input(
54 doc="Warped and PSF-matched template used to create the difference image.",
55 dimensions=("instrument", "visit", "detector"),
56 storageClass="ExposureF",
57 name="{fakesType}{coaddName}Diff_matchedExp",
58 )
59 difference = pipeBase.connectionTypes.Input(
60 doc="Result of subtracting template from science.",
61 dimensions=("instrument", "visit", "detector"),
62 storageClass="ExposureF",
63 name="{fakesType}{coaddName}Diff_differenceTempExp",
64 )
65 outputSchema = pipeBase.connectionTypes.InitOutput(
66 doc="Schema (as an example catalog) for output DIASource catalog.",
67 storageClass="SourceCatalog",
68 name="{fakesType}{coaddName}Diff_diaSrc_schema",
69 )
70 diaSources = pipeBase.connectionTypes.Output(
71 doc="Detected diaSources on the difference image.",
72 dimensions=("instrument", "visit", "detector"),
73 storageClass="SourceCatalog",
74 name="{fakesType}{coaddName}Diff_diaSrc",
75 )
76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
77 doc="Difference image with detection mask plane filled in.",
78 dimensions=("instrument", "visit", "detector"),
79 storageClass="ExposureF",
80 name="{fakesType}{coaddName}Diff_differenceExp",
81 )
82
83
84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
85 pipelineConnections=DetectAndMeasureConnections):
86 """Config for DetectAndMeasureTask
87 """
88 doMerge = pexConfig.Field(
89 dtype=bool,
90 default=True,
91 doc="Merge positive and negative diaSources with grow radius "
92 "set by growFootprint"
93 )
94 doForcedMeasurement = pexConfig.Field(
95 dtype=bool,
96 default=True,
97 doc="Force photometer diaSource locations on PVI?")
98 doAddMetrics = pexConfig.Field(
99 dtype=bool,
100 default=False,
101 doc="Add columns to the source table to hold analysis metrics?"
102 )
103 detection = pexConfig.ConfigurableField(
104 target=SourceDetectionTask,
105 doc="Final source detection for diaSource measurement",
106 )
107 measurement = pexConfig.ConfigurableField(
108 target=DipoleFitTask,
109 doc="Task to measure sources on the difference image.",
110 )
111 doApCorr = lsst.pex.config.Field(
112 dtype=bool,
113 default=True,
114 doc="Run subtask to apply aperture corrections"
115 )
116 applyApCorr = lsst.pex.config.ConfigurableField(
117 target=ApplyApCorrTask,
118 doc="Task to apply aperture corrections"
119 )
120 forcedMeasurement = pexConfig.ConfigurableField(
121 target=ForcedMeasurementTask,
122 doc="Task to force photometer science image at diaSource locations.",
123 )
124 growFootprint = pexConfig.Field(
125 dtype=int,
126 default=2,
127 doc="Grow positive and negative footprints by this many pixels before merging"
128 )
129 diaSourceMatchRadius = pexConfig.Field(
130 dtype=float,
131 default=0.5,
132 doc="Match radius (in arcseconds) for DiaSource to Source association"
133 )
134 doSkySources = pexConfig.Field(
135 dtype=bool,
136 default=False,
137 doc="Generate sky sources?",
138 )
139 skySources = pexConfig.ConfigurableField(
140 target=SkyObjectsTask,
141 doc="Generate sky sources",
142 )
143 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
144
145 def setDefaults(self):
146 # DiaSource Detection
147 self.detection.thresholdPolarity = "both"
148 self.detection.thresholdValue = 5.0
149 self.detection.reEstimateBackground = False
150 self.detection.thresholdType = "pixel_stdev"
151
152 # Add filtered flux measurement, the correct measurement for pre-convolved images.
153 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
154 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
155 'base_LocalPhotoCalib',
156 'base_LocalWcs',
157 'ext_shapeHSM_HsmSourceMoments',
158 'ext_shapeHSM_HsmPsfMoments',
159 ]
160 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
161 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
162
163 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
164 self.forcedMeasurement.copyColumns = {
165 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
166 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
167 self.forcedMeasurement.slots.shape = None
168
169
170class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
171 """Detect and measure sources on a difference image.
172 """
173 ConfigClass = DetectAndMeasureConfig
174 _DefaultName = "detectAndMeasure"
175
176 def __init__(self, **kwargs):
177 super().__init__(**kwargs)
178 self.schema = afwTable.SourceTable.makeMinimalSchema()
179
180 self.algMetadata = dafBase.PropertyList()
181 self.makeSubtask("detection", schema=self.schema)
182 self.makeSubtask("measurement", schema=self.schema,
183 algMetadata=self.algMetadata)
184 if self.config.doApCorr:
185 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
186 if self.config.doForcedMeasurement:
187 self.schema.addField(
188 "ip_diffim_forced_PsfFlux_instFlux", "D",
189 "Forced PSF flux measured on the direct image.",
190 units="count")
191 self.schema.addField(
192 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
193 "Forced PSF flux error measured on the direct image.",
194 units="count")
195 self.schema.addField(
196 "ip_diffim_forced_PsfFlux_area", "F",
197 "Forced PSF flux effective area of PSF.",
198 units="pixel")
199 self.schema.addField(
200 "ip_diffim_forced_PsfFlux_flag", "Flag",
201 "Forced PSF flux general failure flag.")
202 self.schema.addField(
203 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
204 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
205 self.schema.addField(
206 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
207 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
208 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
209
210 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
211 self.schema.addField("srcMatchId", "L", "unique id of source match")
212 if self.config.doSkySources:
213 self.makeSubtask("skySources")
214 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
215
216 # initialize InitOutputs
217 self.outputSchema = afwTable.SourceCatalog(self.schema)
218 self.outputSchema.getTable().setMetadata(self.algMetadata)
219
220 @staticmethod
221 @deprecated(
222 reason=(
223 "ID factory construction now depends on configuration; use the "
224 "idGenerator config field. Will be removed after v27."
225 ),
226 version="v26.0",
227 category=FutureWarning,
228 )
229 def makeIdFactory(expId, expBits):
230 """Create IdFactory instance for unique 64 bit diaSource id-s.
231
232 Parameters
233 ----------
234 expId : `int`
235 Exposure id.
236
237 expBits: `int`
238 Number of used bits in ``expId``.
239
240 Notes
241 -----
242 The diasource id-s consists of the ``expId`` stored fixed in the highest value
243 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
244 low value end of the integer.
245
246 Returns
247 -------
248 idFactory: `lsst.afw.table.IdFactory`
249 """
250 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
251
252 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
253 inputRefs: pipeBase.InputQuantizedConnection,
254 outputRefs: pipeBase.OutputQuantizedConnection):
255 inputs = butlerQC.get(inputRefs)
256 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
257 idFactory = idGenerator.make_table_id_factory()
258 outputs = self.run(**inputs, idFactory=idFactory)
259 butlerQC.put(outputs, outputRefs)
260
261 @timeMethod
262 def run(self, science, matchedTemplate, difference,
263 idFactory=None):
264 """Detect and measure sources on a difference image.
265
266 The difference image will be convolved with a gaussian approximation of
267 the PSF to form a maximum likelihood image for detection.
268 Close positive and negative detections will optionally be merged into
269 dipole diaSources.
270 Sky sources, or forced detections in background regions, will optionally
271 be added, and the configured measurement algorithm will be run on all
272 detections.
273
274 Parameters
275 ----------
276 science : `lsst.afw.image.ExposureF`
277 Science exposure that the template was subtracted from.
278 matchedTemplate : `lsst.afw.image.ExposureF`
279 Warped and PSF-matched template that was used produce the
280 difference image.
281 difference : `lsst.afw.image.ExposureF`
282 Result of subtracting template from the science image.
283 idFactory : `lsst.afw.table.IdFactory`, optional
284 Generator object to assign ids to detected sources in the difference image.
285
286 Returns
287 -------
288 measurementResults : `lsst.pipe.base.Struct`
289
290 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
291 Subtracted exposure with detection mask applied.
292 ``diaSources`` : `lsst.afw.table.SourceCatalog`
293 The catalog of detected sources.
294 """
295 # Ensure that we start with an empty detection mask.
296 mask = difference.mask
297 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
298
299 table = afwTable.SourceTable.make(self.schema, idFactory)
300 table.setMetadata(self.algMetadata)
301 results = self.detection.run(
302 table=table,
303 exposure=difference,
304 doSmooth=True,
305 )
306
307 return self.processResults(science, matchedTemplate, difference, results.sources, table,
308 positiveFootprints=results.positive, negativeFootprints=results.negative)
309
310 def processResults(self, science, matchedTemplate, difference, sources, table,
311 positiveFootprints=None, negativeFootprints=None,):
312 """Measure and process the results of source detection.
313
314 Parameters
315 ----------
317 Detected sources on the difference exposure.
318 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
319 Positive polarity footprints.
320 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
321 Negative polarity footprints.
323 Table object that will be used to create the SourceCatalog.
324 science : `lsst.afw.image.ExposureF`
325 Science exposure that the template was subtracted from.
326 matchedTemplate : `lsst.afw.image.ExposureF`
327 Warped and PSF-matched template that was used produce the
328 difference image.
329 difference : `lsst.afw.image.ExposureF`
330 Result of subtracting template from the science image.
331
332 Returns
333 -------
334 measurementResults : `lsst.pipe.base.Struct`
335
336 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
337 Subtracted exposure with detection mask applied.
338 ``diaSources`` : `lsst.afw.table.SourceCatalog`
339 The catalog of detected sources.
340 """
341 if self.config.doMerge:
342 fpSet = positiveFootprints
343 fpSet.merge(negativeFootprints, self.config.growFootprint,
344 self.config.growFootprint, False)
345 diaSources = afwTable.SourceCatalog(table)
346 fpSet.makeSources(diaSources)
347 self.log.info("Merging detections into %d sources", len(diaSources))
348 else:
349 diaSources = sources
350
351 if self.config.doSkySources:
352 self.addSkySources(diaSources, difference.mask, difference.info.id)
353
354 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
355
356 if self.config.doForcedMeasurement:
357 self.measureForcedSources(diaSources, science, difference.getWcs())
358
359 measurementResults = pipeBase.Struct(
360 subtractedMeasuredExposure=difference,
361 diaSources=diaSources,
362 )
363
364 return measurementResults
365
366 def addSkySources(self, diaSources, mask, seed):
367 """Add sources in empty regions of the difference image
368 for measuring the background.
369
370 Parameters
371 ----------
372 diaSources : `lsst.afw.table.SourceCatalog`
373 The catalog of detected sources.
374 mask : `lsst.afw.image.Mask`
375 Mask plane for determining regions where Sky sources can be added.
376 seed : `int`
377 Seed value to initialize the random number generator.
378 """
379 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
380 if skySourceFootprints:
381 for foot in skySourceFootprints:
382 s = diaSources.addNew()
383 s.setFootprint(foot)
384 s.set(self.skySourceKey, True)
385
386 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
387 """Use (matched) template and science image to constrain dipole fitting.
388
389 Parameters
390 ----------
391 diaSources : `lsst.afw.table.SourceCatalog`
392 The catalog of detected sources.
393 science : `lsst.afw.image.ExposureF`
394 Science exposure that the template was subtracted from.
395 difference : `lsst.afw.image.ExposureF`
396 Result of subtracting template from the science image.
397 matchedTemplate : `lsst.afw.image.ExposureF`
398 Warped and PSF-matched template that was used produce the
399 difference image.
400 """
401 # Note that this may not be correct if we convolved the science image.
402 # In the future we may wish to persist the matchedScience image.
403 self.measurement.run(diaSources, difference, science, matchedTemplate)
404 if self.config.doApCorr:
405 self.applyApCorr.run(
406 catalog=diaSources,
407 apCorrMap=difference.getInfo().getApCorrMap()
408 )
409
410 def measureForcedSources(self, diaSources, science, wcs):
411 """Perform forced measurement of the diaSources on the science image.
412
413 Parameters
414 ----------
415 diaSources : `lsst.afw.table.SourceCatalog`
416 The catalog of detected sources.
417 science : `lsst.afw.image.ExposureF`
418 Science exposure that the template was subtracted from.
420 Coordinate system definition (wcs) for the exposure.
421 """
422 # Run forced psf photometry on the PVI at the diaSource locations.
423 # Copy the measured flux and error into the diaSource.
424 forcedSources = self.forcedMeasurement.generateMeasCat(
425 science, diaSources, wcs)
426 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
427 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
428 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
429 "ip_diffim_forced_PsfFlux_instFlux", True)
430 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
431 "ip_diffim_forced_PsfFlux_instFluxErr", True)
432 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
433 "ip_diffim_forced_PsfFlux_area", True)
434 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
435 "ip_diffim_forced_PsfFlux_flag", True)
436 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
437 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
438 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
439 "ip_diffim_forced_PsfFlux_flag_edge", True)
440 for diaSource, forcedSource in zip(diaSources, forcedSources):
441 diaSource.assign(forcedSource, mapper)
442
443
444class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
445 scoreExposure = pipeBase.connectionTypes.Input(
446 doc="Maximum likelihood image for detection.",
447 dimensions=("instrument", "visit", "detector"),
448 storageClass="ExposureF",
449 name="{fakesType}{coaddName}Diff_scoreExp",
450 )
451
452
453class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
454 pipelineConnections=DetectAndMeasureScoreConnections):
455 pass
456
457
458class DetectAndMeasureScoreTask(DetectAndMeasureTask):
459 """Detect DIA sources using a score image,
460 and measure the detections on the difference image.
461
462 Source detection is run on the supplied score, or maximum likelihood,
463 image. Note that no additional convolution will be done in this case.
464 Close positive and negative detections will optionally be merged into
465 dipole diaSources.
466 Sky sources, or forced detections in background regions, will optionally
467 be added, and the configured measurement algorithm will be run on all
468 detections.
469 """
470 ConfigClass = DetectAndMeasureScoreConfig
471 _DefaultName = "detectAndMeasureScore"
472
473 @timeMethod
474 def run(self, science, matchedTemplate, difference, scoreExposure,
475 idFactory=None):
476 """Detect and measure sources on a score image.
477
478 Parameters
479 ----------
480 science : `lsst.afw.image.ExposureF`
481 Science exposure that the template was subtracted from.
482 matchedTemplate : `lsst.afw.image.ExposureF`
483 Warped and PSF-matched template that was used produce the
484 difference image.
485 difference : `lsst.afw.image.ExposureF`
486 Result of subtracting template from the science image.
487 scoreExposure : `lsst.afw.image.ExposureF`
488 Score or maximum likelihood difference image
489 idFactory : `lsst.afw.table.IdFactory`, optional
490 Generator object to assign ids to detected sources in the difference image.
491
492 Returns
493 -------
494 measurementResults : `lsst.pipe.base.Struct`
495
496 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
497 Subtracted exposure with detection mask applied.
498 ``diaSources`` : `lsst.afw.table.SourceCatalog`
499 The catalog of detected sources.
500 """
501 # Ensure that we start with an empty detection mask.
502 mask = scoreExposure.mask
503 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
504
505 table = afwTable.SourceTable.make(self.schema, idFactory)
506 table.setMetadata(self.algMetadata)
507 # Exclude the edge of the CCD from detection.
508 # This operation would be performed in the detection subtask if doSmooth=True
509 # but we need to apply the cut here since we are using a preconvolved image.
510 goodBBox = scoreExposure.getPsf().getKernel().shrinkBBox(scoreExposure.getBBox())
511 results = self.detection.run(
512 table=table,
513 exposure=scoreExposure[goodBBox],
514 doSmooth=False,
515 )
516 # Copy the detection mask from the Score image to the difference image
517 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
518
519 return self.processResults(science, matchedTemplate, difference, results.sources, table,
520 positiveFootprints=results.positive, negativeFootprints=results.negative)
def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs)
Definition: getTemplate.py:484