lsst.ip.diffim g2959009a90+78a34153b8
Loading...
Searching...
No Matches
detectAndMeasure.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22from deprecated.sphinx import deprecated
23
24import lsst.afw.table as afwTable
25import lsst.daf.base as dafBase
26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources # noqa: F401
29import lsst.meas.extensions.shapeHSM
30from lsst.obs.base import ExposureIdInfo
31import lsst.pex.config as pexConfig
32import lsst.pipe.base as pipeBase
33import lsst.utils
34from lsst.utils.timer import timeMethod
35
36from . import DipoleFitTask
37
38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
40
41
42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
43 dimensions=("instrument", "visit", "detector"),
44 defaultTemplates={"coaddName": "deep",
45 "warpTypeSuffix": "",
46 "fakesType": ""}):
47 science = pipeBase.connectionTypes.Input(
48 doc="Input science exposure.",
49 dimensions=("instrument", "visit", "detector"),
50 storageClass="ExposureF",
51 name="{fakesType}calexp"
52 )
53 matchedTemplate = pipeBase.connectionTypes.Input(
54 doc="Warped and PSF-matched template used to create the difference image.",
55 dimensions=("instrument", "visit", "detector"),
56 storageClass="ExposureF",
57 name="{fakesType}{coaddName}Diff_matchedExp",
58 )
59 difference = pipeBase.connectionTypes.Input(
60 doc="Result of subtracting template from science.",
61 dimensions=("instrument", "visit", "detector"),
62 storageClass="ExposureF",
63 name="{fakesType}{coaddName}Diff_differenceTempExp",
64 )
65 outputSchema = pipeBase.connectionTypes.InitOutput(
66 doc="Schema (as an example catalog) for output DIASource catalog.",
67 storageClass="SourceCatalog",
68 name="{fakesType}{coaddName}Diff_diaSrc_schema",
69 )
70 diaSources = pipeBase.connectionTypes.Output(
71 doc="Detected diaSources on the difference image.",
72 dimensions=("instrument", "visit", "detector"),
73 storageClass="SourceCatalog",
74 name="{fakesType}{coaddName}Diff_diaSrc",
75 )
76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
77 doc="Difference image with detection mask plane filled in.",
78 dimensions=("instrument", "visit", "detector"),
79 storageClass="ExposureF",
80 name="{fakesType}{coaddName}Diff_differenceExp",
81 )
82
83
84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
85 pipelineConnections=DetectAndMeasureConnections):
86 """Config for DetectAndMeasureTask
87 """
88 doMerge = pexConfig.Field(
89 dtype=bool,
90 default=True,
91 doc="Merge positive and negative diaSources with grow radius "
92 "set by growFootprint"
93 )
94 doForcedMeasurement = pexConfig.Field(
95 dtype=bool,
96 default=True,
97 doc="Force photometer diaSource locations on PVI?")
98 doAddMetrics = pexConfig.Field(
99 dtype=bool,
100 default=False,
101 doc="Add columns to the source table to hold analysis metrics?"
102 )
103 detection = pexConfig.ConfigurableField(
104 target=SourceDetectionTask,
105 doc="Final source detection for diaSource measurement",
106 )
107 measurement = pexConfig.ConfigurableField(
108 target=DipoleFitTask,
109 doc="Task to measure sources on the difference image.",
110 )
111 doApCorr = lsst.pex.config.Field(
112 dtype=bool,
113 default=True,
114 doc="Run subtask to apply aperture corrections"
115 )
116 applyApCorr = lsst.pex.config.ConfigurableField(
117 target=ApplyApCorrTask,
118 doc="Task to apply aperture corrections"
119 )
120 forcedMeasurement = pexConfig.ConfigurableField(
121 target=ForcedMeasurementTask,
122 doc="Task to force photometer science image at diaSource locations.",
123 )
124 growFootprint = pexConfig.Field(
125 dtype=int,
126 default=2,
127 doc="Grow positive and negative footprints by this many pixels before merging"
128 )
129 diaSourceMatchRadius = pexConfig.Field(
130 dtype=float,
131 default=0.5,
132 doc="Match radius (in arcseconds) for DiaSource to Source association"
133 )
134 doSkySources = pexConfig.Field(
135 dtype=bool,
136 default=False,
137 doc="Generate sky sources?",
138 )
139 skySources = pexConfig.ConfigurableField(
140 target=SkyObjectsTask,
141 doc="Generate sky sources",
142 )
143 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
144
145 def setDefaults(self):
146 # DiaSource Detection
147 self.detection.thresholdPolarity = "both"
148 self.detection.thresholdValue = 5.0
149 self.detection.reEstimateBackground = False
150 self.detection.thresholdType = "pixel_stdev"
151 self.detection.excludeMaskPlanes = ["EDGE"]
152
153 # Add filtered flux measurement, the correct measurement for pre-convolved images.
154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
155 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
156 'base_LocalPhotoCalib',
157 'base_LocalWcs',
158 'ext_shapeHSM_HsmSourceMoments',
159 'ext_shapeHSM_HsmPsfMoments',
160 ]
161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
163 self.measurement.plugins["base_NaiveCentroid"].maxDistToPeak = 5.0
164 self.measurement.plugins["base_SdssCentroid"].maxDistToPeak = 5.0
165 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
166 self.forcedMeasurement.copyColumns = {
167 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
168 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
169 self.forcedMeasurement.slots.shape = None
170
171
172class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
173 """Detect and measure sources on a difference image.
174 """
175 ConfigClass = DetectAndMeasureConfig
176 _DefaultName = "detectAndMeasure"
177
178 def __init__(self, **kwargs):
179 super().__init__(**kwargs)
180 self.schema = afwTable.SourceTable.makeMinimalSchema()
181
182 self.algMetadata = dafBase.PropertyList()
183 self.makeSubtask("detection", schema=self.schema)
184 self.makeSubtask("measurement", schema=self.schema,
185 algMetadata=self.algMetadata)
186 if self.config.doApCorr:
187 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
188 if self.config.doForcedMeasurement:
189 self.schema.addField(
190 "ip_diffim_forced_PsfFlux_instFlux", "D",
191 "Forced PSF flux measured on the direct image.",
192 units="count")
193 self.schema.addField(
194 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
195 "Forced PSF flux error measured on the direct image.",
196 units="count")
197 self.schema.addField(
198 "ip_diffim_forced_PsfFlux_area", "F",
199 "Forced PSF flux effective area of PSF.",
200 units="pixel")
201 self.schema.addField(
202 "ip_diffim_forced_PsfFlux_flag", "Flag",
203 "Forced PSF flux general failure flag.")
204 self.schema.addField(
205 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
206 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
207 self.schema.addField(
208 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
209 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
210 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
211
212 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
213 self.schema.addField("srcMatchId", "L", "unique id of source match")
214 if self.config.doSkySources:
215 self.makeSubtask("skySources")
216 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
217
218 # initialize InitOutputs
219 self.outputSchema = afwTable.SourceCatalog(self.schema)
220 self.outputSchema.getTable().setMetadata(self.algMetadata)
221
222 @staticmethod
223 @deprecated(
224 reason=(
225 "ID factory construction now depends on configuration; use the "
226 "idGenerator config field. Will be removed after v27."
227 ),
228 version="v26.0",
229 category=FutureWarning,
230 )
231 def makeIdFactory(expId, expBits):
232 """Create IdFactory instance for unique 64 bit diaSource id-s.
233
234 Parameters
235 ----------
236 expId : `int`
237 Exposure id.
238
239 expBits: `int`
240 Number of used bits in ``expId``.
241
242 Notes
243 -----
244 The diasource id-s consists of the ``expId`` stored fixed in the highest value
245 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
246 low value end of the integer.
247
248 Returns
249 -------
250 idFactory: `lsst.afw.table.IdFactory`
251 """
252 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
253
254 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
255 inputRefs: pipeBase.InputQuantizedConnection,
256 outputRefs: pipeBase.OutputQuantizedConnection):
257 inputs = butlerQC.get(inputRefs)
258 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
259 idFactory = idGenerator.make_table_id_factory()
260 outputs = self.run(**inputs, idFactory=idFactory)
261 butlerQC.put(outputs, outputRefs)
262
263 @timeMethod
264 def run(self, science, matchedTemplate, difference,
265 idFactory=None):
266 """Detect and measure sources on a difference image.
267
268 The difference image will be convolved with a gaussian approximation of
269 the PSF to form a maximum likelihood image for detection.
270 Close positive and negative detections will optionally be merged into
271 dipole diaSources.
272 Sky sources, or forced detections in background regions, will optionally
273 be added, and the configured measurement algorithm will be run on all
274 detections.
275
276 Parameters
277 ----------
278 science : `lsst.afw.image.ExposureF`
279 Science exposure that the template was subtracted from.
280 matchedTemplate : `lsst.afw.image.ExposureF`
281 Warped and PSF-matched template that was used produce the
282 difference image.
283 difference : `lsst.afw.image.ExposureF`
284 Result of subtracting template from the science image.
285 idFactory : `lsst.afw.table.IdFactory`, optional
286 Generator object to assign ids to detected sources in the difference image.
287
288 Returns
289 -------
290 measurementResults : `lsst.pipe.base.Struct`
291
292 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
293 Subtracted exposure with detection mask applied.
294 ``diaSources`` : `lsst.afw.table.SourceCatalog`
295 The catalog of detected sources.
296 """
297 # Ensure that we start with an empty detection mask.
298 mask = difference.mask
299 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
300
301 table = afwTable.SourceTable.make(self.schema, idFactory)
302 table.setMetadata(self.algMetadata)
303 results = self.detection.run(
304 table=table,
305 exposure=difference,
306 doSmooth=True,
307 )
308
309 return self.processResults(science, matchedTemplate, difference, results.sources, table,
310 positiveFootprints=results.positive, negativeFootprints=results.negative)
311
312 def processResults(self, science, matchedTemplate, difference, sources, table,
313 positiveFootprints=None, negativeFootprints=None,):
314 """Measure and process the results of source detection.
315
316 Parameters
317 ----------
319 Detected sources on the difference exposure.
320 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
321 Positive polarity footprints.
322 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
323 Negative polarity footprints.
325 Table object that will be used to create the SourceCatalog.
326 science : `lsst.afw.image.ExposureF`
327 Science exposure that the template was subtracted from.
328 matchedTemplate : `lsst.afw.image.ExposureF`
329 Warped and PSF-matched template that was used produce the
330 difference image.
331 difference : `lsst.afw.image.ExposureF`
332 Result of subtracting template from the science image.
333
334 Returns
335 -------
336 measurementResults : `lsst.pipe.base.Struct`
337
338 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
339 Subtracted exposure with detection mask applied.
340 ``diaSources`` : `lsst.afw.table.SourceCatalog`
341 The catalog of detected sources.
342 """
343 if self.config.doMerge:
344 fpSet = positiveFootprints
345 fpSet.merge(negativeFootprints, self.config.growFootprint,
346 self.config.growFootprint, False)
347 diaSources = afwTable.SourceCatalog(table)
348 fpSet.makeSources(diaSources)
349 self.log.info("Merging detections into %d sources", len(diaSources))
350 else:
351 diaSources = sources
352
353 if self.config.doSkySources:
354 self.addSkySources(diaSources, difference.mask, difference.info.id)
355
356 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
357
358 if self.config.doForcedMeasurement:
359 self.measureForcedSources(diaSources, science, difference.getWcs())
360
361 measurementResults = pipeBase.Struct(
362 subtractedMeasuredExposure=difference,
363 diaSources=diaSources,
364 )
365
366 return measurementResults
367
368 def addSkySources(self, diaSources, mask, seed):
369 """Add sources in empty regions of the difference image
370 for measuring the background.
371
372 Parameters
373 ----------
374 diaSources : `lsst.afw.table.SourceCatalog`
375 The catalog of detected sources.
376 mask : `lsst.afw.image.Mask`
377 Mask plane for determining regions where Sky sources can be added.
378 seed : `int`
379 Seed value to initialize the random number generator.
380 """
381 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
382 if skySourceFootprints:
383 for foot in skySourceFootprints:
384 s = diaSources.addNew()
385 s.setFootprint(foot)
386 s.set(self.skySourceKey, True)
387
388 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
389 """Use (matched) template and science image to constrain dipole fitting.
390
391 Parameters
392 ----------
393 diaSources : `lsst.afw.table.SourceCatalog`
394 The catalog of detected sources.
395 science : `lsst.afw.image.ExposureF`
396 Science exposure that the template was subtracted from.
397 difference : `lsst.afw.image.ExposureF`
398 Result of subtracting template from the science image.
399 matchedTemplate : `lsst.afw.image.ExposureF`
400 Warped and PSF-matched template that was used produce the
401 difference image.
402 """
403 # Note that this may not be correct if we convolved the science image.
404 # In the future we may wish to persist the matchedScience image.
405 self.measurement.run(diaSources, difference, science, matchedTemplate)
406 if self.config.doApCorr:
407 self.applyApCorr.run(
408 catalog=diaSources,
409 apCorrMap=difference.getInfo().getApCorrMap()
410 )
411
412 def measureForcedSources(self, diaSources, science, wcs):
413 """Perform forced measurement of the diaSources on the science image.
414
415 Parameters
416 ----------
417 diaSources : `lsst.afw.table.SourceCatalog`
418 The catalog of detected sources.
419 science : `lsst.afw.image.ExposureF`
420 Science exposure that the template was subtracted from.
422 Coordinate system definition (wcs) for the exposure.
423 """
424 # Run forced psf photometry on the PVI at the diaSource locations.
425 # Copy the measured flux and error into the diaSource.
426 forcedSources = self.forcedMeasurement.generateMeasCat(
427 science, diaSources, wcs)
428 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
429 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
430 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
431 "ip_diffim_forced_PsfFlux_instFlux", True)
432 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
433 "ip_diffim_forced_PsfFlux_instFluxErr", True)
434 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
435 "ip_diffim_forced_PsfFlux_area", True)
436 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
437 "ip_diffim_forced_PsfFlux_flag", True)
438 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
439 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
440 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
441 "ip_diffim_forced_PsfFlux_flag_edge", True)
442 for diaSource, forcedSource in zip(diaSources, forcedSources):
443 diaSource.assign(forcedSource, mapper)
444
445
446class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
447 scoreExposure = pipeBase.connectionTypes.Input(
448 doc="Maximum likelihood image for detection.",
449 dimensions=("instrument", "visit", "detector"),
450 storageClass="ExposureF",
451 name="{fakesType}{coaddName}Diff_scoreExp",
452 )
453
454
455class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
456 pipelineConnections=DetectAndMeasureScoreConnections):
457 pass
458
459
460class DetectAndMeasureScoreTask(DetectAndMeasureTask):
461 """Detect DIA sources using a score image,
462 and measure the detections on the difference image.
463
464 Source detection is run on the supplied score, or maximum likelihood,
465 image. Note that no additional convolution will be done in this case.
466 Close positive and negative detections will optionally be merged into
467 dipole diaSources.
468 Sky sources, or forced detections in background regions, will optionally
469 be added, and the configured measurement algorithm will be run on all
470 detections.
471 """
472 ConfigClass = DetectAndMeasureScoreConfig
473 _DefaultName = "detectAndMeasureScore"
474
475 @timeMethod
476 def run(self, science, matchedTemplate, difference, scoreExposure,
477 idFactory=None):
478 """Detect and measure sources on a score image.
479
480 Parameters
481 ----------
482 science : `lsst.afw.image.ExposureF`
483 Science exposure that the template was subtracted from.
484 matchedTemplate : `lsst.afw.image.ExposureF`
485 Warped and PSF-matched template that was used produce the
486 difference image.
487 difference : `lsst.afw.image.ExposureF`
488 Result of subtracting template from the science image.
489 scoreExposure : `lsst.afw.image.ExposureF`
490 Score or maximum likelihood difference image
491 idFactory : `lsst.afw.table.IdFactory`, optional
492 Generator object to assign ids to detected sources in the difference image.
493
494 Returns
495 -------
496 measurementResults : `lsst.pipe.base.Struct`
497
498 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
499 Subtracted exposure with detection mask applied.
500 ``diaSources`` : `lsst.afw.table.SourceCatalog`
501 The catalog of detected sources.
502 """
503 # Ensure that we start with an empty detection mask.
504 mask = scoreExposure.mask
505 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
506
507 table = afwTable.SourceTable.make(self.schema, idFactory)
508 table.setMetadata(self.algMetadata)
509 results = self.detection.run(
510 table=table,
511 exposure=scoreExposure,
512 doSmooth=False,
513 )
514 # Copy the detection mask from the Score image to the difference image
515 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
516
517 return self.processResults(science, matchedTemplate, difference, results.sources, table,
518 positiveFootprints=results.positive, negativeFootprints=results.negative)
def run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)
Definition: getTemplate.py:188