lsst.ip.diffim g41be43c305+0e4ba543ab
Loading...
Searching...
No Matches
detectAndMeasure.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22from deprecated.sphinx import deprecated
23
24import lsst.afw.table as afwTable
25import lsst.daf.base as dafBase
26from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
27from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig
28import lsst.meas.extensions.trailedSources # noqa: F401
29import lsst.meas.extensions.shapeHSM
30from lsst.obs.base import ExposureIdInfo
31import lsst.pex.config as pexConfig
32import lsst.pipe.base as pipeBase
33import lsst.utils
34from lsst.utils.timer import timeMethod
35
36from . import DipoleFitTask
37
38__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask",
39 "DetectAndMeasureScoreConfig", "DetectAndMeasureScoreTask"]
40
41
42class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
43 dimensions=("instrument", "visit", "detector"),
44 defaultTemplates={"coaddName": "deep",
45 "warpTypeSuffix": "",
46 "fakesType": ""}):
47 science = pipeBase.connectionTypes.Input(
48 doc="Input science exposure.",
49 dimensions=("instrument", "visit", "detector"),
50 storageClass="ExposureF",
51 name="{fakesType}calexp"
52 )
53 matchedTemplate = pipeBase.connectionTypes.Input(
54 doc="Warped and PSF-matched template used to create the difference image.",
55 dimensions=("instrument", "visit", "detector"),
56 storageClass="ExposureF",
57 name="{fakesType}{coaddName}Diff_matchedExp",
58 )
59 difference = pipeBase.connectionTypes.Input(
60 doc="Result of subtracting template from science.",
61 dimensions=("instrument", "visit", "detector"),
62 storageClass="ExposureF",
63 name="{fakesType}{coaddName}Diff_differenceTempExp",
64 )
65 outputSchema = pipeBase.connectionTypes.InitOutput(
66 doc="Schema (as an example catalog) for output DIASource catalog.",
67 storageClass="SourceCatalog",
68 name="{fakesType}{coaddName}Diff_diaSrc_schema",
69 )
70 diaSources = pipeBase.connectionTypes.Output(
71 doc="Detected diaSources on the difference image.",
72 dimensions=("instrument", "visit", "detector"),
73 storageClass="SourceCatalog",
74 name="{fakesType}{coaddName}Diff_diaSrc",
75 )
76 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
77 doc="Difference image with detection mask plane filled in.",
78 dimensions=("instrument", "visit", "detector"),
79 storageClass="ExposureF",
80 name="{fakesType}{coaddName}Diff_differenceExp",
81 )
82
83
84class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
85 pipelineConnections=DetectAndMeasureConnections):
86 """Config for DetectAndMeasureTask
87 """
88 doMerge = pexConfig.Field(
89 dtype=bool,
90 default=True,
91 doc="Merge positive and negative diaSources with grow radius "
92 "set by growFootprint"
93 )
94 doForcedMeasurement = pexConfig.Field(
95 dtype=bool,
96 default=True,
97 doc="Force photometer diaSource locations on PVI?")
98 doAddMetrics = pexConfig.Field(
99 dtype=bool,
100 default=False,
101 doc="Add columns to the source table to hold analysis metrics?"
102 )
103 detection = pexConfig.ConfigurableField(
104 target=SourceDetectionTask,
105 doc="Final source detection for diaSource measurement",
106 )
107 measurement = pexConfig.ConfigurableField(
108 target=DipoleFitTask,
109 doc="Task to measure sources on the difference image.",
110 )
111 doApCorr = lsst.pex.config.Field(
112 dtype=bool,
113 default=True,
114 doc="Run subtask to apply aperture corrections"
115 )
116 applyApCorr = lsst.pex.config.ConfigurableField(
117 target=ApplyApCorrTask,
118 doc="Task to apply aperture corrections"
119 )
120 forcedMeasurement = pexConfig.ConfigurableField(
121 target=ForcedMeasurementTask,
122 doc="Task to force photometer science image at diaSource locations.",
123 )
124 growFootprint = pexConfig.Field(
125 dtype=int,
126 default=2,
127 doc="Grow positive and negative footprints by this many pixels before merging"
128 )
129 diaSourceMatchRadius = pexConfig.Field(
130 dtype=float,
131 default=0.5,
132 doc="Match radius (in arcseconds) for DiaSource to Source association"
133 )
134 doSkySources = pexConfig.Field(
135 dtype=bool,
136 default=False,
137 doc="Generate sky sources?",
138 )
139 skySources = pexConfig.ConfigurableField(
140 target=SkyObjectsTask,
141 doc="Generate sky sources",
142 )
143 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
144
145 def setDefaults(self):
146 # DiaSource Detection
147 self.detection.thresholdPolarity = "both"
148 self.detection.thresholdValue = 5.0
149 self.detection.reEstimateBackground = False
150 self.detection.thresholdType = "pixel_stdev"
151 self.detection.excludeMaskPlanes = ["EDGE"]
152
153 # Add filtered flux measurement, the correct measurement for pre-convolved images.
154 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
155 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
156 'base_LocalPhotoCalib',
157 'base_LocalWcs',
158 'ext_shapeHSM_HsmSourceMoments',
159 'ext_shapeHSM_HsmPsfMoments',
160 ]
161 self.measurement.slots.psfShape = "ext_shapeHSM_HsmPsfMoments"
162 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
163
164 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
165 self.forcedMeasurement.copyColumns = {
166 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
167 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
168 self.forcedMeasurement.slots.shape = None
169
170
171class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
172 """Detect and measure sources on a difference image.
173 """
174 ConfigClass = DetectAndMeasureConfig
175 _DefaultName = "detectAndMeasure"
176
177 def __init__(self, **kwargs):
178 super().__init__(**kwargs)
179 self.schema = afwTable.SourceTable.makeMinimalSchema()
180
181 self.algMetadata = dafBase.PropertyList()
182 self.makeSubtask("detection", schema=self.schema)
183 self.makeSubtask("measurement", schema=self.schema,
184 algMetadata=self.algMetadata)
185 if self.config.doApCorr:
186 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
187 if self.config.doForcedMeasurement:
188 self.schema.addField(
189 "ip_diffim_forced_PsfFlux_instFlux", "D",
190 "Forced PSF flux measured on the direct image.",
191 units="count")
192 self.schema.addField(
193 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
194 "Forced PSF flux error measured on the direct image.",
195 units="count")
196 self.schema.addField(
197 "ip_diffim_forced_PsfFlux_area", "F",
198 "Forced PSF flux effective area of PSF.",
199 units="pixel")
200 self.schema.addField(
201 "ip_diffim_forced_PsfFlux_flag", "Flag",
202 "Forced PSF flux general failure flag.")
203 self.schema.addField(
204 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
205 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
206 self.schema.addField(
207 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
208 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
209 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
210
211 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
212 self.schema.addField("srcMatchId", "L", "unique id of source match")
213 if self.config.doSkySources:
214 self.makeSubtask("skySources")
215 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
216
217 # initialize InitOutputs
218 self.outputSchema = afwTable.SourceCatalog(self.schema)
219 self.outputSchema.getTable().setMetadata(self.algMetadata)
220
221 @staticmethod
222 @deprecated(
223 reason=(
224 "ID factory construction now depends on configuration; use the "
225 "idGenerator config field. Will be removed after v27."
226 ),
227 version="v26.0",
228 category=FutureWarning,
229 )
230 def makeIdFactory(expId, expBits):
231 """Create IdFactory instance for unique 64 bit diaSource id-s.
232
233 Parameters
234 ----------
235 expId : `int`
236 Exposure id.
237
238 expBits: `int`
239 Number of used bits in ``expId``.
240
241 Notes
242 -----
243 The diasource id-s consists of the ``expId`` stored fixed in the highest value
244 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
245 low value end of the integer.
246
247 Returns
248 -------
249 idFactory: `lsst.afw.table.IdFactory`
250 """
251 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
252
253 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
254 inputRefs: pipeBase.InputQuantizedConnection,
255 outputRefs: pipeBase.OutputQuantizedConnection):
256 inputs = butlerQC.get(inputRefs)
257 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
258 idFactory = idGenerator.make_table_id_factory()
259 outputs = self.run(**inputs, idFactory=idFactory)
260 butlerQC.put(outputs, outputRefs)
261
262 @timeMethod
263 def run(self, science, matchedTemplate, difference,
264 idFactory=None):
265 """Detect and measure sources on a difference image.
266
267 The difference image will be convolved with a gaussian approximation of
268 the PSF to form a maximum likelihood image for detection.
269 Close positive and negative detections will optionally be merged into
270 dipole diaSources.
271 Sky sources, or forced detections in background regions, will optionally
272 be added, and the configured measurement algorithm will be run on all
273 detections.
274
275 Parameters
276 ----------
277 science : `lsst.afw.image.ExposureF`
278 Science exposure that the template was subtracted from.
279 matchedTemplate : `lsst.afw.image.ExposureF`
280 Warped and PSF-matched template that was used produce the
281 difference image.
282 difference : `lsst.afw.image.ExposureF`
283 Result of subtracting template from the science image.
284 idFactory : `lsst.afw.table.IdFactory`, optional
285 Generator object to assign ids to detected sources in the difference image.
286
287 Returns
288 -------
289 measurementResults : `lsst.pipe.base.Struct`
290
291 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
292 Subtracted exposure with detection mask applied.
293 ``diaSources`` : `lsst.afw.table.SourceCatalog`
294 The catalog of detected sources.
295 """
296 # Ensure that we start with an empty detection mask.
297 mask = difference.mask
298 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
299
300 table = afwTable.SourceTable.make(self.schema, idFactory)
301 table.setMetadata(self.algMetadata)
302 results = self.detection.run(
303 table=table,
304 exposure=difference,
305 doSmooth=True,
306 )
307
308 return self.processResults(science, matchedTemplate, difference, results.sources, table,
309 positiveFootprints=results.positive, negativeFootprints=results.negative)
310
311 def processResults(self, science, matchedTemplate, difference, sources, table,
312 positiveFootprints=None, negativeFootprints=None,):
313 """Measure and process the results of source detection.
314
315 Parameters
316 ----------
318 Detected sources on the difference exposure.
319 positiveFootprints : `lsst.afw.detection.FootprintSet`, optional
320 Positive polarity footprints.
321 negativeFootprints : `lsst.afw.detection.FootprintSet`, optional
322 Negative polarity footprints.
324 Table object that will be used to create the SourceCatalog.
325 science : `lsst.afw.image.ExposureF`
326 Science exposure that the template was subtracted from.
327 matchedTemplate : `lsst.afw.image.ExposureF`
328 Warped and PSF-matched template that was used produce the
329 difference image.
330 difference : `lsst.afw.image.ExposureF`
331 Result of subtracting template from the science image.
332
333 Returns
334 -------
335 measurementResults : `lsst.pipe.base.Struct`
336
337 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
338 Subtracted exposure with detection mask applied.
339 ``diaSources`` : `lsst.afw.table.SourceCatalog`
340 The catalog of detected sources.
341 """
342 if self.config.doMerge:
343 fpSet = positiveFootprints
344 fpSet.merge(negativeFootprints, self.config.growFootprint,
345 self.config.growFootprint, False)
346 diaSources = afwTable.SourceCatalog(table)
347 fpSet.makeSources(diaSources)
348 self.log.info("Merging detections into %d sources", len(diaSources))
349 else:
350 diaSources = sources
351
352 if self.config.doSkySources:
353 self.addSkySources(diaSources, difference.mask, difference.info.id)
354
355 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
356
357 if self.config.doForcedMeasurement:
358 self.measureForcedSources(diaSources, science, difference.getWcs())
359
360 measurementResults = pipeBase.Struct(
361 subtractedMeasuredExposure=difference,
362 diaSources=diaSources,
363 )
364
365 return measurementResults
366
367 def addSkySources(self, diaSources, mask, seed):
368 """Add sources in empty regions of the difference image
369 for measuring the background.
370
371 Parameters
372 ----------
373 diaSources : `lsst.afw.table.SourceCatalog`
374 The catalog of detected sources.
375 mask : `lsst.afw.image.Mask`
376 Mask plane for determining regions where Sky sources can be added.
377 seed : `int`
378 Seed value to initialize the random number generator.
379 """
380 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
381 if skySourceFootprints:
382 for foot in skySourceFootprints:
383 s = diaSources.addNew()
384 s.setFootprint(foot)
385 s.set(self.skySourceKey, True)
386
387 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
388 """Use (matched) template and science image to constrain dipole fitting.
389
390 Parameters
391 ----------
392 diaSources : `lsst.afw.table.SourceCatalog`
393 The catalog of detected sources.
394 science : `lsst.afw.image.ExposureF`
395 Science exposure that the template was subtracted from.
396 difference : `lsst.afw.image.ExposureF`
397 Result of subtracting template from the science image.
398 matchedTemplate : `lsst.afw.image.ExposureF`
399 Warped and PSF-matched template that was used produce the
400 difference image.
401 """
402 # Note that this may not be correct if we convolved the science image.
403 # In the future we may wish to persist the matchedScience image.
404 self.measurement.run(diaSources, difference, science, matchedTemplate)
405 if self.config.doApCorr:
406 self.applyApCorr.run(
407 catalog=diaSources,
408 apCorrMap=difference.getInfo().getApCorrMap()
409 )
410
411 def measureForcedSources(self, diaSources, science, wcs):
412 """Perform forced measurement of the diaSources on the science image.
413
414 Parameters
415 ----------
416 diaSources : `lsst.afw.table.SourceCatalog`
417 The catalog of detected sources.
418 science : `lsst.afw.image.ExposureF`
419 Science exposure that the template was subtracted from.
421 Coordinate system definition (wcs) for the exposure.
422 """
423 # Run forced psf photometry on the PVI at the diaSource locations.
424 # Copy the measured flux and error into the diaSource.
425 forcedSources = self.forcedMeasurement.generateMeasCat(
426 science, diaSources, wcs)
427 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
428 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
429 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
430 "ip_diffim_forced_PsfFlux_instFlux", True)
431 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
432 "ip_diffim_forced_PsfFlux_instFluxErr", True)
433 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
434 "ip_diffim_forced_PsfFlux_area", True)
435 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
436 "ip_diffim_forced_PsfFlux_flag", True)
437 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
438 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
439 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
440 "ip_diffim_forced_PsfFlux_flag_edge", True)
441 for diaSource, forcedSource in zip(diaSources, forcedSources):
442 diaSource.assign(forcedSource, mapper)
443
444
445class DetectAndMeasureScoreConnections(DetectAndMeasureConnections):
446 scoreExposure = pipeBase.connectionTypes.Input(
447 doc="Maximum likelihood image for detection.",
448 dimensions=("instrument", "visit", "detector"),
449 storageClass="ExposureF",
450 name="{fakesType}{coaddName}Diff_scoreExp",
451 )
452
453
454class DetectAndMeasureScoreConfig(DetectAndMeasureConfig,
455 pipelineConnections=DetectAndMeasureScoreConnections):
456 pass
457
458
459class DetectAndMeasureScoreTask(DetectAndMeasureTask):
460 """Detect DIA sources using a score image,
461 and measure the detections on the difference image.
462
463 Source detection is run on the supplied score, or maximum likelihood,
464 image. Note that no additional convolution will be done in this case.
465 Close positive and negative detections will optionally be merged into
466 dipole diaSources.
467 Sky sources, or forced detections in background regions, will optionally
468 be added, and the configured measurement algorithm will be run on all
469 detections.
470 """
471 ConfigClass = DetectAndMeasureScoreConfig
472 _DefaultName = "detectAndMeasureScore"
473
474 @timeMethod
475 def run(self, science, matchedTemplate, difference, scoreExposure,
476 idFactory=None):
477 """Detect and measure sources on a score image.
478
479 Parameters
480 ----------
481 science : `lsst.afw.image.ExposureF`
482 Science exposure that the template was subtracted from.
483 matchedTemplate : `lsst.afw.image.ExposureF`
484 Warped and PSF-matched template that was used produce the
485 difference image.
486 difference : `lsst.afw.image.ExposureF`
487 Result of subtracting template from the science image.
488 scoreExposure : `lsst.afw.image.ExposureF`
489 Score or maximum likelihood difference image
490 idFactory : `lsst.afw.table.IdFactory`, optional
491 Generator object to assign ids to detected sources in the difference image.
492
493 Returns
494 -------
495 measurementResults : `lsst.pipe.base.Struct`
496
497 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
498 Subtracted exposure with detection mask applied.
499 ``diaSources`` : `lsst.afw.table.SourceCatalog`
500 The catalog of detected sources.
501 """
502 # Ensure that we start with an empty detection mask.
503 mask = scoreExposure.mask
504 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
505
506 table = afwTable.SourceTable.make(self.schema, idFactory)
507 table.setMetadata(self.algMetadata)
508 results = self.detection.run(
509 table=table,
510 exposure=scoreExposure,
511 doSmooth=False,
512 )
513 # Copy the detection mask from the Score image to the difference image
514 difference.mask.assign(scoreExposure.mask, scoreExposure.getBBox())
515
516 return self.processResults(science, matchedTemplate, difference, results.sources, table,
517 positiveFootprints=results.positive, negativeFootprints=results.negative)
def run(self, coaddExposures, bbox, wcs, dataIds, physical_filter=None, **kwargs)
Definition: getTemplate.py:188