Coverage for python/lsst/ip/diffim/detectAndMeasure.py: 35%
122 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-30 03:17 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-30 03:17 -0700
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import lsst.afw.table as afwTable
23import lsst.daf.base as dafBase
24from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask
25from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
26import lsst.meas.extensions.trailedSources # noqa: F401
27from lsst.obs.base import ExposureIdInfo
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.utils
31from lsst.utils.timer import timeMethod
33from . import DipoleFitTask
35__all__ = ["DetectAndMeasureConfig", "DetectAndMeasureTask"]
38class DetectAndMeasureConnections(pipeBase.PipelineTaskConnections,
39 dimensions=("instrument", "visit", "detector"),
40 defaultTemplates={"coaddName": "deep",
41 "warpTypeSuffix": "",
42 "fakesType": ""}):
43 science = pipeBase.connectionTypes.Input(
44 doc="Input science exposure.",
45 dimensions=("instrument", "visit", "detector"),
46 storageClass="ExposureF",
47 name="{fakesType}calexp"
48 )
49 matchedTemplate = pipeBase.connectionTypes.Input(
50 doc="Warped and PSF-matched template used to create the difference image.",
51 dimensions=("instrument", "visit", "detector"),
52 storageClass="ExposureF",
53 name="{fakesType}{coaddName}Diff_matchedExp",
54 )
55 difference = pipeBase.connectionTypes.Input(
56 doc="Result of subtracting template from science.",
57 dimensions=("instrument", "visit", "detector"),
58 storageClass="ExposureF",
59 name="{fakesType}{coaddName}Diff_differenceTempExp",
60 )
61 selectSources = pipeBase.connectionTypes.Input(
62 doc="Sources measured on the science exposure; will be matched to the "
63 "detected sources on the difference image.",
64 dimensions=("instrument", "visit", "detector"),
65 storageClass="SourceCatalog",
66 name="{fakesType}src",
67 )
68 outputSchema = pipeBase.connectionTypes.InitOutput(
69 doc="Schema (as an example catalog) for output DIASource catalog.",
70 storageClass="SourceCatalog",
71 name="{fakesType}{coaddName}Diff_diaSrc_schema",
72 )
73 diaSources = pipeBase.connectionTypes.Output(
74 doc="Detected diaSources on the difference image.",
75 dimensions=("instrument", "visit", "detector"),
76 storageClass="SourceCatalog",
77 name="{fakesType}{coaddName}Diff_diaSrc",
78 )
79 subtractedMeasuredExposure = pipeBase.connectionTypes.Output(
80 doc="Difference image with detection mask plane filled in.",
81 dimensions=("instrument", "visit", "detector"),
82 storageClass="ExposureF",
83 name="{fakesType}{coaddName}Diff_differenceExp",
84 )
87class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig,
88 pipelineConnections=DetectAndMeasureConnections):
89 """Config for DetectAndMeasureTask
90 """
91 doMerge = pexConfig.Field(
92 dtype=bool,
93 default=True,
94 doc="Merge positive and negative diaSources with grow radius "
95 "set by growFootprint"
96 )
97 doForcedMeasurement = pexConfig.Field(
98 dtype=bool,
99 default=True,
100 doc="Force photometer diaSource locations on PVI?")
101 doAddMetrics = pexConfig.Field(
102 dtype=bool,
103 default=False,
104 doc="Add columns to the source table to hold analysis metrics?"
105 )
106 detection = pexConfig.ConfigurableField(
107 target=SourceDetectionTask,
108 doc="Final source detection for diaSource measurement",
109 )
110 measurement = pexConfig.ConfigurableField(
111 target=DipoleFitTask,
112 doc="Task to measure sources on the difference image.",
113 )
114 doApCorr = lsst.pex.config.Field(
115 dtype=bool,
116 default=True,
117 doc="Run subtask to apply aperture corrections"
118 )
119 applyApCorr = lsst.pex.config.ConfigurableField(
120 target=ApplyApCorrTask,
121 doc="Task to apply aperture corrections"
122 )
123 forcedMeasurement = pexConfig.ConfigurableField(
124 target=ForcedMeasurementTask,
125 doc="Task to force photometer science image at diaSource locations.",
126 )
127 growFootprint = pexConfig.Field(
128 dtype=int,
129 default=2,
130 doc="Grow positive and negative footprints by this many pixels before merging"
131 )
132 diaSourceMatchRadius = pexConfig.Field(
133 dtype=float,
134 default=0.5,
135 doc="Match radius (in arcseconds) for DiaSource to Source association"
136 )
137 doSkySources = pexConfig.Field(
138 dtype=bool,
139 default=False,
140 doc="Generate sky sources?",
141 )
142 skySources = pexConfig.ConfigurableField(
143 target=SkyObjectsTask,
144 doc="Generate sky sources",
145 )
147 def setDefaults(self):
148 # DiaSource Detection
149 self.detection.thresholdPolarity = "both"
150 self.detection.thresholdValue = 5.0
151 self.detection.reEstimateBackground = False
152 self.detection.thresholdType = "pixel_stdev"
154 # Add filtered flux measurement, the correct measurement for pre-convolved images.
155 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
156 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
157 'base_LocalPhotoCalib',
158 'base_LocalWcs']
160 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
161 self.forcedMeasurement.copyColumns = {
162 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
163 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
164 self.forcedMeasurement.slots.shape = None
167class DetectAndMeasureTask(lsst.pipe.base.PipelineTask):
168 """Detect and measure sources on a difference image.
169 """
170 ConfigClass = DetectAndMeasureConfig
171 _DefaultName = "detectAndMeasure"
173 def __init__(self, **kwargs):
174 super().__init__(**kwargs)
175 self.schema = afwTable.SourceTable.makeMinimalSchema()
177 self.algMetadata = dafBase.PropertyList()
178 self.makeSubtask("detection", schema=self.schema)
179 self.makeSubtask("measurement", schema=self.schema,
180 algMetadata=self.algMetadata)
181 if self.config.doApCorr:
182 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
183 if self.config.doForcedMeasurement:
184 self.schema.addField(
185 "ip_diffim_forced_PsfFlux_instFlux", "D",
186 "Forced PSF flux measured on the direct image.",
187 units="count")
188 self.schema.addField(
189 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
190 "Forced PSF flux error measured on the direct image.",
191 units="count")
192 self.schema.addField(
193 "ip_diffim_forced_PsfFlux_area", "F",
194 "Forced PSF flux effective area of PSF.",
195 units="pixel")
196 self.schema.addField(
197 "ip_diffim_forced_PsfFlux_flag", "Flag",
198 "Forced PSF flux general failure flag.")
199 self.schema.addField(
200 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
201 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
202 self.schema.addField(
203 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
204 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
205 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
207 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
208 self.schema.addField("srcMatchId", "L", "unique id of source match")
209 if self.config.doSkySources:
210 self.makeSubtask("skySources")
211 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
213 # initialize InitOutputs
214 self.outputSchema = afwTable.SourceCatalog(self.schema)
215 self.outputSchema.getTable().setMetadata(self.algMetadata)
217 @staticmethod
218 def makeIdFactory(expId, expBits):
219 """Create IdFactory instance for unique 64 bit diaSource id-s.
221 Parameters
222 ----------
223 expId : `int`
224 Exposure id.
226 expBits: `int`
227 Number of used bits in ``expId``.
229 Notes
230 -----
231 The diasource id-s consists of the ``expId`` stored fixed in the highest value
232 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
233 low value end of the integer.
235 Returns
236 -------
237 idFactory: `lsst.afw.table.IdFactory`
238 """
239 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
241 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
242 inputRefs: pipeBase.InputQuantizedConnection,
243 outputRefs: pipeBase.OutputQuantizedConnection):
244 inputs = butlerQC.get(inputRefs)
245 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
246 returnMaxBits=True)
247 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
249 outputs = self.run(inputs['science'],
250 inputs['matchedTemplate'],
251 inputs['difference'],
252 inputs['selectSources'],
253 idFactory=idFactory)
254 butlerQC.put(outputs, outputRefs)
256 @timeMethod
257 def run(self, science, matchedTemplate, difference, selectSources,
258 idFactory=None):
259 """Detect and measure sources on a difference image.
261 Parameters
262 ----------
263 science : `lsst.afw.image.ExposureF`
264 Science exposure that the template was subtracted from.
265 matchedTemplate : `lsst.afw.image.ExposureF`
266 Warped and PSF-matched template that was used produce the
267 difference image.
268 difference : `lsst.afw.image.ExposureF`
269 Result of subtracting template from the science image.
270 selectSources : `lsst.afw.table.SourceCatalog`
271 Identified sources on the science exposure.
272 idFactory : `lsst.afw.table.IdFactory`, optional
273 Generator object to assign ids to detected sources in the difference image.
275 Returns
276 -------
277 results : `lsst.pipe.base.Struct`
279 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF`
280 Subtracted exposure with detection mask applied.
281 ``diaSources`` : `lsst.afw.table.SourceCatalog`
282 The catalog of detected sources.
283 """
284 # Ensure that we start with an empty detection mask.
285 mask = difference.mask
286 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
288 table = afwTable.SourceTable.make(self.schema, idFactory)
289 table.setMetadata(self.algMetadata)
290 results = self.detection.run(
291 table=table,
292 exposure=difference,
293 doSmooth=True,
294 )
296 if self.config.doMerge:
297 fpSet = results.fpSets.positive
298 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
299 self.config.growFootprint, False)
300 diaSources = afwTable.SourceCatalog(table)
301 fpSet.makeSources(diaSources)
302 self.log.info("Merging detections into %d sources", len(diaSources))
303 else:
304 diaSources = results.sources
306 if self.config.doSkySources:
307 self.addSkySources(diaSources, difference.mask, difference.info.id)
309 self.measureDiaSources(diaSources, science, difference, matchedTemplate)
311 if self.config.doForcedMeasurement:
312 self.measureForcedSources(diaSources, science, difference.getWcs())
314 return pipeBase.Struct(
315 subtractedMeasuredExposure=difference,
316 diaSources=diaSources,
317 )
319 def addSkySources(self, diaSources, mask, seed):
320 """Add sources in empty regions of the difference image
321 for measuring the background.
323 Parameters
324 ----------
325 diaSources : `lsst.afw.table.SourceCatalog`
326 The catalog of detected sources.
327 mask : `lsst.afw.image.Mask`
328 Mask plane for determining regions where Sky sources can be added.
329 seed : `int`
330 Seed value to initialize the random number generator.
331 """
332 skySourceFootprints = self.skySources.run(mask=mask, seed=seed)
333 if skySourceFootprints:
334 for foot in skySourceFootprints:
335 s = diaSources.addNew()
336 s.setFootprint(foot)
337 s.set(self.skySourceKey, True)
339 def measureDiaSources(self, diaSources, science, difference, matchedTemplate):
340 """Use (matched) template and science image to constrain dipole fitting.
342 Parameters
343 ----------
344 diaSources : `lsst.afw.table.SourceCatalog`
345 The catalog of detected sources.
346 science : `lsst.afw.image.ExposureF`
347 Science exposure that the template was subtracted from.
348 difference : `lsst.afw.image.ExposureF`
349 Result of subtracting template from the science image.
350 matchedTemplate : `lsst.afw.image.ExposureF`
351 Warped and PSF-matched template that was used produce the
352 difference image.
353 """
354 # Note that this may not be correct if we convolved the science image.
355 # In the future we may wish to persist the matchedScience image.
356 self.measurement.run(diaSources, difference, science, matchedTemplate)
357 if self.config.doApCorr:
358 self.applyApCorr.run(
359 catalog=diaSources,
360 apCorrMap=difference.getInfo().getApCorrMap()
361 )
363 def measureForcedSources(self, diaSources, science, wcs):
364 """Perform forced measurement of the diaSources on the science image.
366 Parameters
367 ----------
368 diaSources : `lsst.afw.table.SourceCatalog`
369 The catalog of detected sources.
370 science : `lsst.afw.image.ExposureF`
371 Science exposure that the template was subtracted from.
372 wcs : `lsst.afw.geom.SkyWcs`
373 Coordinate system definition (wcs) for the exposure.
374 """
375 # Run forced psf photometry on the PVI at the diaSource locations.
376 # Copy the measured flux and error into the diaSource.
377 forcedSources = self.forcedMeasurement.generateMeasCat(
378 science, diaSources, wcs)
379 self.forcedMeasurement.run(forcedSources, science, diaSources, wcs)
380 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
381 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
382 "ip_diffim_forced_PsfFlux_instFlux", True)
383 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
384 "ip_diffim_forced_PsfFlux_instFluxErr", True)
385 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
386 "ip_diffim_forced_PsfFlux_area", True)
387 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
388 "ip_diffim_forced_PsfFlux_flag", True)
389 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
390 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
391 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
392 "ip_diffim_forced_PsfFlux_flag_edge", True)
393 for diaSource, forcedSource in zip(diaSources, forcedSources):
394 diaSource.assign(forcedSource, mapper)