Coverage for python/lsst/pipe/tasks/imageDifference.py: 15%
551 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-21 10:01 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-21 10:01 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
24import math
25import random
26import numpy
28import lsst.utils
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31import lsst.daf.base as dafBase
32import lsst.geom as geom
33import lsst.afw.math as afwMath
34import lsst.afw.table as afwTable
35import lsst.meas.extensions.trailedSources # noqa: F401
36from lsst.meas.algorithms import (SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask,
37 LoadReferenceObjectsConfig, SkyObjectsTask,
38 ScaleVarianceTask)
39from lsst.meas.astrom import AstrometryConfig, AstrometryTask
40from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
41from lsst.pipe.tasks.registerImage import RegisterTask
42from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
43 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
44 GetCoaddAsTemplateTask, DipoleFitTask,
45 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
46import lsst.ip.diffim.diffimTools as diffimTools
47import lsst.ip.diffim.utils as diUtils
48import lsst.afw.display as afwDisplay
49from lsst.skymap import BaseSkyMap
50from lsst.obs.base import ExposureIdInfo
51from lsst.utils.timer import timeMethod
53from deprecated.sphinx import deprecated
55FwhmPerSigma = 2*math.sqrt(2*math.log(2))
56IqrToSigma = 0.741
59class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
60 dimensions=("instrument", "visit", "detector", "skymap"),
61 defaultTemplates={"coaddName": "deep",
62 "skyMapName": "deep",
63 "warpTypeSuffix": "",
64 "fakesType": ""}):
66 exposure = pipeBase.connectionTypes.Input(
67 doc="Input science exposure to subtract from.",
68 dimensions=("instrument", "visit", "detector"),
69 storageClass="ExposureF",
70 name="{fakesType}calexp"
71 )
73 # TODO DM-22953
74 # kernelSources = pipeBase.connectionTypes.Input(
75 # doc="Source catalog produced in calibrate task for kernel candidate sources",
76 # name="src",
77 # storageClass="SourceCatalog",
78 # dimensions=("instrument", "visit", "detector"),
79 # )
81 skyMap = pipeBase.connectionTypes.Input(
82 doc="Input definition of geometry/bbox and projection/wcs for template exposures",
83 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
84 dimensions=("skymap", ),
85 storageClass="SkyMap",
86 )
87 coaddExposures = pipeBase.connectionTypes.Input(
88 doc="Input template to match and subtract from the exposure",
89 dimensions=("tract", "patch", "skymap", "band"),
90 storageClass="ExposureF",
91 name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
92 multiple=True,
93 deferLoad=True
94 )
95 dcrCoadds = pipeBase.connectionTypes.Input(
96 doc="Input DCR template to match and subtract from the exposure",
97 name="{fakesType}dcrCoadd{warpTypeSuffix}",
98 storageClass="ExposureF",
99 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
100 multiple=True,
101 deferLoad=True
102 )
103 finalizedPsfApCorrCatalog = pipeBase.connectionTypes.Input(
104 doc=("Per-visit finalized psf models and aperture correction maps. "
105 "These catalogs use the detector id for the catalog id, "
106 "sorted on id for fast lookup."),
107 name="finalized_psf_ap_corr_catalog",
108 storageClass="ExposureCatalog",
109 dimensions=("instrument", "visit"),
110 )
111 outputSchema = pipeBase.connectionTypes.InitOutput(
112 doc="Schema (as an example catalog) for output DIASource catalog.",
113 storageClass="SourceCatalog",
114 name="{fakesType}{coaddName}Diff_diaSrc_schema",
115 )
116 subtractedExposure = pipeBase.connectionTypes.Output(
117 doc="Output AL difference or Zogy proper difference image",
118 dimensions=("instrument", "visit", "detector"),
119 storageClass="ExposureF",
120 name="{fakesType}{coaddName}Diff_differenceExp",
121 )
122 scoreExposure = pipeBase.connectionTypes.Output(
123 doc="Output AL likelihood or Zogy score image",
124 dimensions=("instrument", "visit", "detector"),
125 storageClass="ExposureF",
126 name="{fakesType}{coaddName}Diff_scoreExp",
127 )
128 warpedExposure = pipeBase.connectionTypes.Output(
129 doc="Warped template used to create `subtractedExposure`.",
130 dimensions=("instrument", "visit", "detector"),
131 storageClass="ExposureF",
132 name="{fakesType}{coaddName}Diff_warpedExp",
133 )
134 matchedExposure = pipeBase.connectionTypes.Output(
135 doc="Warped template used to create `subtractedExposure`.",
136 dimensions=("instrument", "visit", "detector"),
137 storageClass="ExposureF",
138 name="{fakesType}{coaddName}Diff_matchedExp",
139 )
140 diaSources = pipeBase.connectionTypes.Output(
141 doc="Output detected diaSources on the difference image",
142 dimensions=("instrument", "visit", "detector"),
143 storageClass="SourceCatalog",
144 name="{fakesType}{coaddName}Diff_diaSrc",
145 )
147 def __init__(self, *, config=None):
148 super().__init__(config=config)
149 if config.coaddName == 'dcr':
150 self.inputs.remove("coaddExposures")
151 else:
152 self.inputs.remove("dcrCoadds")
153 if not config.doWriteSubtractedExp:
154 self.outputs.remove("subtractedExposure")
155 if not config.doWriteScoreExp:
156 self.outputs.remove("scoreExposure")
157 if not config.doWriteWarpedExp:
158 self.outputs.remove("warpedExposure")
159 if not config.doWriteMatchedExp:
160 self.outputs.remove("matchedExposure")
161 if not config.doWriteSources:
162 self.outputs.remove("diaSources")
163 if not config.doApplyFinalizedPsf:
164 self.inputs.remove("finalizedPsfApCorrCatalog")
166 # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
167 # Make kernelSources optional
170class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
171 pipelineConnections=ImageDifferenceTaskConnections):
172 """Config for ImageDifferenceTask.
173 """
175 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
176 doc="Add background to calexp before processing it. "
177 "Useful as ipDiffim does background matching.")
178 doUseRegister = pexConfig.Field(dtype=bool, default=False,
179 doc="Re-compute astrometry on the template. "
180 "Use image-to-image registration to align template with "
181 "science image (AL only).")
182 doDebugRegister = pexConfig.Field(dtype=bool, default=False,
183 doc="Writing debugging data for doUseRegister")
184 doSelectSources = pexConfig.Field(dtype=bool, default=False,
185 doc="Select stars to use for kernel fitting (AL only)")
186 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
187 doc="Select stars of extreme color as part "
188 "of the control sample (AL only)")
189 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
190 doc="Select stars that are variable to be part "
191 "of the control sample (AL only)")
192 doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
193 doPreConvolve = pexConfig.Field(dtype=bool, default=False,
194 doc="Not in use. Superseded by useScoreImageDetection.",
195 deprecated="This option superseded by useScoreImageDetection."
196 " Will be removed after v22.")
197 useScoreImageDetection = pexConfig.Field(
198 dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or "
199 "the Zogy score image. Use it for source detection (if doDetection=True).")
200 doWriteScoreExp = pexConfig.Field(
201 dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?")
202 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
203 doc="Scale variance of the template before PSF matching")
204 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
205 doc="Scale variance of the diffim before PSF matching. "
206 "You may do either this or template variance scaling, "
207 "or neither. (Doing both is a waste of CPU.)")
208 useGaussianForPreConvolution = pexConfig.Field(
209 dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution "
210 "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)")
211 doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
212 doDecorrelation = pexConfig.Field(dtype=bool, default=True,
213 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
214 "kernel convolution (AL only)? If True, also update the diffim PSF.")
215 doMerge = pexConfig.Field(dtype=bool, default=True,
216 doc="Merge positive and negative diaSources with grow radius "
217 "set by growFootprint")
218 doMatchSources = pexConfig.Field(dtype=bool, default=False,
219 doc="Match diaSources with input calexp sources and ref catalog sources")
220 doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
221 doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
222 doForcedMeasurement = pexConfig.Field(
223 dtype=bool,
224 default=True,
225 doc="Force photometer diaSource locations on PVI?")
226 doWriteSubtractedExp = pexConfig.Field(
227 dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
228 doWriteWarpedExp = pexConfig.Field(
229 dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
230 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
231 doc="Write warped and PSF-matched template coadd exposure?")
232 doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
233 doAddMetrics = pexConfig.Field(dtype=bool, default=False,
234 doc="Add columns to the source table to hold analysis metrics?")
235 doApplyFinalizedPsf = pexConfig.Field(
236 doc="Whether to apply finalized psf models and aperture correction map.",
237 dtype=bool,
238 default=False,
239 )
241 coaddName = pexConfig.Field(
242 doc="coadd name: typically one of deep, goodSeeing, or dcr",
243 dtype=str,
244 default="deep",
245 )
246 convolveTemplate = pexConfig.Field(
247 doc="Which image gets convolved (default = template)",
248 dtype=bool,
249 default=True
250 )
251 refObjLoader = pexConfig.ConfigField(
252 dtype=LoadReferenceObjectsConfig,
253 doc="reference object loader",
254 )
255 astrometer = pexConfig.ConfigurableField(
256 target=AstrometryTask,
257 doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
258 )
259 sourceSelector = pexConfig.ConfigurableField(
260 target=ObjectSizeStarSelectorTask,
261 doc="Source selection algorithm",
262 )
263 subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
264 decorrelate = pexConfig.ConfigurableField(
265 target=DecorrelateALKernelSpatialTask,
266 doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
267 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
268 "default of 5.5).",
269 )
270 # Old style ImageMapper grid. ZogyTask has its own grid option
271 doSpatiallyVarying = pexConfig.Field(
272 dtype=bool,
273 default=False,
274 doc="Perform A&L decorrelation on a grid across the "
275 "image in order to allow for spatial variations. Zogy does not use this option."
276 )
277 detection = pexConfig.ConfigurableField(
278 target=SourceDetectionTask,
279 doc="Low-threshold detection for final measurement",
280 )
281 measurement = pexConfig.ConfigurableField(
282 target=DipoleFitTask,
283 doc="Enable updated dipole fitting method",
284 )
285 doApCorr = lsst.pex.config.Field(
286 dtype=bool,
287 default=True,
288 doc="Run subtask to apply aperture corrections"
289 )
290 applyApCorr = lsst.pex.config.ConfigurableField(
291 target=ApplyApCorrTask,
292 doc="Subtask to apply aperture corrections"
293 )
294 forcedMeasurement = pexConfig.ConfigurableField(
295 target=ForcedMeasurementTask,
296 doc="Subtask to force photometer PVI at diaSource location.",
297 )
298 getTemplate = pexConfig.ConfigurableField(
299 target=GetCoaddAsTemplateTask,
300 doc="Subtask to retrieve template exposure and sources",
301 )
302 scaleVariance = pexConfig.ConfigurableField(
303 target=ScaleVarianceTask,
304 doc="Subtask to rescale the variance of the template "
305 "to the statistically expected level"
306 )
307 controlStepSize = pexConfig.Field(
308 doc="What step size (every Nth one) to select a control sample from the kernelSources",
309 dtype=int,
310 default=5
311 )
312 controlRandomSeed = pexConfig.Field(
313 doc="Random seed for shuffing the control sample",
314 dtype=int,
315 default=10
316 )
317 register = pexConfig.ConfigurableField(
318 target=RegisterTask,
319 doc="Task to enable image-to-image image registration (warping)",
320 )
321 kernelSourcesFromRef = pexConfig.Field(
322 doc="Select sources to measure kernel from reference catalog if True, template if false",
323 dtype=bool,
324 default=False
325 )
326 templateSipOrder = pexConfig.Field(
327 dtype=int, default=2,
328 doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
329 )
330 growFootprint = pexConfig.Field(
331 dtype=int, default=2,
332 doc="Grow positive and negative footprints by this amount before merging"
333 )
334 diaSourceMatchRadius = pexConfig.Field(
335 dtype=float, default=0.5,
336 doc="Match radius (in arcseconds) for DiaSource to Source association"
337 )
338 requiredTemplateFraction = pexConfig.Field(
339 dtype=float, default=0.1,
340 doc="Do not attempt to run task if template covers less than this fraction of pixels."
341 "Setting to 0 will always attempt image subtraction"
342 )
343 doSkySources = pexConfig.Field(
344 dtype=bool,
345 default=False,
346 doc="Generate sky sources?",
347 )
348 skySources = pexConfig.ConfigurableField(
349 target=SkyObjectsTask,
350 doc="Generate sky sources",
351 )
353 def setDefaults(self):
354 # defaults are OK for catalog and diacatalog
356 self.subtract['al'].kernel.name = "AL"
357 self.subtract['al'].kernel.active.fitForBackground = True
358 self.subtract['al'].kernel.active.spatialKernelOrder = 1
359 self.subtract['al'].kernel.active.spatialBgOrder = 2
361 # DiaSource Detection
362 self.detection.thresholdPolarity = "both"
363 self.detection.thresholdValue = 5.0
364 self.detection.reEstimateBackground = False
365 self.detection.thresholdType = "pixel_stdev"
367 # Add filtered flux measurement, the correct measurement for pre-convolved images.
368 # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
369 # To change that you must modify algorithms.names in the task's applyOverrides method,
370 # after the user has set doPreConvolve.
371 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
372 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
373 'base_LocalPhotoCalib',
374 'base_LocalWcs']
376 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
377 self.forcedMeasurement.copyColumns = {
378 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
379 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
380 self.forcedMeasurement.slots.shape = None
382 # For shuffling the control sample
383 random.seed(self.controlRandomSeed)
385 def validate(self):
386 pexConfig.Config.validate(self)
387 if not self.doSubtract and not self.doDetection:
388 raise ValueError("Either doSubtract or doDetection must be enabled.")
389 if self.doMeasurement and not self.doDetection:
390 raise ValueError("Cannot run source measurement without source detection.")
391 if self.doMerge and not self.doDetection:
392 raise ValueError("Cannot run source merging without source detection.")
393 if self.doSkySources and not self.doDetection:
394 raise ValueError("Cannot run sky source creation without source detection.")
395 if self.doUseRegister and not self.doSelectSources:
396 raise ValueError("doUseRegister=True and doSelectSources=False. "
397 "Cannot run RegisterTask without selecting sources.")
398 if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
399 raise ValueError("Scaling the diffim variance and scaling the template variance "
400 "are both set. Please choose one or the other.")
401 # We cannot allow inconsistencies that would lead to None or not available output products
402 if self.subtract.name == 'zogy':
403 if self.doWriteMatchedExp:
404 raise ValueError("doWriteMatchedExp=True Matched exposure is not "
405 "calculated in zogy subtraction.")
406 if self.doAddMetrics:
407 raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
408 if self.doDecorrelation:
409 raise ValueError(
410 "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
411 if self.doSelectSources:
412 raise ValueError(
413 "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
414 if self.useGaussianForPreConvolution:
415 raise ValueError(
416 "useGaussianForPreConvolution=True This is an AL subtraction only option.")
417 else:
418 # AL only consistency checks
419 if self.useScoreImageDetection and not self.convolveTemplate:
420 raise ValueError(
421 "convolveTemplate=False and useScoreImageDetection=True "
422 "Pre-convolution and matching of the science image is not a supported operation.")
423 if self.doWriteSubtractedExp and self.useScoreImageDetection:
424 raise ValueError(
425 "doWriteSubtractedExp=True and useScoreImageDetection=True "
426 "Regular difference image is not calculated. "
427 "AL subtraction calculates either the regular difference image or the score image.")
428 if self.doWriteScoreExp and not self.useScoreImageDetection:
429 raise ValueError(
430 "doWriteScoreExp=True and useScoreImageDetection=False "
431 "Score image is not calculated. "
432 "AL subtraction calculates either the regular difference image or the score image.")
433 if self.doAddMetrics and not self.doSubtract:
434 raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
435 if self.useGaussianForPreConvolution and not self.useScoreImageDetection:
436 raise ValueError(
437 "useGaussianForPreConvolution=True and useScoreImageDetection=False "
438 "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.")
441@deprecated(reason="This Task has been replaced with lsst.ip.diffim.subtractImages"
442 " and lsst.ip.diffim.detectAndMeasure. Will be removed after v25.",
443 version="v24.0", category=FutureWarning)
444class ImageDifferenceTask(pipeBase.PipelineTask):
445 """Subtract an image from a template and measure the result.
447 Parameters
448 ----------
449 butler : `lsst.daf.butler.Butler` or `None`, optional
450 Butler object to use in constructing reference object loaders.
451 **kwargs
452 Additional keyword arguments.
453 """
454 ConfigClass = ImageDifferenceConfig
455 _DefaultName = "imageDifference"
457 def __init__(self, butler=None, **kwargs):
458 super().__init__(**kwargs)
459 self.makeSubtask("getTemplate")
461 self.makeSubtask("subtract")
463 if self.config.subtract.name == 'al' and self.config.doDecorrelation:
464 self.makeSubtask("decorrelate")
466 if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
467 self.makeSubtask("scaleVariance")
469 if self.config.doUseRegister:
470 self.makeSubtask("register")
471 self.schema = afwTable.SourceTable.makeMinimalSchema()
473 if self.config.doSelectSources:
474 self.makeSubtask("sourceSelector")
475 if self.config.kernelSourcesFromRef:
476 self.makeSubtask('refObjLoader', butler=butler)
477 self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
479 self.algMetadata = dafBase.PropertyList()
480 if self.config.doDetection:
481 self.makeSubtask("detection", schema=self.schema)
482 if self.config.doMeasurement:
483 self.makeSubtask("measurement", schema=self.schema,
484 algMetadata=self.algMetadata)
485 if self.config.doApCorr:
486 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
487 if self.config.doForcedMeasurement:
488 self.schema.addField(
489 "ip_diffim_forced_PsfFlux_instFlux", "D",
490 "Forced PSF flux measured on the direct image.",
491 units="count")
492 self.schema.addField(
493 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
494 "Forced PSF flux error measured on the direct image.",
495 units="count")
496 self.schema.addField(
497 "ip_diffim_forced_PsfFlux_area", "F",
498 "Forced PSF flux effective area of PSF.",
499 units="pixel")
500 self.schema.addField(
501 "ip_diffim_forced_PsfFlux_flag", "Flag",
502 "Forced PSF flux general failure flag.")
503 self.schema.addField(
504 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
505 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
506 self.schema.addField(
507 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
508 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
509 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
510 if self.config.doMatchSources:
511 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
512 self.schema.addField("srcMatchId", "L", "unique id of source match")
513 if self.config.doSkySources:
514 self.makeSubtask("skySources")
515 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
517 # initialize InitOutputs
518 self.outputSchema = afwTable.SourceCatalog(self.schema)
519 self.outputSchema.getTable().setMetadata(self.algMetadata)
521 @staticmethod
522 def makeIdFactory(expId, expBits):
523 """Create IdFactory instance for unique 64 bit diaSource id-s.
525 Parameters
526 ----------
527 expId : `int`
528 Exposure ID.
530 expBits : `int`
531 Number of used bits in ``expId``.
533 Returns
534 -------
535 idFactory : `lsst.afw.table.IdFactory`
536 Generator object to assign ids to detected sources in the difference image.
538 Notes
539 -----
540 The diasource id-s consists of the ``expId`` stored fixed in the highest value
541 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
542 low value end of the integer.
543 """
544 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
546 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
547 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
548 inputRefs: pipeBase.InputQuantizedConnection,
549 outputRefs: pipeBase.OutputQuantizedConnection):
550 inputs = butlerQC.get(inputRefs)
551 self.log.info("Processing %s", butlerQC.quantum.dataId)
553 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None)
554 exposure = self.prepareCalibratedExposure(
555 inputs["exposure"],
556 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog
557 )
559 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
560 returnMaxBits=True)
561 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
562 if self.config.coaddName == 'dcr':
563 templateExposures = inputRefs.dcrCoadds
564 else:
565 templateExposures = inputRefs.coaddExposures
566 templateStruct = self.getTemplate.runQuantum(
567 exposure, butlerQC, inputRefs.skyMap, templateExposures
568 )
570 self.checkTemplateIsSufficient(templateStruct.exposure)
572 outputs = self.run(exposure=exposure,
573 templateExposure=templateStruct.exposure,
574 idFactory=idFactory)
575 # Consistency with runDataref gen2 handling
576 if outputs.diaSources is None:
577 del outputs.diaSources
578 butlerQC.put(outputs, outputRefs)
580 def prepareCalibratedExposure(self, exposure, finalizedPsfApCorrCatalog=None):
581 """Prepare a calibrated exposure and apply finalized psf if so configured.
583 Parameters
584 ----------
585 exposure : `lsst.afw.image.exposure.Exposure`
586 Input exposure to adjust calibrations.
587 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
588 Exposure catalog with finalized psf models and aperture correction
589 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
590 the detector id for the catalog id, sorted on id for fast lookup.
592 Returns
593 -------
594 exposure : `lsst.afw.image.exposure.Exposure`
595 Exposure with adjusted calibrations.
596 """
597 detectorId = exposure.getInfo().getDetector().getId()
599 if finalizedPsfApCorrCatalog is not None:
600 row = finalizedPsfApCorrCatalog.find(detectorId)
601 if row is None:
602 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; "
603 "Using original psf.", detectorId)
604 else:
605 psf = row.getPsf()
606 apCorrMap = row.getApCorrMap()
607 if psf is None or apCorrMap is None:
608 self.log.warning("Detector id %s has None for psf/apCorrMap in "
609 "finalizedPsfApCorrCatalog; Using original psf.", detectorId)
610 else:
611 exposure.setPsf(psf)
612 exposure.info.setApCorrMap(apCorrMap)
614 return exposure
616 @timeMethod
617 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
618 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
619 """PSF matches, subtract two images and perform detection on the difference image.
621 Parameters
622 ----------
623 exposure : `lsst.afw.image.ExposureF`, optional
624 The science exposure, the minuend in the image subtraction.
625 Can be None only if ``config.doSubtract==False``.
626 selectSources : `lsst.afw.table.SourceCatalog`, optional
627 Identified sources on the science exposure. This catalog is used to
628 select sources in order to perform the AL PSF matching on stamp images
629 around them. The selection steps depend on config options and whether
630 ``templateSources`` and ``matchingSources`` specified.
631 templateExposure : `lsst.afw.image.ExposureF`, optional
632 The template to be subtracted from ``exposure`` in the image subtraction.
633 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
634 The template exposure should cover the same sky area as the science exposure.
635 It is either a stich of patches of a coadd skymap image or a calexp
636 of the same pointing as the science exposure. Can be None only
637 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
638 templateSources : `lsst.afw.table.SourceCatalog`, optional
639 Identified sources on the template exposure.
640 idFactory : `lsst.afw.table.IdFactory`
641 Generator object to assign ids to detected sources in the difference image.
642 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
643 Background exposure to be added back to the science exposure
644 if ``config.doAddCalexpBackground==True``.
645 subtractedExposure : `lsst.afw.image.ExposureF`, optional
646 If ``config.doSubtract==False`` and ``config.doDetection==True``,
647 performs the post subtraction source detection only on this exposure.
648 Otherwise should be None.
650 Returns
651 -------
652 results : `lsst.pipe.base.Struct`
653 Results as a struct with attributes:
655 ``subtractedExposure``
656 Difference image (`lsst.afw.image.ExposureF`).
657 ``scoreExposure``
658 The zogy score exposure, if calculated (`lsst.afw.image.ExposureF` or `None`).
659 ``matchedExposure``
660 The matched PSF exposure (`lsst.afw.image.ExposureF`).
661 ``subtractRes``
662 The returned result structure of the ImagePsfMatchTask subtask (`lsst.pipe.base.Struct`).
663 ``diaSources``
664 The catalog of detected sources (`lsst.afw.table.SourceCatalog`).
665 ``selectSources``
666 The input source catalog with optionally added Qa information
667 (`lsst.afw.table.SourceCatalog`).
669 Notes
670 -----
671 The following major steps are included:
673 - warp template coadd to match WCS of image
674 - PSF match image to warped template
675 - subtract image from PSF-matched, warped template
676 - detect sources
677 - measure sources
679 For details about the image subtraction configuration modes
680 see `lsst.ip.diffim`.
681 """
682 subtractRes = None
683 controlSources = None
684 subtractedExposure = None
685 scoreExposure = None
686 diaSources = None
687 kernelSources = None
688 # We'll clone exposure if modified but will still need the original
689 exposureOrig = exposure
691 if self.config.doAddCalexpBackground:
692 mi = exposure.getMaskedImage()
693 mi += calexpBackgroundExposure.getImage()
695 if not exposure.hasPsf():
696 raise pipeBase.TaskError("Exposure has no psf")
697 sciencePsf = exposure.getPsf()
699 if self.config.doSubtract:
700 if self.config.doScaleTemplateVariance:
701 self.log.info("Rescaling template variance")
702 templateVarFactor = self.scaleVariance.run(
703 templateExposure.getMaskedImage())
704 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
705 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
706 self.metadata.add("psfMatchingAlgorithm", self.config.subtract.name)
708 if self.config.subtract.name == 'zogy':
709 subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
710 scoreExposure = subtractRes.scoreExp
711 subtractedExposure = subtractRes.diffExp
712 subtractRes.subtractedExposure = subtractedExposure
713 subtractRes.matchedExposure = None
715 elif self.config.subtract.name == 'al':
716 # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
717 # Just need a rough estimate; average positions are fine
718 sciAvgPos = sciencePsf.getAveragePosition()
719 scienceSigmaOrig = sciencePsf.computeShape(sciAvgPos).getDeterminantRadius()
721 templatePsf = templateExposure.getPsf()
722 templateAvgPos = templatePsf.getAveragePosition()
723 templateSigma = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()
725 # if requested, convolve the science exposure with its PSF
726 # (properly, this should be a cross-correlation, but our code does not yet support that)
727 # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
728 # else sigma of original science exposure
729 # TODO: DM-22762 This functional block should be moved into its own method
730 preConvPsf = None
731 if self.config.useScoreImageDetection:
732 self.log.warning("AL likelihood image: pre-convolution of PSF is not implemented.")
733 convControl = afwMath.ConvolutionControl()
734 # cannot convolve in place, so need a new image anyway
735 srcMI = exposure.maskedImage
736 exposure = exposure.clone() # New deep copy
737 srcPsf = sciencePsf
738 if self.config.useGaussianForPreConvolution:
739 self.log.info(
740 "AL likelihood image: Using Gaussian (sigma=%.2f) PSF estimation "
741 "for science image pre-convolution", scienceSigmaOrig)
742 # convolve with a simplified PSF model: a double Gaussian
743 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
744 preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
745 else:
746 # convolve with science exposure's PSF model
747 self.log.info(
748 "AL likelihood image: Using the science image PSF for pre-convolution.")
749 preConvPsf = srcPsf
750 afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
751 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
752 else:
753 scienceSigmaPost = scienceSigmaOrig
755 # If requested, find and select sources from the image
756 # else, AL subtraction will do its own source detection
757 # TODO: DM-22762 This functional block should be moved into its own method
758 if self.config.doSelectSources:
759 if selectSources is None:
760 self.log.warning("Src product does not exist; running detection, measurement,"
761 " selection")
762 # Run own detection and measurement; necessary in nightly processing
763 selectSources = self.subtract.getSelectSources(
764 exposure,
765 sigma=scienceSigmaPost,
766 doSmooth=not self.config.useScoreImageDetection,
767 idFactory=idFactory,
768 )
770 if self.config.doAddMetrics:
771 # Number of basis functions
773 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
774 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
775 targetFwhmPix=templateSigma*FwhmPerSigma))
776 # Modify the schema of all Sources
777 # DEPRECATED: This is a data dependent (nparam) output product schema
778 # outside the task constructor.
779 # NOTE: The pre-determination of nparam at this point
780 # may be incorrect as the template psf is warped later in
781 # ImagePsfMatchTask.matchExposures()
782 kcQa = KernelCandidateQa(nparam)
783 selectSources = kcQa.addToSchema(selectSources)
784 if self.config.kernelSourcesFromRef:
785 # match exposure sources to reference catalog
786 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
787 matches = astromRet.matches
788 elif templateSources:
789 # match exposure sources to template sources
790 mc = afwTable.MatchControl()
791 mc.findOnlyClosest = False
792 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
793 mc)
794 else:
795 raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
796 "but template sources not available. Cannot match science "
797 "sources with template sources. Run process* on data from "
798 "which templates are built.")
800 kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
801 matches=matches).sourceCat
802 random.shuffle(kernelSources, random.random)
803 controlSources = kernelSources[::self.config.controlStepSize]
804 kernelSources = [k for i, k in enumerate(kernelSources)
805 if i % self.config.controlStepSize]
807 if self.config.doSelectDcrCatalog:
808 redSelector = DiaCatalogSourceSelectorTask(
809 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
810 grMax=99.999))
811 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
812 controlSources.extend(redSources)
814 blueSelector = DiaCatalogSourceSelectorTask(
815 DiaCatalogSourceSelectorConfig(grMin=-99.999,
816 grMax=self.sourceSelector.config.grMin))
817 blueSources = blueSelector.selectStars(exposure, selectSources,
818 matches=matches).starCat
819 controlSources.extend(blueSources)
821 if self.config.doSelectVariableCatalog:
822 varSelector = DiaCatalogSourceSelectorTask(
823 DiaCatalogSourceSelectorConfig(includeVariable=True))
824 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
825 controlSources.extend(varSources)
827 self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)",
828 len(kernelSources), len(selectSources), len(controlSources))
830 allresids = {}
831 # TODO: DM-22762 This functional block should be moved into its own method
832 if self.config.doUseRegister:
833 self.log.info("Registering images")
835 if templateSources is None:
836 # Run detection on the template, which is
837 # temporarily background-subtracted
838 # sigma of PSF of template image before warping
839 templateSources = self.subtract.getSelectSources(
840 templateExposure,
841 sigma=templateSigma,
842 doSmooth=True,
843 idFactory=idFactory
844 )
846 # Third step: we need to fit the relative astrometry.
847 #
848 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
849 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
850 exposure.getWcs(), exposure.getBBox())
851 templateExposure = warpedExp
853 # Create debugging outputs on the astrometric
854 # residuals as a function of position. Persistence
855 # not yet implemented; expected on (I believe) #2636.
856 if self.config.doDebugRegister:
857 # Grab matches to reference catalog
858 srcToMatch = {x.second.getId(): x.first for x in matches}
860 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
861 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
862 sids = [m.first.getId() for m in wcsResults.matches]
863 positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
864 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
865 m.second.get(inCentroidKey))) for m in wcsResults.matches]
866 allresids = dict(zip(sids, zip(positions, residuals)))
868 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
869 wcsResults.wcs.pixelToSky(
870 m.second.get(inCentroidKey))) for m in wcsResults.matches]
871 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
872 + 2.5*numpy.log10(srcToMatch[x].get("r"))
873 for x in sids if x in srcToMatch.keys()])
874 dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
875 if s in srcToMatch.keys()])
876 dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
877 if s in srcToMatch.keys()])
878 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
879 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
880 & (colors <= self.sourceSelector.config.grMax))
881 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
882 rms1Long = IqrToSigma*(
883 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
884 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
885 - numpy.percentile(dlat[idx1], 25))
886 rms2Long = IqrToSigma*(
887 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
888 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
889 - numpy.percentile(dlat[idx2], 25))
890 rms3Long = IqrToSigma*(
891 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
892 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
893 - numpy.percentile(dlat[idx3], 25))
894 self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f",
895 numpy.median(dlong[idx1]), rms1Long,
896 numpy.median(dlat[idx1]), rms1Lat)
897 self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f",
898 numpy.median(dlong[idx2]), rms2Long,
899 numpy.median(dlat[idx2]), rms2Lat)
900 self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f",
901 numpy.median(dlong[idx3]), rms3Long,
902 numpy.median(dlat[idx3]), rms3Lat)
904 self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
905 self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
906 self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
907 self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
908 self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
909 self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
911 self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
912 self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
913 self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
914 self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
915 self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
916 self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
918 # warp template exposure to match exposure,
919 # PSF match template exposure to exposure,
920 # then return the difference
922 # Return warped template... Construct sourceKernelCand list after subtract
923 self.log.info("Subtracting images")
924 subtractRes = self.subtract.subtractExposures(
925 templateExposure=templateExposure,
926 scienceExposure=exposure,
927 candidateList=kernelSources,
928 convolveTemplate=self.config.convolveTemplate,
929 doWarping=not self.config.doUseRegister
930 )
931 if self.config.useScoreImageDetection:
932 scoreExposure = subtractRes.subtractedExposure
933 else:
934 subtractedExposure = subtractRes.subtractedExposure
936 if self.config.doDetection:
937 self.log.info("Computing diffim PSF")
939 # Get Psf from the appropriate input image if it doesn't exist
940 if subtractedExposure is not None and not subtractedExposure.hasPsf():
941 if self.config.convolveTemplate:
942 subtractedExposure.setPsf(exposure.getPsf())
943 else:
944 subtractedExposure.setPsf(templateExposure.getPsf())
946 # If doSubtract is False, then subtractedExposure was fetched from disk (above),
947 # thus it may have already been decorrelated. Thus, we do not decorrelate if
948 # doSubtract is False.
950 # NOTE: At this point doSubtract == True
951 if self.config.doDecorrelation and self.config.doSubtract:
952 preConvKernel = None
953 if self.config.useGaussianForPreConvolution:
954 preConvKernel = preConvPsf.getLocalKernel()
955 if self.config.useScoreImageDetection:
956 scoreExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
957 scoreExposure,
958 subtractRes.psfMatchingKernel,
959 spatiallyVarying=self.config.doSpatiallyVarying,
960 preConvKernel=preConvKernel,
961 templateMatched=True,
962 preConvMode=True).correctedExposure
963 # Note that the subtracted exposure is always decorrelated,
964 # even if the score image is used for detection
965 subtractedExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
966 subtractedExposure,
967 subtractRes.psfMatchingKernel,
968 spatiallyVarying=self.config.doSpatiallyVarying,
969 preConvKernel=None,
970 templateMatched=self.config.convolveTemplate,
971 preConvMode=False).correctedExposure
972 # END (if subtractAlgorithm == 'AL')
973 # END (if self.config.doSubtract)
974 if self.config.doDetection:
975 self.log.info("Running diaSource detection")
977 # subtractedExposure - reserved for task return value
978 # in zogy, it is always the proper difference image
979 # in AL, it may be (yet) pre-convolved and/or decorrelated
980 #
981 # detectionExposure - controls which exposure to use for detection
982 # in-place modifications will appear in task return
983 if self.config.useScoreImageDetection:
984 # zogy with score image detection enabled
985 self.log.info("Detection, diffim rescaling and measurements are "
986 "on AL likelihood or Zogy score image.")
987 detectionExposure = scoreExposure
988 else:
989 # AL or zogy with no score image detection
990 detectionExposure = subtractedExposure
992 # Rescale difference image variance plane
993 if self.config.doScaleDiffimVariance:
994 self.log.info("Rescaling diffim variance")
995 diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
996 self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor)
997 self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
999 # Erase existing detection mask planes
1000 mask = detectionExposure.getMaskedImage().getMask()
1001 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
1003 table = afwTable.SourceTable.make(self.schema, idFactory)
1004 table.setMetadata(self.algMetadata)
1005 results = self.detection.run(
1006 table=table,
1007 exposure=detectionExposure,
1008 doSmooth=not self.config.useScoreImageDetection
1009 )
1011 if self.config.doMerge:
1012 fpSet = results.fpSets.positive
1013 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1014 self.config.growFootprint, False)
1015 diaSources = afwTable.SourceCatalog(table)
1016 fpSet.makeSources(diaSources)
1017 self.log.info("Merging detections into %d sources", len(diaSources))
1018 else:
1019 diaSources = results.sources
1020 # Inject skySources before measurement.
1021 if self.config.doSkySources:
1022 skySourceFootprints = self.skySources.run(
1023 mask=detectionExposure.mask,
1024 seed=detectionExposure.info.id)
1025 if skySourceFootprints:
1026 for foot in skySourceFootprints:
1027 s = diaSources.addNew()
1028 s.setFootprint(foot)
1029 s.set(self.skySourceKey, True)
1031 if self.config.doMeasurement:
1032 newDipoleFitting = self.config.doDipoleFitting
1033 self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1034 if not newDipoleFitting:
1035 # Just fit dipole in diffim
1036 self.measurement.run(diaSources, detectionExposure)
1037 else:
1038 # Use (matched) template and science image (if avail.) to constrain dipole fitting
1039 if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1040 self.measurement.run(diaSources, detectionExposure, exposure,
1041 subtractRes.matchedExposure)
1042 else:
1043 self.measurement.run(diaSources, detectionExposure, exposure)
1044 if self.config.doApCorr:
1045 self.applyApCorr.run(
1046 catalog=diaSources,
1047 apCorrMap=detectionExposure.getInfo().getApCorrMap()
1048 )
1050 if self.config.doForcedMeasurement:
1051 # Run forced psf photometry on the PVI at the diaSource locations.
1052 # Copy the measured flux and error into the diaSource.
1053 forcedSources = self.forcedMeasurement.generateMeasCat(
1054 exposure, diaSources, detectionExposure.getWcs())
1055 self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1056 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1057 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1058 "ip_diffim_forced_PsfFlux_instFlux", True)
1059 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1060 "ip_diffim_forced_PsfFlux_instFluxErr", True)
1061 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1062 "ip_diffim_forced_PsfFlux_area", True)
1063 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1064 "ip_diffim_forced_PsfFlux_flag", True)
1065 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1066 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1067 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1068 "ip_diffim_forced_PsfFlux_flag_edge", True)
1069 for diaSource, forcedSource in zip(diaSources, forcedSources):
1070 diaSource.assign(forcedSource, mapper)
1072 # Match with the calexp sources if possible
1073 if self.config.doMatchSources:
1074 if selectSources is not None:
1075 # Create key,val pair where key=diaSourceId and val=sourceId
1076 matchRadAsec = self.config.diaSourceMatchRadius
1077 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1079 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1080 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1081 srcMatch in srcMatches])
1082 self.log.info("Matched %d / %d diaSources to sources",
1083 len(srcMatchDict), len(diaSources))
1084 else:
1085 self.log.warning("Src product does not exist; cannot match with diaSources")
1086 srcMatchDict = {}
1088 # Create key,val pair where key=diaSourceId and val=refId
1089 refAstromConfig = AstrometryConfig()
1090 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1091 refAstrometer = AstrometryTask(self.refObjLoader, config=refAstromConfig)
1092 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1093 refMatches = astromRet.matches
1094 if refMatches is None:
1095 self.log.warning("No diaSource matches with reference catalog")
1096 refMatchDict = {}
1097 else:
1098 self.log.info("Matched %d / %d diaSources to reference catalog",
1099 len(refMatches), len(diaSources))
1100 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1101 refMatch in refMatches])
1103 # Assign source Ids
1104 for diaSource in diaSources:
1105 sid = diaSource.getId()
1106 if sid in srcMatchDict:
1107 diaSource.set("srcMatchId", srcMatchDict[sid])
1108 if sid in refMatchDict:
1109 diaSource.set("refMatchId", refMatchDict[sid])
1111 if self.config.doAddMetrics and self.config.doSelectSources:
1112 self.log.info("Evaluating metrics and control sample")
1114 kernelCandList = []
1115 for cell in subtractRes.kernelCellSet.getCellList():
1116 for cand in cell.begin(False): # include bad candidates
1117 kernelCandList.append(cand)
1119 # Get basis list to build control sample kernels
1120 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1121 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1123 controlCandList = (
1124 diffimTools.sourceTableToCandidateList(controlSources,
1125 subtractRes.warpedExposure, exposure,
1126 self.config.subtract.kernel.active,
1127 self.config.subtract.kernel.active.detectionConfig,
1128 self.log, doBuild=True, basisList=basisList))
1130 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1131 subtractRes.backgroundModel, dof=nparam)
1132 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1133 subtractRes.backgroundModel)
1135 if self.config.doDetection:
1136 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1137 else:
1138 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1140 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1141 return pipeBase.Struct(
1142 subtractedExposure=subtractedExposure,
1143 scoreExposure=scoreExposure,
1144 warpedExposure=subtractRes.warpedExposure,
1145 matchedExposure=subtractRes.matchedExposure,
1146 subtractRes=subtractRes,
1147 diaSources=diaSources,
1148 selectSources=selectSources
1149 )
1151 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1152 """Fit the relative astrometry between templateSources and selectSources
1154 Notes
1155 -----
1156 TODO: Remove this method. It originally fit a new WCS to the template before calling register.run
1157 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1158 It remains because a subtask overrides it.
1159 """
1160 results = self.register.run(templateSources, templateExposure.getWcs(),
1161 templateExposure.getBBox(), selectSources)
1162 return results
1164 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1165 """Make debug plots and displays.
1167 Parameters
1168 ----------
1169 exposure : `lsst.afw.image.exposure.Exposure`
1170 Input exposure.
1171 subtractRes : `lsst.pipe.base.Struct`
1172 Returned result structure of the ImagePsfMatchTask subtask.
1173 selectSources : `lsst.afw.table.SourceCatalog`
1174 Input source catalog.
1175 kernelSources : `lsst.afw.table.SourceCatalog`
1176 unknown
1177 diaSources : `lsst.afw.table.SourceCatalog`
1178 The catalog of detected sources.
1180 Notes
1181 -----
1182 TODO: Test and update for current debug display and slot names.
1183 """
1184 import lsstDebug
1185 display = lsstDebug.Info(__name__).display
1186 showSubtracted = lsstDebug.Info(__name__).showSubtracted
1187 showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1188 showDiaSources = lsstDebug.Info(__name__).showDiaSources
1189 showDipoles = lsstDebug.Info(__name__).showDipoles
1190 maskTransparency = lsstDebug.Info(__name__).maskTransparency
1191 if display:
1192 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1193 if not maskTransparency:
1194 maskTransparency = 0
1195 disp.setMaskTransparency(maskTransparency)
1197 if display and showSubtracted:
1198 disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1199 mi = subtractRes.subtractedExposure.getMaskedImage()
1200 x0, y0 = mi.getX0(), mi.getY0()
1201 with disp.Buffering():
1202 for s in diaSources:
1203 x, y = s.getX() - x0, s.getY() - y0
1204 ctype = "red" if s.get("flags_negative") else "yellow"
1205 if (s.get("base_PixelFlags_flag_interpolatedCenter")
1206 or s.get("base_PixelFlags_flag_saturatedCenter")
1207 or s.get("base_PixelFlags_flag_crCenter")):
1208 ptype = "x"
1209 elif (s.get("base_PixelFlags_flag_interpolated")
1210 or s.get("base_PixelFlags_flag_saturated")
1211 or s.get("base_PixelFlags_flag_cr")):
1212 ptype = "+"
1213 else:
1214 ptype = "o"
1215 disp.dot(ptype, x, y, size=4, ctype=ctype)
1216 lsstDebug.frame += 1
1218 if display and showPixelResiduals and selectSources:
1219 nonKernelSources = []
1220 for source in selectSources:
1221 if source not in kernelSources:
1222 nonKernelSources.append(source)
1224 diUtils.plotPixelResiduals(exposure,
1225 subtractRes.warpedExposure,
1226 subtractRes.subtractedExposure,
1227 subtractRes.kernelCellSet,
1228 subtractRes.psfMatchingKernel,
1229 subtractRes.backgroundModel,
1230 nonKernelSources,
1231 self.subtract.config.kernel.active.detectionConfig,
1232 origVariance=False)
1233 diUtils.plotPixelResiduals(exposure,
1234 subtractRes.warpedExposure,
1235 subtractRes.subtractedExposure,
1236 subtractRes.kernelCellSet,
1237 subtractRes.psfMatchingKernel,
1238 subtractRes.backgroundModel,
1239 nonKernelSources,
1240 self.subtract.config.kernel.active.detectionConfig,
1241 origVariance=True)
1242 if display and showDiaSources:
1243 flagChecker = SourceFlagChecker(diaSources)
1244 isFlagged = [flagChecker(x) for x in diaSources]
1245 isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1246 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1247 frame=lsstDebug.frame)
1248 lsstDebug.frame += 1
1250 if display and showDipoles:
1251 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1252 frame=lsstDebug.frame)
1253 lsstDebug.frame += 1
1255 def checkTemplateIsSufficient(self, templateExposure):
1256 """Raise NoWorkFound if template coverage < requiredTemplateFraction.
1258 Parameters
1259 ----------
1260 templateExposure : `lsst.afw.image.ExposureF`
1261 The template exposure to check.
1263 Raises
1264 ------
1265 NoWorkFound
1266 Raised if fraction of good pixels, defined as not having NO_DATA
1267 set, is less then the configured requiredTemplateFraction.
1268 """
1269 # Count the number of pixels with the NO_DATA mask bit set
1270 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
1271 pixNoData = numpy.count_nonzero(templateExposure.mask.array
1272 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
1273 pixGood = templateExposure.getBBox().getArea() - pixNoData
1274 self.log.info("template has %d good pixels (%.1f%%)", pixGood,
1275 100*pixGood/templateExposure.getBBox().getArea())
1277 if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction:
1278 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
1279 "To force subtraction, set config requiredTemplateFraction=0." % (
1280 100*pixGood/templateExposure.getBBox().getArea(),
1281 100*self.config.requiredTemplateFraction))
1282 raise pipeBase.NoWorkFound(message)
1285class ImageDifferenceFromTemplateConnections(ImageDifferenceTaskConnections,
1286 defaultTemplates={"coaddName": "goodSeeing"}
1287 ):
1288 inputTemplate = pipeBase.connectionTypes.Input(
1289 doc=("Warped template produced by GetMultiTractCoaddTemplate"),
1290 dimensions=("instrument", "visit", "detector"),
1291 storageClass="ExposureF",
1292 name="{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}",
1293 )
1295 def __init__(self, *, config=None):
1296 super().__init__(config=config)
1297 # ImageDifferenceConnections will have removed one of these.
1298 # Make sure they're both gone, because no coadds are needed.
1299 if "coaddExposures" in self.inputs:
1300 self.inputs.remove("coaddExposures")
1301 if "dcrCoadds" in self.inputs:
1302 self.inputs.remove("dcrCoadds")
1305class ImageDifferenceFromTemplateConfig(ImageDifferenceConfig,
1306 pipelineConnections=ImageDifferenceFromTemplateConnections):
1307 pass
1310class ImageDifferenceFromTemplateTask(ImageDifferenceTask):
1311 ConfigClass = ImageDifferenceFromTemplateConfig
1312 _DefaultName = "imageDifference"
1314 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
1315 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1316 inputs = butlerQC.get(inputRefs)
1317 self.log.info("Processing %s", butlerQC.quantum.dataId)
1318 self.checkTemplateIsSufficient(inputs['inputTemplate'])
1319 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
1320 returnMaxBits=True)
1321 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
1323 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None)
1324 exposure = self.prepareCalibratedExposure(
1325 inputs["exposure"],
1326 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog
1327 )
1329 outputs = self.run(exposure=exposure,
1330 templateExposure=inputs['inputTemplate'],
1331 idFactory=idFactory)
1333 # Consistency with runDataref gen2 handling
1334 if outputs.diaSources is None:
1335 del outputs.diaSources
1336 butlerQC.put(outputs, outputRefs)