lsst.pipe.tasks gf10b05e212+a1470f6de0
imageDifference.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import math
23import random
24import numpy
25
26import lsst.utils
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29import lsst.daf.base as dafBase
30import lsst.geom as geom
31import lsst.afw.math as afwMath
32import lsst.afw.table as afwTable
33import lsst.meas.extensions.trailedSources # noqa: F401
34from lsst.meas.algorithms import (SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask,
35 LoadReferenceObjectsConfig, SkyObjectsTask,
36 ScaleVarianceTask)
37from lsst.meas.astrom import AstrometryConfig, AstrometryTask
38from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
39from lsst.pipe.tasks.registerImage import RegisterTask
40from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
41 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
42 GetCoaddAsTemplateTask, DipoleFitTask,
43 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
44import lsst.ip.diffim.diffimTools as diffimTools
45import lsst.ip.diffim.utils as diUtils
46import lsst.afw.display as afwDisplay
47from lsst.skymap import BaseSkyMap
48from lsst.obs.base import ExposureIdInfo
49from lsst.utils.timer import timeMethod
50
51from deprecated.sphinx import deprecated
52
53__all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
54FwhmPerSigma = 2*math.sqrt(2*math.log(2))
55IqrToSigma = 0.741
56
57
58class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
59 dimensions=("instrument", "visit", "detector", "skymap"),
60 defaultTemplates={"coaddName": "deep",
61 "skyMapName": "deep",
62 "warpTypeSuffix": "",
63 "fakesType": ""}):
64
65 exposure = pipeBase.connectionTypes.Input(
66 doc="Input science exposure to subtract from.",
67 dimensions=("instrument", "visit", "detector"),
68 storageClass="ExposureF",
69 name="{fakesType}calexp"
70 )
71
72 # TODO DM-22953
73 # kernelSources = pipeBase.connectionTypes.Input(
74 # doc="Source catalog produced in calibrate task for kernel candidate sources",
75 # name="src",
76 # storageClass="SourceCatalog",
77 # dimensions=("instrument", "visit", "detector"),
78 # )
79
80 skyMap = pipeBase.connectionTypes.Input(
81 doc="Input definition of geometry/bbox and projection/wcs for template exposures",
82 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
83 dimensions=("skymap", ),
84 storageClass="SkyMap",
85 )
86 coaddExposures = pipeBase.connectionTypes.Input(
87 doc="Input template to match and subtract from the exposure",
88 dimensions=("tract", "patch", "skymap", "band"),
89 storageClass="ExposureF",
90 name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
91 multiple=True,
92 deferLoad=True
93 )
94 dcrCoadds = pipeBase.connectionTypes.Input(
95 doc="Input DCR template to match and subtract from the exposure",
96 name="{fakesType}dcrCoadd{warpTypeSuffix}",
97 storageClass="ExposureF",
98 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
99 multiple=True,
100 deferLoad=True
101 )
102 finalizedPsfApCorrCatalog = pipeBase.connectionTypes.Input(
103 doc=("Per-visit finalized psf models and aperture correction maps. "
104 "These catalogs use the detector id for the catalog id, "
105 "sorted on id for fast lookup."),
106 name="finalized_psf_ap_corr_catalog",
107 storageClass="ExposureCatalog",
108 dimensions=("instrument", "visit"),
109 )
110 outputSchema = pipeBase.connectionTypes.InitOutput(
111 doc="Schema (as an example catalog) for output DIASource catalog.",
112 storageClass="SourceCatalog",
113 name="{fakesType}{coaddName}Diff_diaSrc_schema",
114 )
115 subtractedExposure = pipeBase.connectionTypes.Output(
116 doc="Output AL difference or Zogy proper difference image",
117 dimensions=("instrument", "visit", "detector"),
118 storageClass="ExposureF",
119 name="{fakesType}{coaddName}Diff_differenceExp",
120 )
121 scoreExposure = pipeBase.connectionTypes.Output(
122 doc="Output AL likelihood or Zogy score image",
123 dimensions=("instrument", "visit", "detector"),
124 storageClass="ExposureF",
125 name="{fakesType}{coaddName}Diff_scoreExp",
126 )
127 warpedExposure = pipeBase.connectionTypes.Output(
128 doc="Warped template used to create `subtractedExposure`.",
129 dimensions=("instrument", "visit", "detector"),
130 storageClass="ExposureF",
131 name="{fakesType}{coaddName}Diff_warpedExp",
132 )
133 matchedExposure = pipeBase.connectionTypes.Output(
134 doc="Warped template used to create `subtractedExposure`.",
135 dimensions=("instrument", "visit", "detector"),
136 storageClass="ExposureF",
137 name="{fakesType}{coaddName}Diff_matchedExp",
138 )
139 diaSources = pipeBase.connectionTypes.Output(
140 doc="Output detected diaSources on the difference image",
141 dimensions=("instrument", "visit", "detector"),
142 storageClass="SourceCatalog",
143 name="{fakesType}{coaddName}Diff_diaSrc",
144 )
145
146 def __init__(self, *, config=None):
147 super().__init__(config=config)
148 if config.coaddName == 'dcr':
149 self.inputs.remove("coaddExposures")
150 else:
151 self.inputs.remove("dcrCoadds")
152 if not config.doWriteSubtractedExp:
153 self.outputs.remove("subtractedExposure")
154 if not config.doWriteScoreExp:
155 self.outputs.remove("scoreExposure")
156 if not config.doWriteWarpedExp:
157 self.outputs.remove("warpedExposure")
158 if not config.doWriteMatchedExp:
159 self.outputs.remove("matchedExposure")
160 if not config.doWriteSources:
161 self.outputs.remove("diaSources")
162 if not config.doApplyFinalizedPsf:
163 self.inputs.remove("finalizedPsfApCorrCatalog")
164
165 # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
166 # Make kernelSources optional
167
168
169class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
170 pipelineConnections=ImageDifferenceTaskConnections):
171 """Config for ImageDifferenceTask
172 """
173 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
174 doc="Add background to calexp before processing it. "
175 "Useful as ipDiffim does background matching.")
176 doUseRegister = pexConfig.Field(dtype=bool, default=False,
177 doc="Re-compute astrometry on the template. "
178 "Use image-to-image registration to align template with "
179 "science image (AL only).")
180 doDebugRegister = pexConfig.Field(dtype=bool, default=False,
181 doc="Writing debugging data for doUseRegister")
182 doSelectSources = pexConfig.Field(dtype=bool, default=False,
183 doc="Select stars to use for kernel fitting (AL only)")
184 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
185 doc="Select stars of extreme color as part "
186 "of the control sample (AL only)")
187 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
188 doc="Select stars that are variable to be part "
189 "of the control sample (AL only)")
190 doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
191 doPreConvolve = pexConfig.Field(dtype=bool, default=False,
192 doc="Not in use. Superseded by useScoreImageDetection.",
193 deprecated="This option superseded by useScoreImageDetection."
194 " Will be removed after v22.")
195 useScoreImageDetection = pexConfig.Field(
196 dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or "
197 "the Zogy score image. Use it for source detection (if doDetection=True).")
198 doWriteScoreExp = pexConfig.Field(
199 dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?")
200 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
201 doc="Scale variance of the template before PSF matching")
202 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
203 doc="Scale variance of the diffim before PSF matching. "
204 "You may do either this or template variance scaling, "
205 "or neither. (Doing both is a waste of CPU.)")
206 useGaussianForPreConvolution = pexConfig.Field(
207 dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution "
208 "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)")
209 doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
210 doDecorrelation = pexConfig.Field(dtype=bool, default=True,
211 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
212 "kernel convolution (AL only)? If True, also update the diffim PSF.")
213 doMerge = pexConfig.Field(dtype=bool, default=True,
214 doc="Merge positive and negative diaSources with grow radius "
215 "set by growFootprint")
216 doMatchSources = pexConfig.Field(dtype=bool, default=False,
217 doc="Match diaSources with input calexp sources and ref catalog sources")
218 doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
219 doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
220 doForcedMeasurement = pexConfig.Field(
221 dtype=bool,
222 default=True,
223 doc="Force photometer diaSource locations on PVI?")
224 doWriteSubtractedExp = pexConfig.Field(
225 dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
226 doWriteWarpedExp = pexConfig.Field(
227 dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
228 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
229 doc="Write warped and PSF-matched template coadd exposure?")
230 doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
231 doAddMetrics = pexConfig.Field(dtype=bool, default=False,
232 doc="Add columns to the source table to hold analysis metrics?")
233 doApplyFinalizedPsf = pexConfig.Field(
234 doc="Whether to apply finalized psf models and aperture correction map.",
235 dtype=bool,
236 default=False,
237 )
238
239 coaddName = pexConfig.Field(
240 doc="coadd name: typically one of deep, goodSeeing, or dcr",
241 dtype=str,
242 default="deep",
243 )
244 convolveTemplate = pexConfig.Field(
245 doc="Which image gets convolved (default = template)",
246 dtype=bool,
247 default=True
248 )
249 refObjLoader = pexConfig.ConfigField(
250 dtype=LoadReferenceObjectsConfig,
251 doc="reference object loader",
252 )
253 astrometer = pexConfig.ConfigurableField(
254 target=AstrometryTask,
255 doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
256 )
257 sourceSelector = pexConfig.ConfigurableField(
258 target=ObjectSizeStarSelectorTask,
259 doc="Source selection algorithm",
260 )
261 subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
262 decorrelate = pexConfig.ConfigurableField(
263 target=DecorrelateALKernelSpatialTask,
264 doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
265 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
266 "default of 5.5).",
267 )
268 # Old style ImageMapper grid. ZogyTask has its own grid option
269 doSpatiallyVarying = pexConfig.Field(
270 dtype=bool,
271 default=False,
272 doc="Perform A&L decorrelation on a grid across the "
273 "image in order to allow for spatial variations. Zogy does not use this option."
274 )
275 detection = pexConfig.ConfigurableField(
276 target=SourceDetectionTask,
277 doc="Low-threshold detection for final measurement",
278 )
279 measurement = pexConfig.ConfigurableField(
280 target=DipoleFitTask,
281 doc="Enable updated dipole fitting method",
282 )
283 doApCorr = lsst.pex.config.Field(
284 dtype=bool,
285 default=True,
286 doc="Run subtask to apply aperture corrections"
287 )
288 applyApCorr = lsst.pex.config.ConfigurableField(
289 target=ApplyApCorrTask,
290 doc="Subtask to apply aperture corrections"
291 )
292 forcedMeasurement = pexConfig.ConfigurableField(
293 target=ForcedMeasurementTask,
294 doc="Subtask to force photometer PVI at diaSource location.",
295 )
296 getTemplate = pexConfig.ConfigurableField(
297 target=GetCoaddAsTemplateTask,
298 doc="Subtask to retrieve template exposure and sources",
299 )
300 scaleVariance = pexConfig.ConfigurableField(
301 target=ScaleVarianceTask,
302 doc="Subtask to rescale the variance of the template "
303 "to the statistically expected level"
304 )
305 controlStepSize = pexConfig.Field(
306 doc="What step size (every Nth one) to select a control sample from the kernelSources",
307 dtype=int,
308 default=5
309 )
310 controlRandomSeed = pexConfig.Field(
311 doc="Random seed for shuffing the control sample",
312 dtype=int,
313 default=10
314 )
315 register = pexConfig.ConfigurableField(
316 target=RegisterTask,
317 doc="Task to enable image-to-image image registration (warping)",
318 )
319 kernelSourcesFromRef = pexConfig.Field(
320 doc="Select sources to measure kernel from reference catalog if True, template if false",
321 dtype=bool,
322 default=False
323 )
324 templateSipOrder = pexConfig.Field(
325 dtype=int, default=2,
326 doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
327 )
328 growFootprint = pexConfig.Field(
329 dtype=int, default=2,
330 doc="Grow positive and negative footprints by this amount before merging"
331 )
332 diaSourceMatchRadius = pexConfig.Field(
333 dtype=float, default=0.5,
334 doc="Match radius (in arcseconds) for DiaSource to Source association"
335 )
336 requiredTemplateFraction = pexConfig.Field(
337 dtype=float, default=0.1,
338 doc="Do not attempt to run task if template covers less than this fraction of pixels."
339 "Setting to 0 will always attempt image subtraction"
340 )
341 doSkySources = pexConfig.Field(
342 dtype=bool,
343 default=False,
344 doc="Generate sky sources?",
345 )
346 skySources = pexConfig.ConfigurableField(
347 target=SkyObjectsTask,
348 doc="Generate sky sources",
349 )
350
351 def setDefaults(self):
352 # defaults are OK for catalog and diacatalog
353
354 self.subtract['al'].kernel.name = "AL"
355 self.subtract['al'].kernel.active.fitForBackground = True
356 self.subtract['al'].kernel.active.spatialKernelOrder = 1
357 self.subtract['al'].kernel.active.spatialBgOrder = 2
358
359 # DiaSource Detection
360 self.detection.thresholdPolarity = "both"
361 self.detection.thresholdValue = 5.0
362 self.detection.reEstimateBackground = False
363 self.detection.thresholdType = "pixel_stdev"
364
365 # Add filtered flux measurement, the correct measurement for pre-convolved images.
366 # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
367 # To change that you must modify algorithms.names in the task's applyOverrides method,
368 # after the user has set doPreConvolve.
369 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
370 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
371 'base_LocalPhotoCalib',
372 'base_LocalWcs']
373
374 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
375 self.forcedMeasurement.copyColumns = {
376 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
377 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
378 self.forcedMeasurement.slots.shape = None
379
380 # For shuffling the control sample
381 random.seed(self.controlRandomSeed)
382
383 def validate(self):
384 pexConfig.Config.validate(self)
385 if not self.doSubtract and not self.doDetection:
386 raise ValueError("Either doSubtract or doDetection must be enabled.")
387 if self.doMeasurement and not self.doDetection:
388 raise ValueError("Cannot run source measurement without source detection.")
389 if self.doMerge and not self.doDetection:
390 raise ValueError("Cannot run source merging without source detection.")
391 if self.doSkySources and not self.doDetection:
392 raise ValueError("Cannot run sky source creation without source detection.")
393 if self.doUseRegister and not self.doSelectSources:
394 raise ValueError("doUseRegister=True and doSelectSources=False. "
395 "Cannot run RegisterTask without selecting sources.")
396 if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
397 raise ValueError("Scaling the diffim variance and scaling the template variance "
398 "are both set. Please choose one or the other.")
399 # We cannot allow inconsistencies that would lead to None or not available output products
400 if self.subtract.name == 'zogy':
401 if self.doWriteMatchedExp:
402 raise ValueError("doWriteMatchedExp=True Matched exposure is not "
403 "calculated in zogy subtraction.")
404 if self.doAddMetrics:
405 raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
406 if self.doDecorrelation:
407 raise ValueError(
408 "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
409 if self.doSelectSources:
410 raise ValueError(
411 "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
412 if self.useGaussianForPreConvolution:
413 raise ValueError(
414 "useGaussianForPreConvolution=True This is an AL subtraction only option.")
415 else:
416 # AL only consistency checks
417 if self.useScoreImageDetection and not self.convolveTemplate:
418 raise ValueError(
419 "convolveTemplate=False and useScoreImageDetection=True "
420 "Pre-convolution and matching of the science image is not a supported operation.")
421 if self.doWriteSubtractedExp and self.useScoreImageDetection:
422 raise ValueError(
423 "doWriteSubtractedExp=True and useScoreImageDetection=True "
424 "Regular difference image is not calculated. "
425 "AL subtraction calculates either the regular difference image or the score image.")
426 if self.doWriteScoreExp and not self.useScoreImageDetection:
427 raise ValueError(
428 "doWriteScoreExp=True and useScoreImageDetection=False "
429 "Score image is not calculated. "
430 "AL subtraction calculates either the regular difference image or the score image.")
431 if self.doAddMetrics and not self.doSubtract:
432 raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
433 if self.useGaussianForPreConvolution and not self.useScoreImageDetection:
434 raise ValueError(
435 "useGaussianForPreConvolution=True and useScoreImageDetection=False "
436 "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.")
437
438
439@deprecated(reason="This Task has been replaced with lsst.ip.diffim.subtractImages"
440 " and lsst.ip.diffim.detectAndMeasure. Will be removed after v25.",
441 version="v24.0", category=FutureWarning)
442class ImageDifferenceTask(pipeBase.PipelineTask):
443 """Subtract an image from a template and measure the result
444 """
445 ConfigClass = ImageDifferenceConfig
446 _DefaultName = "imageDifference"
447
448 def __init__(self, butler=None, **kwargs):
449 """!Construct an ImageDifference Task
450
451 @param[in] butler Butler object to use in constructing reference object loaders
452 """
453 super().__init__(**kwargs)
454 self.makeSubtask("getTemplate")
455
456 self.makeSubtask("subtract")
457
458 if self.config.subtract.name == 'al' and self.config.doDecorrelation:
459 self.makeSubtask("decorrelate")
460
461 if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
462 self.makeSubtask("scaleVariance")
463
464 if self.config.doUseRegister:
465 self.makeSubtask("register")
466 self.schema = afwTable.SourceTable.makeMinimalSchema()
467
468 if self.config.doSelectSources:
469 self.makeSubtask("sourceSelector")
470 if self.config.kernelSourcesFromRef:
471 self.makeSubtask('refObjLoader', butler=butler)
472 self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
473
474 self.algMetadata = dafBase.PropertyList()
475 if self.config.doDetection:
476 self.makeSubtask("detection", schema=self.schema)
477 if self.config.doMeasurement:
478 self.makeSubtask("measurement", schema=self.schema,
479 algMetadata=self.algMetadata)
480 if self.config.doApCorr:
481 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
482 if self.config.doForcedMeasurement:
483 self.schema.addField(
484 "ip_diffim_forced_PsfFlux_instFlux", "D",
485 "Forced PSF flux measured on the direct image.",
486 units="count")
487 self.schema.addField(
488 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
489 "Forced PSF flux error measured on the direct image.",
490 units="count")
491 self.schema.addField(
492 "ip_diffim_forced_PsfFlux_area", "F",
493 "Forced PSF flux effective area of PSF.",
494 units="pixel")
495 self.schema.addField(
496 "ip_diffim_forced_PsfFlux_flag", "Flag",
497 "Forced PSF flux general failure flag.")
498 self.schema.addField(
499 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
500 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
501 self.schema.addField(
502 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
503 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
504 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
505 if self.config.doMatchSources:
506 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
507 self.schema.addField("srcMatchId", "L", "unique id of source match")
508 if self.config.doSkySources:
509 self.makeSubtask("skySources")
510 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
511
512 # initialize InitOutputs
513 self.outputSchema = afwTable.SourceCatalog(self.schema)
514 self.outputSchema.getTable().setMetadata(self.algMetadata)
515
516 @staticmethod
517 def makeIdFactory(expId, expBits):
518 """Create IdFactory instance for unique 64 bit diaSource id-s.
519
520 Parameters
521 ----------
522 expId : `int`
523 Exposure id.
524
525 expBits: `int`
526 Number of used bits in ``expId``.
527
528 Note
529 ----
530 The diasource id-s consists of the ``expId`` stored fixed in the highest value
531 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
532 low value end of the integer.
533
534 Returns
535 -------
536 idFactory: `lsst.afw.table.IdFactory`
537 """
538 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
539
540 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
541 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
542 inputRefs: pipeBase.InputQuantizedConnection,
543 outputRefs: pipeBase.OutputQuantizedConnection):
544 inputs = butlerQC.get(inputRefs)
545 self.log.info("Processing %s", butlerQC.quantum.dataId)
546
547 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None)
548 exposure = self.prepareCalibratedExposure(
549 inputs["exposure"],
550 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog
551 )
552
553 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
554 returnMaxBits=True)
555 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
556 if self.config.coaddName == 'dcr':
557 templateExposures = inputRefs.dcrCoadds
558 else:
559 templateExposures = inputRefs.coaddExposures
560 templateStruct = self.getTemplate.runQuantum(
561 exposure, butlerQC, inputRefs.skyMap, templateExposures
562 )
563
564 self.checkTemplateIsSufficient(templateStruct.exposure)
565
566 outputs = self.run(exposure=exposure,
567 templateExposure=templateStruct.exposure,
568 idFactory=idFactory)
569 # Consistency with runDataref gen2 handling
570 if outputs.diaSources is None:
571 del outputs.diaSources
572 butlerQC.put(outputs, outputRefs)
573
574 def prepareCalibratedExposure(self, exposure, finalizedPsfApCorrCatalog=None):
575 """Prepare a calibrated exposure and apply finalized psf if so configured.
576
577 Parameters
578 ----------
579 exposure : `lsst.afw.image.exposure.Exposure`
580 Input exposure to adjust calibrations.
581 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
582 Exposure catalog with finalized psf models and aperture correction
583 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
584 the detector id for the catalog id, sorted on id for fast lookup.
585
586 Returns
587 -------
588 exposure : `lsst.afw.image.exposure.Exposure`
589 Exposure with adjusted calibrations.
590 """
591 detectorId = exposure.getInfo().getDetector().getId()
592
593 if finalizedPsfApCorrCatalog is not None:
594 row = finalizedPsfApCorrCatalog.find(detectorId)
595 if row is None:
596 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; "
597 "Using original psf.", detectorId)
598 else:
599 psf = row.getPsf()
600 apCorrMap = row.getApCorrMap()
601 if psf is None or apCorrMap is None:
602 self.log.warning("Detector id %s has None for psf/apCorrMap in "
603 "finalizedPsfApCorrCatalog; Using original psf.", detectorId)
604 else:
605 exposure.setPsf(psf)
606 exposure.info.setApCorrMap(apCorrMap)
607
608 return exposure
609
610 @timeMethod
611 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
612 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
613 """PSF matches, subtract two images and perform detection on the difference image.
614
615 Parameters
616 ----------
617 exposure : `lsst.afw.image.ExposureF`, optional
618 The science exposure, the minuend in the image subtraction.
619 Can be None only if ``config.doSubtract==False``.
620 selectSources : `lsst.afw.table.SourceCatalog`, optional
621 Identified sources on the science exposure. This catalog is used to
622 select sources in order to perform the AL PSF matching on stamp images
623 around them. The selection steps depend on config options and whether
624 ``templateSources`` and ``matchingSources`` specified.
625 templateExposure : `lsst.afw.image.ExposureF`, optional
626 The template to be subtracted from ``exposure`` in the image subtraction.
627 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
628 The template exposure should cover the same sky area as the science exposure.
629 It is either a stich of patches of a coadd skymap image or a calexp
630 of the same pointing as the science exposure. Can be None only
631 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
632 templateSources : `lsst.afw.table.SourceCatalog`, optional
633 Identified sources on the template exposure.
634 idFactory : `lsst.afw.table.IdFactory`
635 Generator object to assign ids to detected sources in the difference image.
636 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
637 Background exposure to be added back to the science exposure
638 if ``config.doAddCalexpBackground==True``
639 subtractedExposure : `lsst.afw.image.ExposureF`, optional
640 If ``config.doSubtract==False`` and ``config.doDetection==True``,
641 performs the post subtraction source detection only on this exposure.
642 Otherwise should be None.
643
644 Returns
645 -------
646 results : `lsst.pipe.base.Struct`
647 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
648 Difference image.
649 ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
650 The zogy score exposure, if calculated.
651 ``matchedExposure`` : `lsst.afw.image.ExposureF`
652 The matched PSF exposure.
653 ``subtractRes`` : `lsst.pipe.base.Struct`
654 The returned result structure of the ImagePsfMatchTask subtask.
655 ``diaSources`` : `lsst.afw.table.SourceCatalog`
656 The catalog of detected sources.
657 ``selectSources`` : `lsst.afw.table.SourceCatalog`
658 The input source catalog with optionally added Qa information.
659
660 Notes
661 -----
662 The following major steps are included:
663
664 - warp template coadd to match WCS of image
665 - PSF match image to warped template
666 - subtract image from PSF-matched, warped template
667 - detect sources
668 - measure sources
669
670 For details about the image subtraction configuration modes
671 see `lsst.ip.diffim`.
672 """
673 subtractRes = None
674 controlSources = None
675 subtractedExposure = None
676 scoreExposure = None
677 diaSources = None
678 kernelSources = None
679 # We'll clone exposure if modified but will still need the original
680 exposureOrig = exposure
681
682 if self.config.doAddCalexpBackground:
683 mi = exposure.getMaskedImage()
684 mi += calexpBackgroundExposure.getImage()
685
686 if not exposure.hasPsf():
687 raise pipeBase.TaskError("Exposure has no psf")
688 sciencePsf = exposure.getPsf()
689
690 if self.config.doSubtract:
691 if self.config.doScaleTemplateVariance:
692 self.log.info("Rescaling template variance")
693 templateVarFactor = self.scaleVariance.run(
694 templateExposure.getMaskedImage())
695 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
696 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
697 self.metadata.add("psfMatchingAlgorithm", self.config.subtract.name)
698
699 if self.config.subtract.name == 'zogy':
700 subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
701 scoreExposure = subtractRes.scoreExp
702 subtractedExposure = subtractRes.diffExp
703 subtractRes.subtractedExposure = subtractedExposure
704 subtractRes.matchedExposure = None
705
706 elif self.config.subtract.name == 'al':
707 # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
708 # Just need a rough estimate; average positions are fine
709 sciAvgPos = sciencePsf.getAveragePosition()
710 scienceSigmaOrig = sciencePsf.computeShape(sciAvgPos).getDeterminantRadius()
711
712 templatePsf = templateExposure.getPsf()
713 templateAvgPos = templatePsf.getAveragePosition()
714 templateSigma = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()
715
716 # if requested, convolve the science exposure with its PSF
717 # (properly, this should be a cross-correlation, but our code does not yet support that)
718 # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
719 # else sigma of original science exposure
720 # TODO: DM-22762 This functional block should be moved into its own method
721 preConvPsf = None
722 if self.config.useScoreImageDetection:
723 self.log.warning("AL likelihood image: pre-convolution of PSF is not implemented.")
724 convControl = afwMath.ConvolutionControl()
725 # cannot convolve in place, so need a new image anyway
726 srcMI = exposure.maskedImage
727 exposure = exposure.clone() # New deep copy
728 srcPsf = sciencePsf
729 if self.config.useGaussianForPreConvolution:
730 self.log.info(
731 "AL likelihood image: Using Gaussian (sigma=%.2f) PSF estimation "
732 "for science image pre-convolution", scienceSigmaOrig)
733 # convolve with a simplified PSF model: a double Gaussian
734 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
735 preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
736 else:
737 # convolve with science exposure's PSF model
738 self.log.info(
739 "AL likelihood image: Using the science image PSF for pre-convolution.")
740 preConvPsf = srcPsf
741 afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
742 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
743 else:
744 scienceSigmaPost = scienceSigmaOrig
745
746 # If requested, find and select sources from the image
747 # else, AL subtraction will do its own source detection
748 # TODO: DM-22762 This functional block should be moved into its own method
749 if self.config.doSelectSources:
750 if selectSources is None:
751 self.log.warning("Src product does not exist; running detection, measurement,"
752 " selection")
753 # Run own detection and measurement; necessary in nightly processing
754 selectSources = self.subtract.getSelectSources(
755 exposure,
756 sigma=scienceSigmaPost,
757 doSmooth=not self.config.useScoreImageDetection,
758 idFactory=idFactory,
759 )
760
761 if self.config.doAddMetrics:
762 # Number of basis functions
763
764 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
765 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
766 targetFwhmPix=templateSigma*FwhmPerSigma))
767 # Modify the schema of all Sources
768 # DEPRECATED: This is a data dependent (nparam) output product schema
769 # outside the task constructor.
770 # NOTE: The pre-determination of nparam at this point
771 # may be incorrect as the template psf is warped later in
772 # ImagePsfMatchTask.matchExposures()
773 kcQa = KernelCandidateQa(nparam)
774 selectSources = kcQa.addToSchema(selectSources)
775 if self.config.kernelSourcesFromRef:
776 # match exposure sources to reference catalog
777 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
778 matches = astromRet.matches
779 elif templateSources:
780 # match exposure sources to template sources
781 mc = afwTable.MatchControl()
782 mc.findOnlyClosest = False
783 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
784 mc)
785 else:
786 raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
787 "but template sources not available. Cannot match science "
788 "sources with template sources. Run process* on data from "
789 "which templates are built.")
790
791 kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
792 matches=matches).sourceCat
793 random.shuffle(kernelSources, random.random)
794 controlSources = kernelSources[::self.config.controlStepSize]
795 kernelSources = [k for i, k in enumerate(kernelSources)
796 if i % self.config.controlStepSize]
797
798 if self.config.doSelectDcrCatalog:
799 redSelector = DiaCatalogSourceSelectorTask(
800 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
801 grMax=99.999))
802 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
803 controlSources.extend(redSources)
804
805 blueSelector = DiaCatalogSourceSelectorTask(
806 DiaCatalogSourceSelectorConfig(grMin=-99.999,
807 grMax=self.sourceSelector.config.grMin))
808 blueSources = blueSelector.selectStars(exposure, selectSources,
809 matches=matches).starCat
810 controlSources.extend(blueSources)
811
812 if self.config.doSelectVariableCatalog:
813 varSelector = DiaCatalogSourceSelectorTask(
814 DiaCatalogSourceSelectorConfig(includeVariable=True))
815 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
816 controlSources.extend(varSources)
817
818 self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)",
819 len(kernelSources), len(selectSources), len(controlSources))
820
821 allresids = {}
822 # TODO: DM-22762 This functional block should be moved into its own method
823 if self.config.doUseRegister:
824 self.log.info("Registering images")
825
826 if templateSources is None:
827 # Run detection on the template, which is
828 # temporarily background-subtracted
829 # sigma of PSF of template image before warping
830 templateSources = self.subtract.getSelectSources(
831 templateExposure,
832 sigma=templateSigma,
833 doSmooth=True,
834 idFactory=idFactory
835 )
836
837 # Third step: we need to fit the relative astrometry.
838 #
839 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
840 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
841 exposure.getWcs(), exposure.getBBox())
842 templateExposure = warpedExp
843
844 # Create debugging outputs on the astrometric
845 # residuals as a function of position. Persistence
846 # not yet implemented; expected on (I believe) #2636.
847 if self.config.doDebugRegister:
848 # Grab matches to reference catalog
849 srcToMatch = {x.second.getId(): x.first for x in matches}
850
851 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
852 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
853 sids = [m.first.getId() for m in wcsResults.matches]
854 positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
855 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
856 m.second.get(inCentroidKey))) for m in wcsResults.matches]
857 allresids = dict(zip(sids, zip(positions, residuals)))
858
859 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
860 wcsResults.wcs.pixelToSky(
861 m.second.get(inCentroidKey))) for m in wcsResults.matches]
862 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
863 + 2.5*numpy.log10(srcToMatch[x].get("r"))
864 for x in sids if x in srcToMatch.keys()])
865 dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
866 if s in srcToMatch.keys()])
867 dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
868 if s in srcToMatch.keys()])
869 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
870 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
871 & (colors <= self.sourceSelector.config.grMax))
872 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
873 rms1Long = IqrToSigma*(
874 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
875 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
876 - numpy.percentile(dlat[idx1], 25))
877 rms2Long = IqrToSigma*(
878 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
879 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
880 - numpy.percentile(dlat[idx2], 25))
881 rms3Long = IqrToSigma*(
882 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
883 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
884 - numpy.percentile(dlat[idx3], 25))
885 self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f",
886 numpy.median(dlong[idx1]), rms1Long,
887 numpy.median(dlat[idx1]), rms1Lat)
888 self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f",
889 numpy.median(dlong[idx2]), rms2Long,
890 numpy.median(dlat[idx2]), rms2Lat)
891 self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f",
892 numpy.median(dlong[idx3]), rms3Long,
893 numpy.median(dlat[idx3]), rms3Lat)
894
895 self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
896 self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
897 self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
898 self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
899 self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
900 self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
901
902 self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
903 self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
904 self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
905 self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
906 self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
907 self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
908
909 # warp template exposure to match exposure,
910 # PSF match template exposure to exposure,
911 # then return the difference
912
913 # Return warped template... Construct sourceKernelCand list after subtract
914 self.log.info("Subtracting images")
915 subtractRes = self.subtract.subtractExposures(
916 templateExposure=templateExposure,
917 scienceExposure=exposure,
918 candidateList=kernelSources,
919 convolveTemplate=self.config.convolveTemplate,
920 doWarping=not self.config.doUseRegister
921 )
922 if self.config.useScoreImageDetection:
923 scoreExposure = subtractRes.subtractedExposure
924 else:
925 subtractedExposure = subtractRes.subtractedExposure
926
927 if self.config.doDetection:
928 self.log.info("Computing diffim PSF")
929
930 # Get Psf from the appropriate input image if it doesn't exist
931 if subtractedExposure is not None and not subtractedExposure.hasPsf():
932 if self.config.convolveTemplate:
933 subtractedExposure.setPsf(exposure.getPsf())
934 else:
935 subtractedExposure.setPsf(templateExposure.getPsf())
936
937 # If doSubtract is False, then subtractedExposure was fetched from disk (above),
938 # thus it may have already been decorrelated. Thus, we do not decorrelate if
939 # doSubtract is False.
940
941 # NOTE: At this point doSubtract == True
942 if self.config.doDecorrelation and self.config.doSubtract:
943 preConvKernel = None
944 if self.config.useGaussianForPreConvolution:
945 preConvKernel = preConvPsf.getLocalKernel()
946 if self.config.useScoreImageDetection:
947 scoreExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
948 scoreExposure,
949 subtractRes.psfMatchingKernel,
950 spatiallyVarying=self.config.doSpatiallyVarying,
951 preConvKernel=preConvKernel,
952 templateMatched=True,
953 preConvMode=True).correctedExposure
954 # Note that the subtracted exposure is always decorrelated,
955 # even if the score image is used for detection
956 subtractedExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
957 subtractedExposure,
958 subtractRes.psfMatchingKernel,
959 spatiallyVarying=self.config.doSpatiallyVarying,
960 preConvKernel=None,
961 templateMatched=self.config.convolveTemplate,
962 preConvMode=False).correctedExposure
963 # END (if subtractAlgorithm == 'AL')
964 # END (if self.config.doSubtract)
965 if self.config.doDetection:
966 self.log.info("Running diaSource detection")
967
968 # subtractedExposure - reserved for task return value
969 # in zogy, it is always the proper difference image
970 # in AL, it may be (yet) pre-convolved and/or decorrelated
971 #
972 # detectionExposure - controls which exposure to use for detection
973 # in-place modifications will appear in task return
974 if self.config.useScoreImageDetection:
975 # zogy with score image detection enabled
976 self.log.info("Detection, diffim rescaling and measurements are "
977 "on AL likelihood or Zogy score image.")
978 detectionExposure = scoreExposure
979 else:
980 # AL or zogy with no score image detection
981 detectionExposure = subtractedExposure
982
983 # Rescale difference image variance plane
984 if self.config.doScaleDiffimVariance:
985 self.log.info("Rescaling diffim variance")
986 diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
987 self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor)
988 self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
989
990 # Erase existing detection mask planes
991 mask = detectionExposure.getMaskedImage().getMask()
992 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
993
994 table = afwTable.SourceTable.make(self.schema, idFactory)
995 table.setMetadata(self.algMetadata)
996 results = self.detection.run(
997 table=table,
998 exposure=detectionExposure,
999 doSmooth=not self.config.useScoreImageDetection
1000 )
1001
1002 if self.config.doMerge:
1003 fpSet = results.fpSets.positive
1004 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1005 self.config.growFootprint, False)
1006 diaSources = afwTable.SourceCatalog(table)
1007 fpSet.makeSources(diaSources)
1008 self.log.info("Merging detections into %d sources", len(diaSources))
1009 else:
1010 diaSources = results.sources
1011 # Inject skySources before measurement.
1012 if self.config.doSkySources:
1013 skySourceFootprints = self.skySources.run(
1014 mask=detectionExposure.mask,
1015 seed=detectionExposure.info.id)
1016 if skySourceFootprints:
1017 for foot in skySourceFootprints:
1018 s = diaSources.addNew()
1019 s.setFootprint(foot)
1020 s.set(self.skySourceKey, True)
1021
1022 if self.config.doMeasurement:
1023 newDipoleFitting = self.config.doDipoleFitting
1024 self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1025 if not newDipoleFitting:
1026 # Just fit dipole in diffim
1027 self.measurement.run(diaSources, detectionExposure)
1028 else:
1029 # Use (matched) template and science image (if avail.) to constrain dipole fitting
1030 if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1031 self.measurement.run(diaSources, detectionExposure, exposure,
1032 subtractRes.matchedExposure)
1033 else:
1034 self.measurement.run(diaSources, detectionExposure, exposure)
1035 if self.config.doApCorr:
1036 self.applyApCorr.run(
1037 catalog=diaSources,
1038 apCorrMap=detectionExposure.getInfo().getApCorrMap()
1039 )
1040
1041 if self.config.doForcedMeasurement:
1042 # Run forced psf photometry on the PVI at the diaSource locations.
1043 # Copy the measured flux and error into the diaSource.
1044 forcedSources = self.forcedMeasurement.generateMeasCat(
1045 exposure, diaSources, detectionExposure.getWcs())
1046 self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1047 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1048 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1049 "ip_diffim_forced_PsfFlux_instFlux", True)
1050 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1051 "ip_diffim_forced_PsfFlux_instFluxErr", True)
1052 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1053 "ip_diffim_forced_PsfFlux_area", True)
1054 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1055 "ip_diffim_forced_PsfFlux_flag", True)
1056 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1057 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1058 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1059 "ip_diffim_forced_PsfFlux_flag_edge", True)
1060 for diaSource, forcedSource in zip(diaSources, forcedSources):
1061 diaSource.assign(forcedSource, mapper)
1062
1063 # Match with the calexp sources if possible
1064 if self.config.doMatchSources:
1065 if selectSources is not None:
1066 # Create key,val pair where key=diaSourceId and val=sourceId
1067 matchRadAsec = self.config.diaSourceMatchRadius
1068 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1069
1070 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1071 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1072 srcMatch in srcMatches])
1073 self.log.info("Matched %d / %d diaSources to sources",
1074 len(srcMatchDict), len(diaSources))
1075 else:
1076 self.log.warning("Src product does not exist; cannot match with diaSources")
1077 srcMatchDict = {}
1078
1079 # Create key,val pair where key=diaSourceId and val=refId
1080 refAstromConfig = AstrometryConfig()
1081 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1082 refAstrometer = AstrometryTask(self.refObjLoader, config=refAstromConfig)
1083 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1084 refMatches = astromRet.matches
1085 if refMatches is None:
1086 self.log.warning("No diaSource matches with reference catalog")
1087 refMatchDict = {}
1088 else:
1089 self.log.info("Matched %d / %d diaSources to reference catalog",
1090 len(refMatches), len(diaSources))
1091 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1092 refMatch in refMatches])
1093
1094 # Assign source Ids
1095 for diaSource in diaSources:
1096 sid = diaSource.getId()
1097 if sid in srcMatchDict:
1098 diaSource.set("srcMatchId", srcMatchDict[sid])
1099 if sid in refMatchDict:
1100 diaSource.set("refMatchId", refMatchDict[sid])
1101
1102 if self.config.doAddMetrics and self.config.doSelectSources:
1103 self.log.info("Evaluating metrics and control sample")
1104
1105 kernelCandList = []
1106 for cell in subtractRes.kernelCellSet.getCellList():
1107 for cand in cell.begin(False): # include bad candidates
1108 kernelCandList.append(cand)
1109
1110 # Get basis list to build control sample kernels
1111 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1112 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1113
1114 controlCandList = (
1115 diffimTools.sourceTableToCandidateList(controlSources,
1116 subtractRes.warpedExposure, exposure,
1117 self.config.subtract.kernel.active,
1118 self.config.subtract.kernel.active.detectionConfig,
1119 self.log, doBuild=True, basisList=basisList))
1120
1121 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1122 subtractRes.backgroundModel, dof=nparam)
1123 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1124 subtractRes.backgroundModel)
1125
1126 if self.config.doDetection:
1127 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1128 else:
1129 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1130
1131 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1132 return pipeBase.Struct(
1133 subtractedExposure=subtractedExposure,
1134 scoreExposure=scoreExposure,
1135 warpedExposure=subtractRes.warpedExposure,
1136 matchedExposure=subtractRes.matchedExposure,
1137 subtractRes=subtractRes,
1138 diaSources=diaSources,
1139 selectSources=selectSources
1140 )
1141
1142 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1143 """Fit the relative astrometry between templateSources and selectSources
1144
1145 Todo
1146 ----
1147
1148 Remove this method. It originally fit a new WCS to the template before calling register.run
1149 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1150 It remains because a subtask overrides it.
1151 """
1152 results = self.register.run(templateSources, templateExposure.getWcs(),
1153 templateExposure.getBBox(), selectSources)
1154 return results
1155
1156 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1157 """Make debug plots and displays.
1158
1159 Todo
1160 ----
1161 Test and update for current debug display and slot names
1162 """
1163 import lsstDebug
1164 display = lsstDebug.Info(__name__).display
1165 showSubtracted = lsstDebug.Info(__name__).showSubtracted
1166 showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1167 showDiaSources = lsstDebug.Info(__name__).showDiaSources
1168 showDipoles = lsstDebug.Info(__name__).showDipoles
1169 maskTransparency = lsstDebug.Info(__name__).maskTransparency
1170 if display:
1171 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1172 if not maskTransparency:
1173 maskTransparency = 0
1174 disp.setMaskTransparency(maskTransparency)
1175
1176 if display and showSubtracted:
1177 disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1178 mi = subtractRes.subtractedExposure.getMaskedImage()
1179 x0, y0 = mi.getX0(), mi.getY0()
1180 with disp.Buffering():
1181 for s in diaSources:
1182 x, y = s.getX() - x0, s.getY() - y0
1183 ctype = "red" if s.get("flags_negative") else "yellow"
1184 if (s.get("base_PixelFlags_flag_interpolatedCenter")
1185 or s.get("base_PixelFlags_flag_saturatedCenter")
1186 or s.get("base_PixelFlags_flag_crCenter")):
1187 ptype = "x"
1188 elif (s.get("base_PixelFlags_flag_interpolated")
1189 or s.get("base_PixelFlags_flag_saturated")
1190 or s.get("base_PixelFlags_flag_cr")):
1191 ptype = "+"
1192 else:
1193 ptype = "o"
1194 disp.dot(ptype, x, y, size=4, ctype=ctype)
1195 lsstDebug.frame += 1
1196
1197 if display and showPixelResiduals and selectSources:
1198 nonKernelSources = []
1199 for source in selectSources:
1200 if source not in kernelSources:
1201 nonKernelSources.append(source)
1202
1203 diUtils.plotPixelResiduals(exposure,
1204 subtractRes.warpedExposure,
1205 subtractRes.subtractedExposure,
1206 subtractRes.kernelCellSet,
1207 subtractRes.psfMatchingKernel,
1208 subtractRes.backgroundModel,
1209 nonKernelSources,
1210 self.subtract.config.kernel.active.detectionConfig,
1211 origVariance=False)
1212 diUtils.plotPixelResiduals(exposure,
1213 subtractRes.warpedExposure,
1214 subtractRes.subtractedExposure,
1215 subtractRes.kernelCellSet,
1216 subtractRes.psfMatchingKernel,
1217 subtractRes.backgroundModel,
1218 nonKernelSources,
1219 self.subtract.config.kernel.active.detectionConfig,
1220 origVariance=True)
1221 if display and showDiaSources:
1222 flagChecker = SourceFlagChecker(diaSources)
1223 isFlagged = [flagChecker(x) for x in diaSources]
1224 isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1225 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1226 frame=lsstDebug.frame)
1227 lsstDebug.frame += 1
1228
1229 if display and showDipoles:
1230 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1231 frame=lsstDebug.frame)
1232 lsstDebug.frame += 1
1233
1234 def checkTemplateIsSufficient(self, templateExposure):
1235 """Raise NoWorkFound if template coverage < requiredTemplateFraction
1236
1237 Parameters
1238 ----------
1239 templateExposure : `lsst.afw.image.ExposureF`
1240 The template exposure to check
1241
1242 Raises
1243 ------
1244 NoWorkFound
1245 Raised if fraction of good pixels, defined as not having NO_DATA
1246 set, is less then the configured requiredTemplateFraction
1247 """
1248 # Count the number of pixels with the NO_DATA mask bit set
1249 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
1250 pixNoData = numpy.count_nonzero(templateExposure.mask.array
1251 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
1252 pixGood = templateExposure.getBBox().getArea() - pixNoData
1253 self.log.info("template has %d good pixels (%.1f%%)", pixGood,
1254 100*pixGood/templateExposure.getBBox().getArea())
1255
1256 if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction:
1257 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
1258 "To force subtraction, set config requiredTemplateFraction=0." % (
1259 100*pixGood/templateExposure.getBBox().getArea(),
1260 100*self.config.requiredTemplateFraction))
1261 raise pipeBase.NoWorkFound(message)
1262
1263
1264class ImageDifferenceFromTemplateConnections(ImageDifferenceTaskConnections,
1265 defaultTemplates={"coaddName": "goodSeeing"}
1266 ):
1267 inputTemplate = pipeBase.connectionTypes.Input(
1268 doc=("Warped template produced by GetMultiTractCoaddTemplate"),
1269 dimensions=("instrument", "visit", "detector"),
1270 storageClass="ExposureF",
1271 name="{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}",
1272 )
1273
1274 def __init__(self, *, config=None):
1275 super().__init__(config=config)
1276 # ImageDifferenceConnections will have removed one of these.
1277 # Make sure they're both gone, because no coadds are needed.
1278 if "coaddExposures" in self.inputs:
1279 self.inputs.remove("coaddExposures")
1280 if "dcrCoadds" in self.inputs:
1281 self.inputs.remove("dcrCoadds")
1282
1283
1284class ImageDifferenceFromTemplateConfig(ImageDifferenceConfig,
1285 pipelineConnections=ImageDifferenceFromTemplateConnections):
1286 pass
1287
1288
1289class ImageDifferenceFromTemplateTask(ImageDifferenceTask):
1290 ConfigClass = ImageDifferenceFromTemplateConfig
1291 _DefaultName = "imageDifference"
1292
1293 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
1294 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1295 inputs = butlerQC.get(inputRefs)
1296 self.log.info("Processing %s", butlerQC.quantum.dataId)
1297 self.checkTemplateIsSufficient(inputs['inputTemplate'])
1298 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
1299 returnMaxBits=True)
1300 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
1301
1302 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None)
1303 exposure = self.prepareCalibratedExposure(
1304 inputs["exposure"],
1305 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog
1306 )
1307
1308 outputs = self.run(exposure=exposure,
1309 templateExposure=inputs['inputTemplate'],
1310 idFactory=idFactory)
1311
1312 # Consistency with runDataref gen2 handling
1313 if outputs.diaSources is None:
1314 del outputs.diaSources
1315 butlerQC.put(outputs, outputRefs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)