lsst.pipe.tasks g57dfbfe8b2+08e3ebb819
imageDifference.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import math
23import random
24import numpy
25
26import lsst.utils
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29import lsst.daf.base as dafBase
30import lsst.geom as geom
31import lsst.afw.math as afwMath
32import lsst.afw.table as afwTable
33import lsst.meas.extensions.trailedSources # noqa: F401
34from lsst.meas.algorithms import (SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask,
35 LoadIndexedReferenceObjectsTask, SkyObjectsTask,
36 ScaleVarianceTask)
37from lsst.meas.astrom import AstrometryConfig, AstrometryTask
38from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
39from lsst.pipe.tasks.registerImage import RegisterTask
40from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
41 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
42 GetCoaddAsTemplateTask, DipoleFitTask,
43 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
44import lsst.ip.diffim.diffimTools as diffimTools
45import lsst.ip.diffim.utils as diUtils
46import lsst.afw.display as afwDisplay
47from lsst.skymap import BaseSkyMap
48from lsst.obs.base import ExposureIdInfo
49from lsst.utils.timer import timeMethod
50
51from deprecated.sphinx import deprecated
52
53__all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
54FwhmPerSigma = 2*math.sqrt(2*math.log(2))
55IqrToSigma = 0.741
56
57
58class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
59 dimensions=("instrument", "visit", "detector", "skymap"),
60 defaultTemplates={"coaddName": "deep",
61 "skyMapName": "deep",
62 "warpTypeSuffix": "",
63 "fakesType": ""}):
64
65 exposure = pipeBase.connectionTypes.Input(
66 doc="Input science exposure to subtract from.",
67 dimensions=("instrument", "visit", "detector"),
68 storageClass="ExposureF",
69 name="{fakesType}calexp"
70 )
71
72 # TODO DM-22953
73 # kernelSources = pipeBase.connectionTypes.Input(
74 # doc="Source catalog produced in calibrate task for kernel candidate sources",
75 # name="src",
76 # storageClass="SourceCatalog",
77 # dimensions=("instrument", "visit", "detector"),
78 # )
79
80 skyMap = pipeBase.connectionTypes.Input(
81 doc="Input definition of geometry/bbox and projection/wcs for template exposures",
82 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
83 dimensions=("skymap", ),
84 storageClass="SkyMap",
85 )
86 coaddExposures = pipeBase.connectionTypes.Input(
87 doc="Input template to match and subtract from the exposure",
88 dimensions=("tract", "patch", "skymap", "band"),
89 storageClass="ExposureF",
90 name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
91 multiple=True,
92 deferLoad=True
93 )
94 dcrCoadds = pipeBase.connectionTypes.Input(
95 doc="Input DCR template to match and subtract from the exposure",
96 name="{fakesType}dcrCoadd{warpTypeSuffix}",
97 storageClass="ExposureF",
98 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
99 multiple=True,
100 deferLoad=True
101 )
102 finalizedPsfApCorrCatalog = pipeBase.connectionTypes.Input(
103 doc=("Per-visit finalized psf models and aperture correction maps. "
104 "These catalogs use the detector id for the catalog id, "
105 "sorted on id for fast lookup."),
106 name="finalized_psf_ap_corr_catalog",
107 storageClass="ExposureCatalog",
108 dimensions=("instrument", "visit"),
109 )
110 outputSchema = pipeBase.connectionTypes.InitOutput(
111 doc="Schema (as an example catalog) for output DIASource catalog.",
112 storageClass="SourceCatalog",
113 name="{fakesType}{coaddName}Diff_diaSrc_schema",
114 )
115 subtractedExposure = pipeBase.connectionTypes.Output(
116 doc="Output AL difference or Zogy proper difference image",
117 dimensions=("instrument", "visit", "detector"),
118 storageClass="ExposureF",
119 name="{fakesType}{coaddName}Diff_differenceExp",
120 )
121 scoreExposure = pipeBase.connectionTypes.Output(
122 doc="Output AL likelihood or Zogy score image",
123 dimensions=("instrument", "visit", "detector"),
124 storageClass="ExposureF",
125 name="{fakesType}{coaddName}Diff_scoreExp",
126 )
127 warpedExposure = pipeBase.connectionTypes.Output(
128 doc="Warped template used to create `subtractedExposure`.",
129 dimensions=("instrument", "visit", "detector"),
130 storageClass="ExposureF",
131 name="{fakesType}{coaddName}Diff_warpedExp",
132 )
133 matchedExposure = pipeBase.connectionTypes.Output(
134 doc="Warped template used to create `subtractedExposure`.",
135 dimensions=("instrument", "visit", "detector"),
136 storageClass="ExposureF",
137 name="{fakesType}{coaddName}Diff_matchedExp",
138 )
139 diaSources = pipeBase.connectionTypes.Output(
140 doc="Output detected diaSources on the difference image",
141 dimensions=("instrument", "visit", "detector"),
142 storageClass="SourceCatalog",
143 name="{fakesType}{coaddName}Diff_diaSrc",
144 )
145
146 def __init__(self, *, config=None):
147 super().__init__(config=config)
148 if config.coaddName == 'dcr':
149 self.inputs.remove("coaddExposures")
150 else:
151 self.inputs.remove("dcrCoadds")
152 if not config.doWriteSubtractedExp:
153 self.outputs.remove("subtractedExposure")
154 if not config.doWriteScoreExp:
155 self.outputs.remove("scoreExposure")
156 if not config.doWriteWarpedExp:
157 self.outputs.remove("warpedExposure")
158 if not config.doWriteMatchedExp:
159 self.outputs.remove("matchedExposure")
160 if not config.doWriteSources:
161 self.outputs.remove("diaSources")
162 if not config.doApplyFinalizedPsf:
163 self.inputs.remove("finalizedPsfApCorrCatalog")
164
165 # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
166 # Make kernelSources optional
167
168
169class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
170 pipelineConnections=ImageDifferenceTaskConnections):
171 """Config for ImageDifferenceTask
172 """
173 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
174 doc="Add background to calexp before processing it. "
175 "Useful as ipDiffim does background matching.")
176 doUseRegister = pexConfig.Field(dtype=bool, default=False,
177 doc="Re-compute astrometry on the template. "
178 "Use image-to-image registration to align template with "
179 "science image (AL only).")
180 doDebugRegister = pexConfig.Field(dtype=bool, default=False,
181 doc="Writing debugging data for doUseRegister")
182 doSelectSources = pexConfig.Field(dtype=bool, default=False,
183 doc="Select stars to use for kernel fitting (AL only)")
184 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
185 doc="Select stars of extreme color as part "
186 "of the control sample (AL only)")
187 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
188 doc="Select stars that are variable to be part "
189 "of the control sample (AL only)")
190 doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
191 doPreConvolve = pexConfig.Field(dtype=bool, default=False,
192 doc="Not in use. Superseded by useScoreImageDetection.",
193 deprecated="This option superseded by useScoreImageDetection."
194 " Will be removed after v22.")
195 useScoreImageDetection = pexConfig.Field(
196 dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or "
197 "the Zogy score image. Use it for source detection (if doDetection=True).")
198 doWriteScoreExp = pexConfig.Field(
199 dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?")
200 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
201 doc="Scale variance of the template before PSF matching")
202 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
203 doc="Scale variance of the diffim before PSF matching. "
204 "You may do either this or template variance scaling, "
205 "or neither. (Doing both is a waste of CPU.)")
206 useGaussianForPreConvolution = pexConfig.Field(
207 dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution "
208 "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)")
209 doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
210 doDecorrelation = pexConfig.Field(dtype=bool, default=True,
211 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
212 "kernel convolution (AL only)? If True, also update the diffim PSF.")
213 doMerge = pexConfig.Field(dtype=bool, default=True,
214 doc="Merge positive and negative diaSources with grow radius "
215 "set by growFootprint")
216 doMatchSources = pexConfig.Field(dtype=bool, default=False,
217 doc="Match diaSources with input calexp sources and ref catalog sources")
218 doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
219 doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
220 doForcedMeasurement = pexConfig.Field(
221 dtype=bool,
222 default=True,
223 doc="Force photometer diaSource locations on PVI?")
224 doWriteSubtractedExp = pexConfig.Field(
225 dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
226 doWriteWarpedExp = pexConfig.Field(
227 dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
228 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
229 doc="Write warped and PSF-matched template coadd exposure?")
230 doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
231 doAddMetrics = pexConfig.Field(dtype=bool, default=False,
232 doc="Add columns to the source table to hold analysis metrics?")
233 doApplyFinalizedPsf = pexConfig.Field(
234 doc="Whether to apply finalized psf models and aperture correction map.",
235 dtype=bool,
236 default=False,
237 )
238
239 coaddName = pexConfig.Field(
240 doc="coadd name: typically one of deep, goodSeeing, or dcr",
241 dtype=str,
242 default="deep",
243 )
244 convolveTemplate = pexConfig.Field(
245 doc="Which image gets convolved (default = template)",
246 dtype=bool,
247 default=True
248 )
249 refObjLoader = pexConfig.ConfigurableField(
250 target=LoadIndexedReferenceObjectsTask,
251 doc="reference object loader",
252 )
253 astrometer = pexConfig.ConfigurableField(
254 target=AstrometryTask,
255 doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
256 )
257 sourceSelector = pexConfig.ConfigurableField(
258 target=ObjectSizeStarSelectorTask,
259 doc="Source selection algorithm",
260 )
261 subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
262 decorrelate = pexConfig.ConfigurableField(
263 target=DecorrelateALKernelSpatialTask,
264 doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
265 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
266 "default of 5.5).",
267 )
268 # Old style ImageMapper grid. ZogyTask has its own grid option
269 doSpatiallyVarying = pexConfig.Field(
270 dtype=bool,
271 default=False,
272 doc="Perform A&L decorrelation on a grid across the "
273 "image in order to allow for spatial variations. Zogy does not use this option."
274 )
275 detection = pexConfig.ConfigurableField(
276 target=SourceDetectionTask,
277 doc="Low-threshold detection for final measurement",
278 )
279 measurement = pexConfig.ConfigurableField(
280 target=DipoleFitTask,
281 doc="Enable updated dipole fitting method",
282 )
283 doApCorr = lsst.pex.config.Field(
284 dtype=bool,
285 default=True,
286 doc="Run subtask to apply aperture corrections"
287 )
288 applyApCorr = lsst.pex.config.ConfigurableField(
289 target=ApplyApCorrTask,
290 doc="Subtask to apply aperture corrections"
291 )
292 forcedMeasurement = pexConfig.ConfigurableField(
293 target=ForcedMeasurementTask,
294 doc="Subtask to force photometer PVI at diaSource location.",
295 )
296 getTemplate = pexConfig.ConfigurableField(
297 target=GetCoaddAsTemplateTask,
298 doc="Subtask to retrieve template exposure and sources",
299 )
300 scaleVariance = pexConfig.ConfigurableField(
301 target=ScaleVarianceTask,
302 doc="Subtask to rescale the variance of the template "
303 "to the statistically expected level"
304 )
305 controlStepSize = pexConfig.Field(
306 doc="What step size (every Nth one) to select a control sample from the kernelSources",
307 dtype=int,
308 default=5
309 )
310 controlRandomSeed = pexConfig.Field(
311 doc="Random seed for shuffing the control sample",
312 dtype=int,
313 default=10
314 )
315 register = pexConfig.ConfigurableField(
316 target=RegisterTask,
317 doc="Task to enable image-to-image image registration (warping)",
318 )
319 kernelSourcesFromRef = pexConfig.Field(
320 doc="Select sources to measure kernel from reference catalog if True, template if false",
321 dtype=bool,
322 default=False
323 )
324 templateSipOrder = pexConfig.Field(
325 dtype=int, default=2,
326 doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
327 )
328 growFootprint = pexConfig.Field(
329 dtype=int, default=2,
330 doc="Grow positive and negative footprints by this amount before merging"
331 )
332 diaSourceMatchRadius = pexConfig.Field(
333 dtype=float, default=0.5,
334 doc="Match radius (in arcseconds) for DiaSource to Source association"
335 )
336 requiredTemplateFraction = pexConfig.Field(
337 dtype=float, default=0.1,
338 doc="Do not attempt to run task if template covers less than this fraction of pixels."
339 "Setting to 0 will always attempt image subtraction"
340 )
341 doSkySources = pexConfig.Field(
342 dtype=bool,
343 default=False,
344 doc="Generate sky sources?",
345 )
346 skySources = pexConfig.ConfigurableField(
347 target=SkyObjectsTask,
348 doc="Generate sky sources",
349 )
350
351 def setDefaults(self):
352 # defaults are OK for catalog and diacatalog
353
354 self.subtract['al'].kernel.name = "AL"
355 self.subtract['al'].kernel.active.fitForBackground = True
356 self.subtract['al'].kernel.active.spatialKernelOrder = 1
357 self.subtract['al'].kernel.active.spatialBgOrder = 2
358
359 # DiaSource Detection
360 self.detection.thresholdPolarity = "both"
361 self.detection.thresholdValue = 5.0
362 self.detection.reEstimateBackground = False
363 self.detection.thresholdType = "pixel_stdev"
364
365 # Add filtered flux measurement, the correct measurement for pre-convolved images.
366 # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
367 # To change that you must modify algorithms.names in the task's applyOverrides method,
368 # after the user has set doPreConvolve.
369 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
370 self.measurement.plugins.names |= ['ext_trailedSources_Naive',
371 'base_LocalPhotoCalib',
372 'base_LocalWcs']
373
374 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
375 self.forcedMeasurement.copyColumns = {
376 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
377 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
378 self.forcedMeasurement.slots.shape = None
379
380 # For shuffling the control sample
381 random.seed(self.controlRandomSeed)
382
383 def validate(self):
384 pexConfig.Config.validate(self)
385 if not self.doSubtract and not self.doDetection:
386 raise ValueError("Either doSubtract or doDetection must be enabled.")
387 if self.doMeasurement and not self.doDetection:
388 raise ValueError("Cannot run source measurement without source detection.")
389 if self.doMerge and not self.doDetection:
390 raise ValueError("Cannot run source merging without source detection.")
391 if self.doSkySources and not self.doDetection:
392 raise ValueError("Cannot run sky source creation without source detection.")
393 if self.doUseRegister and not self.doSelectSources:
394 raise ValueError("doUseRegister=True and doSelectSources=False. "
395 "Cannot run RegisterTask without selecting sources.")
396 if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
397 raise ValueError("Scaling the diffim variance and scaling the template variance "
398 "are both set. Please choose one or the other.")
399 # We cannot allow inconsistencies that would lead to None or not available output products
400 if self.subtract.name == 'zogy':
401 if self.doWriteMatchedExp:
402 raise ValueError("doWriteMatchedExp=True Matched exposure is not "
403 "calculated in zogy subtraction.")
404 if self.doAddMetrics:
405 raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
406 if self.doDecorrelation:
407 raise ValueError(
408 "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
409 if self.doSelectSources:
410 raise ValueError(
411 "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
412 if self.useGaussianForPreConvolution:
413 raise ValueError(
414 "useGaussianForPreConvolution=True This is an AL subtraction only option.")
415 else:
416 # AL only consistency checks
417 if self.useScoreImageDetection and not self.convolveTemplate:
418 raise ValueError(
419 "convolveTemplate=False and useScoreImageDetection=True "
420 "Pre-convolution and matching of the science image is not a supported operation.")
421 if self.doWriteSubtractedExp and self.useScoreImageDetection:
422 raise ValueError(
423 "doWriteSubtractedExp=True and useScoreImageDetection=True "
424 "Regular difference image is not calculated. "
425 "AL subtraction calculates either the regular difference image or the score image.")
426 if self.doWriteScoreExp and not self.useScoreImageDetection:
427 raise ValueError(
428 "doWriteScoreExp=True and useScoreImageDetection=False "
429 "Score image is not calculated. "
430 "AL subtraction calculates either the regular difference image or the score image.")
431 if self.doAddMetrics and not self.doSubtract:
432 raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
433 if self.useGaussianForPreConvolution and not self.useScoreImageDetection:
434 raise ValueError(
435 "useGaussianForPreConvolution=True and useScoreImageDetection=False "
436 "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.")
437
438
439class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
440
441 @staticmethod
442 def getTargetList(parsedCmd, **kwargs):
443 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
444 **kwargs)
445
446
447@deprecated(reason="This Task has been replaced with lsst.ip.diffim.subtractImages"
448 " and lsst.ip.diffim.detectAndMeasure. Will be removed after v25.",
449 version="v24.0", category=FutureWarning)
450class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
451 """Subtract an image from a template and measure the result
452 """
453 ConfigClass = ImageDifferenceConfig
454 RunnerClass = ImageDifferenceTaskRunner
455 _DefaultName = "imageDifference"
456
457 def __init__(self, butler=None, **kwargs):
458 """!Construct an ImageDifference Task
459
460 @param[in] butler Butler object to use in constructing reference object loaders
461 """
462 super().__init__(**kwargs)
463 self.makeSubtask("getTemplate")
464
465 self.makeSubtask("subtract")
466
467 if self.config.subtract.name == 'al' and self.config.doDecorrelation:
468 self.makeSubtask("decorrelate")
469
470 if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
471 self.makeSubtask("scaleVariance")
472
473 if self.config.doUseRegister:
474 self.makeSubtask("register")
475 self.schema = afwTable.SourceTable.makeMinimalSchema()
476
477 if self.config.doSelectSources:
478 self.makeSubtask("sourceSelector")
479 if self.config.kernelSourcesFromRef:
480 self.makeSubtask('refObjLoader', butler=butler)
481 self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
482
483 self.algMetadata = dafBase.PropertyList()
484 if self.config.doDetection:
485 self.makeSubtask("detection", schema=self.schema)
486 if self.config.doMeasurement:
487 self.makeSubtask("measurement", schema=self.schema,
488 algMetadata=self.algMetadata)
489 if self.config.doApCorr:
490 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
491 if self.config.doForcedMeasurement:
492 self.schema.addField(
493 "ip_diffim_forced_PsfFlux_instFlux", "D",
494 "Forced PSF flux measured on the direct image.",
495 units="count")
496 self.schema.addField(
497 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
498 "Forced PSF flux error measured on the direct image.",
499 units="count")
500 self.schema.addField(
501 "ip_diffim_forced_PsfFlux_area", "F",
502 "Forced PSF flux effective area of PSF.",
503 units="pixel")
504 self.schema.addField(
505 "ip_diffim_forced_PsfFlux_flag", "Flag",
506 "Forced PSF flux general failure flag.")
507 self.schema.addField(
508 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
509 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
510 self.schema.addField(
511 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
512 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
513 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
514 if self.config.doMatchSources:
515 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
516 self.schema.addField("srcMatchId", "L", "unique id of source match")
517 if self.config.doSkySources:
518 self.makeSubtask("skySources")
519 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
520
521 # initialize InitOutputs
522 self.outputSchema = afwTable.SourceCatalog(self.schema)
523 self.outputSchema.getTable().setMetadata(self.algMetadata)
524
525 @staticmethod
526 def makeIdFactory(expId, expBits):
527 """Create IdFactory instance for unique 64 bit diaSource id-s.
528
529 Parameters
530 ----------
531 expId : `int`
532 Exposure id.
533
534 expBits: `int`
535 Number of used bits in ``expId``.
536
537 Note
538 ----
539 The diasource id-s consists of the ``expId`` stored fixed in the highest value
540 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
541 low value end of the integer.
542
543 Returns
544 -------
545 idFactory: `lsst.afw.table.IdFactory`
546 """
547 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
548
549 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
550 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
551 inputRefs: pipeBase.InputQuantizedConnection,
552 outputRefs: pipeBase.OutputQuantizedConnection):
553 inputs = butlerQC.get(inputRefs)
554 self.log.info("Processing %s", butlerQC.quantum.dataId)
555
556 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None)
557 exposure = self.prepareCalibratedExposure(
558 inputs["exposure"],
559 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog
560 )
561
562 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
563 returnMaxBits=True)
564 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
565 if self.config.coaddName == 'dcr':
566 templateExposures = inputRefs.dcrCoadds
567 else:
568 templateExposures = inputRefs.coaddExposures
569 templateStruct = self.getTemplate.runQuantum(
570 exposure, butlerQC, inputRefs.skyMap, templateExposures
571 )
572
573 self.checkTemplateIsSufficient(templateStruct.exposure)
574
575 outputs = self.run(exposure=exposure,
576 templateExposure=templateStruct.exposure,
577 idFactory=idFactory)
578 # Consistency with runDataref gen2 handling
579 if outputs.diaSources is None:
580 del outputs.diaSources
581 butlerQC.put(outputs, outputRefs)
582
583 @timeMethod
584 def runDataRef(self, sensorRef, templateIdList=None):
585 """Subtract an image from a template coadd and measure the result.
586
587 Data I/O wrapper around `run` using the butler in Gen2.
588
589 Parameters
590 ----------
591 sensorRef : `lsst.daf.persistence.ButlerDataRef`
592 Sensor-level butler data reference, used for the following data products:
593
594 Input only:
595 - calexp
596 - psf
597 - ccdExposureId
598 - ccdExposureId_bits
599 - self.config.coaddName + "Coadd_skyMap"
600 - self.config.coaddName + "Coadd"
601 Input or output, depending on config:
602 - self.config.coaddName + "Diff_subtractedExp"
603 Output, depending on config:
604 - self.config.coaddName + "Diff_matchedExp"
605 - self.config.coaddName + "Diff_src"
606
607 Returns
608 -------
609 results : `lsst.pipe.base.Struct`
610 Returns the Struct by `run`.
611 """
612 subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
613 subtractedExposure = None
614 selectSources = None
615 calexpBackgroundExposure = None
616 self.log.info("Processing %s", sensorRef.dataId)
617
618 # We make one IdFactory that will be used by both icSrc and src datasets;
619 # I don't know if this is the way we ultimately want to do things, but at least
620 # this ensures the source IDs are fully unique.
621 idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
622 expBits=sensorRef.get("ccdExposureId_bits"))
623 if self.config.doAddCalexpBackground:
624 calexpBackgroundExposure = sensorRef.get("calexpBackground")
625
626 # Retrieve the science image we wish to analyze
627 exposure = sensorRef.get("calexp", immediate=True)
628
629 # Retrieve the template image
630 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
631
632 if sensorRef.datasetExists("src"):
633 self.log.info("Source selection via src product")
634 # Sources already exist; for data release processing
635 selectSources = sensorRef.get("src")
636
637 if not self.config.doSubtract and self.config.doDetection:
638 # If we don't do subtraction, we need the subtracted exposure from the repo
639 subtractedExposure = sensorRef.get(subtractedExposureName)
640 # Both doSubtract and doDetection cannot be False
641
642 results = self.run(exposure=exposure,
643 selectSources=selectSources,
644 templateExposure=template.exposure,
645 templateSources=template.sources,
646 idFactory=idFactory,
647 calexpBackgroundExposure=calexpBackgroundExposure,
648 subtractedExposure=subtractedExposure)
649
650 if self.config.doWriteSources and results.diaSources is not None:
651 sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
652 if self.config.doWriteWarpedExp:
653 sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
654 if self.config.doWriteMatchedExp:
655 sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
656 if self.config.doAddMetrics and self.config.doSelectSources:
657 sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
658 if self.config.doWriteSubtractedExp:
659 sensorRef.put(results.subtractedExposure, subtractedExposureName)
660 if self.config.doWriteScoreExp:
661 sensorRef.put(results.scoreExposure, self.config.coaddName + "Diff_scoreExp")
662 return results
663
664 def prepareCalibratedExposure(self, exposure, finalizedPsfApCorrCatalog=None):
665 """Prepare a calibrated exposure and apply finalized psf if so configured.
666
667 Parameters
668 ----------
669 exposure : `lsst.afw.image.exposure.Exposure`
670 Input exposure to adjust calibrations.
671 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
672 Exposure catalog with finalized psf models and aperture correction
673 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
674 the detector id for the catalog id, sorted on id for fast lookup.
675
676 Returns
677 -------
678 exposure : `lsst.afw.image.exposure.Exposure`
679 Exposure with adjusted calibrations.
680 """
681 detectorId = exposure.getInfo().getDetector().getId()
682
683 if finalizedPsfApCorrCatalog is not None:
684 row = finalizedPsfApCorrCatalog.find(detectorId)
685 if row is None:
686 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; "
687 "Using original psf.", detectorId)
688 else:
689 psf = row.getPsf()
690 apCorrMap = row.getApCorrMap()
691 if psf is None or apCorrMap is None:
692 self.log.warning("Detector id %s has None for psf/apCorrMap in "
693 "finalizedPsfApCorrCatalog; Using original psf.", detectorId)
694 else:
695 exposure.setPsf(psf)
696 exposure.info.setApCorrMap(apCorrMap)
697
698 return exposure
699
700 @timeMethod
701 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
702 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
703 """PSF matches, subtract two images and perform detection on the difference image.
704
705 Parameters
706 ----------
707 exposure : `lsst.afw.image.ExposureF`, optional
708 The science exposure, the minuend in the image subtraction.
709 Can be None only if ``config.doSubtract==False``.
710 selectSources : `lsst.afw.table.SourceCatalog`, optional
711 Identified sources on the science exposure. This catalog is used to
712 select sources in order to perform the AL PSF matching on stamp images
713 around them. The selection steps depend on config options and whether
714 ``templateSources`` and ``matchingSources`` specified.
715 templateExposure : `lsst.afw.image.ExposureF`, optional
716 The template to be subtracted from ``exposure`` in the image subtraction.
717 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
718 The template exposure should cover the same sky area as the science exposure.
719 It is either a stich of patches of a coadd skymap image or a calexp
720 of the same pointing as the science exposure. Can be None only
721 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
722 templateSources : `lsst.afw.table.SourceCatalog`, optional
723 Identified sources on the template exposure.
724 idFactory : `lsst.afw.table.IdFactory`
725 Generator object to assign ids to detected sources in the difference image.
726 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
727 Background exposure to be added back to the science exposure
728 if ``config.doAddCalexpBackground==True``
729 subtractedExposure : `lsst.afw.image.ExposureF`, optional
730 If ``config.doSubtract==False`` and ``config.doDetection==True``,
731 performs the post subtraction source detection only on this exposure.
732 Otherwise should be None.
733
734 Returns
735 -------
736 results : `lsst.pipe.base.Struct`
737 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
738 Difference image.
739 ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
740 The zogy score exposure, if calculated.
741 ``matchedExposure`` : `lsst.afw.image.ExposureF`
742 The matched PSF exposure.
743 ``subtractRes`` : `lsst.pipe.base.Struct`
744 The returned result structure of the ImagePsfMatchTask subtask.
745 ``diaSources`` : `lsst.afw.table.SourceCatalog`
746 The catalog of detected sources.
747 ``selectSources`` : `lsst.afw.table.SourceCatalog`
748 The input source catalog with optionally added Qa information.
749
750 Notes
751 -----
752 The following major steps are included:
753
754 - warp template coadd to match WCS of image
755 - PSF match image to warped template
756 - subtract image from PSF-matched, warped template
757 - detect sources
758 - measure sources
759
760 For details about the image subtraction configuration modes
761 see `lsst.ip.diffim`.
762 """
763 subtractRes = None
764 controlSources = None
765 subtractedExposure = None
766 scoreExposure = None
767 diaSources = None
768 kernelSources = None
769 # We'll clone exposure if modified but will still need the original
770 exposureOrig = exposure
771
772 if self.config.doAddCalexpBackground:
773 mi = exposure.getMaskedImage()
774 mi += calexpBackgroundExposure.getImage()
775
776 if not exposure.hasPsf():
777 raise pipeBase.TaskError("Exposure has no psf")
778 sciencePsf = exposure.getPsf()
779
780 if self.config.doSubtract:
781 if self.config.doScaleTemplateVariance:
782 self.log.info("Rescaling template variance")
783 templateVarFactor = self.scaleVariance.run(
784 templateExposure.getMaskedImage())
785 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
786 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
787 self.metadata.add("psfMatchingAlgorithm", self.config.subtract.name)
788
789 if self.config.subtract.name == 'zogy':
790 subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
791 scoreExposure = subtractRes.scoreExp
792 subtractedExposure = subtractRes.diffExp
793 subtractRes.subtractedExposure = subtractedExposure
794 subtractRes.matchedExposure = None
795
796 elif self.config.subtract.name == 'al':
797 # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
798 # Just need a rough estimate; average positions are fine
799 sciAvgPos = sciencePsf.getAveragePosition()
800 scienceSigmaOrig = sciencePsf.computeShape(sciAvgPos).getDeterminantRadius()
801
802 templatePsf = templateExposure.getPsf()
803 templateAvgPos = templatePsf.getAveragePosition()
804 templateSigma = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()
805
806 # if requested, convolve the science exposure with its PSF
807 # (properly, this should be a cross-correlation, but our code does not yet support that)
808 # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
809 # else sigma of original science exposure
810 # TODO: DM-22762 This functional block should be moved into its own method
811 preConvPsf = None
812 if self.config.useScoreImageDetection:
813 self.log.warning("AL likelihood image: pre-convolution of PSF is not implemented.")
814 convControl = afwMath.ConvolutionControl()
815 # cannot convolve in place, so need a new image anyway
816 srcMI = exposure.maskedImage
817 exposure = exposure.clone() # New deep copy
818 srcPsf = sciencePsf
819 if self.config.useGaussianForPreConvolution:
820 self.log.info(
821 "AL likelihood image: Using Gaussian (sigma=%.2f) PSF estimation "
822 "for science image pre-convolution", scienceSigmaOrig)
823 # convolve with a simplified PSF model: a double Gaussian
824 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
825 preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
826 else:
827 # convolve with science exposure's PSF model
828 self.log.info(
829 "AL likelihood image: Using the science image PSF for pre-convolution.")
830 preConvPsf = srcPsf
831 afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
832 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
833 else:
834 scienceSigmaPost = scienceSigmaOrig
835
836 # If requested, find and select sources from the image
837 # else, AL subtraction will do its own source detection
838 # TODO: DM-22762 This functional block should be moved into its own method
839 if self.config.doSelectSources:
840 if selectSources is None:
841 self.log.warning("Src product does not exist; running detection, measurement,"
842 " selection")
843 # Run own detection and measurement; necessary in nightly processing
844 selectSources = self.subtract.getSelectSources(
845 exposure,
846 sigma=scienceSigmaPost,
847 doSmooth=not self.config.useScoreImageDetection,
848 idFactory=idFactory,
849 )
850
851 if self.config.doAddMetrics:
852 # Number of basis functions
853
854 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
855 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
856 targetFwhmPix=templateSigma*FwhmPerSigma))
857 # Modify the schema of all Sources
858 # DEPRECATED: This is a data dependent (nparam) output product schema
859 # outside the task constructor.
860 # NOTE: The pre-determination of nparam at this point
861 # may be incorrect as the template psf is warped later in
862 # ImagePsfMatchTask.matchExposures()
863 kcQa = KernelCandidateQa(nparam)
864 selectSources = kcQa.addToSchema(selectSources)
865 if self.config.kernelSourcesFromRef:
866 # match exposure sources to reference catalog
867 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
868 matches = astromRet.matches
869 elif templateSources:
870 # match exposure sources to template sources
871 mc = afwTable.MatchControl()
872 mc.findOnlyClosest = False
873 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
874 mc)
875 else:
876 raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
877 "but template sources not available. Cannot match science "
878 "sources with template sources. Run process* on data from "
879 "which templates are built.")
880
881 kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
882 matches=matches).sourceCat
883 random.shuffle(kernelSources, random.random)
884 controlSources = kernelSources[::self.config.controlStepSize]
885 kernelSources = [k for i, k in enumerate(kernelSources)
886 if i % self.config.controlStepSize]
887
888 if self.config.doSelectDcrCatalog:
889 redSelector = DiaCatalogSourceSelectorTask(
890 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
891 grMax=99.999))
892 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
893 controlSources.extend(redSources)
894
895 blueSelector = DiaCatalogSourceSelectorTask(
896 DiaCatalogSourceSelectorConfig(grMin=-99.999,
897 grMax=self.sourceSelector.config.grMin))
898 blueSources = blueSelector.selectStars(exposure, selectSources,
899 matches=matches).starCat
900 controlSources.extend(blueSources)
901
902 if self.config.doSelectVariableCatalog:
903 varSelector = DiaCatalogSourceSelectorTask(
904 DiaCatalogSourceSelectorConfig(includeVariable=True))
905 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
906 controlSources.extend(varSources)
907
908 self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)",
909 len(kernelSources), len(selectSources), len(controlSources))
910
911 allresids = {}
912 # TODO: DM-22762 This functional block should be moved into its own method
913 if self.config.doUseRegister:
914 self.log.info("Registering images")
915
916 if templateSources is None:
917 # Run detection on the template, which is
918 # temporarily background-subtracted
919 # sigma of PSF of template image before warping
920 templateSources = self.subtract.getSelectSources(
921 templateExposure,
922 sigma=templateSigma,
923 doSmooth=True,
924 idFactory=idFactory
925 )
926
927 # Third step: we need to fit the relative astrometry.
928 #
929 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
930 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
931 exposure.getWcs(), exposure.getBBox())
932 templateExposure = warpedExp
933
934 # Create debugging outputs on the astrometric
935 # residuals as a function of position. Persistence
936 # not yet implemented; expected on (I believe) #2636.
937 if self.config.doDebugRegister:
938 # Grab matches to reference catalog
939 srcToMatch = {x.second.getId(): x.first for x in matches}
940
941 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
942 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
943 sids = [m.first.getId() for m in wcsResults.matches]
944 positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
945 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
946 m.second.get(inCentroidKey))) for m in wcsResults.matches]
947 allresids = dict(zip(sids, zip(positions, residuals)))
948
949 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
950 wcsResults.wcs.pixelToSky(
951 m.second.get(inCentroidKey))) for m in wcsResults.matches]
952 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
953 + 2.5*numpy.log10(srcToMatch[x].get("r"))
954 for x in sids if x in srcToMatch.keys()])
955 dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
956 if s in srcToMatch.keys()])
957 dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
958 if s in srcToMatch.keys()])
959 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
960 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
961 & (colors <= self.sourceSelector.config.grMax))
962 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
963 rms1Long = IqrToSigma*(
964 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
965 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
966 - numpy.percentile(dlat[idx1], 25))
967 rms2Long = IqrToSigma*(
968 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
969 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
970 - numpy.percentile(dlat[idx2], 25))
971 rms3Long = IqrToSigma*(
972 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
973 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
974 - numpy.percentile(dlat[idx3], 25))
975 self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f",
976 numpy.median(dlong[idx1]), rms1Long,
977 numpy.median(dlat[idx1]), rms1Lat)
978 self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f",
979 numpy.median(dlong[idx2]), rms2Long,
980 numpy.median(dlat[idx2]), rms2Lat)
981 self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f",
982 numpy.median(dlong[idx3]), rms3Long,
983 numpy.median(dlat[idx3]), rms3Lat)
984
985 self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
986 self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
987 self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
988 self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
989 self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
990 self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
991
992 self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
993 self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
994 self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
995 self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
996 self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
997 self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
998
999 # warp template exposure to match exposure,
1000 # PSF match template exposure to exposure,
1001 # then return the difference
1002
1003 # Return warped template... Construct sourceKernelCand list after subtract
1004 self.log.info("Subtracting images")
1005 subtractRes = self.subtract.subtractExposures(
1006 templateExposure=templateExposure,
1007 scienceExposure=exposure,
1008 candidateList=kernelSources,
1009 convolveTemplate=self.config.convolveTemplate,
1010 doWarping=not self.config.doUseRegister
1011 )
1012 if self.config.useScoreImageDetection:
1013 scoreExposure = subtractRes.subtractedExposure
1014 else:
1015 subtractedExposure = subtractRes.subtractedExposure
1016
1017 if self.config.doDetection:
1018 self.log.info("Computing diffim PSF")
1019
1020 # Get Psf from the appropriate input image if it doesn't exist
1021 if subtractedExposure is not None and not subtractedExposure.hasPsf():
1022 if self.config.convolveTemplate:
1023 subtractedExposure.setPsf(exposure.getPsf())
1024 else:
1025 subtractedExposure.setPsf(templateExposure.getPsf())
1026
1027 # If doSubtract is False, then subtractedExposure was fetched from disk (above),
1028 # thus it may have already been decorrelated. Thus, we do not decorrelate if
1029 # doSubtract is False.
1030
1031 # NOTE: At this point doSubtract == True
1032 if self.config.doDecorrelation and self.config.doSubtract:
1033 preConvKernel = None
1034 if self.config.useGaussianForPreConvolution:
1035 preConvKernel = preConvPsf.getLocalKernel()
1036 if self.config.useScoreImageDetection:
1037 scoreExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
1038 scoreExposure,
1039 subtractRes.psfMatchingKernel,
1040 spatiallyVarying=self.config.doSpatiallyVarying,
1041 preConvKernel=preConvKernel,
1042 templateMatched=True,
1043 preConvMode=True).correctedExposure
1044 # Note that the subtracted exposure is always decorrelated,
1045 # even if the score image is used for detection
1046 subtractedExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
1047 subtractedExposure,
1048 subtractRes.psfMatchingKernel,
1049 spatiallyVarying=self.config.doSpatiallyVarying,
1050 preConvKernel=None,
1051 templateMatched=self.config.convolveTemplate,
1052 preConvMode=False).correctedExposure
1053 # END (if subtractAlgorithm == 'AL')
1054 # END (if self.config.doSubtract)
1055 if self.config.doDetection:
1056 self.log.info("Running diaSource detection")
1057
1058 # subtractedExposure - reserved for task return value
1059 # in zogy, it is always the proper difference image
1060 # in AL, it may be (yet) pre-convolved and/or decorrelated
1061 #
1062 # detectionExposure - controls which exposure to use for detection
1063 # in-place modifications will appear in task return
1064 if self.config.useScoreImageDetection:
1065 # zogy with score image detection enabled
1066 self.log.info("Detection, diffim rescaling and measurements are "
1067 "on AL likelihood or Zogy score image.")
1068 detectionExposure = scoreExposure
1069 else:
1070 # AL or zogy with no score image detection
1071 detectionExposure = subtractedExposure
1072
1073 # Rescale difference image variance plane
1074 if self.config.doScaleDiffimVariance:
1075 self.log.info("Rescaling diffim variance")
1076 diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
1077 self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor)
1078 self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
1079
1080 # Erase existing detection mask planes
1081 mask = detectionExposure.getMaskedImage().getMask()
1082 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
1083
1084 table = afwTable.SourceTable.make(self.schema, idFactory)
1085 table.setMetadata(self.algMetadata)
1086 results = self.detection.run(
1087 table=table,
1088 exposure=detectionExposure,
1089 doSmooth=not self.config.useScoreImageDetection
1090 )
1091
1092 if self.config.doMerge:
1093 fpSet = results.fpSets.positive
1094 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1095 self.config.growFootprint, False)
1096 diaSources = afwTable.SourceCatalog(table)
1097 fpSet.makeSources(diaSources)
1098 self.log.info("Merging detections into %d sources", len(diaSources))
1099 else:
1100 diaSources = results.sources
1101 # Inject skySources before measurement.
1102 if self.config.doSkySources:
1103 skySourceFootprints = self.skySources.run(
1104 mask=detectionExposure.mask,
1105 seed=detectionExposure.info.id)
1106 if skySourceFootprints:
1107 for foot in skySourceFootprints:
1108 s = diaSources.addNew()
1109 s.setFootprint(foot)
1110 s.set(self.skySourceKey, True)
1111
1112 if self.config.doMeasurement:
1113 newDipoleFitting = self.config.doDipoleFitting
1114 self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1115 if not newDipoleFitting:
1116 # Just fit dipole in diffim
1117 self.measurement.run(diaSources, detectionExposure)
1118 else:
1119 # Use (matched) template and science image (if avail.) to constrain dipole fitting
1120 if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1121 self.measurement.run(diaSources, detectionExposure, exposure,
1122 subtractRes.matchedExposure)
1123 else:
1124 self.measurement.run(diaSources, detectionExposure, exposure)
1125 if self.config.doApCorr:
1126 self.applyApCorr.run(
1127 catalog=diaSources,
1128 apCorrMap=detectionExposure.getInfo().getApCorrMap()
1129 )
1130
1131 if self.config.doForcedMeasurement:
1132 # Run forced psf photometry on the PVI at the diaSource locations.
1133 # Copy the measured flux and error into the diaSource.
1134 forcedSources = self.forcedMeasurement.generateMeasCat(
1135 exposure, diaSources, detectionExposure.getWcs())
1136 self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1137 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1138 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1139 "ip_diffim_forced_PsfFlux_instFlux", True)
1140 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1141 "ip_diffim_forced_PsfFlux_instFluxErr", True)
1142 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1143 "ip_diffim_forced_PsfFlux_area", True)
1144 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1145 "ip_diffim_forced_PsfFlux_flag", True)
1146 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1147 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1148 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1149 "ip_diffim_forced_PsfFlux_flag_edge", True)
1150 for diaSource, forcedSource in zip(diaSources, forcedSources):
1151 diaSource.assign(forcedSource, mapper)
1152
1153 # Match with the calexp sources if possible
1154 if self.config.doMatchSources:
1155 if selectSources is not None:
1156 # Create key,val pair where key=diaSourceId and val=sourceId
1157 matchRadAsec = self.config.diaSourceMatchRadius
1158 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1159
1160 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1161 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1162 srcMatch in srcMatches])
1163 self.log.info("Matched %d / %d diaSources to sources",
1164 len(srcMatchDict), len(diaSources))
1165 else:
1166 self.log.warning("Src product does not exist; cannot match with diaSources")
1167 srcMatchDict = {}
1168
1169 # Create key,val pair where key=diaSourceId and val=refId
1170 refAstromConfig = AstrometryConfig()
1171 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1172 refAstrometer = AstrometryTask(self.refObjLoader, config=refAstromConfig)
1173 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1174 refMatches = astromRet.matches
1175 if refMatches is None:
1176 self.log.warning("No diaSource matches with reference catalog")
1177 refMatchDict = {}
1178 else:
1179 self.log.info("Matched %d / %d diaSources to reference catalog",
1180 len(refMatches), len(diaSources))
1181 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1182 refMatch in refMatches])
1183
1184 # Assign source Ids
1185 for diaSource in diaSources:
1186 sid = diaSource.getId()
1187 if sid in srcMatchDict:
1188 diaSource.set("srcMatchId", srcMatchDict[sid])
1189 if sid in refMatchDict:
1190 diaSource.set("refMatchId", refMatchDict[sid])
1191
1192 if self.config.doAddMetrics and self.config.doSelectSources:
1193 self.log.info("Evaluating metrics and control sample")
1194
1195 kernelCandList = []
1196 for cell in subtractRes.kernelCellSet.getCellList():
1197 for cand in cell.begin(False): # include bad candidates
1198 kernelCandList.append(cand)
1199
1200 # Get basis list to build control sample kernels
1201 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1202 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1203
1204 controlCandList = (
1205 diffimTools.sourceTableToCandidateList(controlSources,
1206 subtractRes.warpedExposure, exposure,
1207 self.config.subtract.kernel.active,
1208 self.config.subtract.kernel.active.detectionConfig,
1209 self.log, doBuild=True, basisList=basisList))
1210
1211 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1212 subtractRes.backgroundModel, dof=nparam)
1213 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1214 subtractRes.backgroundModel)
1215
1216 if self.config.doDetection:
1217 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1218 else:
1219 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1220
1221 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1222 return pipeBase.Struct(
1223 subtractedExposure=subtractedExposure,
1224 scoreExposure=scoreExposure,
1225 warpedExposure=subtractRes.warpedExposure,
1226 matchedExposure=subtractRes.matchedExposure,
1227 subtractRes=subtractRes,
1228 diaSources=diaSources,
1229 selectSources=selectSources
1230 )
1231
1232 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1233 """Fit the relative astrometry between templateSources and selectSources
1234
1235 Todo
1236 ----
1237
1238 Remove this method. It originally fit a new WCS to the template before calling register.run
1239 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1240 It remains because a subtask overrides it.
1241 """
1242 results = self.register.run(templateSources, templateExposure.getWcs(),
1243 templateExposure.getBBox(), selectSources)
1244 return results
1245
1246 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1247 """Make debug plots and displays.
1248
1249 Todo
1250 ----
1251 Test and update for current debug display and slot names
1252 """
1253 import lsstDebug
1254 display = lsstDebug.Info(__name__).display
1255 showSubtracted = lsstDebug.Info(__name__).showSubtracted
1256 showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1257 showDiaSources = lsstDebug.Info(__name__).showDiaSources
1258 showDipoles = lsstDebug.Info(__name__).showDipoles
1259 maskTransparency = lsstDebug.Info(__name__).maskTransparency
1260 if display:
1261 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1262 if not maskTransparency:
1263 maskTransparency = 0
1264 disp.setMaskTransparency(maskTransparency)
1265
1266 if display and showSubtracted:
1267 disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1268 mi = subtractRes.subtractedExposure.getMaskedImage()
1269 x0, y0 = mi.getX0(), mi.getY0()
1270 with disp.Buffering():
1271 for s in diaSources:
1272 x, y = s.getX() - x0, s.getY() - y0
1273 ctype = "red" if s.get("flags_negative") else "yellow"
1274 if (s.get("base_PixelFlags_flag_interpolatedCenter")
1275 or s.get("base_PixelFlags_flag_saturatedCenter")
1276 or s.get("base_PixelFlags_flag_crCenter")):
1277 ptype = "x"
1278 elif (s.get("base_PixelFlags_flag_interpolated")
1279 or s.get("base_PixelFlags_flag_saturated")
1280 or s.get("base_PixelFlags_flag_cr")):
1281 ptype = "+"
1282 else:
1283 ptype = "o"
1284 disp.dot(ptype, x, y, size=4, ctype=ctype)
1285 lsstDebug.frame += 1
1286
1287 if display and showPixelResiduals and selectSources:
1288 nonKernelSources = []
1289 for source in selectSources:
1290 if source not in kernelSources:
1291 nonKernelSources.append(source)
1292
1293 diUtils.plotPixelResiduals(exposure,
1294 subtractRes.warpedExposure,
1295 subtractRes.subtractedExposure,
1296 subtractRes.kernelCellSet,
1297 subtractRes.psfMatchingKernel,
1298 subtractRes.backgroundModel,
1299 nonKernelSources,
1300 self.subtract.config.kernel.active.detectionConfig,
1301 origVariance=False)
1302 diUtils.plotPixelResiduals(exposure,
1303 subtractRes.warpedExposure,
1304 subtractRes.subtractedExposure,
1305 subtractRes.kernelCellSet,
1306 subtractRes.psfMatchingKernel,
1307 subtractRes.backgroundModel,
1308 nonKernelSources,
1309 self.subtract.config.kernel.active.detectionConfig,
1310 origVariance=True)
1311 if display and showDiaSources:
1312 flagChecker = SourceFlagChecker(diaSources)
1313 isFlagged = [flagChecker(x) for x in diaSources]
1314 isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1315 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1316 frame=lsstDebug.frame)
1317 lsstDebug.frame += 1
1318
1319 if display and showDipoles:
1320 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1321 frame=lsstDebug.frame)
1322 lsstDebug.frame += 1
1323
1324 def checkTemplateIsSufficient(self, templateExposure):
1325 """Raise NoWorkFound if template coverage < requiredTemplateFraction
1326
1327 Parameters
1328 ----------
1329 templateExposure : `lsst.afw.image.ExposureF`
1330 The template exposure to check
1331
1332 Raises
1333 ------
1334 NoWorkFound
1335 Raised if fraction of good pixels, defined as not having NO_DATA
1336 set, is less then the configured requiredTemplateFraction
1337 """
1338 # Count the number of pixels with the NO_DATA mask bit set
1339 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
1340 pixNoData = numpy.count_nonzero(templateExposure.mask.array
1341 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
1342 pixGood = templateExposure.getBBox().getArea() - pixNoData
1343 self.log.info("template has %d good pixels (%.1f%%)", pixGood,
1344 100*pixGood/templateExposure.getBBox().getArea())
1345
1346 if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction:
1347 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
1348 "To force subtraction, set config requiredTemplateFraction=0." % (
1349 100*pixGood/templateExposure.getBBox().getArea(),
1350 100*self.config.requiredTemplateFraction))
1351 raise pipeBase.NoWorkFound(message)
1352
1353 def _getConfigName(self):
1354 """Return the name of the config dataset
1355 """
1356 return "%sDiff_config" % (self.config.coaddName,)
1357
1358 def _getMetadataName(self):
1359 """Return the name of the metadata dataset
1360 """
1361 return "%sDiff_metadata" % (self.config.coaddName,)
1362
1363 def getSchemaCatalogs(self):
1364 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1365 return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1366
1367 @classmethod
1368 def _makeArgumentParser(cls):
1369 """Create an argument parser
1370 """
1371 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1372 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1373 parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1374 help="Template data ID in case of calexp template,"
1375 " e.g. --templateId visit=6789")
1376 return parser
1377
1378
1379class ImageDifferenceFromTemplateConnections(ImageDifferenceTaskConnections,
1380 defaultTemplates={"coaddName": "goodSeeing"}
1381 ):
1382 inputTemplate = pipeBase.connectionTypes.Input(
1383 doc=("Warped template produced by GetMultiTractCoaddTemplate"),
1384 dimensions=("instrument", "visit", "detector"),
1385 storageClass="ExposureF",
1386 name="{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}",
1387 )
1388
1389 def __init__(self, *, config=None):
1390 super().__init__(config=config)
1391 # ImageDifferenceConnections will have removed one of these.
1392 # Make sure they're both gone, because no coadds are needed.
1393 if "coaddExposures" in self.inputs:
1394 self.inputs.remove("coaddExposures")
1395 if "dcrCoadds" in self.inputs:
1396 self.inputs.remove("dcrCoadds")
1397
1398
1399class ImageDifferenceFromTemplateConfig(ImageDifferenceConfig,
1400 pipelineConnections=ImageDifferenceFromTemplateConnections):
1401 pass
1402
1403
1404class ImageDifferenceFromTemplateTask(ImageDifferenceTask):
1405 ConfigClass = ImageDifferenceFromTemplateConfig
1406 _DefaultName = "imageDifference"
1407
1408 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
1409 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1410 inputs = butlerQC.get(inputRefs)
1411 self.log.info("Processing %s", butlerQC.quantum.dataId)
1412 self.checkTemplateIsSufficient(inputs['inputTemplate'])
1413 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
1414 returnMaxBits=True)
1415 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
1416
1417 finalizedPsfApCorrCatalog = inputs.get("finalizedPsfApCorrCatalog", None)
1418 exposure = self.prepareCalibratedExposure(
1419 inputs["exposure"],
1420 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog
1421 )
1422
1423 outputs = self.run(exposure=exposure,
1424 templateExposure=inputs['inputTemplate'],
1425 idFactory=idFactory)
1426
1427 # Consistency with runDataref gen2 handling
1428 if outputs.diaSources is None:
1429 del outputs.diaSources
1430 butlerQC.put(outputs, outputRefs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)