lsst.pipe.tasks 23.0.0+f2fbba1123
imageDifference.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import math
23import random
24import numpy
25
26import lsst.utils
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29import lsst.daf.base as dafBase
30import lsst.geom as geom
31import lsst.afw.math as afwMath
32import lsst.afw.table as afwTable
33from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
35from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
36from lsst.pipe.tasks.registerImage import RegisterTask
37from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43import lsst.ip.diffim.diffimTools as diffimTools
44import lsst.ip.diffim.utils as diUtils
45import lsst.afw.display as afwDisplay
46from lsst.skymap import BaseSkyMap
47from lsst.obs.base import ExposureIdInfo
48
49__all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
50FwhmPerSigma = 2*math.sqrt(2*math.log(2))
51IqrToSigma = 0.741
52
53
54class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
55 dimensions=("instrument", "visit", "detector", "skymap"),
56 defaultTemplates={"coaddName": "deep",
57 "skyMapName": "deep",
58 "warpTypeSuffix": "",
59 "fakesType": ""}):
60
61 exposure = pipeBase.connectionTypes.Input(
62 doc="Input science exposure to subtract from.",
63 dimensions=("instrument", "visit", "detector"),
64 storageClass="ExposureF",
65 name="{fakesType}calexp"
66 )
67
68 # TODO DM-22953
69 # kernelSources = pipeBase.connectionTypes.Input(
70 # doc="Source catalog produced in calibrate task for kernel candidate sources",
71 # name="src",
72 # storageClass="SourceCatalog",
73 # dimensions=("instrument", "visit", "detector"),
74 # )
75
76 skyMap = pipeBase.connectionTypes.Input(
77 doc="Input definition of geometry/bbox and projection/wcs for template exposures",
78 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
79 dimensions=("skymap", ),
80 storageClass="SkyMap",
81 )
82 coaddExposures = pipeBase.connectionTypes.Input(
83 doc="Input template to match and subtract from the exposure",
84 dimensions=("tract", "patch", "skymap", "band"),
85 storageClass="ExposureF",
86 name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
87 multiple=True,
88 deferLoad=True
89 )
90 dcrCoadds = pipeBase.connectionTypes.Input(
91 doc="Input DCR template to match and subtract from the exposure",
92 name="{fakesType}dcrCoadd{warpTypeSuffix}",
93 storageClass="ExposureF",
94 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
95 multiple=True,
96 deferLoad=True
97 )
98 outputSchema = pipeBase.connectionTypes.InitOutput(
99 doc="Schema (as an example catalog) for output DIASource catalog.",
100 storageClass="SourceCatalog",
101 name="{fakesType}{coaddName}Diff_diaSrc_schema",
102 )
103 subtractedExposure = pipeBase.connectionTypes.Output(
104 doc="Output AL difference or Zogy proper difference image",
105 dimensions=("instrument", "visit", "detector"),
106 storageClass="ExposureF",
107 name="{fakesType}{coaddName}Diff_differenceExp",
108 )
109 scoreExposure = pipeBase.connectionTypes.Output(
110 doc="Output AL likelihood or Zogy score image",
111 dimensions=("instrument", "visit", "detector"),
112 storageClass="ExposureF",
113 name="{fakesType}{coaddName}Diff_scoreExp",
114 )
115 warpedExposure = pipeBase.connectionTypes.Output(
116 doc="Warped template used to create `subtractedExposure`.",
117 dimensions=("instrument", "visit", "detector"),
118 storageClass="ExposureF",
119 name="{fakesType}{coaddName}Diff_warpedExp",
120 )
121 matchedExposure = pipeBase.connectionTypes.Output(
122 doc="Warped template used to create `subtractedExposure`.",
123 dimensions=("instrument", "visit", "detector"),
124 storageClass="ExposureF",
125 name="{fakesType}{coaddName}Diff_matchedExp",
126 )
127 diaSources = pipeBase.connectionTypes.Output(
128 doc="Output detected diaSources on the difference image",
129 dimensions=("instrument", "visit", "detector"),
130 storageClass="SourceCatalog",
131 name="{fakesType}{coaddName}Diff_diaSrc",
132 )
133
134 def __init__(self, *, config=None):
135 super().__init__(config=config)
136 if config.coaddName == 'dcr':
137 self.inputs.remove("coaddExposures")
138 else:
139 self.inputs.remove("dcrCoadds")
140 if not config.doWriteSubtractedExp:
141 self.outputs.remove("subtractedExposure")
142 if not config.doWriteScoreExp:
143 self.outputs.remove("scoreExposure")
144 if not config.doWriteWarpedExp:
145 self.outputs.remove("warpedExposure")
146 if not config.doWriteMatchedExp:
147 self.outputs.remove("matchedExposure")
148 if not config.doWriteSources:
149 self.outputs.remove("diaSources")
150
151 # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
152 # Make kernelSources optional
153
154
155class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
156 pipelineConnections=ImageDifferenceTaskConnections):
157 """Config for ImageDifferenceTask
158 """
159 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
160 doc="Add background to calexp before processing it. "
161 "Useful as ipDiffim does background matching.")
162 doUseRegister = pexConfig.Field(dtype=bool, default=False,
163 doc="Re-compute astrometry on the template. "
164 "Use image-to-image registration to align template with "
165 "science image (AL only).")
166 doDebugRegister = pexConfig.Field(dtype=bool, default=False,
167 doc="Writing debugging data for doUseRegister")
168 doSelectSources = pexConfig.Field(dtype=bool, default=False,
169 doc="Select stars to use for kernel fitting (AL only)")
170 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
171 doc="Select stars of extreme color as part "
172 "of the control sample (AL only)")
173 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
174 doc="Select stars that are variable to be part "
175 "of the control sample (AL only)")
176 doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
177 doPreConvolve = pexConfig.Field(dtype=bool, default=False,
178 doc="Not in use. Superseded by useScoreImageDetection.",
179 deprecated="This option superseded by useScoreImageDetection."
180 " Will be removed after v22.")
181 useScoreImageDetection = pexConfig.Field(
182 dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or "
183 "the Zogy score image. Use it for source detection (if doDetection=True).")
184 doWriteScoreExp = pexConfig.Field(
185 dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?")
186 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
187 doc="Scale variance of the template before PSF matching")
188 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
189 doc="Scale variance of the diffim before PSF matching. "
190 "You may do either this or template variance scaling, "
191 "or neither. (Doing both is a waste of CPU.)")
192 useGaussianForPreConvolution = pexConfig.Field(
193 dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution "
194 "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)")
195 doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
196 doDecorrelation = pexConfig.Field(dtype=bool, default=True,
197 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
198 "kernel convolution (AL only)? If True, also update the diffim PSF.")
199 doMerge = pexConfig.Field(dtype=bool, default=True,
200 doc="Merge positive and negative diaSources with grow radius "
201 "set by growFootprint")
202 doMatchSources = pexConfig.Field(dtype=bool, default=False,
203 doc="Match diaSources with input calexp sources and ref catalog sources")
204 doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
205 doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
206 doForcedMeasurement = pexConfig.Field(
207 dtype=bool,
208 default=True,
209 doc="Force photometer diaSource locations on PVI?")
210 doWriteSubtractedExp = pexConfig.Field(
211 dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
212 doWriteWarpedExp = pexConfig.Field(
213 dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
214 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
215 doc="Write warped and PSF-matched template coadd exposure?")
216 doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
217 doAddMetrics = pexConfig.Field(dtype=bool, default=False,
218 doc="Add columns to the source table to hold analysis metrics?")
219
220 coaddName = pexConfig.Field(
221 doc="coadd name: typically one of deep, goodSeeing, or dcr",
222 dtype=str,
223 default="deep",
224 )
225 convolveTemplate = pexConfig.Field(
226 doc="Which image gets convolved (default = template)",
227 dtype=bool,
228 default=True
229 )
230 refObjLoader = pexConfig.ConfigurableField(
231 target=LoadIndexedReferenceObjectsTask,
232 doc="reference object loader",
233 )
234 astrometer = pexConfig.ConfigurableField(
235 target=AstrometryTask,
236 doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
237 )
238 sourceSelector = pexConfig.ConfigurableField(
239 target=ObjectSizeStarSelectorTask,
240 doc="Source selection algorithm",
241 )
242 subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
243 decorrelate = pexConfig.ConfigurableField(
244 target=DecorrelateALKernelSpatialTask,
245 doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
246 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
247 "default of 5.5).",
248 )
249 # Old style ImageMapper grid. ZogyTask has its own grid option
250 doSpatiallyVarying = pexConfig.Field(
251 dtype=bool,
252 default=False,
253 doc="Perform A&L decorrelation on a grid across the "
254 "image in order to allow for spatial variations. Zogy does not use this option."
255 )
256 detection = pexConfig.ConfigurableField(
257 target=SourceDetectionTask,
258 doc="Low-threshold detection for final measurement",
259 )
260 measurement = pexConfig.ConfigurableField(
261 target=DipoleFitTask,
262 doc="Enable updated dipole fitting method",
263 )
264 doApCorr = lsst.pex.config.Field(
265 dtype=bool,
266 default=True,
267 doc="Run subtask to apply aperture corrections"
268 )
269 applyApCorr = lsst.pex.config.ConfigurableField(
270 target=ApplyApCorrTask,
271 doc="Subtask to apply aperture corrections"
272 )
273 forcedMeasurement = pexConfig.ConfigurableField(
274 target=ForcedMeasurementTask,
275 doc="Subtask to force photometer PVI at diaSource location.",
276 )
277 getTemplate = pexConfig.ConfigurableField(
278 target=GetCoaddAsTemplateTask,
279 doc="Subtask to retrieve template exposure and sources",
280 )
281 scaleVariance = pexConfig.ConfigurableField(
282 target=ScaleVarianceTask,
283 doc="Subtask to rescale the variance of the template "
284 "to the statistically expected level"
285 )
286 controlStepSize = pexConfig.Field(
287 doc="What step size (every Nth one) to select a control sample from the kernelSources",
288 dtype=int,
289 default=5
290 )
291 controlRandomSeed = pexConfig.Field(
292 doc="Random seed for shuffing the control sample",
293 dtype=int,
294 default=10
295 )
296 register = pexConfig.ConfigurableField(
297 target=RegisterTask,
298 doc="Task to enable image-to-image image registration (warping)",
299 )
300 kernelSourcesFromRef = pexConfig.Field(
301 doc="Select sources to measure kernel from reference catalog if True, template if false",
302 dtype=bool,
303 default=False
304 )
305 templateSipOrder = pexConfig.Field(
306 dtype=int, default=2,
307 doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
308 )
309 growFootprint = pexConfig.Field(
310 dtype=int, default=2,
311 doc="Grow positive and negative footprints by this amount before merging"
312 )
313 diaSourceMatchRadius = pexConfig.Field(
314 dtype=float, default=0.5,
315 doc="Match radius (in arcseconds) for DiaSource to Source association"
316 )
317 requiredTemplateFraction = pexConfig.Field(
318 dtype=float, default=0.1,
319 doc="Do not attempt to run task if template covers less than this fraction of pixels."
320 "Setting to 0 will always attempt image subtraction"
321 )
322 doSkySources = pexConfig.Field(
323 dtype=bool,
324 default=False,
325 doc="Generate sky sources?",
326 )
327 skySources = pexConfig.ConfigurableField(
328 target=SkyObjectsTask,
329 doc="Generate sky sources",
330 )
331
332 def setDefaults(self):
333 # defaults are OK for catalog and diacatalog
334
335 self.subtract['al'].kernel.name = "AL"
336 self.subtract['al'].kernel.active.fitForBackground = True
337 self.subtract['al'].kernel.active.spatialKernelOrder = 1
338 self.subtract['al'].kernel.active.spatialBgOrder = 2
339
340 # DiaSource Detection
341 self.detection.thresholdPolarity = "both"
342 self.detection.thresholdValue = 5.0
343 self.detection.reEstimateBackground = False
344 self.detection.thresholdType = "pixel_stdev"
345
346 # Add filtered flux measurement, the correct measurement for pre-convolved images.
347 # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
348 # To change that you must modify algorithms.names in the task's applyOverrides method,
349 # after the user has set doPreConvolve.
350 self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
351 self.measurement.plugins.names |= ['base_LocalPhotoCalib',
352 'base_LocalWcs']
353
354 self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
355 self.forcedMeasurement.copyColumns = {
356 "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
357 self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
358 self.forcedMeasurement.slots.shape = None
359
360 # For shuffling the control sample
361 random.seed(self.controlRandomSeed)
362
363 def validate(self):
364 pexConfig.Config.validate(self)
365 if not self.doSubtract and not self.doDetection:
366 raise ValueError("Either doSubtract or doDetection must be enabled.")
367 if self.doMeasurement and not self.doDetection:
368 raise ValueError("Cannot run source measurement without source detection.")
369 if self.doMerge and not self.doDetection:
370 raise ValueError("Cannot run source merging without source detection.")
371 if self.doSkySources and not self.doDetection:
372 raise ValueError("Cannot run sky source creation without source detection.")
373 if self.doUseRegister and not self.doSelectSources:
374 raise ValueError("doUseRegister=True and doSelectSources=False. "
375 "Cannot run RegisterTask without selecting sources.")
376 if hasattr(self.getTemplate, "coaddName"):
377 if self.getTemplate.coaddName != self.coaddName:
378 raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
379 if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
380 raise ValueError("Scaling the diffim variance and scaling the template variance "
381 "are both set. Please choose one or the other.")
382 # We cannot allow inconsistencies that would lead to None or not available output products
383 if self.subtract.name == 'zogy':
384 if self.doWriteMatchedExp:
385 raise ValueError("doWriteMatchedExp=True Matched exposure is not "
386 "calculated in zogy subtraction.")
387 if self.doAddMetrics:
388 raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
389 if self.doDecorrelation:
390 raise ValueError(
391 "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
392 if self.doSelectSources:
393 raise ValueError(
394 "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
395 if self.useGaussianForPreConvolution:
396 raise ValueError(
397 "useGaussianForPreConvolution=True This is an AL subtraction only option.")
398 else:
399 # AL only consistency checks
400 if self.useScoreImageDetection and not self.convolveTemplate:
401 raise ValueError(
402 "convolveTemplate=False and useScoreImageDetection=True "
403 "Pre-convolution and matching of the science image is not a supported operation.")
404 if self.doWriteSubtractedExp and self.useScoreImageDetection:
405 raise ValueError(
406 "doWriteSubtractedExp=True and useScoreImageDetection=True "
407 "Regular difference image is not calculated. "
408 "AL subtraction calculates either the regular difference image or the score image.")
409 if self.doWriteScoreExp and not self.useScoreImageDetection:
410 raise ValueError(
411 "doWriteScoreExp=True and useScoreImageDetection=False "
412 "Score image is not calculated. "
413 "AL subtraction calculates either the regular difference image or the score image.")
414 if self.doAddMetrics and not self.doSubtract:
415 raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
416 if self.useGaussianForPreConvolution and not self.useScoreImageDetection:
417 raise ValueError(
418 "useGaussianForPreConvolution=True and useScoreImageDetection=False "
419 "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.")
420
421
422class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
423
424 @staticmethod
425 def getTargetList(parsedCmd, **kwargs):
426 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
427 **kwargs)
428
429
430class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
431 """Subtract an image from a template and measure the result
432 """
433 ConfigClass = ImageDifferenceConfig
434 RunnerClass = ImageDifferenceTaskRunner
435 _DefaultName = "imageDifference"
436
437 def __init__(self, butler=None, **kwargs):
438 """!Construct an ImageDifference Task
439
440 @param[in] butler Butler object to use in constructing reference object loaders
441 """
442 super().__init__(**kwargs)
443 self.makeSubtask("getTemplate")
444
445 self.makeSubtask("subtract")
446
447 if self.config.subtract.name == 'al' and self.config.doDecorrelation:
448 self.makeSubtask("decorrelate")
449
450 if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
451 self.makeSubtask("scaleVariance")
452
453 if self.config.doUseRegister:
454 self.makeSubtask("register")
455 self.schema = afwTable.SourceTable.makeMinimalSchema()
456
457 if self.config.doSelectSources:
458 self.makeSubtask("sourceSelector")
459 if self.config.kernelSourcesFromRef:
460 self.makeSubtask('refObjLoader', butler=butler)
461 self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
462
463 self.algMetadata = dafBase.PropertyList()
464 if self.config.doDetection:
465 self.makeSubtask("detection", schema=self.schema)
466 if self.config.doMeasurement:
467 self.makeSubtask("measurement", schema=self.schema,
468 algMetadata=self.algMetadata)
469 if self.config.doApCorr:
470 self.makeSubtask("applyApCorr", schema=self.measurement.schema)
471 if self.config.doForcedMeasurement:
472 self.schema.addField(
473 "ip_diffim_forced_PsfFlux_instFlux", "D",
474 "Forced PSF flux measured on the direct image.",
475 units="count")
476 self.schema.addField(
477 "ip_diffim_forced_PsfFlux_instFluxErr", "D",
478 "Forced PSF flux error measured on the direct image.",
479 units="count")
480 self.schema.addField(
481 "ip_diffim_forced_PsfFlux_area", "F",
482 "Forced PSF flux effective area of PSF.",
483 units="pixel")
484 self.schema.addField(
485 "ip_diffim_forced_PsfFlux_flag", "Flag",
486 "Forced PSF flux general failure flag.")
487 self.schema.addField(
488 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
489 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
490 self.schema.addField(
491 "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
492 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
493 self.makeSubtask("forcedMeasurement", refSchema=self.schema)
494 if self.config.doMatchSources:
495 self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
496 self.schema.addField("srcMatchId", "L", "unique id of source match")
497 if self.config.doSkySources:
498 self.makeSubtask("skySources")
499 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
500
501 # initialize InitOutputs
502 self.outputSchema = afwTable.SourceCatalog(self.schema)
503 self.outputSchema.getTable().setMetadata(self.algMetadata)
504
505 @staticmethod
506 def makeIdFactory(expId, expBits):
507 """Create IdFactory instance for unique 64 bit diaSource id-s.
508
509 Parameters
510 ----------
511 expId : `int`
512 Exposure id.
513
514 expBits: `int`
515 Number of used bits in ``expId``.
516
517 Note
518 ----
519 The diasource id-s consists of the ``expId`` stored fixed in the highest value
520 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
521 low value end of the integer.
522
523 Returns
524 -------
525 idFactory: `lsst.afw.table.IdFactory`
526 """
527 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
528
529 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
530 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
531 inputRefs: pipeBase.InputQuantizedConnection,
532 outputRefs: pipeBase.OutputQuantizedConnection):
533 inputs = butlerQC.get(inputRefs)
534 self.log.info("Processing %s", butlerQC.quantum.dataId)
535 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
536 returnMaxBits=True)
537 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
538 if self.config.coaddName == 'dcr':
539 templateExposures = inputRefs.dcrCoadds
540 else:
541 templateExposures = inputRefs.coaddExposures
542 templateStruct = self.getTemplate.runQuantum(
543 inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
544 )
545
546 self.checkTemplateIsSufficient(templateStruct.exposure)
547
548 outputs = self.run(exposure=inputs['exposure'],
549 templateExposure=templateStruct.exposure,
550 idFactory=idFactory)
551 # Consistency with runDataref gen2 handling
552 if outputs.diaSources is None:
553 del outputs.diaSources
554 butlerQC.put(outputs, outputRefs)
555
556 @pipeBase.timeMethod
557 def runDataRef(self, sensorRef, templateIdList=None):
558 """Subtract an image from a template coadd and measure the result.
559
560 Data I/O wrapper around `run` using the butler in Gen2.
561
562 Parameters
563 ----------
564 sensorRef : `lsst.daf.persistence.ButlerDataRef`
565 Sensor-level butler data reference, used for the following data products:
566
567 Input only:
568 - calexp
569 - psf
570 - ccdExposureId
571 - ccdExposureId_bits
572 - self.config.coaddName + "Coadd_skyMap"
573 - self.config.coaddName + "Coadd"
574 Input or output, depending on config:
575 - self.config.coaddName + "Diff_subtractedExp"
576 Output, depending on config:
577 - self.config.coaddName + "Diff_matchedExp"
578 - self.config.coaddName + "Diff_src"
579
580 Returns
581 -------
582 results : `lsst.pipe.base.Struct`
583 Returns the Struct by `run`.
584 """
585 subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
586 subtractedExposure = None
587 selectSources = None
588 calexpBackgroundExposure = None
589 self.log.info("Processing %s", sensorRef.dataId)
590
591 # We make one IdFactory that will be used by both icSrc and src datasets;
592 # I don't know if this is the way we ultimately want to do things, but at least
593 # this ensures the source IDs are fully unique.
594 idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
595 expBits=sensorRef.get("ccdExposureId_bits"))
596 if self.config.doAddCalexpBackground:
597 calexpBackgroundExposure = sensorRef.get("calexpBackground")
598
599 # Retrieve the science image we wish to analyze
600 exposure = sensorRef.get("calexp", immediate=True)
601
602 # Retrieve the template image
603 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
604
605 if sensorRef.datasetExists("src"):
606 self.log.info("Source selection via src product")
607 # Sources already exist; for data release processing
608 selectSources = sensorRef.get("src")
609
610 if not self.config.doSubtract and self.config.doDetection:
611 # If we don't do subtraction, we need the subtracted exposure from the repo
612 subtractedExposure = sensorRef.get(subtractedExposureName)
613 # Both doSubtract and doDetection cannot be False
614
615 results = self.run(exposure=exposure,
616 selectSources=selectSources,
617 templateExposure=template.exposure,
618 templateSources=template.sources,
619 idFactory=idFactory,
620 calexpBackgroundExposure=calexpBackgroundExposure,
621 subtractedExposure=subtractedExposure)
622
623 if self.config.doWriteSources and results.diaSources is not None:
624 sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
625 if self.config.doWriteWarpedExp:
626 sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
627 if self.config.doWriteMatchedExp:
628 sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
629 if self.config.doAddMetrics and self.config.doSelectSources:
630 sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
631 if self.config.doWriteSubtractedExp:
632 sensorRef.put(results.subtractedExposure, subtractedExposureName)
633 if self.config.doWriteScoreExp:
634 sensorRef.put(results.scoreExposure, self.config.coaddName + "Diff_scoreExp")
635 return results
636
637 @pipeBase.timeMethod
638 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
639 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
640 """PSF matches, subtract two images and perform detection on the difference image.
641
642 Parameters
643 ----------
644 exposure : `lsst.afw.image.ExposureF`, optional
645 The science exposure, the minuend in the image subtraction.
646 Can be None only if ``config.doSubtract==False``.
647 selectSources : `lsst.afw.table.SourceCatalog`, optional
648 Identified sources on the science exposure. This catalog is used to
649 select sources in order to perform the AL PSF matching on stamp images
650 around them. The selection steps depend on config options and whether
651 ``templateSources`` and ``matchingSources`` specified.
652 templateExposure : `lsst.afw.image.ExposureF`, optional
653 The template to be subtracted from ``exposure`` in the image subtraction.
654 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
655 The template exposure should cover the same sky area as the science exposure.
656 It is either a stich of patches of a coadd skymap image or a calexp
657 of the same pointing as the science exposure. Can be None only
658 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
659 templateSources : `lsst.afw.table.SourceCatalog`, optional
660 Identified sources on the template exposure.
661 idFactory : `lsst.afw.table.IdFactory`
662 Generator object to assign ids to detected sources in the difference image.
663 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
664 Background exposure to be added back to the science exposure
665 if ``config.doAddCalexpBackground==True``
666 subtractedExposure : `lsst.afw.image.ExposureF`, optional
667 If ``config.doSubtract==False`` and ``config.doDetection==True``,
668 performs the post subtraction source detection only on this exposure.
669 Otherwise should be None.
670
671 Returns
672 -------
673 results : `lsst.pipe.base.Struct`
674 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
675 Difference image.
676 ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
677 The zogy score exposure, if calculated.
678 ``matchedExposure`` : `lsst.afw.image.ExposureF`
679 The matched PSF exposure.
680 ``subtractRes`` : `lsst.pipe.base.Struct`
681 The returned result structure of the ImagePsfMatchTask subtask.
682 ``diaSources`` : `lsst.afw.table.SourceCatalog`
683 The catalog of detected sources.
684 ``selectSources`` : `lsst.afw.table.SourceCatalog`
685 The input source catalog with optionally added Qa information.
686
687 Notes
688 -----
689 The following major steps are included:
690
691 - warp template coadd to match WCS of image
692 - PSF match image to warped template
693 - subtract image from PSF-matched, warped template
694 - detect sources
695 - measure sources
696
697 For details about the image subtraction configuration modes
698 see `lsst.ip.diffim`.
699 """
700 subtractRes = None
701 controlSources = None
702 subtractedExposure = None
703 scoreExposure = None
704 diaSources = None
705 kernelSources = None
706 # We'll clone exposure if modified but will still need the original
707 exposureOrig = exposure
708
709 if self.config.doAddCalexpBackground:
710 mi = exposure.getMaskedImage()
711 mi += calexpBackgroundExposure.getImage()
712
713 if not exposure.hasPsf():
714 raise pipeBase.TaskError("Exposure has no psf")
715 sciencePsf = exposure.getPsf()
716
717 if self.config.doSubtract:
718 if self.config.doScaleTemplateVariance:
719 self.log.info("Rescaling template variance")
720 templateVarFactor = self.scaleVariance.run(
721 templateExposure.getMaskedImage())
722 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
723 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
724 self.metadata.add("psfMatchingAlgorithm", self.config.subtract.name)
725
726 if self.config.subtract.name == 'zogy':
727 subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
728 scoreExposure = subtractRes.scoreExp
729 subtractedExposure = subtractRes.diffExp
730 subtractRes.subtractedExposure = subtractedExposure
731 subtractRes.matchedExposure = None
732
733 elif self.config.subtract.name == 'al':
734 # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
735 # Just need a rough estimate; average positions are fine
736 sciAvgPos = sciencePsf.getAveragePosition()
737 scienceSigmaOrig = sciencePsf.computeShape(sciAvgPos).getDeterminantRadius()
738
739 templatePsf = templateExposure.getPsf()
740 templateAvgPos = templatePsf.getAveragePosition()
741 templateSigma = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()
742
743 # if requested, convolve the science exposure with its PSF
744 # (properly, this should be a cross-correlation, but our code does not yet support that)
745 # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
746 # else sigma of original science exposure
747 # TODO: DM-22762 This functional block should be moved into its own method
748 preConvPsf = None
749 if self.config.useScoreImageDetection:
750 self.log.warn("AL likelihood image: pre-convolution of PSF is not implemented.")
751 convControl = afwMath.ConvolutionControl()
752 # cannot convolve in place, so need a new image anyway
753 srcMI = exposure.maskedImage
754 exposure = exposure.clone() # New deep copy
755 srcPsf = sciencePsf
756 if self.config.useGaussianForPreConvolution:
757 self.log.infof(
758 "AL likelihood image: Using Gaussian (sigma={:.2f}) PSF estimation "
759 "for science image pre-convolution", scienceSigmaOrig)
760 # convolve with a simplified PSF model: a double Gaussian
761 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
762 preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
763 else:
764 # convolve with science exposure's PSF model
765 self.log.infof(
766 "AL likelihood image: Using the science image PSF for pre-convolution.")
767 preConvPsf = srcPsf
768 afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
769 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
770 else:
771 scienceSigmaPost = scienceSigmaOrig
772
773 # If requested, find and select sources from the image
774 # else, AL subtraction will do its own source detection
775 # TODO: DM-22762 This functional block should be moved into its own method
776 if self.config.doSelectSources:
777 if selectSources is None:
778 self.log.warning("Src product does not exist; running detection, measurement,"
779 " selection")
780 # Run own detection and measurement; necessary in nightly processing
781 selectSources = self.subtract.getSelectSources(
782 exposure,
783 sigma=scienceSigmaPost,
784 doSmooth=not self.config.useScoreImageDetection,
785 idFactory=idFactory,
786 )
787
788 if self.config.doAddMetrics:
789 # Number of basis functions
790
791 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
792 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
793 targetFwhmPix=templateSigma*FwhmPerSigma))
794 # Modify the schema of all Sources
795 # DEPRECATED: This is a data dependent (nparam) output product schema
796 # outside the task constructor.
797 # NOTE: The pre-determination of nparam at this point
798 # may be incorrect as the template psf is warped later in
799 # ImagePsfMatchTask.matchExposures()
800 kcQa = KernelCandidateQa(nparam)
801 selectSources = kcQa.addToSchema(selectSources)
802 if self.config.kernelSourcesFromRef:
803 # match exposure sources to reference catalog
804 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
805 matches = astromRet.matches
806 elif templateSources:
807 # match exposure sources to template sources
808 mc = afwTable.MatchControl()
809 mc.findOnlyClosest = False
810 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
811 mc)
812 else:
813 raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
814 "but template sources not available. Cannot match science "
815 "sources with template sources. Run process* on data from "
816 "which templates are built.")
817
818 kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
819 matches=matches).sourceCat
820 random.shuffle(kernelSources, random.random)
821 controlSources = kernelSources[::self.config.controlStepSize]
822 kernelSources = [k for i, k in enumerate(kernelSources)
823 if i % self.config.controlStepSize]
824
825 if self.config.doSelectDcrCatalog:
826 redSelector = DiaCatalogSourceSelectorTask(
827 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
828 grMax=99.999))
829 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
830 controlSources.extend(redSources)
831
832 blueSelector = DiaCatalogSourceSelectorTask(
833 DiaCatalogSourceSelectorConfig(grMin=-99.999,
834 grMax=self.sourceSelector.config.grMin))
835 blueSources = blueSelector.selectStars(exposure, selectSources,
836 matches=matches).starCat
837 controlSources.extend(blueSources)
838
839 if self.config.doSelectVariableCatalog:
840 varSelector = DiaCatalogSourceSelectorTask(
841 DiaCatalogSourceSelectorConfig(includeVariable=True))
842 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
843 controlSources.extend(varSources)
844
845 self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)",
846 len(kernelSources), len(selectSources), len(controlSources))
847
848 allresids = {}
849 # TODO: DM-22762 This functional block should be moved into its own method
850 if self.config.doUseRegister:
851 self.log.info("Registering images")
852
853 if templateSources is None:
854 # Run detection on the template, which is
855 # temporarily background-subtracted
856 # sigma of PSF of template image before warping
857 templateSources = self.subtract.getSelectSources(
858 templateExposure,
859 sigma=templateSigma,
860 doSmooth=True,
861 idFactory=idFactory
862 )
863
864 # Third step: we need to fit the relative astrometry.
865 #
866 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
867 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
868 exposure.getWcs(), exposure.getBBox())
869 templateExposure = warpedExp
870
871 # Create debugging outputs on the astrometric
872 # residuals as a function of position. Persistence
873 # not yet implemented; expected on (I believe) #2636.
874 if self.config.doDebugRegister:
875 # Grab matches to reference catalog
876 srcToMatch = {x.second.getId(): x.first for x in matches}
877
878 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
879 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
880 sids = [m.first.getId() for m in wcsResults.matches]
881 positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
882 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
883 m.second.get(inCentroidKey))) for m in wcsResults.matches]
884 allresids = dict(zip(sids, zip(positions, residuals)))
885
886 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
887 wcsResults.wcs.pixelToSky(
888 m.second.get(inCentroidKey))) for m in wcsResults.matches]
889 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
890 + 2.5*numpy.log10(srcToMatch[x].get("r"))
891 for x in sids if x in srcToMatch.keys()])
892 dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
893 if s in srcToMatch.keys()])
894 dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
895 if s in srcToMatch.keys()])
896 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
897 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
898 & (colors <= self.sourceSelector.config.grMax))
899 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
900 rms1Long = IqrToSigma*(
901 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
902 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
903 - numpy.percentile(dlat[idx1], 25))
904 rms2Long = IqrToSigma*(
905 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
906 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
907 - numpy.percentile(dlat[idx2], 25))
908 rms3Long = IqrToSigma*(
909 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
910 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
911 - numpy.percentile(dlat[idx3], 25))
912 self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f",
913 numpy.median(dlong[idx1]), rms1Long,
914 numpy.median(dlat[idx1]), rms1Lat)
915 self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f",
916 numpy.median(dlong[idx2]), rms2Long,
917 numpy.median(dlat[idx2]), rms2Lat)
918 self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f",
919 numpy.median(dlong[idx3]), rms3Long,
920 numpy.median(dlat[idx3]), rms3Lat)
921
922 self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
923 self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
924 self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
925 self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
926 self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
927 self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
928
929 self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
930 self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
931 self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
932 self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
933 self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
934 self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
935
936 # warp template exposure to match exposure,
937 # PSF match template exposure to exposure,
938 # then return the difference
939
940 # Return warped template... Construct sourceKernelCand list after subtract
941 self.log.info("Subtracting images")
942 subtractRes = self.subtract.subtractExposures(
943 templateExposure=templateExposure,
944 scienceExposure=exposure,
945 candidateList=kernelSources,
946 convolveTemplate=self.config.convolveTemplate,
947 doWarping=not self.config.doUseRegister
948 )
949 if self.config.useScoreImageDetection:
950 scoreExposure = subtractRes.subtractedExposure
951 else:
952 subtractedExposure = subtractRes.subtractedExposure
953
954 if self.config.doDetection:
955 self.log.info("Computing diffim PSF")
956
957 # Get Psf from the appropriate input image if it doesn't exist
958 if subtractedExposure is not None and not subtractedExposure.hasPsf():
959 if self.config.convolveTemplate:
960 subtractedExposure.setPsf(exposure.getPsf())
961 else:
962 subtractedExposure.setPsf(templateExposure.getPsf())
963
964 # If doSubtract is False, then subtractedExposure was fetched from disk (above),
965 # thus it may have already been decorrelated. Thus, we do not decorrelate if
966 # doSubtract is False.
967
968 # NOTE: At this point doSubtract == True
969 if self.config.doDecorrelation and self.config.doSubtract:
970 preConvKernel = None
971 if self.config.useGaussianForPreConvolution:
972 preConvKernel = preConvPsf.getLocalKernel()
973 if self.config.useScoreImageDetection:
974 scoreExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
975 scoreExposure,
976 subtractRes.psfMatchingKernel,
977 spatiallyVarying=self.config.doSpatiallyVarying,
978 preConvKernel=preConvKernel,
979 templateMatched=True,
980 preConvMode=True).correctedExposure
981 # Note that the subtracted exposure is always decorrelated,
982 # even if the score image is used for detection
983 subtractedExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
984 subtractedExposure,
985 subtractRes.psfMatchingKernel,
986 spatiallyVarying=self.config.doSpatiallyVarying,
987 preConvKernel=None,
988 templateMatched=self.config.convolveTemplate,
989 preConvMode=False).correctedExposure
990 # END (if subtractAlgorithm == 'AL')
991 # END (if self.config.doSubtract)
992 if self.config.doDetection:
993 self.log.info("Running diaSource detection")
994
995 # subtractedExposure - reserved for task return value
996 # in zogy, it is always the proper difference image
997 # in AL, it may be (yet) pre-convolved and/or decorrelated
998 #
999 # detectionExposure - controls which exposure to use for detection
1000 # in-place modifications will appear in task return
1001 if self.config.useScoreImageDetection:
1002 # zogy with score image detection enabled
1003 self.log.info("Detection, diffim rescaling and measurements are "
1004 "on AL likelihood or Zogy score image.")
1005 detectionExposure = scoreExposure
1006 else:
1007 # AL or zogy with no score image detection
1008 detectionExposure = subtractedExposure
1009
1010 # Rescale difference image variance plane
1011 if self.config.doScaleDiffimVariance:
1012 self.log.info("Rescaling diffim variance")
1013 diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
1014 self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor)
1015 self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
1016
1017 # Erase existing detection mask planes
1018 mask = detectionExposure.getMaskedImage().getMask()
1019 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
1020
1021 table = afwTable.SourceTable.make(self.schema, idFactory)
1022 table.setMetadata(self.algMetadata)
1023 results = self.detection.run(
1024 table=table,
1025 exposure=detectionExposure,
1026 doSmooth=not self.config.useScoreImageDetection
1027 )
1028
1029 if self.config.doMerge:
1030 fpSet = results.fpSets.positive
1031 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1032 self.config.growFootprint, False)
1033 diaSources = afwTable.SourceCatalog(table)
1034 fpSet.makeSources(diaSources)
1035 self.log.info("Merging detections into %d sources", len(diaSources))
1036 else:
1037 diaSources = results.sources
1038 # Inject skySources before measurement.
1039 if self.config.doSkySources:
1040 skySourceFootprints = self.skySources.run(
1041 mask=detectionExposure.mask,
1042 seed=detectionExposure.getInfo().getVisitInfo().getExposureId())
1043 if skySourceFootprints:
1044 for foot in skySourceFootprints:
1045 s = diaSources.addNew()
1046 s.setFootprint(foot)
1047 s.set(self.skySourceKey, True)
1048
1049 if self.config.doMeasurement:
1050 newDipoleFitting = self.config.doDipoleFitting
1051 self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1052 if not newDipoleFitting:
1053 # Just fit dipole in diffim
1054 self.measurement.run(diaSources, detectionExposure)
1055 else:
1056 # Use (matched) template and science image (if avail.) to constrain dipole fitting
1057 if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1058 self.measurement.run(diaSources, detectionExposure, exposure,
1059 subtractRes.matchedExposure)
1060 else:
1061 self.measurement.run(diaSources, detectionExposure, exposure)
1062 if self.config.doApCorr:
1063 self.applyApCorr.run(
1064 catalog=diaSources,
1065 apCorrMap=detectionExposure.getInfo().getApCorrMap()
1066 )
1067
1068 if self.config.doForcedMeasurement:
1069 # Run forced psf photometry on the PVI at the diaSource locations.
1070 # Copy the measured flux and error into the diaSource.
1071 forcedSources = self.forcedMeasurement.generateMeasCat(
1072 exposure, diaSources, detectionExposure.getWcs())
1073 self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1074 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1075 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1076 "ip_diffim_forced_PsfFlux_instFlux", True)
1077 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1078 "ip_diffim_forced_PsfFlux_instFluxErr", True)
1079 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1080 "ip_diffim_forced_PsfFlux_area", True)
1081 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1082 "ip_diffim_forced_PsfFlux_flag", True)
1083 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1084 "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1085 mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1086 "ip_diffim_forced_PsfFlux_flag_edge", True)
1087 for diaSource, forcedSource in zip(diaSources, forcedSources):
1088 diaSource.assign(forcedSource, mapper)
1089
1090 # Match with the calexp sources if possible
1091 if self.config.doMatchSources:
1092 if selectSources is not None:
1093 # Create key,val pair where key=diaSourceId and val=sourceId
1094 matchRadAsec = self.config.diaSourceMatchRadius
1095 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1096
1097 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1098 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1099 srcMatch in srcMatches])
1100 self.log.info("Matched %d / %d diaSources to sources",
1101 len(srcMatchDict), len(diaSources))
1102 else:
1103 self.log.warning("Src product does not exist; cannot match with diaSources")
1104 srcMatchDict = {}
1105
1106 # Create key,val pair where key=diaSourceId and val=refId
1107 refAstromConfig = AstrometryConfig()
1108 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1109 refAstrometer = AstrometryTask(refAstromConfig)
1110 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1111 refMatches = astromRet.matches
1112 if refMatches is None:
1113 self.log.warning("No diaSource matches with reference catalog")
1114 refMatchDict = {}
1115 else:
1116 self.log.info("Matched %d / %d diaSources to reference catalog",
1117 len(refMatches), len(diaSources))
1118 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1119 refMatch in refMatches])
1120
1121 # Assign source Ids
1122 for diaSource in diaSources:
1123 sid = diaSource.getId()
1124 if sid in srcMatchDict:
1125 diaSource.set("srcMatchId", srcMatchDict[sid])
1126 if sid in refMatchDict:
1127 diaSource.set("refMatchId", refMatchDict[sid])
1128
1129 if self.config.doAddMetrics and self.config.doSelectSources:
1130 self.log.info("Evaluating metrics and control sample")
1131
1132 kernelCandList = []
1133 for cell in subtractRes.kernelCellSet.getCellList():
1134 for cand in cell.begin(False): # include bad candidates
1135 kernelCandList.append(cand)
1136
1137 # Get basis list to build control sample kernels
1138 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1139 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1140
1141 controlCandList = (
1142 diffimTools.sourceTableToCandidateList(controlSources,
1143 subtractRes.warpedExposure, exposure,
1144 self.config.subtract.kernel.active,
1145 self.config.subtract.kernel.active.detectionConfig,
1146 self.log, doBuild=True, basisList=basisList))
1147
1148 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1149 subtractRes.backgroundModel, dof=nparam)
1150 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1151 subtractRes.backgroundModel)
1152
1153 if self.config.doDetection:
1154 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1155 else:
1156 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1157
1158 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1159 return pipeBase.Struct(
1160 subtractedExposure=subtractedExposure,
1161 scoreExposure=scoreExposure,
1162 warpedExposure=subtractRes.warpedExposure,
1163 matchedExposure=subtractRes.matchedExposure,
1164 subtractRes=subtractRes,
1165 diaSources=diaSources,
1166 selectSources=selectSources
1167 )
1168
1169 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1170 """Fit the relative astrometry between templateSources and selectSources
1171
1172 Todo
1173 ----
1174
1175 Remove this method. It originally fit a new WCS to the template before calling register.run
1176 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1177 It remains because a subtask overrides it.
1178 """
1179 results = self.register.run(templateSources, templateExposure.getWcs(),
1180 templateExposure.getBBox(), selectSources)
1181 return results
1182
1183 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1184 """Make debug plots and displays.
1185
1186 Todo
1187 ----
1188 Test and update for current debug display and slot names
1189 """
1190 import lsstDebug
1191 display = lsstDebug.Info(__name__).display
1192 showSubtracted = lsstDebug.Info(__name__).showSubtracted
1193 showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1194 showDiaSources = lsstDebug.Info(__name__).showDiaSources
1195 showDipoles = lsstDebug.Info(__name__).showDipoles
1196 maskTransparency = lsstDebug.Info(__name__).maskTransparency
1197 if display:
1198 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1199 if not maskTransparency:
1200 maskTransparency = 0
1201 disp.setMaskTransparency(maskTransparency)
1202
1203 if display and showSubtracted:
1204 disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1205 mi = subtractRes.subtractedExposure.getMaskedImage()
1206 x0, y0 = mi.getX0(), mi.getY0()
1207 with disp.Buffering():
1208 for s in diaSources:
1209 x, y = s.getX() - x0, s.getY() - y0
1210 ctype = "red" if s.get("flags_negative") else "yellow"
1211 if (s.get("base_PixelFlags_flag_interpolatedCenter")
1212 or s.get("base_PixelFlags_flag_saturatedCenter")
1213 or s.get("base_PixelFlags_flag_crCenter")):
1214 ptype = "x"
1215 elif (s.get("base_PixelFlags_flag_interpolated")
1216 or s.get("base_PixelFlags_flag_saturated")
1217 or s.get("base_PixelFlags_flag_cr")):
1218 ptype = "+"
1219 else:
1220 ptype = "o"
1221 disp.dot(ptype, x, y, size=4, ctype=ctype)
1222 lsstDebug.frame += 1
1223
1224 if display and showPixelResiduals and selectSources:
1225 nonKernelSources = []
1226 for source in selectSources:
1227 if source not in kernelSources:
1228 nonKernelSources.append(source)
1229
1230 diUtils.plotPixelResiduals(exposure,
1231 subtractRes.warpedExposure,
1232 subtractRes.subtractedExposure,
1233 subtractRes.kernelCellSet,
1234 subtractRes.psfMatchingKernel,
1235 subtractRes.backgroundModel,
1236 nonKernelSources,
1237 self.subtract.config.kernel.active.detectionConfig,
1238 origVariance=False)
1239 diUtils.plotPixelResiduals(exposure,
1240 subtractRes.warpedExposure,
1241 subtractRes.subtractedExposure,
1242 subtractRes.kernelCellSet,
1243 subtractRes.psfMatchingKernel,
1244 subtractRes.backgroundModel,
1245 nonKernelSources,
1246 self.subtract.config.kernel.active.detectionConfig,
1247 origVariance=True)
1248 if display and showDiaSources:
1249 flagChecker = SourceFlagChecker(diaSources)
1250 isFlagged = [flagChecker(x) for x in diaSources]
1251 isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1252 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1253 frame=lsstDebug.frame)
1254 lsstDebug.frame += 1
1255
1256 if display and showDipoles:
1257 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1258 frame=lsstDebug.frame)
1259 lsstDebug.frame += 1
1260
1261 def checkTemplateIsSufficient(self, templateExposure):
1262 """Raise NoWorkFound if template coverage < requiredTemplateFraction
1263
1264 Parameters
1265 ----------
1266 templateExposure : `lsst.afw.image.ExposureF`
1267 The template exposure to check
1268
1269 Raises
1270 ------
1271 NoWorkFound
1272 Raised if fraction of good pixels, defined as not having NO_DATA
1273 set, is less then the configured requiredTemplateFraction
1274 """
1275 # Count the number of pixels with the NO_DATA mask bit set
1276 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
1277 pixNoData = numpy.count_nonzero(templateExposure.mask.array
1278 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
1279 pixGood = templateExposure.getBBox().getArea() - pixNoData
1280 self.log.info("template has %d good pixels (%.1f%%)", pixGood,
1281 100*pixGood/templateExposure.getBBox().getArea())
1282
1283 if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction:
1284 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
1285 "To force subtraction, set config requiredTemplateFraction=0." % (
1286 100*pixGood/templateExposure.getBBox().getArea(),
1287 100*self.config.requiredTemplateFraction))
1288 raise pipeBase.NoWorkFound(message)
1289
1290 def _getConfigName(self):
1291 """Return the name of the config dataset
1292 """
1293 return "%sDiff_config" % (self.config.coaddName,)
1294
1295 def _getMetadataName(self):
1296 """Return the name of the metadata dataset
1297 """
1298 return "%sDiff_metadata" % (self.config.coaddName,)
1299
1300 def getSchemaCatalogs(self):
1301 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1302 return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1303
1304 @classmethod
1305 def _makeArgumentParser(cls):
1306 """Create an argument parser
1307 """
1308 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1309 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1310 parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1311 help="Template data ID in case of calexp template,"
1312 " e.g. --templateId visit=6789")
1313 return parser
1314
1315
1316class ImageDifferenceFromTemplateConnections(ImageDifferenceTaskConnections,
1317 defaultTemplates={"coaddName": "goodSeeing"}
1318 ):
1319 inputTemplate = pipeBase.connectionTypes.Input(
1320 doc=("Warped template produced by GetMultiTractCoaddTemplate"),
1321 dimensions=("instrument", "visit", "detector"),
1322 storageClass="ExposureF",
1323 name="{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}",
1324 )
1325
1326 def __init__(self, *, config=None):
1327 super().__init__(config=config)
1328 # ImageDifferenceConnections will have removed one of these.
1329 # Make sure they're both gone, because no coadds are needed.
1330 if "coaddExposures" in self.inputs:
1331 self.inputs.remove("coaddExposures")
1332 if "dcrCoadds" in self.inputs:
1333 self.inputs.remove("dcrCoadds")
1334
1335
1336class ImageDifferenceFromTemplateConfig(ImageDifferenceConfig,
1337 pipelineConnections=ImageDifferenceFromTemplateConnections):
1338 pass
1339
1340
1341class ImageDifferenceFromTemplateTask(ImageDifferenceTask):
1342 ConfigClass = ImageDifferenceFromTemplateConfig
1343 _DefaultName = "imageDifference"
1344
1345 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
1346 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1347 inputs = butlerQC.get(inputRefs)
1348 self.log.info("Processing %s", butlerQC.quantum.dataId)
1349 self.checkTemplateIsSufficient(inputs['inputTemplate'])
1350 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
1351 returnMaxBits=True)
1352 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
1353
1354 outputs = self.run(exposure=inputs['exposure'],
1355 templateExposure=inputs['inputTemplate'],
1356 idFactory=idFactory)
1357
1358 # Consistency with runDataref gen2 handling
1359 if outputs.diaSources is None:
1360 del outputs.diaSources
1361 butlerQC.put(outputs, outputRefs)
1362
1363
1364class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1365 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1366 doc="Shift stars going into RegisterTask by this amount")
1367 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1368 doc="Perturb stars going into RegisterTask by this amount")
1369
1370 def setDefaults(self):
1371 ImageDifferenceConfig.setDefaults(self)
1372 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1373
1374
1375class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1376 """!Image difference Task used in the Winter 2013 data challege.
1377 Enables testing the effects of registration shifts and scatter.
1378
1379 For use with winter 2013 simulated images:
1380 Use --templateId visit=88868666 for sparse data
1381 --templateId visit=22222200 for dense data (g)
1382 --templateId visit=11111100 for dense data (i)
1383 """
1384 ConfigClass = Winter2013ImageDifferenceConfig
1385 _DefaultName = "winter2013ImageDifference"
1386
1387 def __init__(self, **kwargs):
1388 ImageDifferenceTask.__init__(self, **kwargs)
1389
1390 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1391 """Fit the relative astrometry between templateSources and selectSources"""
1392 if self.config.winter2013WcsShift > 0.0:
1393 offset = geom.Extent2D(self.config.winter2013WcsShift,
1394 self.config.winter2013WcsShift)
1395 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1396 for source in templateSources:
1397 centroid = source.get(cKey)
1398 source.set(cKey, centroid + offset)
1399 elif self.config.winter2013WcsRms > 0.0:
1400 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1401 for source in templateSources:
1402 offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1403 self.config.winter2013WcsRms*numpy.random.normal())
1404 centroid = source.get(cKey)
1405 source.set(cKey, centroid + offset)
1406
1407 results = self.register.run(templateSources, templateExposure.getWcs(),
1408 templateExposure.getBBox(), selectSources)
1409 return results
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)