lsst.pipe.tasks  21.0.0-153-gd15f111f+620bfe6b9b
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 import lsst.meas.extensions.trailedSources # noqa: F401
34 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
35 from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
36 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
37 from lsst.pipe.tasks.registerImage import RegisterTask
38 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
39 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
40 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
41  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
42  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
43  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
44 import lsst.ip.diffim.diffimTools as diffimTools
45 import lsst.ip.diffim.utils as diUtils
46 import lsst.afw.display as afwDisplay
47 from lsst.skymap import BaseSkyMap
48 from lsst.obs.base import ExposureIdInfo
49 
50 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
51 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
52 IqrToSigma = 0.741
53 
54 
55 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
56  dimensions=("instrument", "visit", "detector", "skymap"),
57  defaultTemplates={"coaddName": "deep",
58  "skyMapName": "deep",
59  "warpTypeSuffix": "",
60  "fakesType": ""}):
61 
62  exposure = pipeBase.connectionTypes.Input(
63  doc="Input science exposure to subtract from.",
64  dimensions=("instrument", "visit", "detector"),
65  storageClass="ExposureF",
66  name="{fakesType}calexp"
67  )
68 
69  # TODO DM-22953
70  # kernelSources = pipeBase.connectionTypes.Input(
71  # doc="Source catalog produced in calibrate task for kernel candidate sources",
72  # name="src",
73  # storageClass="SourceCatalog",
74  # dimensions=("instrument", "visit", "detector"),
75  # )
76 
77  skyMap = pipeBase.connectionTypes.Input(
78  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
79  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
80  dimensions=("skymap", ),
81  storageClass="SkyMap",
82  )
83  coaddExposures = pipeBase.connectionTypes.Input(
84  doc="Input template to match and subtract from the exposure",
85  dimensions=("tract", "patch", "skymap", "band"),
86  storageClass="ExposureF",
87  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
88  multiple=True,
89  deferLoad=True
90  )
91  dcrCoadds = pipeBase.connectionTypes.Input(
92  doc="Input DCR template to match and subtract from the exposure",
93  name="{fakesType}dcrCoadd{warpTypeSuffix}",
94  storageClass="ExposureF",
95  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
96  multiple=True,
97  deferLoad=True
98  )
99  outputSchema = pipeBase.connectionTypes.InitOutput(
100  doc="Schema (as an example catalog) for output DIASource catalog.",
101  storageClass="SourceCatalog",
102  name="{fakesType}{coaddName}Diff_diaSrc_schema",
103  )
104  subtractedExposure = pipeBase.connectionTypes.Output(
105  doc="Output AL difference or Zogy proper difference image",
106  dimensions=("instrument", "visit", "detector"),
107  storageClass="ExposureF",
108  name="{fakesType}{coaddName}Diff_differenceExp",
109  )
110  scoreExposure = pipeBase.connectionTypes.Output(
111  doc="Output AL likelihood or Zogy score image",
112  dimensions=("instrument", "visit", "detector"),
113  storageClass="ExposureF",
114  name="{fakesType}{coaddName}Diff_scoreExp",
115  )
116  warpedExposure = pipeBase.connectionTypes.Output(
117  doc="Warped template used to create `subtractedExposure`.",
118  dimensions=("instrument", "visit", "detector"),
119  storageClass="ExposureF",
120  name="{fakesType}{coaddName}Diff_warpedExp",
121  )
122  matchedExposure = pipeBase.connectionTypes.Output(
123  doc="Warped template used to create `subtractedExposure`.",
124  dimensions=("instrument", "visit", "detector"),
125  storageClass="ExposureF",
126  name="{fakesType}{coaddName}Diff_matchedExp",
127  )
128  diaSources = pipeBase.connectionTypes.Output(
129  doc="Output detected diaSources on the difference image",
130  dimensions=("instrument", "visit", "detector"),
131  storageClass="SourceCatalog",
132  name="{fakesType}{coaddName}Diff_diaSrc",
133  )
134 
135  def __init__(self, *, config=None):
136  super().__init__(config=config)
137  if config.coaddName == 'dcr':
138  self.inputs.remove("coaddExposures")
139  else:
140  self.inputs.remove("dcrCoadds")
141  if not config.doWriteSubtractedExp:
142  self.outputs.remove("subtractedExposure")
143  if not config.doWriteScoreExp:
144  self.outputs.remove("scoreExposure")
145  if not config.doWriteWarpedExp:
146  self.outputs.remove("warpedExposure")
147  if not config.doWriteMatchedExp:
148  self.outputs.remove("matchedExposure")
149  if not config.doWriteSources:
150  self.outputs.remove("diaSources")
151 
152  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
153  # Make kernelSources optional
154 
155 
156 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
157  pipelineConnections=ImageDifferenceTaskConnections):
158  """Config for ImageDifferenceTask
159  """
160  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
161  doc="Add background to calexp before processing it. "
162  "Useful as ipDiffim does background matching.")
163  doUseRegister = pexConfig.Field(dtype=bool, default=False,
164  doc="Re-compute astrometry on the template. "
165  "Use image-to-image registration to align template with "
166  "science image (AL only).")
167  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
168  doc="Writing debugging data for doUseRegister")
169  doSelectSources = pexConfig.Field(dtype=bool, default=False,
170  doc="Select stars to use for kernel fitting (AL only)")
171  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
172  doc="Select stars of extreme color as part "
173  "of the control sample (AL only)")
174  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
175  doc="Select stars that are variable to be part "
176  "of the control sample (AL only)")
177  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
178  doPreConvolve = pexConfig.Field(dtype=bool, default=False,
179  doc="Not in use. Superseded by useScoreImageDetection.",
180  deprecated="This option superseded by useScoreImageDetection."
181  " Will be removed after v22.")
182  useScoreImageDetection = pexConfig.Field(
183  dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or "
184  "the Zogy score image. Use it for source detection (if doDetection=True).")
185  doWriteScoreExp = pexConfig.Field(
186  dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?")
187  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
188  doc="Scale variance of the template before PSF matching")
189  doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
190  doc="Scale variance of the diffim before PSF matching. "
191  "You may do either this or template variance scaling, "
192  "or neither. (Doing both is a waste of CPU.)")
193  useGaussianForPreConvolution = pexConfig.Field(
194  dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution "
195  "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)")
196  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
197  doDecorrelation = pexConfig.Field(dtype=bool, default=True,
198  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
199  "kernel convolution (AL only)? If True, also update the diffim PSF.")
200  doMerge = pexConfig.Field(dtype=bool, default=True,
201  doc="Merge positive and negative diaSources with grow radius "
202  "set by growFootprint")
203  doMatchSources = pexConfig.Field(dtype=bool, default=False,
204  doc="Match diaSources with input calexp sources and ref catalog sources")
205  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
206  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
207  doForcedMeasurement = pexConfig.Field(
208  dtype=bool,
209  default=True,
210  doc="Force photometer diaSource locations on PVI?")
211  doWriteSubtractedExp = pexConfig.Field(
212  dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
213  doWriteWarpedExp = pexConfig.Field(
214  dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
215  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
216  doc="Write warped and PSF-matched template coadd exposure?")
217  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
218  doAddMetrics = pexConfig.Field(dtype=bool, default=False,
219  doc="Add columns to the source table to hold analysis metrics?")
220 
221  coaddName = pexConfig.Field(
222  doc="coadd name: typically one of deep, goodSeeing, or dcr",
223  dtype=str,
224  default="deep",
225  )
226  convolveTemplate = pexConfig.Field(
227  doc="Which image gets convolved (default = template)",
228  dtype=bool,
229  default=True
230  )
231  refObjLoader = pexConfig.ConfigurableField(
232  target=LoadIndexedReferenceObjectsTask,
233  doc="reference object loader",
234  )
235  astrometer = pexConfig.ConfigurableField(
236  target=AstrometryTask,
237  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
238  )
239  sourceSelector = pexConfig.ConfigurableField(
240  target=ObjectSizeStarSelectorTask,
241  doc="Source selection algorithm",
242  )
243  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
244  decorrelate = pexConfig.ConfigurableField(
245  target=DecorrelateALKernelSpatialTask,
246  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
247  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
248  "default of 5.5).",
249  )
250  # Old style ImageMapper grid. ZogyTask has its own grid option
251  doSpatiallyVarying = pexConfig.Field(
252  dtype=bool,
253  default=False,
254  doc="Perform A&L decorrelation on a grid across the "
255  "image in order to allow for spatial variations. Zogy does not use this option."
256  )
257  detection = pexConfig.ConfigurableField(
258  target=SourceDetectionTask,
259  doc="Low-threshold detection for final measurement",
260  )
261  measurement = pexConfig.ConfigurableField(
262  target=DipoleFitTask,
263  doc="Enable updated dipole fitting method",
264  )
265  doApCorr = lsst.pex.config.Field(
266  dtype=bool,
267  default=True,
268  doc="Run subtask to apply aperture corrections"
269  )
270  applyApCorr = lsst.pex.config.ConfigurableField(
271  target=ApplyApCorrTask,
272  doc="Subtask to apply aperture corrections"
273  )
274  forcedMeasurement = pexConfig.ConfigurableField(
275  target=ForcedMeasurementTask,
276  doc="Subtask to force photometer PVI at diaSource location.",
277  )
278  getTemplate = pexConfig.ConfigurableField(
279  target=GetCoaddAsTemplateTask,
280  doc="Subtask to retrieve template exposure and sources",
281  )
282  scaleVariance = pexConfig.ConfigurableField(
283  target=ScaleVarianceTask,
284  doc="Subtask to rescale the variance of the template "
285  "to the statistically expected level"
286  )
287  controlStepSize = pexConfig.Field(
288  doc="What step size (every Nth one) to select a control sample from the kernelSources",
289  dtype=int,
290  default=5
291  )
292  controlRandomSeed = pexConfig.Field(
293  doc="Random seed for shuffing the control sample",
294  dtype=int,
295  default=10
296  )
297  register = pexConfig.ConfigurableField(
298  target=RegisterTask,
299  doc="Task to enable image-to-image image registration (warping)",
300  )
301  kernelSourcesFromRef = pexConfig.Field(
302  doc="Select sources to measure kernel from reference catalog if True, template if false",
303  dtype=bool,
304  default=False
305  )
306  templateSipOrder = pexConfig.Field(
307  dtype=int, default=2,
308  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
309  )
310  growFootprint = pexConfig.Field(
311  dtype=int, default=2,
312  doc="Grow positive and negative footprints by this amount before merging"
313  )
314  diaSourceMatchRadius = pexConfig.Field(
315  dtype=float, default=0.5,
316  doc="Match radius (in arcseconds) for DiaSource to Source association"
317  )
318  requiredTemplateFraction = pexConfig.Field(
319  dtype=float, default=0.1,
320  doc="Do not attempt to run task if template covers less than this fraction of pixels."
321  "Setting to 0 will always attempt image subtraction"
322  )
323  doSkySources = pexConfig.Field(
324  dtype=bool,
325  default=False,
326  doc="Generate sky sources?",
327  )
328  skySources = pexConfig.ConfigurableField(
329  target=SkyObjectsTask,
330  doc="Generate sky sources",
331  )
332 
333  def setDefaults(self):
334  # defaults are OK for catalog and diacatalog
335 
336  self.subtract['al'].kernel.name = "AL"
337  self.subtract['al'].kernel.active.fitForBackground = True
338  self.subtract['al'].kernel.active.spatialKernelOrder = 1
339  self.subtract['al'].kernel.active.spatialBgOrder = 2
340 
341  # DiaSource Detection
342  self.detection.thresholdPolarity = "both"
343  self.detection.thresholdValue = 5.0
344  self.detection.reEstimateBackground = False
345  self.detection.thresholdType = "pixel_stdev"
346 
347  # Add filtered flux measurement, the correct measurement for pre-convolved images.
348  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
349  # To change that you must modify algorithms.names in the task's applyOverrides method,
350  # after the user has set doPreConvolve.
351  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
352  self.measurement.plugins.names |= ['ext_trailedSources_Naive',
353  'base_LocalPhotoCalib',
354  'base_LocalWcs']
355 
356  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
357  self.forcedMeasurement.copyColumns = {
358  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
359  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
360  self.forcedMeasurement.slots.shape = None
361 
362  # For shuffling the control sample
363  random.seed(self.controlRandomSeed)
364 
365  def validate(self):
366  pexConfig.Config.validate(self)
367  if not self.doSubtract and not self.doDetection:
368  raise ValueError("Either doSubtract or doDetection must be enabled.")
369  if self.doMeasurement and not self.doDetection:
370  raise ValueError("Cannot run source measurement without source detection.")
371  if self.doMerge and not self.doDetection:
372  raise ValueError("Cannot run source merging without source detection.")
373  if self.doSkySources and not self.doDetection:
374  raise ValueError("Cannot run sky source creation without source detection.")
375  if self.doUseRegister and not self.doSelectSources:
376  raise ValueError("doUseRegister=True and doSelectSources=False. "
377  "Cannot run RegisterTask without selecting sources.")
378  if hasattr(self.getTemplate, "coaddName"):
379  if self.getTemplate.coaddName != self.coaddName:
380  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
381  if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
382  raise ValueError("Scaling the diffim variance and scaling the template variance "
383  "are both set. Please choose one or the other.")
384  # We cannot allow inconsistencies that would lead to None or not available output products
385  if self.subtract.name == 'zogy':
386  if self.doWriteMatchedExp:
387  raise ValueError("doWriteMatchedExp=True Matched exposure is not "
388  "calculated in zogy subtraction.")
389  if self.doAddMetrics:
390  raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
391  if self.doDecorrelation:
392  raise ValueError(
393  "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
394  if self.doSelectSources:
395  raise ValueError(
396  "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
397  if self.useGaussianForPreConvolution:
398  raise ValueError(
399  "useGaussianForPreConvolution=True This is an AL subtraction only option.")
400  else:
401  # AL only consistency checks
402  if self.useScoreImageDetection and not self.convolveTemplate:
403  raise ValueError(
404  "convolveTemplate=False and useScoreImageDetection=True "
405  "Pre-convolution and matching of the science image is not a supported operation.")
406  if self.doWriteSubtractedExp and self.useScoreImageDetection:
407  raise ValueError(
408  "doWriteSubtractedExp=True and useScoreImageDetection=True "
409  "Regular difference image is not calculated. "
410  "AL subtraction calculates either the regular difference image or the score image.")
411  if self.doWriteScoreExp and not self.useScoreImageDetection:
412  raise ValueError(
413  "doWriteScoreExp=True and useScoreImageDetection=False "
414  "Score image is not calculated. "
415  "AL subtraction calculates either the regular difference image or the score image.")
416  if self.doAddMetrics and not self.doSubtract:
417  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
418  if self.useGaussianForPreConvolution and not self.useScoreImageDetection:
419  raise ValueError(
420  "useGaussianForPreConvolution=True and useScoreImageDetection=False "
421  "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.")
422 
423 
424 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
425 
426  @staticmethod
427  def getTargetList(parsedCmd, **kwargs):
428  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
429  **kwargs)
430 
431 
432 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
433  """Subtract an image from a template and measure the result
434  """
435  ConfigClass = ImageDifferenceConfig
436  RunnerClass = ImageDifferenceTaskRunner
437  _DefaultName = "imageDifference"
438 
439  def __init__(self, butler=None, **kwargs):
440  """!Construct an ImageDifference Task
441 
442  @param[in] butler Butler object to use in constructing reference object loaders
443  """
444  super().__init__(**kwargs)
445  self.makeSubtask("getTemplate")
446 
447  self.makeSubtask("subtract")
448 
449  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
450  self.makeSubtask("decorrelate")
451 
452  if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
453  self.makeSubtask("scaleVariance")
454 
455  if self.config.doUseRegister:
456  self.makeSubtask("register")
457  self.schema = afwTable.SourceTable.makeMinimalSchema()
458 
459  if self.config.doSelectSources:
460  self.makeSubtask("sourceSelector")
461  if self.config.kernelSourcesFromRef:
462  self.makeSubtask('refObjLoader', butler=butler)
463  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
464 
465  self.algMetadata = dafBase.PropertyList()
466  if self.config.doDetection:
467  self.makeSubtask("detection", schema=self.schema)
468  if self.config.doMeasurement:
469  self.makeSubtask("measurement", schema=self.schema,
470  algMetadata=self.algMetadata)
471  if self.config.doApCorr:
472  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
473  if self.config.doForcedMeasurement:
474  self.schema.addField(
475  "ip_diffim_forced_PsfFlux_instFlux", "D",
476  "Forced PSF flux measured on the direct image.",
477  units="count")
478  self.schema.addField(
479  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
480  "Forced PSF flux error measured on the direct image.",
481  units="count")
482  self.schema.addField(
483  "ip_diffim_forced_PsfFlux_area", "F",
484  "Forced PSF flux effective area of PSF.",
485  units="pixel")
486  self.schema.addField(
487  "ip_diffim_forced_PsfFlux_flag", "Flag",
488  "Forced PSF flux general failure flag.")
489  self.schema.addField(
490  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
491  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
492  self.schema.addField(
493  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
494  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
495  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
496  if self.config.doMatchSources:
497  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
498  self.schema.addField("srcMatchId", "L", "unique id of source match")
499  if self.config.doSkySources:
500  self.makeSubtask("skySources")
501  self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
502 
503  # initialize InitOutputs
504  self.outputSchema = afwTable.SourceCatalog(self.schema)
505  self.outputSchema.getTable().setMetadata(self.algMetadata)
506 
507  @staticmethod
508  def makeIdFactory(expId, expBits):
509  """Create IdFactory instance for unique 64 bit diaSource id-s.
510 
511  Parameters
512  ----------
513  expId : `int`
514  Exposure id.
515 
516  expBits: `int`
517  Number of used bits in ``expId``.
518 
519  Note
520  ----
521  The diasource id-s consists of the ``expId`` stored fixed in the highest value
522  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
523  low value end of the integer.
524 
525  Returns
526  -------
527  idFactory: `lsst.afw.table.IdFactory`
528  """
529  return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
530 
531  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
532  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
533  inputRefs: pipeBase.InputQuantizedConnection,
534  outputRefs: pipeBase.OutputQuantizedConnection):
535  inputs = butlerQC.get(inputRefs)
536  self.log.info("Processing %s", butlerQC.quantum.dataId)
537  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
538  returnMaxBits=True)
539  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
540  if self.config.coaddName == 'dcr':
541  templateExposures = inputRefs.dcrCoadds
542  else:
543  templateExposures = inputRefs.coaddExposures
544  templateStruct = self.getTemplate.runQuantum(
545  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
546  )
547 
548  self.checkTemplateIsSufficient(templateStruct.exposure)
549 
550  outputs = self.run(exposure=inputs['exposure'],
551  templateExposure=templateStruct.exposure,
552  idFactory=idFactory)
553  # Consistency with runDataref gen2 handling
554  if outputs.diaSources is None:
555  del outputs.diaSources
556  butlerQC.put(outputs, outputRefs)
557 
558  @pipeBase.timeMethod
559  def runDataRef(self, sensorRef, templateIdList=None):
560  """Subtract an image from a template coadd and measure the result.
561 
562  Data I/O wrapper around `run` using the butler in Gen2.
563 
564  Parameters
565  ----------
566  sensorRef : `lsst.daf.persistence.ButlerDataRef`
567  Sensor-level butler data reference, used for the following data products:
568 
569  Input only:
570  - calexp
571  - psf
572  - ccdExposureId
573  - ccdExposureId_bits
574  - self.config.coaddName + "Coadd_skyMap"
575  - self.config.coaddName + "Coadd"
576  Input or output, depending on config:
577  - self.config.coaddName + "Diff_subtractedExp"
578  Output, depending on config:
579  - self.config.coaddName + "Diff_matchedExp"
580  - self.config.coaddName + "Diff_src"
581 
582  Returns
583  -------
584  results : `lsst.pipe.base.Struct`
585  Returns the Struct by `run`.
586  """
587  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
588  subtractedExposure = None
589  selectSources = None
590  calexpBackgroundExposure = None
591  self.log.info("Processing %s", sensorRef.dataId)
592 
593  # We make one IdFactory that will be used by both icSrc and src datasets;
594  # I don't know if this is the way we ultimately want to do things, but at least
595  # this ensures the source IDs are fully unique.
596  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
597  expBits=sensorRef.get("ccdExposureId_bits"))
598  if self.config.doAddCalexpBackground:
599  calexpBackgroundExposure = sensorRef.get("calexpBackground")
600 
601  # Retrieve the science image we wish to analyze
602  exposure = sensorRef.get("calexp", immediate=True)
603 
604  # Retrieve the template image
605  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
606 
607  if sensorRef.datasetExists("src"):
608  self.log.info("Source selection via src product")
609  # Sources already exist; for data release processing
610  selectSources = sensorRef.get("src")
611 
612  if not self.config.doSubtract and self.config.doDetection:
613  # If we don't do subtraction, we need the subtracted exposure from the repo
614  subtractedExposure = sensorRef.get(subtractedExposureName)
615  # Both doSubtract and doDetection cannot be False
616 
617  results = self.run(exposure=exposure,
618  selectSources=selectSources,
619  templateExposure=template.exposure,
620  templateSources=template.sources,
621  idFactory=idFactory,
622  calexpBackgroundExposure=calexpBackgroundExposure,
623  subtractedExposure=subtractedExposure)
624 
625  if self.config.doWriteSources and results.diaSources is not None:
626  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
627  if self.config.doWriteWarpedExp:
628  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
629  if self.config.doWriteMatchedExp:
630  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
631  if self.config.doAddMetrics and self.config.doSelectSources:
632  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
633  if self.config.doWriteSubtractedExp:
634  sensorRef.put(results.subtractedExposure, subtractedExposureName)
635  if self.config.doWriteScoreExp:
636  sensorRef.put(results.scoreExposure, self.config.coaddName + "Diff_scoreExp")
637  return results
638 
639  @pipeBase.timeMethod
640  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
641  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
642  """PSF matches, subtract two images and perform detection on the difference image.
643 
644  Parameters
645  ----------
646  exposure : `lsst.afw.image.ExposureF`, optional
647  The science exposure, the minuend in the image subtraction.
648  Can be None only if ``config.doSubtract==False``.
649  selectSources : `lsst.afw.table.SourceCatalog`, optional
650  Identified sources on the science exposure. This catalog is used to
651  select sources in order to perform the AL PSF matching on stamp images
652  around them. The selection steps depend on config options and whether
653  ``templateSources`` and ``matchingSources`` specified.
654  templateExposure : `lsst.afw.image.ExposureF`, optional
655  The template to be subtracted from ``exposure`` in the image subtraction.
656  ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
657  The template exposure should cover the same sky area as the science exposure.
658  It is either a stich of patches of a coadd skymap image or a calexp
659  of the same pointing as the science exposure. Can be None only
660  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
661  templateSources : `lsst.afw.table.SourceCatalog`, optional
662  Identified sources on the template exposure.
663  idFactory : `lsst.afw.table.IdFactory`
664  Generator object to assign ids to detected sources in the difference image.
665  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
666  Background exposure to be added back to the science exposure
667  if ``config.doAddCalexpBackground==True``
668  subtractedExposure : `lsst.afw.image.ExposureF`, optional
669  If ``config.doSubtract==False`` and ``config.doDetection==True``,
670  performs the post subtraction source detection only on this exposure.
671  Otherwise should be None.
672 
673  Returns
674  -------
675  results : `lsst.pipe.base.Struct`
676  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
677  Difference image.
678  ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
679  The zogy score exposure, if calculated.
680  ``matchedExposure`` : `lsst.afw.image.ExposureF`
681  The matched PSF exposure.
682  ``subtractRes`` : `lsst.pipe.base.Struct`
683  The returned result structure of the ImagePsfMatchTask subtask.
684  ``diaSources`` : `lsst.afw.table.SourceCatalog`
685  The catalog of detected sources.
686  ``selectSources`` : `lsst.afw.table.SourceCatalog`
687  The input source catalog with optionally added Qa information.
688 
689  Notes
690  -----
691  The following major steps are included:
692 
693  - warp template coadd to match WCS of image
694  - PSF match image to warped template
695  - subtract image from PSF-matched, warped template
696  - detect sources
697  - measure sources
698 
699  For details about the image subtraction configuration modes
700  see `lsst.ip.diffim`.
701  """
702  subtractRes = None
703  controlSources = None
704  subtractedExposure = None
705  scoreExposure = None
706  diaSources = None
707  kernelSources = None
708  # We'll clone exposure if modified but will still need the original
709  exposureOrig = exposure
710 
711  if self.config.doAddCalexpBackground:
712  mi = exposure.getMaskedImage()
713  mi += calexpBackgroundExposure.getImage()
714 
715  if not exposure.hasPsf():
716  raise pipeBase.TaskError("Exposure has no psf")
717  sciencePsf = exposure.getPsf()
718 
719  if self.config.doSubtract:
720  if self.config.doScaleTemplateVariance:
721  self.log.info("Rescaling template variance")
722  templateVarFactor = self.scaleVariance.run(
723  templateExposure.getMaskedImage())
724  self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
725  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
726  self.metadata.add("psfMatchingAlgorithm", self.config.subtract.name)
727 
728  if self.config.subtract.name == 'zogy':
729  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
730  scoreExposure = subtractRes.scoreExp
731  subtractedExposure = subtractRes.diffExp
732  subtractRes.subtractedExposure = subtractedExposure
733  subtractRes.matchedExposure = None
734 
735  elif self.config.subtract.name == 'al':
736  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
737  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
738  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
739 
740  # if requested, convolve the science exposure with its PSF
741  # (properly, this should be a cross-correlation, but our code does not yet support that)
742  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
743  # else sigma of original science exposure
744  # TODO: DM-22762 This functional block should be moved into its own method
745  preConvPsf = None
746  if self.config.useScoreImageDetection:
747  self.log.warn("AL likelihood image: pre-convolution of PSF is not implemented.")
748  convControl = afwMath.ConvolutionControl()
749  # cannot convolve in place, so need a new image anyway
750  srcMI = exposure.maskedImage
751  exposure = exposure.clone() # New deep copy
752  srcPsf = sciencePsf
753  if self.config.useGaussianForPreConvolution:
754  self.log.infof(
755  "AL likelihood image: Using Gaussian (sigma={:.2f}) PSF estimation "
756  "for science image pre-convolution", scienceSigmaOrig)
757  # convolve with a simplified PSF model: a double Gaussian
758  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
759  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
760  else:
761  # convolve with science exposure's PSF model
762  self.log.infof(
763  "AL likelihood image: Using the science image PSF for pre-convolution.")
764  preConvPsf = srcPsf
765  afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
766  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
767  else:
768  scienceSigmaPost = scienceSigmaOrig
769 
770  # If requested, find and select sources from the image
771  # else, AL subtraction will do its own source detection
772  # TODO: DM-22762 This functional block should be moved into its own method
773  if self.config.doSelectSources:
774  if selectSources is None:
775  self.log.warning("Src product does not exist; running detection, measurement,"
776  " selection")
777  # Run own detection and measurement; necessary in nightly processing
778  selectSources = self.subtract.getSelectSources(
779  exposure,
780  sigma=scienceSigmaPost,
781  doSmooth=not self.config.useScoreImageDetection,
782  idFactory=idFactory,
783  )
784 
785  if self.config.doAddMetrics:
786  # Number of basis functions
787 
788  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
789  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
790  targetFwhmPix=templateSigma*FwhmPerSigma))
791  # Modify the schema of all Sources
792  # DEPRECATED: This is a data dependent (nparam) output product schema
793  # outside the task constructor.
794  # NOTE: The pre-determination of nparam at this point
795  # may be incorrect as the template psf is warped later in
796  # ImagePsfMatchTask.matchExposures()
797  kcQa = KernelCandidateQa(nparam)
798  selectSources = kcQa.addToSchema(selectSources)
799  if self.config.kernelSourcesFromRef:
800  # match exposure sources to reference catalog
801  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
802  matches = astromRet.matches
803  elif templateSources:
804  # match exposure sources to template sources
805  mc = afwTable.MatchControl()
806  mc.findOnlyClosest = False
807  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
808  mc)
809  else:
810  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
811  "but template sources not available. Cannot match science "
812  "sources with template sources. Run process* on data from "
813  "which templates are built.")
814 
815  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
816  matches=matches).sourceCat
817  random.shuffle(kernelSources, random.random)
818  controlSources = kernelSources[::self.config.controlStepSize]
819  kernelSources = [k for i, k in enumerate(kernelSources)
820  if i % self.config.controlStepSize]
821 
822  if self.config.doSelectDcrCatalog:
823  redSelector = DiaCatalogSourceSelectorTask(
824  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
825  grMax=99.999))
826  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
827  controlSources.extend(redSources)
828 
829  blueSelector = DiaCatalogSourceSelectorTask(
830  DiaCatalogSourceSelectorConfig(grMin=-99.999,
831  grMax=self.sourceSelector.config.grMin))
832  blueSources = blueSelector.selectStars(exposure, selectSources,
833  matches=matches).starCat
834  controlSources.extend(blueSources)
835 
836  if self.config.doSelectVariableCatalog:
837  varSelector = DiaCatalogSourceSelectorTask(
838  DiaCatalogSourceSelectorConfig(includeVariable=True))
839  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
840  controlSources.extend(varSources)
841 
842  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)",
843  len(kernelSources), len(selectSources), len(controlSources))
844 
845  allresids = {}
846  # TODO: DM-22762 This functional block should be moved into its own method
847  if self.config.doUseRegister:
848  self.log.info("Registering images")
849 
850  if templateSources is None:
851  # Run detection on the template, which is
852  # temporarily background-subtracted
853  # sigma of PSF of template image before warping
854  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
855  templateSources = self.subtract.getSelectSources(
856  templateExposure,
857  sigma=templateSigma,
858  doSmooth=True,
859  idFactory=idFactory
860  )
861 
862  # Third step: we need to fit the relative astrometry.
863  #
864  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
865  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
866  exposure.getWcs(), exposure.getBBox())
867  templateExposure = warpedExp
868 
869  # Create debugging outputs on the astrometric
870  # residuals as a function of position. Persistence
871  # not yet implemented; expected on (I believe) #2636.
872  if self.config.doDebugRegister:
873  # Grab matches to reference catalog
874  srcToMatch = {x.second.getId(): x.first for x in matches}
875 
876  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
877  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
878  sids = [m.first.getId() for m in wcsResults.matches]
879  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
880  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
881  m.second.get(inCentroidKey))) for m in wcsResults.matches]
882  allresids = dict(zip(sids, zip(positions, residuals)))
883 
884  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
885  wcsResults.wcs.pixelToSky(
886  m.second.get(inCentroidKey))) for m in wcsResults.matches]
887  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
888  + 2.5*numpy.log10(srcToMatch[x].get("r"))
889  for x in sids if x in srcToMatch.keys()])
890  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
891  if s in srcToMatch.keys()])
892  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
893  if s in srcToMatch.keys()])
894  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
895  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
896  & (colors <= self.sourceSelector.config.grMax))
897  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
898  rms1Long = IqrToSigma*(
899  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
900  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
901  - numpy.percentile(dlat[idx1], 25))
902  rms2Long = IqrToSigma*(
903  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
904  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
905  - numpy.percentile(dlat[idx2], 25))
906  rms3Long = IqrToSigma*(
907  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
908  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
909  - numpy.percentile(dlat[idx3], 25))
910  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f",
911  numpy.median(dlong[idx1]), rms1Long,
912  numpy.median(dlat[idx1]), rms1Lat)
913  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f",
914  numpy.median(dlong[idx2]), rms2Long,
915  numpy.median(dlat[idx2]), rms2Lat)
916  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f",
917  numpy.median(dlong[idx3]), rms3Long,
918  numpy.median(dlat[idx3]), rms3Lat)
919 
920  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
921  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
922  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
923  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
924  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
925  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
926 
927  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
928  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
929  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
930  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
931  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
932  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
933 
934  # warp template exposure to match exposure,
935  # PSF match template exposure to exposure,
936  # then return the difference
937 
938  # Return warped template... Construct sourceKernelCand list after subtract
939  self.log.info("Subtracting images")
940  subtractRes = self.subtract.subtractExposures(
941  templateExposure=templateExposure,
942  scienceExposure=exposure,
943  candidateList=kernelSources,
944  convolveTemplate=self.config.convolveTemplate,
945  doWarping=not self.config.doUseRegister
946  )
947  if self.config.useScoreImageDetection:
948  scoreExposure = subtractRes.subtractedExposure
949  else:
950  subtractedExposure = subtractRes.subtractedExposure
951 
952  if self.config.doDetection:
953  self.log.info("Computing diffim PSF")
954 
955  # Get Psf from the appropriate input image if it doesn't exist
956  if subtractedExposure is not None and not subtractedExposure.hasPsf():
957  if self.config.convolveTemplate:
958  subtractedExposure.setPsf(exposure.getPsf())
959  else:
960  subtractedExposure.setPsf(templateExposure.getPsf())
961 
962  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
963  # thus it may have already been decorrelated. Thus, we do not decorrelate if
964  # doSubtract is False.
965 
966  # NOTE: At this point doSubtract == True
967  if self.config.doDecorrelation and self.config.doSubtract:
968  preConvKernel = None
969  if self.config.useGaussianForPreConvolution:
970  preConvKernel = preConvPsf.getLocalKernel()
971  if self.config.useScoreImageDetection:
972  scoreExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
973  scoreExposure,
974  subtractRes.psfMatchingKernel,
975  spatiallyVarying=self.config.doSpatiallyVarying,
976  preConvKernel=preConvKernel,
977  templateMatched=True,
978  preConvMode=True).correctedExposure
979  # Note that the subtracted exposure is always decorrelated,
980  # even if the score image is used for detection
981  subtractedExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
982  subtractedExposure,
983  subtractRes.psfMatchingKernel,
984  spatiallyVarying=self.config.doSpatiallyVarying,
985  preConvKernel=None,
986  templateMatched=self.config.convolveTemplate,
987  preConvMode=False).correctedExposure
988  # END (if subtractAlgorithm == 'AL')
989  # END (if self.config.doSubtract)
990  if self.config.doDetection:
991  self.log.info("Running diaSource detection")
992 
993  # subtractedExposure - reserved for task return value
994  # in zogy, it is always the proper difference image
995  # in AL, it may be (yet) pre-convolved and/or decorrelated
996  #
997  # detectionExposure - controls which exposure to use for detection
998  # in-place modifications will appear in task return
999  if self.config.useScoreImageDetection:
1000  # zogy with score image detection enabled
1001  self.log.info("Detection, diffim rescaling and measurements are "
1002  "on AL likelihood or Zogy score image.")
1003  detectionExposure = scoreExposure
1004  else:
1005  # AL or zogy with no score image detection
1006  detectionExposure = subtractedExposure
1007 
1008  # Rescale difference image variance plane
1009  if self.config.doScaleDiffimVariance:
1010  self.log.info("Rescaling diffim variance")
1011  diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
1012  self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor)
1013  self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
1014 
1015  # Erase existing detection mask planes
1016  mask = detectionExposure.getMaskedImage().getMask()
1017  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
1018 
1019  table = afwTable.SourceTable.make(self.schema, idFactory)
1020  table.setMetadata(self.algMetadata)
1021  results = self.detection.run(
1022  table=table,
1023  exposure=detectionExposure,
1024  doSmooth=not self.config.useScoreImageDetection
1025  )
1026 
1027  if self.config.doMerge:
1028  fpSet = results.fpSets.positive
1029  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1030  self.config.growFootprint, False)
1031  diaSources = afwTable.SourceCatalog(table)
1032  fpSet.makeSources(diaSources)
1033  self.log.info("Merging detections into %d sources", len(diaSources))
1034  else:
1035  diaSources = results.sources
1036  # Inject skySources before measurement.
1037  if self.config.doSkySources:
1038  skySourceFootprints = self.skySources.run(
1039  mask=detectionExposure.mask,
1040  seed=detectionExposure.getInfo().getVisitInfo().getExposureId())
1041  if skySourceFootprints:
1042  for foot in skySourceFootprints:
1043  s = diaSources.addNew()
1044  s.setFootprint(foot)
1045  s.set(self.skySourceKey, True)
1046 
1047  if self.config.doMeasurement:
1048  newDipoleFitting = self.config.doDipoleFitting
1049  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1050  if not newDipoleFitting:
1051  # Just fit dipole in diffim
1052  self.measurement.run(diaSources, detectionExposure)
1053  else:
1054  # Use (matched) template and science image (if avail.) to constrain dipole fitting
1055  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1056  self.measurement.run(diaSources, detectionExposure, exposure,
1057  subtractRes.matchedExposure)
1058  else:
1059  self.measurement.run(diaSources, detectionExposure, exposure)
1060  if self.config.doApCorr:
1061  self.applyApCorr.run(
1062  catalog=diaSources,
1063  apCorrMap=detectionExposure.getInfo().getApCorrMap()
1064  )
1065 
1066  if self.config.doForcedMeasurement:
1067  # Run forced psf photometry on the PVI at the diaSource locations.
1068  # Copy the measured flux and error into the diaSource.
1069  forcedSources = self.forcedMeasurement.generateMeasCat(
1070  exposure, diaSources, detectionExposure.getWcs())
1071  self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1072  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1073  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1074  "ip_diffim_forced_PsfFlux_instFlux", True)
1075  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1076  "ip_diffim_forced_PsfFlux_instFluxErr", True)
1077  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1078  "ip_diffim_forced_PsfFlux_area", True)
1079  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1080  "ip_diffim_forced_PsfFlux_flag", True)
1081  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1082  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1083  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1084  "ip_diffim_forced_PsfFlux_flag_edge", True)
1085  for diaSource, forcedSource in zip(diaSources, forcedSources):
1086  diaSource.assign(forcedSource, mapper)
1087 
1088  # Match with the calexp sources if possible
1089  if self.config.doMatchSources:
1090  if selectSources is not None:
1091  # Create key,val pair where key=diaSourceId and val=sourceId
1092  matchRadAsec = self.config.diaSourceMatchRadius
1093  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1094 
1095  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1096  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1097  srcMatch in srcMatches])
1098  self.log.info("Matched %d / %d diaSources to sources",
1099  len(srcMatchDict), len(diaSources))
1100  else:
1101  self.log.warning("Src product does not exist; cannot match with diaSources")
1102  srcMatchDict = {}
1103 
1104  # Create key,val pair where key=diaSourceId and val=refId
1105  refAstromConfig = AstrometryConfig()
1106  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1107  refAstrometer = AstrometryTask(refAstromConfig)
1108  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1109  refMatches = astromRet.matches
1110  if refMatches is None:
1111  self.log.warning("No diaSource matches with reference catalog")
1112  refMatchDict = {}
1113  else:
1114  self.log.info("Matched %d / %d diaSources to reference catalog",
1115  len(refMatches), len(diaSources))
1116  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1117  refMatch in refMatches])
1118 
1119  # Assign source Ids
1120  for diaSource in diaSources:
1121  sid = diaSource.getId()
1122  if sid in srcMatchDict:
1123  diaSource.set("srcMatchId", srcMatchDict[sid])
1124  if sid in refMatchDict:
1125  diaSource.set("refMatchId", refMatchDict[sid])
1126 
1127  if self.config.doAddMetrics and self.config.doSelectSources:
1128  self.log.info("Evaluating metrics and control sample")
1129 
1130  kernelCandList = []
1131  for cell in subtractRes.kernelCellSet.getCellList():
1132  for cand in cell.begin(False): # include bad candidates
1133  kernelCandList.append(cand)
1134 
1135  # Get basis list to build control sample kernels
1136  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1137  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1138 
1139  controlCandList = (
1140  diffimTools.sourceTableToCandidateList(controlSources,
1141  subtractRes.warpedExposure, exposure,
1142  self.config.subtract.kernel.active,
1143  self.config.subtract.kernel.active.detectionConfig,
1144  self.log, doBuild=True, basisList=basisList))
1145 
1146  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1147  subtractRes.backgroundModel, dof=nparam)
1148  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1149  subtractRes.backgroundModel)
1150 
1151  if self.config.doDetection:
1152  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1153  else:
1154  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1155 
1156  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1157  return pipeBase.Struct(
1158  subtractedExposure=subtractedExposure,
1159  scoreExposure=scoreExposure,
1160  warpedExposure=subtractRes.warpedExposure,
1161  matchedExposure=subtractRes.matchedExposure,
1162  subtractRes=subtractRes,
1163  diaSources=diaSources,
1164  selectSources=selectSources
1165  )
1166 
1167  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1168  """Fit the relative astrometry between templateSources and selectSources
1169 
1170  Todo
1171  ----
1172 
1173  Remove this method. It originally fit a new WCS to the template before calling register.run
1174  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1175  It remains because a subtask overrides it.
1176  """
1177  results = self.register.run(templateSources, templateExposure.getWcs(),
1178  templateExposure.getBBox(), selectSources)
1179  return results
1180 
1181  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1182  """Make debug plots and displays.
1183 
1184  Todo
1185  ----
1186  Test and update for current debug display and slot names
1187  """
1188  import lsstDebug
1189  display = lsstDebug.Info(__name__).display
1190  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1191  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1192  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1193  showDipoles = lsstDebug.Info(__name__).showDipoles
1194  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1195  if display:
1196  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1197  if not maskTransparency:
1198  maskTransparency = 0
1199  disp.setMaskTransparency(maskTransparency)
1200 
1201  if display and showSubtracted:
1202  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1203  mi = subtractRes.subtractedExposure.getMaskedImage()
1204  x0, y0 = mi.getX0(), mi.getY0()
1205  with disp.Buffering():
1206  for s in diaSources:
1207  x, y = s.getX() - x0, s.getY() - y0
1208  ctype = "red" if s.get("flags_negative") else "yellow"
1209  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1210  or s.get("base_PixelFlags_flag_saturatedCenter")
1211  or s.get("base_PixelFlags_flag_crCenter")):
1212  ptype = "x"
1213  elif (s.get("base_PixelFlags_flag_interpolated")
1214  or s.get("base_PixelFlags_flag_saturated")
1215  or s.get("base_PixelFlags_flag_cr")):
1216  ptype = "+"
1217  else:
1218  ptype = "o"
1219  disp.dot(ptype, x, y, size=4, ctype=ctype)
1220  lsstDebug.frame += 1
1221 
1222  if display and showPixelResiduals and selectSources:
1223  nonKernelSources = []
1224  for source in selectSources:
1225  if source not in kernelSources:
1226  nonKernelSources.append(source)
1227 
1228  diUtils.plotPixelResiduals(exposure,
1229  subtractRes.warpedExposure,
1230  subtractRes.subtractedExposure,
1231  subtractRes.kernelCellSet,
1232  subtractRes.psfMatchingKernel,
1233  subtractRes.backgroundModel,
1234  nonKernelSources,
1235  self.subtract.config.kernel.active.detectionConfig,
1236  origVariance=False)
1237  diUtils.plotPixelResiduals(exposure,
1238  subtractRes.warpedExposure,
1239  subtractRes.subtractedExposure,
1240  subtractRes.kernelCellSet,
1241  subtractRes.psfMatchingKernel,
1242  subtractRes.backgroundModel,
1243  nonKernelSources,
1244  self.subtract.config.kernel.active.detectionConfig,
1245  origVariance=True)
1246  if display and showDiaSources:
1247  flagChecker = SourceFlagChecker(diaSources)
1248  isFlagged = [flagChecker(x) for x in diaSources]
1249  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1250  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1251  frame=lsstDebug.frame)
1252  lsstDebug.frame += 1
1253 
1254  if display and showDipoles:
1255  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1256  frame=lsstDebug.frame)
1257  lsstDebug.frame += 1
1258 
1259  def checkTemplateIsSufficient(self, templateExposure):
1260  """Raise NoWorkFound if template coverage < requiredTemplateFraction
1261 
1262  Parameters
1263  ----------
1264  templateExposure : `lsst.afw.image.ExposureF`
1265  The template exposure to check
1266 
1267  Raises
1268  ------
1269  NoWorkFound
1270  Raised if fraction of good pixels, defined as not having NO_DATA
1271  set, is less then the configured requiredTemplateFraction
1272  """
1273  # Count the number of pixels with the NO_DATA mask bit set
1274  # counting NaN pixels is insufficient because pixels without data are often intepolated over)
1275  pixNoData = numpy.count_nonzero(templateExposure.mask.array
1276  & templateExposure.mask.getPlaneBitMask('NO_DATA'))
1277  pixGood = templateExposure.getBBox().getArea() - pixNoData
1278  self.log.info("template has %d good pixels (%.1f%%)", pixGood,
1279  100*pixGood/templateExposure.getBBox().getArea())
1280 
1281  if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction:
1282  message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
1283  "To force subtraction, set config requiredTemplateFraction=0." % (
1284  100*pixGood/templateExposure.getBBox().getArea(),
1285  100*self.config.requiredTemplateFraction))
1286  raise pipeBase.NoWorkFound(message)
1287 
1288  def _getConfigName(self):
1289  """Return the name of the config dataset
1290  """
1291  return "%sDiff_config" % (self.config.coaddName,)
1292 
1293  def _getMetadataName(self):
1294  """Return the name of the metadata dataset
1295  """
1296  return "%sDiff_metadata" % (self.config.coaddName,)
1297 
1298  def getSchemaCatalogs(self):
1299  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1300  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1301 
1302  @classmethod
1303  def _makeArgumentParser(cls):
1304  """Create an argument parser
1305  """
1306  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1307  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1308  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1309  help="Template data ID in case of calexp template,"
1310  " e.g. --templateId visit=6789")
1311  return parser
1312 
1313 
1314 class ImageDifferenceFromTemplateConnections(ImageDifferenceTaskConnections,
1315  defaultTemplates={"coaddName": "goodSeeing"}
1316  ):
1317  inputTemplate = pipeBase.connectionTypes.Input(
1318  doc=("Warped template produced by GetMultiTractCoaddTemplate"),
1319  dimensions=("instrument", "visit", "detector"),
1320  storageClass="ExposureF",
1321  name="{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}",
1322  )
1323 
1324  def __init__(self, *, config=None):
1325  super().__init__(config=config)
1326  # ImageDifferenceConnections will have removed one of these.
1327  # Make sure they're both gone, because no coadds are needed.
1328  if "coaddExposures" in self.inputs:
1329  self.inputs.remove("coaddExposures")
1330  if "dcrCoadds" in self.inputs:
1331  self.inputs.remove("dcrCoadds")
1332 
1333 
1334 class ImageDifferenceFromTemplateConfig(ImageDifferenceConfig,
1335  pipelineConnections=ImageDifferenceFromTemplateConnections):
1336  pass
1337 
1338 
1339 class ImageDifferenceFromTemplateTask(ImageDifferenceTask):
1340  ConfigClass = ImageDifferenceFromTemplateConfig
1341  _DefaultName = "imageDifference"
1342 
1343  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
1344  def runQuantum(self, butlerQC, inputRefs, outputRefs):
1345  inputs = butlerQC.get(inputRefs)
1346  self.log.info("Processing %s", butlerQC.quantum.dataId)
1347  self.checkTemplateIsSufficient(inputs['inputTemplate'])
1348  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
1349  returnMaxBits=True)
1350  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
1351 
1352  outputs = self.run(exposure=inputs['exposure'],
1353  templateExposure=inputs['inputTemplate'],
1354  idFactory=idFactory)
1355 
1356  # Consistency with runDataref gen2 handling
1357  if outputs.diaSources is None:
1358  del outputs.diaSources
1359  butlerQC.put(outputs, outputRefs)
1360 
1361 
1362 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1363  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1364  doc="Shift stars going into RegisterTask by this amount")
1365  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1366  doc="Perturb stars going into RegisterTask by this amount")
1367 
1368  def setDefaults(self):
1369  ImageDifferenceConfig.setDefaults(self)
1370  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1371 
1372 
1373 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1374  """!Image difference Task used in the Winter 2013 data challege.
1375  Enables testing the effects of registration shifts and scatter.
1376 
1377  For use with winter 2013 simulated images:
1378  Use --templateId visit=88868666 for sparse data
1379  --templateId visit=22222200 for dense data (g)
1380  --templateId visit=11111100 for dense data (i)
1381  """
1382  ConfigClass = Winter2013ImageDifferenceConfig
1383  _DefaultName = "winter2013ImageDifference"
1384 
1385  def __init__(self, **kwargs):
1386  ImageDifferenceTask.__init__(self, **kwargs)
1387 
1388  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1389  """Fit the relative astrometry between templateSources and selectSources"""
1390  if self.config.winter2013WcsShift > 0.0:
1391  offset = geom.Extent2D(self.config.winter2013WcsShift,
1392  self.config.winter2013WcsShift)
1393  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1394  for source in templateSources:
1395  centroid = source.get(cKey)
1396  source.set(cKey, centroid + offset)
1397  elif self.config.winter2013WcsRms > 0.0:
1398  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1399  for source in templateSources:
1400  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1401  self.config.winter2013WcsRms*numpy.random.normal())
1402  centroid = source.get(cKey)
1403  source.set(cKey, centroid + offset)
1404 
1405  results = self.register.run(templateSources, templateExposure.getWcs(),
1406  templateExposure.getBBox(), selectSources)
1407  return results
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)