lsst.pipe.tasks  21.0.0-120-g57749b33+77c36da417
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 from lsst.skymap import BaseSkyMap
47 from lsst.obs.base import ExposureIdInfo
48 
49 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
50 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
51 IqrToSigma = 0.741
52 
53 
54 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
55  dimensions=("instrument", "visit", "detector", "skymap"),
56  defaultTemplates={"coaddName": "deep",
57  "skyMapName": "deep",
58  "warpTypeSuffix": "",
59  "fakesType": ""}):
60 
61  exposure = pipeBase.connectionTypes.Input(
62  doc="Input science exposure to subtract from.",
63  dimensions=("instrument", "visit", "detector"),
64  storageClass="ExposureF",
65  name="{fakesType}calexp"
66  )
67 
68  # TODO DM-22953
69  # kernelSources = pipeBase.connectionTypes.Input(
70  # doc="Source catalog produced in calibrate task for kernel candidate sources",
71  # name="src",
72  # storageClass="SourceCatalog",
73  # dimensions=("instrument", "visit", "detector"),
74  # )
75 
76  skyMap = pipeBase.connectionTypes.Input(
77  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
78  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
79  dimensions=("skymap", ),
80  storageClass="SkyMap",
81  )
82  coaddExposures = pipeBase.connectionTypes.Input(
83  doc="Input template to match and subtract from the exposure",
84  dimensions=("tract", "patch", "skymap", "band"),
85  storageClass="ExposureF",
86  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
87  multiple=True,
88  deferLoad=True
89  )
90  dcrCoadds = pipeBase.connectionTypes.Input(
91  doc="Input DCR template to match and subtract from the exposure",
92  name="{fakesType}dcrCoadd{warpTypeSuffix}",
93  storageClass="ExposureF",
94  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
95  multiple=True,
96  deferLoad=True
97  )
98  outputSchema = pipeBase.connectionTypes.InitOutput(
99  doc="Schema (as an example catalog) for output DIASource catalog.",
100  storageClass="SourceCatalog",
101  name="{fakesType}{coaddName}Diff_diaSrc_schema",
102  )
103  # TODO DM-29965: Currently the AL likelihood image is not a separate data product
104  subtractedExposure = pipeBase.connectionTypes.Output(
105  doc="Output AL difference or Zogy proper difference image",
106  dimensions=("instrument", "visit", "detector"),
107  storageClass="ExposureF",
108  name="{fakesType}{coaddName}Diff_differenceExp",
109  )
110  scoreExposure = pipeBase.connectionTypes.Output(
111  doc="Output Zogy score (likelihood) image",
112  dimensions=("instrument", "visit", "detector"),
113  storageClass="ExposureF",
114  name="{fakesType}{coaddName}Diff_scoreExp",
115  )
116  warpedExposure = pipeBase.connectionTypes.Output(
117  doc="Warped template used to create `subtractedExposure`.",
118  dimensions=("instrument", "visit", "detector"),
119  storageClass="ExposureF",
120  name="{fakesType}{coaddName}Diff_warpedExp",
121  )
122  matchedExposure = pipeBase.connectionTypes.Output(
123  doc="Warped template used to create `subtractedExposure`.",
124  dimensions=("instrument", "visit", "detector"),
125  storageClass="ExposureF",
126  name="{fakesType}{coaddName}Diff_matchedExp",
127  )
128  diaSources = pipeBase.connectionTypes.Output(
129  doc="Output detected diaSources on the difference image",
130  dimensions=("instrument", "visit", "detector"),
131  storageClass="SourceCatalog",
132  name="{fakesType}{coaddName}Diff_diaSrc",
133  )
134 
135  def __init__(self, *, config=None):
136  super().__init__(config=config)
137  if config.coaddName == 'dcr':
138  self.inputs.remove("coaddExposures")
139  else:
140  self.inputs.remove("dcrCoadds")
141  if not config.doWriteSubtractedExp:
142  self.outputs.remove("subtractedExposure")
143  if not config.doWriteScoreExp:
144  self.outputs.remove("scoreExposure")
145  if not config.doWriteWarpedExp:
146  self.outputs.remove("warpedExposure")
147  if not config.doWriteMatchedExp:
148  self.outputs.remove("matchedExposure")
149  if not config.doWriteSources:
150  self.outputs.remove("diaSources")
151 
152  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
153  # Make kernelSources optional
154 
155 
156 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
157  pipelineConnections=ImageDifferenceTaskConnections):
158  """Config for ImageDifferenceTask
159  """
160  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
161  doc="Add background to calexp before processing it. "
162  "Useful as ipDiffim does background matching.")
163  doUseRegister = pexConfig.Field(dtype=bool, default=False,
164  doc="Re-compute astrometry on the template. "
165  "Use image-to-image registration to align template with "
166  "science image (AL only).")
167  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
168  doc="Writing debugging data for doUseRegister")
169  doSelectSources = pexConfig.Field(dtype=bool, default=False,
170  doc="Select stars to use for kernel fitting (AL only)")
171  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
172  doc="Select stars of extreme color as part "
173  "of the control sample (AL only)")
174  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
175  doc="Select stars that are variable to be part "
176  "of the control sample (AL only)")
177  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
178  doPreConvolve = pexConfig.Field(dtype=bool, default=False,
179  doc="Convolve science image by its PSF before PSF-matching (AL only)."
180  " The difference image becomes a likelihood image.")
181  useScoreImageDetection = pexConfig.Field(
182  dtype=bool, default=False, doc="Calculate and detect sources on the Zogy score image (Zogy only).")
183  doWriteScoreExp = pexConfig.Field(
184  dtype=bool, default=False, doc="Write score exposure (Zogy only) ?")
185  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
186  doc="Scale variance of the template before PSF matching")
187  doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
188  doc="Scale variance of the diffim before PSF matching. "
189  "You may do either this or template variance scaling, "
190  "or neither. (Doing both is a waste of CPU.)")
191  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
192  doc="Use a simple gaussian PSF model for pre-convolution "
193  "(else use fit PSF)? Ignored if doPreConvolve false.")
194  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
195  doDecorrelation = pexConfig.Field(dtype=bool, default=True,
196  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
197  "kernel convolution (AL only)? If True, also update the diffim PSF.")
198  doMerge = pexConfig.Field(dtype=bool, default=True,
199  doc="Merge positive and negative diaSources with grow radius "
200  "set by growFootprint")
201  doMatchSources = pexConfig.Field(dtype=bool, default=False,
202  doc="Match diaSources with input calexp sources and ref catalog sources")
203  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
204  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
205  doForcedMeasurement = pexConfig.Field(
206  dtype=bool,
207  default=True,
208  doc="Force photometer diaSource locations on PVI?")
209  doWriteSubtractedExp = pexConfig.Field(
210  dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
211  doWriteWarpedExp = pexConfig.Field(
212  dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
213  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
214  doc="Write warped and PSF-matched template coadd exposure?")
215  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
216  doAddMetrics = pexConfig.Field(dtype=bool, default=False,
217  doc="Add columns to the source table to hold analysis metrics?")
218 
219  coaddName = pexConfig.Field(
220  doc="coadd name: typically one of deep, goodSeeing, or dcr",
221  dtype=str,
222  default="deep",
223  )
224  convolveTemplate = pexConfig.Field(
225  doc="Which image gets convolved (default = template)",
226  dtype=bool,
227  default=True
228  )
229  refObjLoader = pexConfig.ConfigurableField(
230  target=LoadIndexedReferenceObjectsTask,
231  doc="reference object loader",
232  )
233  astrometer = pexConfig.ConfigurableField(
234  target=AstrometryTask,
235  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
236  )
237  sourceSelector = pexConfig.ConfigurableField(
238  target=ObjectSizeStarSelectorTask,
239  doc="Source selection algorithm",
240  )
241  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
242  decorrelate = pexConfig.ConfigurableField(
243  target=DecorrelateALKernelSpatialTask,
244  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
245  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
246  "default of 5.5).",
247  )
248  # Old style ImageMapper grid. ZogyTask has its own grid option
249  doSpatiallyVarying = pexConfig.Field(
250  dtype=bool,
251  default=False,
252  doc="Perform A&L decorrelation on a grid across the "
253  "image in order to allow for spatial variations. Zogy does not use this option."
254  )
255  detection = pexConfig.ConfigurableField(
256  target=SourceDetectionTask,
257  doc="Low-threshold detection for final measurement",
258  )
259  measurement = pexConfig.ConfigurableField(
260  target=DipoleFitTask,
261  doc="Enable updated dipole fitting method",
262  )
263  doApCorr = lsst.pex.config.Field(
264  dtype=bool,
265  default=True,
266  doc="Run subtask to apply aperture corrections"
267  )
268  applyApCorr = lsst.pex.config.ConfigurableField(
269  target=ApplyApCorrTask,
270  doc="Subtask to apply aperture corrections"
271  )
272  forcedMeasurement = pexConfig.ConfigurableField(
273  target=ForcedMeasurementTask,
274  doc="Subtask to force photometer PVI at diaSource location.",
275  )
276  getTemplate = pexConfig.ConfigurableField(
277  target=GetCoaddAsTemplateTask,
278  doc="Subtask to retrieve template exposure and sources",
279  )
280  scaleVariance = pexConfig.ConfigurableField(
281  target=ScaleVarianceTask,
282  doc="Subtask to rescale the variance of the template "
283  "to the statistically expected level"
284  )
285  controlStepSize = pexConfig.Field(
286  doc="What step size (every Nth one) to select a control sample from the kernelSources",
287  dtype=int,
288  default=5
289  )
290  controlRandomSeed = pexConfig.Field(
291  doc="Random seed for shuffing the control sample",
292  dtype=int,
293  default=10
294  )
295  register = pexConfig.ConfigurableField(
296  target=RegisterTask,
297  doc="Task to enable image-to-image image registration (warping)",
298  )
299  kernelSourcesFromRef = pexConfig.Field(
300  doc="Select sources to measure kernel from reference catalog if True, template if false",
301  dtype=bool,
302  default=False
303  )
304  templateSipOrder = pexConfig.Field(
305  dtype=int, default=2,
306  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
307  )
308  growFootprint = pexConfig.Field(
309  dtype=int, default=2,
310  doc="Grow positive and negative footprints by this amount before merging"
311  )
312  diaSourceMatchRadius = pexConfig.Field(
313  dtype=float, default=0.5,
314  doc="Match radius (in arcseconds) for DiaSource to Source association"
315  )
316  requiredTemplateFraction = pexConfig.Field(
317  dtype=float, default=0.1,
318  doc="Do not attempt to run task if template covers less than this fraction of pixels."
319  "Setting to 0 will always attempt image subtraction"
320  )
321  doSkySources = pexConfig.Field(
322  dtype=bool,
323  default=False,
324  doc="Generate sky sources?",
325  )
326  skySources = pexConfig.ConfigurableField(
327  target=SkyObjectsTask,
328  doc="Generate sky sources",
329  )
330 
331  def setDefaults(self):
332  # defaults are OK for catalog and diacatalog
333 
334  self.subtract['al'].kernel.name = "AL"
335  self.subtract['al'].kernel.active.fitForBackground = True
336  self.subtract['al'].kernel.active.spatialKernelOrder = 1
337  self.subtract['al'].kernel.active.spatialBgOrder = 2
338 
339  # DiaSource Detection
340  self.detection.thresholdPolarity = "both"
341  self.detection.thresholdValue = 5.0
342  self.detection.reEstimateBackground = False
343  self.detection.thresholdType = "pixel_stdev"
344 
345  # Add filtered flux measurement, the correct measurement for pre-convolved images.
346  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
347  # To change that you must modify algorithms.names in the task's applyOverrides method,
348  # after the user has set doPreConvolve.
349  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
350  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
351  'base_LocalWcs']
352 
353  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
354  self.forcedMeasurement.copyColumns = {
355  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
356  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
357  self.forcedMeasurement.slots.shape = None
358 
359  # For shuffling the control sample
360  random.seed(self.controlRandomSeed)
361 
362  def validate(self):
363  pexConfig.Config.validate(self)
364  if not self.doSubtract and not self.doDetection:
365  raise ValueError("Either doSubtract or doDetection must be enabled.")
366  if self.doMeasurement and not self.doDetection:
367  raise ValueError("Cannot run source measurement without source detection.")
368  if self.doMerge and not self.doDetection:
369  raise ValueError("Cannot run source merging without source detection.")
370  if self.doSkySources and not self.doDetection:
371  raise ValueError("Cannot run sky source creation without source detection.")
372  if self.doUseRegister and not self.doSelectSources:
373  raise ValueError("doUseRegister=True and doSelectSources=False. "
374  "Cannot run RegisterTask without selecting sources.")
375  if hasattr(self.getTemplate, "coaddName"):
376  if self.getTemplate.coaddName != self.coaddName:
377  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
378  if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
379  raise ValueError("Scaling the diffim variance and scaling the template variance "
380  "are both set. Please choose one or the other.")
381  # We cannot allow inconsistencies that would lead to None or not available output products
382  if self.subtract.name == 'zogy':
383  if self.doWriteScoreExp and not self.useScoreImageDetection:
384  raise ValueError("doWriteScoreExp=True and useScoreImageDetection=False "
385  "is not supported. Score image is not calculated.")
386  if self.doWriteMatchedExp:
387  raise ValueError("doWriteMatchedExp=True Matched exposure is not "
388  "calculated in zogy subtraction.")
389  if self.doAddMetrics:
390  raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
391  if self.doDecorrelation:
392  raise ValueError(
393  "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
394  if self.doPreConvolve:
395  raise ValueError(
396  "doPreConvolve=True Pre-convolution is not a zogy option.")
397  if self.doSelectSources:
398  raise ValueError(
399  "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
400  else:
401  if self.useScoreImageDetection:
402  raise ValueError("useScoreImageDetection=True Score exposure does not "
403  "exist in AL subtraction.")
404  if self.doWriteScoreExp: # Ensure output is not None
405  raise ValueError("doWriteScoreExp=True Score exposure does not exist in AL subtraction.")
406  if self.doAddMetrics and not self.doSubtract:
407  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
408  if self.doPreConvolve and self.doDecorrelation:
409  raise NotImplementedError(
410  "doPreConvolve=True and doDecorrelation=True "
411  "The decorrelation afterburner cannot handle pre-convolved exposures.")
412 
413 
414 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
415 
416  @staticmethod
417  def getTargetList(parsedCmd, **kwargs):
418  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
419  **kwargs)
420 
421 
422 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
423  """Subtract an image from a template and measure the result
424  """
425  ConfigClass = ImageDifferenceConfig
426  RunnerClass = ImageDifferenceTaskRunner
427  _DefaultName = "imageDifference"
428 
429  def __init__(self, butler=None, **kwargs):
430  """!Construct an ImageDifference Task
431 
432  @param[in] butler Butler object to use in constructing reference object loaders
433  """
434  super().__init__(**kwargs)
435  self.makeSubtask("getTemplate")
436 
437  self.makeSubtask("subtract")
438 
439  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
440  self.makeSubtask("decorrelate")
441 
442  if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
443  self.makeSubtask("scaleVariance")
444 
445  if self.config.doUseRegister:
446  self.makeSubtask("register")
447  self.schema = afwTable.SourceTable.makeMinimalSchema()
448 
449  if self.config.doSelectSources:
450  self.makeSubtask("sourceSelector")
451  if self.config.kernelSourcesFromRef:
452  self.makeSubtask('refObjLoader', butler=butler)
453  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
454 
455  self.algMetadata = dafBase.PropertyList()
456  if self.config.doDetection:
457  self.makeSubtask("detection", schema=self.schema)
458  if self.config.doMeasurement:
459  self.makeSubtask("measurement", schema=self.schema,
460  algMetadata=self.algMetadata)
461  if self.config.doApCorr:
462  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
463  if self.config.doForcedMeasurement:
464  self.schema.addField(
465  "ip_diffim_forced_PsfFlux_instFlux", "D",
466  "Forced PSF flux measured on the direct image.",
467  units="count")
468  self.schema.addField(
469  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
470  "Forced PSF flux error measured on the direct image.",
471  units="count")
472  self.schema.addField(
473  "ip_diffim_forced_PsfFlux_area", "F",
474  "Forced PSF flux effective area of PSF.",
475  units="pixel")
476  self.schema.addField(
477  "ip_diffim_forced_PsfFlux_flag", "Flag",
478  "Forced PSF flux general failure flag.")
479  self.schema.addField(
480  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
481  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
482  self.schema.addField(
483  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
484  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
485  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
486  if self.config.doMatchSources:
487  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
488  self.schema.addField("srcMatchId", "L", "unique id of source match")
489  if self.config.doSkySources:
490  self.makeSubtask("skySources")
491  self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
492 
493  # initialize InitOutputs
494  self.outputSchema = afwTable.SourceCatalog(self.schema)
495  self.outputSchema.getTable().setMetadata(self.algMetadata)
496 
497  @staticmethod
498  def makeIdFactory(expId, expBits):
499  """Create IdFactory instance for unique 64 bit diaSource id-s.
500 
501  Parameters
502  ----------
503  expId : `int`
504  Exposure id.
505 
506  expBits: `int`
507  Number of used bits in ``expId``.
508 
509  Note
510  ----
511  The diasource id-s consists of the ``expId`` stored fixed in the highest value
512  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
513  low value end of the integer.
514 
515  Returns
516  -------
517  idFactory: `lsst.afw.table.IdFactory`
518  """
519  return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
520 
521  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
522  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
523  inputRefs: pipeBase.InputQuantizedConnection,
524  outputRefs: pipeBase.OutputQuantizedConnection):
525  inputs = butlerQC.get(inputRefs)
526  self.log.info(f"Processing {butlerQC.quantum.dataId}")
527  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
528  returnMaxBits=True)
529  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
530  if self.config.coaddName == 'dcr':
531  templateExposures = inputRefs.dcrCoadds
532  else:
533  templateExposures = inputRefs.coaddExposures
534  templateStruct = self.getTemplate.runQuantum(
535  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
536  )
537 
538  if templateStruct.area/inputs['exposure'].getBBox().getArea() < self.config.requiredTemplateFraction:
539  message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
540  "To force subtraction, set config requiredTemplateFraction=0." % (
541  100*templateStruct.area/inputs['exposure'].getBBox().getArea(),
542  100*self.config.requiredTemplateFraction))
543  raise pipeBase.NoWorkFound(message)
544  else:
545  outputs = self.run(exposure=inputs['exposure'],
546  templateExposure=templateStruct.exposure,
547  idFactory=idFactory)
548  # Consistency with runDataref gen2 handling
549  if outputs.diaSources is None:
550  del outputs.diaSources
551  butlerQC.put(outputs, outputRefs)
552 
553  @pipeBase.timeMethod
554  def runDataRef(self, sensorRef, templateIdList=None):
555  """Subtract an image from a template coadd and measure the result.
556 
557  Data I/O wrapper around `run` using the butler in Gen2.
558 
559  Parameters
560  ----------
561  sensorRef : `lsst.daf.persistence.ButlerDataRef`
562  Sensor-level butler data reference, used for the following data products:
563 
564  Input only:
565  - calexp
566  - psf
567  - ccdExposureId
568  - ccdExposureId_bits
569  - self.config.coaddName + "Coadd_skyMap"
570  - self.config.coaddName + "Coadd"
571  Input or output, depending on config:
572  - self.config.coaddName + "Diff_subtractedExp"
573  Output, depending on config:
574  - self.config.coaddName + "Diff_matchedExp"
575  - self.config.coaddName + "Diff_src"
576 
577  Returns
578  -------
579  results : `lsst.pipe.base.Struct`
580  Returns the Struct by `run`.
581  """
582  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
583  subtractedExposure = None
584  selectSources = None
585  calexpBackgroundExposure = None
586  self.log.info("Processing %s" % (sensorRef.dataId))
587 
588  # We make one IdFactory that will be used by both icSrc and src datasets;
589  # I don't know if this is the way we ultimately want to do things, but at least
590  # this ensures the source IDs are fully unique.
591  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
592  expBits=sensorRef.get("ccdExposureId_bits"))
593  if self.config.doAddCalexpBackground:
594  calexpBackgroundExposure = sensorRef.get("calexpBackground")
595 
596  # Retrieve the science image we wish to analyze
597  exposure = sensorRef.get("calexp", immediate=True)
598 
599  # Retrieve the template image
600  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
601 
602  if sensorRef.datasetExists("src"):
603  self.log.info("Source selection via src product")
604  # Sources already exist; for data release processing
605  selectSources = sensorRef.get("src")
606 
607  if not self.config.doSubtract and self.config.doDetection:
608  # If we don't do subtraction, we need the subtracted exposure from the repo
609  subtractedExposure = sensorRef.get(subtractedExposureName)
610  # Both doSubtract and doDetection cannot be False
611 
612  results = self.run(exposure=exposure,
613  selectSources=selectSources,
614  templateExposure=template.exposure,
615  templateSources=template.sources,
616  idFactory=idFactory,
617  calexpBackgroundExposure=calexpBackgroundExposure,
618  subtractedExposure=subtractedExposure)
619 
620  if self.config.doWriteSources and results.diaSources is not None:
621  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
622  if self.config.doWriteWarpedExp:
623  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
624  if self.config.doWriteMatchedExp:
625  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
626  if self.config.doAddMetrics and self.config.doSelectSources:
627  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
628  if self.config.doWriteSubtractedExp:
629  sensorRef.put(results.subtractedExposure, subtractedExposureName)
630  if self.config.doWriteScoreExp:
631  sensorRef.put(results.scoreExposure, self.config.coaddName + "Diff_scoreExp")
632  return results
633 
634  @pipeBase.timeMethod
635  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
636  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
637  """PSF matches, subtract two images and perform detection on the difference image.
638 
639  Parameters
640  ----------
641  exposure : `lsst.afw.image.ExposureF`, optional
642  The science exposure, the minuend in the image subtraction.
643  Can be None only if ``config.doSubtract==False``.
644  selectSources : `lsst.afw.table.SourceCatalog`, optional
645  Identified sources on the science exposure. This catalog is used to
646  select sources in order to perform the AL PSF matching on stamp images
647  around them. The selection steps depend on config options and whether
648  ``templateSources`` and ``matchingSources`` specified.
649  templateExposure : `lsst.afw.image.ExposureF`, optional
650  The template to be subtracted from ``exposure`` in the image subtraction.
651  ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
652  The template exposure should cover the same sky area as the science exposure.
653  It is either a stich of patches of a coadd skymap image or a calexp
654  of the same pointing as the science exposure. Can be None only
655  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
656  templateSources : `lsst.afw.table.SourceCatalog`, optional
657  Identified sources on the template exposure.
658  idFactory : `lsst.afw.table.IdFactory`
659  Generator object to assign ids to detected sources in the difference image.
660  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
661  Background exposure to be added back to the science exposure
662  if ``config.doAddCalexpBackground==True``
663  subtractedExposure : `lsst.afw.image.ExposureF`, optional
664  If ``config.doSubtract==False`` and ``config.doDetection==True``,
665  performs the post subtraction source detection only on this exposure.
666  Otherwise should be None.
667 
668  Returns
669  -------
670  results : `lsst.pipe.base.Struct`
671  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
672  Difference image.
673  ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
674  The zogy score exposure, if calculated.
675  ``matchedExposure`` : `lsst.afw.image.ExposureF`
676  The matched PSF exposure.
677  ``subtractRes`` : `lsst.pipe.base.Struct`
678  The returned result structure of the ImagePsfMatchTask subtask.
679  ``diaSources`` : `lsst.afw.table.SourceCatalog`
680  The catalog of detected sources.
681  ``selectSources`` : `lsst.afw.table.SourceCatalog`
682  The input source catalog with optionally added Qa information.
683 
684  Notes
685  -----
686  The following major steps are included:
687 
688  - warp template coadd to match WCS of image
689  - PSF match image to warped template
690  - subtract image from PSF-matched, warped template
691  - detect sources
692  - measure sources
693 
694  For details about the image subtraction configuration modes
695  see `lsst.ip.diffim`.
696  """
697  subtractRes = None
698  controlSources = None
699  scoreExposure = None
700  diaSources = None
701  kernelSources = None
702  # We'll clone exposure if modified but will still need the original
703  exposureOrig = exposure
704 
705  if self.config.doAddCalexpBackground:
706  mi = exposure.getMaskedImage()
707  mi += calexpBackgroundExposure.getImage()
708 
709  if not exposure.hasPsf():
710  raise pipeBase.TaskError("Exposure has no psf")
711  sciencePsf = exposure.getPsf()
712 
713  if self.config.doSubtract:
714  if self.config.doScaleTemplateVariance:
715  self.log.info("Rescaling template variance")
716  templateVarFactor = self.scaleVariance.run(
717  templateExposure.getMaskedImage())
718  self.log.info("Template variance scaling factor: %.2f" % templateVarFactor)
719  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
720 
721  if self.config.subtract.name == 'zogy':
722  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
723  scoreExposure = subtractRes.scoreExp
724  subtractedExposure = subtractRes.diffExp
725  subtractRes.subtractedExposure = subtractedExposure
726  subtractRes.matchedExposure = None
727 
728  elif self.config.subtract.name == 'al':
729  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
730  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
731  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
732 
733  # if requested, convolve the science exposure with its PSF
734  # (properly, this should be a cross-correlation, but our code does not yet support that)
735  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
736  # else sigma of original science exposure
737  # TODO: DM-22762 This functional block should be moved into its own method
738  preConvPsf = None
739  if self.config.doPreConvolve:
740  convControl = afwMath.ConvolutionControl()
741  # cannot convolve in place, so need a new image anyway
742  srcMI = exposure.maskedImage
743  exposure = exposure.clone() # New deep copy
744  srcPsf = sciencePsf
745  if self.config.useGaussianForPreConvolution:
746  # convolve with a simplified PSF model: a double Gaussian
747  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
748  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
749  else:
750  # convolve with science exposure's PSF model
751  preConvPsf = srcPsf
752  afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
753  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
754  else:
755  scienceSigmaPost = scienceSigmaOrig
756 
757  # If requested, find and select sources from the image
758  # else, AL subtraction will do its own source detection
759  # TODO: DM-22762 This functional block should be moved into its own method
760  if self.config.doSelectSources:
761  if selectSources is None:
762  self.log.warn("Src product does not exist; running detection, measurement, selection")
763  # Run own detection and measurement; necessary in nightly processing
764  selectSources = self.subtract.getSelectSources(
765  exposure,
766  sigma=scienceSigmaPost,
767  doSmooth=not self.config.doPreConvolve,
768  idFactory=idFactory,
769  )
770 
771  if self.config.doAddMetrics:
772  # Number of basis functions
773 
774  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
775  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
776  targetFwhmPix=templateSigma*FwhmPerSigma))
777  # Modify the schema of all Sources
778  # DEPRECATED: This is a data dependent (nparam) output product schema
779  # outside the task constructor.
780  # NOTE: The pre-determination of nparam at this point
781  # may be incorrect as the template psf is warped later in
782  # ImagePsfMatchTask.matchExposures()
783  kcQa = KernelCandidateQa(nparam)
784  selectSources = kcQa.addToSchema(selectSources)
785  if self.config.kernelSourcesFromRef:
786  # match exposure sources to reference catalog
787  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
788  matches = astromRet.matches
789  elif templateSources:
790  # match exposure sources to template sources
791  mc = afwTable.MatchControl()
792  mc.findOnlyClosest = False
793  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
794  mc)
795  else:
796  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
797  "but template sources not available. Cannot match science "
798  "sources with template sources. Run process* on data from "
799  "which templates are built.")
800 
801  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
802  matches=matches).sourceCat
803  random.shuffle(kernelSources, random.random)
804  controlSources = kernelSources[::self.config.controlStepSize]
805  kernelSources = [k for i, k in enumerate(kernelSources)
806  if i % self.config.controlStepSize]
807 
808  if self.config.doSelectDcrCatalog:
809  redSelector = DiaCatalogSourceSelectorTask(
810  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
811  grMax=99.999))
812  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
813  controlSources.extend(redSources)
814 
815  blueSelector = DiaCatalogSourceSelectorTask(
816  DiaCatalogSourceSelectorConfig(grMin=-99.999,
817  grMax=self.sourceSelector.config.grMin))
818  blueSources = blueSelector.selectStars(exposure, selectSources,
819  matches=matches).starCat
820  controlSources.extend(blueSources)
821 
822  if self.config.doSelectVariableCatalog:
823  varSelector = DiaCatalogSourceSelectorTask(
824  DiaCatalogSourceSelectorConfig(includeVariable=True))
825  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
826  controlSources.extend(varSources)
827 
828  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
829  % (len(kernelSources), len(selectSources), len(controlSources)))
830 
831  allresids = {}
832  # TODO: DM-22762 This functional block should be moved into its own method
833  if self.config.doUseRegister:
834  self.log.info("Registering images")
835 
836  if templateSources is None:
837  # Run detection on the template, which is
838  # temporarily background-subtracted
839  # sigma of PSF of template image before warping
840  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
841  templateSources = self.subtract.getSelectSources(
842  templateExposure,
843  sigma=templateSigma,
844  doSmooth=True,
845  idFactory=idFactory
846  )
847 
848  # Third step: we need to fit the relative astrometry.
849  #
850  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
851  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
852  exposure.getWcs(), exposure.getBBox())
853  templateExposure = warpedExp
854 
855  # Create debugging outputs on the astrometric
856  # residuals as a function of position. Persistence
857  # not yet implemented; expected on (I believe) #2636.
858  if self.config.doDebugRegister:
859  # Grab matches to reference catalog
860  srcToMatch = {x.second.getId(): x.first for x in matches}
861 
862  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
863  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
864  sids = [m.first.getId() for m in wcsResults.matches]
865  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
866  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
867  m.second.get(inCentroidKey))) for m in wcsResults.matches]
868  allresids = dict(zip(sids, zip(positions, residuals)))
869 
870  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
871  wcsResults.wcs.pixelToSky(
872  m.second.get(inCentroidKey))) for m in wcsResults.matches]
873  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
874  + 2.5*numpy.log10(srcToMatch[x].get("r"))
875  for x in sids if x in srcToMatch.keys()])
876  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
877  if s in srcToMatch.keys()])
878  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
879  if s in srcToMatch.keys()])
880  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
881  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
882  & (colors <= self.sourceSelector.config.grMax))
883  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
884  rms1Long = IqrToSigma*(
885  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
886  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
887  - numpy.percentile(dlat[idx1], 25))
888  rms2Long = IqrToSigma*(
889  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
890  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
891  - numpy.percentile(dlat[idx2], 25))
892  rms3Long = IqrToSigma*(
893  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
894  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
895  - numpy.percentile(dlat[idx3], 25))
896  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
897  (numpy.median(dlong[idx1]), rms1Long,
898  numpy.median(dlat[idx1]), rms1Lat))
899  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
900  (numpy.median(dlong[idx2]), rms2Long,
901  numpy.median(dlat[idx2]), rms2Lat))
902  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
903  (numpy.median(dlong[idx3]), rms3Long,
904  numpy.median(dlat[idx3]), rms3Lat))
905 
906  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
907  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
908  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
909  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
910  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
911  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
912 
913  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
914  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
915  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
916  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
917  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
918  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
919 
920  # warp template exposure to match exposure,
921  # PSF match template exposure to exposure,
922  # then return the difference
923 
924  # Return warped template... Construct sourceKernelCand list after subtract
925  self.log.info("Subtracting images")
926  subtractRes = self.subtract.subtractExposures(
927  templateExposure=templateExposure,
928  scienceExposure=exposure,
929  candidateList=kernelSources,
930  convolveTemplate=self.config.convolveTemplate,
931  doWarping=not self.config.doUseRegister
932  )
933  subtractedExposure = subtractRes.subtractedExposure
934 
935  if self.config.doDetection:
936  self.log.info("Computing diffim PSF")
937 
938  # Get Psf from the appropriate input image if it doesn't exist
939  if not subtractedExposure.hasPsf():
940  if self.config.convolveTemplate:
941  subtractedExposure.setPsf(exposure.getPsf())
942  else:
943  subtractedExposure.setPsf(templateExposure.getPsf())
944 
945  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
946  # thus it may have already been decorrelated. Thus, we do not decorrelate if
947  # doSubtract is False.
948 
949  # NOTE: At this point doSubtract == True
950  if self.config.doDecorrelation and self.config.doSubtract:
951  preConvKernel = None
952  if preConvPsf is not None:
953  preConvKernel = preConvPsf.getLocalKernel()
954  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
955  subtractedExposure,
956  subtractRes.psfMatchingKernel,
957  spatiallyVarying=self.config.doSpatiallyVarying,
958  preConvKernel=preConvKernel,
959  templateMatched=self.config.convolveTemplate)
960  subtractedExposure = decorrResult.correctedExposure
961 
962  # END (if subtractAlgorithm == 'AL')
963  # END (if self.config.doSubtract)
964  if self.config.doDetection:
965  self.log.info("Running diaSource detection")
966 
967  # subtractedExposure - reserved for task return value
968  # in zogy, it is always the proper difference image
969  # in AL, it may be (yet) pre-convolved and/or decorrelated
970  #
971  # detectionExposure - controls which exposure to use for detection
972  # in-place modifications will appear in task return
973  if self.config.useScoreImageDetection:
974  # zogy with score image detection enabled
975  self.log.info("Detection, diffim rescaling and measurements are on Zogy score image.")
976  detectionExposure = scoreExposure
977  detectOnLikelihood = True
978  else:
979  # AL or zogy with no score image detection
980  detectionExposure = subtractedExposure
981  detectOnLikelihood = False
982  if self.config.doPreConvolve:
983  # In AL, the likelihood image is not a separate product, yet
984  self.log.info("Detection, diffim rescaling and measurements are on AL pre-convolved "
985  "difference (likelihood) image.")
986  detectOnLikelihood = True
987  else:
988  self.log.info("Detection, diffim rescaling and measurements are on "
989  "(proper) difference image.")
990 
991  # Rescale difference image variance plane
992  if self.config.doScaleDiffimVariance:
993  self.log.info("Rescaling diffim variance")
994  diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
995  self.log.info("Diffim variance scaling factor: %.2f" % diffimVarFactor)
996  self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
997 
998  # Erase existing detection mask planes
999  mask = detectionExposure.getMaskedImage().getMask()
1000  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
1001 
1002  table = afwTable.SourceTable.make(self.schema, idFactory)
1003  table.setMetadata(self.algMetadata)
1004  results = self.detection.run(
1005  table=table,
1006  exposure=detectionExposure,
1007  doSmooth=not detectOnLikelihood
1008  )
1009 
1010  if self.config.doMerge:
1011  fpSet = results.fpSets.positive
1012  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1013  self.config.growFootprint, False)
1014  diaSources = afwTable.SourceCatalog(table)
1015  fpSet.makeSources(diaSources)
1016  self.log.info("Merging detections into %d sources" % (len(diaSources)))
1017  else:
1018  diaSources = results.sources
1019  # Inject skySources before measurement.
1020  if self.config.doSkySources:
1021  skySourceFootprints = self.skySources.run(
1022  mask=detectionExposure.mask,
1023  seed=detectionExposure.getInfo().getVisitInfo().getExposureId())
1024  if skySourceFootprints:
1025  for foot in skySourceFootprints:
1026  s = diaSources.addNew()
1027  s.setFootprint(foot)
1028  s.set(self.skySourceKey, True)
1029 
1030  if self.config.doMeasurement:
1031  newDipoleFitting = self.config.doDipoleFitting
1032  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1033  if not newDipoleFitting:
1034  # Just fit dipole in diffim
1035  self.measurement.run(diaSources, detectionExposure)
1036  else:
1037  # Use (matched) template and science image (if avail.) to constrain dipole fitting
1038  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1039  self.measurement.run(diaSources, detectionExposure, exposure,
1040  subtractRes.matchedExposure)
1041  else:
1042  self.measurement.run(diaSources, detectionExposure, exposure)
1043  if self.config.doApCorr:
1044  self.applyApCorr.run(
1045  catalog=diaSources,
1046  apCorrMap=detectionExposure.getInfo().getApCorrMap()
1047  )
1048 
1049  if self.config.doForcedMeasurement:
1050  # Run forced psf photometry on the PVI at the diaSource locations.
1051  # Copy the measured flux and error into the diaSource.
1052  forcedSources = self.forcedMeasurement.generateMeasCat(
1053  exposure, diaSources, detectionExposure.getWcs())
1054  self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1055  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1056  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1057  "ip_diffim_forced_PsfFlux_instFlux", True)
1058  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1059  "ip_diffim_forced_PsfFlux_instFluxErr", True)
1060  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1061  "ip_diffim_forced_PsfFlux_area", True)
1062  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1063  "ip_diffim_forced_PsfFlux_flag", True)
1064  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1065  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1066  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1067  "ip_diffim_forced_PsfFlux_flag_edge", True)
1068  for diaSource, forcedSource in zip(diaSources, forcedSources):
1069  diaSource.assign(forcedSource, mapper)
1070 
1071  # Match with the calexp sources if possible
1072  if self.config.doMatchSources:
1073  if selectSources is not None:
1074  # Create key,val pair where key=diaSourceId and val=sourceId
1075  matchRadAsec = self.config.diaSourceMatchRadius
1076  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1077 
1078  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1079  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1080  srcMatch in srcMatches])
1081  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
1082  len(diaSources)))
1083  else:
1084  self.log.warn("Src product does not exist; cannot match with diaSources")
1085  srcMatchDict = {}
1086 
1087  # Create key,val pair where key=diaSourceId and val=refId
1088  refAstromConfig = AstrometryConfig()
1089  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1090  refAstrometer = AstrometryTask(refAstromConfig)
1091  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1092  refMatches = astromRet.matches
1093  if refMatches is None:
1094  self.log.warn("No diaSource matches with reference catalog")
1095  refMatchDict = {}
1096  else:
1097  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
1098  len(diaSources)))
1099  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1100  refMatch in refMatches])
1101 
1102  # Assign source Ids
1103  for diaSource in diaSources:
1104  sid = diaSource.getId()
1105  if sid in srcMatchDict:
1106  diaSource.set("srcMatchId", srcMatchDict[sid])
1107  if sid in refMatchDict:
1108  diaSource.set("refMatchId", refMatchDict[sid])
1109 
1110  if self.config.doAddMetrics and self.config.doSelectSources:
1111  self.log.info("Evaluating metrics and control sample")
1112 
1113  kernelCandList = []
1114  for cell in subtractRes.kernelCellSet.getCellList():
1115  for cand in cell.begin(False): # include bad candidates
1116  kernelCandList.append(cand)
1117 
1118  # Get basis list to build control sample kernels
1119  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1120  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1121 
1122  controlCandList = (
1123  diffimTools.sourceTableToCandidateList(controlSources,
1124  subtractRes.warpedExposure, exposure,
1125  self.config.subtract.kernel.active,
1126  self.config.subtract.kernel.active.detectionConfig,
1127  self.log, doBuild=True, basisList=basisList))
1128 
1129  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1130  subtractRes.backgroundModel, dof=nparam)
1131  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1132  subtractRes.backgroundModel)
1133 
1134  if self.config.doDetection:
1135  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1136  else:
1137  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1138 
1139  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1140  return pipeBase.Struct(
1141  subtractedExposure=subtractedExposure,
1142  scoreExposure=scoreExposure,
1143  warpedExposure=subtractRes.warpedExposure,
1144  matchedExposure=subtractRes.matchedExposure,
1145  subtractRes=subtractRes,
1146  diaSources=diaSources,
1147  selectSources=selectSources
1148  )
1149 
1150  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1151  """Fit the relative astrometry between templateSources and selectSources
1152 
1153  Todo
1154  ----
1155 
1156  Remove this method. It originally fit a new WCS to the template before calling register.run
1157  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1158  It remains because a subtask overrides it.
1159  """
1160  results = self.register.run(templateSources, templateExposure.getWcs(),
1161  templateExposure.getBBox(), selectSources)
1162  return results
1163 
1164  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1165  """Make debug plots and displays.
1166 
1167  Todo
1168  ----
1169  Test and update for current debug display and slot names
1170  """
1171  import lsstDebug
1172  display = lsstDebug.Info(__name__).display
1173  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1174  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1175  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1176  showDipoles = lsstDebug.Info(__name__).showDipoles
1177  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1178  if display:
1179  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1180  if not maskTransparency:
1181  maskTransparency = 0
1182  disp.setMaskTransparency(maskTransparency)
1183 
1184  if display and showSubtracted:
1185  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1186  mi = subtractRes.subtractedExposure.getMaskedImage()
1187  x0, y0 = mi.getX0(), mi.getY0()
1188  with disp.Buffering():
1189  for s in diaSources:
1190  x, y = s.getX() - x0, s.getY() - y0
1191  ctype = "red" if s.get("flags_negative") else "yellow"
1192  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1193  or s.get("base_PixelFlags_flag_saturatedCenter")
1194  or s.get("base_PixelFlags_flag_crCenter")):
1195  ptype = "x"
1196  elif (s.get("base_PixelFlags_flag_interpolated")
1197  or s.get("base_PixelFlags_flag_saturated")
1198  or s.get("base_PixelFlags_flag_cr")):
1199  ptype = "+"
1200  else:
1201  ptype = "o"
1202  disp.dot(ptype, x, y, size=4, ctype=ctype)
1203  lsstDebug.frame += 1
1204 
1205  if display and showPixelResiduals and selectSources:
1206  nonKernelSources = []
1207  for source in selectSources:
1208  if source not in kernelSources:
1209  nonKernelSources.append(source)
1210 
1211  diUtils.plotPixelResiduals(exposure,
1212  subtractRes.warpedExposure,
1213  subtractRes.subtractedExposure,
1214  subtractRes.kernelCellSet,
1215  subtractRes.psfMatchingKernel,
1216  subtractRes.backgroundModel,
1217  nonKernelSources,
1218  self.subtract.config.kernel.active.detectionConfig,
1219  origVariance=False)
1220  diUtils.plotPixelResiduals(exposure,
1221  subtractRes.warpedExposure,
1222  subtractRes.subtractedExposure,
1223  subtractRes.kernelCellSet,
1224  subtractRes.psfMatchingKernel,
1225  subtractRes.backgroundModel,
1226  nonKernelSources,
1227  self.subtract.config.kernel.active.detectionConfig,
1228  origVariance=True)
1229  if display and showDiaSources:
1230  flagChecker = SourceFlagChecker(diaSources)
1231  isFlagged = [flagChecker(x) for x in diaSources]
1232  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1233  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1234  frame=lsstDebug.frame)
1235  lsstDebug.frame += 1
1236 
1237  if display and showDipoles:
1238  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1239  frame=lsstDebug.frame)
1240  lsstDebug.frame += 1
1241 
1242  def _getConfigName(self):
1243  """Return the name of the config dataset
1244  """
1245  return "%sDiff_config" % (self.config.coaddName,)
1246 
1247  def _getMetadataName(self):
1248  """Return the name of the metadata dataset
1249  """
1250  return "%sDiff_metadata" % (self.config.coaddName,)
1251 
1252  def getSchemaCatalogs(self):
1253  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1254  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1255 
1256  @classmethod
1257  def _makeArgumentParser(cls):
1258  """Create an argument parser
1259  """
1260  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1261  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1262  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1263  help="Template data ID in case of calexp template,"
1264  " e.g. --templateId visit=6789")
1265  return parser
1266 
1267 
1268 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1269  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1270  doc="Shift stars going into RegisterTask by this amount")
1271  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1272  doc="Perturb stars going into RegisterTask by this amount")
1273 
1274  def setDefaults(self):
1275  ImageDifferenceConfig.setDefaults(self)
1276  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1277 
1278 
1279 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1280  """!Image difference Task used in the Winter 2013 data challege.
1281  Enables testing the effects of registration shifts and scatter.
1282 
1283  For use with winter 2013 simulated images:
1284  Use --templateId visit=88868666 for sparse data
1285  --templateId visit=22222200 for dense data (g)
1286  --templateId visit=11111100 for dense data (i)
1287  """
1288  ConfigClass = Winter2013ImageDifferenceConfig
1289  _DefaultName = "winter2013ImageDifference"
1290 
1291  def __init__(self, **kwargs):
1292  ImageDifferenceTask.__init__(self, **kwargs)
1293 
1294  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1295  """Fit the relative astrometry between templateSources and selectSources"""
1296  if self.config.winter2013WcsShift > 0.0:
1297  offset = geom.Extent2D(self.config.winter2013WcsShift,
1298  self.config.winter2013WcsShift)
1299  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1300  for source in templateSources:
1301  centroid = source.get(cKey)
1302  source.set(cKey, centroid + offset)
1303  elif self.config.winter2013WcsRms > 0.0:
1304  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1305  for source in templateSources:
1306  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1307  self.config.winter2013WcsRms*numpy.random.normal())
1308  centroid = source.get(cKey)
1309  source.set(cKey, centroid + offset)
1310 
1311  results = self.register.run(templateSources, templateExposure.getWcs(),
1312  templateExposure.getBBox(), selectSources)
1313  return results
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)