lsst.pipe.tasks  21.0.0-100-g362f0c00+4647f1f7f0
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 from lsst.skymap import BaseSkyMap
47 from lsst.obs.base import ExposureIdInfo
48 
49 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
50 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
51 IqrToSigma = 0.741
52 
53 
54 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
55  dimensions=("instrument", "visit", "detector", "skymap"),
56  defaultTemplates={"coaddName": "deep",
57  "skyMapName": "deep",
58  "warpTypeSuffix": "",
59  "fakesType": ""}):
60 
61  exposure = pipeBase.connectionTypes.Input(
62  doc="Input science exposure to subtract from.",
63  dimensions=("instrument", "visit", "detector"),
64  storageClass="ExposureF",
65  name="{fakesType}calexp"
66  )
67 
68  # TODO DM-22953
69  # kernelSources = pipeBase.connectionTypes.Input(
70  # doc="Source catalog produced in calibrate task for kernel candidate sources",
71  # name="src",
72  # storageClass="SourceCatalog",
73  # dimensions=("instrument", "visit", "detector"),
74  # )
75 
76  skyMap = pipeBase.connectionTypes.Input(
77  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
78  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
79  dimensions=("skymap", ),
80  storageClass="SkyMap",
81  )
82  coaddExposures = pipeBase.connectionTypes.Input(
83  doc="Input template to match and subtract from the exposure",
84  dimensions=("tract", "patch", "skymap", "band"),
85  storageClass="ExposureF",
86  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
87  multiple=True,
88  deferLoad=True
89  )
90  dcrCoadds = pipeBase.connectionTypes.Input(
91  doc="Input DCR template to match and subtract from the exposure",
92  name="{fakesType}dcrCoadd{warpTypeSuffix}",
93  storageClass="ExposureF",
94  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
95  multiple=True,
96  deferLoad=True
97  )
98  outputSchema = pipeBase.connectionTypes.InitOutput(
99  doc="Schema (as an example catalog) for output DIASource catalog.",
100  storageClass="SourceCatalog",
101  name="{fakesType}{coaddName}Diff_diaSrc_schema",
102  )
103  # TODO DM-29965: Currently the AL likelihood image is not a separate data product
104  subtractedExposure = pipeBase.connectionTypes.Output(
105  doc="Output AL difference or Zogy proper difference image",
106  dimensions=("instrument", "visit", "detector"),
107  storageClass="ExposureF",
108  name="{fakesType}{coaddName}Diff_differenceExp",
109  )
110  scoreExposure = pipeBase.connectionTypes.Output(
111  doc="Output Zogy score (likelihood) image",
112  dimensions=("instrument", "visit", "detector"),
113  storageClass="ExposureF",
114  name="{fakesType}{coaddName}Diff_scoreExp",
115  )
116  warpedExposure = pipeBase.connectionTypes.Output(
117  doc="Warped template used to create `subtractedExposure`.",
118  dimensions=("instrument", "visit", "detector"),
119  storageClass="ExposureF",
120  name="{fakesType}{coaddName}Diff_warpedExp",
121  )
122  matchedExposure = pipeBase.connectionTypes.Output(
123  doc="Warped template used to create `subtractedExposure`.",
124  dimensions=("instrument", "visit", "detector"),
125  storageClass="ExposureF",
126  name="{fakesType}{coaddName}Diff_matchedExp",
127  )
128  diaSources = pipeBase.connectionTypes.Output(
129  doc="Output detected diaSources on the difference image",
130  dimensions=("instrument", "visit", "detector"),
131  storageClass="SourceCatalog",
132  name="{fakesType}{coaddName}Diff_diaSrc",
133  )
134 
135  def __init__(self, *, config=None):
136  super().__init__(config=config)
137  if config.coaddName == 'dcr':
138  self.inputs.remove("coaddExposures")
139  else:
140  self.inputs.remove("dcrCoadds")
141  if not config.doWriteSubtractedExp:
142  self.outputs.remove("subtractedExposure")
143  if not config.doWriteScoreExp:
144  self.outputs.remove("scoreExposure")
145  if not config.doWriteWarpedExp:
146  self.outputs.remove("warpedExposure")
147  if not config.doWriteMatchedExp:
148  self.outputs.remove("matchedExposure")
149  if not config.doWriteSources:
150  self.outputs.remove("diaSources")
151 
152  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
153  # Make kernelSources optional
154 
155 
156 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
157  pipelineConnections=ImageDifferenceTaskConnections):
158  """Config for ImageDifferenceTask
159  """
160  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
161  doc="Add background to calexp before processing it. "
162  "Useful as ipDiffim does background matching.")
163  doUseRegister = pexConfig.Field(dtype=bool, default=False,
164  doc="Re-compute astrometry on the template. "
165  "Use image-to-image registration to align template with "
166  "science image (AL only).")
167  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
168  doc="Writing debugging data for doUseRegister")
169  doSelectSources = pexConfig.Field(dtype=bool, default=False,
170  doc="Select stars to use for kernel fitting (AL only)")
171  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
172  doc="Select stars of extreme color as part "
173  "of the control sample (AL only)")
174  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
175  doc="Select stars that are variable to be part "
176  "of the control sample (AL only)")
177  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
178  doPreConvolve = pexConfig.Field(dtype=bool, default=False,
179  doc="Convolve science image by its PSF before PSF-matching (AL only)."
180  " The difference image becomes a likelihood image.")
181  useScoreImageDetection = pexConfig.Field(
182  dtype=bool, default=False, doc="Calculate and detect sources on the Zogy score image (Zogy only).")
183  doWriteScoreExp = pexConfig.Field(
184  dtype=bool, default=False, doc="Write score exposure (Zogy only) ?")
185  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
186  doc="Scale variance of the template before PSF matching")
187  doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
188  doc="Scale variance of the diffim before PSF matching. "
189  "You may do either this or template variance scaling, "
190  "or neither. (Doing both is a waste of CPU.)")
191  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
192  doc="Use a simple gaussian PSF model for pre-convolution "
193  "(else use fit PSF)? Ignored if doPreConvolve false.")
194  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
195  doDecorrelation = pexConfig.Field(dtype=bool, default=True,
196  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
197  "kernel convolution (AL only)? If True, also update the diffim PSF.")
198  doMerge = pexConfig.Field(dtype=bool, default=True,
199  doc="Merge positive and negative diaSources with grow radius "
200  "set by growFootprint")
201  doMatchSources = pexConfig.Field(dtype=bool, default=False,
202  doc="Match diaSources with input calexp sources and ref catalog sources")
203  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
204  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
205  doForcedMeasurement = pexConfig.Field(
206  dtype=bool,
207  default=True,
208  doc="Force photometer diaSource locations on PVI?")
209  doWriteSubtractedExp = pexConfig.Field(
210  dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
211  doWriteWarpedExp = pexConfig.Field(
212  dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
213  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
214  doc="Write warped and PSF-matched template coadd exposure?")
215  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
216  doAddMetrics = pexConfig.Field(dtype=bool, default=False,
217  doc="Add columns to the source table to hold analysis metrics?")
218 
219  coaddName = pexConfig.Field(
220  doc="coadd name: typically one of deep, goodSeeing, or dcr",
221  dtype=str,
222  default="deep",
223  )
224  convolveTemplate = pexConfig.Field(
225  doc="Which image gets convolved (default = template)",
226  dtype=bool,
227  default=True
228  )
229  refObjLoader = pexConfig.ConfigurableField(
230  target=LoadIndexedReferenceObjectsTask,
231  doc="reference object loader",
232  )
233  astrometer = pexConfig.ConfigurableField(
234  target=AstrometryTask,
235  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
236  )
237  sourceSelector = pexConfig.ConfigurableField(
238  target=ObjectSizeStarSelectorTask,
239  doc="Source selection algorithm",
240  )
241  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
242  decorrelate = pexConfig.ConfigurableField(
243  target=DecorrelateALKernelSpatialTask,
244  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
245  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
246  "default of 5.5).",
247  )
248  # Old style ImageMapper grid. ZogyTask has its own grid option
249  doSpatiallyVarying = pexConfig.Field(
250  dtype=bool,
251  default=False,
252  doc="Perform A&L decorrelation on a grid across the "
253  "image in order to allow for spatial variations. Zogy does not use this option."
254  )
255  detection = pexConfig.ConfigurableField(
256  target=SourceDetectionTask,
257  doc="Low-threshold detection for final measurement",
258  )
259  measurement = pexConfig.ConfigurableField(
260  target=DipoleFitTask,
261  doc="Enable updated dipole fitting method",
262  )
263  doApCorr = lsst.pex.config.Field(
264  dtype=bool,
265  default=True,
266  doc="Run subtask to apply aperture corrections"
267  )
268  applyApCorr = lsst.pex.config.ConfigurableField(
269  target=ApplyApCorrTask,
270  doc="Subtask to apply aperture corrections"
271  )
272  forcedMeasurement = pexConfig.ConfigurableField(
273  target=ForcedMeasurementTask,
274  doc="Subtask to force photometer PVI at diaSource location.",
275  )
276  getTemplate = pexConfig.ConfigurableField(
277  target=GetCoaddAsTemplateTask,
278  doc="Subtask to retrieve template exposure and sources",
279  )
280  scaleVariance = pexConfig.ConfigurableField(
281  target=ScaleVarianceTask,
282  doc="Subtask to rescale the variance of the template "
283  "to the statistically expected level"
284  )
285  controlStepSize = pexConfig.Field(
286  doc="What step size (every Nth one) to select a control sample from the kernelSources",
287  dtype=int,
288  default=5
289  )
290  controlRandomSeed = pexConfig.Field(
291  doc="Random seed for shuffing the control sample",
292  dtype=int,
293  default=10
294  )
295  register = pexConfig.ConfigurableField(
296  target=RegisterTask,
297  doc="Task to enable image-to-image image registration (warping)",
298  )
299  kernelSourcesFromRef = pexConfig.Field(
300  doc="Select sources to measure kernel from reference catalog if True, template if false",
301  dtype=bool,
302  default=False
303  )
304  templateSipOrder = pexConfig.Field(
305  dtype=int, default=2,
306  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
307  )
308  growFootprint = pexConfig.Field(
309  dtype=int, default=2,
310  doc="Grow positive and negative footprints by this amount before merging"
311  )
312  diaSourceMatchRadius = pexConfig.Field(
313  dtype=float, default=0.5,
314  doc="Match radius (in arcseconds) for DiaSource to Source association"
315  )
316  requiredTemplateFraction = pexConfig.Field(
317  dtype=float, default=0.1,
318  doc="Do not attempt to run task if template covers less than this fraction of pixels."
319  "Setting to 0 will always attempt image subtraction"
320  )
321 
322  def setDefaults(self):
323  # defaults are OK for catalog and diacatalog
324 
325  self.subtract['al'].kernel.name = "AL"
326  self.subtract['al'].kernel.active.fitForBackground = True
327  self.subtract['al'].kernel.active.spatialKernelOrder = 1
328  self.subtract['al'].kernel.active.spatialBgOrder = 2
329 
330  # DiaSource Detection
331  self.detection.thresholdPolarity = "both"
332  self.detection.thresholdValue = 5.0
333  self.detection.reEstimateBackground = False
334  self.detection.thresholdType = "pixel_stdev"
335 
336  # Add filtered flux measurement, the correct measurement for pre-convolved images.
337  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
338  # To change that you must modify algorithms.names in the task's applyOverrides method,
339  # after the user has set doPreConvolve.
340  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
341  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
342  'base_LocalWcs']
343 
344  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
345  self.forcedMeasurement.copyColumns = {
346  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
347  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
348  self.forcedMeasurement.slots.shape = None
349 
350  # For shuffling the control sample
351  random.seed(self.controlRandomSeed)
352 
353  def validate(self):
354  pexConfig.Config.validate(self)
355  if not self.doSubtract and not self.doDetection:
356  raise ValueError("Either doSubtract or doDetection must be enabled.")
357  if self.doMeasurement and not self.doDetection:
358  raise ValueError("Cannot run source measurement without source detection.")
359  if self.doMerge and not self.doDetection:
360  raise ValueError("Cannot run source merging without source detection.")
361  if self.doUseRegister and not self.doSelectSources:
362  raise ValueError("doUseRegister=True and doSelectSources=False. "
363  "Cannot run RegisterTask without selecting sources.")
364  if hasattr(self.getTemplate, "coaddName"):
365  if self.getTemplate.coaddName != self.coaddName:
366  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
367  if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
368  raise ValueError("Scaling the diffim variance and scaling the template variance "
369  "are both set. Please choose one or the other.")
370  # We cannot allow inconsistencies that would lead to None or not available output products
371  if self.subtract.name == 'zogy':
372  if self.doWriteScoreExp and not self.useScoreImageDetection:
373  raise ValueError("doWriteScoreExp=True and useScoreImageDetection=False "
374  "is not supported. Score image is not calculated.")
375  if self.doWriteMatchedExp:
376  raise ValueError("doWriteMatchedExp=True Matched exposure is not "
377  "calculated in zogy subtraction.")
378  if self.doAddMetrics:
379  raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
380  if self.doDecorrelation:
381  raise ValueError(
382  "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
383  if self.doPreConvolve:
384  raise ValueError(
385  "doPreConvolve=True Pre-convolution is not a zogy option.")
386  if self.doSelectSources:
387  raise ValueError(
388  "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
389  else:
390  if self.useScoreImageDetection:
391  raise ValueError("useScoreImageDetection=True Score exposure does not "
392  "exist in AL subtraction.")
393  if self.doWriteScoreExp: # Ensure output is not None
394  raise ValueError("doWriteScoreExp=True Score exposure does not exist in AL subtraction.")
395  if self.doAddMetrics and not self.doSubtract:
396  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
397  if self.doPreConvolve and self.doDecorrelation:
398  raise NotImplementedError(
399  "doPreConvolve=True and doDecorrelation=True "
400  "The decorrelation afterburner cannot handle pre-convolved exposures.")
401 
402 
403 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
404 
405  @staticmethod
406  def getTargetList(parsedCmd, **kwargs):
407  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
408  **kwargs)
409 
410 
411 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
412  """Subtract an image from a template and measure the result
413  """
414  ConfigClass = ImageDifferenceConfig
415  RunnerClass = ImageDifferenceTaskRunner
416  _DefaultName = "imageDifference"
417 
418  def __init__(self, butler=None, **kwargs):
419  """!Construct an ImageDifference Task
420 
421  @param[in] butler Butler object to use in constructing reference object loaders
422  """
423  super().__init__(**kwargs)
424  self.makeSubtask("getTemplate")
425 
426  self.makeSubtask("subtract")
427 
428  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
429  self.makeSubtask("decorrelate")
430 
431  if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
432  self.makeSubtask("scaleVariance")
433 
434  if self.config.doUseRegister:
435  self.makeSubtask("register")
436  self.schema = afwTable.SourceTable.makeMinimalSchema()
437 
438  if self.config.doSelectSources:
439  self.makeSubtask("sourceSelector")
440  if self.config.kernelSourcesFromRef:
441  self.makeSubtask('refObjLoader', butler=butler)
442  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
443 
444  self.algMetadata = dafBase.PropertyList()
445  if self.config.doDetection:
446  self.makeSubtask("detection", schema=self.schema)
447  if self.config.doMeasurement:
448  self.makeSubtask("measurement", schema=self.schema,
449  algMetadata=self.algMetadata)
450  if self.config.doApCorr:
451  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
452  if self.config.doForcedMeasurement:
453  self.schema.addField(
454  "ip_diffim_forced_PsfFlux_instFlux", "D",
455  "Forced PSF flux measured on the direct image.",
456  units="count")
457  self.schema.addField(
458  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
459  "Forced PSF flux error measured on the direct image.",
460  units="count")
461  self.schema.addField(
462  "ip_diffim_forced_PsfFlux_area", "F",
463  "Forced PSF flux effective area of PSF.",
464  units="pixel")
465  self.schema.addField(
466  "ip_diffim_forced_PsfFlux_flag", "Flag",
467  "Forced PSF flux general failure flag.")
468  self.schema.addField(
469  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
470  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
471  self.schema.addField(
472  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
473  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
474  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
475  if self.config.doMatchSources:
476  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
477  self.schema.addField("srcMatchId", "L", "unique id of source match")
478 
479  # initialize InitOutputs
480  self.outputSchema = afwTable.SourceCatalog(self.schema)
481  self.outputSchema.getTable().setMetadata(self.algMetadata)
482 
483  @staticmethod
484  def makeIdFactory(expId, expBits):
485  """Create IdFactory instance for unique 64 bit diaSource id-s.
486 
487  Parameters
488  ----------
489  expId : `int`
490  Exposure id.
491 
492  expBits: `int`
493  Number of used bits in ``expId``.
494 
495  Note
496  ----
497  The diasource id-s consists of the ``expId`` stored fixed in the highest value
498  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
499  low value end of the integer.
500 
501  Returns
502  -------
503  idFactory: `lsst.afw.table.IdFactory`
504  """
505  return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
506 
507  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
508  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
509  inputRefs: pipeBase.InputQuantizedConnection,
510  outputRefs: pipeBase.OutputQuantizedConnection):
511  inputs = butlerQC.get(inputRefs)
512  self.log.info(f"Processing {butlerQC.quantum.dataId}")
513  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
514  returnMaxBits=True)
515  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
516  if self.config.coaddName == 'dcr':
517  templateExposures = inputRefs.dcrCoadds
518  else:
519  templateExposures = inputRefs.coaddExposures
520  templateStruct = self.getTemplate.runQuantum(
521  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
522  )
523 
524  if templateStruct.area/inputs['exposure'].getBBox().getArea() < self.config.requiredTemplateFraction:
525  # TO DO DM-30649: when available, use exit code for expected failures
526  # In the meantime, raise to exit and register as a failure
527  message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
528  "To force subtraction, set config requiredTemplateFraction=0." % (
529  100*templateStruct.area/inputs['exposure'].getBBox().getArea(),
530  100*self.config.requiredTemplateFraction))
531  raise pipeBase.TaskError("Expected Failure: %s" % (message))
532  else:
533  outputs = self.run(exposure=inputs['exposure'],
534  templateExposure=templateStruct.exposure,
535  idFactory=idFactory)
536  # Consistency with runDataref gen2 handling
537  if outputs.diaSources is None:
538  del outputs.diaSources
539  butlerQC.put(outputs, outputRefs)
540 
541  @pipeBase.timeMethod
542  def runDataRef(self, sensorRef, templateIdList=None):
543  """Subtract an image from a template coadd and measure the result.
544 
545  Data I/O wrapper around `run` using the butler in Gen2.
546 
547  Parameters
548  ----------
549  sensorRef : `lsst.daf.persistence.ButlerDataRef`
550  Sensor-level butler data reference, used for the following data products:
551 
552  Input only:
553  - calexp
554  - psf
555  - ccdExposureId
556  - ccdExposureId_bits
557  - self.config.coaddName + "Coadd_skyMap"
558  - self.config.coaddName + "Coadd"
559  Input or output, depending on config:
560  - self.config.coaddName + "Diff_subtractedExp"
561  Output, depending on config:
562  - self.config.coaddName + "Diff_matchedExp"
563  - self.config.coaddName + "Diff_src"
564 
565  Returns
566  -------
567  results : `lsst.pipe.base.Struct`
568  Returns the Struct by `run`.
569  """
570  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
571  subtractedExposure = None
572  selectSources = None
573  calexpBackgroundExposure = None
574  self.log.info("Processing %s" % (sensorRef.dataId))
575 
576  # We make one IdFactory that will be used by both icSrc and src datasets;
577  # I don't know if this is the way we ultimately want to do things, but at least
578  # this ensures the source IDs are fully unique.
579  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
580  expBits=sensorRef.get("ccdExposureId_bits"))
581  if self.config.doAddCalexpBackground:
582  calexpBackgroundExposure = sensorRef.get("calexpBackground")
583 
584  # Retrieve the science image we wish to analyze
585  exposure = sensorRef.get("calexp", immediate=True)
586 
587  # Retrieve the template image
588  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
589 
590  if sensorRef.datasetExists("src"):
591  self.log.info("Source selection via src product")
592  # Sources already exist; for data release processing
593  selectSources = sensorRef.get("src")
594 
595  if not self.config.doSubtract and self.config.doDetection:
596  # If we don't do subtraction, we need the subtracted exposure from the repo
597  subtractedExposure = sensorRef.get(subtractedExposureName)
598  # Both doSubtract and doDetection cannot be False
599 
600  results = self.run(exposure=exposure,
601  selectSources=selectSources,
602  templateExposure=template.exposure,
603  templateSources=template.sources,
604  idFactory=idFactory,
605  calexpBackgroundExposure=calexpBackgroundExposure,
606  subtractedExposure=subtractedExposure)
607 
608  if self.config.doWriteSources and results.diaSources is not None:
609  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
610  if self.config.doWriteWarpedExp:
611  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
612  if self.config.doWriteMatchedExp:
613  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
614  if self.config.doAddMetrics and self.config.doSelectSources:
615  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
616  if self.config.doWriteSubtractedExp:
617  sensorRef.put(results.subtractedExposure, subtractedExposureName)
618  if self.config.doWriteScoreExp:
619  sensorRef.put(results.scoreExposure, self.config.coaddName + "Diff_scoreExp")
620  return results
621 
622  @pipeBase.timeMethod
623  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
624  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
625  """PSF matches, subtract two images and perform detection on the difference image.
626 
627  Parameters
628  ----------
629  exposure : `lsst.afw.image.ExposureF`, optional
630  The science exposure, the minuend in the image subtraction.
631  Can be None only if ``config.doSubtract==False``.
632  selectSources : `lsst.afw.table.SourceCatalog`, optional
633  Identified sources on the science exposure. This catalog is used to
634  select sources in order to perform the AL PSF matching on stamp images
635  around them. The selection steps depend on config options and whether
636  ``templateSources`` and ``matchingSources`` specified.
637  templateExposure : `lsst.afw.image.ExposureF`, optional
638  The template to be subtracted from ``exposure`` in the image subtraction.
639  ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
640  The template exposure should cover the same sky area as the science exposure.
641  It is either a stich of patches of a coadd skymap image or a calexp
642  of the same pointing as the science exposure. Can be None only
643  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
644  templateSources : `lsst.afw.table.SourceCatalog`, optional
645  Identified sources on the template exposure.
646  idFactory : `lsst.afw.table.IdFactory`
647  Generator object to assign ids to detected sources in the difference image.
648  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
649  Background exposure to be added back to the science exposure
650  if ``config.doAddCalexpBackground==True``
651  subtractedExposure : `lsst.afw.image.ExposureF`, optional
652  If ``config.doSubtract==False`` and ``config.doDetection==True``,
653  performs the post subtraction source detection only on this exposure.
654  Otherwise should be None.
655 
656  Returns
657  -------
658  results : `lsst.pipe.base.Struct`
659  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
660  Difference image.
661  ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
662  The zogy score exposure, if calculated.
663  ``matchedExposure`` : `lsst.afw.image.ExposureF`
664  The matched PSF exposure.
665  ``subtractRes`` : `lsst.pipe.base.Struct`
666  The returned result structure of the ImagePsfMatchTask subtask.
667  ``diaSources`` : `lsst.afw.table.SourceCatalog`
668  The catalog of detected sources.
669  ``selectSources`` : `lsst.afw.table.SourceCatalog`
670  The input source catalog with optionally added Qa information.
671 
672  Notes
673  -----
674  The following major steps are included:
675 
676  - warp template coadd to match WCS of image
677  - PSF match image to warped template
678  - subtract image from PSF-matched, warped template
679  - detect sources
680  - measure sources
681 
682  For details about the image subtraction configuration modes
683  see `lsst.ip.diffim`.
684  """
685  subtractRes = None
686  controlSources = None
687  scoreExposure = None
688  diaSources = None
689  kernelSources = None
690  # We'll clone exposure if modified but will still need the original
691  exposureOrig = exposure
692 
693  if self.config.doAddCalexpBackground:
694  mi = exposure.getMaskedImage()
695  mi += calexpBackgroundExposure.getImage()
696 
697  if not exposure.hasPsf():
698  raise pipeBase.TaskError("Exposure has no psf")
699  sciencePsf = exposure.getPsf()
700 
701  if self.config.doSubtract:
702  if self.config.doScaleTemplateVariance:
703  self.log.info("Rescaling template variance")
704  templateVarFactor = self.scaleVariance.run(
705  templateExposure.getMaskedImage())
706  self.log.info("Template variance scaling factor: %.2f" % templateVarFactor)
707  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
708 
709  if self.config.subtract.name == 'zogy':
710  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
711  scoreExposure = subtractRes.scoreExp
712  subtractedExposure = subtractRes.diffExp
713  subtractRes.subtractedExposure = subtractedExposure
714  subtractRes.matchedExposure = None
715 
716  elif self.config.subtract.name == 'al':
717  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
718  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
719  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
720 
721  # if requested, convolve the science exposure with its PSF
722  # (properly, this should be a cross-correlation, but our code does not yet support that)
723  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
724  # else sigma of original science exposure
725  # TODO: DM-22762 This functional block should be moved into its own method
726  preConvPsf = None
727  if self.config.doPreConvolve:
728  convControl = afwMath.ConvolutionControl()
729  # cannot convolve in place, so need a new image anyway
730  srcMI = exposure.maskedImage
731  exposure = exposure.clone() # New deep copy
732  srcPsf = sciencePsf
733  if self.config.useGaussianForPreConvolution:
734  # convolve with a simplified PSF model: a double Gaussian
735  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
736  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
737  else:
738  # convolve with science exposure's PSF model
739  preConvPsf = srcPsf
740  afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
741  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
742  else:
743  scienceSigmaPost = scienceSigmaOrig
744 
745  # If requested, find and select sources from the image
746  # else, AL subtraction will do its own source detection
747  # TODO: DM-22762 This functional block should be moved into its own method
748  if self.config.doSelectSources:
749  if selectSources is None:
750  self.log.warn("Src product does not exist; running detection, measurement, selection")
751  # Run own detection and measurement; necessary in nightly processing
752  selectSources = self.subtract.getSelectSources(
753  exposure,
754  sigma=scienceSigmaPost,
755  doSmooth=not self.config.doPreConvolve,
756  idFactory=idFactory,
757  )
758 
759  if self.config.doAddMetrics:
760  # Number of basis functions
761 
762  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
763  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
764  targetFwhmPix=templateSigma*FwhmPerSigma))
765  # Modify the schema of all Sources
766  # DEPRECATED: This is a data dependent (nparam) output product schema
767  # outside the task constructor.
768  # NOTE: The pre-determination of nparam at this point
769  # may be incorrect as the template psf is warped later in
770  # ImagePsfMatchTask.matchExposures()
771  kcQa = KernelCandidateQa(nparam)
772  selectSources = kcQa.addToSchema(selectSources)
773  if self.config.kernelSourcesFromRef:
774  # match exposure sources to reference catalog
775  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
776  matches = astromRet.matches
777  elif templateSources:
778  # match exposure sources to template sources
779  mc = afwTable.MatchControl()
780  mc.findOnlyClosest = False
781  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
782  mc)
783  else:
784  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
785  "but template sources not available. Cannot match science "
786  "sources with template sources. Run process* on data from "
787  "which templates are built.")
788 
789  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
790  matches=matches).sourceCat
791  random.shuffle(kernelSources, random.random)
792  controlSources = kernelSources[::self.config.controlStepSize]
793  kernelSources = [k for i, k in enumerate(kernelSources)
794  if i % self.config.controlStepSize]
795 
796  if self.config.doSelectDcrCatalog:
797  redSelector = DiaCatalogSourceSelectorTask(
798  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
799  grMax=99.999))
800  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
801  controlSources.extend(redSources)
802 
803  blueSelector = DiaCatalogSourceSelectorTask(
804  DiaCatalogSourceSelectorConfig(grMin=-99.999,
805  grMax=self.sourceSelector.config.grMin))
806  blueSources = blueSelector.selectStars(exposure, selectSources,
807  matches=matches).starCat
808  controlSources.extend(blueSources)
809 
810  if self.config.doSelectVariableCatalog:
811  varSelector = DiaCatalogSourceSelectorTask(
812  DiaCatalogSourceSelectorConfig(includeVariable=True))
813  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
814  controlSources.extend(varSources)
815 
816  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
817  % (len(kernelSources), len(selectSources), len(controlSources)))
818 
819  allresids = {}
820  # TODO: DM-22762 This functional block should be moved into its own method
821  if self.config.doUseRegister:
822  self.log.info("Registering images")
823 
824  if templateSources is None:
825  # Run detection on the template, which is
826  # temporarily background-subtracted
827  # sigma of PSF of template image before warping
828  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
829  templateSources = self.subtract.getSelectSources(
830  templateExposure,
831  sigma=templateSigma,
832  doSmooth=True,
833  idFactory=idFactory
834  )
835 
836  # Third step: we need to fit the relative astrometry.
837  #
838  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
839  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
840  exposure.getWcs(), exposure.getBBox())
841  templateExposure = warpedExp
842 
843  # Create debugging outputs on the astrometric
844  # residuals as a function of position. Persistence
845  # not yet implemented; expected on (I believe) #2636.
846  if self.config.doDebugRegister:
847  # Grab matches to reference catalog
848  srcToMatch = {x.second.getId(): x.first for x in matches}
849 
850  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
851  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
852  sids = [m.first.getId() for m in wcsResults.matches]
853  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
854  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
855  m.second.get(inCentroidKey))) for m in wcsResults.matches]
856  allresids = dict(zip(sids, zip(positions, residuals)))
857 
858  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
859  wcsResults.wcs.pixelToSky(
860  m.second.get(inCentroidKey))) for m in wcsResults.matches]
861  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
862  + 2.5*numpy.log10(srcToMatch[x].get("r"))
863  for x in sids if x in srcToMatch.keys()])
864  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
865  if s in srcToMatch.keys()])
866  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
867  if s in srcToMatch.keys()])
868  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
869  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
870  & (colors <= self.sourceSelector.config.grMax))
871  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
872  rms1Long = IqrToSigma*(
873  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
874  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
875  - numpy.percentile(dlat[idx1], 25))
876  rms2Long = IqrToSigma*(
877  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
878  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
879  - numpy.percentile(dlat[idx2], 25))
880  rms3Long = IqrToSigma*(
881  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
882  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
883  - numpy.percentile(dlat[idx3], 25))
884  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
885  (numpy.median(dlong[idx1]), rms1Long,
886  numpy.median(dlat[idx1]), rms1Lat))
887  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
888  (numpy.median(dlong[idx2]), rms2Long,
889  numpy.median(dlat[idx2]), rms2Lat))
890  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
891  (numpy.median(dlong[idx3]), rms3Long,
892  numpy.median(dlat[idx3]), rms3Lat))
893 
894  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
895  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
896  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
897  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
898  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
899  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
900 
901  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
902  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
903  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
904  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
905  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
906  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
907 
908  # warp template exposure to match exposure,
909  # PSF match template exposure to exposure,
910  # then return the difference
911 
912  # Return warped template... Construct sourceKernelCand list after subtract
913  self.log.info("Subtracting images")
914  subtractRes = self.subtract.subtractExposures(
915  templateExposure=templateExposure,
916  scienceExposure=exposure,
917  candidateList=kernelSources,
918  convolveTemplate=self.config.convolveTemplate,
919  doWarping=not self.config.doUseRegister
920  )
921  subtractedExposure = subtractRes.subtractedExposure
922 
923  if self.config.doDetection:
924  self.log.info("Computing diffim PSF")
925 
926  # Get Psf from the appropriate input image if it doesn't exist
927  if not subtractedExposure.hasPsf():
928  if self.config.convolveTemplate:
929  subtractedExposure.setPsf(exposure.getPsf())
930  else:
931  subtractedExposure.setPsf(templateExposure.getPsf())
932 
933  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
934  # thus it may have already been decorrelated. Thus, we do not decorrelate if
935  # doSubtract is False.
936 
937  # NOTE: At this point doSubtract == True
938  if self.config.doDecorrelation and self.config.doSubtract:
939  preConvKernel = None
940  if preConvPsf is not None:
941  preConvKernel = preConvPsf.getLocalKernel()
942  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
943  subtractedExposure,
944  subtractRes.psfMatchingKernel,
945  spatiallyVarying=self.config.doSpatiallyVarying,
946  preConvKernel=preConvKernel,
947  templateMatched=self.config.convolveTemplate)
948  subtractedExposure = decorrResult.correctedExposure
949 
950  # END (if subtractAlgorithm == 'AL')
951  # END (if self.config.doSubtract)
952  if self.config.doDetection:
953  self.log.info("Running diaSource detection")
954 
955  # subtractedExposure - reserved for task return value
956  # in zogy, it is always the proper difference image
957  # in AL, it may be (yet) pre-convolved and/or decorrelated
958  #
959  # detectionExposure - controls which exposure to use for detection
960  # in-place modifications will appear in task return
961  if self.config.useScoreImageDetection:
962  # zogy with score image detection enabled
963  self.log.info("Detection, diffim rescaling and measurements are on Zogy score image.")
964  detectionExposure = scoreExposure
965  detectOnLikelihood = True
966  else:
967  # AL or zogy with no score image detection
968  detectionExposure = subtractedExposure
969  detectOnLikelihood = False
970  if self.config.doPreConvolve:
971  # In AL, the likelihood image is not a separate product, yet
972  self.log.info("Detection, diffim rescaling and measurements are on AL pre-convolved "
973  "difference (likelihood) image.")
974  detectOnLikelihood = True
975  else:
976  self.log.info("Detection, diffim rescaling and measurements are on "
977  "(proper) difference image.")
978 
979  # Rescale difference image variance plane
980  if self.config.doScaleDiffimVariance:
981  self.log.info("Rescaling diffim variance")
982  diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
983  self.log.info("Diffim variance scaling factor: %.2f" % diffimVarFactor)
984  self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
985 
986  # Erase existing detection mask planes
987  mask = detectionExposure.getMaskedImage().getMask()
988  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
989 
990  table = afwTable.SourceTable.make(self.schema, idFactory)
991  table.setMetadata(self.algMetadata)
992  results = self.detection.run(
993  table=table,
994  exposure=detectionExposure,
995  doSmooth=not detectOnLikelihood
996  )
997 
998  if self.config.doMerge:
999  fpSet = results.fpSets.positive
1000  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1001  self.config.growFootprint, False)
1002  diaSources = afwTable.SourceCatalog(table)
1003  fpSet.makeSources(diaSources)
1004  self.log.info("Merging detections into %d sources" % (len(diaSources)))
1005  else:
1006  diaSources = results.sources
1007 
1008  if self.config.doMeasurement:
1009  newDipoleFitting = self.config.doDipoleFitting
1010  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1011  if not newDipoleFitting:
1012  # Just fit dipole in diffim
1013  self.measurement.run(diaSources, detectionExposure)
1014  else:
1015  # Use (matched) template and science image (if avail.) to constrain dipole fitting
1016  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1017  self.measurement.run(diaSources, detectionExposure, exposure,
1018  subtractRes.matchedExposure)
1019  else:
1020  self.measurement.run(diaSources, detectionExposure, exposure)
1021  if self.config.doApCorr:
1022  self.applyApCorr.run(
1023  catalog=diaSources,
1024  apCorrMap=detectionExposure.getInfo().getApCorrMap()
1025  )
1026 
1027  if self.config.doForcedMeasurement:
1028  # Run forced psf photometry on the PVI at the diaSource locations.
1029  # Copy the measured flux and error into the diaSource.
1030  forcedSources = self.forcedMeasurement.generateMeasCat(
1031  exposure, diaSources, detectionExposure.getWcs())
1032  self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1033  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1034  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1035  "ip_diffim_forced_PsfFlux_instFlux", True)
1036  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1037  "ip_diffim_forced_PsfFlux_instFluxErr", True)
1038  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1039  "ip_diffim_forced_PsfFlux_area", True)
1040  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1041  "ip_diffim_forced_PsfFlux_flag", True)
1042  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1043  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1044  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1045  "ip_diffim_forced_PsfFlux_flag_edge", True)
1046  for diaSource, forcedSource in zip(diaSources, forcedSources):
1047  diaSource.assign(forcedSource, mapper)
1048 
1049  # Match with the calexp sources if possible
1050  if self.config.doMatchSources:
1051  if selectSources is not None:
1052  # Create key,val pair where key=diaSourceId and val=sourceId
1053  matchRadAsec = self.config.diaSourceMatchRadius
1054  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1055 
1056  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1057  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1058  srcMatch in srcMatches])
1059  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
1060  len(diaSources)))
1061  else:
1062  self.log.warn("Src product does not exist; cannot match with diaSources")
1063  srcMatchDict = {}
1064 
1065  # Create key,val pair where key=diaSourceId and val=refId
1066  refAstromConfig = AstrometryConfig()
1067  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1068  refAstrometer = AstrometryTask(refAstromConfig)
1069  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1070  refMatches = astromRet.matches
1071  if refMatches is None:
1072  self.log.warn("No diaSource matches with reference catalog")
1073  refMatchDict = {}
1074  else:
1075  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
1076  len(diaSources)))
1077  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1078  refMatch in refMatches])
1079 
1080  # Assign source Ids
1081  for diaSource in diaSources:
1082  sid = diaSource.getId()
1083  if sid in srcMatchDict:
1084  diaSource.set("srcMatchId", srcMatchDict[sid])
1085  if sid in refMatchDict:
1086  diaSource.set("refMatchId", refMatchDict[sid])
1087 
1088  if self.config.doAddMetrics and self.config.doSelectSources:
1089  self.log.info("Evaluating metrics and control sample")
1090 
1091  kernelCandList = []
1092  for cell in subtractRes.kernelCellSet.getCellList():
1093  for cand in cell.begin(False): # include bad candidates
1094  kernelCandList.append(cand)
1095 
1096  # Get basis list to build control sample kernels
1097  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1098  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1099 
1100  controlCandList = (
1101  diffimTools.sourceTableToCandidateList(controlSources,
1102  subtractRes.warpedExposure, exposure,
1103  self.config.subtract.kernel.active,
1104  self.config.subtract.kernel.active.detectionConfig,
1105  self.log, doBuild=True, basisList=basisList))
1106 
1107  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1108  subtractRes.backgroundModel, dof=nparam)
1109  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1110  subtractRes.backgroundModel)
1111 
1112  if self.config.doDetection:
1113  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1114  else:
1115  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1116 
1117  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1118  return pipeBase.Struct(
1119  subtractedExposure=subtractedExposure,
1120  scoreExposure=scoreExposure,
1121  warpedExposure=subtractRes.warpedExposure,
1122  matchedExposure=subtractRes.matchedExposure,
1123  subtractRes=subtractRes,
1124  diaSources=diaSources,
1125  selectSources=selectSources
1126  )
1127 
1128  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1129  """Fit the relative astrometry between templateSources and selectSources
1130 
1131  Todo
1132  ----
1133 
1134  Remove this method. It originally fit a new WCS to the template before calling register.run
1135  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1136  It remains because a subtask overrides it.
1137  """
1138  results = self.register.run(templateSources, templateExposure.getWcs(),
1139  templateExposure.getBBox(), selectSources)
1140  return results
1141 
1142  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1143  """Make debug plots and displays.
1144 
1145  Todo
1146  ----
1147  Test and update for current debug display and slot names
1148  """
1149  import lsstDebug
1150  display = lsstDebug.Info(__name__).display
1151  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1152  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1153  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1154  showDipoles = lsstDebug.Info(__name__).showDipoles
1155  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1156  if display:
1157  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1158  if not maskTransparency:
1159  maskTransparency = 0
1160  disp.setMaskTransparency(maskTransparency)
1161 
1162  if display and showSubtracted:
1163  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1164  mi = subtractRes.subtractedExposure.getMaskedImage()
1165  x0, y0 = mi.getX0(), mi.getY0()
1166  with disp.Buffering():
1167  for s in diaSources:
1168  x, y = s.getX() - x0, s.getY() - y0
1169  ctype = "red" if s.get("flags_negative") else "yellow"
1170  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1171  or s.get("base_PixelFlags_flag_saturatedCenter")
1172  or s.get("base_PixelFlags_flag_crCenter")):
1173  ptype = "x"
1174  elif (s.get("base_PixelFlags_flag_interpolated")
1175  or s.get("base_PixelFlags_flag_saturated")
1176  or s.get("base_PixelFlags_flag_cr")):
1177  ptype = "+"
1178  else:
1179  ptype = "o"
1180  disp.dot(ptype, x, y, size=4, ctype=ctype)
1181  lsstDebug.frame += 1
1182 
1183  if display and showPixelResiduals and selectSources:
1184  nonKernelSources = []
1185  for source in selectSources:
1186  if source not in kernelSources:
1187  nonKernelSources.append(source)
1188 
1189  diUtils.plotPixelResiduals(exposure,
1190  subtractRes.warpedExposure,
1191  subtractRes.subtractedExposure,
1192  subtractRes.kernelCellSet,
1193  subtractRes.psfMatchingKernel,
1194  subtractRes.backgroundModel,
1195  nonKernelSources,
1196  self.subtract.config.kernel.active.detectionConfig,
1197  origVariance=False)
1198  diUtils.plotPixelResiduals(exposure,
1199  subtractRes.warpedExposure,
1200  subtractRes.subtractedExposure,
1201  subtractRes.kernelCellSet,
1202  subtractRes.psfMatchingKernel,
1203  subtractRes.backgroundModel,
1204  nonKernelSources,
1205  self.subtract.config.kernel.active.detectionConfig,
1206  origVariance=True)
1207  if display and showDiaSources:
1208  flagChecker = SourceFlagChecker(diaSources)
1209  isFlagged = [flagChecker(x) for x in diaSources]
1210  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1211  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1212  frame=lsstDebug.frame)
1213  lsstDebug.frame += 1
1214 
1215  if display and showDipoles:
1216  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1217  frame=lsstDebug.frame)
1218  lsstDebug.frame += 1
1219 
1220  def _getConfigName(self):
1221  """Return the name of the config dataset
1222  """
1223  return "%sDiff_config" % (self.config.coaddName,)
1224 
1225  def _getMetadataName(self):
1226  """Return the name of the metadata dataset
1227  """
1228  return "%sDiff_metadata" % (self.config.coaddName,)
1229 
1230  def getSchemaCatalogs(self):
1231  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1232  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1233 
1234  @classmethod
1235  def _makeArgumentParser(cls):
1236  """Create an argument parser
1237  """
1238  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1239  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1240  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1241  help="Template data ID in case of calexp template,"
1242  " e.g. --templateId visit=6789")
1243  return parser
1244 
1245 
1246 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1247  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1248  doc="Shift stars going into RegisterTask by this amount")
1249  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1250  doc="Perturb stars going into RegisterTask by this amount")
1251 
1252  def setDefaults(self):
1253  ImageDifferenceConfig.setDefaults(self)
1254  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1255 
1256 
1257 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1258  """!Image difference Task used in the Winter 2013 data challege.
1259  Enables testing the effects of registration shifts and scatter.
1260 
1261  For use with winter 2013 simulated images:
1262  Use --templateId visit=88868666 for sparse data
1263  --templateId visit=22222200 for dense data (g)
1264  --templateId visit=11111100 for dense data (i)
1265  """
1266  ConfigClass = Winter2013ImageDifferenceConfig
1267  _DefaultName = "winter2013ImageDifference"
1268 
1269  def __init__(self, **kwargs):
1270  ImageDifferenceTask.__init__(self, **kwargs)
1271 
1272  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1273  """Fit the relative astrometry between templateSources and selectSources"""
1274  if self.config.winter2013WcsShift > 0.0:
1275  offset = geom.Extent2D(self.config.winter2013WcsShift,
1276  self.config.winter2013WcsShift)
1277  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1278  for source in templateSources:
1279  centroid = source.get(cKey)
1280  source.set(cKey, centroid + offset)
1281  elif self.config.winter2013WcsRms > 0.0:
1282  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1283  for source in templateSources:
1284  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1285  self.config.winter2013WcsRms*numpy.random.normal())
1286  centroid = source.get(cKey)
1287  source.set(cKey, centroid + offset)
1288 
1289  results = self.register.run(templateSources, templateExposure.getWcs(),
1290  templateExposure.getBBox(), selectSources)
1291  return results
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)