lsst.pipe.tasks  21.0.0-42-g5afaedd3+13d2e51252
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 from lsst.skymap import BaseSkyMap
47 
48 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
49 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
50 IqrToSigma = 0.741
51 
52 
53 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
54  dimensions=("instrument", "visit", "detector", "skymap"),
55  defaultTemplates={"coaddName": "deep",
56  "skyMapName": "deep",
57  "warpTypeSuffix": "",
58  "fakesType": ""}):
59 
60  exposure = pipeBase.connectionTypes.Input(
61  doc="Input science exposure to subtract from.",
62  dimensions=("instrument", "visit", "detector"),
63  storageClass="ExposureF",
64  name="{fakesType}calexp"
65  )
66 
67  # TODO DM-22953
68  # kernelSources = pipeBase.connectionTypes.Input(
69  # doc="Source catalog produced in calibrate task for kernel candidate sources",
70  # name="src",
71  # storageClass="SourceCatalog",
72  # dimensions=("instrument", "visit", "detector"),
73  # )
74 
75  skyMap = pipeBase.connectionTypes.Input(
76  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
77  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78  dimensions=("skymap", ),
79  storageClass="SkyMap",
80  )
81  coaddExposures = pipeBase.connectionTypes.Input(
82  doc="Input template to match and subtract from the exposure",
83  dimensions=("tract", "patch", "skymap", "band"),
84  storageClass="ExposureF",
85  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
86  multiple=True,
87  deferLoad=True
88  )
89  dcrCoadds = pipeBase.connectionTypes.Input(
90  doc="Input DCR template to match and subtract from the exposure",
91  name="{fakesType}dcrCoadd{warpTypeSuffix}",
92  storageClass="ExposureF",
93  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
94  multiple=True,
95  deferLoad=True
96  )
97  outputSchema = pipeBase.connectionTypes.InitOutput(
98  doc="Schema (as an example catalog) for output DIASource catalog.",
99  storageClass="SourceCatalog",
100  name="{fakesType}{coaddName}Diff_diaSrc_schema",
101  )
102  subtractedExposure = pipeBase.connectionTypes.Output(
103  doc="Output difference image",
104  dimensions=("instrument", "visit", "detector"),
105  storageClass="ExposureF",
106  name="{fakesType}{coaddName}Diff_differenceExp",
107  )
108  warpedExposure = pipeBase.connectionTypes.Output(
109  doc="Warped template used to create `subtractedExposure`.",
110  dimensions=("instrument", "visit", "detector"),
111  storageClass="ExposureF",
112  name="{fakesType}{coaddName}Diff_warpedExp",
113  )
114  diaSources = pipeBase.connectionTypes.Output(
115  doc="Output detected diaSources on the difference image",
116  dimensions=("instrument", "visit", "detector"),
117  storageClass="SourceCatalog",
118  name="{fakesType}{coaddName}Diff_diaSrc",
119  )
120 
121  def __init__(self, *, config=None):
122  super().__init__(config=config)
123  if config.coaddName == 'dcr':
124  self.inputs.remove("coaddExposures")
125  else:
126  self.inputs.remove("dcrCoadds")
127 
128  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
129  # Make kernelSources optional
130 
131 
132 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
133  pipelineConnections=ImageDifferenceTaskConnections):
134  """Config for ImageDifferenceTask
135  """
136  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
137  doc="Add background to calexp before processing it. "
138  "Useful as ipDiffim does background matching.")
139  doUseRegister = pexConfig.Field(dtype=bool, default=True,
140  doc="Use image-to-image registration to align template with "
141  "science image")
142  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
143  doc="Writing debugging data for doUseRegister")
144  doSelectSources = pexConfig.Field(dtype=bool, default=True,
145  doc="Select stars to use for kernel fitting")
146  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
147  doc="Select stars of extreme color as part of the control sample")
148  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
149  doc="Select stars that are variable to be part "
150  "of the control sample")
151  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
152  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
153  doc="Convolve science image by its PSF before PSF-matching?")
154  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
155  doc="Scale variance of the template before PSF matching")
156  doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=False,
157  doc="Scale variance of the diffim before PSF matching. "
158  "You may do either this or template variance scaling, "
159  "or neither. (Doing both is a waste of CPU.)")
160  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
161  doc="Use a simple gaussian PSF model for pre-convolution "
162  "(else use fit PSF)? Ignored if doPreConvolve false.")
163  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
164  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
165  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
166  "kernel convolution? If True, also update the diffim PSF.")
167  doMerge = pexConfig.Field(dtype=bool, default=True,
168  doc="Merge positive and negative diaSources with grow radius "
169  "set by growFootprint")
170  doMatchSources = pexConfig.Field(dtype=bool, default=True,
171  doc="Match diaSources with input calexp sources and ref catalog sources")
172  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
173  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
174  doForcedMeasurement = pexConfig.Field(
175  dtype=bool,
176  default=True,
177  doc="Force photometer diaSource locations on PVI?")
178  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
179  doWriteWarpedExp = pexConfig.Field(dtype=bool, default=False,
180  doc="Write WCS, warped template coadd exposure?")
181  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
182  doc="Write warped and PSF-matched template coadd exposure?")
183  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
184  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
185  doc="Add columns to the source table to hold analysis metrics?")
186 
187  coaddName = pexConfig.Field(
188  doc="coadd name: typically one of deep, goodSeeing, or dcr",
189  dtype=str,
190  default="deep",
191  )
192  convolveTemplate = pexConfig.Field(
193  doc="Which image gets convolved (default = template)",
194  dtype=bool,
195  default=True
196  )
197  refObjLoader = pexConfig.ConfigurableField(
198  target=LoadIndexedReferenceObjectsTask,
199  doc="reference object loader",
200  )
201  astrometer = pexConfig.ConfigurableField(
202  target=AstrometryTask,
203  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
204  )
205  sourceSelector = pexConfig.ConfigurableField(
206  target=ObjectSizeStarSelectorTask,
207  doc="Source selection algorithm",
208  )
209  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
210  decorrelate = pexConfig.ConfigurableField(
211  target=DecorrelateALKernelSpatialTask,
212  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
213  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
214  "default of 5.5).",
215  )
216  doSpatiallyVarying = pexConfig.Field(
217  dtype=bool,
218  default=False,
219  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
220  "image in order to allow for spatial variations"
221  )
222  detection = pexConfig.ConfigurableField(
223  target=SourceDetectionTask,
224  doc="Low-threshold detection for final measurement",
225  )
226  measurement = pexConfig.ConfigurableField(
227  target=DipoleFitTask,
228  doc="Enable updated dipole fitting method",
229  )
230  doApCorr = lsst.pex.config.Field(
231  dtype=bool,
232  default=True,
233  doc="Run subtask to apply aperture corrections"
234  )
235  applyApCorr = lsst.pex.config.ConfigurableField(
236  target=ApplyApCorrTask,
237  doc="Subtask to apply aperture corrections"
238  )
239  forcedMeasurement = pexConfig.ConfigurableField(
240  target=ForcedMeasurementTask,
241  doc="Subtask to force photometer PVI at diaSource location.",
242  )
243  getTemplate = pexConfig.ConfigurableField(
244  target=GetCoaddAsTemplateTask,
245  doc="Subtask to retrieve template exposure and sources",
246  )
247  scaleVariance = pexConfig.ConfigurableField(
248  target=ScaleVarianceTask,
249  doc="Subtask to rescale the variance of the template "
250  "to the statistically expected level"
251  )
252  controlStepSize = pexConfig.Field(
253  doc="What step size (every Nth one) to select a control sample from the kernelSources",
254  dtype=int,
255  default=5
256  )
257  controlRandomSeed = pexConfig.Field(
258  doc="Random seed for shuffing the control sample",
259  dtype=int,
260  default=10
261  )
262  register = pexConfig.ConfigurableField(
263  target=RegisterTask,
264  doc="Task to enable image-to-image image registration (warping)",
265  )
266  kernelSourcesFromRef = pexConfig.Field(
267  doc="Select sources to measure kernel from reference catalog if True, template if false",
268  dtype=bool,
269  default=False
270  )
271  templateSipOrder = pexConfig.Field(
272  dtype=int, default=2,
273  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
274  )
275  growFootprint = pexConfig.Field(
276  dtype=int, default=2,
277  doc="Grow positive and negative footprints by this amount before merging"
278  )
279  diaSourceMatchRadius = pexConfig.Field(
280  dtype=float, default=0.5,
281  doc="Match radius (in arcseconds) for DiaSource to Source association"
282  )
283 
284  def setDefaults(self):
285  # defaults are OK for catalog and diacatalog
286 
287  self.subtract['al'].kernel.name = "AL"
288  self.subtract['al'].kernel.active.fitForBackground = True
289  self.subtract['al'].kernel.active.spatialKernelOrder = 1
290  self.subtract['al'].kernel.active.spatialBgOrder = 2
291  self.doPreConvolve = False
292  self.doMatchSources = False
293  self.doAddMetrics = False
294  self.doUseRegister = False
295 
296  # DiaSource Detection
297  self.detection.thresholdPolarity = "both"
298  self.detection.thresholdValue = 5.5
299  self.detection.reEstimateBackground = False
300  self.detection.thresholdType = "pixel_stdev"
301 
302  # Add filtered flux measurement, the correct measurement for pre-convolved images.
303  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
304  # To change that you must modify algorithms.names in the task's applyOverrides method,
305  # after the user has set doPreConvolve.
306  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
307  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
308  'base_LocalWcs']
309 
310  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
311  self.forcedMeasurement.copyColumns = {
312  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
313  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
314  self.forcedMeasurement.slots.shape = None
315 
316  # For shuffling the control sample
317  random.seed(self.controlRandomSeed)
318 
319  def validate(self):
320  pexConfig.Config.validate(self)
321  if self.doAddMetrics and not self.doSubtract:
322  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
323  if not self.doSubtract and not self.doDetection:
324  raise ValueError("Either doSubtract or doDetection must be enabled.")
325  if self.subtract.name == 'zogy' and self.doAddMetrics:
326  raise ValueError("Kernel metrics does not exist in zogy subtraction.")
327  if self.doMeasurement and not self.doDetection:
328  raise ValueError("Cannot run source measurement without source detection.")
329  if self.doMerge and not self.doDetection:
330  raise ValueError("Cannot run source merging without source detection.")
331  if self.doUseRegister and not self.doSelectSources:
332  raise ValueError("doUseRegister=True and doSelectSources=False. "
333  "Cannot run RegisterTask without selecting sources.")
334  if self.doPreConvolve and self.doDecorrelation and not self.convolveTemplate:
335  raise ValueError("doPreConvolve=True and doDecorrelation=True and "
336  "convolveTemplate=False is not supported.")
337  if hasattr(self.getTemplate, "coaddName"):
338  if self.getTemplate.coaddName != self.coaddName:
339  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
340  if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
341  raise ValueError("Scaling the diffim variance and scaling the template variance "
342  "are both set. Please choose one or the other.")
343 
344 
345 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
346 
347  @staticmethod
348  def getTargetList(parsedCmd, **kwargs):
349  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
350  **kwargs)
351 
352 
353 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
354  """Subtract an image from a template and measure the result
355  """
356  ConfigClass = ImageDifferenceConfig
357  RunnerClass = ImageDifferenceTaskRunner
358  _DefaultName = "imageDifference"
359 
360  def __init__(self, butler=None, **kwargs):
361  """!Construct an ImageDifference Task
362 
363  @param[in] butler Butler object to use in constructing reference object loaders
364  """
365  super().__init__(**kwargs)
366  self.makeSubtask("getTemplate")
367 
368  self.makeSubtask("subtract")
369 
370  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
371  self.makeSubtask("decorrelate")
372 
373  if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
374  self.makeSubtask("scaleVariance")
375 
376  if self.config.doUseRegister:
377  self.makeSubtask("register")
378  self.schema = afwTable.SourceTable.makeMinimalSchema()
379 
380  if self.config.doSelectSources:
381  self.makeSubtask("sourceSelector")
382  if self.config.kernelSourcesFromRef:
383  self.makeSubtask('refObjLoader', butler=butler)
384  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
385 
386  self.algMetadata = dafBase.PropertyList()
387  if self.config.doDetection:
388  self.makeSubtask("detection", schema=self.schema)
389  if self.config.doMeasurement:
390  self.makeSubtask("measurement", schema=self.schema,
391  algMetadata=self.algMetadata)
392  if self.config.doApCorr:
393  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
394  if self.config.doForcedMeasurement:
395  self.schema.addField(
396  "ip_diffim_forced_PsfFlux_instFlux", "D",
397  "Forced PSF flux measured on the direct image.",
398  units="count")
399  self.schema.addField(
400  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
401  "Forced PSF flux error measured on the direct image.",
402  units="count")
403  self.schema.addField(
404  "ip_diffim_forced_PsfFlux_area", "F",
405  "Forced PSF flux effective area of PSF.",
406  units="pixel")
407  self.schema.addField(
408  "ip_diffim_forced_PsfFlux_flag", "Flag",
409  "Forced PSF flux general failure flag.")
410  self.schema.addField(
411  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
412  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
413  self.schema.addField(
414  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
415  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
416  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
417  if self.config.doMatchSources:
418  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
419  self.schema.addField("srcMatchId", "L", "unique id of source match")
420 
421  # initialize InitOutputs
422  self.outputSchema = afwTable.SourceCatalog(self.schema)
423  self.outputSchema.getTable().setMetadata(self.algMetadata)
424 
425  @staticmethod
426  def makeIdFactory(expId, expBits):
427  """Create IdFactory instance for unique 64 bit diaSource id-s.
428 
429  Parameters
430  ----------
431  expId : `int`
432  Exposure id.
433 
434  expBits: `int`
435  Number of used bits in ``expId``.
436 
437  Note
438  ----
439  The diasource id-s consists of the ``expId`` stored fixed in the highest value
440  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
441  low value end of the integer.
442 
443  Returns
444  -------
445  idFactory: `lsst.afw.table.IdFactory`
446  """
447  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
448 
449  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
450  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
451  inputRefs: pipeBase.InputQuantizedConnection,
452  outputRefs: pipeBase.OutputQuantizedConnection):
453  inputs = butlerQC.get(inputRefs)
454  self.log.info(f"Processing {butlerQC.quantum.dataId}")
455  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
456  returnMaxBits=True)
457  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
458  if self.config.coaddName == 'dcr':
459  templateExposures = inputRefs.dcrCoadds
460  else:
461  templateExposures = inputRefs.coaddExposures
462  templateStruct = self.getTemplate.runQuantum(
463  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
464  )
465 
466  outputs = self.run(exposure=inputs['exposure'],
467  templateExposure=templateStruct.exposure,
468  idFactory=idFactory)
469  butlerQC.put(outputs, outputRefs)
470 
471  @pipeBase.timeMethod
472  def runDataRef(self, sensorRef, templateIdList=None):
473  """Subtract an image from a template coadd and measure the result.
474 
475  Data I/O wrapper around `run` using the butler in Gen2.
476 
477  Parameters
478  ----------
479  sensorRef : `lsst.daf.persistence.ButlerDataRef`
480  Sensor-level butler data reference, used for the following data products:
481 
482  Input only:
483  - calexp
484  - psf
485  - ccdExposureId
486  - ccdExposureId_bits
487  - self.config.coaddName + "Coadd_skyMap"
488  - self.config.coaddName + "Coadd"
489  Input or output, depending on config:
490  - self.config.coaddName + "Diff_subtractedExp"
491  Output, depending on config:
492  - self.config.coaddName + "Diff_matchedExp"
493  - self.config.coaddName + "Diff_src"
494 
495  Returns
496  -------
497  results : `lsst.pipe.base.Struct`
498  Returns the Struct by `run`.
499  """
500  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
501  subtractedExposure = None
502  selectSources = None
503  calexpBackgroundExposure = None
504  self.log.info("Processing %s" % (sensorRef.dataId))
505 
506  # We make one IdFactory that will be used by both icSrc and src datasets;
507  # I don't know if this is the way we ultimately want to do things, but at least
508  # this ensures the source IDs are fully unique.
509  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
510  expBits=sensorRef.get("ccdExposureId_bits"))
511  if self.config.doAddCalexpBackground:
512  calexpBackgroundExposure = sensorRef.get("calexpBackground")
513 
514  # Retrieve the science image we wish to analyze
515  exposure = sensorRef.get("calexp", immediate=True)
516 
517  # Retrieve the template image
518  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
519 
520  if sensorRef.datasetExists("src"):
521  self.log.info("Source selection via src product")
522  # Sources already exist; for data release processing
523  selectSources = sensorRef.get("src")
524 
525  if not self.config.doSubtract and self.config.doDetection:
526  # If we don't do subtraction, we need the subtracted exposure from the repo
527  subtractedExposure = sensorRef.get(subtractedExposureName)
528  # Both doSubtract and doDetection cannot be False
529 
530  results = self.run(exposure=exposure,
531  selectSources=selectSources,
532  templateExposure=template.exposure,
533  templateSources=template.sources,
534  idFactory=idFactory,
535  calexpBackgroundExposure=calexpBackgroundExposure,
536  subtractedExposure=subtractedExposure)
537 
538  if self.config.doWriteSources and results.diaSources is not None:
539  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
540  if self.config.doWriteWarpedExp:
541  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
542  if self.config.doWriteMatchedExp:
543  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
544  if self.config.doAddMetrics and self.config.doSelectSources:
545  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
546  if self.config.doWriteSubtractedExp:
547  sensorRef.put(results.subtractedExposure, subtractedExposureName)
548  return results
549 
550  @pipeBase.timeMethod
551  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
552  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
553  """PSF matches, subtract two images and perform detection on the difference image.
554 
555  Parameters
556  ----------
557  exposure : `lsst.afw.image.ExposureF`, optional
558  The science exposure, the minuend in the image subtraction.
559  Can be None only if ``config.doSubtract==False``.
560  selectSources : `lsst.afw.table.SourceCatalog`, optional
561  Identified sources on the science exposure. This catalog is used to
562  select sources in order to perform the AL PSF matching on stamp images
563  around them. The selection steps depend on config options and whether
564  ``templateSources`` and ``matchingSources`` specified.
565  templateExposure : `lsst.afw.image.ExposureF`, optional
566  The template to be subtracted from ``exposure`` in the image subtraction.
567  The template exposure should cover the same sky area as the science exposure.
568  It is either a stich of patches of a coadd skymap image or a calexp
569  of the same pointing as the science exposure. Can be None only
570  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
571  templateSources : `lsst.afw.table.SourceCatalog`, optional
572  Identified sources on the template exposure.
573  idFactory : `lsst.afw.table.IdFactory`
574  Generator object to assign ids to detected sources in the difference image.
575  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
576  Background exposure to be added back to the science exposure
577  if ``config.doAddCalexpBackground==True``
578  subtractedExposure : `lsst.afw.image.ExposureF`, optional
579  If ``config.doSubtract==False`` and ``config.doDetection==True``,
580  performs the post subtraction source detection only on this exposure.
581  Otherwise should be None.
582 
583  Returns
584  -------
585  results : `lsst.pipe.base.Struct`
586  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
587  Difference image.
588  ``matchedExposure`` : `lsst.afw.image.ExposureF`
589  The matched PSF exposure.
590  ``subtractRes`` : `lsst.pipe.base.Struct`
591  The returned result structure of the ImagePsfMatchTask subtask.
592  ``diaSources`` : `lsst.afw.table.SourceCatalog`
593  The catalog of detected sources.
594  ``selectSources`` : `lsst.afw.table.SourceCatalog`
595  The input source catalog with optionally added Qa information.
596 
597  Notes
598  -----
599  The following major steps are included:
600 
601  - warp template coadd to match WCS of image
602  - PSF match image to warped template
603  - subtract image from PSF-matched, warped template
604  - detect sources
605  - measure sources
606 
607  For details about the image subtraction configuration modes
608  see `lsst.ip.diffim`.
609  """
610  subtractRes = None
611  controlSources = None
612  diaSources = None
613  kernelSources = None
614 
615  if self.config.doAddCalexpBackground:
616  mi = exposure.getMaskedImage()
617  mi += calexpBackgroundExposure.getImage()
618 
619  if not exposure.hasPsf():
620  raise pipeBase.TaskError("Exposure has no psf")
621  sciencePsf = exposure.getPsf()
622 
623  if self.config.doSubtract:
624  if self.config.doScaleTemplateVariance:
625  self.log.info("Rescaling template variance")
626  templateVarFactor = self.scaleVariance.run(
627  templateExposure.getMaskedImage())
628  self.log.info("Template variance scaling factor: %.2f" % templateVarFactor)
629  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
630 
631  if self.config.subtract.name == 'zogy':
632  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
633  if self.config.doPreConvolve:
634  subtractedExposure = subtractRes.scoreExp
635  else:
636  subtractedExposure = subtractRes.diffExp
637  subtractRes.subtractedExposure = subtractedExposure
638  subtractRes.matchedExposure = None
639 
640  elif self.config.subtract.name == 'al':
641  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
642  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
643  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
644 
645  # if requested, convolve the science exposure with its PSF
646  # (properly, this should be a cross-correlation, but our code does not yet support that)
647  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
648  # else sigma of original science exposure
649  # TODO: DM-22762 This functional block should be moved into its own method
650  preConvPsf = None
651  if self.config.doPreConvolve:
652  convControl = afwMath.ConvolutionControl()
653  # cannot convolve in place, so make a new MI to receive convolved image
654  srcMI = exposure.getMaskedImage()
655  exposureOrig = exposure.clone()
656  destMI = srcMI.Factory(srcMI.getDimensions())
657  srcPsf = sciencePsf
658  if self.config.useGaussianForPreConvolution:
659  # convolve with a simplified PSF model: a double Gaussian
660  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
661  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
662  else:
663  # convolve with science exposure's PSF model
664  preConvPsf = srcPsf
665  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
666  exposure.setMaskedImage(destMI)
667  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
668  else:
669  scienceSigmaPost = scienceSigmaOrig
670  exposureOrig = exposure
671 
672  # If requested, find and select sources from the image
673  # else, AL subtraction will do its own source detection
674  # TODO: DM-22762 This functional block should be moved into its own method
675  if self.config.doSelectSources:
676  if selectSources is None:
677  self.log.warn("Src product does not exist; running detection, measurement, selection")
678  # Run own detection and measurement; necessary in nightly processing
679  selectSources = self.subtract.getSelectSources(
680  exposure,
681  sigma=scienceSigmaPost,
682  doSmooth=not self.config.doPreConvolve,
683  idFactory=idFactory,
684  )
685 
686  if self.config.doAddMetrics:
687  # Number of basis functions
688 
689  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
690  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
691  targetFwhmPix=templateSigma*FwhmPerSigma))
692  # Modify the schema of all Sources
693  # DEPRECATED: This is a data dependent (nparam) output product schema
694  # outside the task constructor.
695  # NOTE: The pre-determination of nparam at this point
696  # may be incorrect as the template psf is warped later in
697  # ImagePsfMatchTask.matchExposures()
698  kcQa = KernelCandidateQa(nparam)
699  selectSources = kcQa.addToSchema(selectSources)
700  if self.config.kernelSourcesFromRef:
701  # match exposure sources to reference catalog
702  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
703  matches = astromRet.matches
704  elif templateSources:
705  # match exposure sources to template sources
706  mc = afwTable.MatchControl()
707  mc.findOnlyClosest = False
708  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
709  mc)
710  else:
711  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
712  "but template sources not available. Cannot match science "
713  "sources with template sources. Run process* on data from "
714  "which templates are built.")
715 
716  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
717  matches=matches).sourceCat
718  random.shuffle(kernelSources, random.random)
719  controlSources = kernelSources[::self.config.controlStepSize]
720  kernelSources = [k for i, k in enumerate(kernelSources)
721  if i % self.config.controlStepSize]
722 
723  if self.config.doSelectDcrCatalog:
724  redSelector = DiaCatalogSourceSelectorTask(
725  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
726  grMax=99.999))
727  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
728  controlSources.extend(redSources)
729 
730  blueSelector = DiaCatalogSourceSelectorTask(
731  DiaCatalogSourceSelectorConfig(grMin=-99.999,
732  grMax=self.sourceSelector.config.grMin))
733  blueSources = blueSelector.selectStars(exposure, selectSources,
734  matches=matches).starCat
735  controlSources.extend(blueSources)
736 
737  if self.config.doSelectVariableCatalog:
738  varSelector = DiaCatalogSourceSelectorTask(
739  DiaCatalogSourceSelectorConfig(includeVariable=True))
740  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
741  controlSources.extend(varSources)
742 
743  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
744  % (len(kernelSources), len(selectSources), len(controlSources)))
745 
746  allresids = {}
747  # TODO: DM-22762 This functional block should be moved into its own method
748  if self.config.doUseRegister:
749  self.log.info("Registering images")
750 
751  if templateSources is None:
752  # Run detection on the template, which is
753  # temporarily background-subtracted
754  # sigma of PSF of template image before warping
755  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
756  templateSources = self.subtract.getSelectSources(
757  templateExposure,
758  sigma=templateSigma,
759  doSmooth=True,
760  idFactory=idFactory
761  )
762 
763  # Third step: we need to fit the relative astrometry.
764  #
765  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
766  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
767  exposure.getWcs(), exposure.getBBox())
768  templateExposure = warpedExp
769 
770  # Create debugging outputs on the astrometric
771  # residuals as a function of position. Persistence
772  # not yet implemented; expected on (I believe) #2636.
773  if self.config.doDebugRegister:
774  # Grab matches to reference catalog
775  srcToMatch = {x.second.getId(): x.first for x in matches}
776 
777  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
778  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
779  sids = [m.first.getId() for m in wcsResults.matches]
780  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
781  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
782  m.second.get(inCentroidKey))) for m in wcsResults.matches]
783  allresids = dict(zip(sids, zip(positions, residuals)))
784 
785  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
786  wcsResults.wcs.pixelToSky(
787  m.second.get(inCentroidKey))) for m in wcsResults.matches]
788  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
789  + 2.5*numpy.log10(srcToMatch[x].get("r"))
790  for x in sids if x in srcToMatch.keys()])
791  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
792  if s in srcToMatch.keys()])
793  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
794  if s in srcToMatch.keys()])
795  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
796  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
797  & (colors <= self.sourceSelector.config.grMax))
798  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
799  rms1Long = IqrToSigma*(
800  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
801  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
802  - numpy.percentile(dlat[idx1], 25))
803  rms2Long = IqrToSigma*(
804  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
805  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
806  - numpy.percentile(dlat[idx2], 25))
807  rms3Long = IqrToSigma*(
808  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
809  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
810  - numpy.percentile(dlat[idx3], 25))
811  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
812  (numpy.median(dlong[idx1]), rms1Long,
813  numpy.median(dlat[idx1]), rms1Lat))
814  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
815  (numpy.median(dlong[idx2]), rms2Long,
816  numpy.median(dlat[idx2]), rms2Lat))
817  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
818  (numpy.median(dlong[idx3]), rms3Long,
819  numpy.median(dlat[idx3]), rms3Lat))
820 
821  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
822  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
823  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
824  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
825  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
826  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
827 
828  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
829  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
830  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
831  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
832  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
833  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
834 
835  # warp template exposure to match exposure,
836  # PSF match template exposure to exposure,
837  # then return the difference
838 
839  # Return warped template... Construct sourceKernelCand list after subtract
840  self.log.info("Subtracting images")
841  subtractRes = self.subtract.subtractExposures(
842  templateExposure=templateExposure,
843  scienceExposure=exposure,
844  candidateList=kernelSources,
845  convolveTemplate=self.config.convolveTemplate,
846  doWarping=not self.config.doUseRegister
847  )
848  subtractedExposure = subtractRes.subtractedExposure
849 
850  if self.config.doDetection:
851  self.log.info("Computing diffim PSF")
852 
853  # Get Psf from the appropriate input image if it doesn't exist
854  if not subtractedExposure.hasPsf():
855  if self.config.convolveTemplate:
856  subtractedExposure.setPsf(exposure.getPsf())
857  else:
858  subtractedExposure.setPsf(templateExposure.getPsf())
859 
860  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
861  # thus it may have already been decorrelated. Thus, we do not decorrelate if
862  # doSubtract is False.
863 
864  # NOTE: At this point doSubtract == True
865  if self.config.doDecorrelation and self.config.doSubtract:
866  preConvKernel = None
867  if preConvPsf is not None:
868  preConvKernel = preConvPsf.getLocalKernel()
869  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
870  subtractedExposure,
871  subtractRes.psfMatchingKernel,
872  spatiallyVarying=self.config.doSpatiallyVarying,
873  preConvKernel=preConvKernel,
874  templateMatched=self.config.convolveTemplate)
875  subtractedExposure = decorrResult.correctedExposure
876 
877  # END (if subtractAlgorithm == 'AL')
878  # END (if self.config.doSubtract)
879  if self.config.doDetection:
880  self.log.info("Running diaSource detection")
881 
882  # Rescale difference image variance plane
883  if self.config.doScaleDiffimVariance:
884  self.log.info("Rescaling diffim variance")
885  diffimVarFactor = self.scaleVariance.run(subtractedExposure.getMaskedImage())
886  self.log.info("Diffim variance scaling factor: %.2f" % diffimVarFactor)
887  self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
888 
889  # Erase existing detection mask planes
890  mask = subtractedExposure.getMaskedImage().getMask()
891  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
892 
893  table = afwTable.SourceTable.make(self.schema, idFactory)
894  table.setMetadata(self.algMetadata)
895  results = self.detection.run(
896  table=table,
897  exposure=subtractedExposure,
898  doSmooth=not self.config.doPreConvolve
899  )
900 
901  if self.config.doMerge:
902  fpSet = results.fpSets.positive
903  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
904  self.config.growFootprint, False)
905  diaSources = afwTable.SourceCatalog(table)
906  fpSet.makeSources(diaSources)
907  self.log.info("Merging detections into %d sources" % (len(diaSources)))
908  else:
909  diaSources = results.sources
910 
911  if self.config.doMeasurement:
912  newDipoleFitting = self.config.doDipoleFitting
913  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
914  if not newDipoleFitting:
915  # Just fit dipole in diffim
916  self.measurement.run(diaSources, subtractedExposure)
917  else:
918  # Use (matched) template and science image (if avail.) to constrain dipole fitting
919  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
920  self.measurement.run(diaSources, subtractedExposure, exposure,
921  subtractRes.matchedExposure)
922  else:
923  self.measurement.run(diaSources, subtractedExposure, exposure)
924  if self.config.doApCorr:
925  self.applyApCorr.run(
926  catalog=diaSources,
927  apCorrMap=subtractedExposure.getInfo().getApCorrMap()
928  )
929 
930  if self.config.doForcedMeasurement:
931  # Run forced psf photometry on the PVI at the diaSource locations.
932  # Copy the measured flux and error into the diaSource.
933  forcedSources = self.forcedMeasurement.generateMeasCat(
934  exposure, diaSources, subtractedExposure.getWcs())
935  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
936  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
937  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
938  "ip_diffim_forced_PsfFlux_instFlux", True)
939  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
940  "ip_diffim_forced_PsfFlux_instFluxErr", True)
941  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
942  "ip_diffim_forced_PsfFlux_area", True)
943  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
944  "ip_diffim_forced_PsfFlux_flag", True)
945  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
946  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
947  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
948  "ip_diffim_forced_PsfFlux_flag_edge", True)
949  for diaSource, forcedSource in zip(diaSources, forcedSources):
950  diaSource.assign(forcedSource, mapper)
951 
952  # Match with the calexp sources if possible
953  if self.config.doMatchSources:
954  if selectSources is not None:
955  # Create key,val pair where key=diaSourceId and val=sourceId
956  matchRadAsec = self.config.diaSourceMatchRadius
957  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
958 
959  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
960  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
961  srcMatch in srcMatches])
962  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
963  len(diaSources)))
964  else:
965  self.log.warn("Src product does not exist; cannot match with diaSources")
966  srcMatchDict = {}
967 
968  # Create key,val pair where key=diaSourceId and val=refId
969  refAstromConfig = AstrometryConfig()
970  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
971  refAstrometer = AstrometryTask(refAstromConfig)
972  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
973  refMatches = astromRet.matches
974  if refMatches is None:
975  self.log.warn("No diaSource matches with reference catalog")
976  refMatchDict = {}
977  else:
978  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
979  len(diaSources)))
980  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
981  refMatch in refMatches])
982 
983  # Assign source Ids
984  for diaSource in diaSources:
985  sid = diaSource.getId()
986  if sid in srcMatchDict:
987  diaSource.set("srcMatchId", srcMatchDict[sid])
988  if sid in refMatchDict:
989  diaSource.set("refMatchId", refMatchDict[sid])
990 
991  if self.config.doAddMetrics and self.config.doSelectSources:
992  self.log.info("Evaluating metrics and control sample")
993 
994  kernelCandList = []
995  for cell in subtractRes.kernelCellSet.getCellList():
996  for cand in cell.begin(False): # include bad candidates
997  kernelCandList.append(cand)
998 
999  # Get basis list to build control sample kernels
1000  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1001  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1002 
1003  controlCandList = (
1004  diffimTools.sourceTableToCandidateList(controlSources,
1005  subtractRes.warpedExposure, exposure,
1006  self.config.subtract.kernel.active,
1007  self.config.subtract.kernel.active.detectionConfig,
1008  self.log, doBuild=True, basisList=basisList))
1009 
1010  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1011  subtractRes.backgroundModel, dof=nparam)
1012  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1013  subtractRes.backgroundModel)
1014 
1015  if self.config.doDetection:
1016  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1017  else:
1018  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1019 
1020  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1021  return pipeBase.Struct(
1022  subtractedExposure=subtractedExposure,
1023  warpedExposure=subtractRes.warpedExposure,
1024  matchedExposure=subtractRes.matchedExposure,
1025  subtractRes=subtractRes,
1026  diaSources=diaSources,
1027  selectSources=selectSources
1028  )
1029 
1030  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1031  """Fit the relative astrometry between templateSources and selectSources
1032 
1033  Todo
1034  ----
1035 
1036  Remove this method. It originally fit a new WCS to the template before calling register.run
1037  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1038  It remains because a subtask overrides it.
1039  """
1040  results = self.register.run(templateSources, templateExposure.getWcs(),
1041  templateExposure.getBBox(), selectSources)
1042  return results
1043 
1044  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1045  """Make debug plots and displays.
1046 
1047  Todo
1048  ----
1049  Test and update for current debug display and slot names
1050  """
1051  import lsstDebug
1052  display = lsstDebug.Info(__name__).display
1053  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1054  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1055  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1056  showDipoles = lsstDebug.Info(__name__).showDipoles
1057  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1058  if display:
1059  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1060  if not maskTransparency:
1061  maskTransparency = 0
1062  disp.setMaskTransparency(maskTransparency)
1063 
1064  if display and showSubtracted:
1065  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1066  mi = subtractRes.subtractedExposure.getMaskedImage()
1067  x0, y0 = mi.getX0(), mi.getY0()
1068  with disp.Buffering():
1069  for s in diaSources:
1070  x, y = s.getX() - x0, s.getY() - y0
1071  ctype = "red" if s.get("flags_negative") else "yellow"
1072  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1073  or s.get("base_PixelFlags_flag_saturatedCenter")
1074  or s.get("base_PixelFlags_flag_crCenter")):
1075  ptype = "x"
1076  elif (s.get("base_PixelFlags_flag_interpolated")
1077  or s.get("base_PixelFlags_flag_saturated")
1078  or s.get("base_PixelFlags_flag_cr")):
1079  ptype = "+"
1080  else:
1081  ptype = "o"
1082  disp.dot(ptype, x, y, size=4, ctype=ctype)
1083  lsstDebug.frame += 1
1084 
1085  if display and showPixelResiduals and selectSources:
1086  nonKernelSources = []
1087  for source in selectSources:
1088  if source not in kernelSources:
1089  nonKernelSources.append(source)
1090 
1091  diUtils.plotPixelResiduals(exposure,
1092  subtractRes.warpedExposure,
1093  subtractRes.subtractedExposure,
1094  subtractRes.kernelCellSet,
1095  subtractRes.psfMatchingKernel,
1096  subtractRes.backgroundModel,
1097  nonKernelSources,
1098  self.subtract.config.kernel.active.detectionConfig,
1099  origVariance=False)
1100  diUtils.plotPixelResiduals(exposure,
1101  subtractRes.warpedExposure,
1102  subtractRes.subtractedExposure,
1103  subtractRes.kernelCellSet,
1104  subtractRes.psfMatchingKernel,
1105  subtractRes.backgroundModel,
1106  nonKernelSources,
1107  self.subtract.config.kernel.active.detectionConfig,
1108  origVariance=True)
1109  if display and showDiaSources:
1110  flagChecker = SourceFlagChecker(diaSources)
1111  isFlagged = [flagChecker(x) for x in diaSources]
1112  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1113  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1114  frame=lsstDebug.frame)
1115  lsstDebug.frame += 1
1116 
1117  if display and showDipoles:
1118  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1119  frame=lsstDebug.frame)
1120  lsstDebug.frame += 1
1121 
1122  def _getConfigName(self):
1123  """Return the name of the config dataset
1124  """
1125  return "%sDiff_config" % (self.config.coaddName,)
1126 
1127  def _getMetadataName(self):
1128  """Return the name of the metadata dataset
1129  """
1130  return "%sDiff_metadata" % (self.config.coaddName,)
1131 
1132  def getSchemaCatalogs(self):
1133  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1134  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1135 
1136  @classmethod
1137  def _makeArgumentParser(cls):
1138  """Create an argument parser
1139  """
1140  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1141  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1142  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1143  help="Template data ID in case of calexp template,"
1144  " e.g. --templateId visit=6789")
1145  return parser
1146 
1147 
1148 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1149  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1150  doc="Shift stars going into RegisterTask by this amount")
1151  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1152  doc="Perturb stars going into RegisterTask by this amount")
1153 
1154  def setDefaults(self):
1155  ImageDifferenceConfig.setDefaults(self)
1156  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1157 
1158 
1159 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1160  """!Image difference Task used in the Winter 2013 data challege.
1161  Enables testing the effects of registration shifts and scatter.
1162 
1163  For use with winter 2013 simulated images:
1164  Use --templateId visit=88868666 for sparse data
1165  --templateId visit=22222200 for dense data (g)
1166  --templateId visit=11111100 for dense data (i)
1167  """
1168  ConfigClass = Winter2013ImageDifferenceConfig
1169  _DefaultName = "winter2013ImageDifference"
1170 
1171  def __init__(self, **kwargs):
1172  ImageDifferenceTask.__init__(self, **kwargs)
1173 
1174  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1175  """Fit the relative astrometry between templateSources and selectSources"""
1176  if self.config.winter2013WcsShift > 0.0:
1177  offset = geom.Extent2D(self.config.winter2013WcsShift,
1178  self.config.winter2013WcsShift)
1179  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1180  for source in templateSources:
1181  centroid = source.get(cKey)
1182  source.set(cKey, centroid + offset)
1183  elif self.config.winter2013WcsRms > 0.0:
1184  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1185  for source in templateSources:
1186  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1187  self.config.winter2013WcsRms*numpy.random.normal())
1188  centroid = source.get(cKey)
1189  source.set(cKey, centroid + offset)
1190 
1191  results = self.register.run(templateSources, templateExposure.getWcs(),
1192  templateExposure.getBBox(), selectSources)
1193  return results
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)