lsst.pipe.tasks  21.0.0-37-gd4ca0074+7bc44fbdd3
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 from lsst.skymap import BaseSkyMap
47 
48 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
49 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
50 IqrToSigma = 0.741
51 
52 
53 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
54  dimensions=("instrument", "visit", "detector", "skymap"),
55  defaultTemplates={"coaddName": "deep",
56  "skyMapName": "deep",
57  "warpTypeSuffix": "",
58  "fakesType": ""}):
59 
60  exposure = pipeBase.connectionTypes.Input(
61  doc="Input science exposure to subtract from.",
62  dimensions=("instrument", "visit", "detector"),
63  storageClass="ExposureF",
64  name="{fakesType}calexp"
65  )
66 
67  # TODO DM-22953
68  # kernelSources = pipeBase.connectionTypes.Input(
69  # doc="Source catalog produced in calibrate task for kernel candidate sources",
70  # name="src",
71  # storageClass="SourceCatalog",
72  # dimensions=("instrument", "visit", "detector"),
73  # )
74 
75  skyMap = pipeBase.connectionTypes.Input(
76  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
77  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78  dimensions=("skymap", ),
79  storageClass="SkyMap",
80  )
81  coaddExposures = pipeBase.connectionTypes.Input(
82  doc="Input template to match and subtract from the exposure",
83  dimensions=("tract", "patch", "skymap", "band"),
84  storageClass="ExposureF",
85  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
86  multiple=True,
87  deferLoad=True
88  )
89  dcrCoadds = pipeBase.connectionTypes.Input(
90  doc="Input DCR template to match and subtract from the exposure",
91  name="{fakesType}dcrCoadd{warpTypeSuffix}",
92  storageClass="ExposureF",
93  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
94  multiple=True,
95  deferLoad=True
96  )
97  outputSchema = pipeBase.connectionTypes.InitOutput(
98  doc="Schema (as an example catalog) for output DIASource catalog.",
99  storageClass="SourceCatalog",
100  name="{fakesType}{coaddName}Diff_diaSrc_schema",
101  )
102  subtractedExposure = pipeBase.connectionTypes.Output(
103  doc="Output difference image",
104  dimensions=("instrument", "visit", "detector"),
105  storageClass="ExposureF",
106  name="{fakesType}{coaddName}Diff_differenceExp",
107  )
108  warpedExposure = pipeBase.connectionTypes.Output(
109  doc="Warped template used to create `subtractedExposure`.",
110  dimensions=("instrument", "visit", "detector"),
111  storageClass="ExposureF",
112  name="{fakesType}{coaddName}Diff_warpedExp",
113  )
114  diaSources = pipeBase.connectionTypes.Output(
115  doc="Output detected diaSources on the difference image",
116  dimensions=("instrument", "visit", "detector"),
117  storageClass="SourceCatalog",
118  name="{fakesType}{coaddName}Diff_diaSrc",
119  )
120 
121  def __init__(self, *, config=None):
122  super().__init__(config=config)
123  if config.coaddName == 'dcr':
124  self.inputs.remove("coaddExposures")
125  else:
126  self.inputs.remove("dcrCoadds")
127 
128  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
129  # Make kernelSources optional
130 
131 
132 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
133  pipelineConnections=ImageDifferenceTaskConnections):
134  """Config for ImageDifferenceTask
135  """
136  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
137  doc="Add background to calexp before processing it. "
138  "Useful as ipDiffim does background matching.")
139  doUseRegister = pexConfig.Field(dtype=bool, default=True,
140  doc="Use image-to-image registration to align template with "
141  "science image")
142  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
143  doc="Writing debugging data for doUseRegister")
144  doSelectSources = pexConfig.Field(dtype=bool, default=True,
145  doc="Select stars to use for kernel fitting")
146  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
147  doc="Select stars of extreme color as part of the control sample")
148  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
149  doc="Select stars that are variable to be part "
150  "of the control sample")
151  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
152  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
153  doc="Convolve science image by its PSF before PSF-matching?")
154  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
155  doc="Scale variance of the template before PSF matching")
156  doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=False,
157  doc="Scale variance of the diffim before PSF matching. "
158  "You may do either this or template variance scaling, "
159  "or neither. (Doing both is a waste of CPU.)")
160  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
161  doc="Use a simple gaussian PSF model for pre-convolution "
162  "(else use fit PSF)? Ignored if doPreConvolve false.")
163  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
164  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
165  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
166  "kernel convolution? If True, also update the diffim PSF.")
167  doMerge = pexConfig.Field(dtype=bool, default=True,
168  doc="Merge positive and negative diaSources with grow radius "
169  "set by growFootprint")
170  doMatchSources = pexConfig.Field(dtype=bool, default=True,
171  doc="Match diaSources with input calexp sources and ref catalog sources")
172  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
173  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
174  doForcedMeasurement = pexConfig.Field(
175  dtype=bool,
176  default=True,
177  doc="Force photometer diaSource locations on PVI?")
178  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
179  doWriteWarpedExp = pexConfig.Field(dtype=bool, default=False,
180  doc="Write WCS, warped template coadd exposure?")
181  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
182  doc="Write warped and PSF-matched template coadd exposure?")
183  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
184  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
185  doc="Add columns to the source table to hold analysis metrics?")
186 
187  coaddName = pexConfig.Field(
188  doc="coadd name: typically one of deep, goodSeeing, or dcr",
189  dtype=str,
190  default="deep",
191  )
192  convolveTemplate = pexConfig.Field(
193  doc="Which image gets convolved (default = template)",
194  dtype=bool,
195  default=True
196  )
197  refObjLoader = pexConfig.ConfigurableField(
198  target=LoadIndexedReferenceObjectsTask,
199  doc="reference object loader",
200  )
201  astrometer = pexConfig.ConfigurableField(
202  target=AstrometryTask,
203  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
204  )
205  sourceSelector = pexConfig.ConfigurableField(
206  target=ObjectSizeStarSelectorTask,
207  doc="Source selection algorithm",
208  )
209  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
210  decorrelate = pexConfig.ConfigurableField(
211  target=DecorrelateALKernelSpatialTask,
212  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
213  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
214  "default of 5.5).",
215  )
216  doSpatiallyVarying = pexConfig.Field(
217  dtype=bool,
218  default=False,
219  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
220  "image in order to allow for spatial variations"
221  )
222  detection = pexConfig.ConfigurableField(
223  target=SourceDetectionTask,
224  doc="Low-threshold detection for final measurement",
225  )
226  measurement = pexConfig.ConfigurableField(
227  target=DipoleFitTask,
228  doc="Enable updated dipole fitting method",
229  )
230  forcedMeasurement = pexConfig.ConfigurableField(
231  target=ForcedMeasurementTask,
232  doc="Subtask to force photometer PVI at diaSource location.",
233  )
234  getTemplate = pexConfig.ConfigurableField(
235  target=GetCoaddAsTemplateTask,
236  doc="Subtask to retrieve template exposure and sources",
237  )
238  scaleVariance = pexConfig.ConfigurableField(
239  target=ScaleVarianceTask,
240  doc="Subtask to rescale the variance of the template "
241  "to the statistically expected level"
242  )
243  controlStepSize = pexConfig.Field(
244  doc="What step size (every Nth one) to select a control sample from the kernelSources",
245  dtype=int,
246  default=5
247  )
248  controlRandomSeed = pexConfig.Field(
249  doc="Random seed for shuffing the control sample",
250  dtype=int,
251  default=10
252  )
253  register = pexConfig.ConfigurableField(
254  target=RegisterTask,
255  doc="Task to enable image-to-image image registration (warping)",
256  )
257  kernelSourcesFromRef = pexConfig.Field(
258  doc="Select sources to measure kernel from reference catalog if True, template if false",
259  dtype=bool,
260  default=False
261  )
262  templateSipOrder = pexConfig.Field(
263  dtype=int, default=2,
264  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
265  )
266  growFootprint = pexConfig.Field(
267  dtype=int, default=2,
268  doc="Grow positive and negative footprints by this amount before merging"
269  )
270  diaSourceMatchRadius = pexConfig.Field(
271  dtype=float, default=0.5,
272  doc="Match radius (in arcseconds) for DiaSource to Source association"
273  )
274 
275  def setDefaults(self):
276  # defaults are OK for catalog and diacatalog
277 
278  self.subtract['al'].kernel.name = "AL"
279  self.subtract['al'].kernel.active.fitForBackground = True
280  self.subtract['al'].kernel.active.spatialKernelOrder = 1
281  self.subtract['al'].kernel.active.spatialBgOrder = 2
282  self.doPreConvolve = False
283  self.doMatchSources = False
284  self.doAddMetrics = False
285  self.doUseRegister = False
286 
287  # DiaSource Detection
288  self.detection.thresholdPolarity = "both"
289  self.detection.thresholdValue = 5.5
290  self.detection.reEstimateBackground = False
291  self.detection.thresholdType = "pixel_stdev"
292 
293  # Add filtered flux measurement, the correct measurement for pre-convolved images.
294  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
295  # To change that you must modify algorithms.names in the task's applyOverrides method,
296  # after the user has set doPreConvolve.
297  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
298  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
299  'base_LocalWcs']
300 
301  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
302  self.forcedMeasurement.copyColumns = {
303  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
304  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
305  self.forcedMeasurement.slots.shape = None
306 
307  # For shuffling the control sample
308  random.seed(self.controlRandomSeed)
309 
310  def validate(self):
311  pexConfig.Config.validate(self)
312  if self.doAddMetrics and not self.doSubtract:
313  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
314  if not self.doSubtract and not self.doDetection:
315  raise ValueError("Either doSubtract or doDetection must be enabled.")
316  if self.subtract.name == 'zogy' and self.doAddMetrics:
317  raise ValueError("Kernel metrics does not exist in zogy subtraction.")
318  if self.doMeasurement and not self.doDetection:
319  raise ValueError("Cannot run source measurement without source detection.")
320  if self.doMerge and not self.doDetection:
321  raise ValueError("Cannot run source merging without source detection.")
322  if self.doUseRegister and not self.doSelectSources:
323  raise ValueError("doUseRegister=True and doSelectSources=False. "
324  "Cannot run RegisterTask without selecting sources.")
325  if self.doPreConvolve and self.doDecorrelation and not self.convolveTemplate:
326  raise ValueError("doPreConvolve=True and doDecorrelation=True and "
327  "convolveTemplate=False is not supported.")
328  if hasattr(self.getTemplate, "coaddName"):
329  if self.getTemplate.coaddName != self.coaddName:
330  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
331  if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
332  raise ValueError("Scaling the diffim variance and scaling the template variance "
333  "are both set. Please choose one or the other.")
334 
335 
336 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
337 
338  @staticmethod
339  def getTargetList(parsedCmd, **kwargs):
340  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
341  **kwargs)
342 
343 
344 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
345  """Subtract an image from a template and measure the result
346  """
347  ConfigClass = ImageDifferenceConfig
348  RunnerClass = ImageDifferenceTaskRunner
349  _DefaultName = "imageDifference"
350 
351  def __init__(self, butler=None, **kwargs):
352  """!Construct an ImageDifference Task
353 
354  @param[in] butler Butler object to use in constructing reference object loaders
355  """
356  super().__init__(**kwargs)
357  self.makeSubtask("getTemplate")
358 
359  self.makeSubtask("subtract")
360 
361  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
362  self.makeSubtask("decorrelate")
363 
364  if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
365  self.makeSubtask("scaleVariance")
366 
367  if self.config.doUseRegister:
368  self.makeSubtask("register")
369  self.schema = afwTable.SourceTable.makeMinimalSchema()
370 
371  if self.config.doSelectSources:
372  self.makeSubtask("sourceSelector")
373  if self.config.kernelSourcesFromRef:
374  self.makeSubtask('refObjLoader', butler=butler)
375  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
376 
377  self.algMetadata = dafBase.PropertyList()
378  if self.config.doDetection:
379  self.makeSubtask("detection", schema=self.schema)
380  if self.config.doMeasurement:
381  self.makeSubtask("measurement", schema=self.schema,
382  algMetadata=self.algMetadata)
383  if self.config.doForcedMeasurement:
384  self.schema.addField(
385  "ip_diffim_forced_PsfFlux_instFlux", "D",
386  "Forced PSF flux measured on the direct image.",
387  units="count")
388  self.schema.addField(
389  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
390  "Forced PSF flux error measured on the direct image.",
391  units="count")
392  self.schema.addField(
393  "ip_diffim_forced_PsfFlux_area", "F",
394  "Forced PSF flux effective area of PSF.",
395  units="pixel")
396  self.schema.addField(
397  "ip_diffim_forced_PsfFlux_flag", "Flag",
398  "Forced PSF flux general failure flag.")
399  self.schema.addField(
400  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
401  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
402  self.schema.addField(
403  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
404  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
405  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
406  if self.config.doMatchSources:
407  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
408  self.schema.addField("srcMatchId", "L", "unique id of source match")
409 
410  # initialize InitOutputs
411  self.outputSchema = afwTable.SourceCatalog(self.schema)
412  self.outputSchema.getTable().setMetadata(self.algMetadata)
413 
414  @staticmethod
415  def makeIdFactory(expId, expBits):
416  """Create IdFactory instance for unique 64 bit diaSource id-s.
417 
418  Parameters
419  ----------
420  expId : `int`
421  Exposure id.
422 
423  expBits: `int`
424  Number of used bits in ``expId``.
425 
426  Note
427  ----
428  The diasource id-s consists of the ``expId`` stored fixed in the highest value
429  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
430  low value end of the integer.
431 
432  Returns
433  -------
434  idFactory: `lsst.afw.table.IdFactory`
435  """
436  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
437 
438  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
439  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
440  inputRefs: pipeBase.InputQuantizedConnection,
441  outputRefs: pipeBase.OutputQuantizedConnection):
442  inputs = butlerQC.get(inputRefs)
443  self.log.info(f"Processing {butlerQC.quantum.dataId}")
444  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
445  returnMaxBits=True)
446  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
447  if self.config.coaddName == 'dcr':
448  templateExposures = inputRefs.dcrCoadds
449  else:
450  templateExposures = inputRefs.coaddExposures
451  templateStruct = self.getTemplate.runQuantum(
452  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
453  )
454 
455  outputs = self.run(exposure=inputs['exposure'],
456  templateExposure=templateStruct.exposure,
457  idFactory=idFactory)
458  butlerQC.put(outputs, outputRefs)
459 
460  @pipeBase.timeMethod
461  def runDataRef(self, sensorRef, templateIdList=None):
462  """Subtract an image from a template coadd and measure the result.
463 
464  Data I/O wrapper around `run` using the butler in Gen2.
465 
466  Parameters
467  ----------
468  sensorRef : `lsst.daf.persistence.ButlerDataRef`
469  Sensor-level butler data reference, used for the following data products:
470 
471  Input only:
472  - calexp
473  - psf
474  - ccdExposureId
475  - ccdExposureId_bits
476  - self.config.coaddName + "Coadd_skyMap"
477  - self.config.coaddName + "Coadd"
478  Input or output, depending on config:
479  - self.config.coaddName + "Diff_subtractedExp"
480  Output, depending on config:
481  - self.config.coaddName + "Diff_matchedExp"
482  - self.config.coaddName + "Diff_src"
483 
484  Returns
485  -------
486  results : `lsst.pipe.base.Struct`
487  Returns the Struct by `run`.
488  """
489  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
490  subtractedExposure = None
491  selectSources = None
492  calexpBackgroundExposure = None
493  self.log.info("Processing %s" % (sensorRef.dataId))
494 
495  # We make one IdFactory that will be used by both icSrc and src datasets;
496  # I don't know if this is the way we ultimately want to do things, but at least
497  # this ensures the source IDs are fully unique.
498  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
499  expBits=sensorRef.get("ccdExposureId_bits"))
500  if self.config.doAddCalexpBackground:
501  calexpBackgroundExposure = sensorRef.get("calexpBackground")
502 
503  # Retrieve the science image we wish to analyze
504  exposure = sensorRef.get("calexp", immediate=True)
505 
506  # Retrieve the template image
507  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
508 
509  if sensorRef.datasetExists("src"):
510  self.log.info("Source selection via src product")
511  # Sources already exist; for data release processing
512  selectSources = sensorRef.get("src")
513 
514  if not self.config.doSubtract and self.config.doDetection:
515  # If we don't do subtraction, we need the subtracted exposure from the repo
516  subtractedExposure = sensorRef.get(subtractedExposureName)
517  # Both doSubtract and doDetection cannot be False
518 
519  results = self.run(exposure=exposure,
520  selectSources=selectSources,
521  templateExposure=template.exposure,
522  templateSources=template.sources,
523  idFactory=idFactory,
524  calexpBackgroundExposure=calexpBackgroundExposure,
525  subtractedExposure=subtractedExposure)
526 
527  if self.config.doWriteSources and results.diaSources is not None:
528  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
529  if self.config.doWriteWarpedExp:
530  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
531  if self.config.doWriteMatchedExp:
532  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
533  if self.config.doAddMetrics and self.config.doSelectSources:
534  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
535  if self.config.doWriteSubtractedExp:
536  sensorRef.put(results.subtractedExposure, subtractedExposureName)
537  return results
538 
539  @pipeBase.timeMethod
540  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
541  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
542  """PSF matches, subtract two images and perform detection on the difference image.
543 
544  Parameters
545  ----------
546  exposure : `lsst.afw.image.ExposureF`, optional
547  The science exposure, the minuend in the image subtraction.
548  Can be None only if ``config.doSubtract==False``.
549  selectSources : `lsst.afw.table.SourceCatalog`, optional
550  Identified sources on the science exposure. This catalog is used to
551  select sources in order to perform the AL PSF matching on stamp images
552  around them. The selection steps depend on config options and whether
553  ``templateSources`` and ``matchingSources`` specified.
554  templateExposure : `lsst.afw.image.ExposureF`, optional
555  The template to be subtracted from ``exposure`` in the image subtraction.
556  The template exposure should cover the same sky area as the science exposure.
557  It is either a stich of patches of a coadd skymap image or a calexp
558  of the same pointing as the science exposure. Can be None only
559  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
560  templateSources : `lsst.afw.table.SourceCatalog`, optional
561  Identified sources on the template exposure.
562  idFactory : `lsst.afw.table.IdFactory`
563  Generator object to assign ids to detected sources in the difference image.
564  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
565  Background exposure to be added back to the science exposure
566  if ``config.doAddCalexpBackground==True``
567  subtractedExposure : `lsst.afw.image.ExposureF`, optional
568  If ``config.doSubtract==False`` and ``config.doDetection==True``,
569  performs the post subtraction source detection only on this exposure.
570  Otherwise should be None.
571 
572  Returns
573  -------
574  results : `lsst.pipe.base.Struct`
575  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
576  Difference image.
577  ``matchedExposure`` : `lsst.afw.image.ExposureF`
578  The matched PSF exposure.
579  ``subtractRes`` : `lsst.pipe.base.Struct`
580  The returned result structure of the ImagePsfMatchTask subtask.
581  ``diaSources`` : `lsst.afw.table.SourceCatalog`
582  The catalog of detected sources.
583  ``selectSources`` : `lsst.afw.table.SourceCatalog`
584  The input source catalog with optionally added Qa information.
585 
586  Notes
587  -----
588  The following major steps are included:
589 
590  - warp template coadd to match WCS of image
591  - PSF match image to warped template
592  - subtract image from PSF-matched, warped template
593  - detect sources
594  - measure sources
595 
596  For details about the image subtraction configuration modes
597  see `lsst.ip.diffim`.
598  """
599  subtractRes = None
600  controlSources = None
601  diaSources = None
602  kernelSources = None
603 
604  if self.config.doAddCalexpBackground:
605  mi = exposure.getMaskedImage()
606  mi += calexpBackgroundExposure.getImage()
607 
608  if not exposure.hasPsf():
609  raise pipeBase.TaskError("Exposure has no psf")
610  sciencePsf = exposure.getPsf()
611 
612  if self.config.doSubtract:
613  if self.config.doScaleTemplateVariance:
614  self.log.info("Rescaling template variance")
615  templateVarFactor = self.scaleVariance.run(
616  templateExposure.getMaskedImage())
617  self.log.info("Template variance scaling factor: %.2f" % templateVarFactor)
618  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
619 
620  if self.config.subtract.name == 'zogy':
621  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
622  if self.config.doPreConvolve:
623  subtractedExposure = subtractRes.scoreExp
624  else:
625  subtractedExposure = subtractRes.diffExp
626  subtractRes.subtractedExposure = subtractedExposure
627  subtractRes.matchedExposure = None
628 
629  elif self.config.subtract.name == 'al':
630  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
631  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
632  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
633 
634  # if requested, convolve the science exposure with its PSF
635  # (properly, this should be a cross-correlation, but our code does not yet support that)
636  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
637  # else sigma of original science exposure
638  # TODO: DM-22762 This functional block should be moved into its own method
639  preConvPsf = None
640  if self.config.doPreConvolve:
641  convControl = afwMath.ConvolutionControl()
642  # cannot convolve in place, so make a new MI to receive convolved image
643  srcMI = exposure.getMaskedImage()
644  exposureOrig = exposure.clone()
645  destMI = srcMI.Factory(srcMI.getDimensions())
646  srcPsf = sciencePsf
647  if self.config.useGaussianForPreConvolution:
648  # convolve with a simplified PSF model: a double Gaussian
649  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
650  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
651  else:
652  # convolve with science exposure's PSF model
653  preConvPsf = srcPsf
654  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
655  exposure.setMaskedImage(destMI)
656  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
657  else:
658  scienceSigmaPost = scienceSigmaOrig
659  exposureOrig = exposure
660 
661  # If requested, find and select sources from the image
662  # else, AL subtraction will do its own source detection
663  # TODO: DM-22762 This functional block should be moved into its own method
664  if self.config.doSelectSources:
665  if selectSources is None:
666  self.log.warn("Src product does not exist; running detection, measurement, selection")
667  # Run own detection and measurement; necessary in nightly processing
668  selectSources = self.subtract.getSelectSources(
669  exposure,
670  sigma=scienceSigmaPost,
671  doSmooth=not self.config.doPreConvolve,
672  idFactory=idFactory,
673  )
674 
675  if self.config.doAddMetrics:
676  # Number of basis functions
677 
678  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
679  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
680  targetFwhmPix=templateSigma*FwhmPerSigma))
681  # Modify the schema of all Sources
682  # DEPRECATED: This is a data dependent (nparam) output product schema
683  # outside the task constructor.
684  # NOTE: The pre-determination of nparam at this point
685  # may be incorrect as the template psf is warped later in
686  # ImagePsfMatchTask.matchExposures()
687  kcQa = KernelCandidateQa(nparam)
688  selectSources = kcQa.addToSchema(selectSources)
689  if self.config.kernelSourcesFromRef:
690  # match exposure sources to reference catalog
691  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
692  matches = astromRet.matches
693  elif templateSources:
694  # match exposure sources to template sources
695  mc = afwTable.MatchControl()
696  mc.findOnlyClosest = False
697  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
698  mc)
699  else:
700  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
701  "but template sources not available. Cannot match science "
702  "sources with template sources. Run process* on data from "
703  "which templates are built.")
704 
705  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
706  matches=matches).sourceCat
707  random.shuffle(kernelSources, random.random)
708  controlSources = kernelSources[::self.config.controlStepSize]
709  kernelSources = [k for i, k in enumerate(kernelSources)
710  if i % self.config.controlStepSize]
711 
712  if self.config.doSelectDcrCatalog:
713  redSelector = DiaCatalogSourceSelectorTask(
714  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
715  grMax=99.999))
716  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
717  controlSources.extend(redSources)
718 
719  blueSelector = DiaCatalogSourceSelectorTask(
720  DiaCatalogSourceSelectorConfig(grMin=-99.999,
721  grMax=self.sourceSelector.config.grMin))
722  blueSources = blueSelector.selectStars(exposure, selectSources,
723  matches=matches).starCat
724  controlSources.extend(blueSources)
725 
726  if self.config.doSelectVariableCatalog:
727  varSelector = DiaCatalogSourceSelectorTask(
728  DiaCatalogSourceSelectorConfig(includeVariable=True))
729  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
730  controlSources.extend(varSources)
731 
732  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
733  % (len(kernelSources), len(selectSources), len(controlSources)))
734 
735  allresids = {}
736  # TODO: DM-22762 This functional block should be moved into its own method
737  if self.config.doUseRegister:
738  self.log.info("Registering images")
739 
740  if templateSources is None:
741  # Run detection on the template, which is
742  # temporarily background-subtracted
743  # sigma of PSF of template image before warping
744  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
745  templateSources = self.subtract.getSelectSources(
746  templateExposure,
747  sigma=templateSigma,
748  doSmooth=True,
749  idFactory=idFactory
750  )
751 
752  # Third step: we need to fit the relative astrometry.
753  #
754  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
755  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
756  exposure.getWcs(), exposure.getBBox())
757  templateExposure = warpedExp
758 
759  # Create debugging outputs on the astrometric
760  # residuals as a function of position. Persistence
761  # not yet implemented; expected on (I believe) #2636.
762  if self.config.doDebugRegister:
763  # Grab matches to reference catalog
764  srcToMatch = {x.second.getId(): x.first for x in matches}
765 
766  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
767  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
768  sids = [m.first.getId() for m in wcsResults.matches]
769  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
770  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
771  m.second.get(inCentroidKey))) for m in wcsResults.matches]
772  allresids = dict(zip(sids, zip(positions, residuals)))
773 
774  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
775  wcsResults.wcs.pixelToSky(
776  m.second.get(inCentroidKey))) for m in wcsResults.matches]
777  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
778  + 2.5*numpy.log10(srcToMatch[x].get("r"))
779  for x in sids if x in srcToMatch.keys()])
780  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
781  if s in srcToMatch.keys()])
782  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
783  if s in srcToMatch.keys()])
784  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
785  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
786  & (colors <= self.sourceSelector.config.grMax))
787  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
788  rms1Long = IqrToSigma*(
789  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
790  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
791  - numpy.percentile(dlat[idx1], 25))
792  rms2Long = IqrToSigma*(
793  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
794  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
795  - numpy.percentile(dlat[idx2], 25))
796  rms3Long = IqrToSigma*(
797  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
798  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
799  - numpy.percentile(dlat[idx3], 25))
800  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
801  (numpy.median(dlong[idx1]), rms1Long,
802  numpy.median(dlat[idx1]), rms1Lat))
803  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
804  (numpy.median(dlong[idx2]), rms2Long,
805  numpy.median(dlat[idx2]), rms2Lat))
806  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
807  (numpy.median(dlong[idx3]), rms3Long,
808  numpy.median(dlat[idx3]), rms3Lat))
809 
810  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
811  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
812  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
813  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
814  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
815  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
816 
817  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
818  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
819  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
820  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
821  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
822  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
823 
824  # warp template exposure to match exposure,
825  # PSF match template exposure to exposure,
826  # then return the difference
827 
828  # Return warped template... Construct sourceKernelCand list after subtract
829  self.log.info("Subtracting images")
830  subtractRes = self.subtract.subtractExposures(
831  templateExposure=templateExposure,
832  scienceExposure=exposure,
833  candidateList=kernelSources,
834  convolveTemplate=self.config.convolveTemplate,
835  doWarping=not self.config.doUseRegister
836  )
837  subtractedExposure = subtractRes.subtractedExposure
838 
839  if self.config.doDetection:
840  self.log.info("Computing diffim PSF")
841 
842  # Get Psf from the appropriate input image if it doesn't exist
843  if not subtractedExposure.hasPsf():
844  if self.config.convolveTemplate:
845  subtractedExposure.setPsf(exposure.getPsf())
846  else:
847  subtractedExposure.setPsf(templateExposure.getPsf())
848 
849  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
850  # thus it may have already been decorrelated. Thus, we do not decorrelate if
851  # doSubtract is False.
852 
853  # NOTE: At this point doSubtract == True
854  if self.config.doDecorrelation and self.config.doSubtract:
855  preConvKernel = None
856  if preConvPsf is not None:
857  preConvKernel = preConvPsf.getLocalKernel()
858  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
859  subtractedExposure,
860  subtractRes.psfMatchingKernel,
861  spatiallyVarying=self.config.doSpatiallyVarying,
862  preConvKernel=preConvKernel,
863  templateMatched=self.config.convolveTemplate)
864  subtractedExposure = decorrResult.correctedExposure
865 
866  # END (if subtractAlgorithm == 'AL')
867  # END (if self.config.doSubtract)
868  if self.config.doDetection:
869  self.log.info("Running diaSource detection")
870 
871  # Rescale difference image variance plane
872  if self.config.doScaleDiffimVariance:
873  self.log.info("Rescaling diffim variance")
874  diffimVarFactor = self.scaleVariance.run(subtractedExposure.getMaskedImage())
875  self.log.info("Diffim variance scaling factor: %.2f" % diffimVarFactor)
876  self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
877 
878  # Erase existing detection mask planes
879  mask = subtractedExposure.getMaskedImage().getMask()
880  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
881 
882  table = afwTable.SourceTable.make(self.schema, idFactory)
883  table.setMetadata(self.algMetadata)
884  results = self.detection.run(
885  table=table,
886  exposure=subtractedExposure,
887  doSmooth=not self.config.doPreConvolve
888  )
889 
890  if self.config.doMerge:
891  fpSet = results.fpSets.positive
892  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
893  self.config.growFootprint, False)
894  diaSources = afwTable.SourceCatalog(table)
895  fpSet.makeSources(diaSources)
896  self.log.info("Merging detections into %d sources" % (len(diaSources)))
897  else:
898  diaSources = results.sources
899 
900  if self.config.doMeasurement:
901  newDipoleFitting = self.config.doDipoleFitting
902  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
903  if not newDipoleFitting:
904  # Just fit dipole in diffim
905  self.measurement.run(diaSources, subtractedExposure)
906  else:
907  # Use (matched) template and science image (if avail.) to constrain dipole fitting
908  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
909  self.measurement.run(diaSources, subtractedExposure, exposure,
910  subtractRes.matchedExposure)
911  else:
912  self.measurement.run(diaSources, subtractedExposure, exposure)
913 
914  if self.config.doForcedMeasurement:
915  # Run forced psf photometry on the PVI at the diaSource locations.
916  # Copy the measured flux and error into the diaSource.
917  forcedSources = self.forcedMeasurement.generateMeasCat(
918  exposure, diaSources, subtractedExposure.getWcs())
919  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
920  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
921  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
922  "ip_diffim_forced_PsfFlux_instFlux", True)
923  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
924  "ip_diffim_forced_PsfFlux_instFluxErr", True)
925  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
926  "ip_diffim_forced_PsfFlux_area", True)
927  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
928  "ip_diffim_forced_PsfFlux_flag", True)
929  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
930  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
931  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
932  "ip_diffim_forced_PsfFlux_flag_edge", True)
933  for diaSource, forcedSource in zip(diaSources, forcedSources):
934  diaSource.assign(forcedSource, mapper)
935 
936  # Match with the calexp sources if possible
937  if self.config.doMatchSources:
938  if selectSources is not None:
939  # Create key,val pair where key=diaSourceId and val=sourceId
940  matchRadAsec = self.config.diaSourceMatchRadius
941  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
942 
943  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
944  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
945  srcMatch in srcMatches])
946  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
947  len(diaSources)))
948  else:
949  self.log.warn("Src product does not exist; cannot match with diaSources")
950  srcMatchDict = {}
951 
952  # Create key,val pair where key=diaSourceId and val=refId
953  refAstromConfig = AstrometryConfig()
954  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
955  refAstrometer = AstrometryTask(refAstromConfig)
956  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
957  refMatches = astromRet.matches
958  if refMatches is None:
959  self.log.warn("No diaSource matches with reference catalog")
960  refMatchDict = {}
961  else:
962  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
963  len(diaSources)))
964  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
965  refMatch in refMatches])
966 
967  # Assign source Ids
968  for diaSource in diaSources:
969  sid = diaSource.getId()
970  if sid in srcMatchDict:
971  diaSource.set("srcMatchId", srcMatchDict[sid])
972  if sid in refMatchDict:
973  diaSource.set("refMatchId", refMatchDict[sid])
974 
975  if self.config.doAddMetrics and self.config.doSelectSources:
976  self.log.info("Evaluating metrics and control sample")
977 
978  kernelCandList = []
979  for cell in subtractRes.kernelCellSet.getCellList():
980  for cand in cell.begin(False): # include bad candidates
981  kernelCandList.append(cand)
982 
983  # Get basis list to build control sample kernels
984  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
985  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
986 
987  controlCandList = (
988  diffimTools.sourceTableToCandidateList(controlSources,
989  subtractRes.warpedExposure, exposure,
990  self.config.subtract.kernel.active,
991  self.config.subtract.kernel.active.detectionConfig,
992  self.log, doBuild=True, basisList=basisList))
993 
994  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
995  subtractRes.backgroundModel, dof=nparam)
996  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
997  subtractRes.backgroundModel)
998 
999  if self.config.doDetection:
1000  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1001  else:
1002  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1003 
1004  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1005  return pipeBase.Struct(
1006  subtractedExposure=subtractedExposure,
1007  warpedExposure=subtractRes.warpedExposure,
1008  matchedExposure=subtractRes.matchedExposure,
1009  subtractRes=subtractRes,
1010  diaSources=diaSources,
1011  selectSources=selectSources
1012  )
1013 
1014  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1015  """Fit the relative astrometry between templateSources and selectSources
1016 
1017  Todo
1018  ----
1019 
1020  Remove this method. It originally fit a new WCS to the template before calling register.run
1021  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1022  It remains because a subtask overrides it.
1023  """
1024  results = self.register.run(templateSources, templateExposure.getWcs(),
1025  templateExposure.getBBox(), selectSources)
1026  return results
1027 
1028  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1029  """Make debug plots and displays.
1030 
1031  Todo
1032  ----
1033  Test and update for current debug display and slot names
1034  """
1035  import lsstDebug
1036  display = lsstDebug.Info(__name__).display
1037  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1038  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1039  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1040  showDipoles = lsstDebug.Info(__name__).showDipoles
1041  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1042  if display:
1043  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1044  if not maskTransparency:
1045  maskTransparency = 0
1046  disp.setMaskTransparency(maskTransparency)
1047 
1048  if display and showSubtracted:
1049  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1050  mi = subtractRes.subtractedExposure.getMaskedImage()
1051  x0, y0 = mi.getX0(), mi.getY0()
1052  with disp.Buffering():
1053  for s in diaSources:
1054  x, y = s.getX() - x0, s.getY() - y0
1055  ctype = "red" if s.get("flags_negative") else "yellow"
1056  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1057  or s.get("base_PixelFlags_flag_saturatedCenter")
1058  or s.get("base_PixelFlags_flag_crCenter")):
1059  ptype = "x"
1060  elif (s.get("base_PixelFlags_flag_interpolated")
1061  or s.get("base_PixelFlags_flag_saturated")
1062  or s.get("base_PixelFlags_flag_cr")):
1063  ptype = "+"
1064  else:
1065  ptype = "o"
1066  disp.dot(ptype, x, y, size=4, ctype=ctype)
1067  lsstDebug.frame += 1
1068 
1069  if display and showPixelResiduals and selectSources:
1070  nonKernelSources = []
1071  for source in selectSources:
1072  if source not in kernelSources:
1073  nonKernelSources.append(source)
1074 
1075  diUtils.plotPixelResiduals(exposure,
1076  subtractRes.warpedExposure,
1077  subtractRes.subtractedExposure,
1078  subtractRes.kernelCellSet,
1079  subtractRes.psfMatchingKernel,
1080  subtractRes.backgroundModel,
1081  nonKernelSources,
1082  self.subtract.config.kernel.active.detectionConfig,
1083  origVariance=False)
1084  diUtils.plotPixelResiduals(exposure,
1085  subtractRes.warpedExposure,
1086  subtractRes.subtractedExposure,
1087  subtractRes.kernelCellSet,
1088  subtractRes.psfMatchingKernel,
1089  subtractRes.backgroundModel,
1090  nonKernelSources,
1091  self.subtract.config.kernel.active.detectionConfig,
1092  origVariance=True)
1093  if display and showDiaSources:
1094  flagChecker = SourceFlagChecker(diaSources)
1095  isFlagged = [flagChecker(x) for x in diaSources]
1096  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1097  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1098  frame=lsstDebug.frame)
1099  lsstDebug.frame += 1
1100 
1101  if display and showDipoles:
1102  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1103  frame=lsstDebug.frame)
1104  lsstDebug.frame += 1
1105 
1106  def _getConfigName(self):
1107  """Return the name of the config dataset
1108  """
1109  return "%sDiff_config" % (self.config.coaddName,)
1110 
1111  def _getMetadataName(self):
1112  """Return the name of the metadata dataset
1113  """
1114  return "%sDiff_metadata" % (self.config.coaddName,)
1115 
1116  def getSchemaCatalogs(self):
1117  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1118  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1119 
1120  @classmethod
1121  def _makeArgumentParser(cls):
1122  """Create an argument parser
1123  """
1124  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1125  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1126  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1127  help="Template data ID in case of calexp template,"
1128  " e.g. --templateId visit=6789")
1129  return parser
1130 
1131 
1132 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1133  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1134  doc="Shift stars going into RegisterTask by this amount")
1135  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1136  doc="Perturb stars going into RegisterTask by this amount")
1137 
1138  def setDefaults(self):
1139  ImageDifferenceConfig.setDefaults(self)
1140  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1141 
1142 
1143 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1144  """!Image difference Task used in the Winter 2013 data challege.
1145  Enables testing the effects of registration shifts and scatter.
1146 
1147  For use with winter 2013 simulated images:
1148  Use --templateId visit=88868666 for sparse data
1149  --templateId visit=22222200 for dense data (g)
1150  --templateId visit=11111100 for dense data (i)
1151  """
1152  ConfigClass = Winter2013ImageDifferenceConfig
1153  _DefaultName = "winter2013ImageDifference"
1154 
1155  def __init__(self, **kwargs):
1156  ImageDifferenceTask.__init__(self, **kwargs)
1157 
1158  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1159  """Fit the relative astrometry between templateSources and selectSources"""
1160  if self.config.winter2013WcsShift > 0.0:
1161  offset = geom.Extent2D(self.config.winter2013WcsShift,
1162  self.config.winter2013WcsShift)
1163  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1164  for source in templateSources:
1165  centroid = source.get(cKey)
1166  source.set(cKey, centroid + offset)
1167  elif self.config.winter2013WcsRms > 0.0:
1168  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1169  for source in templateSources:
1170  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1171  self.config.winter2013WcsRms*numpy.random.normal())
1172  centroid = source.get(cKey)
1173  source.set(cKey, centroid + offset)
1174 
1175  results = self.register.run(templateSources, templateExposure.getWcs(),
1176  templateExposure.getBBox(), selectSources)
1177  return results
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)