lsst.pipe.tasks  21.0.0-15-g986934a8+2928135b22
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 from lsst.skymap import BaseSkyMap
47 
48 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
49 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
50 IqrToSigma = 0.741
51 
52 
53 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
54  dimensions=("instrument", "visit", "detector", "skymap"),
55  defaultTemplates={"coaddName": "deep",
56  "skyMapName": "deep",
57  "warpTypeSuffix": "",
58  "fakesType": ""}):
59 
60  exposure = pipeBase.connectionTypes.Input(
61  doc="Input science exposure to subtract from.",
62  dimensions=("instrument", "visit", "detector"),
63  storageClass="ExposureF",
64  name="calexp"
65  )
66 
67  # TODO DM-22953
68  # kernelSources = pipeBase.connectionTypes.Input(
69  # doc="Source catalog produced in calibrate task for kernel candidate sources",
70  # name="src",
71  # storageClass="SourceCatalog",
72  # dimensions=("instrument", "visit", "detector"),
73  # )
74 
75  skyMap = pipeBase.connectionTypes.Input(
76  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
77  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
78  dimensions=("skymap", ),
79  storageClass="SkyMap",
80  )
81  coaddExposures = pipeBase.connectionTypes.Input(
82  doc="Input template to match and subtract from the exposure",
83  dimensions=("tract", "patch", "skymap", "band"),
84  storageClass="ExposureF",
85  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
86  multiple=True,
87  deferLoad=True
88  )
89  dcrCoadds = pipeBase.connectionTypes.Input(
90  doc="Input DCR template to match and subtract from the exposure",
91  name="{fakesType}dcrCoadd{warpTypeSuffix}",
92  storageClass="ExposureF",
93  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
94  multiple=True,
95  deferLoad=True
96  )
97  outputSchema = pipeBase.connectionTypes.InitOutput(
98  doc="Schema (as an example catalog) for output DIASource catalog.",
99  storageClass="SourceCatalog",
100  name="{fakesType}{coaddName}Diff_diaSrc_schema",
101  )
102  subtractedExposure = pipeBase.connectionTypes.Output(
103  doc="Output difference image",
104  dimensions=("instrument", "visit", "detector"),
105  storageClass="ExposureF",
106  name="{fakesType}{coaddName}Diff_differenceExp",
107  )
108  warpedExposure = pipeBase.connectionTypes.Output(
109  doc="Warped template used to create `subtractedExposure`.",
110  dimensions=("instrument", "visit", "detector"),
111  storageClass="ExposureF",
112  name="{fakesType}{coaddName}Diff_warpedExp",
113  )
114  diaSources = pipeBase.connectionTypes.Output(
115  doc="Output detected diaSources on the difference image",
116  dimensions=("instrument", "visit", "detector"),
117  storageClass="SourceCatalog",
118  name="{fakesType}{coaddName}Diff_diaSrc",
119  )
120 
121  def __init__(self, *, config=None):
122  super().__init__(config=config)
123  if config.coaddName == 'dcr':
124  self.inputs.remove("coaddExposures")
125  else:
126  self.inputs.remove("dcrCoadds")
127 
128  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
129  # Make kernelSources optional
130 
131 
132 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
133  pipelineConnections=ImageDifferenceTaskConnections):
134  """Config for ImageDifferenceTask
135  """
136  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
137  doc="Add background to calexp before processing it. "
138  "Useful as ipDiffim does background matching.")
139  doUseRegister = pexConfig.Field(dtype=bool, default=True,
140  doc="Use image-to-image registration to align template with "
141  "science image")
142  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
143  doc="Writing debugging data for doUseRegister")
144  doSelectSources = pexConfig.Field(dtype=bool, default=True,
145  doc="Select stars to use for kernel fitting")
146  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
147  doc="Select stars of extreme color as part of the control sample")
148  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
149  doc="Select stars that are variable to be part "
150  "of the control sample")
151  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
152  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
153  doc="Convolve science image by its PSF before PSF-matching?")
154  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
155  doc="Scale variance of the template before PSF matching")
156  doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=False,
157  doc="Scale variance of the diffim before PSF matching. "
158  "You may do either this or template variance scaling, "
159  "or neither. (Doing both is a waste of CPU.)")
160  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
161  doc="Use a simple gaussian PSF model for pre-convolution "
162  "(else use fit PSF)? Ignored if doPreConvolve false.")
163  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
164  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
165  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
166  "kernel convolution? If True, also update the diffim PSF.")
167  doMerge = pexConfig.Field(dtype=bool, default=True,
168  doc="Merge positive and negative diaSources with grow radius "
169  "set by growFootprint")
170  doMatchSources = pexConfig.Field(dtype=bool, default=True,
171  doc="Match diaSources with input calexp sources and ref catalog sources")
172  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
173  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
174  doForcedMeasurement = pexConfig.Field(
175  dtype=bool,
176  default=True,
177  doc="Force photometer diaSource locations on PVI?")
178  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
179  doWriteWarpedExp = pexConfig.Field(dtype=bool, default=False,
180  doc="Write WCS, warped template coadd exposure?")
181  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
182  doc="Write warped and PSF-matched template coadd exposure?")
183  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
184  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
185  doc="Add columns to the source table to hold analysis metrics?")
186 
187  coaddName = pexConfig.Field(
188  doc="coadd name: typically one of deep, goodSeeing, or dcr",
189  dtype=str,
190  default="deep",
191  )
192  convolveTemplate = pexConfig.Field(
193  doc="Which image gets convolved (default = template)",
194  dtype=bool,
195  default=True
196  )
197  refObjLoader = pexConfig.ConfigurableField(
198  target=LoadIndexedReferenceObjectsTask,
199  doc="reference object loader",
200  )
201  astrometer = pexConfig.ConfigurableField(
202  target=AstrometryTask,
203  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
204  )
205  sourceSelector = pexConfig.ConfigurableField(
206  target=ObjectSizeStarSelectorTask,
207  doc="Source selection algorithm",
208  )
209  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
210  decorrelate = pexConfig.ConfigurableField(
211  target=DecorrelateALKernelSpatialTask,
212  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
213  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
214  "default of 5.5).",
215  )
216  doSpatiallyVarying = pexConfig.Field(
217  dtype=bool,
218  default=False,
219  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
220  "image in order to allow for spatial variations"
221  )
222  detection = pexConfig.ConfigurableField(
223  target=SourceDetectionTask,
224  doc="Low-threshold detection for final measurement",
225  )
226  measurement = pexConfig.ConfigurableField(
227  target=DipoleFitTask,
228  doc="Enable updated dipole fitting method",
229  )
230  forcedMeasurement = pexConfig.ConfigurableField(
231  target=ForcedMeasurementTask,
232  doc="Subtask to force photometer PVI at diaSource location.",
233  )
234  getTemplate = pexConfig.ConfigurableField(
235  target=GetCoaddAsTemplateTask,
236  doc="Subtask to retrieve template exposure and sources",
237  )
238  scaleVariance = pexConfig.ConfigurableField(
239  target=ScaleVarianceTask,
240  doc="Subtask to rescale the variance of the template "
241  "to the statistically expected level"
242  )
243  controlStepSize = pexConfig.Field(
244  doc="What step size (every Nth one) to select a control sample from the kernelSources",
245  dtype=int,
246  default=5
247  )
248  controlRandomSeed = pexConfig.Field(
249  doc="Random seed for shuffing the control sample",
250  dtype=int,
251  default=10
252  )
253  register = pexConfig.ConfigurableField(
254  target=RegisterTask,
255  doc="Task to enable image-to-image image registration (warping)",
256  )
257  kernelSourcesFromRef = pexConfig.Field(
258  doc="Select sources to measure kernel from reference catalog if True, template if false",
259  dtype=bool,
260  default=False
261  )
262  templateSipOrder = pexConfig.Field(
263  dtype=int, default=2,
264  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
265  )
266  growFootprint = pexConfig.Field(
267  dtype=int, default=2,
268  doc="Grow positive and negative footprints by this amount before merging"
269  )
270  diaSourceMatchRadius = pexConfig.Field(
271  dtype=float, default=0.5,
272  doc="Match radius (in arcseconds) for DiaSource to Source association"
273  )
274 
275  def setDefaults(self):
276  # defaults are OK for catalog and diacatalog
277 
278  self.subtract['al'].kernel.name = "AL"
279  self.subtract['al'].kernel.active.fitForBackground = True
280  self.subtract['al'].kernel.active.spatialKernelOrder = 1
281  self.subtract['al'].kernel.active.spatialBgOrder = 2
282  self.doPreConvolve = False
283  self.doMatchSources = False
284  self.doAddMetrics = False
285  self.doUseRegister = False
286 
287  # DiaSource Detection
288  self.detection.thresholdPolarity = "both"
289  self.detection.thresholdValue = 5.5
290  self.detection.reEstimateBackground = False
291  self.detection.thresholdType = "pixel_stdev"
292 
293  # Add filtered flux measurement, the correct measurement for pre-convolved images.
294  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
295  # To change that you must modify algorithms.names in the task's applyOverrides method,
296  # after the user has set doPreConvolve.
297  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
298  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
299  'base_LocalWcs']
300 
301  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
302  self.forcedMeasurement.copyColumns = {
303  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
304  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
305  self.forcedMeasurement.slots.shape = None
306 
307  # For shuffling the control sample
308  random.seed(self.controlRandomSeed)
309 
310  def validate(self):
311  pexConfig.Config.validate(self)
312  if self.doAddMetrics and not self.doSubtract:
313  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
314  if not self.doSubtract and not self.doDetection:
315  raise ValueError("Either doSubtract or doDetection must be enabled.")
316  if self.subtract.name == 'zogy' and self.doAddMetrics:
317  raise ValueError("Kernel metrics does not exist in zogy subtraction.")
318  if self.doMeasurement and not self.doDetection:
319  raise ValueError("Cannot run source measurement without source detection.")
320  if self.doMerge and not self.doDetection:
321  raise ValueError("Cannot run source merging without source detection.")
322  if self.doUseRegister and not self.doSelectSources:
323  raise ValueError("doUseRegister=True and doSelectSources=False. "
324  "Cannot run RegisterTask without selecting sources.")
325  if self.doPreConvolve and self.doDecorrelation and not self.convolveTemplate:
326  raise ValueError("doPreConvolve=True and doDecorrelation=True and "
327  "convolveTemplate=False is not supported.")
328  if hasattr(self.getTemplate, "coaddName"):
329  if self.getTemplate.coaddName != self.coaddName:
330  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
331  if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
332  raise ValueError("Scaling the diffim variance and scaling the template variance "
333  "are both set. Please choose one or the other.")
334 
335 
336 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
337 
338  @staticmethod
339  def getTargetList(parsedCmd, **kwargs):
340  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
341  **kwargs)
342 
343 
344 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
345  """Subtract an image from a template and measure the result
346  """
347  ConfigClass = ImageDifferenceConfig
348  RunnerClass = ImageDifferenceTaskRunner
349  _DefaultName = "imageDifference"
350 
351  def __init__(self, butler=None, **kwargs):
352  """!Construct an ImageDifference Task
353 
354  @param[in] butler Butler object to use in constructing reference object loaders
355  """
356  super().__init__(**kwargs)
357  self.makeSubtask("getTemplate")
358 
359  self.makeSubtask("subtract")
360 
361  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
362  self.makeSubtask("decorrelate")
363 
364  if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
365  self.makeSubtask("scaleVariance")
366 
367  if self.config.doUseRegister:
368  self.makeSubtask("register")
369  self.schema = afwTable.SourceTable.makeMinimalSchema()
370 
371  if self.config.doSelectSources:
372  self.makeSubtask("sourceSelector")
373  if self.config.kernelSourcesFromRef:
374  self.makeSubtask('refObjLoader', butler=butler)
375  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
376 
377  self.algMetadata = dafBase.PropertyList()
378  if self.config.doDetection:
379  self.makeSubtask("detection", schema=self.schema)
380  if self.config.doMeasurement:
381  self.makeSubtask("measurement", schema=self.schema,
382  algMetadata=self.algMetadata)
383  if self.config.doForcedMeasurement:
384  self.schema.addField(
385  "ip_diffim_forced_PsfFlux_instFlux", "D",
386  "Forced PSF flux measured on the direct image.")
387  self.schema.addField(
388  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
389  "Forced PSF flux error measured on the direct image.")
390  self.schema.addField(
391  "ip_diffim_forced_PsfFlux_area", "F",
392  "Forced PSF flux effective area of PSF.",
393  units="pixel")
394  self.schema.addField(
395  "ip_diffim_forced_PsfFlux_flag", "Flag",
396  "Forced PSF flux general failure flag.")
397  self.schema.addField(
398  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
399  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
400  self.schema.addField(
401  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
402  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
403  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
404  if self.config.doMatchSources:
405  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
406  self.schema.addField("srcMatchId", "L", "unique id of source match")
407 
408  # initialize InitOutputs
409  self.outputSchema = afwTable.SourceCatalog(self.schema)
410  self.outputSchema.getTable().setMetadata(self.algMetadata)
411 
412  @staticmethod
413  def makeIdFactory(expId, expBits):
414  """Create IdFactory instance for unique 64 bit diaSource id-s.
415 
416  Parameters
417  ----------
418  expId : `int`
419  Exposure id.
420 
421  expBits: `int`
422  Number of used bits in ``expId``.
423 
424  Note
425  ----
426  The diasource id-s consists of the ``expId`` stored fixed in the highest value
427  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
428  low value end of the integer.
429 
430  Returns
431  -------
432  idFactory: `lsst.afw.table.IdFactory`
433  """
434  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
435 
436  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
437  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
438  inputRefs: pipeBase.InputQuantizedConnection,
439  outputRefs: pipeBase.OutputQuantizedConnection):
440  inputs = butlerQC.get(inputRefs)
441  self.log.info(f"Processing {butlerQC.quantum.dataId}")
442  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
443  returnMaxBits=True)
444  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
445  if self.config.coaddName == 'dcr':
446  templateExposures = inputRefs.dcrCoadds
447  else:
448  templateExposures = inputRefs.coaddExposures
449  templateStruct = self.getTemplate.runQuantum(
450  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
451  )
452 
453  outputs = self.run(exposure=inputs['exposure'],
454  templateExposure=templateStruct.exposure,
455  idFactory=idFactory)
456  butlerQC.put(outputs, outputRefs)
457 
458  @pipeBase.timeMethod
459  def runDataRef(self, sensorRef, templateIdList=None):
460  """Subtract an image from a template coadd and measure the result.
461 
462  Data I/O wrapper around `run` using the butler in Gen2.
463 
464  Parameters
465  ----------
466  sensorRef : `lsst.daf.persistence.ButlerDataRef`
467  Sensor-level butler data reference, used for the following data products:
468 
469  Input only:
470  - calexp
471  - psf
472  - ccdExposureId
473  - ccdExposureId_bits
474  - self.config.coaddName + "Coadd_skyMap"
475  - self.config.coaddName + "Coadd"
476  Input or output, depending on config:
477  - self.config.coaddName + "Diff_subtractedExp"
478  Output, depending on config:
479  - self.config.coaddName + "Diff_matchedExp"
480  - self.config.coaddName + "Diff_src"
481 
482  Returns
483  -------
484  results : `lsst.pipe.base.Struct`
485  Returns the Struct by `run`.
486  """
487  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
488  subtractedExposure = None
489  selectSources = None
490  calexpBackgroundExposure = None
491  self.log.info("Processing %s" % (sensorRef.dataId))
492 
493  # We make one IdFactory that will be used by both icSrc and src datasets;
494  # I don't know if this is the way we ultimately want to do things, but at least
495  # this ensures the source IDs are fully unique.
496  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
497  expBits=sensorRef.get("ccdExposureId_bits"))
498  if self.config.doAddCalexpBackground:
499  calexpBackgroundExposure = sensorRef.get("calexpBackground")
500 
501  # Retrieve the science image we wish to analyze
502  exposure = sensorRef.get("calexp", immediate=True)
503 
504  # Retrieve the template image
505  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
506 
507  if sensorRef.datasetExists("src"):
508  self.log.info("Source selection via src product")
509  # Sources already exist; for data release processing
510  selectSources = sensorRef.get("src")
511 
512  if not self.config.doSubtract and self.config.doDetection:
513  # If we don't do subtraction, we need the subtracted exposure from the repo
514  subtractedExposure = sensorRef.get(subtractedExposureName)
515  # Both doSubtract and doDetection cannot be False
516 
517  results = self.run(exposure=exposure,
518  selectSources=selectSources,
519  templateExposure=template.exposure,
520  templateSources=template.sources,
521  idFactory=idFactory,
522  calexpBackgroundExposure=calexpBackgroundExposure,
523  subtractedExposure=subtractedExposure)
524 
525  if self.config.doWriteSources and results.diaSources is not None:
526  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
527  if self.config.doWriteWarpedExp:
528  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
529  if self.config.doWriteMatchedExp:
530  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
531  if self.config.doAddMetrics and self.config.doSelectSources:
532  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
533  if self.config.doWriteSubtractedExp:
534  sensorRef.put(results.subtractedExposure, subtractedExposureName)
535  return results
536 
537  @pipeBase.timeMethod
538  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
539  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
540  """PSF matches, subtract two images and perform detection on the difference image.
541 
542  Parameters
543  ----------
544  exposure : `lsst.afw.image.ExposureF`, optional
545  The science exposure, the minuend in the image subtraction.
546  Can be None only if ``config.doSubtract==False``.
547  selectSources : `lsst.afw.table.SourceCatalog`, optional
548  Identified sources on the science exposure. This catalog is used to
549  select sources in order to perform the AL PSF matching on stamp images
550  around them. The selection steps depend on config options and whether
551  ``templateSources`` and ``matchingSources`` specified.
552  templateExposure : `lsst.afw.image.ExposureF`, optional
553  The template to be subtracted from ``exposure`` in the image subtraction.
554  The template exposure should cover the same sky area as the science exposure.
555  It is either a stich of patches of a coadd skymap image or a calexp
556  of the same pointing as the science exposure. Can be None only
557  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
558  templateSources : `lsst.afw.table.SourceCatalog`, optional
559  Identified sources on the template exposure.
560  idFactory : `lsst.afw.table.IdFactory`
561  Generator object to assign ids to detected sources in the difference image.
562  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
563  Background exposure to be added back to the science exposure
564  if ``config.doAddCalexpBackground==True``
565  subtractedExposure : `lsst.afw.image.ExposureF`, optional
566  If ``config.doSubtract==False`` and ``config.doDetection==True``,
567  performs the post subtraction source detection only on this exposure.
568  Otherwise should be None.
569 
570  Returns
571  -------
572  results : `lsst.pipe.base.Struct`
573  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
574  Difference image.
575  ``matchedExposure`` : `lsst.afw.image.ExposureF`
576  The matched PSF exposure.
577  ``subtractRes`` : `lsst.pipe.base.Struct`
578  The returned result structure of the ImagePsfMatchTask subtask.
579  ``diaSources`` : `lsst.afw.table.SourceCatalog`
580  The catalog of detected sources.
581  ``selectSources`` : `lsst.afw.table.SourceCatalog`
582  The input source catalog with optionally added Qa information.
583 
584  Notes
585  -----
586  The following major steps are included:
587 
588  - warp template coadd to match WCS of image
589  - PSF match image to warped template
590  - subtract image from PSF-matched, warped template
591  - detect sources
592  - measure sources
593 
594  For details about the image subtraction configuration modes
595  see `lsst.ip.diffim`.
596  """
597  subtractRes = None
598  controlSources = None
599  diaSources = None
600  kernelSources = None
601 
602  if self.config.doAddCalexpBackground:
603  mi = exposure.getMaskedImage()
604  mi += calexpBackgroundExposure.getImage()
605 
606  if not exposure.hasPsf():
607  raise pipeBase.TaskError("Exposure has no psf")
608  sciencePsf = exposure.getPsf()
609 
610  if self.config.doSubtract:
611  if self.config.doScaleTemplateVariance:
612  self.log.info("Rescaling template variance")
613  templateVarFactor = self.scaleVariance.run(
614  templateExposure.getMaskedImage())
615  self.log.info("Template variance scaling factor: %.2f" % templateVarFactor)
616  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
617 
618  if self.config.subtract.name == 'zogy':
619  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
620  if self.config.doPreConvolve:
621  subtractedExposure = subtractRes.scoreExp
622  else:
623  subtractedExposure = subtractRes.diffExp
624  subtractRes.subtractedExposure = subtractedExposure
625  subtractRes.matchedExposure = None
626 
627  elif self.config.subtract.name == 'al':
628  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
629  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
630  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
631 
632  # if requested, convolve the science exposure with its PSF
633  # (properly, this should be a cross-correlation, but our code does not yet support that)
634  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
635  # else sigma of original science exposure
636  # TODO: DM-22762 This functional block should be moved into its own method
637  preConvPsf = None
638  if self.config.doPreConvolve:
639  convControl = afwMath.ConvolutionControl()
640  # cannot convolve in place, so make a new MI to receive convolved image
641  srcMI = exposure.getMaskedImage()
642  exposureOrig = exposure.clone()
643  destMI = srcMI.Factory(srcMI.getDimensions())
644  srcPsf = sciencePsf
645  if self.config.useGaussianForPreConvolution:
646  # convolve with a simplified PSF model: a double Gaussian
647  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
648  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
649  else:
650  # convolve with science exposure's PSF model
651  preConvPsf = srcPsf
652  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
653  exposure.setMaskedImage(destMI)
654  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
655  else:
656  scienceSigmaPost = scienceSigmaOrig
657  exposureOrig = exposure
658 
659  # If requested, find and select sources from the image
660  # else, AL subtraction will do its own source detection
661  # TODO: DM-22762 This functional block should be moved into its own method
662  if self.config.doSelectSources:
663  if selectSources is None:
664  self.log.warn("Src product does not exist; running detection, measurement, selection")
665  # Run own detection and measurement; necessary in nightly processing
666  selectSources = self.subtract.getSelectSources(
667  exposure,
668  sigma=scienceSigmaPost,
669  doSmooth=not self.config.doPreConvolve,
670  idFactory=idFactory,
671  )
672 
673  if self.config.doAddMetrics:
674  # Number of basis functions
675 
676  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
677  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
678  targetFwhmPix=templateSigma*FwhmPerSigma))
679  # Modify the schema of all Sources
680  # DEPRECATED: This is a data dependent (nparam) output product schema
681  # outside the task constructor.
682  # NOTE: The pre-determination of nparam at this point
683  # may be incorrect as the template psf is warped later in
684  # ImagePsfMatchTask.matchExposures()
685  kcQa = KernelCandidateQa(nparam)
686  selectSources = kcQa.addToSchema(selectSources)
687  if self.config.kernelSourcesFromRef:
688  # match exposure sources to reference catalog
689  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
690  matches = astromRet.matches
691  elif templateSources:
692  # match exposure sources to template sources
693  mc = afwTable.MatchControl()
694  mc.findOnlyClosest = False
695  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
696  mc)
697  else:
698  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
699  "but template sources not available. Cannot match science "
700  "sources with template sources. Run process* on data from "
701  "which templates are built.")
702 
703  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
704  matches=matches).sourceCat
705  random.shuffle(kernelSources, random.random)
706  controlSources = kernelSources[::self.config.controlStepSize]
707  kernelSources = [k for i, k in enumerate(kernelSources)
708  if i % self.config.controlStepSize]
709 
710  if self.config.doSelectDcrCatalog:
711  redSelector = DiaCatalogSourceSelectorTask(
712  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
713  grMax=99.999))
714  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
715  controlSources.extend(redSources)
716 
717  blueSelector = DiaCatalogSourceSelectorTask(
718  DiaCatalogSourceSelectorConfig(grMin=-99.999,
719  grMax=self.sourceSelector.config.grMin))
720  blueSources = blueSelector.selectStars(exposure, selectSources,
721  matches=matches).starCat
722  controlSources.extend(blueSources)
723 
724  if self.config.doSelectVariableCatalog:
725  varSelector = DiaCatalogSourceSelectorTask(
726  DiaCatalogSourceSelectorConfig(includeVariable=True))
727  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
728  controlSources.extend(varSources)
729 
730  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
731  % (len(kernelSources), len(selectSources), len(controlSources)))
732 
733  allresids = {}
734  # TODO: DM-22762 This functional block should be moved into its own method
735  if self.config.doUseRegister:
736  self.log.info("Registering images")
737 
738  if templateSources is None:
739  # Run detection on the template, which is
740  # temporarily background-subtracted
741  # sigma of PSF of template image before warping
742  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
743  templateSources = self.subtract.getSelectSources(
744  templateExposure,
745  sigma=templateSigma,
746  doSmooth=True,
747  idFactory=idFactory
748  )
749 
750  # Third step: we need to fit the relative astrometry.
751  #
752  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
753  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
754  exposure.getWcs(), exposure.getBBox())
755  templateExposure = warpedExp
756 
757  # Create debugging outputs on the astrometric
758  # residuals as a function of position. Persistence
759  # not yet implemented; expected on (I believe) #2636.
760  if self.config.doDebugRegister:
761  # Grab matches to reference catalog
762  srcToMatch = {x.second.getId(): x.first for x in matches}
763 
764  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
765  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
766  sids = [m.first.getId() for m in wcsResults.matches]
767  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
768  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
769  m.second.get(inCentroidKey))) for m in wcsResults.matches]
770  allresids = dict(zip(sids, zip(positions, residuals)))
771 
772  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
773  wcsResults.wcs.pixelToSky(
774  m.second.get(inCentroidKey))) for m in wcsResults.matches]
775  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
776  + 2.5*numpy.log10(srcToMatch[x].get("r"))
777  for x in sids if x in srcToMatch.keys()])
778  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
779  if s in srcToMatch.keys()])
780  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
781  if s in srcToMatch.keys()])
782  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
783  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
784  & (colors <= self.sourceSelector.config.grMax))
785  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
786  rms1Long = IqrToSigma*(
787  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
788  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
789  - numpy.percentile(dlat[idx1], 25))
790  rms2Long = IqrToSigma*(
791  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
792  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
793  - numpy.percentile(dlat[idx2], 25))
794  rms3Long = IqrToSigma*(
795  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
796  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
797  - numpy.percentile(dlat[idx3], 25))
798  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
799  (numpy.median(dlong[idx1]), rms1Long,
800  numpy.median(dlat[idx1]), rms1Lat))
801  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
802  (numpy.median(dlong[idx2]), rms2Long,
803  numpy.median(dlat[idx2]), rms2Lat))
804  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
805  (numpy.median(dlong[idx3]), rms3Long,
806  numpy.median(dlat[idx3]), rms3Lat))
807 
808  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
809  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
810  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
811  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
812  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
813  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
814 
815  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
816  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
817  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
818  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
819  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
820  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
821 
822  # warp template exposure to match exposure,
823  # PSF match template exposure to exposure,
824  # then return the difference
825 
826  # Return warped template... Construct sourceKernelCand list after subtract
827  self.log.info("Subtracting images")
828  subtractRes = self.subtract.subtractExposures(
829  templateExposure=templateExposure,
830  scienceExposure=exposure,
831  candidateList=kernelSources,
832  convolveTemplate=self.config.convolveTemplate,
833  doWarping=not self.config.doUseRegister
834  )
835  subtractedExposure = subtractRes.subtractedExposure
836 
837  if self.config.doDetection:
838  self.log.info("Computing diffim PSF")
839 
840  # Get Psf from the appropriate input image if it doesn't exist
841  if not subtractedExposure.hasPsf():
842  if self.config.convolveTemplate:
843  subtractedExposure.setPsf(exposure.getPsf())
844  else:
845  subtractedExposure.setPsf(templateExposure.getPsf())
846 
847  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
848  # thus it may have already been decorrelated. Thus, we do not decorrelate if
849  # doSubtract is False.
850 
851  # NOTE: At this point doSubtract == True
852  if self.config.doDecorrelation and self.config.doSubtract:
853  preConvKernel = None
854  if preConvPsf is not None:
855  preConvKernel = preConvPsf.getLocalKernel()
856  if self.config.convolveTemplate:
857  self.log.info("Decorrelation after template image convolution")
858  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
859  subtractedExposure,
860  subtractRes.psfMatchingKernel,
861  spatiallyVarying=self.config.doSpatiallyVarying,
862  preConvKernel=preConvKernel)
863  else:
864  self.log.info("Decorrelation after science image convolution")
865  decorrResult = self.decorrelate.run(subtractRes.warpedExposure, exposureOrig,
866  subtractedExposure,
867  subtractRes.psfMatchingKernel,
868  spatiallyVarying=self.config.doSpatiallyVarying,
869  preConvKernel=preConvKernel)
870  subtractedExposure = decorrResult.correctedExposure
871 
872  # END (if subtractAlgorithm == 'AL')
873  # END (if self.config.doSubtract)
874  if self.config.doDetection:
875  self.log.info("Running diaSource detection")
876 
877  # Rescale difference image variance plane
878  if self.config.doScaleDiffimVariance:
879  self.log.info("Rescaling diffim variance")
880  diffimVarFactor = self.scaleVariance.run(subtractedExposure.getMaskedImage())
881  self.log.info("Diffim variance scaling factor: %.2f" % diffimVarFactor)
882  self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
883 
884  # Erase existing detection mask planes
885  mask = subtractedExposure.getMaskedImage().getMask()
886  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
887 
888  table = afwTable.SourceTable.make(self.schema, idFactory)
889  table.setMetadata(self.algMetadata)
890  results = self.detection.run(
891  table=table,
892  exposure=subtractedExposure,
893  doSmooth=not self.config.doPreConvolve
894  )
895 
896  if self.config.doMerge:
897  fpSet = results.fpSets.positive
898  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
899  self.config.growFootprint, False)
900  diaSources = afwTable.SourceCatalog(table)
901  fpSet.makeSources(diaSources)
902  self.log.info("Merging detections into %d sources" % (len(diaSources)))
903  else:
904  diaSources = results.sources
905 
906  if self.config.doMeasurement:
907  newDipoleFitting = self.config.doDipoleFitting
908  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
909  if not newDipoleFitting:
910  # Just fit dipole in diffim
911  self.measurement.run(diaSources, subtractedExposure)
912  else:
913  # Use (matched) template and science image (if avail.) to constrain dipole fitting
914  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
915  self.measurement.run(diaSources, subtractedExposure, exposure,
916  subtractRes.matchedExposure)
917  else:
918  self.measurement.run(diaSources, subtractedExposure, exposure)
919 
920  if self.config.doForcedMeasurement:
921  # Run forced psf photometry on the PVI at the diaSource locations.
922  # Copy the measured flux and error into the diaSource.
923  forcedSources = self.forcedMeasurement.generateMeasCat(
924  exposure, diaSources, subtractedExposure.getWcs())
925  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
926  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
927  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
928  "ip_diffim_forced_PsfFlux_instFlux", True)
929  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
930  "ip_diffim_forced_PsfFlux_instFluxErr", True)
931  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
932  "ip_diffim_forced_PsfFlux_area", True)
933  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
934  "ip_diffim_forced_PsfFlux_flag", True)
935  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
936  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
937  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
938  "ip_diffim_forced_PsfFlux_flag_edge", True)
939  for diaSource, forcedSource in zip(diaSources, forcedSources):
940  diaSource.assign(forcedSource, mapper)
941 
942  # Match with the calexp sources if possible
943  if self.config.doMatchSources:
944  if selectSources is not None:
945  # Create key,val pair where key=diaSourceId and val=sourceId
946  matchRadAsec = self.config.diaSourceMatchRadius
947  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
948 
949  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
950  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
951  srcMatch in srcMatches])
952  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
953  len(diaSources)))
954  else:
955  self.log.warn("Src product does not exist; cannot match with diaSources")
956  srcMatchDict = {}
957 
958  # Create key,val pair where key=diaSourceId and val=refId
959  refAstromConfig = AstrometryConfig()
960  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
961  refAstrometer = AstrometryTask(refAstromConfig)
962  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
963  refMatches = astromRet.matches
964  if refMatches is None:
965  self.log.warn("No diaSource matches with reference catalog")
966  refMatchDict = {}
967  else:
968  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
969  len(diaSources)))
970  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
971  refMatch in refMatches])
972 
973  # Assign source Ids
974  for diaSource in diaSources:
975  sid = diaSource.getId()
976  if sid in srcMatchDict:
977  diaSource.set("srcMatchId", srcMatchDict[sid])
978  if sid in refMatchDict:
979  diaSource.set("refMatchId", refMatchDict[sid])
980 
981  if self.config.doAddMetrics and self.config.doSelectSources:
982  self.log.info("Evaluating metrics and control sample")
983 
984  kernelCandList = []
985  for cell in subtractRes.kernelCellSet.getCellList():
986  for cand in cell.begin(False): # include bad candidates
987  kernelCandList.append(cand)
988 
989  # Get basis list to build control sample kernels
990  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
991  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
992 
993  controlCandList = (
994  diffimTools.sourceTableToCandidateList(controlSources,
995  subtractRes.warpedExposure, exposure,
996  self.config.subtract.kernel.active,
997  self.config.subtract.kernel.active.detectionConfig,
998  self.log, doBuild=True, basisList=basisList))
999 
1000  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1001  subtractRes.backgroundModel, dof=nparam)
1002  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1003  subtractRes.backgroundModel)
1004 
1005  if self.config.doDetection:
1006  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1007  else:
1008  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1009 
1010  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1011  return pipeBase.Struct(
1012  subtractedExposure=subtractedExposure,
1013  warpedExposure=subtractRes.warpedExposure,
1014  matchedExposure=subtractRes.matchedExposure,
1015  subtractRes=subtractRes,
1016  diaSources=diaSources,
1017  selectSources=selectSources
1018  )
1019 
1020  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1021  """Fit the relative astrometry between templateSources and selectSources
1022 
1023  Todo
1024  ----
1025 
1026  Remove this method. It originally fit a new WCS to the template before calling register.run
1027  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1028  It remains because a subtask overrides it.
1029  """
1030  results = self.register.run(templateSources, templateExposure.getWcs(),
1031  templateExposure.getBBox(), selectSources)
1032  return results
1033 
1034  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1035  """Make debug plots and displays.
1036 
1037  Todo
1038  ----
1039  Test and update for current debug display and slot names
1040  """
1041  import lsstDebug
1042  display = lsstDebug.Info(__name__).display
1043  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1044  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1045  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1046  showDipoles = lsstDebug.Info(__name__).showDipoles
1047  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1048  if display:
1049  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1050  if not maskTransparency:
1051  maskTransparency = 0
1052  disp.setMaskTransparency(maskTransparency)
1053 
1054  if display and showSubtracted:
1055  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1056  mi = subtractRes.subtractedExposure.getMaskedImage()
1057  x0, y0 = mi.getX0(), mi.getY0()
1058  with disp.Buffering():
1059  for s in diaSources:
1060  x, y = s.getX() - x0, s.getY() - y0
1061  ctype = "red" if s.get("flags_negative") else "yellow"
1062  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1063  or s.get("base_PixelFlags_flag_saturatedCenter")
1064  or s.get("base_PixelFlags_flag_crCenter")):
1065  ptype = "x"
1066  elif (s.get("base_PixelFlags_flag_interpolated")
1067  or s.get("base_PixelFlags_flag_saturated")
1068  or s.get("base_PixelFlags_flag_cr")):
1069  ptype = "+"
1070  else:
1071  ptype = "o"
1072  disp.dot(ptype, x, y, size=4, ctype=ctype)
1073  lsstDebug.frame += 1
1074 
1075  if display and showPixelResiduals and selectSources:
1076  nonKernelSources = []
1077  for source in selectSources:
1078  if source not in kernelSources:
1079  nonKernelSources.append(source)
1080 
1081  diUtils.plotPixelResiduals(exposure,
1082  subtractRes.warpedExposure,
1083  subtractRes.subtractedExposure,
1084  subtractRes.kernelCellSet,
1085  subtractRes.psfMatchingKernel,
1086  subtractRes.backgroundModel,
1087  nonKernelSources,
1088  self.subtract.config.kernel.active.detectionConfig,
1089  origVariance=False)
1090  diUtils.plotPixelResiduals(exposure,
1091  subtractRes.warpedExposure,
1092  subtractRes.subtractedExposure,
1093  subtractRes.kernelCellSet,
1094  subtractRes.psfMatchingKernel,
1095  subtractRes.backgroundModel,
1096  nonKernelSources,
1097  self.subtract.config.kernel.active.detectionConfig,
1098  origVariance=True)
1099  if display and showDiaSources:
1100  flagChecker = SourceFlagChecker(diaSources)
1101  isFlagged = [flagChecker(x) for x in diaSources]
1102  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1103  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1104  frame=lsstDebug.frame)
1105  lsstDebug.frame += 1
1106 
1107  if display and showDipoles:
1108  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1109  frame=lsstDebug.frame)
1110  lsstDebug.frame += 1
1111 
1112  def _getConfigName(self):
1113  """Return the name of the config dataset
1114  """
1115  return "%sDiff_config" % (self.config.coaddName,)
1116 
1117  def _getMetadataName(self):
1118  """Return the name of the metadata dataset
1119  """
1120  return "%sDiff_metadata" % (self.config.coaddName,)
1121 
1122  def getSchemaCatalogs(self):
1123  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1124  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1125 
1126  @classmethod
1127  def _makeArgumentParser(cls):
1128  """Create an argument parser
1129  """
1130  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1131  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1132  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1133  help="Template data ID in case of calexp template,"
1134  " e.g. --templateId visit=6789")
1135  return parser
1136 
1137 
1138 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1139  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1140  doc="Shift stars going into RegisterTask by this amount")
1141  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1142  doc="Perturb stars going into RegisterTask by this amount")
1143 
1144  def setDefaults(self):
1145  ImageDifferenceConfig.setDefaults(self)
1146  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1147 
1148 
1149 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1150  """!Image difference Task used in the Winter 2013 data challege.
1151  Enables testing the effects of registration shifts and scatter.
1152 
1153  For use with winter 2013 simulated images:
1154  Use --templateId visit=88868666 for sparse data
1155  --templateId visit=22222200 for dense data (g)
1156  --templateId visit=11111100 for dense data (i)
1157  """
1158  ConfigClass = Winter2013ImageDifferenceConfig
1159  _DefaultName = "winter2013ImageDifference"
1160 
1161  def __init__(self, **kwargs):
1162  ImageDifferenceTask.__init__(self, **kwargs)
1163 
1164  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1165  """Fit the relative astrometry between templateSources and selectSources"""
1166  if self.config.winter2013WcsShift > 0.0:
1167  offset = geom.Extent2D(self.config.winter2013WcsShift,
1168  self.config.winter2013WcsShift)
1169  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1170  for source in templateSources:
1171  centroid = source.get(cKey)
1172  source.set(cKey, centroid + offset)
1173  elif self.config.winter2013WcsRms > 0.0:
1174  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1175  for source in templateSources:
1176  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1177  self.config.winter2013WcsRms*numpy.random.normal())
1178  centroid = source.get(cKey)
1179  source.set(cKey, centroid + offset)
1180 
1181  results = self.register.run(templateSources, templateExposure.getWcs(),
1182  templateExposure.getBBox(), selectSources)
1183  return results
lsst.pipe.tasks.registerImage
Definition: registerImage.py:1
lsst::ip::diffim
lsst::meas::astrom
lsst::afw::display
lsst::ip::diffim::utils
lsst::utils::inheritDoc
lsst.pipe.tasks.scaleVariance
Definition: scaleVariance.py:1
lsst::ip::diffim::diffimTools
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:721
lsstDebug::Info
lsst::meas::base
lsst::pex::config
lsst::afw::table
lsst::utils
lsst::geom
lsst.pipe.tasks.imageDifference.ImageDifferenceTaskConnections
Definition: imageDifference.py:55
lsst::daf::base
lsst::meas::algorithms::SingleGaussianPsf
lsst::afw::math
lsst::skymap
lsst.pipe::base
lsst::meas::algorithms
Extent< double, 2 >