lsst.pipe.tasks  20.0.0-4-gd489f3fc
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 
47 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
49 IqrToSigma = 0.741
50 
51 
52 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
53  dimensions=("instrument", "visit", "detector", "skymap"),
54  defaultTemplates={"coaddName": "deep",
55  "skyMapName": "deep",
56  "warpTypeSuffix": "",
57  "fakesType": ""}):
58 
59  exposure = pipeBase.connectionTypes.Input(
60  doc="Input science exposure to subtract from.",
61  dimensions=("instrument", "visit", "detector"),
62  storageClass="ExposureF",
63  name="calexp"
64  )
65 
66  # TODO DM-22953
67  # kernelSources = pipeBase.connectionTypes.Input(
68  # doc="Source catalog produced in calibrate task for kernel candidate sources",
69  # name="src",
70  # storageClass="SourceCatalog",
71  # dimensions=("instrument", "visit", "detector"),
72  # )
73 
74  skyMap = pipeBase.connectionTypes.Input(
75  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
76  name="{skyMapName}Coadd_skyMap",
77  dimensions=("skymap", ),
78  storageClass="SkyMap",
79  )
80  coaddExposures = pipeBase.connectionTypes.Input(
81  doc="Input template to match and subtract from the exposure",
82  dimensions=("tract", "patch", "skymap", "abstract_filter"),
83  storageClass="ExposureF",
84  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
85  multiple=True,
86  deferLoad=True
87  )
88  dcrCoadds = pipeBase.connectionTypes.Input(
89  doc="Input DCR template to match and subtract from the exposure",
90  name="{fakesType}dcrCoadd{warpTypeSuffix}",
91  storageClass="ExposureF",
92  dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
93  multiple=True,
94  deferLoad=True
95  )
96  outputSchema = pipeBase.connectionTypes.InitOutput(
97  doc="Schema (as an example catalog) for output DIASource catalog.",
98  storageClass="SourceCatalog",
99  name="{fakesType}{coaddName}Diff_diaSrc_schema",
100  )
101  subtractedExposure = pipeBase.connectionTypes.Output(
102  doc="Output difference image",
103  dimensions=("instrument", "visit", "detector"),
104  storageClass="ExposureF",
105  name="{fakesType}{coaddName}Diff_differenceExp",
106  )
107  diaSources = pipeBase.connectionTypes.Output(
108  doc="Output detected diaSources on the difference image",
109  dimensions=("instrument", "visit", "detector"),
110  storageClass="SourceCatalog",
111  name="{fakesType}{coaddName}Diff_diaSrc",
112  )
113 
114  def __init__(self, *, config=None):
115  super().__init__(config=config)
116  if config.coaddName == 'dcr':
117  self.inputs.remove("coaddExposures")
118  else:
119  self.inputs.remove("dcrCoadds")
120 
121  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
122  # Make kernelSources optional
123 
124 
125 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
126  pipelineConnections=ImageDifferenceTaskConnections):
127  """Config for ImageDifferenceTask
128  """
129  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
130  doc="Add background to calexp before processing it. "
131  "Useful as ipDiffim does background matching.")
132  doUseRegister = pexConfig.Field(dtype=bool, default=True,
133  doc="Use image-to-image registration to align template with "
134  "science image")
135  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
136  doc="Writing debugging data for doUseRegister")
137  doSelectSources = pexConfig.Field(dtype=bool, default=True,
138  doc="Select stars to use for kernel fitting")
139  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
140  doc="Select stars of extreme color as part of the control sample")
141  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
142  doc="Select stars that are variable to be part "
143  "of the control sample")
144  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
145  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
146  doc="Convolve science image by its PSF before PSF-matching?")
147  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
148  doc="Scale variance of the template before PSF matching")
149  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
150  doc="Use a simple gaussian PSF model for pre-convolution "
151  "(else use fit PSF)? Ignored if doPreConvolve false.")
152  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
153  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
154  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
155  "kernel convolution? If True, also update the diffim PSF.")
156  doMerge = pexConfig.Field(dtype=bool, default=True,
157  doc="Merge positive and negative diaSources with grow radius "
158  "set by growFootprint")
159  doMatchSources = pexConfig.Field(dtype=bool, default=True,
160  doc="Match diaSources with input calexp sources and ref catalog sources")
161  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
162  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
163  doForcedMeasurement = pexConfig.Field(
164  dtype=bool,
165  default=True,
166  doc="Force photometer diaSource locations on PVI?")
167  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
168  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
169  doc="Write warped and PSF-matched template coadd exposure?")
170  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
171  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
172  doc="Add columns to the source table to hold analysis metrics?")
173 
174  coaddName = pexConfig.Field(
175  doc="coadd name: typically one of deep, goodSeeing, or dcr",
176  dtype=str,
177  default="deep",
178  )
179  convolveTemplate = pexConfig.Field(
180  doc="Which image gets convolved (default = template)",
181  dtype=bool,
182  default=True
183  )
184  refObjLoader = pexConfig.ConfigurableField(
185  target=LoadIndexedReferenceObjectsTask,
186  doc="reference object loader",
187  )
188  astrometer = pexConfig.ConfigurableField(
189  target=AstrometryTask,
190  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
191  )
192  sourceSelector = pexConfig.ConfigurableField(
193  target=ObjectSizeStarSelectorTask,
194  doc="Source selection algorithm",
195  )
196  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
197  decorrelate = pexConfig.ConfigurableField(
198  target=DecorrelateALKernelSpatialTask,
199  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
200  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
201  "default of 5.5).",
202  )
203  doSpatiallyVarying = pexConfig.Field(
204  dtype=bool,
205  default=False,
206  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
207  "image in order to allow for spatial variations"
208  )
209  detection = pexConfig.ConfigurableField(
210  target=SourceDetectionTask,
211  doc="Low-threshold detection for final measurement",
212  )
213  measurement = pexConfig.ConfigurableField(
214  target=DipoleFitTask,
215  doc="Enable updated dipole fitting method",
216  )
217  forcedMeasurement = pexConfig.ConfigurableField(
218  target=ForcedMeasurementTask,
219  doc="Subtask to force photometer PVI at diaSource location.",
220  )
221  getTemplate = pexConfig.ConfigurableField(
222  target=GetCoaddAsTemplateTask,
223  doc="Subtask to retrieve template exposure and sources",
224  )
225  scaleVariance = pexConfig.ConfigurableField(
226  target=ScaleVarianceTask,
227  doc="Subtask to rescale the variance of the template "
228  "to the statistically expected level"
229  )
230  controlStepSize = pexConfig.Field(
231  doc="What step size (every Nth one) to select a control sample from the kernelSources",
232  dtype=int,
233  default=5
234  )
235  controlRandomSeed = pexConfig.Field(
236  doc="Random seed for shuffing the control sample",
237  dtype=int,
238  default=10
239  )
240  register = pexConfig.ConfigurableField(
241  target=RegisterTask,
242  doc="Task to enable image-to-image image registration (warping)",
243  )
244  kernelSourcesFromRef = pexConfig.Field(
245  doc="Select sources to measure kernel from reference catalog if True, template if false",
246  dtype=bool,
247  default=False
248  )
249  templateSipOrder = pexConfig.Field(
250  dtype=int, default=2,
251  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
252  )
253  growFootprint = pexConfig.Field(
254  dtype=int, default=2,
255  doc="Grow positive and negative footprints by this amount before merging"
256  )
257  diaSourceMatchRadius = pexConfig.Field(
258  dtype=float, default=0.5,
259  doc="Match radius (in arcseconds) for DiaSource to Source association"
260  )
261 
262  def setDefaults(self):
263  # defaults are OK for catalog and diacatalog
264 
265  self.subtract['al'].kernel.name = "AL"
266  self.subtract['al'].kernel.active.fitForBackground = True
267  self.subtract['al'].kernel.active.spatialKernelOrder = 1
268  self.subtract['al'].kernel.active.spatialBgOrder = 2
269  self.doPreConvolve = False
270  self.doMatchSources = False
271  self.doAddMetrics = False
272  self.doUseRegister = False
273 
274  # DiaSource Detection
275  self.detection.thresholdPolarity = "both"
276  self.detection.thresholdValue = 5.5
277  self.detection.reEstimateBackground = False
278  self.detection.thresholdType = "pixel_stdev"
279 
280  # Add filtered flux measurement, the correct measurement for pre-convolved images.
281  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
282  # To change that you must modify algorithms.names in the task's applyOverrides method,
283  # after the user has set doPreConvolve.
284  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
285  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
286  'base_LocalWcs']
287 
288  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
289  self.forcedMeasurement.copyColumns = {
290  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
291  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
292  self.forcedMeasurement.slots.shape = None
293 
294  # For shuffling the control sample
295  random.seed(self.controlRandomSeed)
296 
297  def validate(self):
298  pexConfig.Config.validate(self)
299  if self.doAddMetrics and not self.doSubtract:
300  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
301  if not self.doSubtract and not self.doDetection:
302  raise ValueError("Either doSubtract or doDetection must be enabled.")
303  if self.subtract.name == 'zogy' and self.doAddMetrics:
304  raise ValueError("Kernel metrics does not exist in zogy subtraction.")
305  if self.doMeasurement and not self.doDetection:
306  raise ValueError("Cannot run source measurement without source detection.")
307  if self.doMerge and not self.doDetection:
308  raise ValueError("Cannot run source merging without source detection.")
309  if self.doUseRegister and not self.doSelectSources:
310  raise ValueError("doUseRegister=True and doSelectSources=False. "
311  "Cannot run RegisterTask without selecting sources.")
312  if self.doPreConvolve and self.doDecorrelation and not self.convolveTemplate:
313  raise ValueError("doPreConvolve=True and doDecorrelation=True and "
314  "convolveTemplate=False is not supported.")
315  if hasattr(self.getTemplate, "coaddName"):
316  if self.getTemplate.coaddName != self.coaddName:
317  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
318 
319 
320 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
321 
322  @staticmethod
323  def getTargetList(parsedCmd, **kwargs):
324  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
325  **kwargs)
326 
327 
328 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
329  """Subtract an image from a template and measure the result
330  """
331  ConfigClass = ImageDifferenceConfig
332  RunnerClass = ImageDifferenceTaskRunner
333  _DefaultName = "imageDifference"
334 
335  def __init__(self, butler=None, **kwargs):
336  """!Construct an ImageDifference Task
337 
338  @param[in] butler Butler object to use in constructing reference object loaders
339  """
340  super().__init__(**kwargs)
341  self.makeSubtask("getTemplate")
342 
343  self.makeSubtask("subtract")
344 
345  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
346  self.makeSubtask("decorrelate")
347 
348  if self.config.doScaleTemplateVariance:
349  self.makeSubtask("scaleVariance")
350 
351  if self.config.doUseRegister:
352  self.makeSubtask("register")
353  self.schema = afwTable.SourceTable.makeMinimalSchema()
354 
355  if self.config.doSelectSources:
356  self.makeSubtask("sourceSelector")
357  if self.config.kernelSourcesFromRef:
358  self.makeSubtask('refObjLoader', butler=butler)
359  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
360 
361  self.algMetadata = dafBase.PropertyList()
362  if self.config.doDetection:
363  self.makeSubtask("detection", schema=self.schema)
364  if self.config.doMeasurement:
365  self.makeSubtask("measurement", schema=self.schema,
366  algMetadata=self.algMetadata)
367  if self.config.doForcedMeasurement:
368  self.schema.addField(
369  "ip_diffim_forced_PsfFlux_instFlux", "D",
370  "Forced PSF flux measured on the direct image.")
371  self.schema.addField(
372  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
373  "Forced PSF flux error measured on the direct image.")
374  self.schema.addField(
375  "ip_diffim_forced_PsfFlux_area", "F",
376  "Forced PSF flux effective area of PSF.",
377  units="pixel")
378  self.schema.addField(
379  "ip_diffim_forced_PsfFlux_flag", "Flag",
380  "Forced PSF flux general failure flag.")
381  self.schema.addField(
382  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
383  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
384  self.schema.addField(
385  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
386  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
387  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
388  if self.config.doMatchSources:
389  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
390  self.schema.addField("srcMatchId", "L", "unique id of source match")
391 
392  # initialize InitOutputs
393  self.outputSchema = afwTable.SourceCatalog(self.schema)
394  self.outputSchema.getTable().setMetadata(self.algMetadata)
395 
396  @staticmethod
397  def makeIdFactory(expId, expBits):
398  """Create IdFactory instance for unique 64 bit diaSource id-s.
399 
400  Parameters
401  ----------
402  expId : `int`
403  Exposure id.
404 
405  expBits: `int`
406  Number of used bits in ``expId``.
407 
408  Note
409  ----
410  The diasource id-s consists of the ``expId`` stored fixed in the highest value
411  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
412  low value end of the integer.
413 
414  Returns
415  -------
416  idFactory: `lsst.afw.table.IdFactory`
417  """
418  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
419 
420  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
421  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
422  inputRefs: pipeBase.InputQuantizedConnection,
423  outputRefs: pipeBase.OutputQuantizedConnection):
424  inputs = butlerQC.get(inputRefs)
425  self.log.info(f"Processing {butlerQC.quantum.dataId}")
426  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
427  returnMaxBits=True)
428  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
429  if self.config.coaddName == 'dcr':
430  templateExposures = inputRefs.dcrCoadds
431  else:
432  templateExposures = inputRefs.coaddExposures
433  templateStruct = self.getTemplate.runQuantum(
434  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
435  )
436 
437  outputs = self.run(exposure=inputs['exposure'],
438  templateExposure=templateStruct.exposure,
439  idFactory=idFactory)
440  butlerQC.put(outputs, outputRefs)
441 
442  @pipeBase.timeMethod
443  def runDataRef(self, sensorRef, templateIdList=None):
444  """Subtract an image from a template coadd and measure the result.
445 
446  Data I/O wrapper around `run` using the butler in Gen2.
447 
448  Parameters
449  ----------
450  sensorRef : `lsst.daf.persistence.ButlerDataRef`
451  Sensor-level butler data reference, used for the following data products:
452 
453  Input only:
454  - calexp
455  - psf
456  - ccdExposureId
457  - ccdExposureId_bits
458  - self.config.coaddName + "Coadd_skyMap"
459  - self.config.coaddName + "Coadd"
460  Input or output, depending on config:
461  - self.config.coaddName + "Diff_subtractedExp"
462  Output, depending on config:
463  - self.config.coaddName + "Diff_matchedExp"
464  - self.config.coaddName + "Diff_src"
465 
466  Returns
467  -------
468  results : `lsst.pipe.base.Struct`
469  Returns the Struct by `run`.
470  """
471  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
472  subtractedExposure = None
473  selectSources = None
474  calexpBackgroundExposure = None
475  self.log.info("Processing %s" % (sensorRef.dataId))
476 
477  # We make one IdFactory that will be used by both icSrc and src datasets;
478  # I don't know if this is the way we ultimately want to do things, but at least
479  # this ensures the source IDs are fully unique.
480  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
481  expBits=sensorRef.get("ccdExposureId_bits"))
482  if self.config.doAddCalexpBackground:
483  calexpBackgroundExposure = sensorRef.get("calexpBackground")
484 
485  # Retrieve the science image we wish to analyze
486  exposure = sensorRef.get("calexp", immediate=True)
487 
488  # Retrieve the template image
489  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
490 
491  if sensorRef.datasetExists("src"):
492  self.log.info("Source selection via src product")
493  # Sources already exist; for data release processing
494  selectSources = sensorRef.get("src")
495 
496  if not self.config.doSubtract and self.config.doDetection:
497  # If we don't do subtraction, we need the subtracted exposure from the repo
498  subtractedExposure = sensorRef.get(subtractedExposureName)
499  # Both doSubtract and doDetection cannot be False
500 
501  results = self.run(exposure=exposure,
502  selectSources=selectSources,
503  templateExposure=template.exposure,
504  templateSources=template.sources,
505  idFactory=idFactory,
506  calexpBackgroundExposure=calexpBackgroundExposure,
507  subtractedExposure=subtractedExposure)
508 
509  if self.config.doWriteSources and results.diaSources is not None:
510  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
511  if self.config.doWriteMatchedExp:
512  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
513  if self.config.doAddMetrics and self.config.doSelectSources:
514  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
515  if self.config.doWriteSubtractedExp:
516  sensorRef.put(results.subtractedExposure, subtractedExposureName)
517  return results
518 
519  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
520  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
521  """PSF matches, subtract two images and perform detection on the difference image.
522 
523  Parameters
524  ----------
525  exposure : `lsst.afw.image.ExposureF`, optional
526  The science exposure, the minuend in the image subtraction.
527  Can be None only if ``config.doSubtract==False``.
528  selectSources : `lsst.afw.table.SourceCatalog`, optional
529  Identified sources on the science exposure. This catalog is used to
530  select sources in order to perform the AL PSF matching on stamp images
531  around them. The selection steps depend on config options and whether
532  ``templateSources`` and ``matchingSources`` specified.
533  templateExposure : `lsst.afw.image.ExposureF`, optional
534  The template to be subtracted from ``exposure`` in the image subtraction.
535  The template exposure should cover the same sky area as the science exposure.
536  It is either a stich of patches of a coadd skymap image or a calexp
537  of the same pointing as the science exposure. Can be None only
538  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
539  templateSources : `lsst.afw.table.SourceCatalog`, optional
540  Identified sources on the template exposure.
541  idFactory : `lsst.afw.table.IdFactory`
542  Generator object to assign ids to detected sources in the difference image.
543  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
544  Background exposure to be added back to the science exposure
545  if ``config.doAddCalexpBackground==True``
546  subtractedExposure : `lsst.afw.image.ExposureF`, optional
547  If ``config.doSubtract==False`` and ``config.doDetection==True``,
548  performs the post subtraction source detection only on this exposure.
549  Otherwise should be None.
550 
551  Returns
552  -------
553  results : `lsst.pipe.base.Struct`
554  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
555  Difference image.
556  ``matchedExposure`` : `lsst.afw.image.ExposureF`
557  The matched PSF exposure.
558  ``subtractRes`` : `lsst.pipe.base.Struct`
559  The returned result structure of the ImagePsfMatchTask subtask.
560  ``diaSources`` : `lsst.afw.table.SourceCatalog`
561  The catalog of detected sources.
562  ``selectSources`` : `lsst.afw.table.SourceCatalog`
563  The input source catalog with optionally added Qa information.
564 
565  Notes
566  -----
567  The following major steps are included:
568 
569  - warp template coadd to match WCS of image
570  - PSF match image to warped template
571  - subtract image from PSF-matched, warped template
572  - detect sources
573  - measure sources
574 
575  For details about the image subtraction configuration modes
576  see `lsst.ip.diffim`.
577  """
578  subtractRes = None
579  controlSources = None
580  diaSources = None
581  kernelSources = None
582 
583  if self.config.doAddCalexpBackground:
584  mi = exposure.getMaskedImage()
585  mi += calexpBackgroundExposure.getImage()
586 
587  if not exposure.hasPsf():
588  raise pipeBase.TaskError("Exposure has no psf")
589  sciencePsf = exposure.getPsf()
590 
591  if self.config.doSubtract:
592  if self.config.doScaleTemplateVariance:
593  templateVarFactor = self.scaleVariance.run(
594  templateExposure.getMaskedImage())
595  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
596 
597  if self.config.subtract.name == 'zogy':
598  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
599  doWarping=True,
600  spatiallyVarying=self.config.doSpatiallyVarying,
601  doPreConvolve=self.config.doPreConvolve)
602  subtractedExposure = subtractRes.subtractedExposure
603 
604  elif self.config.subtract.name == 'al':
605  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
606  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
607  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
608 
609  # if requested, convolve the science exposure with its PSF
610  # (properly, this should be a cross-correlation, but our code does not yet support that)
611  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
612  # else sigma of original science exposure
613  # TODO: DM-22762 This functional block should be moved into its own method
614  preConvPsf = None
615  if self.config.doPreConvolve:
616  convControl = afwMath.ConvolutionControl()
617  # cannot convolve in place, so make a new MI to receive convolved image
618  srcMI = exposure.getMaskedImage()
619  exposureOrig = exposure.clone()
620  destMI = srcMI.Factory(srcMI.getDimensions())
621  srcPsf = sciencePsf
622  if self.config.useGaussianForPreConvolution:
623  # convolve with a simplified PSF model: a double Gaussian
624  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
625  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
626  else:
627  # convolve with science exposure's PSF model
628  preConvPsf = srcPsf
629  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
630  exposure.setMaskedImage(destMI)
631  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
632  else:
633  scienceSigmaPost = scienceSigmaOrig
634  exposureOrig = exposure
635 
636  # If requested, find and select sources from the image
637  # else, AL subtraction will do its own source detection
638  # TODO: DM-22762 This functional block should be moved into its own method
639  if self.config.doSelectSources:
640  if selectSources is None:
641  self.log.warn("Src product does not exist; running detection, measurement, selection")
642  # Run own detection and measurement; necessary in nightly processing
643  selectSources = self.subtract.getSelectSources(
644  exposure,
645  sigma=scienceSigmaPost,
646  doSmooth=not self.config.doPreConvolve,
647  idFactory=idFactory,
648  )
649 
650  if self.config.doAddMetrics:
651  # Number of basis functions
652 
653  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
654  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
655  targetFwhmPix=templateSigma*FwhmPerSigma))
656  # Modify the schema of all Sources
657  # DEPRECATED: This is a data dependent (nparam) output product schema
658  # outside the task constructor.
659  # NOTE: The pre-determination of nparam at this point
660  # may be incorrect as the template psf is warped later in
661  # ImagePsfMatchTask.matchExposures()
662  kcQa = KernelCandidateQa(nparam)
663  selectSources = kcQa.addToSchema(selectSources)
664  if self.config.kernelSourcesFromRef:
665  # match exposure sources to reference catalog
666  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
667  matches = astromRet.matches
668  elif templateSources:
669  # match exposure sources to template sources
670  mc = afwTable.MatchControl()
671  mc.findOnlyClosest = False
672  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
673  mc)
674  else:
675  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
676  "but template sources not available. Cannot match science "
677  "sources with template sources. Run process* on data from "
678  "which templates are built.")
679 
680  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
681  matches=matches).sourceCat
682  random.shuffle(kernelSources, random.random)
683  controlSources = kernelSources[::self.config.controlStepSize]
684  kernelSources = [k for i, k in enumerate(kernelSources)
685  if i % self.config.controlStepSize]
686 
687  if self.config.doSelectDcrCatalog:
688  redSelector = DiaCatalogSourceSelectorTask(
689  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
690  grMax=99.999))
691  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
692  controlSources.extend(redSources)
693 
694  blueSelector = DiaCatalogSourceSelectorTask(
695  DiaCatalogSourceSelectorConfig(grMin=-99.999,
696  grMax=self.sourceSelector.config.grMin))
697  blueSources = blueSelector.selectStars(exposure, selectSources,
698  matches=matches).starCat
699  controlSources.extend(blueSources)
700 
701  if self.config.doSelectVariableCatalog:
702  varSelector = DiaCatalogSourceSelectorTask(
703  DiaCatalogSourceSelectorConfig(includeVariable=True))
704  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
705  controlSources.extend(varSources)
706 
707  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
708  % (len(kernelSources), len(selectSources), len(controlSources)))
709 
710  allresids = {}
711  # TODO: DM-22762 This functional block should be moved into its own method
712  if self.config.doUseRegister:
713  self.log.info("Registering images")
714 
715  if templateSources is None:
716  # Run detection on the template, which is
717  # temporarily background-subtracted
718  # sigma of PSF of template image before warping
719  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
720  templateSources = self.subtract.getSelectSources(
721  templateExposure,
722  sigma=templateSigma,
723  doSmooth=True,
724  idFactory=idFactory
725  )
726 
727  # Third step: we need to fit the relative astrometry.
728  #
729  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
730  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
731  exposure.getWcs(), exposure.getBBox())
732  templateExposure = warpedExp
733 
734  # Create debugging outputs on the astrometric
735  # residuals as a function of position. Persistence
736  # not yet implemented; expected on (I believe) #2636.
737  if self.config.doDebugRegister:
738  # Grab matches to reference catalog
739  srcToMatch = {x.second.getId(): x.first for x in matches}
740 
741  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
742  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
743  sids = [m.first.getId() for m in wcsResults.matches]
744  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
745  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
746  m.second.get(inCentroidKey))) for m in wcsResults.matches]
747  allresids = dict(zip(sids, zip(positions, residuals)))
748 
749  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
750  wcsResults.wcs.pixelToSky(
751  m.second.get(inCentroidKey))) for m in wcsResults.matches]
752  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
753  + 2.5*numpy.log10(srcToMatch[x].get("r"))
754  for x in sids if x in srcToMatch.keys()])
755  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
756  if s in srcToMatch.keys()])
757  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
758  if s in srcToMatch.keys()])
759  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
760  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
761  & (colors <= self.sourceSelector.config.grMax))
762  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
763  rms1Long = IqrToSigma*(
764  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
765  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
766  - numpy.percentile(dlat[idx1], 25))
767  rms2Long = IqrToSigma*(
768  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
769  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
770  - numpy.percentile(dlat[idx2], 25))
771  rms3Long = IqrToSigma*(
772  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
773  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
774  - numpy.percentile(dlat[idx3], 25))
775  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
776  (numpy.median(dlong[idx1]), rms1Long,
777  numpy.median(dlat[idx1]), rms1Lat))
778  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
779  (numpy.median(dlong[idx2]), rms2Long,
780  numpy.median(dlat[idx2]), rms2Lat))
781  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
782  (numpy.median(dlong[idx3]), rms3Long,
783  numpy.median(dlat[idx3]), rms3Lat))
784 
785  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
786  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
787  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
788  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
789  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
790  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
791 
792  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
793  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
794  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
795  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
796  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
797  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
798 
799  # warp template exposure to match exposure,
800  # PSF match template exposure to exposure,
801  # then return the difference
802 
803  # Return warped template... Construct sourceKernelCand list after subtract
804  self.log.info("Subtracting images")
805  subtractRes = self.subtract.subtractExposures(
806  templateExposure=templateExposure,
807  scienceExposure=exposure,
808  candidateList=kernelSources,
809  convolveTemplate=self.config.convolveTemplate,
810  doWarping=not self.config.doUseRegister
811  )
812  subtractedExposure = subtractRes.subtractedExposure
813 
814  if self.config.doDetection:
815  self.log.info("Computing diffim PSF")
816 
817  # Get Psf from the appropriate input image if it doesn't exist
818  if not subtractedExposure.hasPsf():
819  if self.config.convolveTemplate:
820  subtractedExposure.setPsf(exposure.getPsf())
821  else:
822  subtractedExposure.setPsf(templateExposure.getPsf())
823 
824  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
825  # thus it may have already been decorrelated. Thus, we do not decorrelate if
826  # doSubtract is False.
827 
828  # NOTE: At this point doSubtract == True
829  if self.config.doDecorrelation and self.config.doSubtract:
830  preConvKernel = None
831  if preConvPsf is not None:
832  preConvKernel = preConvPsf.getLocalKernel()
833  if self.config.convolveTemplate:
834  self.log.info("Decorrelation after template image convolution")
835  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
836  subtractedExposure,
837  subtractRes.psfMatchingKernel,
838  spatiallyVarying=self.config.doSpatiallyVarying,
839  preConvKernel=preConvKernel)
840  else:
841  self.log.info("Decorrelation after science image convolution")
842  decorrResult = self.decorrelate.run(subtractRes.warpedExposure, exposureOrig,
843  subtractedExposure,
844  subtractRes.psfMatchingKernel,
845  spatiallyVarying=self.config.doSpatiallyVarying,
846  preConvKernel=preConvKernel)
847  subtractedExposure = decorrResult.correctedExposure
848 
849  # END (if subtractAlgorithm == 'AL')
850  # END (if self.config.doSubtract)
851  if self.config.doDetection:
852  self.log.info("Running diaSource detection")
853  # Erase existing detection mask planes
854  mask = subtractedExposure.getMaskedImage().getMask()
855  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
856 
857  table = afwTable.SourceTable.make(self.schema, idFactory)
858  table.setMetadata(self.algMetadata)
859  results = self.detection.run(
860  table=table,
861  exposure=subtractedExposure,
862  doSmooth=not self.config.doPreConvolve
863  )
864 
865  if self.config.doMerge:
866  fpSet = results.fpSets.positive
867  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
868  self.config.growFootprint, False)
869  diaSources = afwTable.SourceCatalog(table)
870  fpSet.makeSources(diaSources)
871  self.log.info("Merging detections into %d sources" % (len(diaSources)))
872  else:
873  diaSources = results.sources
874 
875  if self.config.doMeasurement:
876  newDipoleFitting = self.config.doDipoleFitting
877  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
878  if not newDipoleFitting:
879  # Just fit dipole in diffim
880  self.measurement.run(diaSources, subtractedExposure)
881  else:
882  # Use (matched) template and science image (if avail.) to constrain dipole fitting
883  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
884  self.measurement.run(diaSources, subtractedExposure, exposure,
885  subtractRes.matchedExposure)
886  else:
887  self.measurement.run(diaSources, subtractedExposure, exposure)
888 
889  if self.config.doForcedMeasurement:
890  # Run forced psf photometry on the PVI at the diaSource locations.
891  # Copy the measured flux and error into the diaSource.
892  forcedSources = self.forcedMeasurement.generateMeasCat(
893  exposure, diaSources, subtractedExposure.getWcs())
894  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
895  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
896  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
897  "ip_diffim_forced_PsfFlux_instFlux", True)
898  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
899  "ip_diffim_forced_PsfFlux_instFluxErr", True)
900  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
901  "ip_diffim_forced_PsfFlux_area", True)
902  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
903  "ip_diffim_forced_PsfFlux_flag", True)
904  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
905  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
906  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
907  "ip_diffim_forced_PsfFlux_flag_edge", True)
908  for diaSource, forcedSource in zip(diaSources, forcedSources):
909  diaSource.assign(forcedSource, mapper)
910 
911  # Match with the calexp sources if possible
912  if self.config.doMatchSources:
913  if selectSources is not None:
914  # Create key,val pair where key=diaSourceId and val=sourceId
915  matchRadAsec = self.config.diaSourceMatchRadius
916  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
917 
918  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
919  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
920  srcMatch in srcMatches])
921  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
922  len(diaSources)))
923  else:
924  self.log.warn("Src product does not exist; cannot match with diaSources")
925  srcMatchDict = {}
926 
927  # Create key,val pair where key=diaSourceId and val=refId
928  refAstromConfig = AstrometryConfig()
929  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
930  refAstrometer = AstrometryTask(refAstromConfig)
931  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
932  refMatches = astromRet.matches
933  if refMatches is None:
934  self.log.warn("No diaSource matches with reference catalog")
935  refMatchDict = {}
936  else:
937  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
938  len(diaSources)))
939  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
940  refMatch in refMatches])
941 
942  # Assign source Ids
943  for diaSource in diaSources:
944  sid = diaSource.getId()
945  if sid in srcMatchDict:
946  diaSource.set("srcMatchId", srcMatchDict[sid])
947  if sid in refMatchDict:
948  diaSource.set("refMatchId", refMatchDict[sid])
949 
950  if self.config.doAddMetrics and self.config.doSelectSources:
951  self.log.info("Evaluating metrics and control sample")
952 
953  kernelCandList = []
954  for cell in subtractRes.kernelCellSet.getCellList():
955  for cand in cell.begin(False): # include bad candidates
956  kernelCandList.append(cand)
957 
958  # Get basis list to build control sample kernels
959  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
960  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
961 
962  controlCandList = (
963  diffimTools.sourceTableToCandidateList(controlSources,
964  subtractRes.warpedExposure, exposure,
965  self.config.subtract.kernel.active,
966  self.config.subtract.kernel.active.detectionConfig,
967  self.log, doBuild=True, basisList=basisList))
968 
969  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
970  subtractRes.backgroundModel, dof=nparam)
971  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
972  subtractRes.backgroundModel)
973 
974  if self.config.doDetection:
975  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
976  else:
977  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
978 
979  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
980  return pipeBase.Struct(
981  subtractedExposure=subtractedExposure,
982  matchedExposure=subtractRes.matchedExposure,
983  subtractRes=subtractRes,
984  diaSources=diaSources,
985  selectSources=selectSources
986  )
987 
988  def fitAstrometry(self, templateSources, templateExposure, selectSources):
989  """Fit the relative astrometry between templateSources and selectSources
990 
991  Todo
992  ----
993 
994  Remove this method. It originally fit a new WCS to the template before calling register.run
995  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
996  It remains because a subtask overrides it.
997  """
998  results = self.register.run(templateSources, templateExposure.getWcs(),
999  templateExposure.getBBox(), selectSources)
1000  return results
1001 
1002  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1003  """Make debug plots and displays.
1004 
1005  Todo
1006  ----
1007  Test and update for current debug display and slot names
1008  """
1009  import lsstDebug
1010  display = lsstDebug.Info(__name__).display
1011  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1012  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1013  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1014  showDipoles = lsstDebug.Info(__name__).showDipoles
1015  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1016  if display:
1017  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1018  if not maskTransparency:
1019  maskTransparency = 0
1020  disp.setMaskTransparency(maskTransparency)
1021 
1022  if display and showSubtracted:
1023  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1024  mi = subtractRes.subtractedExposure.getMaskedImage()
1025  x0, y0 = mi.getX0(), mi.getY0()
1026  with disp.Buffering():
1027  for s in diaSources:
1028  x, y = s.getX() - x0, s.getY() - y0
1029  ctype = "red" if s.get("flags_negative") else "yellow"
1030  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1031  or s.get("base_PixelFlags_flag_saturatedCenter")
1032  or s.get("base_PixelFlags_flag_crCenter")):
1033  ptype = "x"
1034  elif (s.get("base_PixelFlags_flag_interpolated")
1035  or s.get("base_PixelFlags_flag_saturated")
1036  or s.get("base_PixelFlags_flag_cr")):
1037  ptype = "+"
1038  else:
1039  ptype = "o"
1040  disp.dot(ptype, x, y, size=4, ctype=ctype)
1041  lsstDebug.frame += 1
1042 
1043  if display and showPixelResiduals and selectSources:
1044  nonKernelSources = []
1045  for source in selectSources:
1046  if source not in kernelSources:
1047  nonKernelSources.append(source)
1048 
1049  diUtils.plotPixelResiduals(exposure,
1050  subtractRes.warpedExposure,
1051  subtractRes.subtractedExposure,
1052  subtractRes.kernelCellSet,
1053  subtractRes.psfMatchingKernel,
1054  subtractRes.backgroundModel,
1055  nonKernelSources,
1056  self.subtract.config.kernel.active.detectionConfig,
1057  origVariance=False)
1058  diUtils.plotPixelResiduals(exposure,
1059  subtractRes.warpedExposure,
1060  subtractRes.subtractedExposure,
1061  subtractRes.kernelCellSet,
1062  subtractRes.psfMatchingKernel,
1063  subtractRes.backgroundModel,
1064  nonKernelSources,
1065  self.subtract.config.kernel.active.detectionConfig,
1066  origVariance=True)
1067  if display and showDiaSources:
1068  flagChecker = SourceFlagChecker(diaSources)
1069  isFlagged = [flagChecker(x) for x in diaSources]
1070  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1071  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1072  frame=lsstDebug.frame)
1073  lsstDebug.frame += 1
1074 
1075  if display and showDipoles:
1076  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1077  frame=lsstDebug.frame)
1078  lsstDebug.frame += 1
1079 
1080  def _getConfigName(self):
1081  """Return the name of the config dataset
1082  """
1083  return "%sDiff_config" % (self.config.coaddName,)
1084 
1085  def _getMetadataName(self):
1086  """Return the name of the metadata dataset
1087  """
1088  return "%sDiff_metadata" % (self.config.coaddName,)
1089 
1090  def getSchemaCatalogs(self):
1091  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1092  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1093 
1094  @classmethod
1095  def _makeArgumentParser(cls):
1096  """Create an argument parser
1097  """
1098  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1099  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1100  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1101  help="Template data ID in case of calexp template,"
1102  " e.g. --templateId visit=6789")
1103  return parser
1104 
1105 
1106 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1107  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1108  doc="Shift stars going into RegisterTask by this amount")
1109  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1110  doc="Perturb stars going into RegisterTask by this amount")
1111 
1112  def setDefaults(self):
1113  ImageDifferenceConfig.setDefaults(self)
1114  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1115 
1116 
1117 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1118  """!Image difference Task used in the Winter 2013 data challege.
1119  Enables testing the effects of registration shifts and scatter.
1120 
1121  For use with winter 2013 simulated images:
1122  Use --templateId visit=88868666 for sparse data
1123  --templateId visit=22222200 for dense data (g)
1124  --templateId visit=11111100 for dense data (i)
1125  """
1126  ConfigClass = Winter2013ImageDifferenceConfig
1127  _DefaultName = "winter2013ImageDifference"
1128 
1129  def __init__(self, **kwargs):
1130  ImageDifferenceTask.__init__(self, **kwargs)
1131 
1132  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1133  """Fit the relative astrometry between templateSources and selectSources"""
1134  if self.config.winter2013WcsShift > 0.0:
1135  offset = geom.Extent2D(self.config.winter2013WcsShift,
1136  self.config.winter2013WcsShift)
1137  cKey = templateSources[0].getTable().getCentroidKey()
1138  for source in templateSources:
1139  centroid = source.get(cKey)
1140  source.set(cKey, centroid + offset)
1141  elif self.config.winter2013WcsRms > 0.0:
1142  cKey = templateSources[0].getTable().getCentroidKey()
1143  for source in templateSources:
1144  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1145  self.config.winter2013WcsRms*numpy.random.normal())
1146  centroid = source.get(cKey)
1147  source.set(cKey, centroid + offset)
1148 
1149  results = self.register.run(templateSources, templateExposure.getWcs(),
1150  templateExposure.getBBox(), selectSources)
1151  return results
lsst.pipe.tasks.registerImage
Definition: registerImage.py:1
lsst::ip::diffim
lsst::meas::astrom
lsst::afw::display
lsst::ip::diffim::utils
lsst::utils::inheritDoc
lsst.pipe.tasks.scaleVariance
Definition: scaleVariance.py:1
lsst::ip::diffim::diffimTools
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:712
lsstDebug::Info
lsst::meas::base
lsst::afw::table
lsst::utils
lsst::geom
lsst.pipe.tasks.imageDifference.ImageDifferenceTaskConnections
Definition: imageDifference.py:54
lsst::daf::base
lsst::meas::algorithms::SingleGaussianPsf
lsst::afw::math
lsst.pipe::base
lsst::meas::algorithms
Extent< double, 2 >