lsst.pipe.tasks  20.0.0-17-g98646ba4+f780282b58
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 
47 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
49 IqrToSigma = 0.741
50 
51 
52 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
53  dimensions=("instrument", "visit", "detector", "skymap"),
54  defaultTemplates={"coaddName": "deep",
55  "skyMapName": "deep",
56  "warpTypeSuffix": "",
57  "fakesType": ""}):
58 
59  exposure = pipeBase.connectionTypes.Input(
60  doc="Input science exposure to subtract from.",
61  dimensions=("instrument", "visit", "detector"),
62  storageClass="ExposureF",
63  name="calexp"
64  )
65 
66  # TODO DM-22953
67  # kernelSources = pipeBase.connectionTypes.Input(
68  # doc="Source catalog produced in calibrate task for kernel candidate sources",
69  # name="src",
70  # storageClass="SourceCatalog",
71  # dimensions=("instrument", "visit", "detector"),
72  # )
73 
74  skyMap = pipeBase.connectionTypes.Input(
75  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
76  name="{skyMapName}Coadd_skyMap",
77  dimensions=("skymap", ),
78  storageClass="SkyMap",
79  )
80  coaddExposures = pipeBase.connectionTypes.Input(
81  doc="Input template to match and subtract from the exposure",
82  dimensions=("tract", "patch", "skymap", "abstract_filter"),
83  storageClass="ExposureF",
84  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
85  multiple=True,
86  deferLoad=True
87  )
88  dcrCoadds = pipeBase.connectionTypes.Input(
89  doc="Input DCR template to match and subtract from the exposure",
90  name="{fakesType}dcrCoadd{warpTypeSuffix}",
91  storageClass="ExposureF",
92  dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
93  multiple=True,
94  deferLoad=True
95  )
96  outputSchema = pipeBase.connectionTypes.InitOutput(
97  doc="Schema (as an example catalog) for output DIASource catalog.",
98  storageClass="SourceCatalog",
99  name="{fakesType}{coaddName}Diff_diaSrc_schema",
100  )
101  subtractedExposure = pipeBase.connectionTypes.Output(
102  doc="Output difference image",
103  dimensions=("instrument", "visit", "detector"),
104  storageClass="ExposureF",
105  name="{fakesType}{coaddName}Diff_differenceExp",
106  )
107  warpedExposure = pipeBase.connectionTypes.Output(
108  doc="Warped template used to create `subtractedExposure`.",
109  dimensions=("instrument", "visit", "detector"),
110  storageClass="ExposureF",
111  name="{fakesType}{coaddName}Diff_warpedExp",
112  )
113  diaSources = pipeBase.connectionTypes.Output(
114  doc="Output detected diaSources on the difference image",
115  dimensions=("instrument", "visit", "detector"),
116  storageClass="SourceCatalog",
117  name="{fakesType}{coaddName}Diff_diaSrc",
118  )
119 
120  def __init__(self, *, config=None):
121  super().__init__(config=config)
122  if config.coaddName == 'dcr':
123  self.inputs.remove("coaddExposures")
124  else:
125  self.inputs.remove("dcrCoadds")
126 
127  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
128  # Make kernelSources optional
129 
130 
131 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
132  pipelineConnections=ImageDifferenceTaskConnections):
133  """Config for ImageDifferenceTask
134  """
135  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
136  doc="Add background to calexp before processing it. "
137  "Useful as ipDiffim does background matching.")
138  doUseRegister = pexConfig.Field(dtype=bool, default=True,
139  doc="Use image-to-image registration to align template with "
140  "science image")
141  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
142  doc="Writing debugging data for doUseRegister")
143  doSelectSources = pexConfig.Field(dtype=bool, default=True,
144  doc="Select stars to use for kernel fitting")
145  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
146  doc="Select stars of extreme color as part of the control sample")
147  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
148  doc="Select stars that are variable to be part "
149  "of the control sample")
150  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
151  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
152  doc="Convolve science image by its PSF before PSF-matching?")
153  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
154  doc="Scale variance of the template before PSF matching")
155  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
156  doc="Use a simple gaussian PSF model for pre-convolution "
157  "(else use fit PSF)? Ignored if doPreConvolve false.")
158  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
159  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
160  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
161  "kernel convolution? If True, also update the diffim PSF.")
162  doMerge = pexConfig.Field(dtype=bool, default=True,
163  doc="Merge positive and negative diaSources with grow radius "
164  "set by growFootprint")
165  doMatchSources = pexConfig.Field(dtype=bool, default=True,
166  doc="Match diaSources with input calexp sources and ref catalog sources")
167  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
168  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
169  doForcedMeasurement = pexConfig.Field(
170  dtype=bool,
171  default=True,
172  doc="Force photometer diaSource locations on PVI?")
173  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
174  doWriteWarpedExp = pexConfig.Field(dtype=bool, default=False,
175  doc="Write WCS, warped template coadd exposure?")
176  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
177  doc="Write warped and PSF-matched template coadd exposure?")
178  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
179  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
180  doc="Add columns to the source table to hold analysis metrics?")
181 
182  coaddName = pexConfig.Field(
183  doc="coadd name: typically one of deep, goodSeeing, or dcr",
184  dtype=str,
185  default="deep",
186  )
187  convolveTemplate = pexConfig.Field(
188  doc="Which image gets convolved (default = template)",
189  dtype=bool,
190  default=True
191  )
192  refObjLoader = pexConfig.ConfigurableField(
193  target=LoadIndexedReferenceObjectsTask,
194  doc="reference object loader",
195  )
196  astrometer = pexConfig.ConfigurableField(
197  target=AstrometryTask,
198  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
199  )
200  sourceSelector = pexConfig.ConfigurableField(
201  target=ObjectSizeStarSelectorTask,
202  doc="Source selection algorithm",
203  )
204  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
205  decorrelate = pexConfig.ConfigurableField(
206  target=DecorrelateALKernelSpatialTask,
207  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
208  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
209  "default of 5.5).",
210  )
211  doSpatiallyVarying = pexConfig.Field(
212  dtype=bool,
213  default=False,
214  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
215  "image in order to allow for spatial variations"
216  )
217  detection = pexConfig.ConfigurableField(
218  target=SourceDetectionTask,
219  doc="Low-threshold detection for final measurement",
220  )
221  measurement = pexConfig.ConfigurableField(
222  target=DipoleFitTask,
223  doc="Enable updated dipole fitting method",
224  )
225  forcedMeasurement = pexConfig.ConfigurableField(
226  target=ForcedMeasurementTask,
227  doc="Subtask to force photometer PVI at diaSource location.",
228  )
229  getTemplate = pexConfig.ConfigurableField(
230  target=GetCoaddAsTemplateTask,
231  doc="Subtask to retrieve template exposure and sources",
232  )
233  scaleVariance = pexConfig.ConfigurableField(
234  target=ScaleVarianceTask,
235  doc="Subtask to rescale the variance of the template "
236  "to the statistically expected level"
237  )
238  controlStepSize = pexConfig.Field(
239  doc="What step size (every Nth one) to select a control sample from the kernelSources",
240  dtype=int,
241  default=5
242  )
243  controlRandomSeed = pexConfig.Field(
244  doc="Random seed for shuffing the control sample",
245  dtype=int,
246  default=10
247  )
248  register = pexConfig.ConfigurableField(
249  target=RegisterTask,
250  doc="Task to enable image-to-image image registration (warping)",
251  )
252  kernelSourcesFromRef = pexConfig.Field(
253  doc="Select sources to measure kernel from reference catalog if True, template if false",
254  dtype=bool,
255  default=False
256  )
257  templateSipOrder = pexConfig.Field(
258  dtype=int, default=2,
259  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
260  )
261  growFootprint = pexConfig.Field(
262  dtype=int, default=2,
263  doc="Grow positive and negative footprints by this amount before merging"
264  )
265  diaSourceMatchRadius = pexConfig.Field(
266  dtype=float, default=0.5,
267  doc="Match radius (in arcseconds) for DiaSource to Source association"
268  )
269 
270  def setDefaults(self):
271  # defaults are OK for catalog and diacatalog
272 
273  self.subtract['al'].kernel.name = "AL"
274  self.subtract['al'].kernel.active.fitForBackground = True
275  self.subtract['al'].kernel.active.spatialKernelOrder = 1
276  self.subtract['al'].kernel.active.spatialBgOrder = 2
277  self.doPreConvolve = False
278  self.doMatchSources = False
279  self.doAddMetrics = False
280  self.doUseRegister = False
281 
282  # DiaSource Detection
283  self.detection.thresholdPolarity = "both"
284  self.detection.thresholdValue = 5.5
285  self.detection.reEstimateBackground = False
286  self.detection.thresholdType = "pixel_stdev"
287 
288  # Add filtered flux measurement, the correct measurement for pre-convolved images.
289  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
290  # To change that you must modify algorithms.names in the task's applyOverrides method,
291  # after the user has set doPreConvolve.
292  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
293  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
294  'base_LocalWcs']
295 
296  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
297  self.forcedMeasurement.copyColumns = {
298  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
299  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
300  self.forcedMeasurement.slots.shape = None
301 
302  # For shuffling the control sample
303  random.seed(self.controlRandomSeed)
304 
305  def validate(self):
306  pexConfig.Config.validate(self)
307  if self.doAddMetrics and not self.doSubtract:
308  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
309  if not self.doSubtract and not self.doDetection:
310  raise ValueError("Either doSubtract or doDetection must be enabled.")
311  if self.subtract.name == 'zogy' and self.doAddMetrics:
312  raise ValueError("Kernel metrics does not exist in zogy subtraction.")
313  if self.doMeasurement and not self.doDetection:
314  raise ValueError("Cannot run source measurement without source detection.")
315  if self.doMerge and not self.doDetection:
316  raise ValueError("Cannot run source merging without source detection.")
317  if self.doUseRegister and not self.doSelectSources:
318  raise ValueError("doUseRegister=True and doSelectSources=False. "
319  "Cannot run RegisterTask without selecting sources.")
320  if self.doPreConvolve and self.doDecorrelation and not self.convolveTemplate:
321  raise ValueError("doPreConvolve=True and doDecorrelation=True and "
322  "convolveTemplate=False is not supported.")
323  if hasattr(self.getTemplate, "coaddName"):
324  if self.getTemplate.coaddName != self.coaddName:
325  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
326 
327 
328 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
329 
330  @staticmethod
331  def getTargetList(parsedCmd, **kwargs):
332  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
333  **kwargs)
334 
335 
336 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
337  """Subtract an image from a template and measure the result
338  """
339  ConfigClass = ImageDifferenceConfig
340  RunnerClass = ImageDifferenceTaskRunner
341  _DefaultName = "imageDifference"
342 
343  def __init__(self, butler=None, **kwargs):
344  """!Construct an ImageDifference Task
345 
346  @param[in] butler Butler object to use in constructing reference object loaders
347  """
348  super().__init__(**kwargs)
349  self.makeSubtask("getTemplate")
350 
351  self.makeSubtask("subtract")
352 
353  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
354  self.makeSubtask("decorrelate")
355 
356  if self.config.doScaleTemplateVariance:
357  self.makeSubtask("scaleVariance")
358 
359  if self.config.doUseRegister:
360  self.makeSubtask("register")
361  self.schema = afwTable.SourceTable.makeMinimalSchema()
362 
363  if self.config.doSelectSources:
364  self.makeSubtask("sourceSelector")
365  if self.config.kernelSourcesFromRef:
366  self.makeSubtask('refObjLoader', butler=butler)
367  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
368 
369  self.algMetadata = dafBase.PropertyList()
370  if self.config.doDetection:
371  self.makeSubtask("detection", schema=self.schema)
372  if self.config.doMeasurement:
373  self.makeSubtask("measurement", schema=self.schema,
374  algMetadata=self.algMetadata)
375  if self.config.doForcedMeasurement:
376  self.schema.addField(
377  "ip_diffim_forced_PsfFlux_instFlux", "D",
378  "Forced PSF flux measured on the direct image.")
379  self.schema.addField(
380  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
381  "Forced PSF flux error measured on the direct image.")
382  self.schema.addField(
383  "ip_diffim_forced_PsfFlux_area", "F",
384  "Forced PSF flux effective area of PSF.",
385  units="pixel")
386  self.schema.addField(
387  "ip_diffim_forced_PsfFlux_flag", "Flag",
388  "Forced PSF flux general failure flag.")
389  self.schema.addField(
390  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
391  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
392  self.schema.addField(
393  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
394  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
395  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
396  if self.config.doMatchSources:
397  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
398  self.schema.addField("srcMatchId", "L", "unique id of source match")
399 
400  # initialize InitOutputs
401  self.outputSchema = afwTable.SourceCatalog(self.schema)
402  self.outputSchema.getTable().setMetadata(self.algMetadata)
403 
404  @staticmethod
405  def makeIdFactory(expId, expBits):
406  """Create IdFactory instance for unique 64 bit diaSource id-s.
407 
408  Parameters
409  ----------
410  expId : `int`
411  Exposure id.
412 
413  expBits: `int`
414  Number of used bits in ``expId``.
415 
416  Note
417  ----
418  The diasource id-s consists of the ``expId`` stored fixed in the highest value
419  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
420  low value end of the integer.
421 
422  Returns
423  -------
424  idFactory: `lsst.afw.table.IdFactory`
425  """
426  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
427 
428  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
429  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
430  inputRefs: pipeBase.InputQuantizedConnection,
431  outputRefs: pipeBase.OutputQuantizedConnection):
432  inputs = butlerQC.get(inputRefs)
433  self.log.info(f"Processing {butlerQC.quantum.dataId}")
434  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
435  returnMaxBits=True)
436  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
437  if self.config.coaddName == 'dcr':
438  templateExposures = inputRefs.dcrCoadds
439  else:
440  templateExposures = inputRefs.coaddExposures
441  templateStruct = self.getTemplate.runQuantum(
442  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
443  )
444 
445  outputs = self.run(exposure=inputs['exposure'],
446  templateExposure=templateStruct.exposure,
447  idFactory=idFactory)
448  butlerQC.put(outputs, outputRefs)
449 
450  @pipeBase.timeMethod
451  def runDataRef(self, sensorRef, templateIdList=None):
452  """Subtract an image from a template coadd and measure the result.
453 
454  Data I/O wrapper around `run` using the butler in Gen2.
455 
456  Parameters
457  ----------
458  sensorRef : `lsst.daf.persistence.ButlerDataRef`
459  Sensor-level butler data reference, used for the following data products:
460 
461  Input only:
462  - calexp
463  - psf
464  - ccdExposureId
465  - ccdExposureId_bits
466  - self.config.coaddName + "Coadd_skyMap"
467  - self.config.coaddName + "Coadd"
468  Input or output, depending on config:
469  - self.config.coaddName + "Diff_subtractedExp"
470  Output, depending on config:
471  - self.config.coaddName + "Diff_matchedExp"
472  - self.config.coaddName + "Diff_src"
473 
474  Returns
475  -------
476  results : `lsst.pipe.base.Struct`
477  Returns the Struct by `run`.
478  """
479  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
480  subtractedExposure = None
481  selectSources = None
482  calexpBackgroundExposure = None
483  self.log.info("Processing %s" % (sensorRef.dataId))
484 
485  # We make one IdFactory that will be used by both icSrc and src datasets;
486  # I don't know if this is the way we ultimately want to do things, but at least
487  # this ensures the source IDs are fully unique.
488  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
489  expBits=sensorRef.get("ccdExposureId_bits"))
490  if self.config.doAddCalexpBackground:
491  calexpBackgroundExposure = sensorRef.get("calexpBackground")
492 
493  # Retrieve the science image we wish to analyze
494  exposure = sensorRef.get("calexp", immediate=True)
495 
496  # Retrieve the template image
497  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
498 
499  if sensorRef.datasetExists("src"):
500  self.log.info("Source selection via src product")
501  # Sources already exist; for data release processing
502  selectSources = sensorRef.get("src")
503 
504  if not self.config.doSubtract and self.config.doDetection:
505  # If we don't do subtraction, we need the subtracted exposure from the repo
506  subtractedExposure = sensorRef.get(subtractedExposureName)
507  # Both doSubtract and doDetection cannot be False
508 
509  results = self.run(exposure=exposure,
510  selectSources=selectSources,
511  templateExposure=template.exposure,
512  templateSources=template.sources,
513  idFactory=idFactory,
514  calexpBackgroundExposure=calexpBackgroundExposure,
515  subtractedExposure=subtractedExposure)
516 
517  if self.config.doWriteSources and results.diaSources is not None:
518  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
519  if self.config.doWriteWarpedExp:
520  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
521  if self.config.doWriteMatchedExp:
522  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
523  if self.config.doAddMetrics and self.config.doSelectSources:
524  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
525  if self.config.doWriteSubtractedExp:
526  sensorRef.put(results.subtractedExposure, subtractedExposureName)
527  return results
528 
529  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
530  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
531  """PSF matches, subtract two images and perform detection on the difference image.
532 
533  Parameters
534  ----------
535  exposure : `lsst.afw.image.ExposureF`, optional
536  The science exposure, the minuend in the image subtraction.
537  Can be None only if ``config.doSubtract==False``.
538  selectSources : `lsst.afw.table.SourceCatalog`, optional
539  Identified sources on the science exposure. This catalog is used to
540  select sources in order to perform the AL PSF matching on stamp images
541  around them. The selection steps depend on config options and whether
542  ``templateSources`` and ``matchingSources`` specified.
543  templateExposure : `lsst.afw.image.ExposureF`, optional
544  The template to be subtracted from ``exposure`` in the image subtraction.
545  The template exposure should cover the same sky area as the science exposure.
546  It is either a stich of patches of a coadd skymap image or a calexp
547  of the same pointing as the science exposure. Can be None only
548  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
549  templateSources : `lsst.afw.table.SourceCatalog`, optional
550  Identified sources on the template exposure.
551  idFactory : `lsst.afw.table.IdFactory`
552  Generator object to assign ids to detected sources in the difference image.
553  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
554  Background exposure to be added back to the science exposure
555  if ``config.doAddCalexpBackground==True``
556  subtractedExposure : `lsst.afw.image.ExposureF`, optional
557  If ``config.doSubtract==False`` and ``config.doDetection==True``,
558  performs the post subtraction source detection only on this exposure.
559  Otherwise should be None.
560 
561  Returns
562  -------
563  results : `lsst.pipe.base.Struct`
564  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
565  Difference image.
566  ``matchedExposure`` : `lsst.afw.image.ExposureF`
567  The matched PSF exposure.
568  ``subtractRes`` : `lsst.pipe.base.Struct`
569  The returned result structure of the ImagePsfMatchTask subtask.
570  ``diaSources`` : `lsst.afw.table.SourceCatalog`
571  The catalog of detected sources.
572  ``selectSources`` : `lsst.afw.table.SourceCatalog`
573  The input source catalog with optionally added Qa information.
574 
575  Notes
576  -----
577  The following major steps are included:
578 
579  - warp template coadd to match WCS of image
580  - PSF match image to warped template
581  - subtract image from PSF-matched, warped template
582  - detect sources
583  - measure sources
584 
585  For details about the image subtraction configuration modes
586  see `lsst.ip.diffim`.
587  """
588  subtractRes = None
589  controlSources = None
590  diaSources = None
591  kernelSources = None
592 
593  if self.config.doAddCalexpBackground:
594  mi = exposure.getMaskedImage()
595  mi += calexpBackgroundExposure.getImage()
596 
597  if not exposure.hasPsf():
598  raise pipeBase.TaskError("Exposure has no psf")
599  sciencePsf = exposure.getPsf()
600 
601  if self.config.doSubtract:
602  if self.config.doScaleTemplateVariance:
603  templateVarFactor = self.scaleVariance.run(
604  templateExposure.getMaskedImage())
605  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
606 
607  if self.config.subtract.name == 'zogy':
608  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
609  if self.config.doPreConvolve:
610  subtractedExposure = subtractRes.scoreExp
611  else:
612  subtractedExposure = subtractRes.diffExp
613  subtractRes.subtractedExposure = subtractedExposure
614  subtractRes.matchedExposure = None
615 
616  elif self.config.subtract.name == 'al':
617  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
618  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
619  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
620 
621  # if requested, convolve the science exposure with its PSF
622  # (properly, this should be a cross-correlation, but our code does not yet support that)
623  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
624  # else sigma of original science exposure
625  # TODO: DM-22762 This functional block should be moved into its own method
626  preConvPsf = None
627  if self.config.doPreConvolve:
628  convControl = afwMath.ConvolutionControl()
629  # cannot convolve in place, so make a new MI to receive convolved image
630  srcMI = exposure.getMaskedImage()
631  exposureOrig = exposure.clone()
632  destMI = srcMI.Factory(srcMI.getDimensions())
633  srcPsf = sciencePsf
634  if self.config.useGaussianForPreConvolution:
635  # convolve with a simplified PSF model: a double Gaussian
636  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
637  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
638  else:
639  # convolve with science exposure's PSF model
640  preConvPsf = srcPsf
641  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
642  exposure.setMaskedImage(destMI)
643  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
644  else:
645  scienceSigmaPost = scienceSigmaOrig
646  exposureOrig = exposure
647 
648  # If requested, find and select sources from the image
649  # else, AL subtraction will do its own source detection
650  # TODO: DM-22762 This functional block should be moved into its own method
651  if self.config.doSelectSources:
652  if selectSources is None:
653  self.log.warn("Src product does not exist; running detection, measurement, selection")
654  # Run own detection and measurement; necessary in nightly processing
655  selectSources = self.subtract.getSelectSources(
656  exposure,
657  sigma=scienceSigmaPost,
658  doSmooth=not self.config.doPreConvolve,
659  idFactory=idFactory,
660  )
661 
662  if self.config.doAddMetrics:
663  # Number of basis functions
664 
665  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
666  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
667  targetFwhmPix=templateSigma*FwhmPerSigma))
668  # Modify the schema of all Sources
669  # DEPRECATED: This is a data dependent (nparam) output product schema
670  # outside the task constructor.
671  # NOTE: The pre-determination of nparam at this point
672  # may be incorrect as the template psf is warped later in
673  # ImagePsfMatchTask.matchExposures()
674  kcQa = KernelCandidateQa(nparam)
675  selectSources = kcQa.addToSchema(selectSources)
676  if self.config.kernelSourcesFromRef:
677  # match exposure sources to reference catalog
678  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
679  matches = astromRet.matches
680  elif templateSources:
681  # match exposure sources to template sources
682  mc = afwTable.MatchControl()
683  mc.findOnlyClosest = False
684  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
685  mc)
686  else:
687  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
688  "but template sources not available. Cannot match science "
689  "sources with template sources. Run process* on data from "
690  "which templates are built.")
691 
692  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
693  matches=matches).sourceCat
694  random.shuffle(kernelSources, random.random)
695  controlSources = kernelSources[::self.config.controlStepSize]
696  kernelSources = [k for i, k in enumerate(kernelSources)
697  if i % self.config.controlStepSize]
698 
699  if self.config.doSelectDcrCatalog:
700  redSelector = DiaCatalogSourceSelectorTask(
701  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
702  grMax=99.999))
703  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
704  controlSources.extend(redSources)
705 
706  blueSelector = DiaCatalogSourceSelectorTask(
707  DiaCatalogSourceSelectorConfig(grMin=-99.999,
708  grMax=self.sourceSelector.config.grMin))
709  blueSources = blueSelector.selectStars(exposure, selectSources,
710  matches=matches).starCat
711  controlSources.extend(blueSources)
712 
713  if self.config.doSelectVariableCatalog:
714  varSelector = DiaCatalogSourceSelectorTask(
715  DiaCatalogSourceSelectorConfig(includeVariable=True))
716  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
717  controlSources.extend(varSources)
718 
719  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
720  % (len(kernelSources), len(selectSources), len(controlSources)))
721 
722  allresids = {}
723  # TODO: DM-22762 This functional block should be moved into its own method
724  if self.config.doUseRegister:
725  self.log.info("Registering images")
726 
727  if templateSources is None:
728  # Run detection on the template, which is
729  # temporarily background-subtracted
730  # sigma of PSF of template image before warping
731  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
732  templateSources = self.subtract.getSelectSources(
733  templateExposure,
734  sigma=templateSigma,
735  doSmooth=True,
736  idFactory=idFactory
737  )
738 
739  # Third step: we need to fit the relative astrometry.
740  #
741  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
742  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
743  exposure.getWcs(), exposure.getBBox())
744  templateExposure = warpedExp
745 
746  # Create debugging outputs on the astrometric
747  # residuals as a function of position. Persistence
748  # not yet implemented; expected on (I believe) #2636.
749  if self.config.doDebugRegister:
750  # Grab matches to reference catalog
751  srcToMatch = {x.second.getId(): x.first for x in matches}
752 
753  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
754  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
755  sids = [m.first.getId() for m in wcsResults.matches]
756  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
757  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
758  m.second.get(inCentroidKey))) for m in wcsResults.matches]
759  allresids = dict(zip(sids, zip(positions, residuals)))
760 
761  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
762  wcsResults.wcs.pixelToSky(
763  m.second.get(inCentroidKey))) for m in wcsResults.matches]
764  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
765  + 2.5*numpy.log10(srcToMatch[x].get("r"))
766  for x in sids if x in srcToMatch.keys()])
767  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
768  if s in srcToMatch.keys()])
769  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
770  if s in srcToMatch.keys()])
771  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
772  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
773  & (colors <= self.sourceSelector.config.grMax))
774  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
775  rms1Long = IqrToSigma*(
776  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
777  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
778  - numpy.percentile(dlat[idx1], 25))
779  rms2Long = IqrToSigma*(
780  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
781  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
782  - numpy.percentile(dlat[idx2], 25))
783  rms3Long = IqrToSigma*(
784  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
785  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
786  - numpy.percentile(dlat[idx3], 25))
787  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
788  (numpy.median(dlong[idx1]), rms1Long,
789  numpy.median(dlat[idx1]), rms1Lat))
790  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
791  (numpy.median(dlong[idx2]), rms2Long,
792  numpy.median(dlat[idx2]), rms2Lat))
793  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
794  (numpy.median(dlong[idx3]), rms3Long,
795  numpy.median(dlat[idx3]), rms3Lat))
796 
797  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
798  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
799  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
800  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
801  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
802  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
803 
804  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
805  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
806  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
807  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
808  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
809  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
810 
811  # warp template exposure to match exposure,
812  # PSF match template exposure to exposure,
813  # then return the difference
814 
815  # Return warped template... Construct sourceKernelCand list after subtract
816  self.log.info("Subtracting images")
817  subtractRes = self.subtract.subtractExposures(
818  templateExposure=templateExposure,
819  scienceExposure=exposure,
820  candidateList=kernelSources,
821  convolveTemplate=self.config.convolveTemplate,
822  doWarping=not self.config.doUseRegister
823  )
824  subtractedExposure = subtractRes.subtractedExposure
825 
826  if self.config.doDetection:
827  self.log.info("Computing diffim PSF")
828 
829  # Get Psf from the appropriate input image if it doesn't exist
830  if not subtractedExposure.hasPsf():
831  if self.config.convolveTemplate:
832  subtractedExposure.setPsf(exposure.getPsf())
833  else:
834  subtractedExposure.setPsf(templateExposure.getPsf())
835 
836  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
837  # thus it may have already been decorrelated. Thus, we do not decorrelate if
838  # doSubtract is False.
839 
840  # NOTE: At this point doSubtract == True
841  if self.config.doDecorrelation and self.config.doSubtract:
842  preConvKernel = None
843  if preConvPsf is not None:
844  preConvKernel = preConvPsf.getLocalKernel()
845  if self.config.convolveTemplate:
846  self.log.info("Decorrelation after template image convolution")
847  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
848  subtractedExposure,
849  subtractRes.psfMatchingKernel,
850  spatiallyVarying=self.config.doSpatiallyVarying,
851  preConvKernel=preConvKernel)
852  else:
853  self.log.info("Decorrelation after science image convolution")
854  decorrResult = self.decorrelate.run(subtractRes.warpedExposure, exposureOrig,
855  subtractedExposure,
856  subtractRes.psfMatchingKernel,
857  spatiallyVarying=self.config.doSpatiallyVarying,
858  preConvKernel=preConvKernel)
859  subtractedExposure = decorrResult.correctedExposure
860 
861  # END (if subtractAlgorithm == 'AL')
862  # END (if self.config.doSubtract)
863  if self.config.doDetection:
864  self.log.info("Running diaSource detection")
865  # Erase existing detection mask planes
866  mask = subtractedExposure.getMaskedImage().getMask()
867  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
868 
869  table = afwTable.SourceTable.make(self.schema, idFactory)
870  table.setMetadata(self.algMetadata)
871  results = self.detection.run(
872  table=table,
873  exposure=subtractedExposure,
874  doSmooth=not self.config.doPreConvolve
875  )
876 
877  if self.config.doMerge:
878  fpSet = results.fpSets.positive
879  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
880  self.config.growFootprint, False)
881  diaSources = afwTable.SourceCatalog(table)
882  fpSet.makeSources(diaSources)
883  self.log.info("Merging detections into %d sources" % (len(diaSources)))
884  else:
885  diaSources = results.sources
886 
887  if self.config.doMeasurement:
888  newDipoleFitting = self.config.doDipoleFitting
889  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
890  if not newDipoleFitting:
891  # Just fit dipole in diffim
892  self.measurement.run(diaSources, subtractedExposure)
893  else:
894  # Use (matched) template and science image (if avail.) to constrain dipole fitting
895  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
896  self.measurement.run(diaSources, subtractedExposure, exposure,
897  subtractRes.matchedExposure)
898  else:
899  self.measurement.run(diaSources, subtractedExposure, exposure)
900 
901  if self.config.doForcedMeasurement:
902  # Run forced psf photometry on the PVI at the diaSource locations.
903  # Copy the measured flux and error into the diaSource.
904  forcedSources = self.forcedMeasurement.generateMeasCat(
905  exposure, diaSources, subtractedExposure.getWcs())
906  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
907  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
908  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
909  "ip_diffim_forced_PsfFlux_instFlux", True)
910  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
911  "ip_diffim_forced_PsfFlux_instFluxErr", True)
912  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
913  "ip_diffim_forced_PsfFlux_area", True)
914  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
915  "ip_diffim_forced_PsfFlux_flag", True)
916  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
917  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
918  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
919  "ip_diffim_forced_PsfFlux_flag_edge", True)
920  for diaSource, forcedSource in zip(diaSources, forcedSources):
921  diaSource.assign(forcedSource, mapper)
922 
923  # Match with the calexp sources if possible
924  if self.config.doMatchSources:
925  if selectSources is not None:
926  # Create key,val pair where key=diaSourceId and val=sourceId
927  matchRadAsec = self.config.diaSourceMatchRadius
928  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
929 
930  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
931  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
932  srcMatch in srcMatches])
933  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
934  len(diaSources)))
935  else:
936  self.log.warn("Src product does not exist; cannot match with diaSources")
937  srcMatchDict = {}
938 
939  # Create key,val pair where key=diaSourceId and val=refId
940  refAstromConfig = AstrometryConfig()
941  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
942  refAstrometer = AstrometryTask(refAstromConfig)
943  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
944  refMatches = astromRet.matches
945  if refMatches is None:
946  self.log.warn("No diaSource matches with reference catalog")
947  refMatchDict = {}
948  else:
949  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
950  len(diaSources)))
951  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
952  refMatch in refMatches])
953 
954  # Assign source Ids
955  for diaSource in diaSources:
956  sid = diaSource.getId()
957  if sid in srcMatchDict:
958  diaSource.set("srcMatchId", srcMatchDict[sid])
959  if sid in refMatchDict:
960  diaSource.set("refMatchId", refMatchDict[sid])
961 
962  if self.config.doAddMetrics and self.config.doSelectSources:
963  self.log.info("Evaluating metrics and control sample")
964 
965  kernelCandList = []
966  for cell in subtractRes.kernelCellSet.getCellList():
967  for cand in cell.begin(False): # include bad candidates
968  kernelCandList.append(cand)
969 
970  # Get basis list to build control sample kernels
971  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
972  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
973 
974  controlCandList = (
975  diffimTools.sourceTableToCandidateList(controlSources,
976  subtractRes.warpedExposure, exposure,
977  self.config.subtract.kernel.active,
978  self.config.subtract.kernel.active.detectionConfig,
979  self.log, doBuild=True, basisList=basisList))
980 
981  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
982  subtractRes.backgroundModel, dof=nparam)
983  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
984  subtractRes.backgroundModel)
985 
986  if self.config.doDetection:
987  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
988  else:
989  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
990 
991  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
992  return pipeBase.Struct(
993  subtractedExposure=subtractedExposure,
994  warpedExposure=subtractRes.warpedExposure,
995  matchedExposure=subtractRes.matchedExposure,
996  subtractRes=subtractRes,
997  diaSources=diaSources,
998  selectSources=selectSources
999  )
1000 
1001  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1002  """Fit the relative astrometry between templateSources and selectSources
1003 
1004  Todo
1005  ----
1006 
1007  Remove this method. It originally fit a new WCS to the template before calling register.run
1008  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1009  It remains because a subtask overrides it.
1010  """
1011  results = self.register.run(templateSources, templateExposure.getWcs(),
1012  templateExposure.getBBox(), selectSources)
1013  return results
1014 
1015  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1016  """Make debug plots and displays.
1017 
1018  Todo
1019  ----
1020  Test and update for current debug display and slot names
1021  """
1022  import lsstDebug
1023  display = lsstDebug.Info(__name__).display
1024  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1025  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1026  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1027  showDipoles = lsstDebug.Info(__name__).showDipoles
1028  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1029  if display:
1030  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1031  if not maskTransparency:
1032  maskTransparency = 0
1033  disp.setMaskTransparency(maskTransparency)
1034 
1035  if display and showSubtracted:
1036  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1037  mi = subtractRes.subtractedExposure.getMaskedImage()
1038  x0, y0 = mi.getX0(), mi.getY0()
1039  with disp.Buffering():
1040  for s in diaSources:
1041  x, y = s.getX() - x0, s.getY() - y0
1042  ctype = "red" if s.get("flags_negative") else "yellow"
1043  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1044  or s.get("base_PixelFlags_flag_saturatedCenter")
1045  or s.get("base_PixelFlags_flag_crCenter")):
1046  ptype = "x"
1047  elif (s.get("base_PixelFlags_flag_interpolated")
1048  or s.get("base_PixelFlags_flag_saturated")
1049  or s.get("base_PixelFlags_flag_cr")):
1050  ptype = "+"
1051  else:
1052  ptype = "o"
1053  disp.dot(ptype, x, y, size=4, ctype=ctype)
1054  lsstDebug.frame += 1
1055 
1056  if display and showPixelResiduals and selectSources:
1057  nonKernelSources = []
1058  for source in selectSources:
1059  if source not in kernelSources:
1060  nonKernelSources.append(source)
1061 
1062  diUtils.plotPixelResiduals(exposure,
1063  subtractRes.warpedExposure,
1064  subtractRes.subtractedExposure,
1065  subtractRes.kernelCellSet,
1066  subtractRes.psfMatchingKernel,
1067  subtractRes.backgroundModel,
1068  nonKernelSources,
1069  self.subtract.config.kernel.active.detectionConfig,
1070  origVariance=False)
1071  diUtils.plotPixelResiduals(exposure,
1072  subtractRes.warpedExposure,
1073  subtractRes.subtractedExposure,
1074  subtractRes.kernelCellSet,
1075  subtractRes.psfMatchingKernel,
1076  subtractRes.backgroundModel,
1077  nonKernelSources,
1078  self.subtract.config.kernel.active.detectionConfig,
1079  origVariance=True)
1080  if display and showDiaSources:
1081  flagChecker = SourceFlagChecker(diaSources)
1082  isFlagged = [flagChecker(x) for x in diaSources]
1083  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1084  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1085  frame=lsstDebug.frame)
1086  lsstDebug.frame += 1
1087 
1088  if display and showDipoles:
1089  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1090  frame=lsstDebug.frame)
1091  lsstDebug.frame += 1
1092 
1093  def _getConfigName(self):
1094  """Return the name of the config dataset
1095  """
1096  return "%sDiff_config" % (self.config.coaddName,)
1097 
1098  def _getMetadataName(self):
1099  """Return the name of the metadata dataset
1100  """
1101  return "%sDiff_metadata" % (self.config.coaddName,)
1102 
1103  def getSchemaCatalogs(self):
1104  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1105  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1106 
1107  @classmethod
1108  def _makeArgumentParser(cls):
1109  """Create an argument parser
1110  """
1111  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1112  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1113  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1114  help="Template data ID in case of calexp template,"
1115  " e.g. --templateId visit=6789")
1116  return parser
1117 
1118 
1119 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1120  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1121  doc="Shift stars going into RegisterTask by this amount")
1122  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1123  doc="Perturb stars going into RegisterTask by this amount")
1124 
1125  def setDefaults(self):
1126  ImageDifferenceConfig.setDefaults(self)
1127  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1128 
1129 
1130 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1131  """!Image difference Task used in the Winter 2013 data challege.
1132  Enables testing the effects of registration shifts and scatter.
1133 
1134  For use with winter 2013 simulated images:
1135  Use --templateId visit=88868666 for sparse data
1136  --templateId visit=22222200 for dense data (g)
1137  --templateId visit=11111100 for dense data (i)
1138  """
1139  ConfigClass = Winter2013ImageDifferenceConfig
1140  _DefaultName = "winter2013ImageDifference"
1141 
1142  def __init__(self, **kwargs):
1143  ImageDifferenceTask.__init__(self, **kwargs)
1144 
1145  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1146  """Fit the relative astrometry between templateSources and selectSources"""
1147  if self.config.winter2013WcsShift > 0.0:
1148  offset = geom.Extent2D(self.config.winter2013WcsShift,
1149  self.config.winter2013WcsShift)
1150  cKey = templateSources[0].getTable().getCentroidKey()
1151  for source in templateSources:
1152  centroid = source.get(cKey)
1153  source.set(cKey, centroid + offset)
1154  elif self.config.winter2013WcsRms > 0.0:
1155  cKey = templateSources[0].getTable().getCentroidKey()
1156  for source in templateSources:
1157  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1158  self.config.winter2013WcsRms*numpy.random.normal())
1159  centroid = source.get(cKey)
1160  source.set(cKey, centroid + offset)
1161 
1162  results = self.register.run(templateSources, templateExposure.getWcs(),
1163  templateExposure.getBBox(), selectSources)
1164  return results
lsst.pipe.tasks.registerImage
Definition: registerImage.py:1
lsst::ip::diffim
lsst::meas::astrom
lsst::afw::display
lsst::ip::diffim::utils
lsst::utils::inheritDoc
lsst.pipe.tasks.scaleVariance
Definition: scaleVariance.py:1
lsst::ip::diffim::diffimTools
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:712
lsstDebug::Info
lsst::meas::base
lsst::afw::table
lsst::utils
lsst::geom
lsst.pipe.tasks.imageDifference.ImageDifferenceTaskConnections
Definition: imageDifference.py:54
lsst::daf::base
lsst::meas::algorithms::SingleGaussianPsf
lsst::afw::math
lsst.pipe::base
lsst::meas::algorithms
Extent< double, 2 >