lsst.pipe.tasks  20.0.0-30-g1d38f5b5+be27eade48
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 
47 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
49 IqrToSigma = 0.741
50 
51 
52 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
53  dimensions=("instrument", "visit", "detector", "skymap"),
54  defaultTemplates={"coaddName": "deep",
55  "skyMapName": "deep",
56  "warpTypeSuffix": "",
57  "fakesType": ""}):
58 
59  exposure = pipeBase.connectionTypes.Input(
60  doc="Input science exposure to subtract from.",
61  dimensions=("instrument", "visit", "detector"),
62  storageClass="ExposureF",
63  name="calexp"
64  )
65 
66  # TODO DM-22953
67  # kernelSources = pipeBase.connectionTypes.Input(
68  # doc="Source catalog produced in calibrate task for kernel candidate sources",
69  # name="src",
70  # storageClass="SourceCatalog",
71  # dimensions=("instrument", "visit", "detector"),
72  # )
73 
74  skyMap = pipeBase.connectionTypes.Input(
75  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
76  name="{skyMapName}Coadd_skyMap",
77  dimensions=("skymap", ),
78  storageClass="SkyMap",
79  )
80  coaddExposures = pipeBase.connectionTypes.Input(
81  doc="Input template to match and subtract from the exposure",
82  dimensions=("tract", "patch", "skymap", "band"),
83  storageClass="ExposureF",
84  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
85  multiple=True,
86  deferLoad=True
87  )
88  dcrCoadds = pipeBase.connectionTypes.Input(
89  doc="Input DCR template to match and subtract from the exposure",
90  name="{fakesType}dcrCoadd{warpTypeSuffix}",
91  storageClass="ExposureF",
92  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
93  multiple=True,
94  deferLoad=True
95  )
96  outputSchema = pipeBase.connectionTypes.InitOutput(
97  doc="Schema (as an example catalog) for output DIASource catalog.",
98  storageClass="SourceCatalog",
99  name="{fakesType}{coaddName}Diff_diaSrc_schema",
100  )
101  subtractedExposure = pipeBase.connectionTypes.Output(
102  doc="Output difference image",
103  dimensions=("instrument", "visit", "detector"),
104  storageClass="ExposureF",
105  name="{fakesType}{coaddName}Diff_differenceExp",
106  )
107  warpedExposure = pipeBase.connectionTypes.Output(
108  doc="Warped template used to create `subtractedExposure`.",
109  dimensions=("instrument", "visit", "detector"),
110  storageClass="ExposureF",
111  name="{fakesType}{coaddName}Diff_warpedExp",
112  )
113  diaSources = pipeBase.connectionTypes.Output(
114  doc="Output detected diaSources on the difference image",
115  dimensions=("instrument", "visit", "detector"),
116  storageClass="SourceCatalog",
117  name="{fakesType}{coaddName}Diff_diaSrc",
118  )
119 
120  def __init__(self, *, config=None):
121  super().__init__(config=config)
122  if config.coaddName == 'dcr':
123  self.inputs.remove("coaddExposures")
124  else:
125  self.inputs.remove("dcrCoadds")
126 
127  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
128  # Make kernelSources optional
129 
130 
131 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
132  pipelineConnections=ImageDifferenceTaskConnections):
133  """Config for ImageDifferenceTask
134  """
135  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
136  doc="Add background to calexp before processing it. "
137  "Useful as ipDiffim does background matching.")
138  doUseRegister = pexConfig.Field(dtype=bool, default=True,
139  doc="Use image-to-image registration to align template with "
140  "science image")
141  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
142  doc="Writing debugging data for doUseRegister")
143  doSelectSources = pexConfig.Field(dtype=bool, default=True,
144  doc="Select stars to use for kernel fitting")
145  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
146  doc="Select stars of extreme color as part of the control sample")
147  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
148  doc="Select stars that are variable to be part "
149  "of the control sample")
150  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
151  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
152  doc="Convolve science image by its PSF before PSF-matching?")
153  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
154  doc="Scale variance of the template before PSF matching")
155  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
156  doc="Use a simple gaussian PSF model for pre-convolution "
157  "(else use fit PSF)? Ignored if doPreConvolve false.")
158  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
159  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
160  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
161  "kernel convolution? If True, also update the diffim PSF.")
162  doMerge = pexConfig.Field(dtype=bool, default=True,
163  doc="Merge positive and negative diaSources with grow radius "
164  "set by growFootprint")
165  doMatchSources = pexConfig.Field(dtype=bool, default=True,
166  doc="Match diaSources with input calexp sources and ref catalog sources")
167  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
168  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
169  doForcedMeasurement = pexConfig.Field(
170  dtype=bool,
171  default=True,
172  doc="Force photometer diaSource locations on PVI?")
173  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
174  doWriteWarpedExp = pexConfig.Field(dtype=bool, default=False,
175  doc="Write WCS, warped template coadd exposure?")
176  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
177  doc="Write warped and PSF-matched template coadd exposure?")
178  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
179  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
180  doc="Add columns to the source table to hold analysis metrics?")
181 
182  coaddName = pexConfig.Field(
183  doc="coadd name: typically one of deep, goodSeeing, or dcr",
184  dtype=str,
185  default="deep",
186  )
187  convolveTemplate = pexConfig.Field(
188  doc="Which image gets convolved (default = template)",
189  dtype=bool,
190  default=True
191  )
192  refObjLoader = pexConfig.ConfigurableField(
193  target=LoadIndexedReferenceObjectsTask,
194  doc="reference object loader",
195  )
196  astrometer = pexConfig.ConfigurableField(
197  target=AstrometryTask,
198  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
199  )
200  sourceSelector = pexConfig.ConfigurableField(
201  target=ObjectSizeStarSelectorTask,
202  doc="Source selection algorithm",
203  )
204  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
205  decorrelate = pexConfig.ConfigurableField(
206  target=DecorrelateALKernelSpatialTask,
207  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
208  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
209  "default of 5.5).",
210  )
211  doSpatiallyVarying = pexConfig.Field(
212  dtype=bool,
213  default=False,
214  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
215  "image in order to allow for spatial variations"
216  )
217  detection = pexConfig.ConfigurableField(
218  target=SourceDetectionTask,
219  doc="Low-threshold detection for final measurement",
220  )
221  measurement = pexConfig.ConfigurableField(
222  target=DipoleFitTask,
223  doc="Enable updated dipole fitting method",
224  )
225  forcedMeasurement = pexConfig.ConfigurableField(
226  target=ForcedMeasurementTask,
227  doc="Subtask to force photometer PVI at diaSource location.",
228  )
229  getTemplate = pexConfig.ConfigurableField(
230  target=GetCoaddAsTemplateTask,
231  doc="Subtask to retrieve template exposure and sources",
232  )
233  scaleVariance = pexConfig.ConfigurableField(
234  target=ScaleVarianceTask,
235  doc="Subtask to rescale the variance of the template "
236  "to the statistically expected level"
237  )
238  controlStepSize = pexConfig.Field(
239  doc="What step size (every Nth one) to select a control sample from the kernelSources",
240  dtype=int,
241  default=5
242  )
243  controlRandomSeed = pexConfig.Field(
244  doc="Random seed for shuffing the control sample",
245  dtype=int,
246  default=10
247  )
248  register = pexConfig.ConfigurableField(
249  target=RegisterTask,
250  doc="Task to enable image-to-image image registration (warping)",
251  )
252  kernelSourcesFromRef = pexConfig.Field(
253  doc="Select sources to measure kernel from reference catalog if True, template if false",
254  dtype=bool,
255  default=False
256  )
257  templateSipOrder = pexConfig.Field(
258  dtype=int, default=2,
259  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
260  )
261  growFootprint = pexConfig.Field(
262  dtype=int, default=2,
263  doc="Grow positive and negative footprints by this amount before merging"
264  )
265  diaSourceMatchRadius = pexConfig.Field(
266  dtype=float, default=0.5,
267  doc="Match radius (in arcseconds) for DiaSource to Source association"
268  )
269 
270  def setDefaults(self):
271  # defaults are OK for catalog and diacatalog
272 
273  self.subtract['al'].kernel.name = "AL"
274  self.subtract['al'].kernel.active.fitForBackground = True
275  self.subtract['al'].kernel.active.spatialKernelOrder = 1
276  self.subtract['al'].kernel.active.spatialBgOrder = 2
277  self.doPreConvolve = False
278  self.doMatchSources = False
279  self.doAddMetrics = False
280  self.doUseRegister = False
281 
282  # DiaSource Detection
283  self.detection.thresholdPolarity = "both"
284  self.detection.thresholdValue = 5.5
285  self.detection.reEstimateBackground = False
286  self.detection.thresholdType = "pixel_stdev"
287 
288  # Add filtered flux measurement, the correct measurement for pre-convolved images.
289  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
290  # To change that you must modify algorithms.names in the task's applyOverrides method,
291  # after the user has set doPreConvolve.
292  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
293  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
294  'base_LocalWcs']
295 
296  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
297  self.forcedMeasurement.copyColumns = {
298  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
299  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
300  self.forcedMeasurement.slots.shape = None
301 
302  # For shuffling the control sample
303  random.seed(self.controlRandomSeed)
304 
305  def validate(self):
306  pexConfig.Config.validate(self)
307  if self.doAddMetrics and not self.doSubtract:
308  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
309  if not self.doSubtract and not self.doDetection:
310  raise ValueError("Either doSubtract or doDetection must be enabled.")
311  if self.subtract.name == 'zogy' and self.doAddMetrics:
312  raise ValueError("Kernel metrics does not exist in zogy subtraction.")
313  if self.doMeasurement and not self.doDetection:
314  raise ValueError("Cannot run source measurement without source detection.")
315  if self.doMerge and not self.doDetection:
316  raise ValueError("Cannot run source merging without source detection.")
317  if self.doUseRegister and not self.doSelectSources:
318  raise ValueError("doUseRegister=True and doSelectSources=False. "
319  "Cannot run RegisterTask without selecting sources.")
320  if self.doPreConvolve and self.doDecorrelation and not self.convolveTemplate:
321  raise ValueError("doPreConvolve=True and doDecorrelation=True and "
322  "convolveTemplate=False is not supported.")
323  if hasattr(self.getTemplate, "coaddName"):
324  if self.getTemplate.coaddName != self.coaddName:
325  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
326 
327 
328 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
329 
330  @staticmethod
331  def getTargetList(parsedCmd, **kwargs):
332  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
333  **kwargs)
334 
335 
336 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
337  """Subtract an image from a template and measure the result
338  """
339  ConfigClass = ImageDifferenceConfig
340  RunnerClass = ImageDifferenceTaskRunner
341  _DefaultName = "imageDifference"
342 
343  def __init__(self, butler=None, **kwargs):
344  """!Construct an ImageDifference Task
345 
346  @param[in] butler Butler object to use in constructing reference object loaders
347  """
348  super().__init__(**kwargs)
349  self.makeSubtask("getTemplate")
350 
351  self.makeSubtask("subtract")
352 
353  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
354  self.makeSubtask("decorrelate")
355 
356  if self.config.doScaleTemplateVariance:
357  self.makeSubtask("scaleVariance")
358 
359  if self.config.doUseRegister:
360  self.makeSubtask("register")
361  self.schema = afwTable.SourceTable.makeMinimalSchema()
362 
363  if self.config.doSelectSources:
364  self.makeSubtask("sourceSelector")
365  if self.config.kernelSourcesFromRef:
366  self.makeSubtask('refObjLoader', butler=butler)
367  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
368 
369  self.algMetadata = dafBase.PropertyList()
370  if self.config.doDetection:
371  self.makeSubtask("detection", schema=self.schema)
372  if self.config.doMeasurement:
373  self.makeSubtask("measurement", schema=self.schema,
374  algMetadata=self.algMetadata)
375  if self.config.doForcedMeasurement:
376  self.schema.addField(
377  "ip_diffim_forced_PsfFlux_instFlux", "D",
378  "Forced PSF flux measured on the direct image.")
379  self.schema.addField(
380  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
381  "Forced PSF flux error measured on the direct image.")
382  self.schema.addField(
383  "ip_diffim_forced_PsfFlux_area", "F",
384  "Forced PSF flux effective area of PSF.",
385  units="pixel")
386  self.schema.addField(
387  "ip_diffim_forced_PsfFlux_flag", "Flag",
388  "Forced PSF flux general failure flag.")
389  self.schema.addField(
390  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
391  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
392  self.schema.addField(
393  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
394  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
395  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
396  if self.config.doMatchSources:
397  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
398  self.schema.addField("srcMatchId", "L", "unique id of source match")
399 
400  # initialize InitOutputs
401  self.outputSchema = afwTable.SourceCatalog(self.schema)
402  self.outputSchema.getTable().setMetadata(self.algMetadata)
403 
404  @staticmethod
405  def makeIdFactory(expId, expBits):
406  """Create IdFactory instance for unique 64 bit diaSource id-s.
407 
408  Parameters
409  ----------
410  expId : `int`
411  Exposure id.
412 
413  expBits: `int`
414  Number of used bits in ``expId``.
415 
416  Note
417  ----
418  The diasource id-s consists of the ``expId`` stored fixed in the highest value
419  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
420  low value end of the integer.
421 
422  Returns
423  -------
424  idFactory: `lsst.afw.table.IdFactory`
425  """
426  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
427 
428  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
429  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
430  inputRefs: pipeBase.InputQuantizedConnection,
431  outputRefs: pipeBase.OutputQuantizedConnection):
432  inputs = butlerQC.get(inputRefs)
433  self.log.info(f"Processing {butlerQC.quantum.dataId}")
434  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
435  returnMaxBits=True)
436  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
437  if self.config.coaddName == 'dcr':
438  templateExposures = inputRefs.dcrCoadds
439  else:
440  templateExposures = inputRefs.coaddExposures
441  templateStruct = self.getTemplate.runQuantum(
442  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
443  )
444 
445  outputs = self.run(exposure=inputs['exposure'],
446  templateExposure=templateStruct.exposure,
447  idFactory=idFactory)
448  butlerQC.put(outputs, outputRefs)
449 
450  @pipeBase.timeMethod
451  def runDataRef(self, sensorRef, templateIdList=None):
452  """Subtract an image from a template coadd and measure the result.
453 
454  Data I/O wrapper around `run` using the butler in Gen2.
455 
456  Parameters
457  ----------
458  sensorRef : `lsst.daf.persistence.ButlerDataRef`
459  Sensor-level butler data reference, used for the following data products:
460 
461  Input only:
462  - calexp
463  - psf
464  - ccdExposureId
465  - ccdExposureId_bits
466  - self.config.coaddName + "Coadd_skyMap"
467  - self.config.coaddName + "Coadd"
468  Input or output, depending on config:
469  - self.config.coaddName + "Diff_subtractedExp"
470  Output, depending on config:
471  - self.config.coaddName + "Diff_matchedExp"
472  - self.config.coaddName + "Diff_src"
473 
474  Returns
475  -------
476  results : `lsst.pipe.base.Struct`
477  Returns the Struct by `run`.
478  """
479  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
480  subtractedExposure = None
481  selectSources = None
482  calexpBackgroundExposure = None
483  self.log.info("Processing %s" % (sensorRef.dataId))
484 
485  # We make one IdFactory that will be used by both icSrc and src datasets;
486  # I don't know if this is the way we ultimately want to do things, but at least
487  # this ensures the source IDs are fully unique.
488  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
489  expBits=sensorRef.get("ccdExposureId_bits"))
490  if self.config.doAddCalexpBackground:
491  calexpBackgroundExposure = sensorRef.get("calexpBackground")
492 
493  # Retrieve the science image we wish to analyze
494  exposure = sensorRef.get("calexp", immediate=True)
495 
496  # Retrieve the template image
497  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
498 
499  if sensorRef.datasetExists("src"):
500  self.log.info("Source selection via src product")
501  # Sources already exist; for data release processing
502  selectSources = sensorRef.get("src")
503 
504  if not self.config.doSubtract and self.config.doDetection:
505  # If we don't do subtraction, we need the subtracted exposure from the repo
506  subtractedExposure = sensorRef.get(subtractedExposureName)
507  # Both doSubtract and doDetection cannot be False
508 
509  results = self.run(exposure=exposure,
510  selectSources=selectSources,
511  templateExposure=template.exposure,
512  templateSources=template.sources,
513  idFactory=idFactory,
514  calexpBackgroundExposure=calexpBackgroundExposure,
515  subtractedExposure=subtractedExposure)
516 
517  if self.config.doWriteSources and results.diaSources is not None:
518  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
519  if self.config.doWriteWarpedExp:
520  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
521  if self.config.doWriteMatchedExp:
522  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
523  if self.config.doAddMetrics and self.config.doSelectSources:
524  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
525  if self.config.doWriteSubtractedExp:
526  sensorRef.put(results.subtractedExposure, subtractedExposureName)
527  return results
528 
529  @pipeBase.timeMethod
530  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
531  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
532  """PSF matches, subtract two images and perform detection on the difference image.
533 
534  Parameters
535  ----------
536  exposure : `lsst.afw.image.ExposureF`, optional
537  The science exposure, the minuend in the image subtraction.
538  Can be None only if ``config.doSubtract==False``.
539  selectSources : `lsst.afw.table.SourceCatalog`, optional
540  Identified sources on the science exposure. This catalog is used to
541  select sources in order to perform the AL PSF matching on stamp images
542  around them. The selection steps depend on config options and whether
543  ``templateSources`` and ``matchingSources`` specified.
544  templateExposure : `lsst.afw.image.ExposureF`, optional
545  The template to be subtracted from ``exposure`` in the image subtraction.
546  The template exposure should cover the same sky area as the science exposure.
547  It is either a stich of patches of a coadd skymap image or a calexp
548  of the same pointing as the science exposure. Can be None only
549  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
550  templateSources : `lsst.afw.table.SourceCatalog`, optional
551  Identified sources on the template exposure.
552  idFactory : `lsst.afw.table.IdFactory`
553  Generator object to assign ids to detected sources in the difference image.
554  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
555  Background exposure to be added back to the science exposure
556  if ``config.doAddCalexpBackground==True``
557  subtractedExposure : `lsst.afw.image.ExposureF`, optional
558  If ``config.doSubtract==False`` and ``config.doDetection==True``,
559  performs the post subtraction source detection only on this exposure.
560  Otherwise should be None.
561 
562  Returns
563  -------
564  results : `lsst.pipe.base.Struct`
565  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
566  Difference image.
567  ``matchedExposure`` : `lsst.afw.image.ExposureF`
568  The matched PSF exposure.
569  ``subtractRes`` : `lsst.pipe.base.Struct`
570  The returned result structure of the ImagePsfMatchTask subtask.
571  ``diaSources`` : `lsst.afw.table.SourceCatalog`
572  The catalog of detected sources.
573  ``selectSources`` : `lsst.afw.table.SourceCatalog`
574  The input source catalog with optionally added Qa information.
575 
576  Notes
577  -----
578  The following major steps are included:
579 
580  - warp template coadd to match WCS of image
581  - PSF match image to warped template
582  - subtract image from PSF-matched, warped template
583  - detect sources
584  - measure sources
585 
586  For details about the image subtraction configuration modes
587  see `lsst.ip.diffim`.
588  """
589  subtractRes = None
590  controlSources = None
591  diaSources = None
592  kernelSources = None
593 
594  if self.config.doAddCalexpBackground:
595  mi = exposure.getMaskedImage()
596  mi += calexpBackgroundExposure.getImage()
597 
598  if not exposure.hasPsf():
599  raise pipeBase.TaskError("Exposure has no psf")
600  sciencePsf = exposure.getPsf()
601 
602  if self.config.doSubtract:
603  if self.config.doScaleTemplateVariance:
604  templateVarFactor = self.scaleVariance.run(
605  templateExposure.getMaskedImage())
606  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
607 
608  if self.config.subtract.name == 'zogy':
609  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
610  if self.config.doPreConvolve:
611  subtractedExposure = subtractRes.scoreExp
612  else:
613  subtractedExposure = subtractRes.diffExp
614  subtractRes.subtractedExposure = subtractedExposure
615  subtractRes.matchedExposure = None
616 
617  elif self.config.subtract.name == 'al':
618  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
619  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
620  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
621 
622  # if requested, convolve the science exposure with its PSF
623  # (properly, this should be a cross-correlation, but our code does not yet support that)
624  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
625  # else sigma of original science exposure
626  # TODO: DM-22762 This functional block should be moved into its own method
627  preConvPsf = None
628  if self.config.doPreConvolve:
629  convControl = afwMath.ConvolutionControl()
630  # cannot convolve in place, so make a new MI to receive convolved image
631  srcMI = exposure.getMaskedImage()
632  exposureOrig = exposure.clone()
633  destMI = srcMI.Factory(srcMI.getDimensions())
634  srcPsf = sciencePsf
635  if self.config.useGaussianForPreConvolution:
636  # convolve with a simplified PSF model: a double Gaussian
637  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
638  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
639  else:
640  # convolve with science exposure's PSF model
641  preConvPsf = srcPsf
642  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
643  exposure.setMaskedImage(destMI)
644  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
645  else:
646  scienceSigmaPost = scienceSigmaOrig
647  exposureOrig = exposure
648 
649  # If requested, find and select sources from the image
650  # else, AL subtraction will do its own source detection
651  # TODO: DM-22762 This functional block should be moved into its own method
652  if self.config.doSelectSources:
653  if selectSources is None:
654  self.log.warn("Src product does not exist; running detection, measurement, selection")
655  # Run own detection and measurement; necessary in nightly processing
656  selectSources = self.subtract.getSelectSources(
657  exposure,
658  sigma=scienceSigmaPost,
659  doSmooth=not self.config.doPreConvolve,
660  idFactory=idFactory,
661  )
662 
663  if self.config.doAddMetrics:
664  # Number of basis functions
665 
666  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
667  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
668  targetFwhmPix=templateSigma*FwhmPerSigma))
669  # Modify the schema of all Sources
670  # DEPRECATED: This is a data dependent (nparam) output product schema
671  # outside the task constructor.
672  # NOTE: The pre-determination of nparam at this point
673  # may be incorrect as the template psf is warped later in
674  # ImagePsfMatchTask.matchExposures()
675  kcQa = KernelCandidateQa(nparam)
676  selectSources = kcQa.addToSchema(selectSources)
677  if self.config.kernelSourcesFromRef:
678  # match exposure sources to reference catalog
679  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
680  matches = astromRet.matches
681  elif templateSources:
682  # match exposure sources to template sources
683  mc = afwTable.MatchControl()
684  mc.findOnlyClosest = False
685  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
686  mc)
687  else:
688  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
689  "but template sources not available. Cannot match science "
690  "sources with template sources. Run process* on data from "
691  "which templates are built.")
692 
693  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
694  matches=matches).sourceCat
695  random.shuffle(kernelSources, random.random)
696  controlSources = kernelSources[::self.config.controlStepSize]
697  kernelSources = [k for i, k in enumerate(kernelSources)
698  if i % self.config.controlStepSize]
699 
700  if self.config.doSelectDcrCatalog:
701  redSelector = DiaCatalogSourceSelectorTask(
702  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
703  grMax=99.999))
704  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
705  controlSources.extend(redSources)
706 
707  blueSelector = DiaCatalogSourceSelectorTask(
708  DiaCatalogSourceSelectorConfig(grMin=-99.999,
709  grMax=self.sourceSelector.config.grMin))
710  blueSources = blueSelector.selectStars(exposure, selectSources,
711  matches=matches).starCat
712  controlSources.extend(blueSources)
713 
714  if self.config.doSelectVariableCatalog:
715  varSelector = DiaCatalogSourceSelectorTask(
716  DiaCatalogSourceSelectorConfig(includeVariable=True))
717  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
718  controlSources.extend(varSources)
719 
720  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
721  % (len(kernelSources), len(selectSources), len(controlSources)))
722 
723  allresids = {}
724  # TODO: DM-22762 This functional block should be moved into its own method
725  if self.config.doUseRegister:
726  self.log.info("Registering images")
727 
728  if templateSources is None:
729  # Run detection on the template, which is
730  # temporarily background-subtracted
731  # sigma of PSF of template image before warping
732  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
733  templateSources = self.subtract.getSelectSources(
734  templateExposure,
735  sigma=templateSigma,
736  doSmooth=True,
737  idFactory=idFactory
738  )
739 
740  # Third step: we need to fit the relative astrometry.
741  #
742  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
743  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
744  exposure.getWcs(), exposure.getBBox())
745  templateExposure = warpedExp
746 
747  # Create debugging outputs on the astrometric
748  # residuals as a function of position. Persistence
749  # not yet implemented; expected on (I believe) #2636.
750  if self.config.doDebugRegister:
751  # Grab matches to reference catalog
752  srcToMatch = {x.second.getId(): x.first for x in matches}
753 
754  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
755  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
756  sids = [m.first.getId() for m in wcsResults.matches]
757  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
758  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
759  m.second.get(inCentroidKey))) for m in wcsResults.matches]
760  allresids = dict(zip(sids, zip(positions, residuals)))
761 
762  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
763  wcsResults.wcs.pixelToSky(
764  m.second.get(inCentroidKey))) for m in wcsResults.matches]
765  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
766  + 2.5*numpy.log10(srcToMatch[x].get("r"))
767  for x in sids if x in srcToMatch.keys()])
768  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
769  if s in srcToMatch.keys()])
770  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
771  if s in srcToMatch.keys()])
772  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
773  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
774  & (colors <= self.sourceSelector.config.grMax))
775  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
776  rms1Long = IqrToSigma*(
777  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
778  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
779  - numpy.percentile(dlat[idx1], 25))
780  rms2Long = IqrToSigma*(
781  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
782  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
783  - numpy.percentile(dlat[idx2], 25))
784  rms3Long = IqrToSigma*(
785  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
786  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
787  - numpy.percentile(dlat[idx3], 25))
788  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
789  (numpy.median(dlong[idx1]), rms1Long,
790  numpy.median(dlat[idx1]), rms1Lat))
791  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
792  (numpy.median(dlong[idx2]), rms2Long,
793  numpy.median(dlat[idx2]), rms2Lat))
794  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
795  (numpy.median(dlong[idx3]), rms3Long,
796  numpy.median(dlat[idx3]), rms3Lat))
797 
798  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
799  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
800  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
801  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
802  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
803  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
804 
805  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
806  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
807  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
808  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
809  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
810  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
811 
812  # warp template exposure to match exposure,
813  # PSF match template exposure to exposure,
814  # then return the difference
815 
816  # Return warped template... Construct sourceKernelCand list after subtract
817  self.log.info("Subtracting images")
818  subtractRes = self.subtract.subtractExposures(
819  templateExposure=templateExposure,
820  scienceExposure=exposure,
821  candidateList=kernelSources,
822  convolveTemplate=self.config.convolveTemplate,
823  doWarping=not self.config.doUseRegister
824  )
825  subtractedExposure = subtractRes.subtractedExposure
826 
827  if self.config.doDetection:
828  self.log.info("Computing diffim PSF")
829 
830  # Get Psf from the appropriate input image if it doesn't exist
831  if not subtractedExposure.hasPsf():
832  if self.config.convolveTemplate:
833  subtractedExposure.setPsf(exposure.getPsf())
834  else:
835  subtractedExposure.setPsf(templateExposure.getPsf())
836 
837  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
838  # thus it may have already been decorrelated. Thus, we do not decorrelate if
839  # doSubtract is False.
840 
841  # NOTE: At this point doSubtract == True
842  if self.config.doDecorrelation and self.config.doSubtract:
843  preConvKernel = None
844  if preConvPsf is not None:
845  preConvKernel = preConvPsf.getLocalKernel()
846  if self.config.convolveTemplate:
847  self.log.info("Decorrelation after template image convolution")
848  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
849  subtractedExposure,
850  subtractRes.psfMatchingKernel,
851  spatiallyVarying=self.config.doSpatiallyVarying,
852  preConvKernel=preConvKernel)
853  else:
854  self.log.info("Decorrelation after science image convolution")
855  decorrResult = self.decorrelate.run(subtractRes.warpedExposure, exposureOrig,
856  subtractedExposure,
857  subtractRes.psfMatchingKernel,
858  spatiallyVarying=self.config.doSpatiallyVarying,
859  preConvKernel=preConvKernel)
860  subtractedExposure = decorrResult.correctedExposure
861 
862  # END (if subtractAlgorithm == 'AL')
863  # END (if self.config.doSubtract)
864  if self.config.doDetection:
865  self.log.info("Running diaSource detection")
866  # Erase existing detection mask planes
867  mask = subtractedExposure.getMaskedImage().getMask()
868  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
869 
870  table = afwTable.SourceTable.make(self.schema, idFactory)
871  table.setMetadata(self.algMetadata)
872  results = self.detection.run(
873  table=table,
874  exposure=subtractedExposure,
875  doSmooth=not self.config.doPreConvolve
876  )
877 
878  if self.config.doMerge:
879  fpSet = results.fpSets.positive
880  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
881  self.config.growFootprint, False)
882  diaSources = afwTable.SourceCatalog(table)
883  fpSet.makeSources(diaSources)
884  self.log.info("Merging detections into %d sources" % (len(diaSources)))
885  else:
886  diaSources = results.sources
887 
888  if self.config.doMeasurement:
889  newDipoleFitting = self.config.doDipoleFitting
890  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
891  if not newDipoleFitting:
892  # Just fit dipole in diffim
893  self.measurement.run(diaSources, subtractedExposure)
894  else:
895  # Use (matched) template and science image (if avail.) to constrain dipole fitting
896  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
897  self.measurement.run(diaSources, subtractedExposure, exposure,
898  subtractRes.matchedExposure)
899  else:
900  self.measurement.run(diaSources, subtractedExposure, exposure)
901 
902  if self.config.doForcedMeasurement:
903  # Run forced psf photometry on the PVI at the diaSource locations.
904  # Copy the measured flux and error into the diaSource.
905  forcedSources = self.forcedMeasurement.generateMeasCat(
906  exposure, diaSources, subtractedExposure.getWcs())
907  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
908  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
909  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
910  "ip_diffim_forced_PsfFlux_instFlux", True)
911  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
912  "ip_diffim_forced_PsfFlux_instFluxErr", True)
913  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
914  "ip_diffim_forced_PsfFlux_area", True)
915  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
916  "ip_diffim_forced_PsfFlux_flag", True)
917  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
918  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
919  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
920  "ip_diffim_forced_PsfFlux_flag_edge", True)
921  for diaSource, forcedSource in zip(diaSources, forcedSources):
922  diaSource.assign(forcedSource, mapper)
923 
924  # Match with the calexp sources if possible
925  if self.config.doMatchSources:
926  if selectSources is not None:
927  # Create key,val pair where key=diaSourceId and val=sourceId
928  matchRadAsec = self.config.diaSourceMatchRadius
929  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
930 
931  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
932  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
933  srcMatch in srcMatches])
934  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
935  len(diaSources)))
936  else:
937  self.log.warn("Src product does not exist; cannot match with diaSources")
938  srcMatchDict = {}
939 
940  # Create key,val pair where key=diaSourceId and val=refId
941  refAstromConfig = AstrometryConfig()
942  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
943  refAstrometer = AstrometryTask(refAstromConfig)
944  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
945  refMatches = astromRet.matches
946  if refMatches is None:
947  self.log.warn("No diaSource matches with reference catalog")
948  refMatchDict = {}
949  else:
950  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
951  len(diaSources)))
952  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
953  refMatch in refMatches])
954 
955  # Assign source Ids
956  for diaSource in diaSources:
957  sid = diaSource.getId()
958  if sid in srcMatchDict:
959  diaSource.set("srcMatchId", srcMatchDict[sid])
960  if sid in refMatchDict:
961  diaSource.set("refMatchId", refMatchDict[sid])
962 
963  if self.config.doAddMetrics and self.config.doSelectSources:
964  self.log.info("Evaluating metrics and control sample")
965 
966  kernelCandList = []
967  for cell in subtractRes.kernelCellSet.getCellList():
968  for cand in cell.begin(False): # include bad candidates
969  kernelCandList.append(cand)
970 
971  # Get basis list to build control sample kernels
972  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
973  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
974 
975  controlCandList = (
976  diffimTools.sourceTableToCandidateList(controlSources,
977  subtractRes.warpedExposure, exposure,
978  self.config.subtract.kernel.active,
979  self.config.subtract.kernel.active.detectionConfig,
980  self.log, doBuild=True, basisList=basisList))
981 
982  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
983  subtractRes.backgroundModel, dof=nparam)
984  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
985  subtractRes.backgroundModel)
986 
987  if self.config.doDetection:
988  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
989  else:
990  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
991 
992  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
993  return pipeBase.Struct(
994  subtractedExposure=subtractedExposure,
995  warpedExposure=subtractRes.warpedExposure,
996  matchedExposure=subtractRes.matchedExposure,
997  subtractRes=subtractRes,
998  diaSources=diaSources,
999  selectSources=selectSources
1000  )
1001 
1002  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1003  """Fit the relative astrometry between templateSources and selectSources
1004 
1005  Todo
1006  ----
1007 
1008  Remove this method. It originally fit a new WCS to the template before calling register.run
1009  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1010  It remains because a subtask overrides it.
1011  """
1012  results = self.register.run(templateSources, templateExposure.getWcs(),
1013  templateExposure.getBBox(), selectSources)
1014  return results
1015 
1016  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1017  """Make debug plots and displays.
1018 
1019  Todo
1020  ----
1021  Test and update for current debug display and slot names
1022  """
1023  import lsstDebug
1024  display = lsstDebug.Info(__name__).display
1025  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1026  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1027  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1028  showDipoles = lsstDebug.Info(__name__).showDipoles
1029  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1030  if display:
1031  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1032  if not maskTransparency:
1033  maskTransparency = 0
1034  disp.setMaskTransparency(maskTransparency)
1035 
1036  if display and showSubtracted:
1037  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1038  mi = subtractRes.subtractedExposure.getMaskedImage()
1039  x0, y0 = mi.getX0(), mi.getY0()
1040  with disp.Buffering():
1041  for s in diaSources:
1042  x, y = s.getX() - x0, s.getY() - y0
1043  ctype = "red" if s.get("flags_negative") else "yellow"
1044  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1045  or s.get("base_PixelFlags_flag_saturatedCenter")
1046  or s.get("base_PixelFlags_flag_crCenter")):
1047  ptype = "x"
1048  elif (s.get("base_PixelFlags_flag_interpolated")
1049  or s.get("base_PixelFlags_flag_saturated")
1050  or s.get("base_PixelFlags_flag_cr")):
1051  ptype = "+"
1052  else:
1053  ptype = "o"
1054  disp.dot(ptype, x, y, size=4, ctype=ctype)
1055  lsstDebug.frame += 1
1056 
1057  if display and showPixelResiduals and selectSources:
1058  nonKernelSources = []
1059  for source in selectSources:
1060  if source not in kernelSources:
1061  nonKernelSources.append(source)
1062 
1063  diUtils.plotPixelResiduals(exposure,
1064  subtractRes.warpedExposure,
1065  subtractRes.subtractedExposure,
1066  subtractRes.kernelCellSet,
1067  subtractRes.psfMatchingKernel,
1068  subtractRes.backgroundModel,
1069  nonKernelSources,
1070  self.subtract.config.kernel.active.detectionConfig,
1071  origVariance=False)
1072  diUtils.plotPixelResiduals(exposure,
1073  subtractRes.warpedExposure,
1074  subtractRes.subtractedExposure,
1075  subtractRes.kernelCellSet,
1076  subtractRes.psfMatchingKernel,
1077  subtractRes.backgroundModel,
1078  nonKernelSources,
1079  self.subtract.config.kernel.active.detectionConfig,
1080  origVariance=True)
1081  if display and showDiaSources:
1082  flagChecker = SourceFlagChecker(diaSources)
1083  isFlagged = [flagChecker(x) for x in diaSources]
1084  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1085  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1086  frame=lsstDebug.frame)
1087  lsstDebug.frame += 1
1088 
1089  if display and showDipoles:
1090  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1091  frame=lsstDebug.frame)
1092  lsstDebug.frame += 1
1093 
1094  def _getConfigName(self):
1095  """Return the name of the config dataset
1096  """
1097  return "%sDiff_config" % (self.config.coaddName,)
1098 
1099  def _getMetadataName(self):
1100  """Return the name of the metadata dataset
1101  """
1102  return "%sDiff_metadata" % (self.config.coaddName,)
1103 
1104  def getSchemaCatalogs(self):
1105  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1106  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1107 
1108  @classmethod
1109  def _makeArgumentParser(cls):
1110  """Create an argument parser
1111  """
1112  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1113  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1114  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1115  help="Template data ID in case of calexp template,"
1116  " e.g. --templateId visit=6789")
1117  return parser
1118 
1119 
1120 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1121  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1122  doc="Shift stars going into RegisterTask by this amount")
1123  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1124  doc="Perturb stars going into RegisterTask by this amount")
1125 
1126  def setDefaults(self):
1127  ImageDifferenceConfig.setDefaults(self)
1128  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1129 
1130 
1131 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1132  """!Image difference Task used in the Winter 2013 data challege.
1133  Enables testing the effects of registration shifts and scatter.
1134 
1135  For use with winter 2013 simulated images:
1136  Use --templateId visit=88868666 for sparse data
1137  --templateId visit=22222200 for dense data (g)
1138  --templateId visit=11111100 for dense data (i)
1139  """
1140  ConfigClass = Winter2013ImageDifferenceConfig
1141  _DefaultName = "winter2013ImageDifference"
1142 
1143  def __init__(self, **kwargs):
1144  ImageDifferenceTask.__init__(self, **kwargs)
1145 
1146  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1147  """Fit the relative astrometry between templateSources and selectSources"""
1148  if self.config.winter2013WcsShift > 0.0:
1149  offset = geom.Extent2D(self.config.winter2013WcsShift,
1150  self.config.winter2013WcsShift)
1151  cKey = templateSources[0].getTable().getCentroidKey()
1152  for source in templateSources:
1153  centroid = source.get(cKey)
1154  source.set(cKey, centroid + offset)
1155  elif self.config.winter2013WcsRms > 0.0:
1156  cKey = templateSources[0].getTable().getCentroidKey()
1157  for source in templateSources:
1158  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1159  self.config.winter2013WcsRms*numpy.random.normal())
1160  centroid = source.get(cKey)
1161  source.set(cKey, centroid + offset)
1162 
1163  results = self.register.run(templateSources, templateExposure.getWcs(),
1164  templateExposure.getBBox(), selectSources)
1165  return results
lsst.pipe.tasks.registerImage
Definition: registerImage.py:1
lsst::ip::diffim
lsst::meas::astrom
lsst::afw::display
lsst::ip::diffim::utils
lsst::utils::inheritDoc
lsst.pipe.tasks.scaleVariance
Definition: scaleVariance.py:1
lsst::ip::diffim::diffimTools
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:720
lsstDebug::Info
lsst::meas::base
lsst::pex::config
lsst::afw::table
lsst::utils
lsst::geom
lsst.pipe.tasks.imageDifference.ImageDifferenceTaskConnections
Definition: imageDifference.py:54
lsst::daf::base
lsst::meas::algorithms::SingleGaussianPsf
lsst::afw::math
lsst.pipe::base
lsst::meas::algorithms
Extent< double, 2 >