lsst.pipe.tasks  20.0.0-10-g1b4d8e16+6
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 
47 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
49 IqrToSigma = 0.741
50 
51 
52 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
53  dimensions=("instrument", "visit", "detector", "skymap"),
54  defaultTemplates={"coaddName": "deep",
55  "skyMapName": "deep",
56  "warpTypeSuffix": "",
57  "fakesType": ""}):
58 
59  exposure = pipeBase.connectionTypes.Input(
60  doc="Input science exposure to subtract from.",
61  dimensions=("instrument", "visit", "detector"),
62  storageClass="ExposureF",
63  name="calexp"
64  )
65 
66  # TODO DM-22953
67  # kernelSources = pipeBase.connectionTypes.Input(
68  # doc="Source catalog produced in calibrate task for kernel candidate sources",
69  # name="src",
70  # storageClass="SourceCatalog",
71  # dimensions=("instrument", "visit", "detector"),
72  # )
73 
74  skyMap = pipeBase.connectionTypes.Input(
75  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
76  name="{skyMapName}Coadd_skyMap",
77  dimensions=("skymap", ),
78  storageClass="SkyMap",
79  )
80  coaddExposures = pipeBase.connectionTypes.Input(
81  doc="Input template to match and subtract from the exposure",
82  dimensions=("tract", "patch", "skymap", "abstract_filter"),
83  storageClass="ExposureF",
84  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
85  multiple=True,
86  deferLoad=True
87  )
88  dcrCoadds = pipeBase.connectionTypes.Input(
89  doc="Input DCR template to match and subtract from the exposure",
90  name="{fakesType}dcrCoadd{warpTypeSuffix}",
91  storageClass="ExposureF",
92  dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
93  multiple=True,
94  deferLoad=True
95  )
96  outputSchema = pipeBase.connectionTypes.InitOutput(
97  doc="Schema (as an example catalog) for output DIASource catalog.",
98  storageClass="SourceCatalog",
99  name="{fakesType}{coaddName}Diff_diaSrc_schema",
100  )
101  subtractedExposure = pipeBase.connectionTypes.Output(
102  doc="Output difference image",
103  dimensions=("instrument", "visit", "detector"),
104  storageClass="ExposureF",
105  name="{fakesType}{coaddName}Diff_differenceExp",
106  )
107  warpedExposure = pipeBase.connectionTypes.Output(
108  doc="Warped template used to create `subtractedExposure`.",
109  dimensions=("instrument", "visit", "detector"),
110  storageClass="ExposureF",
111  name="{fakesType}{coaddName}Diff_warpedExp",
112  )
113  diaSources = pipeBase.connectionTypes.Output(
114  doc="Output detected diaSources on the difference image",
115  dimensions=("instrument", "visit", "detector"),
116  storageClass="SourceCatalog",
117  name="{fakesType}{coaddName}Diff_diaSrc",
118  )
119 
120  def __init__(self, *, config=None):
121  super().__init__(config=config)
122  if config.coaddName == 'dcr':
123  self.inputs.remove("coaddExposures")
124  else:
125  self.inputs.remove("dcrCoadds")
126 
127  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
128  # Make kernelSources optional
129 
130 
131 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
132  pipelineConnections=ImageDifferenceTaskConnections):
133  """Config for ImageDifferenceTask
134  """
135  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
136  doc="Add background to calexp before processing it. "
137  "Useful as ipDiffim does background matching.")
138  doUseRegister = pexConfig.Field(dtype=bool, default=True,
139  doc="Use image-to-image registration to align template with "
140  "science image")
141  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
142  doc="Writing debugging data for doUseRegister")
143  doSelectSources = pexConfig.Field(dtype=bool, default=True,
144  doc="Select stars to use for kernel fitting")
145  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
146  doc="Select stars of extreme color as part of the control sample")
147  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
148  doc="Select stars that are variable to be part "
149  "of the control sample")
150  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
151  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
152  doc="Convolve science image by its PSF before PSF-matching?")
153  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
154  doc="Scale variance of the template before PSF matching")
155  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
156  doc="Use a simple gaussian PSF model for pre-convolution "
157  "(else use fit PSF)? Ignored if doPreConvolve false.")
158  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
159  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
160  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
161  "kernel convolution? If True, also update the diffim PSF.")
162  doMerge = pexConfig.Field(dtype=bool, default=True,
163  doc="Merge positive and negative diaSources with grow radius "
164  "set by growFootprint")
165  doMatchSources = pexConfig.Field(dtype=bool, default=True,
166  doc="Match diaSources with input calexp sources and ref catalog sources")
167  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
168  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
169  doForcedMeasurement = pexConfig.Field(
170  dtype=bool,
171  default=True,
172  doc="Force photometer diaSource locations on PVI?")
173  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
174  doWriteWarpedExp = pexConfig.Field(dtype=bool, default=False,
175  doc="Write WCS, warped template coadd exposure?")
176  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
177  doc="Write warped and PSF-matched template coadd exposure?")
178  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
179  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
180  doc="Add columns to the source table to hold analysis metrics?")
181 
182  coaddName = pexConfig.Field(
183  doc="coadd name: typically one of deep, goodSeeing, or dcr",
184  dtype=str,
185  default="deep",
186  )
187  convolveTemplate = pexConfig.Field(
188  doc="Which image gets convolved (default = template)",
189  dtype=bool,
190  default=True
191  )
192  refObjLoader = pexConfig.ConfigurableField(
193  target=LoadIndexedReferenceObjectsTask,
194  doc="reference object loader",
195  )
196  astrometer = pexConfig.ConfigurableField(
197  target=AstrometryTask,
198  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
199  )
200  sourceSelector = pexConfig.ConfigurableField(
201  target=ObjectSizeStarSelectorTask,
202  doc="Source selection algorithm",
203  )
204  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
205  decorrelate = pexConfig.ConfigurableField(
206  target=DecorrelateALKernelSpatialTask,
207  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
208  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
209  "default of 5.5).",
210  )
211  doSpatiallyVarying = pexConfig.Field(
212  dtype=bool,
213  default=False,
214  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
215  "image in order to allow for spatial variations"
216  )
217  detection = pexConfig.ConfigurableField(
218  target=SourceDetectionTask,
219  doc="Low-threshold detection for final measurement",
220  )
221  measurement = pexConfig.ConfigurableField(
222  target=DipoleFitTask,
223  doc="Enable updated dipole fitting method",
224  )
225  forcedMeasurement = pexConfig.ConfigurableField(
226  target=ForcedMeasurementTask,
227  doc="Subtask to force photometer PVI at diaSource location.",
228  )
229  getTemplate = pexConfig.ConfigurableField(
230  target=GetCoaddAsTemplateTask,
231  doc="Subtask to retrieve template exposure and sources",
232  )
233  scaleVariance = pexConfig.ConfigurableField(
234  target=ScaleVarianceTask,
235  doc="Subtask to rescale the variance of the template "
236  "to the statistically expected level"
237  )
238  controlStepSize = pexConfig.Field(
239  doc="What step size (every Nth one) to select a control sample from the kernelSources",
240  dtype=int,
241  default=5
242  )
243  controlRandomSeed = pexConfig.Field(
244  doc="Random seed for shuffing the control sample",
245  dtype=int,
246  default=10
247  )
248  register = pexConfig.ConfigurableField(
249  target=RegisterTask,
250  doc="Task to enable image-to-image image registration (warping)",
251  )
252  kernelSourcesFromRef = pexConfig.Field(
253  doc="Select sources to measure kernel from reference catalog if True, template if false",
254  dtype=bool,
255  default=False
256  )
257  templateSipOrder = pexConfig.Field(
258  dtype=int, default=2,
259  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
260  )
261  growFootprint = pexConfig.Field(
262  dtype=int, default=2,
263  doc="Grow positive and negative footprints by this amount before merging"
264  )
265  diaSourceMatchRadius = pexConfig.Field(
266  dtype=float, default=0.5,
267  doc="Match radius (in arcseconds) for DiaSource to Source association"
268  )
269 
270  def setDefaults(self):
271  # defaults are OK for catalog and diacatalog
272 
273  self.subtract['al'].kernel.name = "AL"
274  self.subtract['al'].kernel.active.fitForBackground = True
275  self.subtract['al'].kernel.active.spatialKernelOrder = 1
276  self.subtract['al'].kernel.active.spatialBgOrder = 2
277  self.doPreConvolve = False
278  self.doMatchSources = False
279  self.doAddMetrics = False
280  self.doUseRegister = False
281 
282  # DiaSource Detection
283  self.detection.thresholdPolarity = "both"
284  self.detection.thresholdValue = 5.5
285  self.detection.reEstimateBackground = False
286  self.detection.thresholdType = "pixel_stdev"
287 
288  # Add filtered flux measurement, the correct measurement for pre-convolved images.
289  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
290  # To change that you must modify algorithms.names in the task's applyOverrides method,
291  # after the user has set doPreConvolve.
292  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
293  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
294  'base_LocalWcs']
295 
296  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
297  self.forcedMeasurement.copyColumns = {
298  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
299  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
300  self.forcedMeasurement.slots.shape = None
301 
302  # For shuffling the control sample
303  random.seed(self.controlRandomSeed)
304 
305  def validate(self):
306  pexConfig.Config.validate(self)
307  if self.doAddMetrics and not self.doSubtract:
308  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
309  if not self.doSubtract and not self.doDetection:
310  raise ValueError("Either doSubtract or doDetection must be enabled.")
311  if self.subtract.name == 'zogy' and self.doAddMetrics:
312  raise ValueError("Kernel metrics does not exist in zogy subtraction.")
313  if self.doMeasurement and not self.doDetection:
314  raise ValueError("Cannot run source measurement without source detection.")
315  if self.doMerge and not self.doDetection:
316  raise ValueError("Cannot run source merging without source detection.")
317  if self.doUseRegister and not self.doSelectSources:
318  raise ValueError("doUseRegister=True and doSelectSources=False. "
319  "Cannot run RegisterTask without selecting sources.")
320  if self.doPreConvolve and self.doDecorrelation and not self.convolveTemplate:
321  raise ValueError("doPreConvolve=True and doDecorrelation=True and "
322  "convolveTemplate=False is not supported.")
323  if hasattr(self.getTemplate, "coaddName"):
324  if self.getTemplate.coaddName != self.coaddName:
325  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
326 
327 
328 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
329 
330  @staticmethod
331  def getTargetList(parsedCmd, **kwargs):
332  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
333  **kwargs)
334 
335 
336 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
337  """Subtract an image from a template and measure the result
338  """
339  ConfigClass = ImageDifferenceConfig
340  RunnerClass = ImageDifferenceTaskRunner
341  _DefaultName = "imageDifference"
342 
343  def __init__(self, butler=None, **kwargs):
344  """!Construct an ImageDifference Task
345 
346  @param[in] butler Butler object to use in constructing reference object loaders
347  """
348  super().__init__(**kwargs)
349  self.makeSubtask("getTemplate")
350 
351  self.makeSubtask("subtract")
352 
353  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
354  self.makeSubtask("decorrelate")
355 
356  if self.config.doScaleTemplateVariance:
357  self.makeSubtask("scaleVariance")
358 
359  if self.config.doUseRegister:
360  self.makeSubtask("register")
361  self.schema = afwTable.SourceTable.makeMinimalSchema()
362 
363  if self.config.doSelectSources:
364  self.makeSubtask("sourceSelector")
365  if self.config.kernelSourcesFromRef:
366  self.makeSubtask('refObjLoader', butler=butler)
367  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
368 
369  self.algMetadata = dafBase.PropertyList()
370  if self.config.doDetection:
371  self.makeSubtask("detection", schema=self.schema)
372  if self.config.doMeasurement:
373  self.makeSubtask("measurement", schema=self.schema,
374  algMetadata=self.algMetadata)
375  if self.config.doForcedMeasurement:
376  self.schema.addField(
377  "ip_diffim_forced_PsfFlux_instFlux", "D",
378  "Forced PSF flux measured on the direct image.")
379  self.schema.addField(
380  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
381  "Forced PSF flux error measured on the direct image.")
382  self.schema.addField(
383  "ip_diffim_forced_PsfFlux_area", "F",
384  "Forced PSF flux effective area of PSF.",
385  units="pixel")
386  self.schema.addField(
387  "ip_diffim_forced_PsfFlux_flag", "Flag",
388  "Forced PSF flux general failure flag.")
389  self.schema.addField(
390  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
391  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
392  self.schema.addField(
393  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
394  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
395  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
396  if self.config.doMatchSources:
397  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
398  self.schema.addField("srcMatchId", "L", "unique id of source match")
399 
400  # initialize InitOutputs
401  self.outputSchema = afwTable.SourceCatalog(self.schema)
402  self.outputSchema.getTable().setMetadata(self.algMetadata)
403 
404  @staticmethod
405  def makeIdFactory(expId, expBits):
406  """Create IdFactory instance for unique 64 bit diaSource id-s.
407 
408  Parameters
409  ----------
410  expId : `int`
411  Exposure id.
412 
413  expBits: `int`
414  Number of used bits in ``expId``.
415 
416  Note
417  ----
418  The diasource id-s consists of the ``expId`` stored fixed in the highest value
419  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
420  low value end of the integer.
421 
422  Returns
423  -------
424  idFactory: `lsst.afw.table.IdFactory`
425  """
426  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
427 
428  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
429  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
430  inputRefs: pipeBase.InputQuantizedConnection,
431  outputRefs: pipeBase.OutputQuantizedConnection):
432  inputs = butlerQC.get(inputRefs)
433  self.log.info(f"Processing {butlerQC.quantum.dataId}")
434  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
435  returnMaxBits=True)
436  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
437  if self.config.coaddName == 'dcr':
438  templateExposures = inputRefs.dcrCoadds
439  else:
440  templateExposures = inputRefs.coaddExposures
441  templateStruct = self.getTemplate.runQuantum(
442  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
443  )
444 
445  outputs = self.run(exposure=inputs['exposure'],
446  templateExposure=templateStruct.exposure,
447  idFactory=idFactory)
448  butlerQC.put(outputs, outputRefs)
449 
450  @pipeBase.timeMethod
451  def runDataRef(self, sensorRef, templateIdList=None):
452  """Subtract an image from a template coadd and measure the result.
453 
454  Data I/O wrapper around `run` using the butler in Gen2.
455 
456  Parameters
457  ----------
458  sensorRef : `lsst.daf.persistence.ButlerDataRef`
459  Sensor-level butler data reference, used for the following data products:
460 
461  Input only:
462  - calexp
463  - psf
464  - ccdExposureId
465  - ccdExposureId_bits
466  - self.config.coaddName + "Coadd_skyMap"
467  - self.config.coaddName + "Coadd"
468  Input or output, depending on config:
469  - self.config.coaddName + "Diff_subtractedExp"
470  Output, depending on config:
471  - self.config.coaddName + "Diff_matchedExp"
472  - self.config.coaddName + "Diff_src"
473 
474  Returns
475  -------
476  results : `lsst.pipe.base.Struct`
477  Returns the Struct by `run`.
478  """
479  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
480  subtractedExposure = None
481  selectSources = None
482  calexpBackgroundExposure = None
483  self.log.info("Processing %s" % (sensorRef.dataId))
484 
485  # We make one IdFactory that will be used by both icSrc and src datasets;
486  # I don't know if this is the way we ultimately want to do things, but at least
487  # this ensures the source IDs are fully unique.
488  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
489  expBits=sensorRef.get("ccdExposureId_bits"))
490  if self.config.doAddCalexpBackground:
491  calexpBackgroundExposure = sensorRef.get("calexpBackground")
492 
493  # Retrieve the science image we wish to analyze
494  exposure = sensorRef.get("calexp", immediate=True)
495 
496  # Retrieve the template image
497  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
498 
499  if sensorRef.datasetExists("src"):
500  self.log.info("Source selection via src product")
501  # Sources already exist; for data release processing
502  selectSources = sensorRef.get("src")
503 
504  if not self.config.doSubtract and self.config.doDetection:
505  # If we don't do subtraction, we need the subtracted exposure from the repo
506  subtractedExposure = sensorRef.get(subtractedExposureName)
507  # Both doSubtract and doDetection cannot be False
508 
509  results = self.run(exposure=exposure,
510  selectSources=selectSources,
511  templateExposure=template.exposure,
512  templateSources=template.sources,
513  idFactory=idFactory,
514  calexpBackgroundExposure=calexpBackgroundExposure,
515  subtractedExposure=subtractedExposure)
516 
517  if self.config.doWriteSources and results.diaSources is not None:
518  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
519  if self.config.doWriteWarpedExp:
520  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
521  if self.config.doWriteMatchedExp:
522  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
523  if self.config.doAddMetrics and self.config.doSelectSources:
524  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
525  if self.config.doWriteSubtractedExp:
526  sensorRef.put(results.subtractedExposure, subtractedExposureName)
527  return results
528 
529  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
530  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
531  """PSF matches, subtract two images and perform detection on the difference image.
532 
533  Parameters
534  ----------
535  exposure : `lsst.afw.image.ExposureF`, optional
536  The science exposure, the minuend in the image subtraction.
537  Can be None only if ``config.doSubtract==False``.
538  selectSources : `lsst.afw.table.SourceCatalog`, optional
539  Identified sources on the science exposure. This catalog is used to
540  select sources in order to perform the AL PSF matching on stamp images
541  around them. The selection steps depend on config options and whether
542  ``templateSources`` and ``matchingSources`` specified.
543  templateExposure : `lsst.afw.image.ExposureF`, optional
544  The template to be subtracted from ``exposure`` in the image subtraction.
545  The template exposure should cover the same sky area as the science exposure.
546  It is either a stich of patches of a coadd skymap image or a calexp
547  of the same pointing as the science exposure. Can be None only
548  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
549  templateSources : `lsst.afw.table.SourceCatalog`, optional
550  Identified sources on the template exposure.
551  idFactory : `lsst.afw.table.IdFactory`
552  Generator object to assign ids to detected sources in the difference image.
553  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
554  Background exposure to be added back to the science exposure
555  if ``config.doAddCalexpBackground==True``
556  subtractedExposure : `lsst.afw.image.ExposureF`, optional
557  If ``config.doSubtract==False`` and ``config.doDetection==True``,
558  performs the post subtraction source detection only on this exposure.
559  Otherwise should be None.
560 
561  Returns
562  -------
563  results : `lsst.pipe.base.Struct`
564  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
565  Difference image.
566  ``matchedExposure`` : `lsst.afw.image.ExposureF`
567  The matched PSF exposure.
568  ``subtractRes`` : `lsst.pipe.base.Struct`
569  The returned result structure of the ImagePsfMatchTask subtask.
570  ``diaSources`` : `lsst.afw.table.SourceCatalog`
571  The catalog of detected sources.
572  ``selectSources`` : `lsst.afw.table.SourceCatalog`
573  The input source catalog with optionally added Qa information.
574 
575  Notes
576  -----
577  The following major steps are included:
578 
579  - warp template coadd to match WCS of image
580  - PSF match image to warped template
581  - subtract image from PSF-matched, warped template
582  - detect sources
583  - measure sources
584 
585  For details about the image subtraction configuration modes
586  see `lsst.ip.diffim`.
587  """
588  subtractRes = None
589  controlSources = None
590  diaSources = None
591  kernelSources = None
592 
593  if self.config.doAddCalexpBackground:
594  mi = exposure.getMaskedImage()
595  mi += calexpBackgroundExposure.getImage()
596 
597  if not exposure.hasPsf():
598  raise pipeBase.TaskError("Exposure has no psf")
599  sciencePsf = exposure.getPsf()
600 
601  if self.config.doSubtract:
602  if self.config.doScaleTemplateVariance:
603  templateVarFactor = self.scaleVariance.run(
604  templateExposure.getMaskedImage())
605  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
606 
607  if self.config.subtract.name == 'zogy':
608  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
609  doWarping=True,
610  spatiallyVarying=self.config.doSpatiallyVarying,
611  doPreConvolve=self.config.doPreConvolve)
612  subtractedExposure = subtractRes.subtractedExposure
613 
614  elif self.config.subtract.name == 'al':
615  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
616  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
617  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
618 
619  # if requested, convolve the science exposure with its PSF
620  # (properly, this should be a cross-correlation, but our code does not yet support that)
621  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
622  # else sigma of original science exposure
623  # TODO: DM-22762 This functional block should be moved into its own method
624  preConvPsf = None
625  if self.config.doPreConvolve:
626  convControl = afwMath.ConvolutionControl()
627  # cannot convolve in place, so make a new MI to receive convolved image
628  srcMI = exposure.getMaskedImage()
629  exposureOrig = exposure.clone()
630  destMI = srcMI.Factory(srcMI.getDimensions())
631  srcPsf = sciencePsf
632  if self.config.useGaussianForPreConvolution:
633  # convolve with a simplified PSF model: a double Gaussian
634  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
635  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
636  else:
637  # convolve with science exposure's PSF model
638  preConvPsf = srcPsf
639  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
640  exposure.setMaskedImage(destMI)
641  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
642  else:
643  scienceSigmaPost = scienceSigmaOrig
644  exposureOrig = exposure
645 
646  # If requested, find and select sources from the image
647  # else, AL subtraction will do its own source detection
648  # TODO: DM-22762 This functional block should be moved into its own method
649  if self.config.doSelectSources:
650  if selectSources is None:
651  self.log.warn("Src product does not exist; running detection, measurement, selection")
652  # Run own detection and measurement; necessary in nightly processing
653  selectSources = self.subtract.getSelectSources(
654  exposure,
655  sigma=scienceSigmaPost,
656  doSmooth=not self.config.doPreConvolve,
657  idFactory=idFactory,
658  )
659 
660  if self.config.doAddMetrics:
661  # Number of basis functions
662 
663  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
664  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
665  targetFwhmPix=templateSigma*FwhmPerSigma))
666  # Modify the schema of all Sources
667  # DEPRECATED: This is a data dependent (nparam) output product schema
668  # outside the task constructor.
669  # NOTE: The pre-determination of nparam at this point
670  # may be incorrect as the template psf is warped later in
671  # ImagePsfMatchTask.matchExposures()
672  kcQa = KernelCandidateQa(nparam)
673  selectSources = kcQa.addToSchema(selectSources)
674  if self.config.kernelSourcesFromRef:
675  # match exposure sources to reference catalog
676  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
677  matches = astromRet.matches
678  elif templateSources:
679  # match exposure sources to template sources
680  mc = afwTable.MatchControl()
681  mc.findOnlyClosest = False
682  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
683  mc)
684  else:
685  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
686  "but template sources not available. Cannot match science "
687  "sources with template sources. Run process* on data from "
688  "which templates are built.")
689 
690  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
691  matches=matches).sourceCat
692  random.shuffle(kernelSources, random.random)
693  controlSources = kernelSources[::self.config.controlStepSize]
694  kernelSources = [k for i, k in enumerate(kernelSources)
695  if i % self.config.controlStepSize]
696 
697  if self.config.doSelectDcrCatalog:
698  redSelector = DiaCatalogSourceSelectorTask(
699  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
700  grMax=99.999))
701  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
702  controlSources.extend(redSources)
703 
704  blueSelector = DiaCatalogSourceSelectorTask(
705  DiaCatalogSourceSelectorConfig(grMin=-99.999,
706  grMax=self.sourceSelector.config.grMin))
707  blueSources = blueSelector.selectStars(exposure, selectSources,
708  matches=matches).starCat
709  controlSources.extend(blueSources)
710 
711  if self.config.doSelectVariableCatalog:
712  varSelector = DiaCatalogSourceSelectorTask(
713  DiaCatalogSourceSelectorConfig(includeVariable=True))
714  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
715  controlSources.extend(varSources)
716 
717  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
718  % (len(kernelSources), len(selectSources), len(controlSources)))
719 
720  allresids = {}
721  # TODO: DM-22762 This functional block should be moved into its own method
722  if self.config.doUseRegister:
723  self.log.info("Registering images")
724 
725  if templateSources is None:
726  # Run detection on the template, which is
727  # temporarily background-subtracted
728  # sigma of PSF of template image before warping
729  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
730  templateSources = self.subtract.getSelectSources(
731  templateExposure,
732  sigma=templateSigma,
733  doSmooth=True,
734  idFactory=idFactory
735  )
736 
737  # Third step: we need to fit the relative astrometry.
738  #
739  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
740  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
741  exposure.getWcs(), exposure.getBBox())
742  templateExposure = warpedExp
743 
744  # Create debugging outputs on the astrometric
745  # residuals as a function of position. Persistence
746  # not yet implemented; expected on (I believe) #2636.
747  if self.config.doDebugRegister:
748  # Grab matches to reference catalog
749  srcToMatch = {x.second.getId(): x.first for x in matches}
750 
751  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
752  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
753  sids = [m.first.getId() for m in wcsResults.matches]
754  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
755  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
756  m.second.get(inCentroidKey))) for m in wcsResults.matches]
757  allresids = dict(zip(sids, zip(positions, residuals)))
758 
759  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
760  wcsResults.wcs.pixelToSky(
761  m.second.get(inCentroidKey))) for m in wcsResults.matches]
762  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
763  + 2.5*numpy.log10(srcToMatch[x].get("r"))
764  for x in sids if x in srcToMatch.keys()])
765  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
766  if s in srcToMatch.keys()])
767  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
768  if s in srcToMatch.keys()])
769  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
770  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
771  & (colors <= self.sourceSelector.config.grMax))
772  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
773  rms1Long = IqrToSigma*(
774  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
775  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
776  - numpy.percentile(dlat[idx1], 25))
777  rms2Long = IqrToSigma*(
778  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
779  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
780  - numpy.percentile(dlat[idx2], 25))
781  rms3Long = IqrToSigma*(
782  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
783  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
784  - numpy.percentile(dlat[idx3], 25))
785  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
786  (numpy.median(dlong[idx1]), rms1Long,
787  numpy.median(dlat[idx1]), rms1Lat))
788  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
789  (numpy.median(dlong[idx2]), rms2Long,
790  numpy.median(dlat[idx2]), rms2Lat))
791  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
792  (numpy.median(dlong[idx3]), rms3Long,
793  numpy.median(dlat[idx3]), rms3Lat))
794 
795  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
796  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
797  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
798  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
799  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
800  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
801 
802  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
803  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
804  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
805  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
806  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
807  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
808 
809  # warp template exposure to match exposure,
810  # PSF match template exposure to exposure,
811  # then return the difference
812 
813  # Return warped template... Construct sourceKernelCand list after subtract
814  self.log.info("Subtracting images")
815  subtractRes = self.subtract.subtractExposures(
816  templateExposure=templateExposure,
817  scienceExposure=exposure,
818  candidateList=kernelSources,
819  convolveTemplate=self.config.convolveTemplate,
820  doWarping=not self.config.doUseRegister
821  )
822  subtractedExposure = subtractRes.subtractedExposure
823 
824  if self.config.doDetection:
825  self.log.info("Computing diffim PSF")
826 
827  # Get Psf from the appropriate input image if it doesn't exist
828  if not subtractedExposure.hasPsf():
829  if self.config.convolveTemplate:
830  subtractedExposure.setPsf(exposure.getPsf())
831  else:
832  subtractedExposure.setPsf(templateExposure.getPsf())
833 
834  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
835  # thus it may have already been decorrelated. Thus, we do not decorrelate if
836  # doSubtract is False.
837 
838  # NOTE: At this point doSubtract == True
839  if self.config.doDecorrelation and self.config.doSubtract:
840  preConvKernel = None
841  if preConvPsf is not None:
842  preConvKernel = preConvPsf.getLocalKernel()
843  if self.config.convolveTemplate:
844  self.log.info("Decorrelation after template image convolution")
845  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
846  subtractedExposure,
847  subtractRes.psfMatchingKernel,
848  spatiallyVarying=self.config.doSpatiallyVarying,
849  preConvKernel=preConvKernel)
850  else:
851  self.log.info("Decorrelation after science image convolution")
852  decorrResult = self.decorrelate.run(subtractRes.warpedExposure, exposureOrig,
853  subtractedExposure,
854  subtractRes.psfMatchingKernel,
855  spatiallyVarying=self.config.doSpatiallyVarying,
856  preConvKernel=preConvKernel)
857  subtractedExposure = decorrResult.correctedExposure
858 
859  # END (if subtractAlgorithm == 'AL')
860  # END (if self.config.doSubtract)
861  if self.config.doDetection:
862  self.log.info("Running diaSource detection")
863  # Erase existing detection mask planes
864  mask = subtractedExposure.getMaskedImage().getMask()
865  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
866 
867  table = afwTable.SourceTable.make(self.schema, idFactory)
868  table.setMetadata(self.algMetadata)
869  results = self.detection.run(
870  table=table,
871  exposure=subtractedExposure,
872  doSmooth=not self.config.doPreConvolve
873  )
874 
875  if self.config.doMerge:
876  fpSet = results.fpSets.positive
877  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
878  self.config.growFootprint, False)
879  diaSources = afwTable.SourceCatalog(table)
880  fpSet.makeSources(diaSources)
881  self.log.info("Merging detections into %d sources" % (len(diaSources)))
882  else:
883  diaSources = results.sources
884 
885  if self.config.doMeasurement:
886  newDipoleFitting = self.config.doDipoleFitting
887  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
888  if not newDipoleFitting:
889  # Just fit dipole in diffim
890  self.measurement.run(diaSources, subtractedExposure)
891  else:
892  # Use (matched) template and science image (if avail.) to constrain dipole fitting
893  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
894  self.measurement.run(diaSources, subtractedExposure, exposure,
895  subtractRes.matchedExposure)
896  else:
897  self.measurement.run(diaSources, subtractedExposure, exposure)
898 
899  if self.config.doForcedMeasurement:
900  # Run forced psf photometry on the PVI at the diaSource locations.
901  # Copy the measured flux and error into the diaSource.
902  forcedSources = self.forcedMeasurement.generateMeasCat(
903  exposure, diaSources, subtractedExposure.getWcs())
904  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
905  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
906  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
907  "ip_diffim_forced_PsfFlux_instFlux", True)
908  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
909  "ip_diffim_forced_PsfFlux_instFluxErr", True)
910  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
911  "ip_diffim_forced_PsfFlux_area", True)
912  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
913  "ip_diffim_forced_PsfFlux_flag", True)
914  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
915  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
916  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
917  "ip_diffim_forced_PsfFlux_flag_edge", True)
918  for diaSource, forcedSource in zip(diaSources, forcedSources):
919  diaSource.assign(forcedSource, mapper)
920 
921  # Match with the calexp sources if possible
922  if self.config.doMatchSources:
923  if selectSources is not None:
924  # Create key,val pair where key=diaSourceId and val=sourceId
925  matchRadAsec = self.config.diaSourceMatchRadius
926  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
927 
928  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
929  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
930  srcMatch in srcMatches])
931  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
932  len(diaSources)))
933  else:
934  self.log.warn("Src product does not exist; cannot match with diaSources")
935  srcMatchDict = {}
936 
937  # Create key,val pair where key=diaSourceId and val=refId
938  refAstromConfig = AstrometryConfig()
939  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
940  refAstrometer = AstrometryTask(refAstromConfig)
941  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
942  refMatches = astromRet.matches
943  if refMatches is None:
944  self.log.warn("No diaSource matches with reference catalog")
945  refMatchDict = {}
946  else:
947  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
948  len(diaSources)))
949  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
950  refMatch in refMatches])
951 
952  # Assign source Ids
953  for diaSource in diaSources:
954  sid = diaSource.getId()
955  if sid in srcMatchDict:
956  diaSource.set("srcMatchId", srcMatchDict[sid])
957  if sid in refMatchDict:
958  diaSource.set("refMatchId", refMatchDict[sid])
959 
960  if self.config.doAddMetrics and self.config.doSelectSources:
961  self.log.info("Evaluating metrics and control sample")
962 
963  kernelCandList = []
964  for cell in subtractRes.kernelCellSet.getCellList():
965  for cand in cell.begin(False): # include bad candidates
966  kernelCandList.append(cand)
967 
968  # Get basis list to build control sample kernels
969  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
970  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
971 
972  controlCandList = (
973  diffimTools.sourceTableToCandidateList(controlSources,
974  subtractRes.warpedExposure, exposure,
975  self.config.subtract.kernel.active,
976  self.config.subtract.kernel.active.detectionConfig,
977  self.log, doBuild=True, basisList=basisList))
978 
979  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
980  subtractRes.backgroundModel, dof=nparam)
981  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
982  subtractRes.backgroundModel)
983 
984  if self.config.doDetection:
985  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
986  else:
987  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
988 
989  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
990  return pipeBase.Struct(
991  subtractedExposure=subtractedExposure,
992  warpedExposure=subtractRes.warpedExposure,
993  matchedExposure=subtractRes.matchedExposure,
994  subtractRes=subtractRes,
995  diaSources=diaSources,
996  selectSources=selectSources
997  )
998 
999  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1000  """Fit the relative astrometry between templateSources and selectSources
1001 
1002  Todo
1003  ----
1004 
1005  Remove this method. It originally fit a new WCS to the template before calling register.run
1006  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1007  It remains because a subtask overrides it.
1008  """
1009  results = self.register.run(templateSources, templateExposure.getWcs(),
1010  templateExposure.getBBox(), selectSources)
1011  return results
1012 
1013  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1014  """Make debug plots and displays.
1015 
1016  Todo
1017  ----
1018  Test and update for current debug display and slot names
1019  """
1020  import lsstDebug
1021  display = lsstDebug.Info(__name__).display
1022  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1023  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1024  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1025  showDipoles = lsstDebug.Info(__name__).showDipoles
1026  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1027  if display:
1028  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1029  if not maskTransparency:
1030  maskTransparency = 0
1031  disp.setMaskTransparency(maskTransparency)
1032 
1033  if display and showSubtracted:
1034  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1035  mi = subtractRes.subtractedExposure.getMaskedImage()
1036  x0, y0 = mi.getX0(), mi.getY0()
1037  with disp.Buffering():
1038  for s in diaSources:
1039  x, y = s.getX() - x0, s.getY() - y0
1040  ctype = "red" if s.get("flags_negative") else "yellow"
1041  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1042  or s.get("base_PixelFlags_flag_saturatedCenter")
1043  or s.get("base_PixelFlags_flag_crCenter")):
1044  ptype = "x"
1045  elif (s.get("base_PixelFlags_flag_interpolated")
1046  or s.get("base_PixelFlags_flag_saturated")
1047  or s.get("base_PixelFlags_flag_cr")):
1048  ptype = "+"
1049  else:
1050  ptype = "o"
1051  disp.dot(ptype, x, y, size=4, ctype=ctype)
1052  lsstDebug.frame += 1
1053 
1054  if display and showPixelResiduals and selectSources:
1055  nonKernelSources = []
1056  for source in selectSources:
1057  if source not in kernelSources:
1058  nonKernelSources.append(source)
1059 
1060  diUtils.plotPixelResiduals(exposure,
1061  subtractRes.warpedExposure,
1062  subtractRes.subtractedExposure,
1063  subtractRes.kernelCellSet,
1064  subtractRes.psfMatchingKernel,
1065  subtractRes.backgroundModel,
1066  nonKernelSources,
1067  self.subtract.config.kernel.active.detectionConfig,
1068  origVariance=False)
1069  diUtils.plotPixelResiduals(exposure,
1070  subtractRes.warpedExposure,
1071  subtractRes.subtractedExposure,
1072  subtractRes.kernelCellSet,
1073  subtractRes.psfMatchingKernel,
1074  subtractRes.backgroundModel,
1075  nonKernelSources,
1076  self.subtract.config.kernel.active.detectionConfig,
1077  origVariance=True)
1078  if display and showDiaSources:
1079  flagChecker = SourceFlagChecker(diaSources)
1080  isFlagged = [flagChecker(x) for x in diaSources]
1081  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1082  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1083  frame=lsstDebug.frame)
1084  lsstDebug.frame += 1
1085 
1086  if display and showDipoles:
1087  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1088  frame=lsstDebug.frame)
1089  lsstDebug.frame += 1
1090 
1091  def _getConfigName(self):
1092  """Return the name of the config dataset
1093  """
1094  return "%sDiff_config" % (self.config.coaddName,)
1095 
1096  def _getMetadataName(self):
1097  """Return the name of the metadata dataset
1098  """
1099  return "%sDiff_metadata" % (self.config.coaddName,)
1100 
1101  def getSchemaCatalogs(self):
1102  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1103  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1104 
1105  @classmethod
1106  def _makeArgumentParser(cls):
1107  """Create an argument parser
1108  """
1109  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1110  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1111  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1112  help="Template data ID in case of calexp template,"
1113  " e.g. --templateId visit=6789")
1114  return parser
1115 
1116 
1117 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1118  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1119  doc="Shift stars going into RegisterTask by this amount")
1120  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1121  doc="Perturb stars going into RegisterTask by this amount")
1122 
1123  def setDefaults(self):
1124  ImageDifferenceConfig.setDefaults(self)
1125  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1126 
1127 
1128 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1129  """!Image difference Task used in the Winter 2013 data challege.
1130  Enables testing the effects of registration shifts and scatter.
1131 
1132  For use with winter 2013 simulated images:
1133  Use --templateId visit=88868666 for sparse data
1134  --templateId visit=22222200 for dense data (g)
1135  --templateId visit=11111100 for dense data (i)
1136  """
1137  ConfigClass = Winter2013ImageDifferenceConfig
1138  _DefaultName = "winter2013ImageDifference"
1139 
1140  def __init__(self, **kwargs):
1141  ImageDifferenceTask.__init__(self, **kwargs)
1142 
1143  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1144  """Fit the relative astrometry between templateSources and selectSources"""
1145  if self.config.winter2013WcsShift > 0.0:
1146  offset = geom.Extent2D(self.config.winter2013WcsShift,
1147  self.config.winter2013WcsShift)
1148  cKey = templateSources[0].getTable().getCentroidKey()
1149  for source in templateSources:
1150  centroid = source.get(cKey)
1151  source.set(cKey, centroid + offset)
1152  elif self.config.winter2013WcsRms > 0.0:
1153  cKey = templateSources[0].getTable().getCentroidKey()
1154  for source in templateSources:
1155  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1156  self.config.winter2013WcsRms*numpy.random.normal())
1157  centroid = source.get(cKey)
1158  source.set(cKey, centroid + offset)
1159 
1160  results = self.register.run(templateSources, templateExposure.getWcs(),
1161  templateExposure.getBBox(), selectSources)
1162  return results
lsst.pipe.tasks.registerImage
Definition: registerImage.py:1
lsst::ip::diffim
lsst::meas::astrom
lsst::afw::display
lsst::ip::diffim::utils
lsst::utils::inheritDoc
lsst.pipe.tasks.scaleVariance
Definition: scaleVariance.py:1
lsst::ip::diffim::diffimTools
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:712
lsstDebug::Info
lsst::meas::base
lsst::afw::table
lsst::utils
lsst::geom
lsst.pipe.tasks.imageDifference.ImageDifferenceTaskConnections
Definition: imageDifference.py:54
lsst::daf::base
lsst::meas::algorithms::SingleGaussianPsf
lsst::afw::math
lsst.pipe::base
lsst::meas::algorithms
Extent< double, 2 >