lsst.pipe.tasks  19.0.0-36-g0c970956+1
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 
47 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
49 IqrToSigma = 0.741
50 
51 
52 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
53  dimensions=("instrument", "visit", "detector", "skymap"),
54  defaultTemplates={"coaddName": "deep",
55  "warpTypeSuffix": "",
56  "fakesType": ""}):
57 
58  exposure = pipeBase.connectionTypes.Input(
59  doc="Input science exposure to subtract from.",
60  dimensions=("instrument", "visit", "detector"),
61  storageClass="ExposureF",
62  name="calexp"
63  )
64 
65  # TODO DM-22953
66  # kernelSources = pipeBase.connectionTypes.Input(
67  # doc="Source catalog produced in calibrate task for kernel candidate sources",
68  # name="src",
69  # storageClass="SourceCatalog",
70  # dimensions=("instrument", "visit", "detector"),
71  # )
72 
73  skyMap = pipeBase.connectionTypes.Input(
74  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
75  name="{coaddName}Coadd_skyMap",
76  dimensions=("skymap", ),
77  storageClass="SkyMap",
78  )
79  coaddExposures = pipeBase.connectionTypes.Input(
80  doc="Input template to match and subtract from the exposure",
81  dimensions=("tract", "patch", "skymap", "abstract_filter"),
82  storageClass="ExposureF",
83  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
84  multiple=True,
85  deferLoad=True
86  )
87  subtractedExposure = pipeBase.connectionTypes.Output(
88  doc="Output difference image",
89  dimensions=("instrument", "visit", "detector"),
90  storageClass="ExposureF",
91  name="{fakesType}{coaddName}Diff_differenceExp",
92  )
93  diaSources = pipeBase.connectionTypes.Output(
94  doc="Output detected diaSources on the difference image",
95  dimensions=("instrument", "visit", "detector"),
96  storageClass="SourceCatalog",
97  name="{fakesType}{coaddName}Diff_diaSrc",
98  )
99 
100  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
101  # Make kernelSources optional
102 
103 
104 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
105  pipelineConnections=ImageDifferenceTaskConnections):
106  """Config for ImageDifferenceTask
107  """
108  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
109  doc="Add background to calexp before processing it. "
110  "Useful as ipDiffim does background matching.")
111  doUseRegister = pexConfig.Field(dtype=bool, default=True,
112  doc="Use image-to-image registration to align template with "
113  "science image")
114  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
115  doc="Writing debugging data for doUseRegister")
116  doSelectSources = pexConfig.Field(dtype=bool, default=True,
117  doc="Select stars to use for kernel fitting")
118  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
119  doc="Select stars of extreme color as part of the control sample")
120  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
121  doc="Select stars that are variable to be part "
122  "of the control sample")
123  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
124  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
125  doc="Convolve science image by its PSF before PSF-matching?")
126  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
127  doc="Scale variance of the template before PSF matching")
128  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
129  doc="Use a simple gaussian PSF model for pre-convolution "
130  "(else use fit PSF)? Ignored if doPreConvolve false.")
131  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
132  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
133  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
134  "kernel convolution? If True, also update the diffim PSF.")
135  doMerge = pexConfig.Field(dtype=bool, default=True,
136  doc="Merge positive and negative diaSources with grow radius "
137  "set by growFootprint")
138  doMatchSources = pexConfig.Field(dtype=bool, default=True,
139  doc="Match diaSources with input calexp sources and ref catalog sources")
140  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
141  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
142  doForcedMeasurement = pexConfig.Field(
143  dtype=bool,
144  default=True,
145  doc="Force photometer diaSource locations on PVI?")
146  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
147  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
148  doc="Write warped and PSF-matched template coadd exposure?")
149  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
150  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
151  doc="Add columns to the source table to hold analysis metrics?")
152 
153  coaddName = pexConfig.Field(
154  doc="coadd name: typically one of deep, goodSeeing, or dcr",
155  dtype=str,
156  default="deep",
157  )
158  convolveTemplate = pexConfig.Field(
159  doc="Which image gets convolved (default = template)",
160  dtype=bool,
161  default=True
162  )
163  refObjLoader = pexConfig.ConfigurableField(
164  target=LoadIndexedReferenceObjectsTask,
165  doc="reference object loader",
166  )
167  astrometer = pexConfig.ConfigurableField(
168  target=AstrometryTask,
169  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
170  )
171  sourceSelector = pexConfig.ConfigurableField(
172  target=ObjectSizeStarSelectorTask,
173  doc="Source selection algorithm",
174  )
175  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
176  decorrelate = pexConfig.ConfigurableField(
177  target=DecorrelateALKernelSpatialTask,
178  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
179  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
180  "default of 5.5).",
181  )
182  doSpatiallyVarying = pexConfig.Field(
183  dtype=bool,
184  default=False,
185  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
186  "image in order to allow for spatial variations"
187  )
188  detection = pexConfig.ConfigurableField(
189  target=SourceDetectionTask,
190  doc="Low-threshold detection for final measurement",
191  )
192  measurement = pexConfig.ConfigurableField(
193  target=DipoleFitTask,
194  doc="Enable updated dipole fitting method",
195  )
196  forcedMeasurement = pexConfig.ConfigurableField(
197  target=ForcedMeasurementTask,
198  doc="Subtask to force photometer PVI at diaSource location.",
199  )
200  getTemplate = pexConfig.ConfigurableField(
201  target=GetCoaddAsTemplateTask,
202  doc="Subtask to retrieve template exposure and sources",
203  )
204  scaleVariance = pexConfig.ConfigurableField(
205  target=ScaleVarianceTask,
206  doc="Subtask to rescale the variance of the template "
207  "to the statistically expected level"
208  )
209  controlStepSize = pexConfig.Field(
210  doc="What step size (every Nth one) to select a control sample from the kernelSources",
211  dtype=int,
212  default=5
213  )
214  controlRandomSeed = pexConfig.Field(
215  doc="Random seed for shuffing the control sample",
216  dtype=int,
217  default=10
218  )
219  register = pexConfig.ConfigurableField(
220  target=RegisterTask,
221  doc="Task to enable image-to-image image registration (warping)",
222  )
223  kernelSourcesFromRef = pexConfig.Field(
224  doc="Select sources to measure kernel from reference catalog if True, template if false",
225  dtype=bool,
226  default=False
227  )
228  templateSipOrder = pexConfig.Field(
229  dtype=int, default=2,
230  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
231  )
232  growFootprint = pexConfig.Field(
233  dtype=int, default=2,
234  doc="Grow positive and negative footprints by this amount before merging"
235  )
236  diaSourceMatchRadius = pexConfig.Field(
237  dtype=float, default=0.5,
238  doc="Match radius (in arcseconds) for DiaSource to Source association"
239  )
240 
241  def setDefaults(self):
242  # defaults are OK for catalog and diacatalog
243 
244  self.subtract['al'].kernel.name = "AL"
245  self.subtract['al'].kernel.active.fitForBackground = True
246  self.subtract['al'].kernel.active.spatialKernelOrder = 1
247  self.subtract['al'].kernel.active.spatialBgOrder = 2
248  self.doPreConvolve = False
249  self.doMatchSources = False
250  self.doAddMetrics = False
251  self.doUseRegister = False
252 
253  # DiaSource Detection
254  self.detection.thresholdPolarity = "both"
255  self.detection.thresholdValue = 5.5
256  self.detection.reEstimateBackground = False
257  self.detection.thresholdType = "pixel_stdev"
258 
259  # Add filtered flux measurement, the correct measurement for pre-convolved images.
260  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
261  # To change that you must modify algorithms.names in the task's applyOverrides method,
262  # after the user has set doPreConvolve.
263  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
264  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
265  'base_LocalWcs']
266 
267  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
268  self.forcedMeasurement.copyColumns = {
269  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
270  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
271  self.forcedMeasurement.slots.shape = None
272 
273  # For shuffling the control sample
274  random.seed(self.controlRandomSeed)
275 
276  def validate(self):
277  pexConfig.Config.validate(self)
278  if self.doAddMetrics and not self.doSubtract:
279  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
280  if not self.doSubtract and not self.doDetection:
281  raise ValueError("Either doSubtract or doDetection must be enabled.")
282  if self.subtract.name == 'zogy' and self.doAddMetrics:
283  raise ValueError("Kernel metrics does not exist in zogy subtraction.")
284  if self.doMeasurement and not self.doDetection:
285  raise ValueError("Cannot run source measurement without source detection.")
286  if self.doMerge and not self.doDetection:
287  raise ValueError("Cannot run source merging without source detection.")
288  if self.doUseRegister and not self.doSelectSources:
289  raise ValueError("doUseRegister=True and doSelectSources=False. "
290  "Cannot run RegisterTask without selecting sources.")
291  if self.doPreConvolve and self.doDecorrelation and not self.convolveTemplate:
292  raise ValueError("doPreConvolve=True and doDecorrelation=True and "
293  "convolveTemplate=False is not supported.")
294  if hasattr(self.getTemplate, "coaddName"):
295  if self.getTemplate.coaddName != self.coaddName:
296  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
297 
298 
299 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
300 
301  @staticmethod
302  def getTargetList(parsedCmd, **kwargs):
303  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
304  **kwargs)
305 
306 
307 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
308  """Subtract an image from a template and measure the result
309  """
310  ConfigClass = ImageDifferenceConfig
311  RunnerClass = ImageDifferenceTaskRunner
312  _DefaultName = "imageDifference"
313 
314  def __init__(self, butler=None, **kwargs):
315  """!Construct an ImageDifference Task
316 
317  @param[in] butler Butler object to use in constructing reference object loaders
318  """
319  super().__init__(**kwargs)
320  self.makeSubtask("getTemplate")
321 
322  self.makeSubtask("subtract")
323 
324  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
325  self.makeSubtask("decorrelate")
326 
327  if self.config.doScaleTemplateVariance:
328  self.makeSubtask("scaleVariance")
329 
330  if self.config.doUseRegister:
331  self.makeSubtask("register")
332  self.schema = afwTable.SourceTable.makeMinimalSchema()
333 
334  if self.config.doSelectSources:
335  self.makeSubtask("sourceSelector")
336  if self.config.kernelSourcesFromRef:
337  self.makeSubtask('refObjLoader', butler=butler)
338  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
339 
340  self.algMetadata = dafBase.PropertyList()
341  if self.config.doDetection:
342  self.makeSubtask("detection", schema=self.schema)
343  if self.config.doMeasurement:
344  self.makeSubtask("measurement", schema=self.schema,
345  algMetadata=self.algMetadata)
346  if self.config.doForcedMeasurement:
347  self.schema.addField(
348  "ip_diffim_forced_PsfFlux_instFlux", "D",
349  "Forced PSF flux measured on the direct image.")
350  self.schema.addField(
351  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
352  "Forced PSF flux error measured on the direct image.")
353  self.schema.addField(
354  "ip_diffim_forced_PsfFlux_area", "F",
355  "Forced PSF flux effective area of PSF.",
356  units="pixel")
357  self.schema.addField(
358  "ip_diffim_forced_PsfFlux_flag", "Flag",
359  "Forced PSF flux general failure flag.")
360  self.schema.addField(
361  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
362  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
363  self.schema.addField(
364  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
365  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
366  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
367  if self.config.doMatchSources:
368  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
369  self.schema.addField("srcMatchId", "L", "unique id of source match")
370 
371  @staticmethod
372  def makeIdFactory(expId, expBits):
373  """Create IdFactory instance for unique 64 bit diaSource id-s.
374 
375  Parameters
376  ----------
377  expId : `int`
378  Exposure id.
379 
380  expBits: `int`
381  Number of used bits in ``expId``.
382 
383  Note
384  ----
385  The diasource id-s consists of the ``expId`` stored fixed in the highest value
386  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
387  low value end of the integer.
388 
389  Returns
390  -------
391  idFactory: `lsst.afw.table.IdFactory`
392  """
393  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
394 
395  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
396  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
397  inputRefs: pipeBase.InputQuantizedConnection,
398  outputRefs: pipeBase.OutputQuantizedConnection):
399  if self.getTemplate.config.coaddName == 'dcr':
400  # TODO DM-22952
401  raise NotImplementedError("TODO DM-22952: Dcr coadd templates are not yet supported in gen3.")
402 
403  inputs = butlerQC.get(inputRefs)
404  self.log.info(f"Processing {butlerQC.quantum.dataId}")
405  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
406  returnMaxBits=True)
407  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
408  templateStruct = self.getTemplate.runQuantum(
409  inputs['exposure'], butlerQC, inputRefs.skyMap, inputRefs.coaddExposures
410  )
411 
412  outputs = self.run(exposure=inputs['exposure'],
413  templateExposure=templateStruct.exposure,
414  idFactory=idFactory)
415  butlerQC.put(outputs, outputRefs)
416 
417  @pipeBase.timeMethod
418  def runDataRef(self, sensorRef, templateIdList=None):
419  """Subtract an image from a template coadd and measure the result.
420 
421  Data I/O wrapper around `run` using the butler in Gen2.
422 
423  Parameters
424  ----------
425  sensorRef : `lsst.daf.persistence.ButlerDataRef`
426  Sensor-level butler data reference, used for the following data products:
427 
428  Input only:
429  - calexp
430  - psf
431  - ccdExposureId
432  - ccdExposureId_bits
433  - self.config.coaddName + "Coadd_skyMap"
434  - self.config.coaddName + "Coadd"
435  Input or output, depending on config:
436  - self.config.coaddName + "Diff_subtractedExp"
437  Output, depending on config:
438  - self.config.coaddName + "Diff_matchedExp"
439  - self.config.coaddName + "Diff_src"
440 
441  Returns
442  -------
443  results : `lsst.pipe.base.Struct`
444  Returns the Struct by `run`.
445  """
446  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
447  subtractedExposure = None
448  selectSources = None
449  calexpBackgroundExposure = None
450  self.log.info("Processing %s" % (sensorRef.dataId))
451 
452  # We make one IdFactory that will be used by both icSrc and src datasets;
453  # I don't know if this is the way we ultimately want to do things, but at least
454  # this ensures the source IDs are fully unique.
455  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
456  expBits=sensorRef.get("ccdExposureId_bits"))
457  if self.config.doAddCalexpBackground:
458  calexpBackgroundExposure = sensorRef.get("calexpBackground")
459 
460  # Retrieve the science image we wish to analyze
461  exposure = sensorRef.get("calexp", immediate=True)
462 
463  # Retrieve the template image
464  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
465 
466  if sensorRef.datasetExists("src"):
467  self.log.info("Source selection via src product")
468  # Sources already exist; for data release processing
469  selectSources = sensorRef.get("src")
470 
471  if not self.config.doSubtract and self.config.doDetection:
472  # If we don't do subtraction, we need the subtracted exposure from the repo
473  subtractedExposure = sensorRef.get(subtractedExposureName)
474  # Both doSubtract and doDetection cannot be False
475 
476  results = self.run(exposure=exposure,
477  selectSources=selectSources,
478  templateExposure=template.exposure,
479  templateSources=template.sources,
480  idFactory=idFactory,
481  calexpBackgroundExposure=calexpBackgroundExposure,
482  subtractedExposure=subtractedExposure)
483 
484  if self.config.doWriteSources and results.diaSources is not None:
485  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
486  if self.config.doWriteMatchedExp:
487  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
488  if self.config.doAddMetrics and self.config.doSelectSources:
489  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
490  if self.config.doWriteSubtractedExp:
491  sensorRef.put(results.subtractedExposure, subtractedExposureName)
492  return results
493 
494  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
495  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
496  """PSF matches, subtract two images and perform detection on the difference image.
497 
498  Parameters
499  ----------
500  exposure : `lsst.afw.image.ExposureF`, optional
501  The science exposure, the minuend in the image subtraction.
502  Can be None only if ``config.doSubtract==False``.
503  selectSources : `lsst.afw.table.SourceCatalog`, optional
504  Identified sources on the science exposure. This catalog is used to
505  select sources in order to perform the AL PSF matching on stamp images
506  around them. The selection steps depend on config options and whether
507  ``templateSources`` and ``matchingSources`` specified.
508  templateExposure : `lsst.afw.image.ExposureF`, optional
509  The template to be subtracted from ``exposure`` in the image subtraction.
510  The template exposure should cover the same sky area as the science exposure.
511  It is either a stich of patches of a coadd skymap image or a calexp
512  of the same pointing as the science exposure. Can be None only
513  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
514  templateSources : `lsst.afw.table.SourceCatalog`, optional
515  Identified sources on the template exposure.
516  idFactory : `lsst.afw.table.IdFactory`
517  Generator object to assign ids to detected sources in the difference image.
518  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
519  Background exposure to be added back to the science exposure
520  if ``config.doAddCalexpBackground==True``
521  subtractedExposure : `lsst.afw.image.ExposureF`, optional
522  If ``config.doSubtract==False`` and ``config.doDetection==True``,
523  performs the post subtraction source detection only on this exposure.
524  Otherwise should be None.
525 
526  Returns
527  -------
528  results : `lsst.pipe.base.Struct`
529  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
530  Difference image.
531  ``matchedExposure`` : `lsst.afw.image.ExposureF`
532  The matched PSF exposure.
533  ``subtractRes`` : `lsst.pipe.base.Struct`
534  The returned result structure of the ImagePsfMatchTask subtask.
535  ``diaSources`` : `lsst.afw.table.SourceCatalog`
536  The catalog of detected sources.
537  ``selectSources`` : `lsst.afw.table.SourceCatalog`
538  The input source catalog with optionally added Qa information.
539 
540  Notes
541  -----
542  The following major steps are included:
543 
544  - warp template coadd to match WCS of image
545  - PSF match image to warped template
546  - subtract image from PSF-matched, warped template
547  - detect sources
548  - measure sources
549 
550  For details about the image subtraction configuration modes
551  see `lsst.ip.diffim`.
552  """
553  subtractRes = None
554  controlSources = None
555  diaSources = None
556  kernelSources = None
557 
558  if self.config.doAddCalexpBackground:
559  mi = exposure.getMaskedImage()
560  mi += calexpBackgroundExposure.getImage()
561 
562  if not exposure.hasPsf():
563  raise pipeBase.TaskError("Exposure has no psf")
564  sciencePsf = exposure.getPsf()
565 
566  if self.config.doSubtract:
567  if self.config.doScaleTemplateVariance:
568  templateVarFactor = self.scaleVariance.run(
569  templateExposure.getMaskedImage())
570  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
571 
572  if self.config.subtract.name == 'zogy':
573  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
574  doWarping=True,
575  spatiallyVarying=self.config.doSpatiallyVarying,
576  doPreConvolve=self.config.doPreConvolve)
577  subtractedExposure = subtractRes.subtractedExposure
578 
579  elif self.config.subtract.name == 'al':
580  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
581  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
582  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
583 
584  # if requested, convolve the science exposure with its PSF
585  # (properly, this should be a cross-correlation, but our code does not yet support that)
586  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
587  # else sigma of original science exposure
588  # TODO: DM-22762 This functional block should be moved into its own method
589  preConvPsf = None
590  if self.config.doPreConvolve:
591  convControl = afwMath.ConvolutionControl()
592  # cannot convolve in place, so make a new MI to receive convolved image
593  srcMI = exposure.getMaskedImage()
594  destMI = srcMI.Factory(srcMI.getDimensions())
595  srcPsf = sciencePsf
596  if self.config.useGaussianForPreConvolution:
597  # convolve with a simplified PSF model: a double Gaussian
598  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
599  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
600  else:
601  # convolve with science exposure's PSF model
602  preConvPsf = srcPsf
603  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
604  exposure.setMaskedImage(destMI)
605  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
606  else:
607  scienceSigmaPost = scienceSigmaOrig
608 
609  # If requested, find and select sources from the image
610  # else, AL subtraction will do its own source detection
611  # TODO: DM-22762 This functional block should be moved into its own method
612  if self.config.doSelectSources:
613  if selectSources is None:
614  self.log.warn("Src product does not exist; running detection, measurement, selection")
615  # Run own detection and measurement; necessary in nightly processing
616  selectSources = self.subtract.getSelectSources(
617  exposure,
618  sigma=scienceSigmaPost,
619  doSmooth=not self.config.doPreConvolve,
620  idFactory=idFactory,
621  )
622 
623  if self.config.doAddMetrics:
624  # Number of basis functions
625 
626  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
627  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
628  targetFwhmPix=templateSigma*FwhmPerSigma))
629  # Modify the schema of all Sources
630  # DEPRECATED: This is a data dependent (nparam) output product schema
631  # outside the task constructor.
632  # NOTE: The pre-determination of nparam at this point
633  # may be incorrect as the template psf is warped later in
634  # ImagePsfMatchTask.matchExposures()
635  kcQa = KernelCandidateQa(nparam)
636  selectSources = kcQa.addToSchema(selectSources)
637  if self.config.kernelSourcesFromRef:
638  # match exposure sources to reference catalog
639  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
640  matches = astromRet.matches
641  elif templateSources:
642  # match exposure sources to template sources
643  mc = afwTable.MatchControl()
644  mc.findOnlyClosest = False
645  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
646  mc)
647  else:
648  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
649  "but template sources not available. Cannot match science "
650  "sources with template sources. Run process* on data from "
651  "which templates are built.")
652 
653  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
654  matches=matches).sourceCat
655  random.shuffle(kernelSources, random.random)
656  controlSources = kernelSources[::self.config.controlStepSize]
657  kernelSources = [k for i, k in enumerate(kernelSources)
658  if i % self.config.controlStepSize]
659 
660  if self.config.doSelectDcrCatalog:
661  redSelector = DiaCatalogSourceSelectorTask(
662  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
663  grMax=99.999))
664  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
665  controlSources.extend(redSources)
666 
667  blueSelector = DiaCatalogSourceSelectorTask(
668  DiaCatalogSourceSelectorConfig(grMin=-99.999,
669  grMax=self.sourceSelector.config.grMin))
670  blueSources = blueSelector.selectStars(exposure, selectSources,
671  matches=matches).starCat
672  controlSources.extend(blueSources)
673 
674  if self.config.doSelectVariableCatalog:
675  varSelector = DiaCatalogSourceSelectorTask(
676  DiaCatalogSourceSelectorConfig(includeVariable=True))
677  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
678  controlSources.extend(varSources)
679 
680  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
681  % (len(kernelSources), len(selectSources), len(controlSources)))
682 
683  allresids = {}
684  # TODO: DM-22762 This functional block should be moved into its own method
685  if self.config.doUseRegister:
686  self.log.info("Registering images")
687 
688  if templateSources is None:
689  # Run detection on the template, which is
690  # temporarily background-subtracted
691  # sigma of PSF of template image before warping
692  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
693  templateSources = self.subtract.getSelectSources(
694  templateExposure,
695  sigma=templateSigma,
696  doSmooth=True,
697  idFactory=idFactory
698  )
699 
700  # Third step: we need to fit the relative astrometry.
701  #
702  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
703  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
704  exposure.getWcs(), exposure.getBBox())
705  templateExposure = warpedExp
706 
707  # Create debugging outputs on the astrometric
708  # residuals as a function of position. Persistence
709  # not yet implemented; expected on (I believe) #2636.
710  if self.config.doDebugRegister:
711  # Grab matches to reference catalog
712  srcToMatch = {x.second.getId(): x.first for x in matches}
713 
714  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
715  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
716  sids = [m.first.getId() for m in wcsResults.matches]
717  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
718  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
719  m.second.get(inCentroidKey))) for m in wcsResults.matches]
720  allresids = dict(zip(sids, zip(positions, residuals)))
721 
722  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
723  wcsResults.wcs.pixelToSky(
724  m.second.get(inCentroidKey))) for m in wcsResults.matches]
725  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
726  2.5*numpy.log10(srcToMatch[x].get("r"))
727  for x in sids if x in srcToMatch.keys()])
728  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
729  if s in srcToMatch.keys()])
730  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
731  if s in srcToMatch.keys()])
732  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
733  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
734  (colors <= self.sourceSelector.config.grMax))
735  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
736  rms1Long = IqrToSigma*(
737  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
738  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
739  numpy.percentile(dlat[idx1], 25))
740  rms2Long = IqrToSigma*(
741  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
742  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
743  numpy.percentile(dlat[idx2], 25))
744  rms3Long = IqrToSigma*(
745  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
746  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
747  numpy.percentile(dlat[idx3], 25))
748  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
749  (numpy.median(dlong[idx1]), rms1Long,
750  numpy.median(dlat[idx1]), rms1Lat))
751  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
752  (numpy.median(dlong[idx2]), rms2Long,
753  numpy.median(dlat[idx2]), rms2Lat))
754  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
755  (numpy.median(dlong[idx3]), rms3Long,
756  numpy.median(dlat[idx3]), rms3Lat))
757 
758  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
759  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
760  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
761  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
762  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
763  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
764 
765  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
766  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
767  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
768  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
769  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
770  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
771 
772  # warp template exposure to match exposure,
773  # PSF match template exposure to exposure,
774  # then return the difference
775 
776  # Return warped template... Construct sourceKernelCand list after subtract
777  self.log.info("Subtracting images")
778  subtractRes = self.subtract.subtractExposures(
779  templateExposure=templateExposure,
780  scienceExposure=exposure,
781  candidateList=kernelSources,
782  convolveTemplate=self.config.convolveTemplate,
783  doWarping=not self.config.doUseRegister
784  )
785  subtractedExposure = subtractRes.subtractedExposure
786 
787  if self.config.doDetection:
788  self.log.info("Computing diffim PSF")
789 
790  # Get Psf from the appropriate input image if it doesn't exist
791  if not subtractedExposure.hasPsf():
792  if self.config.convolveTemplate:
793  subtractedExposure.setPsf(exposure.getPsf())
794  else:
795  subtractedExposure.setPsf(templateExposure.getPsf())
796 
797  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
798  # thus it may have already been decorrelated. Thus, we do not decorrelate if
799  # doSubtract is False.
800 
801  # NOTE: At this point doSubtract == True
802  if self.config.doDecorrelation and self.config.doSubtract:
803  preConvKernel = None
804  if preConvPsf is not None:
805  preConvKernel = preConvPsf.getLocalKernel()
806  if self.config.convolveTemplate:
807  self.log.info("Decorrelation after template image convolution")
808  decorrResult = self.decorrelate.run(exposure, templateExposure,
809  subtractedExposure,
810  subtractRes.psfMatchingKernel,
811  spatiallyVarying=self.config.doSpatiallyVarying,
812  preConvKernel=preConvKernel)
813  else:
814  self.log.info("Decorrelation after science image convolution")
815  decorrResult = self.decorrelate.run(templateExposure, exposure,
816  subtractedExposure,
817  subtractRes.psfMatchingKernel,
818  spatiallyVarying=self.config.doSpatiallyVarying,
819  preConvKernel=preConvKernel)
820  subtractedExposure = decorrResult.correctedExposure
821 
822  # END (if subtractAlgorithm == 'AL')
823  # END (if self.config.doSubtract)
824  if self.config.doDetection:
825  self.log.info("Running diaSource detection")
826  # Erase existing detection mask planes
827  mask = subtractedExposure.getMaskedImage().getMask()
828  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
829 
830  table = afwTable.SourceTable.make(self.schema, idFactory)
831  table.setMetadata(self.algMetadata)
832  results = self.detection.run(
833  table=table,
834  exposure=subtractedExposure,
835  doSmooth=not self.config.doPreConvolve
836  )
837 
838  if self.config.doMerge:
839  fpSet = results.fpSets.positive
840  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
841  self.config.growFootprint, False)
842  diaSources = afwTable.SourceCatalog(table)
843  fpSet.makeSources(diaSources)
844  self.log.info("Merging detections into %d sources" % (len(diaSources)))
845  else:
846  diaSources = results.sources
847 
848  if self.config.doMeasurement:
849  newDipoleFitting = self.config.doDipoleFitting
850  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
851  if not newDipoleFitting:
852  # Just fit dipole in diffim
853  self.measurement.run(diaSources, subtractedExposure)
854  else:
855  # Use (matched) template and science image (if avail.) to constrain dipole fitting
856  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
857  self.measurement.run(diaSources, subtractedExposure, exposure,
858  subtractRes.matchedExposure)
859  else:
860  self.measurement.run(diaSources, subtractedExposure, exposure)
861 
862  if self.config.doForcedMeasurement:
863  # Run forced psf photometry on the PVI at the diaSource locations.
864  # Copy the measured flux and error into the diaSource.
865  forcedSources = self.forcedMeasurement.generateMeasCat(
866  exposure, diaSources, subtractedExposure.getWcs())
867  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
868  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
869  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
870  "ip_diffim_forced_PsfFlux_instFlux", True)
871  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
872  "ip_diffim_forced_PsfFlux_instFluxErr", True)
873  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
874  "ip_diffim_forced_PsfFlux_area", True)
875  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
876  "ip_diffim_forced_PsfFlux_flag", True)
877  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
878  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
879  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
880  "ip_diffim_forced_PsfFlux_flag_edge", True)
881  for diaSource, forcedSource in zip(diaSources, forcedSources):
882  diaSource.assign(forcedSource, mapper)
883 
884  # Match with the calexp sources if possible
885  if self.config.doMatchSources:
886  if selectSources is not None:
887  # Create key,val pair where key=diaSourceId and val=sourceId
888  matchRadAsec = self.config.diaSourceMatchRadius
889  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
890 
891  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
892  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
893  srcMatch in srcMatches])
894  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
895  len(diaSources)))
896  else:
897  self.log.warn("Src product does not exist; cannot match with diaSources")
898  srcMatchDict = {}
899 
900  # Create key,val pair where key=diaSourceId and val=refId
901  refAstromConfig = AstrometryConfig()
902  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
903  refAstrometer = AstrometryTask(refAstromConfig)
904  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
905  refMatches = astromRet.matches
906  if refMatches is None:
907  self.log.warn("No diaSource matches with reference catalog")
908  refMatchDict = {}
909  else:
910  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
911  len(diaSources)))
912  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
913  refMatch in refMatches])
914 
915  # Assign source Ids
916  for diaSource in diaSources:
917  sid = diaSource.getId()
918  if sid in srcMatchDict:
919  diaSource.set("srcMatchId", srcMatchDict[sid])
920  if sid in refMatchDict:
921  diaSource.set("refMatchId", refMatchDict[sid])
922 
923  if self.config.doAddMetrics and self.config.doSelectSources:
924  self.log.info("Evaluating metrics and control sample")
925 
926  kernelCandList = []
927  for cell in subtractRes.kernelCellSet.getCellList():
928  for cand in cell.begin(False): # include bad candidates
929  kernelCandList.append(cand)
930 
931  # Get basis list to build control sample kernels
932  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
933  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
934 
935  controlCandList = (
936  diffimTools.sourceTableToCandidateList(controlSources,
937  subtractRes.warpedExposure, exposure,
938  self.config.subtract.kernel.active,
939  self.config.subtract.kernel.active.detectionConfig,
940  self.log, doBuild=True, basisList=basisList))
941 
942  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
943  subtractRes.backgroundModel, dof=nparam)
944  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
945  subtractRes.backgroundModel)
946 
947  if self.config.doDetection:
948  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
949  else:
950  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
951 
952  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
953  return pipeBase.Struct(
954  subtractedExposure=subtractedExposure,
955  matchedExposure=subtractRes.matchedExposure,
956  subtractRes=subtractRes,
957  diaSources=diaSources,
958  selectSources=selectSources
959  )
960 
961  def fitAstrometry(self, templateSources, templateExposure, selectSources):
962  """Fit the relative astrometry between templateSources and selectSources
963 
964  Todo
965  ----
966 
967  Remove this method. It originally fit a new WCS to the template before calling register.run
968  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
969  It remains because a subtask overrides it.
970  """
971  results = self.register.run(templateSources, templateExposure.getWcs(),
972  templateExposure.getBBox(), selectSources)
973  return results
974 
975  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
976  """Make debug plots and displays.
977 
978  Todo
979  ----
980  Test and update for current debug display and slot names
981  """
982  import lsstDebug
983  display = lsstDebug.Info(__name__).display
984  showSubtracted = lsstDebug.Info(__name__).showSubtracted
985  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
986  showDiaSources = lsstDebug.Info(__name__).showDiaSources
987  showDipoles = lsstDebug.Info(__name__).showDipoles
988  maskTransparency = lsstDebug.Info(__name__).maskTransparency
989  if display:
990  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
991  if not maskTransparency:
992  maskTransparency = 0
993  disp.setMaskTransparency(maskTransparency)
994 
995  if display and showSubtracted:
996  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
997  mi = subtractRes.subtractedExposure.getMaskedImage()
998  x0, y0 = mi.getX0(), mi.getY0()
999  with disp.Buffering():
1000  for s in diaSources:
1001  x, y = s.getX() - x0, s.getY() - y0
1002  ctype = "red" if s.get("flags_negative") else "yellow"
1003  if (s.get("base_PixelFlags_flag_interpolatedCenter") or
1004  s.get("base_PixelFlags_flag_saturatedCenter") or
1005  s.get("base_PixelFlags_flag_crCenter")):
1006  ptype = "x"
1007  elif (s.get("base_PixelFlags_flag_interpolated") or
1008  s.get("base_PixelFlags_flag_saturated") or
1009  s.get("base_PixelFlags_flag_cr")):
1010  ptype = "+"
1011  else:
1012  ptype = "o"
1013  disp.dot(ptype, x, y, size=4, ctype=ctype)
1014  lsstDebug.frame += 1
1015 
1016  if display and showPixelResiduals and selectSources:
1017  nonKernelSources = []
1018  for source in selectSources:
1019  if source not in kernelSources:
1020  nonKernelSources.append(source)
1021 
1022  diUtils.plotPixelResiduals(exposure,
1023  subtractRes.warpedExposure,
1024  subtractRes.subtractedExposure,
1025  subtractRes.kernelCellSet,
1026  subtractRes.psfMatchingKernel,
1027  subtractRes.backgroundModel,
1028  nonKernelSources,
1029  self.subtract.config.kernel.active.detectionConfig,
1030  origVariance=False)
1031  diUtils.plotPixelResiduals(exposure,
1032  subtractRes.warpedExposure,
1033  subtractRes.subtractedExposure,
1034  subtractRes.kernelCellSet,
1035  subtractRes.psfMatchingKernel,
1036  subtractRes.backgroundModel,
1037  nonKernelSources,
1038  self.subtract.config.kernel.active.detectionConfig,
1039  origVariance=True)
1040  if display and showDiaSources:
1041  flagChecker = SourceFlagChecker(diaSources)
1042  isFlagged = [flagChecker(x) for x in diaSources]
1043  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1044  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1045  frame=lsstDebug.frame)
1046  lsstDebug.frame += 1
1047 
1048  if display and showDipoles:
1049  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1050  frame=lsstDebug.frame)
1051  lsstDebug.frame += 1
1052 
1053  def _getConfigName(self):
1054  """Return the name of the config dataset
1055  """
1056  return "%sDiff_config" % (self.config.coaddName,)
1057 
1058  def _getMetadataName(self):
1059  """Return the name of the metadata dataset
1060  """
1061  return "%sDiff_metadata" % (self.config.coaddName,)
1062 
1063  def getSchemaCatalogs(self):
1064  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1065  diaSrc = afwTable.SourceCatalog(self.schema)
1066  diaSrc.getTable().setMetadata(self.algMetadata)
1067  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
1068 
1069  @classmethod
1070  def _makeArgumentParser(cls):
1071  """Create an argument parser
1072  """
1073  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1074  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1075  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1076  help="Template data ID in case of calexp template,"
1077  " e.g. --templateId visit=6789")
1078  return parser
1079 
1080 
1081 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1082  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1083  doc="Shift stars going into RegisterTask by this amount")
1084  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1085  doc="Perturb stars going into RegisterTask by this amount")
1086 
1087  def setDefaults(self):
1088  ImageDifferenceConfig.setDefaults(self)
1089  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1090 
1091 
1092 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1093  """!Image difference Task used in the Winter 2013 data challege.
1094  Enables testing the effects of registration shifts and scatter.
1095 
1096  For use with winter 2013 simulated images:
1097  Use --templateId visit=88868666 for sparse data
1098  --templateId visit=22222200 for dense data (g)
1099  --templateId visit=11111100 for dense data (i)
1100  """
1101  ConfigClass = Winter2013ImageDifferenceConfig
1102  _DefaultName = "winter2013ImageDifference"
1103 
1104  def __init__(self, **kwargs):
1105  ImageDifferenceTask.__init__(self, **kwargs)
1106 
1107  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1108  """Fit the relative astrometry between templateSources and selectSources"""
1109  if self.config.winter2013WcsShift > 0.0:
1110  offset = geom.Extent2D(self.config.winter2013WcsShift,
1111  self.config.winter2013WcsShift)
1112  cKey = templateSources[0].getTable().getCentroidKey()
1113  for source in templateSources:
1114  centroid = source.get(cKey)
1115  source.set(cKey, centroid + offset)
1116  elif self.config.winter2013WcsRms > 0.0:
1117  cKey = templateSources[0].getTable().getCentroidKey()
1118  for source in templateSources:
1119  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1120  self.config.winter2013WcsRms*numpy.random.normal())
1121  centroid = source.get(cKey)
1122  source.set(cKey, centroid + offset)
1123 
1124  results = self.register.run(templateSources, templateExposure.getWcs(),
1125  templateExposure.getBBox(), selectSources)
1126  return results
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)