lsst.pipe.tasks  13.0-60-g3ba4059d+1
 All Classes Namespaces Files Functions Variables Groups Pages
imageDifference.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 from builtins import zip
24 import math
25 import random
26 import numpy
27 
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.daf.base as dafBase
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.math as afwMath
33 import lsst.afw.table as afwTable
34 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
35 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, \
38  ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import DipoleAnalysis, \
40  SourceFlagChecker, KernelCandidateF, makeKernelBasisList, \
41  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, \
42  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask, \
43  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry
44 import lsst.ip.diffim.diffimTools as diffimTools
45 import lsst.ip.diffim.utils as diUtils
46 
47 FwhmPerSigma = 2 * math.sqrt(2 * math.log(2))
48 IqrToSigma = 0.741
49 
50 
51 class ImageDifferenceConfig(pexConfig.Config):
52  """Config for ImageDifferenceTask
53  """
54  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=True,
55  doc="Add background to calexp before processing it. "
56  "Useful as ipDiffim does background matching.")
57  doUseRegister = pexConfig.Field(dtype=bool, default=True,
58  doc="Use image-to-image registration to align template with "
59  "science image")
60  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
61  doc="Writing debugging data for doUseRegister")
62  doSelectSources = pexConfig.Field(dtype=bool, default=True,
63  doc="Select stars to use for kernel fitting")
64  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
65  doc="Select stars of extreme color as part of the control sample")
66  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
67  doc="Select stars that are variable to be part "
68  "of the control sample")
69  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
70  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
71  doc="Convolve science image by its PSF before PSF-matching?")
72  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
73  doc="Use a simple gaussian PSF model for pre-convolution "
74  "(else use fit PSF)? Ignored if doPreConvolve false.")
75  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
76  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
77  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
78  "kernel convolution? If True, also update the diffim PSF.")
79  doMerge = pexConfig.Field(dtype=bool, default=True,
80  doc="Merge positive and negative diaSources with grow radius "
81  "set by growFootprint")
82  doMatchSources = pexConfig.Field(dtype=bool, default=True,
83  doc="Match diaSources with input calexp sources and ref catalog sources")
84  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
85  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
86  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
87  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
88  doc="Write warped and PSF-matched template coadd exposure?")
89  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
90  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
91  doc="Add columns to the source table to hold analysis metrics?")
92 
93  coaddName = pexConfig.Field(
94  doc="coadd name: typically one of deep or goodSeeing",
95  dtype=str,
96  default="deep",
97  )
98  convolveTemplate = pexConfig.Field(
99  doc="Which image gets convolved (default = template)",
100  dtype=bool,
101  default=True
102  )
103  refObjLoader = pexConfig.ConfigurableField(
104  target=LoadAstrometryNetObjectsTask,
105  doc="reference object loader",
106  )
107  astrometer = pexConfig.ConfigurableField(
108  target=AstrometryTask,
109  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
110  )
111  sourceSelector = pexConfig.ConfigurableField(
112  target=ObjectSizeStarSelectorTask,
113  doc="Source selection algorithm",
114  )
115  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
116  decorrelate = pexConfig.ConfigurableField(
117  target=DecorrelateALKernelSpatialTask,
118  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
119  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
120  "default of 5.5).",
121  )
122  doSpatiallyVarying = pexConfig.Field(
123  dtype=bool,
124  default=False,
125  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
126  "image in order to allow for spatial variations"
127  )
128  detection = pexConfig.ConfigurableField(
129  target=SourceDetectionTask,
130  doc="Low-threshold detection for final measurement",
131  )
132  measurement = pexConfig.ConfigurableField(
133  target=DipoleFitTask,
134  doc="Enable updated dipole fitting method",
135  )
136  getTemplate = pexConfig.ConfigurableField(
137  target=GetCoaddAsTemplateTask,
138  doc="Subtask to retrieve template exposure and sources",
139  )
140  controlStepSize = pexConfig.Field(
141  doc="What step size (every Nth one) to select a control sample from the kernelSources",
142  dtype=int,
143  default=5
144  )
145  controlRandomSeed = pexConfig.Field(
146  doc = "Random seed for shuffing the control sample",
147  dtype = int,
148  default = 10
149  )
150  register = pexConfig.ConfigurableField(
151  target=RegisterTask,
152  doc="Task to enable image-to-image image registration (warping)",
153  )
154  kernelSourcesFromRef = pexConfig.Field(
155  doc="Select sources to measure kernel from reference catalog if True, template if false",
156  dtype=bool,
157  default=False
158  )
159  templateSipOrder = pexConfig.Field(dtype=int, default=2,
160  doc="Sip Order for fitting the Template Wcs "
161  "(default is too high, overfitting)")
162 
163  growFootprint = pexConfig.Field(dtype=int, default=2,
164  doc="Grow positive and negative footprints by this amount before merging")
165 
166  diaSourceMatchRadius = pexConfig.Field(dtype=float, default=0.5,
167  doc="Match radius (in arcseconds) "
168  "for DiaSource to Source association")
169 
170  def setDefaults(self):
171  # defaults are OK for catalog and diacatalog
172 
173  self.subtract['al'].kernel.name = "AL"
174  self.subtract['al'].kernel.active.fitForBackground = True
175  self.subtract['al'].kernel.active.spatialKernelOrder = 1
176  self.subtract['al'].kernel.active.spatialBgOrder = 0
177  self.doPreConvolve = False
178  self.doMatchSources = False
179  self.doAddMetrics = False
180  self.doUseRegister = False
181 
182  # DiaSource Detection
183  self.detection.thresholdPolarity = "both"
184  self.detection.thresholdValue = 5.5
185  self.detection.reEstimateBackground = False
186  self.detection.thresholdType = "pixel_stdev"
187 
188  # Add filtered flux measurement, the correct measurement for pre-convolved images.
189  # Enable all measurements, regardless of doPreConvolved, as it makes data harvesting easier.
190  # To change that you must modify algorithms.names in the task's applyOverrides method,
191  # after the user has set doPreConvolved.
192  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
193 
194  # For shuffling the control sample
195  random.seed(self.controlRandomSeed)
196 
197  def validate(self):
198  pexConfig.Config.validate(self)
199  if self.doMeasurement and not self.doDetection:
200  raise ValueError("Cannot run source measurement without source detection.")
201  if self.doMerge and not self.doDetection:
202  raise ValueError("Cannot run source merging without source detection.")
203  if self.doUseRegister and not self.doSelectSources:
204  raise ValueError("doUseRegister=True and doSelectSources=False. " +
205  "Cannot run RegisterTask without selecting sources.")
206 
207 
208 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
209 
210  @staticmethod
211  def getTargetList(parsedCmd, **kwargs):
212  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
213  **kwargs)
214 
215 
216 class ImageDifferenceTask(pipeBase.CmdLineTask):
217  """Subtract an image from a template and measure the result
218  """
219  ConfigClass = ImageDifferenceConfig
220  RunnerClass = ImageDifferenceTaskRunner
221  _DefaultName = "imageDifference"
222 
223  def __init__(self, butler=None, **kwargs):
224  """!Construct an ImageDifference Task
225 
226  @param[in] butler Butler object to use in constructing reference object loaders
227  """
228  pipeBase.CmdLineTask.__init__(self, **kwargs)
229  self.makeSubtask("getTemplate")
230 
231  self.makeSubtask("subtract")
232 
233  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
234  self.makeSubtask("decorrelate")
235 
236  if self.config.doUseRegister:
237  self.makeSubtask("register")
238  self.schema = afwTable.SourceTable.makeMinimalSchema()
239 
240  if self.config.doSelectSources:
241  self.makeSubtask("sourceSelector", schema=self.schema)
242  self.makeSubtask('refObjLoader', butler=butler)
243  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
244 
245  self.algMetadata = dafBase.PropertyList()
246  if self.config.doDetection:
247  self.makeSubtask("detection", schema=self.schema)
248  if self.config.doMeasurement:
249  self.makeSubtask("measurement", schema=self.schema,
250  algMetadata=self.algMetadata)
251  if self.config.doMatchSources:
252  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
253  self.schema.addField("srcMatchId", "L", "unique id of source match")
254 
255  @pipeBase.timeMethod
256  def run(self, sensorRef, templateIdList=None):
257  """Subtract an image from a template coadd and measure the result
258 
259  Steps include:
260  - warp template coadd to match WCS of image
261  - PSF match image to warped template
262  - subtract image from PSF-matched, warped template
263  - persist difference image
264  - detect sources
265  - measure sources
266 
267  @param sensorRef: sensor-level butler data reference, used for the following data products:
268  Input only:
269  - calexp
270  - psf
271  - ccdExposureId
272  - ccdExposureId_bits
273  - self.config.coaddName + "Coadd_skyMap"
274  - self.config.coaddName + "Coadd"
275  Input or output, depending on config:
276  - self.config.coaddName + "Diff_subtractedExp"
277  Output, depending on config:
278  - self.config.coaddName + "Diff_matchedExp"
279  - self.config.coaddName + "Diff_src"
280 
281  @return pipe_base Struct containing these fields:
282  - subtractedExposure: exposure after subtracting template;
283  the unpersisted version if subtraction not run but detection run
284  None if neither subtraction nor detection run (i.e. nothing useful done)
285  - subtractRes: results of subtraction task; None if subtraction not run
286  - sources: detected and possibly measured sources; None if detection not run
287  """
288  self.log.info("Processing %s" % (sensorRef.dataId))
289 
290  # initialize outputs and some intermediate products
291  subtractedExposure = None
292  subtractRes = None
293  selectSources = None
294  kernelSources = None
295  controlSources = None
296  diaSources = None
297 
298  # We make one IdFactory that will be used by both icSrc and src datasets;
299  # I don't know if this is the way we ultimately want to do things, but at least
300  # this ensures the source IDs are fully unique.
301  expBits = sensorRef.get("ccdExposureId_bits")
302  expId = int(sensorRef.get("ccdExposureId"))
303  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
304 
305  # Retrieve the science image we wish to analyze
306  exposure = sensorRef.get("calexp", immediate=True)
307  if self.config.doAddCalexpBackground:
308  mi = exposure.getMaskedImage()
309  mi += sensorRef.get("calexpBackground").getImage()
310  if not exposure.hasPsf():
311  raise pipeBase.TaskError("Exposure has no psf")
312  sciencePsf = exposure.getPsf()
313 
314  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
315  templateExposure = None # Stitched coadd exposure
316  templateSources = None # Sources on the template image
317  if self.config.doSubtract:
318  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
319  templateExposure = template.exposure
320  templateSources = template.sources
321 
322  if self.config.subtract.name == 'zogy':
323  spatiallyVarying = self.config.doSpatiallyVarying
324  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
325  doWarping=True,
326  spatiallyVarying=spatiallyVarying)
327  subtractedExposure = subtractRes.subtractedExposure
328 
329  elif self.config.subtract.name == 'al':
330  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
331  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
332 
333  # sigma of PSF of template image before warping
334  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
335 
336  # if requested, convolve the science exposure with its PSF
337  # (properly, this should be a cross-correlation, but our code does not yet support that)
338  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
339  # else sigma of original science exposure
340  if self.config.doPreConvolve:
341  convControl = afwMath.ConvolutionControl()
342  # cannot convolve in place, so make a new MI to receive convolved image
343  srcMI = exposure.getMaskedImage()
344  destMI = srcMI.Factory(srcMI.getDimensions())
345  srcPsf = sciencePsf
346  if self.config.useGaussianForPreConvolution:
347  # convolve with a simplified PSF model: a double Gaussian
348  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
349  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
350  else:
351  # convolve with science exposure's PSF model
352  preConvPsf = srcPsf
353  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
354  exposure.setMaskedImage(destMI)
355  scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
356  else:
357  scienceSigmaPost = scienceSigmaOrig
358 
359  # If requested, find sources in the image
360  if self.config.doSelectSources:
361  if not sensorRef.datasetExists("src"):
362  self.log.warn("Src product does not exist; running detection, measurement, selection")
363  # Run own detection and measurement; necessary in nightly processing
364  selectSources = self.subtract.getSelectSources(
365  exposure,
366  sigma=scienceSigmaPost,
367  doSmooth=not self.doPreConvolve,
368  idFactory=idFactory,
369  )
370  else:
371  self.log.info("Source selection via src product")
372  # Sources already exist; for data release processing
373  selectSources = sensorRef.get("src")
374 
375  # Number of basis functions
376  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
377  referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
378  targetFwhmPix=templateSigma * FwhmPerSigma))
379 
380  if self.config.doAddMetrics:
381  # Modify the schema of all Sources
382  kcQa = KernelCandidateQa(nparam)
383  selectSources = kcQa.addToSchema(selectSources)
384 
385  if self.config.kernelSourcesFromRef:
386  # match exposure sources to reference catalog
387  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
388  matches = astromRet.matches
389  elif templateSources:
390  # match exposure sources to template sources
391  mc = afwTable.MatchControl()
392  mc.findOnlyClosest = False
393  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
394  mc)
395  else:
396  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
397  "but template sources not available. Cannot match science " +
398  "sources with template sources. Run process* on data from " +
399  "which templates are built.")
400 
401  kernelSources = self.sourceSelector.selectStars(exposure, selectSources,
402  matches=matches).starCat
403 
404  random.shuffle(kernelSources, random.random)
405  controlSources = kernelSources[::self.config.controlStepSize]
406  kernelSources = [k for i, k in enumerate(kernelSources) if i % self.config.controlStepSize]
407 
408  if self.config.doSelectDcrCatalog:
409  redSelector = DiaCatalogSourceSelectorTask(
410  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999))
411  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
412  controlSources.extend(redSources)
413 
414  blueSelector = DiaCatalogSourceSelectorTask(
415  DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin))
416  blueSources = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat
417  controlSources.extend(blueSources)
418 
419  if self.config.doSelectVariableCatalog:
420  varSelector = DiaCatalogSourceSelectorTask(
421  DiaCatalogSourceSelectorConfig(includeVariable=True))
422  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
423  controlSources.extend(varSources)
424 
425  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
426  % (len(kernelSources), len(selectSources), len(controlSources)))
427  allresids = {}
428  if self.config.doUseRegister:
429  self.log.info("Registering images")
430 
431  if templateSources is None:
432  # Run detection on the template, which is
433  # temporarily background-subtracted
434  templateSources = self.subtract.getSelectSources(
435  templateExposure,
436  sigma=templateSigma,
437  doSmooth=True,
438  idFactory=idFactory
439  )
440 
441  # Third step: we need to fit the relative astrometry.
442  #
443  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
444  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
445  exposure.getWcs(), exposure.getBBox())
446  templateExposure = warpedExp
447 
448  # Create debugging outputs on the astrometric
449  # residuals as a function of position. Persistence
450  # not yet implemented; expected on (I believe) #2636.
451  if self.config.doDebugRegister:
452  # Grab matches to reference catalog
453  srcToMatch = {x.second.getId(): x.first for x in matches}
454 
455  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
456  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
457  sids = [m.first.getId() for m in wcsResults.matches]
458  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
459  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
460  m.second.get(inCentroidKey))) for m in wcsResults.matches]
461  allresids = dict(zip(sids, zip(positions, residuals)))
462 
463  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
464  wcsResults.wcs.pixelToSky(
465  m.second.get(inCentroidKey))) for m in wcsResults.matches]
466  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
467  2.5*numpy.log10(srcToMatch[x].get("r"))
468  for x in sids if x in srcToMatch.keys()])
469  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
470  if s in srcToMatch.keys()])
471  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
472  if s in srcToMatch.keys()])
473  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
474  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
475  (colors <= self.sourceSelector.config.grMax))
476  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
477  rms1Long = IqrToSigma * \
478  (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
479  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)-numpy.percentile(dlat[idx1], 25))
480  rms2Long = IqrToSigma * \
481  (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
482  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)-numpy.percentile(dlat[idx2], 25))
483  rms3Long = IqrToSigma * \
484  (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
485  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)-numpy.percentile(dlat[idx3], 25))
486  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]),
487  rms1Long,
488  numpy.median(dlat[idx1]),
489  rms1Lat))
490  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]),
491  rms2Long,
492  numpy.median(dlat[idx2]),
493  rms2Lat))
494  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]),
495  rms3Long,
496  numpy.median(dlat[idx3]),
497  rms3Lat))
498 
499  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
500  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
501  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
502  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
503  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
504  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
505 
506  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
507  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
508  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
509  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
510  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
511  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
512 
513  # warp template exposure to match exposure,
514  # PSF match template exposure to exposure,
515  # then return the difference
516 
517  # Return warped template... Construct sourceKernelCand list after subtract
518  self.log.info("Subtracting images")
519  subtractRes = self.subtract.subtractExposures(
520  templateExposure=templateExposure,
521  scienceExposure=exposure,
522  candidateList=kernelSources,
523  convolveTemplate=self.config.convolveTemplate,
524  doWarping=not self.config.doUseRegister
525  )
526  subtractedExposure = subtractRes.subtractedExposure
527 
528  if self.config.doWriteMatchedExp:
529  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
530 
531  if self.config.doDetection:
532  self.log.info("Computing diffim PSF")
533  if subtractedExposure is None:
534  subtractedExposure = sensorRef.get(subtractedExposureName)
535 
536  # Get Psf from the appropriate input image if it doesn't exist
537  if not subtractedExposure.hasPsf():
538  if self.config.convolveTemplate:
539  subtractedExposure.setPsf(exposure.getPsf())
540  else:
541  if templateExposure is None:
542  template = self.getTemplate.run(exposure, sensorRef,
543  templateIdList=templateIdList)
544  subtractedExposure.setPsf(template.exposure.getPsf())
545 
546  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
547  # thus it may have already been decorrelated. Thus, we do not decorrelate if
548  # doSubtract is False.
549  if self.config.doDecorrelation and self.config.doSubtract:
550  spatiallyVarying = self.config.doSpatiallyVarying
551  decorrResult = self.decorrelate.run(exposure, templateExposure,
552  subtractedExposure,
553  subtractRes.psfMatchingKernel,
554  spatiallyVarying=spatiallyVarying)
555  subtractedExposure = decorrResult.correctedExposure
556 
557  # END (if subtractAlgorithm == 'AL')
558 
559  if self.config.doDetection:
560  self.log.info("Running diaSource detection")
561  # Erase existing detection mask planes
562  mask = subtractedExposure.getMaskedImage().getMask()
563  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
564 
565  table = afwTable.SourceTable.make(self.schema, idFactory)
566  table.setMetadata(self.algMetadata)
567  results = self.detection.makeSourceCatalog(
568  table=table,
569  exposure=subtractedExposure,
570  doSmooth=not self.config.doPreConvolve
571  )
572 
573  if self.config.doMerge:
574  fpSet = results.fpSets.positive
575  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
576  self.config.growFootprint, False)
577  diaSources = afwTable.SourceCatalog(table)
578  fpSet.makeSources(diaSources)
579  self.log.info("Merging detections into %d sources" % (len(diaSources)))
580  else:
581  diaSources = results.sources
582 
583  if self.config.doMeasurement:
584  newDipoleFitting = self.config.doDipoleFitting
585  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
586  if not newDipoleFitting:
587  # Just fit dipole in diffim
588  self.measurement.run(diaSources, subtractedExposure)
589  else:
590  # Use (matched) template and science image (if avail.) to constrain dipole fitting
591  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
592  self.measurement.run(diaSources, subtractedExposure, exposure,
593  subtractRes.matchedExposure)
594  else:
595  self.measurement.run(diaSources, subtractedExposure, exposure)
596 
597  # Match with the calexp sources if possible
598  if self.config.doMatchSources:
599  if sensorRef.datasetExists("src"):
600  # Create key,val pair where key=diaSourceId and val=sourceId
601  matchRadAsec = self.config.diaSourceMatchRadius
602  matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
603 
604  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
605  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
606  srcMatch in srcMatches])
607  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
608  len(diaSources)))
609  else:
610  self.log.warn("Src product does not exist; cannot match with diaSources")
611  srcMatchDict = {}
612 
613  # Create key,val pair where key=diaSourceId and val=refId
614  refAstromConfig = AstrometryConfig()
615  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
616  refAstrometer = AstrometryTask(refAstromConfig)
617  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
618  refMatches = astromRet.matches
619  if refMatches is None:
620  self.log.warn("No diaSource matches with reference catalog")
621  refMatchDict = {}
622  else:
623  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
624  len(diaSources)))
625  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
626  refMatch in refMatches])
627 
628  # Assign source Ids
629  for diaSource in diaSources:
630  sid = diaSource.getId()
631  if sid in srcMatchDict:
632  diaSource.set("srcMatchId", srcMatchDict[sid])
633  if sid in refMatchDict:
634  diaSource.set("refMatchId", refMatchDict[sid])
635 
636  if diaSources is not None and self.config.doWriteSources:
637  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
638 
639  if self.config.doAddMetrics and self.config.doSelectSources:
640  self.log.info("Evaluating metrics and control sample")
641 
642  kernelCandList = []
643  for cell in subtractRes.kernelCellSet.getCellList():
644  for cand in cell.begin(False): # include bad candidates
645  kernelCandList.append(cand)
646 
647  # Get basis list to build control sample kernels
648  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
649 
650  controlCandList = \
651  diffimTools.sourceTableToCandidateList(controlSources,
652  subtractRes.warpedExposure, exposure,
653  self.config.subtract.kernel.active,
654  self.config.subtract.kernel.active.detectionConfig,
655  self.log, doBuild=True, basisList=basisList)
656 
657  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
658  dof=nparam)
659  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
660 
661  if self.config.doDetection:
662  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
663  else:
664  kcQa.aggregate(selectSources, self.metadata, allresids)
665 
666  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
667 
668  if self.config.doWriteSubtractedExp:
669  sensorRef.put(subtractedExposure, subtractedExposureName)
670 
671  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
672  return pipeBase.Struct(
673  subtractedExposure=subtractedExposure,
674  subtractRes=subtractRes,
675  sources=diaSources,
676  )
677 
678  def fitAstrometry(self, templateSources, templateExposure, selectSources):
679  """Fit the relative astrometry between templateSources and selectSources
680 
681  @todo remove this method. It originally fit a new WCS to the template before calling register.run
682  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
683  It remains because a subtask overrides it.
684  """
685  results = self.register.run(templateSources, templateExposure.getWcs(),
686  templateExposure.getBBox(), selectSources)
687  return results
688 
689  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
690  """@todo Test and update for current debug display and slot names
691  """
692  import lsstDebug
693  display = lsstDebug.Info(__name__).display
694  showSubtracted = lsstDebug.Info(__name__).showSubtracted
695  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
696  showDiaSources = lsstDebug.Info(__name__).showDiaSources
697  showDipoles = lsstDebug.Info(__name__).showDipoles
698  maskTransparency = lsstDebug.Info(__name__).maskTransparency
699  if display:
700  import lsst.afw.display.ds9 as ds9
701  if not maskTransparency:
702  maskTransparency = 0
703  ds9.setMaskTransparency(maskTransparency)
704 
705  if display and showSubtracted:
706  ds9.mtv(subtractRes.subtractedExposure, frame=lsstDebug.frame, title="Subtracted image")
707  mi = subtractRes.subtractedExposure.getMaskedImage()
708  x0, y0 = mi.getX0(), mi.getY0()
709  with ds9.Buffering():
710  for s in diaSources:
711  x, y = s.getX() - x0, s.getY() - y0
712  ctype = "red" if s.get("flags.negative") else "yellow"
713  if (s.get("flags.pixel.interpolated.center") or s.get("flags.pixel.saturated.center") or
714  s.get("flags.pixel.cr.center")):
715  ptype = "x"
716  elif (s.get("flags.pixel.interpolated.any") or s.get("flags.pixel.saturated.any") or
717  s.get("flags.pixel.cr.any")):
718  ptype = "+"
719  else:
720  ptype = "o"
721  ds9.dot(ptype, x, y, size=4, frame=lsstDebug.frame, ctype=ctype)
722  lsstDebug.frame += 1
723 
724  if display and showPixelResiduals and selectSources:
725  nonKernelSources = []
726  for source in selectSources:
727  if source not in kernelSources:
728  nonKernelSources.append(source)
729 
730  diUtils.plotPixelResiduals(exposure,
731  subtractRes.warpedExposure,
732  subtractRes.subtractedExposure,
733  subtractRes.kernelCellSet,
734  subtractRes.psfMatchingKernel,
735  subtractRes.backgroundModel,
736  nonKernelSources,
737  self.subtract.config.kernel.active.detectionConfig,
738  origVariance=False)
739  diUtils.plotPixelResiduals(exposure,
740  subtractRes.warpedExposure,
741  subtractRes.subtractedExposure,
742  subtractRes.kernelCellSet,
743  subtractRes.psfMatchingKernel,
744  subtractRes.backgroundModel,
745  nonKernelSources,
746  self.subtract.config.kernel.active.detectionConfig,
747  origVariance=True)
748  if display and showDiaSources:
749  flagChecker = SourceFlagChecker(diaSources)
750  isFlagged = [flagChecker(x) for x in diaSources]
751  isDipole = [x.get("classification.dipole") for x in diaSources]
752  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
753  frame=lsstDebug.frame)
754  lsstDebug.frame += 1
755 
756  if display and showDipoles:
757  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
758  frame=lsstDebug.frame)
759  lsstDebug.frame += 1
760 
761  def _getConfigName(self):
762  """Return the name of the config dataset
763  """
764  return "%sDiff_config" % (self.config.coaddName,)
765 
766  def _getMetadataName(self):
767  """Return the name of the metadata dataset
768  """
769  return "%sDiff_metadata" % (self.config.coaddName,)
770 
771  def getSchemaCatalogs(self):
772  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
773  diaSrc = afwTable.SourceCatalog(self.schema)
774  diaSrc.getTable().setMetadata(self.algMetadata)
775  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
776 
777  @classmethod
778  def _makeArgumentParser(cls):
779  """Create an argument parser
780  """
781  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
782  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
783  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
784  help="Optional template data ID (visit only), e.g. --templateId visit=6789")
785  return parser
786 
787 
789  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
790  doc="Shift stars going into RegisterTask by this amount")
791  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
792  doc="Perturb stars going into RegisterTask by this amount")
793 
794  def setDefaults(self):
795  ImageDifferenceConfig.setDefaults(self)
796  self.getTemplate.retarget(GetCalexpAsTemplateTask)
797 
798 
800  """!Image difference Task used in the Winter 2013 data challege.
801  Enables testing the effects of registration shifts and scatter.
802 
803  For use with winter 2013 simulated images:
804  Use --templateId visit=88868666 for sparse data
805  --templateId visit=22222200 for dense data (g)
806  --templateId visit=11111100 for dense data (i)
807  """
808  ConfigClass = Winter2013ImageDifferenceConfig
809  _DefaultName = "winter2013ImageDifference"
810 
811  def __init__(self, **kwargs):
812  ImageDifferenceTask.__init__(self, **kwargs)
813 
814  def fitAstrometry(self, templateSources, templateExposure, selectSources):
815  """Fit the relative astrometry between templateSources and selectSources"""
816  if self.config.winter2013WcsShift > 0.0:
817  offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
818  self.config.winter2013WcsShift)
819  cKey = templateSources[0].getTable().getCentroidKey()
820  for source in templateSources:
821  centroid = source.get(cKey)
822  source.set(cKey, centroid+offset)
823  elif self.config.winter2013WcsRms > 0.0:
824  cKey = templateSources[0].getTable().getCentroidKey()
825  for source in templateSources:
826  offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
827  self.config.winter2013WcsRms*numpy.random.normal())
828  centroid = source.get(cKey)
829  source.set(cKey, centroid+offset)
830 
831  results = self.register.run(templateSources, templateExposure.getWcs(),
832  templateExposure.getBBox(), selectSources)
833  return results
Image difference Task used in the Winter 2013 data challege.
def __init__
Construct an ImageDifference Task.