lsst.pipe.tasks  14.0-53-g87cac093+4
imageDifference.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 from builtins import zip
24 import math
25 import random
26 import numpy
27 
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.daf.base as dafBase
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.math as afwMath
33 import lsst.afw.table as afwTable
34 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
35 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, \
38  ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import DipoleAnalysis, \
40  SourceFlagChecker, KernelCandidateF, makeKernelBasisList, \
41  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, \
42  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask, \
43  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry
44 import lsst.ip.diffim.diffimTools as diffimTools
45 import lsst.ip.diffim.utils as diUtils
46 
47 FwhmPerSigma = 2 * math.sqrt(2 * math.log(2))
48 IqrToSigma = 0.741
49 
50 
51 class ImageDifferenceConfig(pexConfig.Config):
52  """Config for ImageDifferenceTask
53  """
54  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=True,
55  doc="Add background to calexp before processing it. "
56  "Useful as ipDiffim does background matching.")
57  doUseRegister = pexConfig.Field(dtype=bool, default=True,
58  doc="Use image-to-image registration to align template with "
59  "science image")
60  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
61  doc="Writing debugging data for doUseRegister")
62  doSelectSources = pexConfig.Field(dtype=bool, default=True,
63  doc="Select stars to use for kernel fitting")
64  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
65  doc="Select stars of extreme color as part of the control sample")
66  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
67  doc="Select stars that are variable to be part "
68  "of the control sample")
69  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
70  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
71  doc="Convolve science image by its PSF before PSF-matching?")
72  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
73  doc="Use a simple gaussian PSF model for pre-convolution "
74  "(else use fit PSF)? Ignored if doPreConvolve false.")
75  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
76  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
77  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
78  "kernel convolution? If True, also update the diffim PSF.")
79  doMerge = pexConfig.Field(dtype=bool, default=True,
80  doc="Merge positive and negative diaSources with grow radius "
81  "set by growFootprint")
82  doMatchSources = pexConfig.Field(dtype=bool, default=True,
83  doc="Match diaSources with input calexp sources and ref catalog sources")
84  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
85  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
86  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
87  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
88  doc="Write warped and PSF-matched template coadd exposure?")
89  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
90  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
91  doc="Add columns to the source table to hold analysis metrics?")
92 
93  coaddName = pexConfig.Field(
94  doc="coadd name: typically one of deep or goodSeeing",
95  dtype=str,
96  default="deep",
97  )
98  convolveTemplate = pexConfig.Field(
99  doc="Which image gets convolved (default = template)",
100  dtype=bool,
101  default=True
102  )
103  refObjLoader = pexConfig.ConfigurableField(
104  target=LoadAstrometryNetObjectsTask,
105  doc="reference object loader",
106  )
107  astrometer = pexConfig.ConfigurableField(
108  target=AstrometryTask,
109  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
110  )
111  sourceSelector = pexConfig.ConfigurableField(
112  target=ObjectSizeStarSelectorTask,
113  doc="Source selection algorithm",
114  )
115  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
116  decorrelate = pexConfig.ConfigurableField(
117  target=DecorrelateALKernelSpatialTask,
118  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
119  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
120  "default of 5.5).",
121  )
122  doSpatiallyVarying = pexConfig.Field(
123  dtype=bool,
124  default=False,
125  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
126  "image in order to allow for spatial variations"
127  )
128  detection = pexConfig.ConfigurableField(
129  target=SourceDetectionTask,
130  doc="Low-threshold detection for final measurement",
131  )
132  measurement = pexConfig.ConfigurableField(
133  target=DipoleFitTask,
134  doc="Enable updated dipole fitting method",
135  )
136  getTemplate = pexConfig.ConfigurableField(
137  target=GetCoaddAsTemplateTask,
138  doc="Subtask to retrieve template exposure and sources",
139  )
140  controlStepSize = pexConfig.Field(
141  doc="What step size (every Nth one) to select a control sample from the kernelSources",
142  dtype=int,
143  default=5
144  )
145  controlRandomSeed = pexConfig.Field(
146  doc = "Random seed for shuffing the control sample",
147  dtype = int,
148  default = 10
149  )
150  register = pexConfig.ConfigurableField(
151  target=RegisterTask,
152  doc="Task to enable image-to-image image registration (warping)",
153  )
154  kernelSourcesFromRef = pexConfig.Field(
155  doc="Select sources to measure kernel from reference catalog if True, template if false",
156  dtype=bool,
157  default=False
158  )
159  templateSipOrder = pexConfig.Field(dtype=int, default=2,
160  doc="Sip Order for fitting the Template Wcs "
161  "(default is too high, overfitting)")
162 
163  growFootprint = pexConfig.Field(dtype=int, default=2,
164  doc="Grow positive and negative footprints by this amount before merging")
165 
166  diaSourceMatchRadius = pexConfig.Field(dtype=float, default=0.5,
167  doc="Match radius (in arcseconds) "
168  "for DiaSource to Source association")
169 
170  def setDefaults(self):
171  # defaults are OK for catalog and diacatalog
172 
173  self.subtract['al'].kernel.name = "AL"
174  self.subtract['al'].kernel.active.fitForBackground = True
175  self.subtract['al'].kernel.active.spatialKernelOrder = 1
176  self.subtract['al'].kernel.active.spatialBgOrder = 0
177  self.doPreConvolve = False
178  self.doMatchSources = False
179  self.doAddMetrics = False
180  self.doUseRegister = False
181 
182  # DiaSource Detection
183  self.detection.thresholdPolarity = "both"
184  self.detection.thresholdValue = 5.5
185  self.detection.reEstimateBackground = False
186  self.detection.thresholdType = "pixel_stdev"
187 
188  # Add filtered flux measurement, the correct measurement for pre-convolved images.
189  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
190  # To change that you must modify algorithms.names in the task's applyOverrides method,
191  # after the user has set doPreConvolve.
192  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
193 
194  # For shuffling the control sample
195  random.seed(self.controlRandomSeed)
196 
197  def validate(self):
198  pexConfig.Config.validate(self)
199  if self.doMeasurement and not self.doDetection:
200  raise ValueError("Cannot run source measurement without source detection.")
201  if self.doMerge and not self.doDetection:
202  raise ValueError("Cannot run source merging without source detection.")
203  if self.doUseRegister and not self.doSelectSources:
204  raise ValueError("doUseRegister=True and doSelectSources=False. " +
205  "Cannot run RegisterTask without selecting sources.")
206 
207 
208 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
209 
210  @staticmethod
211  def getTargetList(parsedCmd, **kwargs):
212  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
213  **kwargs)
214 
215 
216 class ImageDifferenceTask(pipeBase.CmdLineTask):
217  """Subtract an image from a template and measure the result
218  """
219  ConfigClass = ImageDifferenceConfig
220  RunnerClass = ImageDifferenceTaskRunner
221  _DefaultName = "imageDifference"
222 
223  def __init__(self, butler=None, **kwargs):
224  """!Construct an ImageDifference Task
225 
226  @param[in] butler Butler object to use in constructing reference object loaders
227  """
228  pipeBase.CmdLineTask.__init__(self, **kwargs)
229  self.makeSubtask("getTemplate")
230 
231  self.makeSubtask("subtract")
232 
233  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
234  self.makeSubtask("decorrelate")
235 
236  if self.config.doUseRegister:
237  self.makeSubtask("register")
238  self.schema = afwTable.SourceTable.makeMinimalSchema()
239 
240  if self.config.doSelectSources:
241  self.makeSubtask("sourceSelector", schema=self.schema)
242  self.makeSubtask('refObjLoader', butler=butler)
243  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
244 
245  self.algMetadata = dafBase.PropertyList()
246  if self.config.doDetection:
247  self.makeSubtask("detection", schema=self.schema)
248  if self.config.doMeasurement:
249  self.makeSubtask("measurement", schema=self.schema,
250  algMetadata=self.algMetadata)
251  if self.config.doMatchSources:
252  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
253  self.schema.addField("srcMatchId", "L", "unique id of source match")
254 
255  @pipeBase.timeMethod
256  def run(self, sensorRef, templateIdList=None):
257  """Subtract an image from a template coadd and measure the result
258 
259  Steps include:
260  - warp template coadd to match WCS of image
261  - PSF match image to warped template
262  - subtract image from PSF-matched, warped template
263  - persist difference image
264  - detect sources
265  - measure sources
266 
267  @param sensorRef: sensor-level butler data reference, used for the following data products:
268  Input only:
269  - calexp
270  - psf
271  - ccdExposureId
272  - ccdExposureId_bits
273  - self.config.coaddName + "Coadd_skyMap"
274  - self.config.coaddName + "Coadd"
275  Input or output, depending on config:
276  - self.config.coaddName + "Diff_subtractedExp"
277  Output, depending on config:
278  - self.config.coaddName + "Diff_matchedExp"
279  - self.config.coaddName + "Diff_src"
280 
281  @return pipe_base Struct containing these fields:
282  - subtractedExposure: exposure after subtracting template;
283  the unpersisted version if subtraction not run but detection run
284  None if neither subtraction nor detection run (i.e. nothing useful done)
285  - subtractRes: results of subtraction task; None if subtraction not run
286  - sources: detected and possibly measured sources; None if detection not run
287  """
288  self.log.info("Processing %s" % (sensorRef.dataId))
289 
290  # initialize outputs and some intermediate products
291  subtractedExposure = None
292  subtractRes = None
293  selectSources = None
294  kernelSources = None
295  controlSources = None
296  diaSources = None
297 
298  # We make one IdFactory that will be used by both icSrc and src datasets;
299  # I don't know if this is the way we ultimately want to do things, but at least
300  # this ensures the source IDs are fully unique.
301  expBits = sensorRef.get("ccdExposureId_bits")
302  expId = int(sensorRef.get("ccdExposureId"))
303  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
304 
305  # Retrieve the science image we wish to analyze
306  exposure = sensorRef.get("calexp", immediate=True)
307  if self.config.doAddCalexpBackground:
308  mi = exposure.getMaskedImage()
309  mi += sensorRef.get("calexpBackground").getImage()
310  if not exposure.hasPsf():
311  raise pipeBase.TaskError("Exposure has no psf")
312  sciencePsf = exposure.getPsf()
313 
314  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
315  templateExposure = None # Stitched coadd exposure
316  templateSources = None # Sources on the template image
317  if self.config.doSubtract:
318  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
319  templateExposure = template.exposure
320  templateSources = template.sources
321 
322  if self.config.subtract.name == 'zogy':
323  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
324  doWarping=True,
325  spatiallyVarying=self.config.doSpatiallyVarying,
326  doPreConvolve=self.config.doPreConvolve)
327  subtractedExposure = subtractRes.subtractedExposure
328 
329  elif self.config.subtract.name == 'al':
330  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
331  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
332 
333  # sigma of PSF of template image before warping
334  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
335 
336  # if requested, convolve the science exposure with its PSF
337  # (properly, this should be a cross-correlation, but our code does not yet support that)
338  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
339  # else sigma of original science exposure
340  preConvPsf = None
341  if self.config.doPreConvolve:
342  convControl = afwMath.ConvolutionControl()
343  # cannot convolve in place, so make a new MI to receive convolved image
344  srcMI = exposure.getMaskedImage()
345  destMI = srcMI.Factory(srcMI.getDimensions())
346  srcPsf = sciencePsf
347  if self.config.useGaussianForPreConvolution:
348  # convolve with a simplified PSF model: a double Gaussian
349  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
350  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
351  else:
352  # convolve with science exposure's PSF model
353  preConvPsf = srcPsf
354  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
355  exposure.setMaskedImage(destMI)
356  scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
357  else:
358  scienceSigmaPost = scienceSigmaOrig
359 
360  # If requested, find sources in the image
361  if self.config.doSelectSources:
362  if not sensorRef.datasetExists("src"):
363  self.log.warn("Src product does not exist; running detection, measurement, selection")
364  # Run own detection and measurement; necessary in nightly processing
365  selectSources = self.subtract.getSelectSources(
366  exposure,
367  sigma=scienceSigmaPost,
368  doSmooth=not self.doPreConvolve,
369  idFactory=idFactory,
370  )
371  else:
372  self.log.info("Source selection via src product")
373  # Sources already exist; for data release processing
374  selectSources = sensorRef.get("src")
375 
376  # Number of basis functions
377  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
378  referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
379  targetFwhmPix=templateSigma * FwhmPerSigma))
380 
381  if self.config.doAddMetrics:
382  # Modify the schema of all Sources
383  kcQa = KernelCandidateQa(nparam)
384  selectSources = kcQa.addToSchema(selectSources)
385 
386  if self.config.kernelSourcesFromRef:
387  # match exposure sources to reference catalog
388  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
389  matches = astromRet.matches
390  elif templateSources:
391  # match exposure sources to template sources
392  mc = afwTable.MatchControl()
393  mc.findOnlyClosest = False
394  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
395  mc)
396  else:
397  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
398  "but template sources not available. Cannot match science " +
399  "sources with template sources. Run process* on data from " +
400  "which templates are built.")
401 
402  kernelSources = self.sourceSelector.selectStars(exposure, selectSources,
403  matches=matches).starCat
404 
405  random.shuffle(kernelSources, random.random)
406  controlSources = kernelSources[::self.config.controlStepSize]
407  kernelSources = [k for i, k in enumerate(kernelSources)
408  if i % self.config.controlStepSize]
409 
410  if self.config.doSelectDcrCatalog:
411  redSelector = DiaCatalogSourceSelectorTask(
412  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
413  grMax=99.999))
414  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
415  controlSources.extend(redSources)
416 
417  blueSelector = DiaCatalogSourceSelectorTask(
418  DiaCatalogSourceSelectorConfig(grMin=-99.999,
419  grMax=self.sourceSelector.config.grMin))
420  blueSources = blueSelector.selectStars(exposure, selectSources,
421  matches=matches).starCat
422  controlSources.extend(blueSources)
423 
424  if self.config.doSelectVariableCatalog:
425  varSelector = DiaCatalogSourceSelectorTask(
426  DiaCatalogSourceSelectorConfig(includeVariable=True))
427  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
428  controlSources.extend(varSources)
429 
430  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
431  % (len(kernelSources), len(selectSources), len(controlSources)))
432  allresids = {}
433  if self.config.doUseRegister:
434  self.log.info("Registering images")
435 
436  if templateSources is None:
437  # Run detection on the template, which is
438  # temporarily background-subtracted
439  templateSources = self.subtract.getSelectSources(
440  templateExposure,
441  sigma=templateSigma,
442  doSmooth=True,
443  idFactory=idFactory
444  )
445 
446  # Third step: we need to fit the relative astrometry.
447  #
448  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
449  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
450  exposure.getWcs(), exposure.getBBox())
451  templateExposure = warpedExp
452 
453  # Create debugging outputs on the astrometric
454  # residuals as a function of position. Persistence
455  # not yet implemented; expected on (I believe) #2636.
456  if self.config.doDebugRegister:
457  # Grab matches to reference catalog
458  srcToMatch = {x.second.getId(): x.first for x in matches}
459 
460  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
461  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
462  sids = [m.first.getId() for m in wcsResults.matches]
463  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
464  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
465  m.second.get(inCentroidKey))) for m in wcsResults.matches]
466  allresids = dict(zip(sids, zip(positions, residuals)))
467 
468  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
469  wcsResults.wcs.pixelToSky(
470  m.second.get(inCentroidKey))) for m in wcsResults.matches]
471  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
472  2.5*numpy.log10(srcToMatch[x].get("r"))
473  for x in sids if x in srcToMatch.keys()])
474  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
475  if s in srcToMatch.keys()])
476  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
477  if s in srcToMatch.keys()])
478  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
479  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
480  (colors <= self.sourceSelector.config.grMax))
481  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
482  rms1Long = IqrToSigma * \
483  (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
484  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
485  numpy.percentile(dlat[idx1], 25))
486  rms2Long = IqrToSigma * \
487  (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
488  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
489  numpy.percentile(dlat[idx2], 25))
490  rms3Long = IqrToSigma * \
491  (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
492  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
493  numpy.percentile(dlat[idx3], 25))
494  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
495  (numpy.median(dlong[idx1]), rms1Long,
496  numpy.median(dlat[idx1]), rms1Lat))
497  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
498  (numpy.median(dlong[idx2]), rms2Long,
499  numpy.median(dlat[idx2]), rms2Lat))
500  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
501  (numpy.median(dlong[idx3]), rms3Long,
502  numpy.median(dlat[idx3]), rms3Lat))
503 
504  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
505  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
506  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
507  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
508  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
509  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
510 
511  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
512  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
513  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
514  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
515  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
516  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
517 
518  # warp template exposure to match exposure,
519  # PSF match template exposure to exposure,
520  # then return the difference
521 
522  # Return warped template... Construct sourceKernelCand list after subtract
523  self.log.info("Subtracting images")
524  subtractRes = self.subtract.subtractExposures(
525  templateExposure=templateExposure,
526  scienceExposure=exposure,
527  candidateList=kernelSources,
528  convolveTemplate=self.config.convolveTemplate,
529  doWarping=not self.config.doUseRegister
530  )
531  subtractedExposure = subtractRes.subtractedExposure
532 
533  if self.config.doWriteMatchedExp:
534  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
535 
536  if self.config.doDetection:
537  self.log.info("Computing diffim PSF")
538  if subtractedExposure is None:
539  subtractedExposure = sensorRef.get(subtractedExposureName)
540 
541  # Get Psf from the appropriate input image if it doesn't exist
542  if not subtractedExposure.hasPsf():
543  if self.config.convolveTemplate:
544  subtractedExposure.setPsf(exposure.getPsf())
545  else:
546  if templateExposure is None:
547  template = self.getTemplate.run(exposure, sensorRef,
548  templateIdList=templateIdList)
549  subtractedExposure.setPsf(template.exposure.getPsf())
550 
551  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
552  # thus it may have already been decorrelated. Thus, we do not decorrelate if
553  # doSubtract is False.
554  if self.config.doDecorrelation and self.config.doSubtract:
555  preConvKernel = None
556  if preConvPsf is not None:
557  preConvKernel = preConvPsf.getLocalKernel()
558  decorrResult = self.decorrelate.run(exposure, templateExposure,
559  subtractedExposure,
560  subtractRes.psfMatchingKernel,
561  spatiallyVarying=self.config.doSpatiallyVarying,
562  preConvKernel=preConvKernel)
563  subtractedExposure = decorrResult.correctedExposure
564 
565  # END (if subtractAlgorithm == 'AL')
566 
567  if self.config.doDetection:
568  self.log.info("Running diaSource detection")
569  # Erase existing detection mask planes
570  mask = subtractedExposure.getMaskedImage().getMask()
571  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
572 
573  table = afwTable.SourceTable.make(self.schema, idFactory)
574  table.setMetadata(self.algMetadata)
575  results = self.detection.makeSourceCatalog(
576  table=table,
577  exposure=subtractedExposure,
578  doSmooth=not self.config.doPreConvolve
579  )
580 
581  if self.config.doMerge:
582  fpSet = results.fpSets.positive
583  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
584  self.config.growFootprint, False)
585  diaSources = afwTable.SourceCatalog(table)
586  fpSet.makeSources(diaSources)
587  self.log.info("Merging detections into %d sources" % (len(diaSources)))
588  else:
589  diaSources = results.sources
590 
591  if self.config.doMeasurement:
592  newDipoleFitting = self.config.doDipoleFitting
593  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
594  if not newDipoleFitting:
595  # Just fit dipole in diffim
596  self.measurement.run(diaSources, subtractedExposure)
597  else:
598  # Use (matched) template and science image (if avail.) to constrain dipole fitting
599  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
600  self.measurement.run(diaSources, subtractedExposure, exposure,
601  subtractRes.matchedExposure)
602  else:
603  self.measurement.run(diaSources, subtractedExposure, exposure)
604 
605  # Match with the calexp sources if possible
606  if self.config.doMatchSources:
607  if sensorRef.datasetExists("src"):
608  # Create key,val pair where key=diaSourceId and val=sourceId
609  matchRadAsec = self.config.diaSourceMatchRadius
610  matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
611 
612  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
613  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
614  srcMatch in srcMatches])
615  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
616  len(diaSources)))
617  else:
618  self.log.warn("Src product does not exist; cannot match with diaSources")
619  srcMatchDict = {}
620 
621  # Create key,val pair where key=diaSourceId and val=refId
622  refAstromConfig = AstrometryConfig()
623  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
624  refAstrometer = AstrometryTask(refAstromConfig)
625  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
626  refMatches = astromRet.matches
627  if refMatches is None:
628  self.log.warn("No diaSource matches with reference catalog")
629  refMatchDict = {}
630  else:
631  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
632  len(diaSources)))
633  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
634  refMatch in refMatches])
635 
636  # Assign source Ids
637  for diaSource in diaSources:
638  sid = diaSource.getId()
639  if sid in srcMatchDict:
640  diaSource.set("srcMatchId", srcMatchDict[sid])
641  if sid in refMatchDict:
642  diaSource.set("refMatchId", refMatchDict[sid])
643 
644  if diaSources is not None and self.config.doWriteSources:
645  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
646 
647  if self.config.doAddMetrics and self.config.doSelectSources:
648  self.log.info("Evaluating metrics and control sample")
649 
650  kernelCandList = []
651  for cell in subtractRes.kernelCellSet.getCellList():
652  for cand in cell.begin(False): # include bad candidates
653  kernelCandList.append(cand)
654 
655  # Get basis list to build control sample kernels
656  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
657 
658  controlCandList = \
659  diffimTools.sourceTableToCandidateList(controlSources,
660  subtractRes.warpedExposure, exposure,
661  self.config.subtract.kernel.active,
662  self.config.subtract.kernel.active.detectionConfig,
663  self.log, doBuild=True, basisList=basisList)
664 
665  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
666  dof=nparam)
667  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
668 
669  if self.config.doDetection:
670  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
671  else:
672  kcQa.aggregate(selectSources, self.metadata, allresids)
673 
674  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
675 
676  if self.config.doWriteSubtractedExp:
677  sensorRef.put(subtractedExposure, subtractedExposureName)
678 
679  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
680  return pipeBase.Struct(
681  subtractedExposure=subtractedExposure,
682  subtractRes=subtractRes,
683  sources=diaSources,
684  )
685 
686  def fitAstrometry(self, templateSources, templateExposure, selectSources):
687  """Fit the relative astrometry between templateSources and selectSources
688 
689  @todo remove this method. It originally fit a new WCS to the template before calling register.run
690  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
691  It remains because a subtask overrides it.
692  """
693  results = self.register.run(templateSources, templateExposure.getWcs(),
694  templateExposure.getBBox(), selectSources)
695  return results
696 
697  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
698  """@todo Test and update for current debug display and slot names
699  """
700  import lsstDebug
701  display = lsstDebug.Info(__name__).display
702  showSubtracted = lsstDebug.Info(__name__).showSubtracted
703  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
704  showDiaSources = lsstDebug.Info(__name__).showDiaSources
705  showDipoles = lsstDebug.Info(__name__).showDipoles
706  maskTransparency = lsstDebug.Info(__name__).maskTransparency
707  if display:
708  import lsst.afw.display.ds9 as ds9
709  if not maskTransparency:
710  maskTransparency = 0
711  ds9.setMaskTransparency(maskTransparency)
712 
713  if display and showSubtracted:
714  ds9.mtv(subtractRes.subtractedExposure, frame=lsstDebug.frame, title="Subtracted image")
715  mi = subtractRes.subtractedExposure.getMaskedImage()
716  x0, y0 = mi.getX0(), mi.getY0()
717  with ds9.Buffering():
718  for s in diaSources:
719  x, y = s.getX() - x0, s.getY() - y0
720  ctype = "red" if s.get("flags.negative") else "yellow"
721  if (s.get("flags.pixel.interpolated.center") or s.get("flags.pixel.saturated.center") or
722  s.get("flags.pixel.cr.center")):
723  ptype = "x"
724  elif (s.get("flags.pixel.interpolated.any") or s.get("flags.pixel.saturated.any") or
725  s.get("flags.pixel.cr.any")):
726  ptype = "+"
727  else:
728  ptype = "o"
729  ds9.dot(ptype, x, y, size=4, frame=lsstDebug.frame, ctype=ctype)
730  lsstDebug.frame += 1
731 
732  if display and showPixelResiduals and selectSources:
733  nonKernelSources = []
734  for source in selectSources:
735  if source not in kernelSources:
736  nonKernelSources.append(source)
737 
738  diUtils.plotPixelResiduals(exposure,
739  subtractRes.warpedExposure,
740  subtractRes.subtractedExposure,
741  subtractRes.kernelCellSet,
742  subtractRes.psfMatchingKernel,
743  subtractRes.backgroundModel,
744  nonKernelSources,
745  self.subtract.config.kernel.active.detectionConfig,
746  origVariance=False)
747  diUtils.plotPixelResiduals(exposure,
748  subtractRes.warpedExposure,
749  subtractRes.subtractedExposure,
750  subtractRes.kernelCellSet,
751  subtractRes.psfMatchingKernel,
752  subtractRes.backgroundModel,
753  nonKernelSources,
754  self.subtract.config.kernel.active.detectionConfig,
755  origVariance=True)
756  if display and showDiaSources:
757  flagChecker = SourceFlagChecker(diaSources)
758  isFlagged = [flagChecker(x) for x in diaSources]
759  isDipole = [x.get("classification.dipole") for x in diaSources]
760  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
761  frame=lsstDebug.frame)
762  lsstDebug.frame += 1
763 
764  if display and showDipoles:
765  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
766  frame=lsstDebug.frame)
767  lsstDebug.frame += 1
768 
769  def _getConfigName(self):
770  """Return the name of the config dataset
771  """
772  return "%sDiff_config" % (self.config.coaddName,)
773 
774  def _getMetadataName(self):
775  """Return the name of the metadata dataset
776  """
777  return "%sDiff_metadata" % (self.config.coaddName,)
778 
779  def getSchemaCatalogs(self):
780  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
781  diaSrc = afwTable.SourceCatalog(self.schema)
782  diaSrc.getTable().setMetadata(self.algMetadata)
783  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
784 
785  @classmethod
786  def _makeArgumentParser(cls):
787  """Create an argument parser
788  """
789  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
790  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
791  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
792  help="Optional template data ID (visit only), e.g. --templateId visit=6789")
793  return parser
794 
795 
797  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
798  doc="Shift stars going into RegisterTask by this amount")
799  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
800  doc="Perturb stars going into RegisterTask by this amount")
801 
802  def setDefaults(self):
803  ImageDifferenceConfig.setDefaults(self)
804  self.getTemplate.retarget(GetCalexpAsTemplateTask)
805 
806 
808  """!Image difference Task used in the Winter 2013 data challege.
809  Enables testing the effects of registration shifts and scatter.
810 
811  For use with winter 2013 simulated images:
812  Use --templateId visit=88868666 for sparse data
813  --templateId visit=22222200 for dense data (g)
814  --templateId visit=11111100 for dense data (i)
815  """
816  ConfigClass = Winter2013ImageDifferenceConfig
817  _DefaultName = "winter2013ImageDifference"
818 
819  def __init__(self, **kwargs):
820  ImageDifferenceTask.__init__(self, **kwargs)
821 
822  def fitAstrometry(self, templateSources, templateExposure, selectSources):
823  """Fit the relative astrometry between templateSources and selectSources"""
824  if self.config.winter2013WcsShift > 0.0:
825  offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
826  self.config.winter2013WcsShift)
827  cKey = templateSources[0].getTable().getCentroidKey()
828  for source in templateSources:
829  centroid = source.get(cKey)
830  source.set(cKey, centroid+offset)
831  elif self.config.winter2013WcsRms > 0.0:
832  cKey = templateSources[0].getTable().getCentroidKey()
833  for source in templateSources:
834  offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
835  self.config.winter2013WcsRms*numpy.random.normal())
836  centroid = source.get(cKey)
837  source.set(cKey, centroid+offset)
838 
839  results = self.register.run(templateSources, templateExposure.getWcs(),
840  templateExposure.getBBox(), selectSources)
841  return results
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def run(self, sensorRef, templateIdList=None)
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)