lsst.pipe.tasks  13.0-66-gfbf2f2ce+5
imageDifference.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 from builtins import zip
24 import math
25 import random
26 import numpy
27 
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.daf.base as dafBase
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.math as afwMath
33 import lsst.afw.table as afwTable
34 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
35 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, \
38  ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import DipoleAnalysis, \
40  SourceFlagChecker, KernelCandidateF, makeKernelBasisList, \
41  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, \
42  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask, \
43  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry
44 import lsst.ip.diffim.diffimTools as diffimTools
45 import lsst.ip.diffim.utils as diUtils
46 
47 FwhmPerSigma = 2 * math.sqrt(2 * math.log(2))
48 IqrToSigma = 0.741
49 
50 
51 class ImageDifferenceConfig(pexConfig.Config):
52  """Config for ImageDifferenceTask
53  """
54  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=True,
55  doc="Add background to calexp before processing it. "
56  "Useful as ipDiffim does background matching.")
57  doUseRegister = pexConfig.Field(dtype=bool, default=True,
58  doc="Use image-to-image registration to align template with "
59  "science image")
60  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
61  doc="Writing debugging data for doUseRegister")
62  doSelectSources = pexConfig.Field(dtype=bool, default=True,
63  doc="Select stars to use for kernel fitting")
64  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
65  doc="Select stars of extreme color as part of the control sample")
66  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
67  doc="Select stars that are variable to be part "
68  "of the control sample")
69  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
70  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
71  doc="Convolve science image by its PSF before PSF-matching?")
72  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
73  doc="Use a simple gaussian PSF model for pre-convolution "
74  "(else use fit PSF)? Ignored if doPreConvolve false.")
75  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
76  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
77  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
78  "kernel convolution? If True, also update the diffim PSF.")
79  doMerge = pexConfig.Field(dtype=bool, default=True,
80  doc="Merge positive and negative diaSources with grow radius "
81  "set by growFootprint")
82  doMatchSources = pexConfig.Field(dtype=bool, default=True,
83  doc="Match diaSources with input calexp sources and ref catalog sources")
84  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
85  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
86  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
87  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
88  doc="Write warped and PSF-matched template coadd exposure?")
89  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
90  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
91  doc="Add columns to the source table to hold analysis metrics?")
92 
93  coaddName = pexConfig.Field(
94  doc="coadd name: typically one of deep or goodSeeing",
95  dtype=str,
96  default="deep",
97  )
98  convolveTemplate = pexConfig.Field(
99  doc="Which image gets convolved (default = template)",
100  dtype=bool,
101  default=True
102  )
103  refObjLoader = pexConfig.ConfigurableField(
104  target=LoadAstrometryNetObjectsTask,
105  doc="reference object loader",
106  )
107  astrometer = pexConfig.ConfigurableField(
108  target=AstrometryTask,
109  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
110  )
111  sourceSelector = pexConfig.ConfigurableField(
112  target=ObjectSizeStarSelectorTask,
113  doc="Source selection algorithm",
114  )
115  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
116  decorrelate = pexConfig.ConfigurableField(
117  target=DecorrelateALKernelSpatialTask,
118  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
119  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
120  "default of 5.5).",
121  )
122  doSpatiallyVarying = pexConfig.Field(
123  dtype=bool,
124  default=False,
125  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
126  "image in order to allow for spatial variations"
127  )
128  detection = pexConfig.ConfigurableField(
129  target=SourceDetectionTask,
130  doc="Low-threshold detection for final measurement",
131  )
132  measurement = pexConfig.ConfigurableField(
133  target=DipoleFitTask,
134  doc="Enable updated dipole fitting method",
135  )
136  getTemplate = pexConfig.ConfigurableField(
137  target=GetCoaddAsTemplateTask,
138  doc="Subtask to retrieve template exposure and sources",
139  )
140  controlStepSize = pexConfig.Field(
141  doc="What step size (every Nth one) to select a control sample from the kernelSources",
142  dtype=int,
143  default=5
144  )
145  controlRandomSeed = pexConfig.Field(
146  doc = "Random seed for shuffing the control sample",
147  dtype = int,
148  default = 10
149  )
150  register = pexConfig.ConfigurableField(
151  target=RegisterTask,
152  doc="Task to enable image-to-image image registration (warping)",
153  )
154  kernelSourcesFromRef = pexConfig.Field(
155  doc="Select sources to measure kernel from reference catalog if True, template if false",
156  dtype=bool,
157  default=False
158  )
159  templateSipOrder = pexConfig.Field(dtype=int, default=2,
160  doc="Sip Order for fitting the Template Wcs "
161  "(default is too high, overfitting)")
162 
163  growFootprint = pexConfig.Field(dtype=int, default=2,
164  doc="Grow positive and negative footprints by this amount before merging")
165 
166  diaSourceMatchRadius = pexConfig.Field(dtype=float, default=0.5,
167  doc="Match radius (in arcseconds) "
168  "for DiaSource to Source association")
169 
170  def setDefaults(self):
171  # defaults are OK for catalog and diacatalog
172 
173  self.subtract['al'].kernel.name = "AL"
174  self.subtract['al'].kernel.active.fitForBackground = True
175  self.subtract['al'].kernel.active.spatialKernelOrder = 1
176  self.subtract['al'].kernel.active.spatialBgOrder = 0
177  self.doPreConvolve = False
178  self.doMatchSources = False
179  self.doAddMetrics = False
180  self.doUseRegister = False
181 
182  # DiaSource Detection
183  self.detection.thresholdPolarity = "both"
184  self.detection.thresholdValue = 5.5
185  self.detection.reEstimateBackground = False
186  self.detection.thresholdType = "pixel_stdev"
187 
188  # Add filtered flux measurement, the correct measurement for pre-convolved images.
189  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
190  # To change that you must modify algorithms.names in the task's applyOverrides method,
191  # after the user has set doPreConvolve.
192  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
193 
194  # For shuffling the control sample
195  random.seed(self.controlRandomSeed)
196 
197  def validate(self):
198  pexConfig.Config.validate(self)
199  if self.doMeasurement and not self.doDetection:
200  raise ValueError("Cannot run source measurement without source detection.")
201  if self.doMerge and not self.doDetection:
202  raise ValueError("Cannot run source merging without source detection.")
203  if self.doUseRegister and not self.doSelectSources:
204  raise ValueError("doUseRegister=True and doSelectSources=False. " +
205  "Cannot run RegisterTask without selecting sources.")
206 
207 
208 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
209 
210  @staticmethod
211  def getTargetList(parsedCmd, **kwargs):
212  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
213  **kwargs)
214 
215 
216 class ImageDifferenceTask(pipeBase.CmdLineTask):
217  """Subtract an image from a template and measure the result
218  """
219  ConfigClass = ImageDifferenceConfig
220  RunnerClass = ImageDifferenceTaskRunner
221  _DefaultName = "imageDifference"
222 
223  def __init__(self, butler=None, **kwargs):
224  """!Construct an ImageDifference Task
225 
226  @param[in] butler Butler object to use in constructing reference object loaders
227  """
228  pipeBase.CmdLineTask.__init__(self, **kwargs)
229  self.makeSubtask("getTemplate")
230 
231  self.makeSubtask("subtract")
232 
233  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
234  self.makeSubtask("decorrelate")
235 
236  if self.config.doUseRegister:
237  self.makeSubtask("register")
238  self.schema = afwTable.SourceTable.makeMinimalSchema()
239 
240  if self.config.doSelectSources:
241  self.makeSubtask("sourceSelector", schema=self.schema)
242  self.makeSubtask('refObjLoader', butler=butler)
243  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
244 
245  self.algMetadata = dafBase.PropertyList()
246  if self.config.doDetection:
247  self.makeSubtask("detection", schema=self.schema)
248  if self.config.doMeasurement:
249  self.makeSubtask("measurement", schema=self.schema,
250  algMetadata=self.algMetadata)
251  if self.config.doMatchSources:
252  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
253  self.schema.addField("srcMatchId", "L", "unique id of source match")
254 
255  @pipeBase.timeMethod
256  def run(self, sensorRef, templateIdList=None):
257  """Subtract an image from a template coadd and measure the result
258 
259  Steps include:
260  - warp template coadd to match WCS of image
261  - PSF match image to warped template
262  - subtract image from PSF-matched, warped template
263  - persist difference image
264  - detect sources
265  - measure sources
266 
267  @param sensorRef: sensor-level butler data reference, used for the following data products:
268  Input only:
269  - calexp
270  - psf
271  - ccdExposureId
272  - ccdExposureId_bits
273  - self.config.coaddName + "Coadd_skyMap"
274  - self.config.coaddName + "Coadd"
275  Input or output, depending on config:
276  - self.config.coaddName + "Diff_subtractedExp"
277  Output, depending on config:
278  - self.config.coaddName + "Diff_matchedExp"
279  - self.config.coaddName + "Diff_src"
280 
281  @return pipe_base Struct containing these fields:
282  - subtractedExposure: exposure after subtracting template;
283  the unpersisted version if subtraction not run but detection run
284  None if neither subtraction nor detection run (i.e. nothing useful done)
285  - subtractRes: results of subtraction task; None if subtraction not run
286  - sources: detected and possibly measured sources; None if detection not run
287  """
288  self.log.info("Processing %s" % (sensorRef.dataId))
289 
290  # initialize outputs and some intermediate products
291  subtractedExposure = None
292  subtractRes = None
293  selectSources = None
294  kernelSources = None
295  controlSources = None
296  diaSources = None
297 
298  # We make one IdFactory that will be used by both icSrc and src datasets;
299  # I don't know if this is the way we ultimately want to do things, but at least
300  # this ensures the source IDs are fully unique.
301  expBits = sensorRef.get("ccdExposureId_bits")
302  expId = int(sensorRef.get("ccdExposureId"))
303  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
304 
305  # Retrieve the science image we wish to analyze
306  exposure = sensorRef.get("calexp", immediate=True)
307  if self.config.doAddCalexpBackground:
308  mi = exposure.getMaskedImage()
309  mi += sensorRef.get("calexpBackground").getImage()
310  if not exposure.hasPsf():
311  raise pipeBase.TaskError("Exposure has no psf")
312  sciencePsf = exposure.getPsf()
313 
314  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
315  templateExposure = None # Stitched coadd exposure
316  templateSources = None # Sources on the template image
317  if self.config.doSubtract:
318  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
319  templateExposure = template.exposure
320  templateSources = template.sources
321 
322  if self.config.subtract.name == 'zogy':
323  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
324  doWarping=True,
325  spatiallyVarying=self.config.doSpatiallyVarying,
326  doPreConvolve=self.config.doPreConvolve)
327  subtractedExposure = subtractRes.subtractedExposure
328 
329  elif self.config.subtract.name == 'al':
330  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
331  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
332 
333  # sigma of PSF of template image before warping
334  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
335 
336  # if requested, convolve the science exposure with its PSF
337  # (properly, this should be a cross-correlation, but our code does not yet support that)
338  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
339  # else sigma of original science exposure
340  preConvPsf = None
341  if self.config.doPreConvolve:
342  convControl = afwMath.ConvolutionControl()
343  # cannot convolve in place, so make a new MI to receive convolved image
344  srcMI = exposure.getMaskedImage()
345  destMI = srcMI.Factory(srcMI.getDimensions())
346  srcPsf = sciencePsf
347  if self.config.useGaussianForPreConvolution:
348  # convolve with a simplified PSF model: a double Gaussian
349  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
350  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
351  else:
352  # convolve with science exposure's PSF model
353  preConvPsf = srcPsf
354  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
355  exposure.setMaskedImage(destMI)
356  scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
357  else:
358  scienceSigmaPost = scienceSigmaOrig
359 
360  # If requested, find sources in the image
361  if self.config.doSelectSources:
362  if not sensorRef.datasetExists("src"):
363  self.log.warn("Src product does not exist; running detection, measurement, selection")
364  # Run own detection and measurement; necessary in nightly processing
365  selectSources = self.subtract.getSelectSources(
366  exposure,
367  sigma=scienceSigmaPost,
368  doSmooth=not self.doPreConvolve,
369  idFactory=idFactory,
370  )
371  else:
372  self.log.info("Source selection via src product")
373  # Sources already exist; for data release processing
374  selectSources = sensorRef.get("src")
375 
376  # Number of basis functions
377  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
378  referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
379  targetFwhmPix=templateSigma * FwhmPerSigma))
380 
381  if self.config.doAddMetrics:
382  # Modify the schema of all Sources
383  kcQa = KernelCandidateQa(nparam)
384  selectSources = kcQa.addToSchema(selectSources)
385 
386  if self.config.kernelSourcesFromRef:
387  # match exposure sources to reference catalog
388  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
389  matches = astromRet.matches
390  elif templateSources:
391  # match exposure sources to template sources
392  mc = afwTable.MatchControl()
393  mc.findOnlyClosest = False
394  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
395  mc)
396  else:
397  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
398  "but template sources not available. Cannot match science " +
399  "sources with template sources. Run process* on data from " +
400  "which templates are built.")
401 
402  kernelSources = self.sourceSelector.selectStars(exposure, selectSources,
403  matches=matches).starCat
404 
405  random.shuffle(kernelSources, random.random)
406  controlSources = kernelSources[::self.config.controlStepSize]
407  kernelSources = [k for i, k in enumerate(kernelSources) if i % self.config.controlStepSize]
408 
409  if self.config.doSelectDcrCatalog:
410  redSelector = DiaCatalogSourceSelectorTask(
411  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999))
412  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
413  controlSources.extend(redSources)
414 
415  blueSelector = DiaCatalogSourceSelectorTask(
416  DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin))
417  blueSources = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat
418  controlSources.extend(blueSources)
419 
420  if self.config.doSelectVariableCatalog:
421  varSelector = DiaCatalogSourceSelectorTask(
422  DiaCatalogSourceSelectorConfig(includeVariable=True))
423  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
424  controlSources.extend(varSources)
425 
426  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
427  % (len(kernelSources), len(selectSources), len(controlSources)))
428  allresids = {}
429  if self.config.doUseRegister:
430  self.log.info("Registering images")
431 
432  if templateSources is None:
433  # Run detection on the template, which is
434  # temporarily background-subtracted
435  templateSources = self.subtract.getSelectSources(
436  templateExposure,
437  sigma=templateSigma,
438  doSmooth=True,
439  idFactory=idFactory
440  )
441 
442  # Third step: we need to fit the relative astrometry.
443  #
444  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
445  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
446  exposure.getWcs(), exposure.getBBox())
447  templateExposure = warpedExp
448 
449  # Create debugging outputs on the astrometric
450  # residuals as a function of position. Persistence
451  # not yet implemented; expected on (I believe) #2636.
452  if self.config.doDebugRegister:
453  # Grab matches to reference catalog
454  srcToMatch = {x.second.getId(): x.first for x in matches}
455 
456  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
457  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
458  sids = [m.first.getId() for m in wcsResults.matches]
459  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
460  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
461  m.second.get(inCentroidKey))) for m in wcsResults.matches]
462  allresids = dict(zip(sids, zip(positions, residuals)))
463 
464  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
465  wcsResults.wcs.pixelToSky(
466  m.second.get(inCentroidKey))) for m in wcsResults.matches]
467  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
468  2.5*numpy.log10(srcToMatch[x].get("r"))
469  for x in sids if x in srcToMatch.keys()])
470  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
471  if s in srcToMatch.keys()])
472  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
473  if s in srcToMatch.keys()])
474  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
475  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
476  (colors <= self.sourceSelector.config.grMax))
477  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
478  rms1Long = IqrToSigma * \
479  (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
480  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)-numpy.percentile(dlat[idx1], 25))
481  rms2Long = IqrToSigma * \
482  (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
483  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)-numpy.percentile(dlat[idx2], 25))
484  rms3Long = IqrToSigma * \
485  (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
486  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)-numpy.percentile(dlat[idx3], 25))
487  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]),
488  rms1Long,
489  numpy.median(dlat[idx1]),
490  rms1Lat))
491  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]),
492  rms2Long,
493  numpy.median(dlat[idx2]),
494  rms2Lat))
495  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]),
496  rms3Long,
497  numpy.median(dlat[idx3]),
498  rms3Lat))
499 
500  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
501  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
502  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
503  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
504  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
505  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
506 
507  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
508  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
509  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
510  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
511  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
512  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
513 
514  # warp template exposure to match exposure,
515  # PSF match template exposure to exposure,
516  # then return the difference
517 
518  # Return warped template... Construct sourceKernelCand list after subtract
519  self.log.info("Subtracting images")
520  subtractRes = self.subtract.subtractExposures(
521  templateExposure=templateExposure,
522  scienceExposure=exposure,
523  candidateList=kernelSources,
524  convolveTemplate=self.config.convolveTemplate,
525  doWarping=not self.config.doUseRegister
526  )
527  subtractedExposure = subtractRes.subtractedExposure
528 
529  if self.config.doWriteMatchedExp:
530  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
531 
532  if self.config.doDetection:
533  self.log.info("Computing diffim PSF")
534  if subtractedExposure is None:
535  subtractedExposure = sensorRef.get(subtractedExposureName)
536 
537  # Get Psf from the appropriate input image if it doesn't exist
538  if not subtractedExposure.hasPsf():
539  if self.config.convolveTemplate:
540  subtractedExposure.setPsf(exposure.getPsf())
541  else:
542  if templateExposure is None:
543  template = self.getTemplate.run(exposure, sensorRef,
544  templateIdList=templateIdList)
545  subtractedExposure.setPsf(template.exposure.getPsf())
546 
547  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
548  # thus it may have already been decorrelated. Thus, we do not decorrelate if
549  # doSubtract is False.
550  if self.config.doDecorrelation and self.config.doSubtract:
551  preConvKernel = None
552  if preConvPsf is not None:
553  preConvKernel = preConvPsf.getLocalKernel()
554  decorrResult = self.decorrelate.run(exposure, templateExposure,
555  subtractedExposure,
556  subtractRes.psfMatchingKernel,
557  spatiallyVarying=self.config.doSpatiallyVarying,
558  preConvKernel=preConvKernel)
559  subtractedExposure = decorrResult.correctedExposure
560 
561  # END (if subtractAlgorithm == 'AL')
562 
563  if self.config.doDetection:
564  self.log.info("Running diaSource detection")
565  # Erase existing detection mask planes
566  mask = subtractedExposure.getMaskedImage().getMask()
567  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
568 
569  table = afwTable.SourceTable.make(self.schema, idFactory)
570  table.setMetadata(self.algMetadata)
571  results = self.detection.makeSourceCatalog(
572  table=table,
573  exposure=subtractedExposure,
574  doSmooth=not self.config.doPreConvolve
575  )
576 
577  if self.config.doMerge:
578  fpSet = results.fpSets.positive
579  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
580  self.config.growFootprint, False)
581  diaSources = afwTable.SourceCatalog(table)
582  fpSet.makeSources(diaSources)
583  self.log.info("Merging detections into %d sources" % (len(diaSources)))
584  else:
585  diaSources = results.sources
586 
587  if self.config.doMeasurement:
588  newDipoleFitting = self.config.doDipoleFitting
589  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
590  if not newDipoleFitting:
591  # Just fit dipole in diffim
592  self.measurement.run(diaSources, subtractedExposure)
593  else:
594  # Use (matched) template and science image (if avail.) to constrain dipole fitting
595  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
596  self.measurement.run(diaSources, subtractedExposure, exposure,
597  subtractRes.matchedExposure)
598  else:
599  self.measurement.run(diaSources, subtractedExposure, exposure)
600 
601  # Match with the calexp sources if possible
602  if self.config.doMatchSources:
603  if sensorRef.datasetExists("src"):
604  # Create key,val pair where key=diaSourceId and val=sourceId
605  matchRadAsec = self.config.diaSourceMatchRadius
606  matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
607 
608  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
609  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
610  srcMatch in srcMatches])
611  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
612  len(diaSources)))
613  else:
614  self.log.warn("Src product does not exist; cannot match with diaSources")
615  srcMatchDict = {}
616 
617  # Create key,val pair where key=diaSourceId and val=refId
618  refAstromConfig = AstrometryConfig()
619  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
620  refAstrometer = AstrometryTask(refAstromConfig)
621  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
622  refMatches = astromRet.matches
623  if refMatches is None:
624  self.log.warn("No diaSource matches with reference catalog")
625  refMatchDict = {}
626  else:
627  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
628  len(diaSources)))
629  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
630  refMatch in refMatches])
631 
632  # Assign source Ids
633  for diaSource in diaSources:
634  sid = diaSource.getId()
635  if sid in srcMatchDict:
636  diaSource.set("srcMatchId", srcMatchDict[sid])
637  if sid in refMatchDict:
638  diaSource.set("refMatchId", refMatchDict[sid])
639 
640  if diaSources is not None and self.config.doWriteSources:
641  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
642 
643  if self.config.doAddMetrics and self.config.doSelectSources:
644  self.log.info("Evaluating metrics and control sample")
645 
646  kernelCandList = []
647  for cell in subtractRes.kernelCellSet.getCellList():
648  for cand in cell.begin(False): # include bad candidates
649  kernelCandList.append(cand)
650 
651  # Get basis list to build control sample kernels
652  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
653 
654  controlCandList = \
655  diffimTools.sourceTableToCandidateList(controlSources,
656  subtractRes.warpedExposure, exposure,
657  self.config.subtract.kernel.active,
658  self.config.subtract.kernel.active.detectionConfig,
659  self.log, doBuild=True, basisList=basisList)
660 
661  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
662  dof=nparam)
663  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
664 
665  if self.config.doDetection:
666  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
667  else:
668  kcQa.aggregate(selectSources, self.metadata, allresids)
669 
670  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
671 
672  if self.config.doWriteSubtractedExp:
673  sensorRef.put(subtractedExposure, subtractedExposureName)
674 
675  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
676  return pipeBase.Struct(
677  subtractedExposure=subtractedExposure,
678  subtractRes=subtractRes,
679  sources=diaSources,
680  )
681 
682  def fitAstrometry(self, templateSources, templateExposure, selectSources):
683  """Fit the relative astrometry between templateSources and selectSources
684 
685  @todo remove this method. It originally fit a new WCS to the template before calling register.run
686  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
687  It remains because a subtask overrides it.
688  """
689  results = self.register.run(templateSources, templateExposure.getWcs(),
690  templateExposure.getBBox(), selectSources)
691  return results
692 
693  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
694  """@todo Test and update for current debug display and slot names
695  """
696  import lsstDebug
697  display = lsstDebug.Info(__name__).display
698  showSubtracted = lsstDebug.Info(__name__).showSubtracted
699  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
700  showDiaSources = lsstDebug.Info(__name__).showDiaSources
701  showDipoles = lsstDebug.Info(__name__).showDipoles
702  maskTransparency = lsstDebug.Info(__name__).maskTransparency
703  if display:
704  import lsst.afw.display.ds9 as ds9
705  if not maskTransparency:
706  maskTransparency = 0
707  ds9.setMaskTransparency(maskTransparency)
708 
709  if display and showSubtracted:
710  ds9.mtv(subtractRes.subtractedExposure, frame=lsstDebug.frame, title="Subtracted image")
711  mi = subtractRes.subtractedExposure.getMaskedImage()
712  x0, y0 = mi.getX0(), mi.getY0()
713  with ds9.Buffering():
714  for s in diaSources:
715  x, y = s.getX() - x0, s.getY() - y0
716  ctype = "red" if s.get("flags.negative") else "yellow"
717  if (s.get("flags.pixel.interpolated.center") or s.get("flags.pixel.saturated.center") or
718  s.get("flags.pixel.cr.center")):
719  ptype = "x"
720  elif (s.get("flags.pixel.interpolated.any") or s.get("flags.pixel.saturated.any") or
721  s.get("flags.pixel.cr.any")):
722  ptype = "+"
723  else:
724  ptype = "o"
725  ds9.dot(ptype, x, y, size=4, frame=lsstDebug.frame, ctype=ctype)
726  lsstDebug.frame += 1
727 
728  if display and showPixelResiduals and selectSources:
729  nonKernelSources = []
730  for source in selectSources:
731  if source not in kernelSources:
732  nonKernelSources.append(source)
733 
734  diUtils.plotPixelResiduals(exposure,
735  subtractRes.warpedExposure,
736  subtractRes.subtractedExposure,
737  subtractRes.kernelCellSet,
738  subtractRes.psfMatchingKernel,
739  subtractRes.backgroundModel,
740  nonKernelSources,
741  self.subtract.config.kernel.active.detectionConfig,
742  origVariance=False)
743  diUtils.plotPixelResiduals(exposure,
744  subtractRes.warpedExposure,
745  subtractRes.subtractedExposure,
746  subtractRes.kernelCellSet,
747  subtractRes.psfMatchingKernel,
748  subtractRes.backgroundModel,
749  nonKernelSources,
750  self.subtract.config.kernel.active.detectionConfig,
751  origVariance=True)
752  if display and showDiaSources:
753  flagChecker = SourceFlagChecker(diaSources)
754  isFlagged = [flagChecker(x) for x in diaSources]
755  isDipole = [x.get("classification.dipole") for x in diaSources]
756  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
757  frame=lsstDebug.frame)
758  lsstDebug.frame += 1
759 
760  if display and showDipoles:
761  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
762  frame=lsstDebug.frame)
763  lsstDebug.frame += 1
764 
765  def _getConfigName(self):
766  """Return the name of the config dataset
767  """
768  return "%sDiff_config" % (self.config.coaddName,)
769 
770  def _getMetadataName(self):
771  """Return the name of the metadata dataset
772  """
773  return "%sDiff_metadata" % (self.config.coaddName,)
774 
775  def getSchemaCatalogs(self):
776  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
777  diaSrc = afwTable.SourceCatalog(self.schema)
778  diaSrc.getTable().setMetadata(self.algMetadata)
779  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
780 
781  @classmethod
782  def _makeArgumentParser(cls):
783  """Create an argument parser
784  """
785  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
786  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
787  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
788  help="Optional template data ID (visit only), e.g. --templateId visit=6789")
789  return parser
790 
791 
793  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
794  doc="Shift stars going into RegisterTask by this amount")
795  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
796  doc="Perturb stars going into RegisterTask by this amount")
797 
798  def setDefaults(self):
799  ImageDifferenceConfig.setDefaults(self)
800  self.getTemplate.retarget(GetCalexpAsTemplateTask)
801 
802 
804  """!Image difference Task used in the Winter 2013 data challege.
805  Enables testing the effects of registration shifts and scatter.
806 
807  For use with winter 2013 simulated images:
808  Use --templateId visit=88868666 for sparse data
809  --templateId visit=22222200 for dense data (g)
810  --templateId visit=11111100 for dense data (i)
811  """
812  ConfigClass = Winter2013ImageDifferenceConfig
813  _DefaultName = "winter2013ImageDifference"
814 
815  def __init__(self, **kwargs):
816  ImageDifferenceTask.__init__(self, **kwargs)
817 
818  def fitAstrometry(self, templateSources, templateExposure, selectSources):
819  """Fit the relative astrometry between templateSources and selectSources"""
820  if self.config.winter2013WcsShift > 0.0:
821  offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
822  self.config.winter2013WcsShift)
823  cKey = templateSources[0].getTable().getCentroidKey()
824  for source in templateSources:
825  centroid = source.get(cKey)
826  source.set(cKey, centroid+offset)
827  elif self.config.winter2013WcsRms > 0.0:
828  cKey = templateSources[0].getTable().getCentroidKey()
829  for source in templateSources:
830  offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
831  self.config.winter2013WcsRms*numpy.random.normal())
832  centroid = source.get(cKey)
833  source.set(cKey, centroid+offset)
834 
835  results = self.register.run(templateSources, templateExposure.getWcs(),
836  templateExposure.getBBox(), selectSources)
837  return results
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def run(self, sensorRef, templateIdList=None)
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)