lsst.pipe.tasks  13.0-37-g58c8d4e+3
 All Classes Namespaces Files Functions Variables Groups Pages
imageDifference.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 from builtins import zip
24 import math
25 import random
26 import numpy
27 
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.daf.base as dafBase
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.math as afwMath
33 import lsst.afw.table as afwTable
34 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
35 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.meas.algorithms import SourceDetectionTask, PsfAttributes, SingleGaussianPsf, \
38  ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import ImagePsfMatchTask, DipoleAnalysis, \
40  SourceFlagChecker, KernelCandidateF, makeKernelBasisList, \
41  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, \
42  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask, DecorrelateALKernelTask
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 
46 FwhmPerSigma = 2 * math.sqrt(2 * math.log(2))
47 IqrToSigma = 0.741
48 
49 
50 class ImageDifferenceConfig(pexConfig.Config):
51  """Config for ImageDifferenceTask
52  """
53  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=True,
54  doc="Add background to calexp before processing it. "
55  "Useful as ipDiffim does background matching.")
56  doUseRegister = pexConfig.Field(dtype=bool, default=True,
57  doc="Use image-to-image registration to align template with "
58  "science image")
59  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
60  doc="Writing debugging data for doUseRegister")
61  doSelectSources = pexConfig.Field(dtype=bool, default=True,
62  doc="Select stars to use for kernel fitting")
63  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
64  doc="Select stars of extreme color as part of the control sample")
65  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
66  doc="Select stars that are variable to be part "
67  "of the control sample")
68  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
69  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
70  doc="Convolve science image by its PSF before PSF-matching?")
71  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
72  doc="Use a simple gaussian PSF model for pre-convolution "
73  "(else use fit PSF)? Ignored if doPreConvolve false.")
74  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
75  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
76  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
77  "kernel convolution? If True, also update the diffim PSF.")
78  doMerge = pexConfig.Field(dtype=bool, default=True,
79  doc="Merge positive and negative diaSources with grow radius "
80  "set by growFootprint")
81  doMatchSources = pexConfig.Field(dtype=bool, default=True,
82  doc="Match diaSources with input calexp sources and ref catalog sources")
83  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
84  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
85  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
86  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
87  doc="Write warped and PSF-matched template coadd exposure?")
88  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
89  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
90  doc="Add columns to the source table to hold analysis metrics?")
91 
92  coaddName = pexConfig.Field(
93  doc="coadd name: typically one of deep or goodSeeing",
94  dtype=str,
95  default="deep",
96  )
97  convolveTemplate = pexConfig.Field(
98  doc="Which image gets convolved (default = template)",
99  dtype=bool,
100  default=True
101  )
102  refObjLoader = pexConfig.ConfigurableField(
103  target=LoadAstrometryNetObjectsTask,
104  doc="reference object loader",
105  )
106  astrometer = pexConfig.ConfigurableField(
107  target=AstrometryTask,
108  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
109  )
110  sourceSelector = pexConfig.ConfigurableField(
111  target=ObjectSizeStarSelectorTask,
112  doc="Source selection algorithm",
113  )
114  subtract = pexConfig.ConfigurableField(
115  target=ImagePsfMatchTask,
116  doc="Warp and PSF match template to exposure, then subtract",
117  )
118  decorrelate = pexConfig.ConfigurableField(
119  target=DecorrelateALKernelTask,
120  doc="""Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True.
121  If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the
122  default of 5.5).""",
123  )
124  detection = pexConfig.ConfigurableField(
125  target=SourceDetectionTask,
126  doc="Low-threshold detection for final measurement",
127  )
128  # measurement = pexConfig.ConfigurableField(
129  # target=DipoleMeasurementTask,
130  # doc="Final source measurement on low-threshold detections; dipole fitting enabled.",
131  # )
132  measurement = pexConfig.ConfigurableField(
133  target=DipoleFitTask,
134  doc="Enable updated dipole fitting method.",
135  )
136  getTemplate = pexConfig.ConfigurableField(
137  target=GetCoaddAsTemplateTask,
138  doc="Subtask to retrieve template exposure and sources",
139  )
140  controlStepSize = pexConfig.Field(
141  doc="What step size (every Nth one) to select a control sample from the kernelSources",
142  dtype=int,
143  default=5
144  )
145  controlRandomSeed = pexConfig.Field(
146  doc = "Random seed for shuffing the control sample",
147  dtype = int,
148  default = 10
149  )
150  register = pexConfig.ConfigurableField(
151  target=RegisterTask,
152  doc="Task to enable image-to-image image registration (warping)",
153  )
154  kernelSourcesFromRef = pexConfig.Field(
155  doc="Select sources to measure kernel from reference catalog if True, template if false",
156  dtype=bool,
157  default=False
158  )
159  templateSipOrder = pexConfig.Field(dtype=int, default=2,
160  doc="Sip Order for fitting the Template Wcs "
161  "(default is too high, overfitting)")
162 
163  growFootprint = pexConfig.Field(dtype=int, default=2,
164  doc="Grow positive and negative footprints by this amount before merging")
165 
166  diaSourceMatchRadius = pexConfig.Field(dtype=float, default=0.5,
167  doc="Match radius (in arcseconds) "
168  "for DiaSource to Source association")
169 
170  def setDefaults(self):
171  # defaults are OK for catalog and diacatalog
172 
173  self.subtract.kernel.name = "AL"
174  self.subtract.kernel.active.fitForBackground = True
175  self.subtract.kernel.active.spatialKernelOrder = 1
176  self.subtract.kernel.active.spatialBgOrder = 0
177  self.doPreConvolve = False
178  self.doMatchSources = False
179  self.doAddMetrics = False
180  self.doUseRegister = False
181 
182  # DiaSource Detection
183  self.detection.thresholdPolarity = "both"
184  self.detection.thresholdValue = 5.5
185  self.detection.reEstimateBackground = False
186  self.detection.thresholdType = "pixel_stdev"
187 
188  # Add filtered flux measurement, the correct measurement for pre-convolved images.
189  # Enable all measurements, regardless of doPreConvolved, as it makes data harvesting easier.
190  # To change that you must modify algorithms.names in the task's applyOverrides method,
191  # after the user has set doPreConvolved.
192  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
193 
194  # For shuffling the control sample
195  random.seed(self.controlRandomSeed)
196 
197  def validate(self):
198  pexConfig.Config.validate(self)
199  if self.doMeasurement and not self.doDetection:
200  raise ValueError("Cannot run source measurement without source detection.")
201  if self.doMerge and not self.doDetection:
202  raise ValueError("Cannot run source merging without source detection.")
203  if self.doUseRegister and not self.doSelectSources:
204  raise ValueError("doUseRegister=True and doSelectSources=False. " +
205  "Cannot run RegisterTask without selecting sources.")
206 
207 
208 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
209 
210  @staticmethod
211  def getTargetList(parsedCmd, **kwargs):
212  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
213  **kwargs)
214 
215 
216 class ImageDifferenceTask(pipeBase.CmdLineTask):
217  """Subtract an image from a template and measure the result
218  """
219  ConfigClass = ImageDifferenceConfig
220  RunnerClass = ImageDifferenceTaskRunner
221  _DefaultName = "imageDifference"
222 
223  def __init__(self, butler=None, **kwargs):
224  """!Construct an ImageDifference Task
225 
226  @param[in] butler Butler object to use in constructing reference object loaders
227  """
228  pipeBase.CmdLineTask.__init__(self, **kwargs)
229  self.makeSubtask("subtract")
230  self.makeSubtask("getTemplate")
231  self.makeSubtask("decorrelate")
232 
233  if self.config.doUseRegister:
234  self.makeSubtask("register")
235  self.schema = afwTable.SourceTable.makeMinimalSchema()
236 
237  if self.config.doSelectSources:
238  self.makeSubtask("sourceSelector", schema=self.schema)
239  self.makeSubtask('refObjLoader', butler=butler)
240  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
241 
242  self.algMetadata = dafBase.PropertyList()
243  if self.config.doDetection:
244  self.makeSubtask("detection", schema=self.schema)
245  if self.config.doMeasurement:
246  if not self.config.doDipoleFitting:
247  self.makeSubtask("measurement", schema=self.schema,
248  algMetadata=self.algMetadata)
249  else:
250  self.makeSubtask("measurement", schema=self.schema)
251  if self.config.doMatchSources:
252  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
253  self.schema.addField("srcMatchId", "L", "unique id of source match")
254 
255  @pipeBase.timeMethod
256  def run(self, sensorRef, templateIdList=None):
257  """Subtract an image from a template coadd and measure the result
258 
259  Steps include:
260  - warp template coadd to match WCS of image
261  - PSF match image to warped template
262  - subtract image from PSF-matched, warped template
263  - persist difference image
264  - detect sources
265  - measure sources
266 
267  @param sensorRef: sensor-level butler data reference, used for the following data products:
268  Input only:
269  - calexp
270  - psf
271  - ccdExposureId
272  - ccdExposureId_bits
273  - self.config.coaddName + "Coadd_skyMap"
274  - self.config.coaddName + "Coadd"
275  Input or output, depending on config:
276  - self.config.coaddName + "Diff_subtractedExp"
277  Output, depending on config:
278  - self.config.coaddName + "Diff_matchedExp"
279  - self.config.coaddName + "Diff_src"
280 
281  @return pipe_base Struct containing these fields:
282  - subtractedExposure: exposure after subtracting template;
283  the unpersisted version if subtraction not run but detection run
284  None if neither subtraction nor detection run (i.e. nothing useful done)
285  - subtractRes: results of subtraction task; None if subtraction not run
286  - sources: detected and possibly measured sources; None if detection not run
287  """
288  self.log.info("Processing %s" % (sensorRef.dataId))
289 
290  # initialize outputs and some intermediate products
291  subtractedExposure = None
292  subtractRes = None
293  selectSources = None
294  kernelSources = None
295  controlSources = None
296  diaSources = None
297 
298  # We make one IdFactory that will be used by both icSrc and src datasets;
299  # I don't know if this is the way we ultimately want to do things, but at least
300  # this ensures the source IDs are fully unique.
301  expBits = sensorRef.get("ccdExposureId_bits")
302  expId = int(sensorRef.get("ccdExposureId"))
303  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
304 
305  # Retrieve the science image we wish to analyze
306  exposure = sensorRef.get("calexp", immediate=True)
307  if self.config.doAddCalexpBackground:
308  mi = exposure.getMaskedImage()
309  mi += sensorRef.get("calexpBackground").getImage()
310  if not exposure.hasPsf():
311  raise pipeBase.TaskError("Exposure has no psf")
312  sciencePsf = exposure.getPsf()
313 
314  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
315  templateExposure = None # Stitched coadd exposure
316  templateSources = None # Sources on the template image
317  if self.config.doSubtract:
318  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
319  templateExposure = template.exposure
320  templateSources = template.sources
321 
322  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
323  ctr = afwGeom.Box2D(exposure.getBBox()).getCenter()
324  psfAttr = PsfAttributes(sciencePsf, afwGeom.Point2I(ctr))
325  scienceSigmaOrig = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
326 
327  # sigma of PSF of template image before warping
328  ctr = afwGeom.Box2D(templateExposure.getBBox()).getCenter()
329  psfAttr = PsfAttributes(templateExposure.getPsf(), afwGeom.Point2I(ctr))
330  templateSigma = psfAttr.computeGaussianWidth(psfAttr.ADAPTIVE_MOMENT)
331 
332  # if requested, convolve the science exposure with its PSF
333  # (properly, this should be a cross-correlation, but our code does not yet support that)
334  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
335  # else sigma of original science exposure
336  if self.config.doPreConvolve:
337  convControl = afwMath.ConvolutionControl()
338  # cannot convolve in place, so make a new MI to receive convolved image
339  srcMI = exposure.getMaskedImage()
340  destMI = srcMI.Factory(srcMI.getDimensions())
341  srcPsf = sciencePsf
342  if self.config.useGaussianForPreConvolution:
343  # convolve with a simplified PSF model: a double Gaussian
344  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
345  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
346  else:
347  # convolve with science exposure's PSF model
348  preConvPsf = srcPsf
349  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
350  exposure.setMaskedImage(destMI)
351  scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
352  else:
353  scienceSigmaPost = scienceSigmaOrig
354 
355  # If requested, find sources in the image
356  if self.config.doSelectSources:
357  if not sensorRef.datasetExists("src"):
358  self.log.warn("Src product does not exist; running detection, measurement, selection")
359  # Run own detection and measurement; necessary in nightly processing
360  selectSources = self.subtract.getSelectSources(
361  exposure,
362  sigma=scienceSigmaPost,
363  doSmooth=not self.doPreConvolve,
364  idFactory=idFactory,
365  )
366  else:
367  self.log.info("Source selection via src product")
368  # Sources already exist; for data release processing
369  selectSources = sensorRef.get("src")
370 
371  # Number of basis functions
372  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
373  referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
374  targetFwhmPix=templateSigma * FwhmPerSigma))
375 
376  if self.config.doAddMetrics:
377  # Modify the schema of all Sources
378  kcQa = KernelCandidateQa(nparam)
379  selectSources = kcQa.addToSchema(selectSources)
380 
381  if self.config.kernelSourcesFromRef:
382  # match exposure sources to reference catalog
383  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
384  matches = astromRet.matches
385  elif templateSources:
386  # match exposure sources to template sources
387  mc = afwTable.MatchControl()
388  mc.findOnlyClosest = False
389  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
390  mc)
391  else:
392  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
393  "but template sources not available. Cannot match science " +
394  "sources with template sources. Run process* on data from " +
395  "which templates are built.")
396 
397  kernelSources = self.sourceSelector.selectStars(exposure, selectSources,
398  matches=matches).starCat
399 
400  random.shuffle(kernelSources, random.random)
401  controlSources = kernelSources[::self.config.controlStepSize]
402  kernelSources = [k for i, k in enumerate(kernelSources) if i % self.config.controlStepSize]
403 
404  if self.config.doSelectDcrCatalog:
405  redSelector = DiaCatalogSourceSelectorTask(
406  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999))
407  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
408  controlSources.extend(redSources)
409 
410  blueSelector = DiaCatalogSourceSelectorTask(
411  DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin))
412  blueSources = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat
413  controlSources.extend(blueSources)
414 
415  if self.config.doSelectVariableCatalog:
416  varSelector = DiaCatalogSourceSelectorTask(
417  DiaCatalogSourceSelectorConfig(includeVariable=True))
418  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
419  controlSources.extend(varSources)
420 
421  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
422  % (len(kernelSources), len(selectSources), len(controlSources)))
423  allresids = {}
424  if self.config.doUseRegister:
425  self.log.info("Registering images")
426 
427  if templateSources is None:
428  # Run detection on the template, which is
429  # temporarily background-subtracted
430  templateSources = self.subtract.getSelectSources(
431  templateExposure,
432  sigma=templateSigma,
433  doSmooth=True,
434  idFactory=idFactory
435  )
436 
437  # Third step: we need to fit the relative astrometry.
438  #
439  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
440  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
441  exposure.getWcs(), exposure.getBBox())
442  templateExposure = warpedExp
443 
444  # Create debugging outputs on the astrometric
445  # residuals as a function of position. Persistence
446  # not yet implemented; expected on (I believe) #2636.
447  if self.config.doDebugRegister:
448  # Grab matches to reference catalog
449  srcToMatch = {x.second.getId(): x.first for x in matches}
450 
451  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
452  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
453  sids = [m.first.getId() for m in wcsResults.matches]
454  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
455  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
456  m.second.get(inCentroidKey))) for m in wcsResults.matches]
457  allresids = dict(zip(sids, zip(positions, residuals)))
458 
459  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
460  wcsResults.wcs.pixelToSky(
461  m.second.get(inCentroidKey))) for m in wcsResults.matches]
462  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
463  2.5*numpy.log10(srcToMatch[x].get("r"))
464  for x in sids if x in srcToMatch.keys()])
465  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
466  if s in srcToMatch.keys()])
467  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
468  if s in srcToMatch.keys()])
469  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
470  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
471  (colors <= self.sourceSelector.config.grMax))
472  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
473  rms1Long = IqrToSigma * \
474  (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
475  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)-numpy.percentile(dlat[idx1], 25))
476  rms2Long = IqrToSigma * \
477  (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
478  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)-numpy.percentile(dlat[idx2], 25))
479  rms3Long = IqrToSigma * \
480  (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
481  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)-numpy.percentile(dlat[idx3], 25))
482  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]),
483  rms1Long,
484  numpy.median(dlat[idx1]),
485  rms1Lat))
486  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]),
487  rms2Long,
488  numpy.median(dlat[idx2]),
489  rms2Lat))
490  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]),
491  rms3Long,
492  numpy.median(dlat[idx3]),
493  rms3Lat))
494 
495  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
496  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
497  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
498  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
499  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
500  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
501 
502  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
503  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
504  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
505  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
506  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
507  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
508 
509  # warp template exposure to match exposure,
510  # PSF match template exposure to exposure,
511  # then return the difference
512 
513  # Return warped template... Construct sourceKernelCand list after subtract
514  self.log.info("Subtracting images")
515  subtractRes = self.subtract.subtractExposures(
516  templateExposure=templateExposure,
517  scienceExposure=exposure,
518  candidateList=kernelSources,
519  convolveTemplate=self.config.convolveTemplate,
520  doWarping=not self.config.doUseRegister
521  )
522  subtractedExposure = subtractRes.subtractedExposure
523 
524  if self.config.doWriteMatchedExp:
525  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
526 
527  if self.config.doDetection:
528  self.log.info("Computing diffim PSF")
529  if subtractedExposure is None:
530  subtractedExposure = sensorRef.get(subtractedExposureName)
531 
532  # Get Psf from the appropriate input image if it doesn't exist
533  if not subtractedExposure.hasPsf():
534  if self.config.convolveTemplate:
535  subtractedExposure.setPsf(exposure.getPsf())
536  else:
537  if templateExposure is None:
538  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
539  subtractedExposure.setPsf(template.exposure.getPsf())
540 
541  # If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have
542  # already been decorrelated. Thus, we do not do decorrelation if doSubtract is False.
543  if self.config.doDecorrelation and self.config.doSubtract:
544  decorrResult = self.decorrelate.run(exposure, templateExposure,
545  subtractedExposure,
546  subtractRes.psfMatchingKernel)
547  subtractedExposure = decorrResult.correctedExposure
548 
549  if self.config.doDetection:
550  self.log.info("Running diaSource detection")
551  # Erase existing detection mask planes
552  mask = subtractedExposure.getMaskedImage().getMask()
553  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
554 
555  table = afwTable.SourceTable.make(self.schema, idFactory)
556  table.setMetadata(self.algMetadata)
557  results = self.detection.makeSourceCatalog(
558  table=table,
559  exposure=subtractedExposure,
560  doSmooth=not self.config.doPreConvolve
561  )
562 
563  if self.config.doMerge:
564  fpSet = results.fpSets.positive
565  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
566  self.config.growFootprint, False)
567  diaSources = afwTable.SourceCatalog(table)
568  fpSet.makeSources(diaSources)
569  self.log.info("Merging detections into %d sources" % (len(diaSources)))
570  else:
571  diaSources = results.sources
572 
573  if self.config.doMeasurement:
574  self.log.info("Running diaSource measurement")
575  if not self.config.doDipoleFitting:
576  self.measurement.run(diaSources, subtractedExposure)
577  else:
578  if self.config.doSubtract:
579  self.measurement.run(diaSources, subtractedExposure, exposure,
580  subtractRes.matchedExposure)
581  else:
582  self.measurement.run(diaSources, subtractedExposure, exposure)
583 
584  # Match with the calexp sources if possible
585  if self.config.doMatchSources:
586  if sensorRef.datasetExists("src"):
587  # Create key,val pair where key=diaSourceId and val=sourceId
588  matchRadAsec = self.config.diaSourceMatchRadius
589  matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
590 
591  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
592  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
593  srcMatch in srcMatches])
594  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
595  len(diaSources)))
596  else:
597  self.log.warn("Src product does not exist; cannot match with diaSources")
598  srcMatchDict = {}
599 
600  # Create key,val pair where key=diaSourceId and val=refId
601  refAstromConfig = AstrometryConfig()
602  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
603  refAstrometer = AstrometryTask(refAstromConfig)
604  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
605  refMatches = astromRet.matches
606  if refMatches is None:
607  self.log.warn("No diaSource matches with reference catalog")
608  refMatchDict = {}
609  else:
610  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
611  len(diaSources)))
612  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
613  refMatch in refMatches])
614 
615  # Assign source Ids
616  for diaSource in diaSources:
617  sid = diaSource.getId()
618  if sid in srcMatchDict:
619  diaSource.set("srcMatchId", srcMatchDict[sid])
620  if sid in refMatchDict:
621  diaSource.set("refMatchId", refMatchDict[sid])
622 
623  if diaSources is not None and self.config.doWriteSources:
624  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
625 
626  if self.config.doAddMetrics and self.config.doSelectSources:
627  self.log.info("Evaluating metrics and control sample")
628 
629  kernelCandList = []
630  for cell in subtractRes.kernelCellSet.getCellList():
631  for cand in cell.begin(False): # include bad candidates
632  kernelCandList.append(cand)
633 
634  # Get basis list to build control sample kernels
635  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
636 
637  controlCandList = \
638  diffimTools.sourceTableToCandidateList(controlSources,
639  subtractRes.warpedExposure, exposure,
640  self.config.subtract.kernel.active,
641  self.config.subtract.kernel.active.detectionConfig,
642  self.log, doBuild=True, basisList=basisList)
643 
644  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
645  dof=nparam)
646  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
647 
648  if self.config.doDetection:
649  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
650  else:
651  kcQa.aggregate(selectSources, self.metadata, allresids)
652 
653  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
654 
655  if self.config.doWriteSubtractedExp:
656  sensorRef.put(subtractedExposure, subtractedExposureName)
657 
658  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
659  return pipeBase.Struct(
660  subtractedExposure=subtractedExposure,
661  subtractRes=subtractRes,
662  sources=diaSources,
663  )
664 
665  def fitAstrometry(self, templateSources, templateExposure, selectSources):
666  """Fit the relative astrometry between templateSources and selectSources
667 
668  @todo remove this method. It originally fit a new WCS to the template before calling register.run
669  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
670  It remains because a subtask overrides it.
671  """
672  results = self.register.run(templateSources, templateExposure.getWcs(),
673  templateExposure.getBBox(), selectSources)
674  return results
675 
676  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
677  """@todo Test and update for current debug display and slot names
678  """
679  import lsstDebug
680  display = lsstDebug.Info(__name__).display
681  showSubtracted = lsstDebug.Info(__name__).showSubtracted
682  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
683  showDiaSources = lsstDebug.Info(__name__).showDiaSources
684  showDipoles = lsstDebug.Info(__name__).showDipoles
685  maskTransparency = lsstDebug.Info(__name__).maskTransparency
686  if display:
687  import lsst.afw.display.ds9 as ds9
688  if not maskTransparency:
689  maskTransparency = 0
690  ds9.setMaskTransparency(maskTransparency)
691 
692  if display and showSubtracted:
693  ds9.mtv(subtractRes.subtractedExposure, frame=lsstDebug.frame, title="Subtracted image")
694  mi = subtractRes.subtractedExposure.getMaskedImage()
695  x0, y0 = mi.getX0(), mi.getY0()
696  with ds9.Buffering():
697  for s in diaSources:
698  x, y = s.getX() - x0, s.getY() - y0
699  ctype = "red" if s.get("flags.negative") else "yellow"
700  if (s.get("flags.pixel.interpolated.center") or s.get("flags.pixel.saturated.center") or
701  s.get("flags.pixel.cr.center")):
702  ptype = "x"
703  elif (s.get("flags.pixel.interpolated.any") or s.get("flags.pixel.saturated.any") or
704  s.get("flags.pixel.cr.any")):
705  ptype = "+"
706  else:
707  ptype = "o"
708  ds9.dot(ptype, x, y, size=4, frame=lsstDebug.frame, ctype=ctype)
709  lsstDebug.frame += 1
710 
711  if display and showPixelResiduals and selectSources:
712  nonKernelSources = []
713  for source in selectSources:
714  if source not in kernelSources:
715  nonKernelSources.append(source)
716 
717  diUtils.plotPixelResiduals(exposure,
718  subtractRes.warpedExposure,
719  subtractRes.subtractedExposure,
720  subtractRes.kernelCellSet,
721  subtractRes.psfMatchingKernel,
722  subtractRes.backgroundModel,
723  nonKernelSources,
724  self.subtract.config.kernel.active.detectionConfig,
725  origVariance=False)
726  diUtils.plotPixelResiduals(exposure,
727  subtractRes.warpedExposure,
728  subtractRes.subtractedExposure,
729  subtractRes.kernelCellSet,
730  subtractRes.psfMatchingKernel,
731  subtractRes.backgroundModel,
732  nonKernelSources,
733  self.subtract.config.kernel.active.detectionConfig,
734  origVariance=True)
735  if display and showDiaSources:
736  flagChecker = SourceFlagChecker(diaSources)
737  isFlagged = [flagChecker(x) for x in diaSources]
738  isDipole = [x.get("classification.dipole") for x in diaSources]
739  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
740  frame=lsstDebug.frame)
741  lsstDebug.frame += 1
742 
743  if display and showDipoles:
744  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
745  frame=lsstDebug.frame)
746  lsstDebug.frame += 1
747 
748  def _getConfigName(self):
749  """Return the name of the config dataset
750  """
751  return "%sDiff_config" % (self.config.coaddName,)
752 
753  def _getMetadataName(self):
754  """Return the name of the metadata dataset
755  """
756  return "%sDiff_metadata" % (self.config.coaddName,)
757 
758  def getSchemaCatalogs(self):
759  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
760  diaSrc = afwTable.SourceCatalog(self.schema)
761  diaSrc.getTable().setMetadata(self.algMetadata)
762  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
763 
764  @classmethod
765  def _makeArgumentParser(cls):
766  """Create an argument parser
767  """
768  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
769  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
770  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
771  help="Optional template data ID (visit only), e.g. --templateId visit=6789")
772  return parser
773 
774 
776  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
777  doc="Shift stars going into RegisterTask by this amount")
778  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
779  doc="Perturb stars going into RegisterTask by this amount")
780 
781  def setDefaults(self):
782  ImageDifferenceConfig.setDefaults(self)
783  self.getTemplate.retarget(GetCalexpAsTemplateTask)
784 
785 
787  """!Image difference Task used in the Winter 2013 data challege.
788  Enables testing the effects of registration shifts and scatter.
789 
790  For use with winter 2013 simulated images:
791  Use --templateId visit=88868666 for sparse data
792  --templateId visit=22222200 for dense data (g)
793  --templateId visit=11111100 for dense data (i)
794  """
795  ConfigClass = Winter2013ImageDifferenceConfig
796  _DefaultName = "winter2013ImageDifference"
797 
798  def __init__(self, **kwargs):
799  ImageDifferenceTask.__init__(self, **kwargs)
800 
801  def fitAstrometry(self, templateSources, templateExposure, selectSources):
802  """Fit the relative astrometry between templateSources and selectSources"""
803  if self.config.winter2013WcsShift > 0.0:
804  offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
805  self.config.winter2013WcsShift)
806  cKey = templateSources[0].getTable().getCentroidKey()
807  for source in templateSources:
808  centroid = source.get(cKey)
809  source.set(cKey, centroid+offset)
810  elif self.config.winter2013WcsRms > 0.0:
811  cKey = templateSources[0].getTable().getCentroidKey()
812  for source in templateSources:
813  offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
814  self.config.winter2013WcsRms*numpy.random.normal())
815  centroid = source.get(cKey)
816  source.set(cKey, centroid+offset)
817 
818  results = self.register.run(templateSources, templateExposure.getWcs(),
819  templateExposure.getBBox(), selectSources)
820  return results
Image difference Task used in the Winter 2013 data challege.
def __init__
Construct an ImageDifference Task.