lsst.pipe.tasks  15.0-16-g6f0eb036+2
imageDifference.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import random
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
28 import lsst.daf.base as dafBase
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.math as afwMath
31 import lsst.afw.table as afwTable
32 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
33 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
34 from lsst.pipe.tasks.registerImage import RegisterTask
35 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, \
36  ObjectSizeStarSelectorTask
37 from lsst.ip.diffim import DipoleAnalysis, \
38  SourceFlagChecker, KernelCandidateF, makeKernelBasisList, \
39  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, \
40  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask, \
41  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry
42 import lsst.ip.diffim.diffimTools as diffimTools
43 import lsst.ip.diffim.utils as diUtils
44 
45 FwhmPerSigma = 2 * math.sqrt(2 * math.log(2))
46 IqrToSigma = 0.741
47 
48 
49 class ImageDifferenceConfig(pexConfig.Config):
50  """Config for ImageDifferenceTask
51  """
52  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=True,
53  doc="Add background to calexp before processing it. "
54  "Useful as ipDiffim does background matching.")
55  doUseRegister = pexConfig.Field(dtype=bool, default=True,
56  doc="Use image-to-image registration to align template with "
57  "science image")
58  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
59  doc="Writing debugging data for doUseRegister")
60  doSelectSources = pexConfig.Field(dtype=bool, default=True,
61  doc="Select stars to use for kernel fitting")
62  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
63  doc="Select stars of extreme color as part of the control sample")
64  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
65  doc="Select stars that are variable to be part "
66  "of the control sample")
67  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
68  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
69  doc="Convolve science image by its PSF before PSF-matching?")
70  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
71  doc="Use a simple gaussian PSF model for pre-convolution "
72  "(else use fit PSF)? Ignored if doPreConvolve false.")
73  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
74  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
75  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
76  "kernel convolution? If True, also update the diffim PSF.")
77  doMerge = pexConfig.Field(dtype=bool, default=True,
78  doc="Merge positive and negative diaSources with grow radius "
79  "set by growFootprint")
80  doMatchSources = pexConfig.Field(dtype=bool, default=True,
81  doc="Match diaSources with input calexp sources and ref catalog sources")
82  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
83  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
84  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
85  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
86  doc="Write warped and PSF-matched template coadd exposure?")
87  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
88  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
89  doc="Add columns to the source table to hold analysis metrics?")
90 
91  coaddName = pexConfig.Field(
92  doc="coadd name: typically one of deep or goodSeeing",
93  dtype=str,
94  default="deep",
95  )
96  convolveTemplate = pexConfig.Field(
97  doc="Which image gets convolved (default = template)",
98  dtype=bool,
99  default=True
100  )
101  refObjLoader = pexConfig.ConfigurableField(
102  target=LoadAstrometryNetObjectsTask,
103  doc="reference object loader",
104  )
105  astrometer = pexConfig.ConfigurableField(
106  target=AstrometryTask,
107  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
108  )
109  sourceSelector = pexConfig.ConfigurableField(
110  target=ObjectSizeStarSelectorTask,
111  doc="Source selection algorithm",
112  )
113  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
114  decorrelate = pexConfig.ConfigurableField(
115  target=DecorrelateALKernelSpatialTask,
116  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
117  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
118  "default of 5.5).",
119  )
120  doSpatiallyVarying = pexConfig.Field(
121  dtype=bool,
122  default=False,
123  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
124  "image in order to allow for spatial variations"
125  )
126  detection = pexConfig.ConfigurableField(
127  target=SourceDetectionTask,
128  doc="Low-threshold detection for final measurement",
129  )
130  measurement = pexConfig.ConfigurableField(
131  target=DipoleFitTask,
132  doc="Enable updated dipole fitting method",
133  )
134  getTemplate = pexConfig.ConfigurableField(
135  target=GetCoaddAsTemplateTask,
136  doc="Subtask to retrieve template exposure and sources",
137  )
138  controlStepSize = pexConfig.Field(
139  doc="What step size (every Nth one) to select a control sample from the kernelSources",
140  dtype=int,
141  default=5
142  )
143  controlRandomSeed = pexConfig.Field(
144  doc="Random seed for shuffing the control sample",
145  dtype=int,
146  default=10
147  )
148  register = pexConfig.ConfigurableField(
149  target=RegisterTask,
150  doc="Task to enable image-to-image image registration (warping)",
151  )
152  kernelSourcesFromRef = pexConfig.Field(
153  doc="Select sources to measure kernel from reference catalog if True, template if false",
154  dtype=bool,
155  default=False
156  )
157  templateSipOrder = pexConfig.Field(dtype=int, default=2,
158  doc="Sip Order for fitting the Template Wcs "
159  "(default is too high, overfitting)")
160 
161  growFootprint = pexConfig.Field(dtype=int, default=2,
162  doc="Grow positive and negative footprints by this amount before merging")
163 
164  diaSourceMatchRadius = pexConfig.Field(dtype=float, default=0.5,
165  doc="Match radius (in arcseconds) "
166  "for DiaSource to Source association")
167 
168  def setDefaults(self):
169  # defaults are OK for catalog and diacatalog
170 
171  self.subtract['al'].kernel.name = "AL"
172  self.subtract['al'].kernel.active.fitForBackground = True
173  self.subtract['al'].kernel.active.spatialKernelOrder = 1
174  self.subtract['al'].kernel.active.spatialBgOrder = 0
175  self.doPreConvolve = False
176  self.doMatchSources = False
177  self.doAddMetrics = False
178  self.doUseRegister = False
179 
180  # DiaSource Detection
181  self.detection.thresholdPolarity = "both"
182  self.detection.thresholdValue = 5.5
183  self.detection.reEstimateBackground = False
184  self.detection.thresholdType = "pixel_stdev"
185 
186  # Add filtered flux measurement, the correct measurement for pre-convolved images.
187  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
188  # To change that you must modify algorithms.names in the task's applyOverrides method,
189  # after the user has set doPreConvolve.
190  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
191 
192  # For shuffling the control sample
193  random.seed(self.controlRandomSeed)
194 
195  def validate(self):
196  pexConfig.Config.validate(self)
197  if self.doMeasurement and not self.doDetection:
198  raise ValueError("Cannot run source measurement without source detection.")
199  if self.doMerge and not self.doDetection:
200  raise ValueError("Cannot run source merging without source detection.")
201  if self.doUseRegister and not self.doSelectSources:
202  raise ValueError("doUseRegister=True and doSelectSources=False. " +
203  "Cannot run RegisterTask without selecting sources.")
204 
205 
206 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
207 
208  @staticmethod
209  def getTargetList(parsedCmd, **kwargs):
210  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
211  **kwargs)
212 
213 
214 class ImageDifferenceTask(pipeBase.CmdLineTask):
215  """Subtract an image from a template and measure the result
216  """
217  ConfigClass = ImageDifferenceConfig
218  RunnerClass = ImageDifferenceTaskRunner
219  _DefaultName = "imageDifference"
220 
221  def __init__(self, butler=None, **kwargs):
222  """!Construct an ImageDifference Task
223 
224  @param[in] butler Butler object to use in constructing reference object loaders
225  """
226  pipeBase.CmdLineTask.__init__(self, **kwargs)
227  self.makeSubtask("getTemplate")
228 
229  self.makeSubtask("subtract")
230 
231  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
232  self.makeSubtask("decorrelate")
233 
234  if self.config.doUseRegister:
235  self.makeSubtask("register")
236  self.schema = afwTable.SourceTable.makeMinimalSchema()
237 
238  if self.config.doSelectSources:
239  self.makeSubtask("sourceSelector")
240  self.makeSubtask('refObjLoader', butler=butler)
241  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
242 
243  self.algMetadata = dafBase.PropertyList()
244  if self.config.doDetection:
245  self.makeSubtask("detection", schema=self.schema)
246  if self.config.doMeasurement:
247  self.makeSubtask("measurement", schema=self.schema,
248  algMetadata=self.algMetadata)
249  if self.config.doMatchSources:
250  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
251  self.schema.addField("srcMatchId", "L", "unique id of source match")
252 
253  @pipeBase.timeMethod
254  def run(self, sensorRef, templateIdList=None):
255  """Subtract an image from a template coadd and measure the result
256 
257  Steps include:
258  - warp template coadd to match WCS of image
259  - PSF match image to warped template
260  - subtract image from PSF-matched, warped template
261  - persist difference image
262  - detect sources
263  - measure sources
264 
265  @param sensorRef: sensor-level butler data reference, used for the following data products:
266  Input only:
267  - calexp
268  - psf
269  - ccdExposureId
270  - ccdExposureId_bits
271  - self.config.coaddName + "Coadd_skyMap"
272  - self.config.coaddName + "Coadd"
273  Input or output, depending on config:
274  - self.config.coaddName + "Diff_subtractedExp"
275  Output, depending on config:
276  - self.config.coaddName + "Diff_matchedExp"
277  - self.config.coaddName + "Diff_src"
278 
279  @return pipe_base Struct containing these fields:
280  - subtractedExposure: exposure after subtracting template;
281  the unpersisted version if subtraction not run but detection run
282  None if neither subtraction nor detection run (i.e. nothing useful done)
283  - subtractRes: results of subtraction task; None if subtraction not run
284  - sources: detected and possibly measured sources; None if detection not run
285  """
286  self.log.info("Processing %s" % (sensorRef.dataId))
287 
288  # initialize outputs and some intermediate products
289  subtractedExposure = None
290  subtractRes = None
291  selectSources = None
292  kernelSources = None
293  controlSources = None
294  diaSources = None
295 
296  # We make one IdFactory that will be used by both icSrc and src datasets;
297  # I don't know if this is the way we ultimately want to do things, but at least
298  # this ensures the source IDs are fully unique.
299  expBits = sensorRef.get("ccdExposureId_bits")
300  expId = int(sensorRef.get("ccdExposureId"))
301  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
302 
303  # Retrieve the science image we wish to analyze
304  exposure = sensorRef.get("calexp", immediate=True)
305  if self.config.doAddCalexpBackground:
306  mi = exposure.getMaskedImage()
307  mi += sensorRef.get("calexpBackground").getImage()
308  if not exposure.hasPsf():
309  raise pipeBase.TaskError("Exposure has no psf")
310  sciencePsf = exposure.getPsf()
311 
312  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
313  templateExposure = None # Stitched coadd exposure
314  templateSources = None # Sources on the template image
315  if self.config.doSubtract:
316  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
317  templateExposure = template.exposure
318  templateSources = template.sources
319 
320  if self.config.subtract.name == 'zogy':
321  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
322  doWarping=True,
323  spatiallyVarying=self.config.doSpatiallyVarying,
324  doPreConvolve=self.config.doPreConvolve)
325  subtractedExposure = subtractRes.subtractedExposure
326 
327  elif self.config.subtract.name == 'al':
328  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
329  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
330 
331  # sigma of PSF of template image before warping
332  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
333 
334  # if requested, convolve the science exposure with its PSF
335  # (properly, this should be a cross-correlation, but our code does not yet support that)
336  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
337  # else sigma of original science exposure
338  preConvPsf = None
339  if self.config.doPreConvolve:
340  convControl = afwMath.ConvolutionControl()
341  # cannot convolve in place, so make a new MI to receive convolved image
342  srcMI = exposure.getMaskedImage()
343  destMI = srcMI.Factory(srcMI.getDimensions())
344  srcPsf = sciencePsf
345  if self.config.useGaussianForPreConvolution:
346  # convolve with a simplified PSF model: a double Gaussian
347  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
348  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
349  else:
350  # convolve with science exposure's PSF model
351  preConvPsf = srcPsf
352  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
353  exposure.setMaskedImage(destMI)
354  scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
355  else:
356  scienceSigmaPost = scienceSigmaOrig
357 
358  # If requested, find sources in the image
359  if self.config.doSelectSources:
360  if not sensorRef.datasetExists("src"):
361  self.log.warn("Src product does not exist; running detection, measurement, selection")
362  # Run own detection and measurement; necessary in nightly processing
363  selectSources = self.subtract.getSelectSources(
364  exposure,
365  sigma=scienceSigmaPost,
366  doSmooth=not self.doPreConvolve,
367  idFactory=idFactory,
368  )
369  else:
370  self.log.info("Source selection via src product")
371  # Sources already exist; for data release processing
372  selectSources = sensorRef.get("src")
373 
374  # Number of basis functions
375  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
376  referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
377  targetFwhmPix=templateSigma * FwhmPerSigma))
378 
379  if self.config.doAddMetrics:
380  # Modify the schema of all Sources
381  kcQa = KernelCandidateQa(nparam)
382  selectSources = kcQa.addToSchema(selectSources)
383 
384  if self.config.kernelSourcesFromRef:
385  # match exposure sources to reference catalog
386  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
387  matches = astromRet.matches
388  elif templateSources:
389  # match exposure sources to template sources
390  mc = afwTable.MatchControl()
391  mc.findOnlyClosest = False
392  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
393  mc)
394  else:
395  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
396  "but template sources not available. Cannot match science " +
397  "sources with template sources. Run process* on data from " +
398  "which templates are built.")
399 
400  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
401  matches=matches).sourceCat
402 
403  random.shuffle(kernelSources, random.random)
404  controlSources = kernelSources[::self.config.controlStepSize]
405  kernelSources = [k for i, k in enumerate(kernelSources)
406  if i % self.config.controlStepSize]
407 
408  if self.config.doSelectDcrCatalog:
409  redSelector = DiaCatalogSourceSelectorTask(
410  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
411  grMax=99.999))
412  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
413  controlSources.extend(redSources)
414 
415  blueSelector = DiaCatalogSourceSelectorTask(
416  DiaCatalogSourceSelectorConfig(grMin=-99.999,
417  grMax=self.sourceSelector.config.grMin))
418  blueSources = blueSelector.selectStars(exposure, selectSources,
419  matches=matches).starCat
420  controlSources.extend(blueSources)
421 
422  if self.config.doSelectVariableCatalog:
423  varSelector = DiaCatalogSourceSelectorTask(
424  DiaCatalogSourceSelectorConfig(includeVariable=True))
425  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
426  controlSources.extend(varSources)
427 
428  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
429  % (len(kernelSources), len(selectSources), len(controlSources)))
430  allresids = {}
431  if self.config.doUseRegister:
432  self.log.info("Registering images")
433 
434  if templateSources is None:
435  # Run detection on the template, which is
436  # temporarily background-subtracted
437  templateSources = self.subtract.getSelectSources(
438  templateExposure,
439  sigma=templateSigma,
440  doSmooth=True,
441  idFactory=idFactory
442  )
443 
444  # Third step: we need to fit the relative astrometry.
445  #
446  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
447  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
448  exposure.getWcs(), exposure.getBBox())
449  templateExposure = warpedExp
450 
451  # Create debugging outputs on the astrometric
452  # residuals as a function of position. Persistence
453  # not yet implemented; expected on (I believe) #2636.
454  if self.config.doDebugRegister:
455  # Grab matches to reference catalog
456  srcToMatch = {x.second.getId(): x.first for x in matches}
457 
458  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
459  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
460  sids = [m.first.getId() for m in wcsResults.matches]
461  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
462  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
463  m.second.get(inCentroidKey))) for m in wcsResults.matches]
464  allresids = dict(zip(sids, zip(positions, residuals)))
465 
466  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
467  wcsResults.wcs.pixelToSky(
468  m.second.get(inCentroidKey))) for m in wcsResults.matches]
469  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
470  2.5*numpy.log10(srcToMatch[x].get("r"))
471  for x in sids if x in srcToMatch.keys()])
472  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
473  if s in srcToMatch.keys()])
474  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
475  if s in srcToMatch.keys()])
476  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
477  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
478  (colors <= self.sourceSelector.config.grMax))
479  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
480  rms1Long = IqrToSigma * \
481  (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
482  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
483  numpy.percentile(dlat[idx1], 25))
484  rms2Long = IqrToSigma * \
485  (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
486  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
487  numpy.percentile(dlat[idx2], 25))
488  rms3Long = IqrToSigma * \
489  (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
490  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
491  numpy.percentile(dlat[idx3], 25))
492  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
493  (numpy.median(dlong[idx1]), rms1Long,
494  numpy.median(dlat[idx1]), rms1Lat))
495  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
496  (numpy.median(dlong[idx2]), rms2Long,
497  numpy.median(dlat[idx2]), rms2Lat))
498  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
499  (numpy.median(dlong[idx3]), rms3Long,
500  numpy.median(dlat[idx3]), rms3Lat))
501 
502  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
503  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
504  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
505  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
506  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
507  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
508 
509  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
510  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
511  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
512  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
513  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
514  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
515 
516  # warp template exposure to match exposure,
517  # PSF match template exposure to exposure,
518  # then return the difference
519 
520  # Return warped template... Construct sourceKernelCand list after subtract
521  self.log.info("Subtracting images")
522  subtractRes = self.subtract.subtractExposures(
523  templateExposure=templateExposure,
524  scienceExposure=exposure,
525  candidateList=kernelSources,
526  convolveTemplate=self.config.convolveTemplate,
527  doWarping=not self.config.doUseRegister
528  )
529  subtractedExposure = subtractRes.subtractedExposure
530 
531  if self.config.doWriteMatchedExp:
532  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
533 
534  if self.config.doDetection:
535  self.log.info("Computing diffim PSF")
536  if subtractedExposure is None:
537  subtractedExposure = sensorRef.get(subtractedExposureName)
538 
539  # Get Psf from the appropriate input image if it doesn't exist
540  if not subtractedExposure.hasPsf():
541  if self.config.convolveTemplate:
542  subtractedExposure.setPsf(exposure.getPsf())
543  else:
544  if templateExposure is None:
545  template = self.getTemplate.run(exposure, sensorRef,
546  templateIdList=templateIdList)
547  subtractedExposure.setPsf(template.exposure.getPsf())
548 
549  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
550  # thus it may have already been decorrelated. Thus, we do not decorrelate if
551  # doSubtract is False.
552  if self.config.doDecorrelation and self.config.doSubtract:
553  preConvKernel = None
554  if preConvPsf is not None:
555  preConvKernel = preConvPsf.getLocalKernel()
556  decorrResult = self.decorrelate.run(exposure, templateExposure,
557  subtractedExposure,
558  subtractRes.psfMatchingKernel,
559  spatiallyVarying=self.config.doSpatiallyVarying,
560  preConvKernel=preConvKernel)
561  subtractedExposure = decorrResult.correctedExposure
562 
563  # END (if subtractAlgorithm == 'AL')
564 
565  if self.config.doDetection:
566  self.log.info("Running diaSource detection")
567  # Erase existing detection mask planes
568  mask = subtractedExposure.getMaskedImage().getMask()
569  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
570 
571  table = afwTable.SourceTable.make(self.schema, idFactory)
572  table.setMetadata(self.algMetadata)
573  results = self.detection.makeSourceCatalog(
574  table=table,
575  exposure=subtractedExposure,
576  doSmooth=not self.config.doPreConvolve
577  )
578 
579  if self.config.doMerge:
580  fpSet = results.fpSets.positive
581  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
582  self.config.growFootprint, False)
583  diaSources = afwTable.SourceCatalog(table)
584  fpSet.makeSources(diaSources)
585  self.log.info("Merging detections into %d sources" % (len(diaSources)))
586  else:
587  diaSources = results.sources
588 
589  if self.config.doMeasurement:
590  newDipoleFitting = self.config.doDipoleFitting
591  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
592  if not newDipoleFitting:
593  # Just fit dipole in diffim
594  self.measurement.run(diaSources, subtractedExposure)
595  else:
596  # Use (matched) template and science image (if avail.) to constrain dipole fitting
597  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
598  self.measurement.run(diaSources, subtractedExposure, exposure,
599  subtractRes.matchedExposure)
600  else:
601  self.measurement.run(diaSources, subtractedExposure, exposure)
602 
603  # Match with the calexp sources if possible
604  if self.config.doMatchSources:
605  if sensorRef.datasetExists("src"):
606  # Create key,val pair where key=diaSourceId and val=sourceId
607  matchRadAsec = self.config.diaSourceMatchRadius
608  matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
609 
610  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
611  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
612  srcMatch in srcMatches])
613  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
614  len(diaSources)))
615  else:
616  self.log.warn("Src product does not exist; cannot match with diaSources")
617  srcMatchDict = {}
618 
619  # Create key,val pair where key=diaSourceId and val=refId
620  refAstromConfig = AstrometryConfig()
621  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
622  refAstrometer = AstrometryTask(refAstromConfig)
623  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
624  refMatches = astromRet.matches
625  if refMatches is None:
626  self.log.warn("No diaSource matches with reference catalog")
627  refMatchDict = {}
628  else:
629  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
630  len(diaSources)))
631  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
632  refMatch in refMatches])
633 
634  # Assign source Ids
635  for diaSource in diaSources:
636  sid = diaSource.getId()
637  if sid in srcMatchDict:
638  diaSource.set("srcMatchId", srcMatchDict[sid])
639  if sid in refMatchDict:
640  diaSource.set("refMatchId", refMatchDict[sid])
641 
642  if diaSources is not None and self.config.doWriteSources:
643  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
644 
645  if self.config.doAddMetrics and self.config.doSelectSources:
646  self.log.info("Evaluating metrics and control sample")
647 
648  kernelCandList = []
649  for cell in subtractRes.kernelCellSet.getCellList():
650  for cand in cell.begin(False): # include bad candidates
651  kernelCandList.append(cand)
652 
653  # Get basis list to build control sample kernels
654  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
655 
656  controlCandList = \
657  diffimTools.sourceTableToCandidateList(controlSources,
658  subtractRes.warpedExposure, exposure,
659  self.config.subtract.kernel.active,
660  self.config.subtract.kernel.active.detectionConfig,
661  self.log, doBuild=True, basisList=basisList)
662 
663  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
664  dof=nparam)
665  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
666 
667  if self.config.doDetection:
668  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
669  else:
670  kcQa.aggregate(selectSources, self.metadata, allresids)
671 
672  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
673 
674  if self.config.doWriteSubtractedExp:
675  sensorRef.put(subtractedExposure, subtractedExposureName)
676 
677  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
678  return pipeBase.Struct(
679  subtractedExposure=subtractedExposure,
680  subtractRes=subtractRes,
681  sources=diaSources,
682  )
683 
684  def fitAstrometry(self, templateSources, templateExposure, selectSources):
685  """Fit the relative astrometry between templateSources and selectSources
686 
687  @todo remove this method. It originally fit a new WCS to the template before calling register.run
688  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
689  It remains because a subtask overrides it.
690  """
691  results = self.register.run(templateSources, templateExposure.getWcs(),
692  templateExposure.getBBox(), selectSources)
693  return results
694 
695  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
696  """@todo Test and update for current debug display and slot names
697  """
698  import lsstDebug
699  display = lsstDebug.Info(__name__).display
700  showSubtracted = lsstDebug.Info(__name__).showSubtracted
701  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
702  showDiaSources = lsstDebug.Info(__name__).showDiaSources
703  showDipoles = lsstDebug.Info(__name__).showDipoles
704  maskTransparency = lsstDebug.Info(__name__).maskTransparency
705  if display:
706  import lsst.afw.display.ds9 as ds9
707  if not maskTransparency:
708  maskTransparency = 0
709  ds9.setMaskTransparency(maskTransparency)
710 
711  if display and showSubtracted:
712  ds9.mtv(subtractRes.subtractedExposure, frame=lsstDebug.frame, title="Subtracted image")
713  mi = subtractRes.subtractedExposure.getMaskedImage()
714  x0, y0 = mi.getX0(), mi.getY0()
715  with ds9.Buffering():
716  for s in diaSources:
717  x, y = s.getX() - x0, s.getY() - y0
718  ctype = "red" if s.get("flags.negative") else "yellow"
719  if (s.get("flags.pixel.interpolated.center") or s.get("flags.pixel.saturated.center") or
720  s.get("flags.pixel.cr.center")):
721  ptype = "x"
722  elif (s.get("flags.pixel.interpolated.any") or s.get("flags.pixel.saturated.any") or
723  s.get("flags.pixel.cr.any")):
724  ptype = "+"
725  else:
726  ptype = "o"
727  ds9.dot(ptype, x, y, size=4, frame=lsstDebug.frame, ctype=ctype)
728  lsstDebug.frame += 1
729 
730  if display and showPixelResiduals and selectSources:
731  nonKernelSources = []
732  for source in selectSources:
733  if source not in kernelSources:
734  nonKernelSources.append(source)
735 
736  diUtils.plotPixelResiduals(exposure,
737  subtractRes.warpedExposure,
738  subtractRes.subtractedExposure,
739  subtractRes.kernelCellSet,
740  subtractRes.psfMatchingKernel,
741  subtractRes.backgroundModel,
742  nonKernelSources,
743  self.subtract.config.kernel.active.detectionConfig,
744  origVariance=False)
745  diUtils.plotPixelResiduals(exposure,
746  subtractRes.warpedExposure,
747  subtractRes.subtractedExposure,
748  subtractRes.kernelCellSet,
749  subtractRes.psfMatchingKernel,
750  subtractRes.backgroundModel,
751  nonKernelSources,
752  self.subtract.config.kernel.active.detectionConfig,
753  origVariance=True)
754  if display and showDiaSources:
755  flagChecker = SourceFlagChecker(diaSources)
756  isFlagged = [flagChecker(x) for x in diaSources]
757  isDipole = [x.get("classification.dipole") for x in diaSources]
758  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
759  frame=lsstDebug.frame)
760  lsstDebug.frame += 1
761 
762  if display and showDipoles:
763  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
764  frame=lsstDebug.frame)
765  lsstDebug.frame += 1
766 
767  def _getConfigName(self):
768  """Return the name of the config dataset
769  """
770  return "%sDiff_config" % (self.config.coaddName,)
771 
772  def _getMetadataName(self):
773  """Return the name of the metadata dataset
774  """
775  return "%sDiff_metadata" % (self.config.coaddName,)
776 
777  def getSchemaCatalogs(self):
778  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
779  diaSrc = afwTable.SourceCatalog(self.schema)
780  diaSrc.getTable().setMetadata(self.algMetadata)
781  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
782 
783  @classmethod
784  def _makeArgumentParser(cls):
785  """Create an argument parser
786  """
787  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
788  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
789  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
790  help="Optional template data ID (visit only), e.g. --templateId visit=6789")
791  return parser
792 
793 
795  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
796  doc="Shift stars going into RegisterTask by this amount")
797  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
798  doc="Perturb stars going into RegisterTask by this amount")
799 
800  def setDefaults(self):
801  ImageDifferenceConfig.setDefaults(self)
802  self.getTemplate.retarget(GetCalexpAsTemplateTask)
803 
804 
806  """!Image difference Task used in the Winter 2013 data challege.
807  Enables testing the effects of registration shifts and scatter.
808 
809  For use with winter 2013 simulated images:
810  Use --templateId visit=88868666 for sparse data
811  --templateId visit=22222200 for dense data (g)
812  --templateId visit=11111100 for dense data (i)
813  """
814  ConfigClass = Winter2013ImageDifferenceConfig
815  _DefaultName = "winter2013ImageDifference"
816 
817  def __init__(self, **kwargs):
818  ImageDifferenceTask.__init__(self, **kwargs)
819 
820  def fitAstrometry(self, templateSources, templateExposure, selectSources):
821  """Fit the relative astrometry between templateSources and selectSources"""
822  if self.config.winter2013WcsShift > 0.0:
823  offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
824  self.config.winter2013WcsShift)
825  cKey = templateSources[0].getTable().getCentroidKey()
826  for source in templateSources:
827  centroid = source.get(cKey)
828  source.set(cKey, centroid+offset)
829  elif self.config.winter2013WcsRms > 0.0:
830  cKey = templateSources[0].getTable().getCentroidKey()
831  for source in templateSources:
832  offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
833  self.config.winter2013WcsRms*numpy.random.normal())
834  centroid = source.get(cKey)
835  source.set(cKey, centroid+offset)
836 
837  results = self.register.run(templateSources, templateExposure.getWcs(),
838  templateExposure.getBBox(), selectSources)
839  return results
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def run(self, sensorRef, templateIdList=None)
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)