lsst.pipe.tasks  13.0-54-gc325bc5
 All Classes Namespaces Files Functions Variables Groups Pages
imageDifference.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 from builtins import zip
24 import math
25 import random
26 import numpy
27 
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.daf.base as dafBase
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.math as afwMath
33 import lsst.afw.table as afwTable
34 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
35 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, \
38  ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import ImagePsfMatchTask, DipoleAnalysis, \
40  SourceFlagChecker, KernelCandidateF, makeKernelBasisList, \
41  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, \
42  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask, DecorrelateALKernelTask
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 
46 FwhmPerSigma = 2 * math.sqrt(2 * math.log(2))
47 IqrToSigma = 0.741
48 
49 
50 class ImageDifferenceConfig(pexConfig.Config):
51  """Config for ImageDifferenceTask
52  """
53  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=True,
54  doc="Add background to calexp before processing it. "
55  "Useful as ipDiffim does background matching.")
56  doUseRegister = pexConfig.Field(dtype=bool, default=True,
57  doc="Use image-to-image registration to align template with "
58  "science image")
59  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
60  doc="Writing debugging data for doUseRegister")
61  doSelectSources = pexConfig.Field(dtype=bool, default=True,
62  doc="Select stars to use for kernel fitting")
63  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
64  doc="Select stars of extreme color as part of the control sample")
65  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
66  doc="Select stars that are variable to be part "
67  "of the control sample")
68  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
69  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
70  doc="Convolve science image by its PSF before PSF-matching?")
71  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
72  doc="Use a simple gaussian PSF model for pre-convolution "
73  "(else use fit PSF)? Ignored if doPreConvolve false.")
74  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
75  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
76  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
77  "kernel convolution? If True, also update the diffim PSF.")
78  doMerge = pexConfig.Field(dtype=bool, default=True,
79  doc="Merge positive and negative diaSources with grow radius "
80  "set by growFootprint")
81  doMatchSources = pexConfig.Field(dtype=bool, default=True,
82  doc="Match diaSources with input calexp sources and ref catalog sources")
83  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
84  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
85  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
86  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
87  doc="Write warped and PSF-matched template coadd exposure?")
88  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
89  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
90  doc="Add columns to the source table to hold analysis metrics?")
91 
92  coaddName = pexConfig.Field(
93  doc="coadd name: typically one of deep or goodSeeing",
94  dtype=str,
95  default="deep",
96  )
97  convolveTemplate = pexConfig.Field(
98  doc="Which image gets convolved (default = template)",
99  dtype=bool,
100  default=True
101  )
102  refObjLoader = pexConfig.ConfigurableField(
103  target=LoadAstrometryNetObjectsTask,
104  doc="reference object loader",
105  )
106  astrometer = pexConfig.ConfigurableField(
107  target=AstrometryTask,
108  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
109  )
110  sourceSelector = pexConfig.ConfigurableField(
111  target=ObjectSizeStarSelectorTask,
112  doc="Source selection algorithm",
113  )
114  subtract = pexConfig.ConfigurableField(
115  target=ImagePsfMatchTask,
116  doc="Warp and PSF match template to exposure, then subtract",
117  )
118  decorrelate = pexConfig.ConfigurableField(
119  target=DecorrelateALKernelTask,
120  doc="""Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True.
121  If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the
122  default of 5.5).""",
123  )
124  detection = pexConfig.ConfigurableField(
125  target=SourceDetectionTask,
126  doc="Low-threshold detection for final measurement",
127  )
128  # measurement = pexConfig.ConfigurableField(
129  # target=DipoleMeasurementTask,
130  # doc="Final source measurement on low-threshold detections; dipole fitting enabled.",
131  # )
132  measurement = pexConfig.ConfigurableField(
133  target=DipoleFitTask,
134  doc="Enable updated dipole fitting method.",
135  )
136  getTemplate = pexConfig.ConfigurableField(
137  target=GetCoaddAsTemplateTask,
138  doc="Subtask to retrieve template exposure and sources",
139  )
140  controlStepSize = pexConfig.Field(
141  doc="What step size (every Nth one) to select a control sample from the kernelSources",
142  dtype=int,
143  default=5
144  )
145  controlRandomSeed = pexConfig.Field(
146  doc = "Random seed for shuffing the control sample",
147  dtype = int,
148  default = 10
149  )
150  register = pexConfig.ConfigurableField(
151  target=RegisterTask,
152  doc="Task to enable image-to-image image registration (warping)",
153  )
154  kernelSourcesFromRef = pexConfig.Field(
155  doc="Select sources to measure kernel from reference catalog if True, template if false",
156  dtype=bool,
157  default=False
158  )
159  templateSipOrder = pexConfig.Field(dtype=int, default=2,
160  doc="Sip Order for fitting the Template Wcs "
161  "(default is too high, overfitting)")
162 
163  growFootprint = pexConfig.Field(dtype=int, default=2,
164  doc="Grow positive and negative footprints by this amount before merging")
165 
166  diaSourceMatchRadius = pexConfig.Field(dtype=float, default=0.5,
167  doc="Match radius (in arcseconds) "
168  "for DiaSource to Source association")
169 
170  def setDefaults(self):
171  # defaults are OK for catalog and diacatalog
172 
173  self.subtract.kernel.name = "AL"
174  self.subtract.kernel.active.fitForBackground = True
175  self.subtract.kernel.active.spatialKernelOrder = 1
176  self.subtract.kernel.active.spatialBgOrder = 0
177  self.doPreConvolve = False
178  self.doMatchSources = False
179  self.doAddMetrics = False
180  self.doUseRegister = False
181 
182  # DiaSource Detection
183  self.detection.thresholdPolarity = "both"
184  self.detection.thresholdValue = 5.5
185  self.detection.reEstimateBackground = False
186  self.detection.thresholdType = "pixel_stdev"
187 
188  # Add filtered flux measurement, the correct measurement for pre-convolved images.
189  # Enable all measurements, regardless of doPreConvolved, as it makes data harvesting easier.
190  # To change that you must modify algorithms.names in the task's applyOverrides method,
191  # after the user has set doPreConvolved.
192  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
193 
194  # For shuffling the control sample
195  random.seed(self.controlRandomSeed)
196 
197  def validate(self):
198  pexConfig.Config.validate(self)
199  if self.doMeasurement and not self.doDetection:
200  raise ValueError("Cannot run source measurement without source detection.")
201  if self.doMerge and not self.doDetection:
202  raise ValueError("Cannot run source merging without source detection.")
203  if self.doUseRegister and not self.doSelectSources:
204  raise ValueError("doUseRegister=True and doSelectSources=False. " +
205  "Cannot run RegisterTask without selecting sources.")
206 
207 
208 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
209 
210  @staticmethod
211  def getTargetList(parsedCmd, **kwargs):
212  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
213  **kwargs)
214 
215 
216 class ImageDifferenceTask(pipeBase.CmdLineTask):
217  """Subtract an image from a template and measure the result
218  """
219  ConfigClass = ImageDifferenceConfig
220  RunnerClass = ImageDifferenceTaskRunner
221  _DefaultName = "imageDifference"
222 
223  def __init__(self, butler=None, **kwargs):
224  """!Construct an ImageDifference Task
225 
226  @param[in] butler Butler object to use in constructing reference object loaders
227  """
228  pipeBase.CmdLineTask.__init__(self, **kwargs)
229  self.makeSubtask("subtract")
230  self.makeSubtask("getTemplate")
231  self.makeSubtask("decorrelate")
232 
233  if self.config.doUseRegister:
234  self.makeSubtask("register")
235  self.schema = afwTable.SourceTable.makeMinimalSchema()
236 
237  if self.config.doSelectSources:
238  self.makeSubtask("sourceSelector", schema=self.schema)
239  self.makeSubtask('refObjLoader', butler=butler)
240  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
241 
242  self.algMetadata = dafBase.PropertyList()
243  if self.config.doDetection:
244  self.makeSubtask("detection", schema=self.schema)
245  if self.config.doMeasurement:
246  if not self.config.doDipoleFitting:
247  self.makeSubtask("measurement", schema=self.schema,
248  algMetadata=self.algMetadata)
249  else:
250  self.makeSubtask("measurement", schema=self.schema)
251  if self.config.doMatchSources:
252  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
253  self.schema.addField("srcMatchId", "L", "unique id of source match")
254 
255  @pipeBase.timeMethod
256  def run(self, sensorRef, templateIdList=None):
257  """Subtract an image from a template coadd and measure the result
258 
259  Steps include:
260  - warp template coadd to match WCS of image
261  - PSF match image to warped template
262  - subtract image from PSF-matched, warped template
263  - persist difference image
264  - detect sources
265  - measure sources
266 
267  @param sensorRef: sensor-level butler data reference, used for the following data products:
268  Input only:
269  - calexp
270  - psf
271  - ccdExposureId
272  - ccdExposureId_bits
273  - self.config.coaddName + "Coadd_skyMap"
274  - self.config.coaddName + "Coadd"
275  Input or output, depending on config:
276  - self.config.coaddName + "Diff_subtractedExp"
277  Output, depending on config:
278  - self.config.coaddName + "Diff_matchedExp"
279  - self.config.coaddName + "Diff_src"
280 
281  @return pipe_base Struct containing these fields:
282  - subtractedExposure: exposure after subtracting template;
283  the unpersisted version if subtraction not run but detection run
284  None if neither subtraction nor detection run (i.e. nothing useful done)
285  - subtractRes: results of subtraction task; None if subtraction not run
286  - sources: detected and possibly measured sources; None if detection not run
287  """
288  self.log.info("Processing %s" % (sensorRef.dataId))
289 
290  # initialize outputs and some intermediate products
291  subtractedExposure = None
292  subtractRes = None
293  selectSources = None
294  kernelSources = None
295  controlSources = None
296  diaSources = None
297 
298  # We make one IdFactory that will be used by both icSrc and src datasets;
299  # I don't know if this is the way we ultimately want to do things, but at least
300  # this ensures the source IDs are fully unique.
301  expBits = sensorRef.get("ccdExposureId_bits")
302  expId = int(sensorRef.get("ccdExposureId"))
303  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
304 
305  # Retrieve the science image we wish to analyze
306  exposure = sensorRef.get("calexp", immediate=True)
307  if self.config.doAddCalexpBackground:
308  mi = exposure.getMaskedImage()
309  mi += sensorRef.get("calexpBackground").getImage()
310  if not exposure.hasPsf():
311  raise pipeBase.TaskError("Exposure has no psf")
312  sciencePsf = exposure.getPsf()
313 
314  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
315  templateExposure = None # Stitched coadd exposure
316  templateSources = None # Sources on the template image
317  if self.config.doSubtract:
318  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
319  templateExposure = template.exposure
320  templateSources = template.sources
321 
322  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
323  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
324 
325  # sigma of PSF of template image before warping
326  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
327 
328  # if requested, convolve the science exposure with its PSF
329  # (properly, this should be a cross-correlation, but our code does not yet support that)
330  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
331  # else sigma of original science exposure
332  if self.config.doPreConvolve:
333  convControl = afwMath.ConvolutionControl()
334  # cannot convolve in place, so make a new MI to receive convolved image
335  srcMI = exposure.getMaskedImage()
336  destMI = srcMI.Factory(srcMI.getDimensions())
337  srcPsf = sciencePsf
338  if self.config.useGaussianForPreConvolution:
339  # convolve with a simplified PSF model: a double Gaussian
340  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
341  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
342  else:
343  # convolve with science exposure's PSF model
344  preConvPsf = srcPsf
345  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
346  exposure.setMaskedImage(destMI)
347  scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
348  else:
349  scienceSigmaPost = scienceSigmaOrig
350 
351  # If requested, find sources in the image
352  if self.config.doSelectSources:
353  if not sensorRef.datasetExists("src"):
354  self.log.warn("Src product does not exist; running detection, measurement, selection")
355  # Run own detection and measurement; necessary in nightly processing
356  selectSources = self.subtract.getSelectSources(
357  exposure,
358  sigma=scienceSigmaPost,
359  doSmooth=not self.doPreConvolve,
360  idFactory=idFactory,
361  )
362  else:
363  self.log.info("Source selection via src product")
364  # Sources already exist; for data release processing
365  selectSources = sensorRef.get("src")
366 
367  # Number of basis functions
368  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
369  referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
370  targetFwhmPix=templateSigma * FwhmPerSigma))
371 
372  if self.config.doAddMetrics:
373  # Modify the schema of all Sources
374  kcQa = KernelCandidateQa(nparam)
375  selectSources = kcQa.addToSchema(selectSources)
376 
377  if self.config.kernelSourcesFromRef:
378  # match exposure sources to reference catalog
379  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
380  matches = astromRet.matches
381  elif templateSources:
382  # match exposure sources to template sources
383  mc = afwTable.MatchControl()
384  mc.findOnlyClosest = False
385  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
386  mc)
387  else:
388  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
389  "but template sources not available. Cannot match science " +
390  "sources with template sources. Run process* on data from " +
391  "which templates are built.")
392 
393  kernelSources = self.sourceSelector.selectStars(exposure, selectSources,
394  matches=matches).starCat
395 
396  random.shuffle(kernelSources, random.random)
397  controlSources = kernelSources[::self.config.controlStepSize]
398  kernelSources = [k for i, k in enumerate(kernelSources) if i % self.config.controlStepSize]
399 
400  if self.config.doSelectDcrCatalog:
401  redSelector = DiaCatalogSourceSelectorTask(
402  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax, grMax=99.999))
403  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
404  controlSources.extend(redSources)
405 
406  blueSelector = DiaCatalogSourceSelectorTask(
407  DiaCatalogSourceSelectorConfig(grMin=-99.999, grMax=self.sourceSelector.config.grMin))
408  blueSources = blueSelector.selectStars(exposure, selectSources, matches=matches).starCat
409  controlSources.extend(blueSources)
410 
411  if self.config.doSelectVariableCatalog:
412  varSelector = DiaCatalogSourceSelectorTask(
413  DiaCatalogSourceSelectorConfig(includeVariable=True))
414  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
415  controlSources.extend(varSources)
416 
417  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
418  % (len(kernelSources), len(selectSources), len(controlSources)))
419  allresids = {}
420  if self.config.doUseRegister:
421  self.log.info("Registering images")
422 
423  if templateSources is None:
424  # Run detection on the template, which is
425  # temporarily background-subtracted
426  templateSources = self.subtract.getSelectSources(
427  templateExposure,
428  sigma=templateSigma,
429  doSmooth=True,
430  idFactory=idFactory
431  )
432 
433  # Third step: we need to fit the relative astrometry.
434  #
435  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
436  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
437  exposure.getWcs(), exposure.getBBox())
438  templateExposure = warpedExp
439 
440  # Create debugging outputs on the astrometric
441  # residuals as a function of position. Persistence
442  # not yet implemented; expected on (I believe) #2636.
443  if self.config.doDebugRegister:
444  # Grab matches to reference catalog
445  srcToMatch = {x.second.getId(): x.first for x in matches}
446 
447  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
448  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
449  sids = [m.first.getId() for m in wcsResults.matches]
450  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
451  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
452  m.second.get(inCentroidKey))) for m in wcsResults.matches]
453  allresids = dict(zip(sids, zip(positions, residuals)))
454 
455  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
456  wcsResults.wcs.pixelToSky(
457  m.second.get(inCentroidKey))) for m in wcsResults.matches]
458  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
459  2.5*numpy.log10(srcToMatch[x].get("r"))
460  for x in sids if x in srcToMatch.keys()])
461  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
462  if s in srcToMatch.keys()])
463  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
464  if s in srcToMatch.keys()])
465  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
466  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
467  (colors <= self.sourceSelector.config.grMax))
468  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
469  rms1Long = IqrToSigma * \
470  (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
471  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)-numpy.percentile(dlat[idx1], 25))
472  rms2Long = IqrToSigma * \
473  (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
474  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)-numpy.percentile(dlat[idx2], 25))
475  rms3Long = IqrToSigma * \
476  (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
477  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)-numpy.percentile(dlat[idx3], 25))
478  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx1]),
479  rms1Long,
480  numpy.median(dlat[idx1]),
481  rms1Lat))
482  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx2]),
483  rms2Long,
484  numpy.median(dlat[idx2]),
485  rms2Lat))
486  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" % (numpy.median(dlong[idx3]),
487  rms3Long,
488  numpy.median(dlat[idx3]),
489  rms3Lat))
490 
491  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
492  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
493  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
494  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
495  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
496  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
497 
498  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
499  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
500  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
501  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
502  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
503  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
504 
505  # warp template exposure to match exposure,
506  # PSF match template exposure to exposure,
507  # then return the difference
508 
509  # Return warped template... Construct sourceKernelCand list after subtract
510  self.log.info("Subtracting images")
511  subtractRes = self.subtract.subtractExposures(
512  templateExposure=templateExposure,
513  scienceExposure=exposure,
514  candidateList=kernelSources,
515  convolveTemplate=self.config.convolveTemplate,
516  doWarping=not self.config.doUseRegister
517  )
518  subtractedExposure = subtractRes.subtractedExposure
519 
520  if self.config.doWriteMatchedExp:
521  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
522 
523  if self.config.doDetection:
524  self.log.info("Computing diffim PSF")
525  if subtractedExposure is None:
526  subtractedExposure = sensorRef.get(subtractedExposureName)
527 
528  # Get Psf from the appropriate input image if it doesn't exist
529  if not subtractedExposure.hasPsf():
530  if self.config.convolveTemplate:
531  subtractedExposure.setPsf(exposure.getPsf())
532  else:
533  if templateExposure is None:
534  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
535  subtractedExposure.setPsf(template.exposure.getPsf())
536 
537  # If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have
538  # already been decorrelated. Thus, we do not do decorrelation if doSubtract is False.
539  if self.config.doDecorrelation and self.config.doSubtract:
540  decorrResult = self.decorrelate.run(exposure, templateExposure,
541  subtractedExposure,
542  subtractRes.psfMatchingKernel)
543  subtractedExposure = decorrResult.correctedExposure
544 
545  if self.config.doDetection:
546  self.log.info("Running diaSource detection")
547  # Erase existing detection mask planes
548  mask = subtractedExposure.getMaskedImage().getMask()
549  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
550 
551  table = afwTable.SourceTable.make(self.schema, idFactory)
552  table.setMetadata(self.algMetadata)
553  results = self.detection.makeSourceCatalog(
554  table=table,
555  exposure=subtractedExposure,
556  doSmooth=not self.config.doPreConvolve
557  )
558 
559  if self.config.doMerge:
560  fpSet = results.fpSets.positive
561  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
562  self.config.growFootprint, False)
563  diaSources = afwTable.SourceCatalog(table)
564  fpSet.makeSources(diaSources)
565  self.log.info("Merging detections into %d sources" % (len(diaSources)))
566  else:
567  diaSources = results.sources
568 
569  if self.config.doMeasurement:
570  self.log.info("Running diaSource measurement")
571  if not self.config.doDipoleFitting:
572  self.measurement.run(diaSources, subtractedExposure)
573  else:
574  if self.config.doSubtract:
575  self.measurement.run(diaSources, subtractedExposure, exposure,
576  subtractRes.matchedExposure)
577  else:
578  self.measurement.run(diaSources, subtractedExposure, exposure)
579 
580  # Match with the calexp sources if possible
581  if self.config.doMatchSources:
582  if sensorRef.datasetExists("src"):
583  # Create key,val pair where key=diaSourceId and val=sourceId
584  matchRadAsec = self.config.diaSourceMatchRadius
585  matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
586 
587  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
588  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
589  srcMatch in srcMatches])
590  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
591  len(diaSources)))
592  else:
593  self.log.warn("Src product does not exist; cannot match with diaSources")
594  srcMatchDict = {}
595 
596  # Create key,val pair where key=diaSourceId and val=refId
597  refAstromConfig = AstrometryConfig()
598  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
599  refAstrometer = AstrometryTask(refAstromConfig)
600  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
601  refMatches = astromRet.matches
602  if refMatches is None:
603  self.log.warn("No diaSource matches with reference catalog")
604  refMatchDict = {}
605  else:
606  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
607  len(diaSources)))
608  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
609  refMatch in refMatches])
610 
611  # Assign source Ids
612  for diaSource in diaSources:
613  sid = diaSource.getId()
614  if sid in srcMatchDict:
615  diaSource.set("srcMatchId", srcMatchDict[sid])
616  if sid in refMatchDict:
617  diaSource.set("refMatchId", refMatchDict[sid])
618 
619  if diaSources is not None and self.config.doWriteSources:
620  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
621 
622  if self.config.doAddMetrics and self.config.doSelectSources:
623  self.log.info("Evaluating metrics and control sample")
624 
625  kernelCandList = []
626  for cell in subtractRes.kernelCellSet.getCellList():
627  for cand in cell.begin(False): # include bad candidates
628  kernelCandList.append(cand)
629 
630  # Get basis list to build control sample kernels
631  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
632 
633  controlCandList = \
634  diffimTools.sourceTableToCandidateList(controlSources,
635  subtractRes.warpedExposure, exposure,
636  self.config.subtract.kernel.active,
637  self.config.subtract.kernel.active.detectionConfig,
638  self.log, doBuild=True, basisList=basisList)
639 
640  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
641  dof=nparam)
642  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
643 
644  if self.config.doDetection:
645  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
646  else:
647  kcQa.aggregate(selectSources, self.metadata, allresids)
648 
649  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
650 
651  if self.config.doWriteSubtractedExp:
652  sensorRef.put(subtractedExposure, subtractedExposureName)
653 
654  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
655  return pipeBase.Struct(
656  subtractedExposure=subtractedExposure,
657  subtractRes=subtractRes,
658  sources=diaSources,
659  )
660 
661  def fitAstrometry(self, templateSources, templateExposure, selectSources):
662  """Fit the relative astrometry between templateSources and selectSources
663 
664  @todo remove this method. It originally fit a new WCS to the template before calling register.run
665  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
666  It remains because a subtask overrides it.
667  """
668  results = self.register.run(templateSources, templateExposure.getWcs(),
669  templateExposure.getBBox(), selectSources)
670  return results
671 
672  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
673  """@todo Test and update for current debug display and slot names
674  """
675  import lsstDebug
676  display = lsstDebug.Info(__name__).display
677  showSubtracted = lsstDebug.Info(__name__).showSubtracted
678  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
679  showDiaSources = lsstDebug.Info(__name__).showDiaSources
680  showDipoles = lsstDebug.Info(__name__).showDipoles
681  maskTransparency = lsstDebug.Info(__name__).maskTransparency
682  if display:
683  import lsst.afw.display.ds9 as ds9
684  if not maskTransparency:
685  maskTransparency = 0
686  ds9.setMaskTransparency(maskTransparency)
687 
688  if display and showSubtracted:
689  ds9.mtv(subtractRes.subtractedExposure, frame=lsstDebug.frame, title="Subtracted image")
690  mi = subtractRes.subtractedExposure.getMaskedImage()
691  x0, y0 = mi.getX0(), mi.getY0()
692  with ds9.Buffering():
693  for s in diaSources:
694  x, y = s.getX() - x0, s.getY() - y0
695  ctype = "red" if s.get("flags.negative") else "yellow"
696  if (s.get("flags.pixel.interpolated.center") or s.get("flags.pixel.saturated.center") or
697  s.get("flags.pixel.cr.center")):
698  ptype = "x"
699  elif (s.get("flags.pixel.interpolated.any") or s.get("flags.pixel.saturated.any") or
700  s.get("flags.pixel.cr.any")):
701  ptype = "+"
702  else:
703  ptype = "o"
704  ds9.dot(ptype, x, y, size=4, frame=lsstDebug.frame, ctype=ctype)
705  lsstDebug.frame += 1
706 
707  if display and showPixelResiduals and selectSources:
708  nonKernelSources = []
709  for source in selectSources:
710  if source not in kernelSources:
711  nonKernelSources.append(source)
712 
713  diUtils.plotPixelResiduals(exposure,
714  subtractRes.warpedExposure,
715  subtractRes.subtractedExposure,
716  subtractRes.kernelCellSet,
717  subtractRes.psfMatchingKernel,
718  subtractRes.backgroundModel,
719  nonKernelSources,
720  self.subtract.config.kernel.active.detectionConfig,
721  origVariance=False)
722  diUtils.plotPixelResiduals(exposure,
723  subtractRes.warpedExposure,
724  subtractRes.subtractedExposure,
725  subtractRes.kernelCellSet,
726  subtractRes.psfMatchingKernel,
727  subtractRes.backgroundModel,
728  nonKernelSources,
729  self.subtract.config.kernel.active.detectionConfig,
730  origVariance=True)
731  if display and showDiaSources:
732  flagChecker = SourceFlagChecker(diaSources)
733  isFlagged = [flagChecker(x) for x in diaSources]
734  isDipole = [x.get("classification.dipole") for x in diaSources]
735  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
736  frame=lsstDebug.frame)
737  lsstDebug.frame += 1
738 
739  if display and showDipoles:
740  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
741  frame=lsstDebug.frame)
742  lsstDebug.frame += 1
743 
744  def _getConfigName(self):
745  """Return the name of the config dataset
746  """
747  return "%sDiff_config" % (self.config.coaddName,)
748 
749  def _getMetadataName(self):
750  """Return the name of the metadata dataset
751  """
752  return "%sDiff_metadata" % (self.config.coaddName,)
753 
754  def getSchemaCatalogs(self):
755  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
756  diaSrc = afwTable.SourceCatalog(self.schema)
757  diaSrc.getTable().setMetadata(self.algMetadata)
758  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
759 
760  @classmethod
761  def _makeArgumentParser(cls):
762  """Create an argument parser
763  """
764  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
765  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
766  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
767  help="Optional template data ID (visit only), e.g. --templateId visit=6789")
768  return parser
769 
770 
772  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
773  doc="Shift stars going into RegisterTask by this amount")
774  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
775  doc="Perturb stars going into RegisterTask by this amount")
776 
777  def setDefaults(self):
778  ImageDifferenceConfig.setDefaults(self)
779  self.getTemplate.retarget(GetCalexpAsTemplateTask)
780 
781 
783  """!Image difference Task used in the Winter 2013 data challege.
784  Enables testing the effects of registration shifts and scatter.
785 
786  For use with winter 2013 simulated images:
787  Use --templateId visit=88868666 for sparse data
788  --templateId visit=22222200 for dense data (g)
789  --templateId visit=11111100 for dense data (i)
790  """
791  ConfigClass = Winter2013ImageDifferenceConfig
792  _DefaultName = "winter2013ImageDifference"
793 
794  def __init__(self, **kwargs):
795  ImageDifferenceTask.__init__(self, **kwargs)
796 
797  def fitAstrometry(self, templateSources, templateExposure, selectSources):
798  """Fit the relative astrometry between templateSources and selectSources"""
799  if self.config.winter2013WcsShift > 0.0:
800  offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
801  self.config.winter2013WcsShift)
802  cKey = templateSources[0].getTable().getCentroidKey()
803  for source in templateSources:
804  centroid = source.get(cKey)
805  source.set(cKey, centroid+offset)
806  elif self.config.winter2013WcsRms > 0.0:
807  cKey = templateSources[0].getTable().getCentroidKey()
808  for source in templateSources:
809  offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
810  self.config.winter2013WcsRms*numpy.random.normal())
811  centroid = source.get(cKey)
812  source.set(cKey, centroid+offset)
813 
814  results = self.register.run(templateSources, templateExposure.getWcs(),
815  templateExposure.getBBox(), selectSources)
816  return results
Image difference Task used in the Winter 2013 data challege.
def __init__
Construct an ImageDifference Task.