lsst.pipe.tasks  16.0-13-g1e751bcc+8
imageDifference.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import random
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
28 import lsst.daf.base as dafBase
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.math as afwMath
31 import lsst.afw.table as afwTable
32 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
33 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
34 from lsst.pipe.tasks.registerImage import RegisterTask
35 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, \
36  ObjectSizeStarSelectorTask
37 from lsst.ip.diffim import DipoleAnalysis, \
38  SourceFlagChecker, KernelCandidateF, makeKernelBasisList, \
39  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, \
40  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask, \
41  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry
42 import lsst.ip.diffim.diffimTools as diffimTools
43 import lsst.ip.diffim.utils as diUtils
44 
45 FwhmPerSigma = 2 * math.sqrt(2 * math.log(2))
46 IqrToSigma = 0.741
47 
48 
49 class ImageDifferenceConfig(pexConfig.Config):
50  """Config for ImageDifferenceTask
51  """
52  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=True,
53  doc="Add background to calexp before processing it. "
54  "Useful as ipDiffim does background matching.")
55  doUseRegister = pexConfig.Field(dtype=bool, default=True,
56  doc="Use image-to-image registration to align template with "
57  "science image")
58  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
59  doc="Writing debugging data for doUseRegister")
60  doSelectSources = pexConfig.Field(dtype=bool, default=True,
61  doc="Select stars to use for kernel fitting")
62  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
63  doc="Select stars of extreme color as part of the control sample")
64  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
65  doc="Select stars that are variable to be part "
66  "of the control sample")
67  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
68  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
69  doc="Convolve science image by its PSF before PSF-matching?")
70  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
71  doc="Use a simple gaussian PSF model for pre-convolution "
72  "(else use fit PSF)? Ignored if doPreConvolve false.")
73  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
74  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
75  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
76  "kernel convolution? If True, also update the diffim PSF.")
77  doMerge = pexConfig.Field(dtype=bool, default=True,
78  doc="Merge positive and negative diaSources with grow radius "
79  "set by growFootprint")
80  doMatchSources = pexConfig.Field(dtype=bool, default=True,
81  doc="Match diaSources with input calexp sources and ref catalog sources")
82  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
83  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
84  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
85  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
86  doc="Write warped and PSF-matched template coadd exposure?")
87  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
88  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
89  doc="Add columns to the source table to hold analysis metrics?")
90 
91  coaddName = pexConfig.Field(
92  doc="coadd name: typically one of deep, goodSeeing, or dcr",
93  dtype=str,
94  default="deep",
95  )
96  convolveTemplate = pexConfig.Field(
97  doc="Which image gets convolved (default = template)",
98  dtype=bool,
99  default=True
100  )
101  refObjLoader = pexConfig.ConfigurableField(
102  target=LoadAstrometryNetObjectsTask,
103  doc="reference object loader",
104  )
105  astrometer = pexConfig.ConfigurableField(
106  target=AstrometryTask,
107  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
108  )
109  sourceSelector = pexConfig.ConfigurableField(
110  target=ObjectSizeStarSelectorTask,
111  doc="Source selection algorithm",
112  )
113  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
114  decorrelate = pexConfig.ConfigurableField(
115  target=DecorrelateALKernelSpatialTask,
116  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
117  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
118  "default of 5.5).",
119  )
120  doSpatiallyVarying = pexConfig.Field(
121  dtype=bool,
122  default=False,
123  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
124  "image in order to allow for spatial variations"
125  )
126  detection = pexConfig.ConfigurableField(
127  target=SourceDetectionTask,
128  doc="Low-threshold detection for final measurement",
129  )
130  measurement = pexConfig.ConfigurableField(
131  target=DipoleFitTask,
132  doc="Enable updated dipole fitting method",
133  )
134  getTemplate = pexConfig.ConfigurableField(
135  target=GetCoaddAsTemplateTask,
136  doc="Subtask to retrieve template exposure and sources",
137  )
138  controlStepSize = pexConfig.Field(
139  doc="What step size (every Nth one) to select a control sample from the kernelSources",
140  dtype=int,
141  default=5
142  )
143  controlRandomSeed = pexConfig.Field(
144  doc="Random seed for shuffing the control sample",
145  dtype=int,
146  default=10
147  )
148  register = pexConfig.ConfigurableField(
149  target=RegisterTask,
150  doc="Task to enable image-to-image image registration (warping)",
151  )
152  kernelSourcesFromRef = pexConfig.Field(
153  doc="Select sources to measure kernel from reference catalog if True, template if false",
154  dtype=bool,
155  default=False
156  )
157  templateSipOrder = pexConfig.Field(dtype=int, default=2,
158  doc="Sip Order for fitting the Template Wcs "
159  "(default is too high, overfitting)")
160 
161  growFootprint = pexConfig.Field(dtype=int, default=2,
162  doc="Grow positive and negative footprints by this amount before merging")
163 
164  diaSourceMatchRadius = pexConfig.Field(dtype=float, default=0.5,
165  doc="Match radius (in arcseconds) "
166  "for DiaSource to Source association")
167 
168  def setDefaults(self):
169  # defaults are OK for catalog and diacatalog
170 
171  self.subtract['al'].kernel.name = "AL"
172  self.subtract['al'].kernel.active.fitForBackground = True
173  self.subtract['al'].kernel.active.spatialKernelOrder = 1
174  self.subtract['al'].kernel.active.spatialBgOrder = 0
175  self.doPreConvolve = False
176  self.doMatchSources = False
177  self.doAddMetrics = False
178  self.doUseRegister = False
179 
180  # DiaSource Detection
181  self.detection.thresholdPolarity = "both"
182  self.detection.thresholdValue = 5.5
183  self.detection.reEstimateBackground = False
184  self.detection.thresholdType = "pixel_stdev"
185 
186  # Add filtered flux measurement, the correct measurement for pre-convolved images.
187  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
188  # To change that you must modify algorithms.names in the task's applyOverrides method,
189  # after the user has set doPreConvolve.
190  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
191 
192  # For shuffling the control sample
193  random.seed(self.controlRandomSeed)
194 
195  def validate(self):
196  pexConfig.Config.validate(self)
197  if self.doMeasurement and not self.doDetection:
198  raise ValueError("Cannot run source measurement without source detection.")
199  if self.doMerge and not self.doDetection:
200  raise ValueError("Cannot run source merging without source detection.")
201  if self.doUseRegister and not self.doSelectSources:
202  raise ValueError("doUseRegister=True and doSelectSources=False. " +
203  "Cannot run RegisterTask without selecting sources.")
204  if hasattr(self.getTemplate, "coaddName"):
205  if self.getTemplate.coaddName != self.coaddName:
206  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
207 
208 
209 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
210 
211  @staticmethod
212  def getTargetList(parsedCmd, **kwargs):
213  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
214  **kwargs)
215 
216 
217 class ImageDifferenceTask(pipeBase.CmdLineTask):
218  """Subtract an image from a template and measure the result
219  """
220  ConfigClass = ImageDifferenceConfig
221  RunnerClass = ImageDifferenceTaskRunner
222  _DefaultName = "imageDifference"
223 
224  def __init__(self, butler=None, **kwargs):
225  """!Construct an ImageDifference Task
226 
227  @param[in] butler Butler object to use in constructing reference object loaders
228  """
229  pipeBase.CmdLineTask.__init__(self, **kwargs)
230  self.makeSubtask("getTemplate")
231 
232  self.makeSubtask("subtract")
233 
234  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
235  self.makeSubtask("decorrelate")
236 
237  if self.config.doUseRegister:
238  self.makeSubtask("register")
239  self.schema = afwTable.SourceTable.makeMinimalSchema()
240 
241  if self.config.doSelectSources:
242  self.makeSubtask("sourceSelector")
243  self.makeSubtask('refObjLoader', butler=butler)
244  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
245 
246  self.algMetadata = dafBase.PropertyList()
247  if self.config.doDetection:
248  self.makeSubtask("detection", schema=self.schema)
249  if self.config.doMeasurement:
250  self.makeSubtask("measurement", schema=self.schema,
251  algMetadata=self.algMetadata)
252  if self.config.doMatchSources:
253  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
254  self.schema.addField("srcMatchId", "L", "unique id of source match")
255 
256  @pipeBase.timeMethod
257  def runDataRef(self, sensorRef, templateIdList=None):
258  """Subtract an image from a template coadd and measure the result
259 
260  Steps include:
261  - warp template coadd to match WCS of image
262  - PSF match image to warped template
263  - subtract image from PSF-matched, warped template
264  - persist difference image
265  - detect sources
266  - measure sources
267 
268  @param sensorRef: sensor-level butler data reference, used for the following data products:
269  Input only:
270  - calexp
271  - psf
272  - ccdExposureId
273  - ccdExposureId_bits
274  - self.config.coaddName + "Coadd_skyMap"
275  - self.config.coaddName + "Coadd"
276  Input or output, depending on config:
277  - self.config.coaddName + "Diff_subtractedExp"
278  Output, depending on config:
279  - self.config.coaddName + "Diff_matchedExp"
280  - self.config.coaddName + "Diff_src"
281 
282  @return pipe_base Struct containing these fields:
283  - subtractedExposure: exposure after subtracting template;
284  the unpersisted version if subtraction not run but detection run
285  None if neither subtraction nor detection run (i.e. nothing useful done)
286  - subtractRes: results of subtraction task; None if subtraction not run
287  - sources: detected and possibly measured sources; None if detection not run
288  """
289  self.log.info("Processing %s" % (sensorRef.dataId))
290 
291  # initialize outputs and some intermediate products
292  subtractedExposure = None
293  subtractRes = None
294  selectSources = None
295  kernelSources = None
296  controlSources = None
297  diaSources = None
298 
299  # We make one IdFactory that will be used by both icSrc and src datasets;
300  # I don't know if this is the way we ultimately want to do things, but at least
301  # this ensures the source IDs are fully unique.
302  expBits = sensorRef.get("ccdExposureId_bits")
303  expId = int(sensorRef.get("ccdExposureId"))
304  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
305 
306  # Retrieve the science image we wish to analyze
307  exposure = sensorRef.get("calexp", immediate=True)
308  if self.config.doAddCalexpBackground:
309  mi = exposure.getMaskedImage()
310  mi += sensorRef.get("calexpBackground").getImage()
311  if not exposure.hasPsf():
312  raise pipeBase.TaskError("Exposure has no psf")
313  sciencePsf = exposure.getPsf()
314 
315  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
316  templateExposure = None # Stitched coadd exposure
317  templateSources = None # Sources on the template image
318  if self.config.doSubtract:
319  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
320  templateExposure = template.exposure
321  templateSources = template.sources
322 
323  if self.config.subtract.name == 'zogy':
324  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
325  doWarping=True,
326  spatiallyVarying=self.config.doSpatiallyVarying,
327  doPreConvolve=self.config.doPreConvolve)
328  subtractedExposure = subtractRes.subtractedExposure
329 
330  elif self.config.subtract.name == 'al':
331  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
332  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
333 
334  # sigma of PSF of template image before warping
335  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
336 
337  # if requested, convolve the science exposure with its PSF
338  # (properly, this should be a cross-correlation, but our code does not yet support that)
339  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
340  # else sigma of original science exposure
341  preConvPsf = None
342  if self.config.doPreConvolve:
343  convControl = afwMath.ConvolutionControl()
344  # cannot convolve in place, so make a new MI to receive convolved image
345  srcMI = exposure.getMaskedImage()
346  destMI = srcMI.Factory(srcMI.getDimensions())
347  srcPsf = sciencePsf
348  if self.config.useGaussianForPreConvolution:
349  # convolve with a simplified PSF model: a double Gaussian
350  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
351  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
352  else:
353  # convolve with science exposure's PSF model
354  preConvPsf = srcPsf
355  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
356  exposure.setMaskedImage(destMI)
357  scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
358  else:
359  scienceSigmaPost = scienceSigmaOrig
360 
361  # If requested, find sources in the image
362  if self.config.doSelectSources:
363  if not sensorRef.datasetExists("src"):
364  self.log.warn("Src product does not exist; running detection, measurement, selection")
365  # Run own detection and measurement; necessary in nightly processing
366  selectSources = self.subtract.getSelectSources(
367  exposure,
368  sigma=scienceSigmaPost,
369  doSmooth=not self.doPreConvolve,
370  idFactory=idFactory,
371  )
372  else:
373  self.log.info("Source selection via src product")
374  # Sources already exist; for data release processing
375  selectSources = sensorRef.get("src")
376 
377  # Number of basis functions
378  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
379  referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
380  targetFwhmPix=templateSigma * FwhmPerSigma))
381 
382  if self.config.doAddMetrics:
383  # Modify the schema of all Sources
384  kcQa = KernelCandidateQa(nparam)
385  selectSources = kcQa.addToSchema(selectSources)
386 
387  if self.config.kernelSourcesFromRef:
388  # match exposure sources to reference catalog
389  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
390  matches = astromRet.matches
391  elif templateSources:
392  # match exposure sources to template sources
393  mc = afwTable.MatchControl()
394  mc.findOnlyClosest = False
395  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
396  mc)
397  else:
398  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
399  "but template sources not available. Cannot match science " +
400  "sources with template sources. Run process* on data from " +
401  "which templates are built.")
402 
403  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
404  matches=matches).sourceCat
405 
406  random.shuffle(kernelSources, random.random)
407  controlSources = kernelSources[::self.config.controlStepSize]
408  kernelSources = [k for i, k in enumerate(kernelSources)
409  if i % self.config.controlStepSize]
410 
411  if self.config.doSelectDcrCatalog:
412  redSelector = DiaCatalogSourceSelectorTask(
413  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
414  grMax=99.999))
415  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
416  controlSources.extend(redSources)
417 
418  blueSelector = DiaCatalogSourceSelectorTask(
419  DiaCatalogSourceSelectorConfig(grMin=-99.999,
420  grMax=self.sourceSelector.config.grMin))
421  blueSources = blueSelector.selectStars(exposure, selectSources,
422  matches=matches).starCat
423  controlSources.extend(blueSources)
424 
425  if self.config.doSelectVariableCatalog:
426  varSelector = DiaCatalogSourceSelectorTask(
427  DiaCatalogSourceSelectorConfig(includeVariable=True))
428  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
429  controlSources.extend(varSources)
430 
431  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
432  % (len(kernelSources), len(selectSources), len(controlSources)))
433  allresids = {}
434  if self.config.doUseRegister:
435  self.log.info("Registering images")
436 
437  if templateSources is None:
438  # Run detection on the template, which is
439  # temporarily background-subtracted
440  templateSources = self.subtract.getSelectSources(
441  templateExposure,
442  sigma=templateSigma,
443  doSmooth=True,
444  idFactory=idFactory
445  )
446 
447  # Third step: we need to fit the relative astrometry.
448  #
449  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
450  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
451  exposure.getWcs(), exposure.getBBox())
452  templateExposure = warpedExp
453 
454  # Create debugging outputs on the astrometric
455  # residuals as a function of position. Persistence
456  # not yet implemented; expected on (I believe) #2636.
457  if self.config.doDebugRegister:
458  # Grab matches to reference catalog
459  srcToMatch = {x.second.getId(): x.first for x in matches}
460 
461  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
462  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
463  sids = [m.first.getId() for m in wcsResults.matches]
464  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
465  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
466  m.second.get(inCentroidKey))) for m in wcsResults.matches]
467  allresids = dict(zip(sids, zip(positions, residuals)))
468 
469  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
470  wcsResults.wcs.pixelToSky(
471  m.second.get(inCentroidKey))) for m in wcsResults.matches]
472  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
473  2.5*numpy.log10(srcToMatch[x].get("r"))
474  for x in sids if x in srcToMatch.keys()])
475  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
476  if s in srcToMatch.keys()])
477  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
478  if s in srcToMatch.keys()])
479  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
480  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
481  (colors <= self.sourceSelector.config.grMax))
482  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
483  rms1Long = IqrToSigma * \
484  (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
485  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
486  numpy.percentile(dlat[idx1], 25))
487  rms2Long = IqrToSigma * \
488  (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
489  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
490  numpy.percentile(dlat[idx2], 25))
491  rms3Long = IqrToSigma * \
492  (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
493  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
494  numpy.percentile(dlat[idx3], 25))
495  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
496  (numpy.median(dlong[idx1]), rms1Long,
497  numpy.median(dlat[idx1]), rms1Lat))
498  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
499  (numpy.median(dlong[idx2]), rms2Long,
500  numpy.median(dlat[idx2]), rms2Lat))
501  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
502  (numpy.median(dlong[idx3]), rms3Long,
503  numpy.median(dlat[idx3]), rms3Lat))
504 
505  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
506  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
507  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
508  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
509  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
510  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
511 
512  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
513  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
514  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
515  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
516  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
517  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
518 
519  # warp template exposure to match exposure,
520  # PSF match template exposure to exposure,
521  # then return the difference
522 
523  # Return warped template... Construct sourceKernelCand list after subtract
524  self.log.info("Subtracting images")
525  subtractRes = self.subtract.subtractExposures(
526  templateExposure=templateExposure,
527  scienceExposure=exposure,
528  candidateList=kernelSources,
529  convolveTemplate=self.config.convolveTemplate,
530  doWarping=not self.config.doUseRegister
531  )
532  subtractedExposure = subtractRes.subtractedExposure
533 
534  if self.config.doWriteMatchedExp:
535  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
536 
537  if self.config.doDetection:
538  self.log.info("Computing diffim PSF")
539  if subtractedExposure is None:
540  subtractedExposure = sensorRef.get(subtractedExposureName)
541 
542  # Get Psf from the appropriate input image if it doesn't exist
543  if not subtractedExposure.hasPsf():
544  if self.config.convolveTemplate:
545  subtractedExposure.setPsf(exposure.getPsf())
546  else:
547  if templateExposure is None:
548  template = self.getTemplate.run(exposure, sensorRef,
549  templateIdList=templateIdList)
550  subtractedExposure.setPsf(template.exposure.getPsf())
551 
552  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
553  # thus it may have already been decorrelated. Thus, we do not decorrelate if
554  # doSubtract is False.
555  if self.config.doDecorrelation and self.config.doSubtract:
556  preConvKernel = None
557  if preConvPsf is not None:
558  preConvKernel = preConvPsf.getLocalKernel()
559  decorrResult = self.decorrelate.run(exposure, templateExposure,
560  subtractedExposure,
561  subtractRes.psfMatchingKernel,
562  spatiallyVarying=self.config.doSpatiallyVarying,
563  preConvKernel=preConvKernel)
564  subtractedExposure = decorrResult.correctedExposure
565 
566  # END (if subtractAlgorithm == 'AL')
567 
568  if self.config.doDetection:
569  self.log.info("Running diaSource detection")
570  # Erase existing detection mask planes
571  mask = subtractedExposure.getMaskedImage().getMask()
572  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
573 
574  table = afwTable.SourceTable.make(self.schema, idFactory)
575  table.setMetadata(self.algMetadata)
576  results = self.detection.makeSourceCatalog(
577  table=table,
578  exposure=subtractedExposure,
579  doSmooth=not self.config.doPreConvolve
580  )
581 
582  if self.config.doMerge:
583  fpSet = results.fpSets.positive
584  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
585  self.config.growFootprint, False)
586  diaSources = afwTable.SourceCatalog(table)
587  fpSet.makeSources(diaSources)
588  self.log.info("Merging detections into %d sources" % (len(diaSources)))
589  else:
590  diaSources = results.sources
591 
592  if self.config.doMeasurement:
593  newDipoleFitting = self.config.doDipoleFitting
594  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
595  if not newDipoleFitting:
596  # Just fit dipole in diffim
597  self.measurement.run(diaSources, subtractedExposure)
598  else:
599  # Use (matched) template and science image (if avail.) to constrain dipole fitting
600  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
601  self.measurement.run(diaSources, subtractedExposure, exposure,
602  subtractRes.matchedExposure)
603  else:
604  self.measurement.run(diaSources, subtractedExposure, exposure)
605 
606  # Match with the calexp sources if possible
607  if self.config.doMatchSources:
608  if sensorRef.datasetExists("src"):
609  # Create key,val pair where key=diaSourceId and val=sourceId
610  matchRadAsec = self.config.diaSourceMatchRadius
611  matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
612 
613  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
614  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
615  srcMatch in srcMatches])
616  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
617  len(diaSources)))
618  else:
619  self.log.warn("Src product does not exist; cannot match with diaSources")
620  srcMatchDict = {}
621 
622  # Create key,val pair where key=diaSourceId and val=refId
623  refAstromConfig = AstrometryConfig()
624  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
625  refAstrometer = AstrometryTask(refAstromConfig)
626  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
627  refMatches = astromRet.matches
628  if refMatches is None:
629  self.log.warn("No diaSource matches with reference catalog")
630  refMatchDict = {}
631  else:
632  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
633  len(diaSources)))
634  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
635  refMatch in refMatches])
636 
637  # Assign source Ids
638  for diaSource in diaSources:
639  sid = diaSource.getId()
640  if sid in srcMatchDict:
641  diaSource.set("srcMatchId", srcMatchDict[sid])
642  if sid in refMatchDict:
643  diaSource.set("refMatchId", refMatchDict[sid])
644 
645  if diaSources is not None and self.config.doWriteSources:
646  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
647 
648  if self.config.doAddMetrics and self.config.doSelectSources:
649  self.log.info("Evaluating metrics and control sample")
650 
651  kernelCandList = []
652  for cell in subtractRes.kernelCellSet.getCellList():
653  for cand in cell.begin(False): # include bad candidates
654  kernelCandList.append(cand)
655 
656  # Get basis list to build control sample kernels
657  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
658 
659  controlCandList = \
660  diffimTools.sourceTableToCandidateList(controlSources,
661  subtractRes.warpedExposure, exposure,
662  self.config.subtract.kernel.active,
663  self.config.subtract.kernel.active.detectionConfig,
664  self.log, doBuild=True, basisList=basisList)
665 
666  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
667  dof=nparam)
668  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
669 
670  if self.config.doDetection:
671  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
672  else:
673  kcQa.aggregate(selectSources, self.metadata, allresids)
674 
675  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
676 
677  if self.config.doWriteSubtractedExp:
678  sensorRef.put(subtractedExposure, subtractedExposureName)
679 
680  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
681  return pipeBase.Struct(
682  subtractedExposure=subtractedExposure,
683  subtractRes=subtractRes,
684  sources=diaSources,
685  )
686 
687  def fitAstrometry(self, templateSources, templateExposure, selectSources):
688  """Fit the relative astrometry between templateSources and selectSources
689 
690  @todo remove this method. It originally fit a new WCS to the template before calling register.run
691  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
692  It remains because a subtask overrides it.
693  """
694  results = self.register.run(templateSources, templateExposure.getWcs(),
695  templateExposure.getBBox(), selectSources)
696  return results
697 
698  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
699  """@todo Test and update for current debug display and slot names
700  """
701  import lsstDebug
702  display = lsstDebug.Info(__name__).display
703  showSubtracted = lsstDebug.Info(__name__).showSubtracted
704  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
705  showDiaSources = lsstDebug.Info(__name__).showDiaSources
706  showDipoles = lsstDebug.Info(__name__).showDipoles
707  maskTransparency = lsstDebug.Info(__name__).maskTransparency
708  if display:
709  import lsst.afw.display.ds9 as ds9
710  if not maskTransparency:
711  maskTransparency = 0
712  ds9.setMaskTransparency(maskTransparency)
713 
714  if display and showSubtracted:
715  ds9.mtv(subtractRes.subtractedExposure, frame=lsstDebug.frame, title="Subtracted image")
716  mi = subtractRes.subtractedExposure.getMaskedImage()
717  x0, y0 = mi.getX0(), mi.getY0()
718  with ds9.Buffering():
719  for s in diaSources:
720  x, y = s.getX() - x0, s.getY() - y0
721  ctype = "red" if s.get("flags.negative") else "yellow"
722  if (s.get("flags.pixel.interpolated.center") or s.get("flags.pixel.saturated.center") or
723  s.get("flags.pixel.cr.center")):
724  ptype = "x"
725  elif (s.get("flags.pixel.interpolated.any") or s.get("flags.pixel.saturated.any") or
726  s.get("flags.pixel.cr.any")):
727  ptype = "+"
728  else:
729  ptype = "o"
730  ds9.dot(ptype, x, y, size=4, frame=lsstDebug.frame, ctype=ctype)
731  lsstDebug.frame += 1
732 
733  if display and showPixelResiduals and selectSources:
734  nonKernelSources = []
735  for source in selectSources:
736  if source not in kernelSources:
737  nonKernelSources.append(source)
738 
739  diUtils.plotPixelResiduals(exposure,
740  subtractRes.warpedExposure,
741  subtractRes.subtractedExposure,
742  subtractRes.kernelCellSet,
743  subtractRes.psfMatchingKernel,
744  subtractRes.backgroundModel,
745  nonKernelSources,
746  self.subtract.config.kernel.active.detectionConfig,
747  origVariance=False)
748  diUtils.plotPixelResiduals(exposure,
749  subtractRes.warpedExposure,
750  subtractRes.subtractedExposure,
751  subtractRes.kernelCellSet,
752  subtractRes.psfMatchingKernel,
753  subtractRes.backgroundModel,
754  nonKernelSources,
755  self.subtract.config.kernel.active.detectionConfig,
756  origVariance=True)
757  if display and showDiaSources:
758  flagChecker = SourceFlagChecker(diaSources)
759  isFlagged = [flagChecker(x) for x in diaSources]
760  isDipole = [x.get("classification.dipole") for x in diaSources]
761  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
762  frame=lsstDebug.frame)
763  lsstDebug.frame += 1
764 
765  if display and showDipoles:
766  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
767  frame=lsstDebug.frame)
768  lsstDebug.frame += 1
769 
770  def _getConfigName(self):
771  """Return the name of the config dataset
772  """
773  return "%sDiff_config" % (self.config.coaddName,)
774 
775  def _getMetadataName(self):
776  """Return the name of the metadata dataset
777  """
778  return "%sDiff_metadata" % (self.config.coaddName,)
779 
780  def getSchemaCatalogs(self):
781  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
782  diaSrc = afwTable.SourceCatalog(self.schema)
783  diaSrc.getTable().setMetadata(self.algMetadata)
784  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
785 
786  @classmethod
787  def _makeArgumentParser(cls):
788  """Create an argument parser
789  """
790  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
791  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
792  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
793  help="Optional template data ID (visit only), e.g. --templateId visit=6789")
794  return parser
795 
796 
798  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
799  doc="Shift stars going into RegisterTask by this amount")
800  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
801  doc="Perturb stars going into RegisterTask by this amount")
802 
803  def setDefaults(self):
804  ImageDifferenceConfig.setDefaults(self)
805  self.getTemplate.retarget(GetCalexpAsTemplateTask)
806 
807 
809  """!Image difference Task used in the Winter 2013 data challege.
810  Enables testing the effects of registration shifts and scatter.
811 
812  For use with winter 2013 simulated images:
813  Use --templateId visit=88868666 for sparse data
814  --templateId visit=22222200 for dense data (g)
815  --templateId visit=11111100 for dense data (i)
816  """
817  ConfigClass = Winter2013ImageDifferenceConfig
818  _DefaultName = "winter2013ImageDifference"
819 
820  def __init__(self, **kwargs):
821  ImageDifferenceTask.__init__(self, **kwargs)
822 
823  def fitAstrometry(self, templateSources, templateExposure, selectSources):
824  """Fit the relative astrometry between templateSources and selectSources"""
825  if self.config.winter2013WcsShift > 0.0:
826  offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
827  self.config.winter2013WcsShift)
828  cKey = templateSources[0].getTable().getCentroidKey()
829  for source in templateSources:
830  centroid = source.get(cKey)
831  source.set(cKey, centroid+offset)
832  elif self.config.winter2013WcsRms > 0.0:
833  cKey = templateSources[0].getTable().getCentroidKey()
834  for source in templateSources:
835  offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
836  self.config.winter2013WcsRms*numpy.random.normal())
837  centroid = source.get(cKey)
838  source.set(cKey, centroid+offset)
839 
840  results = self.register.run(templateSources, templateExposure.getWcs(),
841  templateExposure.getBBox(), selectSources)
842  return results
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDataRef(self, sensorRef, templateIdList=None)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)