lsst.pipe.tasks  16.0-23-ge8a9b866+2
imageDifference.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import random
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
28 import lsst.daf.base as dafBase
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.math as afwMath
31 import lsst.afw.table as afwTable
32 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
33 from lsst.meas.base import ForcedMeasurementTask
34 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
35 from lsst.pipe.tasks.registerImage import RegisterTask
36 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, \
37  ObjectSizeStarSelectorTask
38 from lsst.ip.diffim import DipoleAnalysis, \
39  SourceFlagChecker, KernelCandidateF, makeKernelBasisList, \
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig, \
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask, \
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 
46 FwhmPerSigma = 2 * math.sqrt(2 * math.log(2))
47 IqrToSigma = 0.741
48 
49 
50 class ImageDifferenceConfig(pexConfig.Config):
51  """Config for ImageDifferenceTask
52  """
53  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=True,
54  doc="Add background to calexp before processing it. "
55  "Useful as ipDiffim does background matching.")
56  doUseRegister = pexConfig.Field(dtype=bool, default=True,
57  doc="Use image-to-image registration to align template with "
58  "science image")
59  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
60  doc="Writing debugging data for doUseRegister")
61  doSelectSources = pexConfig.Field(dtype=bool, default=True,
62  doc="Select stars to use for kernel fitting")
63  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
64  doc="Select stars of extreme color as part of the control sample")
65  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
66  doc="Select stars that are variable to be part "
67  "of the control sample")
68  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
69  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
70  doc="Convolve science image by its PSF before PSF-matching?")
71  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
72  doc="Use a simple gaussian PSF model for pre-convolution "
73  "(else use fit PSF)? Ignored if doPreConvolve false.")
74  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
75  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
76  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
77  "kernel convolution? If True, also update the diffim PSF.")
78  doMerge = pexConfig.Field(dtype=bool, default=True,
79  doc="Merge positive and negative diaSources with grow radius "
80  "set by growFootprint")
81  doMatchSources = pexConfig.Field(dtype=bool, default=True,
82  doc="Match diaSources with input calexp sources and ref catalog sources")
83  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
84  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
85  doForcedMeasurement = pexConfig.Field(dtype=bool, default=True,
86  doc="Force photometer diaSource locations on PVI?")
87  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
88  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
89  doc="Write warped and PSF-matched template coadd exposure?")
90  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
91  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
92  doc="Add columns to the source table to hold analysis metrics?")
93 
94  coaddName = pexConfig.Field(
95  doc="coadd name: typically one of deep, goodSeeing, or dcr",
96  dtype=str,
97  default="deep",
98  )
99  convolveTemplate = pexConfig.Field(
100  doc="Which image gets convolved (default = template)",
101  dtype=bool,
102  default=True
103  )
104  refObjLoader = pexConfig.ConfigurableField(
105  target=LoadAstrometryNetObjectsTask,
106  doc="reference object loader",
107  )
108  astrometer = pexConfig.ConfigurableField(
109  target=AstrometryTask,
110  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
111  )
112  sourceSelector = pexConfig.ConfigurableField(
113  target=ObjectSizeStarSelectorTask,
114  doc="Source selection algorithm",
115  )
116  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
117  decorrelate = pexConfig.ConfigurableField(
118  target=DecorrelateALKernelSpatialTask,
119  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
120  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
121  "default of 5.5).",
122  )
123  doSpatiallyVarying = pexConfig.Field(
124  dtype=bool,
125  default=False,
126  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
127  "image in order to allow for spatial variations"
128  )
129  detection = pexConfig.ConfigurableField(
130  target=SourceDetectionTask,
131  doc="Low-threshold detection for final measurement",
132  )
133  measurement = pexConfig.ConfigurableField(
134  target=DipoleFitTask,
135  doc="Enable updated dipole fitting method",
136  )
137  forcedMeasurement = pexConfig.ConfigurableField(
138  target=ForcedMeasurementTask,
139  doc="Subtask to force photometer PVI at diaSource location.",
140  )
141  getTemplate = pexConfig.ConfigurableField(
142  target=GetCoaddAsTemplateTask,
143  doc="Subtask to retrieve template exposure and sources",
144  )
145  controlStepSize = pexConfig.Field(
146  doc="What step size (every Nth one) to select a control sample from the kernelSources",
147  dtype=int,
148  default=5
149  )
150  controlRandomSeed = pexConfig.Field(
151  doc="Random seed for shuffing the control sample",
152  dtype=int,
153  default=10
154  )
155  register = pexConfig.ConfigurableField(
156  target=RegisterTask,
157  doc="Task to enable image-to-image image registration (warping)",
158  )
159  kernelSourcesFromRef = pexConfig.Field(
160  doc="Select sources to measure kernel from reference catalog if True, template if false",
161  dtype=bool,
162  default=False
163  )
164  templateSipOrder = pexConfig.Field(dtype=int, default=2,
165  doc="Sip Order for fitting the Template Wcs "
166  "(default is too high, overfitting)")
167 
168  growFootprint = pexConfig.Field(dtype=int, default=2,
169  doc="Grow positive and negative footprints by this amount before merging")
170 
171  diaSourceMatchRadius = pexConfig.Field(dtype=float, default=0.5,
172  doc="Match radius (in arcseconds) "
173  "for DiaSource to Source association")
174 
175  def setDefaults(self):
176  # defaults are OK for catalog and diacatalog
177 
178  self.subtract['al'].kernel.name = "AL"
179  self.subtract['al'].kernel.active.fitForBackground = True
180  self.subtract['al'].kernel.active.spatialKernelOrder = 1
181  self.subtract['al'].kernel.active.spatialBgOrder = 0
182  self.doPreConvolve = False
183  self.doMatchSources = False
184  self.doAddMetrics = False
185  self.doUseRegister = False
186 
187  # DiaSource Detection
188  self.detection.thresholdPolarity = "both"
189  self.detection.thresholdValue = 5.5
190  self.detection.reEstimateBackground = False
191  self.detection.thresholdType = "pixel_stdev"
192 
193  # Add filtered flux measurement, the correct measurement for pre-convolved images.
194  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
195  # To change that you must modify algorithms.names in the task's applyOverrides method,
196  # after the user has set doPreConvolve.
197  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
198 
199  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
200  self.forcedMeasurement.copyColumns = {
201  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
202  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
203  self.forcedMeasurement.slots.shape = None
204 
205  # For shuffling the control sample
206  random.seed(self.controlRandomSeed)
207 
208  def validate(self):
209  pexConfig.Config.validate(self)
210  if self.doMeasurement and not self.doDetection:
211  raise ValueError("Cannot run source measurement without source detection.")
212  if self.doMerge and not self.doDetection:
213  raise ValueError("Cannot run source merging without source detection.")
214  if self.doUseRegister and not self.doSelectSources:
215  raise ValueError("doUseRegister=True and doSelectSources=False. " +
216  "Cannot run RegisterTask without selecting sources.")
217  if hasattr(self.getTemplate, "coaddName"):
218  if self.getTemplate.coaddName != self.coaddName:
219  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
220 
221 
222 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
223 
224  @staticmethod
225  def getTargetList(parsedCmd, **kwargs):
226  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
227  **kwargs)
228 
229 
230 class ImageDifferenceTask(pipeBase.CmdLineTask):
231  """Subtract an image from a template and measure the result
232  """
233  ConfigClass = ImageDifferenceConfig
234  RunnerClass = ImageDifferenceTaskRunner
235  _DefaultName = "imageDifference"
236 
237  def __init__(self, butler=None, **kwargs):
238  """!Construct an ImageDifference Task
239 
240  @param[in] butler Butler object to use in constructing reference object loaders
241  """
242  pipeBase.CmdLineTask.__init__(self, **kwargs)
243  self.makeSubtask("getTemplate")
244 
245  self.makeSubtask("subtract")
246 
247  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
248  self.makeSubtask("decorrelate")
249 
250  if self.config.doUseRegister:
251  self.makeSubtask("register")
252  self.schema = afwTable.SourceTable.makeMinimalSchema()
253 
254  if self.config.doSelectSources:
255  self.makeSubtask("sourceSelector")
256  self.makeSubtask('refObjLoader', butler=butler)
257  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
258 
259  self.algMetadata = dafBase.PropertyList()
260  if self.config.doDetection:
261  self.makeSubtask("detection", schema=self.schema)
262  if self.config.doMeasurement:
263  self.makeSubtask("measurement", schema=self.schema,
264  algMetadata=self.algMetadata)
265  if self.config.doForcedMeasurement:
266  self.totFluxKey = self.schema.addField(
267  "totFlux", "D", "Forced flux measured on the PVI")
268  self.totFluxErrKey = self.schema.addField(
269  "totFluxErr", "D", "Forced flux error measured on the PVI")
270  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
271  if self.config.doMatchSources:
272  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
273  self.schema.addField("srcMatchId", "L", "unique id of source match")
274 
275  @pipeBase.timeMethod
276  def runDataRef(self, sensorRef, templateIdList=None):
277  """Subtract an image from a template coadd and measure the result
278 
279  Steps include:
280  - warp template coadd to match WCS of image
281  - PSF match image to warped template
282  - subtract image from PSF-matched, warped template
283  - persist difference image
284  - detect sources
285  - measure sources
286 
287  @param sensorRef: sensor-level butler data reference, used for the following data products:
288  Input only:
289  - calexp
290  - psf
291  - ccdExposureId
292  - ccdExposureId_bits
293  - self.config.coaddName + "Coadd_skyMap"
294  - self.config.coaddName + "Coadd"
295  Input or output, depending on config:
296  - self.config.coaddName + "Diff_subtractedExp"
297  Output, depending on config:
298  - self.config.coaddName + "Diff_matchedExp"
299  - self.config.coaddName + "Diff_src"
300 
301  @return pipe_base Struct containing these fields:
302  - subtractedExposure: exposure after subtracting template;
303  the unpersisted version if subtraction not run but detection run
304  None if neither subtraction nor detection run (i.e. nothing useful done)
305  - subtractRes: results of subtraction task; None if subtraction not run
306  - sources: detected and possibly measured sources; None if detection not run
307  """
308  self.log.info("Processing %s" % (sensorRef.dataId))
309 
310  # initialize outputs and some intermediate products
311  subtractedExposure = None
312  subtractRes = None
313  selectSources = None
314  kernelSources = None
315  controlSources = None
316  diaSources = None
317 
318  # We make one IdFactory that will be used by both icSrc and src datasets;
319  # I don't know if this is the way we ultimately want to do things, but at least
320  # this ensures the source IDs are fully unique.
321  expBits = sensorRef.get("ccdExposureId_bits")
322  expId = int(sensorRef.get("ccdExposureId"))
323  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
324 
325  # Retrieve the science image we wish to analyze
326  exposure = sensorRef.get("calexp", immediate=True)
327  if self.config.doAddCalexpBackground:
328  mi = exposure.getMaskedImage()
329  mi += sensorRef.get("calexpBackground").getImage()
330  if not exposure.hasPsf():
331  raise pipeBase.TaskError("Exposure has no psf")
332  sciencePsf = exposure.getPsf()
333 
334  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
335  templateExposure = None # Stitched coadd exposure
336  templateSources = None # Sources on the template image
337  if self.config.doSubtract:
338  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
339  templateExposure = template.exposure
340  templateSources = template.sources
341 
342  if self.config.subtract.name == 'zogy':
343  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
344  doWarping=True,
345  spatiallyVarying=self.config.doSpatiallyVarying,
346  doPreConvolve=self.config.doPreConvolve)
347  subtractedExposure = subtractRes.subtractedExposure
348 
349  elif self.config.subtract.name == 'al':
350  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
351  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
352 
353  # sigma of PSF of template image before warping
354  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
355 
356  # if requested, convolve the science exposure with its PSF
357  # (properly, this should be a cross-correlation, but our code does not yet support that)
358  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
359  # else sigma of original science exposure
360  preConvPsf = None
361  if self.config.doPreConvolve:
362  convControl = afwMath.ConvolutionControl()
363  # cannot convolve in place, so make a new MI to receive convolved image
364  srcMI = exposure.getMaskedImage()
365  destMI = srcMI.Factory(srcMI.getDimensions())
366  srcPsf = sciencePsf
367  if self.config.useGaussianForPreConvolution:
368  # convolve with a simplified PSF model: a double Gaussian
369  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
370  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
371  else:
372  # convolve with science exposure's PSF model
373  preConvPsf = srcPsf
374  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
375  exposure.setMaskedImage(destMI)
376  scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
377  else:
378  scienceSigmaPost = scienceSigmaOrig
379 
380  # If requested, find sources in the image
381  if self.config.doSelectSources:
382  if not sensorRef.datasetExists("src"):
383  self.log.warn("Src product does not exist; running detection, measurement, selection")
384  # Run own detection and measurement; necessary in nightly processing
385  selectSources = self.subtract.getSelectSources(
386  exposure,
387  sigma=scienceSigmaPost,
388  doSmooth=not self.doPreConvolve,
389  idFactory=idFactory,
390  )
391  else:
392  self.log.info("Source selection via src product")
393  # Sources already exist; for data release processing
394  selectSources = sensorRef.get("src")
395 
396  # Number of basis functions
397  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
398  referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
399  targetFwhmPix=templateSigma * FwhmPerSigma))
400 
401  if self.config.doAddMetrics:
402  # Modify the schema of all Sources
403  kcQa = KernelCandidateQa(nparam)
404  selectSources = kcQa.addToSchema(selectSources)
405 
406  if self.config.kernelSourcesFromRef:
407  # match exposure sources to reference catalog
408  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
409  matches = astromRet.matches
410  elif templateSources:
411  # match exposure sources to template sources
412  mc = afwTable.MatchControl()
413  mc.findOnlyClosest = False
414  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
415  mc)
416  else:
417  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False," +
418  "but template sources not available. Cannot match science " +
419  "sources with template sources. Run process* on data from " +
420  "which templates are built.")
421 
422  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
423  matches=matches).sourceCat
424 
425  random.shuffle(kernelSources, random.random)
426  controlSources = kernelSources[::self.config.controlStepSize]
427  kernelSources = [k for i, k in enumerate(kernelSources)
428  if i % self.config.controlStepSize]
429 
430  if self.config.doSelectDcrCatalog:
431  redSelector = DiaCatalogSourceSelectorTask(
432  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
433  grMax=99.999))
434  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
435  controlSources.extend(redSources)
436 
437  blueSelector = DiaCatalogSourceSelectorTask(
438  DiaCatalogSourceSelectorConfig(grMin=-99.999,
439  grMax=self.sourceSelector.config.grMin))
440  blueSources = blueSelector.selectStars(exposure, selectSources,
441  matches=matches).starCat
442  controlSources.extend(blueSources)
443 
444  if self.config.doSelectVariableCatalog:
445  varSelector = DiaCatalogSourceSelectorTask(
446  DiaCatalogSourceSelectorConfig(includeVariable=True))
447  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
448  controlSources.extend(varSources)
449 
450  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
451  % (len(kernelSources), len(selectSources), len(controlSources)))
452  allresids = {}
453  if self.config.doUseRegister:
454  self.log.info("Registering images")
455 
456  if templateSources is None:
457  # Run detection on the template, which is
458  # temporarily background-subtracted
459  templateSources = self.subtract.getSelectSources(
460  templateExposure,
461  sigma=templateSigma,
462  doSmooth=True,
463  idFactory=idFactory
464  )
465 
466  # Third step: we need to fit the relative astrometry.
467  #
468  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
469  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
470  exposure.getWcs(), exposure.getBBox())
471  templateExposure = warpedExp
472 
473  # Create debugging outputs on the astrometric
474  # residuals as a function of position. Persistence
475  # not yet implemented; expected on (I believe) #2636.
476  if self.config.doDebugRegister:
477  # Grab matches to reference catalog
478  srcToMatch = {x.second.getId(): x.first for x in matches}
479 
480  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
481  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
482  sids = [m.first.getId() for m in wcsResults.matches]
483  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
484  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
485  m.second.get(inCentroidKey))) for m in wcsResults.matches]
486  allresids = dict(zip(sids, zip(positions, residuals)))
487 
488  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
489  wcsResults.wcs.pixelToSky(
490  m.second.get(inCentroidKey))) for m in wcsResults.matches]
491  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
492  2.5*numpy.log10(srcToMatch[x].get("r"))
493  for x in sids if x in srcToMatch.keys()])
494  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
495  if s in srcToMatch.keys()])
496  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
497  if s in srcToMatch.keys()])
498  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
499  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
500  (colors <= self.sourceSelector.config.grMax))
501  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
502  rms1Long = IqrToSigma * \
503  (numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
504  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
505  numpy.percentile(dlat[idx1], 25))
506  rms2Long = IqrToSigma * \
507  (numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
508  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
509  numpy.percentile(dlat[idx2], 25))
510  rms3Long = IqrToSigma * \
511  (numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
512  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
513  numpy.percentile(dlat[idx3], 25))
514  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
515  (numpy.median(dlong[idx1]), rms1Long,
516  numpy.median(dlat[idx1]), rms1Lat))
517  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
518  (numpy.median(dlong[idx2]), rms2Long,
519  numpy.median(dlat[idx2]), rms2Lat))
520  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
521  (numpy.median(dlong[idx3]), rms3Long,
522  numpy.median(dlat[idx3]), rms3Lat))
523 
524  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
525  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
526  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
527  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
528  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
529  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
530 
531  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
532  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
533  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
534  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
535  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
536  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
537 
538  # warp template exposure to match exposure,
539  # PSF match template exposure to exposure,
540  # then return the difference
541 
542  # Return warped template... Construct sourceKernelCand list after subtract
543  self.log.info("Subtracting images")
544  subtractRes = self.subtract.subtractExposures(
545  templateExposure=templateExposure,
546  scienceExposure=exposure,
547  candidateList=kernelSources,
548  convolveTemplate=self.config.convolveTemplate,
549  doWarping=not self.config.doUseRegister
550  )
551  subtractedExposure = subtractRes.subtractedExposure
552 
553  if self.config.doWriteMatchedExp:
554  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
555 
556  if self.config.doDetection:
557  self.log.info("Computing diffim PSF")
558  if subtractedExposure is None:
559  subtractedExposure = sensorRef.get(subtractedExposureName)
560 
561  # Get Psf from the appropriate input image if it doesn't exist
562  if not subtractedExposure.hasPsf():
563  if self.config.convolveTemplate:
564  subtractedExposure.setPsf(exposure.getPsf())
565  else:
566  if templateExposure is None:
567  template = self.getTemplate.run(exposure, sensorRef,
568  templateIdList=templateIdList)
569  subtractedExposure.setPsf(template.exposure.getPsf())
570 
571  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
572  # thus it may have already been decorrelated. Thus, we do not decorrelate if
573  # doSubtract is False.
574  if self.config.doDecorrelation and self.config.doSubtract:
575  preConvKernel = None
576  if preConvPsf is not None:
577  preConvKernel = preConvPsf.getLocalKernel()
578  decorrResult = self.decorrelate.run(exposure, templateExposure,
579  subtractedExposure,
580  subtractRes.psfMatchingKernel,
581  spatiallyVarying=self.config.doSpatiallyVarying,
582  preConvKernel=preConvKernel)
583  subtractedExposure = decorrResult.correctedExposure
584 
585  # END (if subtractAlgorithm == 'AL')
586 
587  if self.config.doDetection:
588  self.log.info("Running diaSource detection")
589  # Erase existing detection mask planes
590  mask = subtractedExposure.getMaskedImage().getMask()
591  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
592 
593  table = afwTable.SourceTable.make(self.schema, idFactory)
594  table.setMetadata(self.algMetadata)
595  results = self.detection.makeSourceCatalog(
596  table=table,
597  exposure=subtractedExposure,
598  doSmooth=not self.config.doPreConvolve
599  )
600 
601  if self.config.doMerge:
602  fpSet = results.fpSets.positive
603  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
604  self.config.growFootprint, False)
605  diaSources = afwTable.SourceCatalog(table)
606  fpSet.makeSources(diaSources)
607  self.log.info("Merging detections into %d sources" % (len(diaSources)))
608  else:
609  diaSources = results.sources
610 
611  if self.config.doMeasurement:
612  newDipoleFitting = self.config.doDipoleFitting
613  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
614  if not newDipoleFitting:
615  # Just fit dipole in diffim
616  self.measurement.run(diaSources, subtractedExposure)
617  else:
618  # Use (matched) template and science image (if avail.) to constrain dipole fitting
619  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
620  self.measurement.run(diaSources, subtractedExposure, exposure,
621  subtractRes.matchedExposure)
622  else:
623  self.measurement.run(diaSources, subtractedExposure, exposure)
624 
625  if self.config.doForcedMeasurement:
626  # Run forced psf photometry on the PVI at the diaSource locations.
627  # Copy the measured flux and error into the diaSource.
628  forcedSources = self.forcedMeasurement.generateMeasCat(
629  exposure, diaSources, subtractedExposure.getWcs())
630  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
631  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
632  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0], "totFlux", True)
633  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
634  "totFluxErr", True)
635  for diaSource, forcedSource in zip(diaSources, forcedSources):
636  diaSource.assign(forcedSource, mapper)
637 
638  # Match with the calexp sources if possible
639  if self.config.doMatchSources:
640  if sensorRef.datasetExists("src"):
641  # Create key,val pair where key=diaSourceId and val=sourceId
642  matchRadAsec = self.config.diaSourceMatchRadius
643  matchRadPixel = matchRadAsec / exposure.getWcs().pixelScale().asArcseconds()
644 
645  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
646  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
647  srcMatch in srcMatches])
648  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
649  len(diaSources)))
650  else:
651  self.log.warn("Src product does not exist; cannot match with diaSources")
652  srcMatchDict = {}
653 
654  # Create key,val pair where key=diaSourceId and val=refId
655  refAstromConfig = AstrometryConfig()
656  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
657  refAstrometer = AstrometryTask(refAstromConfig)
658  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
659  refMatches = astromRet.matches
660  if refMatches is None:
661  self.log.warn("No diaSource matches with reference catalog")
662  refMatchDict = {}
663  else:
664  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
665  len(diaSources)))
666  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
667  refMatch in refMatches])
668 
669  # Assign source Ids
670  for diaSource in diaSources:
671  sid = diaSource.getId()
672  if sid in srcMatchDict:
673  diaSource.set("srcMatchId", srcMatchDict[sid])
674  if sid in refMatchDict:
675  diaSource.set("refMatchId", refMatchDict[sid])
676 
677  if diaSources is not None and self.config.doWriteSources:
678  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
679 
680  if self.config.doAddMetrics and self.config.doSelectSources:
681  self.log.info("Evaluating metrics and control sample")
682 
683  kernelCandList = []
684  for cell in subtractRes.kernelCellSet.getCellList():
685  for cand in cell.begin(False): # include bad candidates
686  kernelCandList.append(cand)
687 
688  # Get basis list to build control sample kernels
689  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
690 
691  controlCandList = \
692  diffimTools.sourceTableToCandidateList(controlSources,
693  subtractRes.warpedExposure, exposure,
694  self.config.subtract.kernel.active,
695  self.config.subtract.kernel.active.detectionConfig,
696  self.log, doBuild=True, basisList=basisList)
697 
698  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
699  dof=nparam)
700  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
701 
702  if self.config.doDetection:
703  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
704  else:
705  kcQa.aggregate(selectSources, self.metadata, allresids)
706 
707  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
708 
709  if self.config.doWriteSubtractedExp:
710  sensorRef.put(subtractedExposure, subtractedExposureName)
711 
712  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
713  return pipeBase.Struct(
714  subtractedExposure=subtractedExposure,
715  subtractRes=subtractRes,
716  sources=diaSources,
717  )
718 
719  def fitAstrometry(self, templateSources, templateExposure, selectSources):
720  """Fit the relative astrometry between templateSources and selectSources
721 
722  @todo remove this method. It originally fit a new WCS to the template before calling register.run
723  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
724  It remains because a subtask overrides it.
725  """
726  results = self.register.run(templateSources, templateExposure.getWcs(),
727  templateExposure.getBBox(), selectSources)
728  return results
729 
730  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
731  """@todo Test and update for current debug display and slot names
732  """
733  import lsstDebug
734  display = lsstDebug.Info(__name__).display
735  showSubtracted = lsstDebug.Info(__name__).showSubtracted
736  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
737  showDiaSources = lsstDebug.Info(__name__).showDiaSources
738  showDipoles = lsstDebug.Info(__name__).showDipoles
739  maskTransparency = lsstDebug.Info(__name__).maskTransparency
740  if display:
741  import lsst.afw.display.ds9 as ds9
742  if not maskTransparency:
743  maskTransparency = 0
744  ds9.setMaskTransparency(maskTransparency)
745 
746  if display and showSubtracted:
747  ds9.mtv(subtractRes.subtractedExposure, frame=lsstDebug.frame, title="Subtracted image")
748  mi = subtractRes.subtractedExposure.getMaskedImage()
749  x0, y0 = mi.getX0(), mi.getY0()
750  with ds9.Buffering():
751  for s in diaSources:
752  x, y = s.getX() - x0, s.getY() - y0
753  ctype = "red" if s.get("flags.negative") else "yellow"
754  if (s.get("flags.pixel.interpolated.center") or s.get("flags.pixel.saturated.center") or
755  s.get("flags.pixel.cr.center")):
756  ptype = "x"
757  elif (s.get("flags.pixel.interpolated.any") or s.get("flags.pixel.saturated.any") or
758  s.get("flags.pixel.cr.any")):
759  ptype = "+"
760  else:
761  ptype = "o"
762  ds9.dot(ptype, x, y, size=4, frame=lsstDebug.frame, ctype=ctype)
763  lsstDebug.frame += 1
764 
765  if display and showPixelResiduals and selectSources:
766  nonKernelSources = []
767  for source in selectSources:
768  if source not in kernelSources:
769  nonKernelSources.append(source)
770 
771  diUtils.plotPixelResiduals(exposure,
772  subtractRes.warpedExposure,
773  subtractRes.subtractedExposure,
774  subtractRes.kernelCellSet,
775  subtractRes.psfMatchingKernel,
776  subtractRes.backgroundModel,
777  nonKernelSources,
778  self.subtract.config.kernel.active.detectionConfig,
779  origVariance=False)
780  diUtils.plotPixelResiduals(exposure,
781  subtractRes.warpedExposure,
782  subtractRes.subtractedExposure,
783  subtractRes.kernelCellSet,
784  subtractRes.psfMatchingKernel,
785  subtractRes.backgroundModel,
786  nonKernelSources,
787  self.subtract.config.kernel.active.detectionConfig,
788  origVariance=True)
789  if display and showDiaSources:
790  flagChecker = SourceFlagChecker(diaSources)
791  isFlagged = [flagChecker(x) for x in diaSources]
792  isDipole = [x.get("classification.dipole") for x in diaSources]
793  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
794  frame=lsstDebug.frame)
795  lsstDebug.frame += 1
796 
797  if display and showDipoles:
798  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
799  frame=lsstDebug.frame)
800  lsstDebug.frame += 1
801 
802  def _getConfigName(self):
803  """Return the name of the config dataset
804  """
805  return "%sDiff_config" % (self.config.coaddName,)
806 
807  def _getMetadataName(self):
808  """Return the name of the metadata dataset
809  """
810  return "%sDiff_metadata" % (self.config.coaddName,)
811 
812  def getSchemaCatalogs(self):
813  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
814  diaSrc = afwTable.SourceCatalog(self.schema)
815  diaSrc.getTable().setMetadata(self.algMetadata)
816  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
817 
818  @classmethod
819  def _makeArgumentParser(cls):
820  """Create an argument parser
821  """
822  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
823  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
824  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
825  help="Optional template data ID (visit only), e.g. --templateId visit=6789")
826  return parser
827 
828 
830  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
831  doc="Shift stars going into RegisterTask by this amount")
832  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
833  doc="Perturb stars going into RegisterTask by this amount")
834 
835  def setDefaults(self):
836  ImageDifferenceConfig.setDefaults(self)
837  self.getTemplate.retarget(GetCalexpAsTemplateTask)
838 
839 
841  """!Image difference Task used in the Winter 2013 data challege.
842  Enables testing the effects of registration shifts and scatter.
843 
844  For use with winter 2013 simulated images:
845  Use --templateId visit=88868666 for sparse data
846  --templateId visit=22222200 for dense data (g)
847  --templateId visit=11111100 for dense data (i)
848  """
849  ConfigClass = Winter2013ImageDifferenceConfig
850  _DefaultName = "winter2013ImageDifference"
851 
852  def __init__(self, **kwargs):
853  ImageDifferenceTask.__init__(self, **kwargs)
854 
855  def fitAstrometry(self, templateSources, templateExposure, selectSources):
856  """Fit the relative astrometry between templateSources and selectSources"""
857  if self.config.winter2013WcsShift > 0.0:
858  offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
859  self.config.winter2013WcsShift)
860  cKey = templateSources[0].getTable().getCentroidKey()
861  for source in templateSources:
862  centroid = source.get(cKey)
863  source.set(cKey, centroid+offset)
864  elif self.config.winter2013WcsRms > 0.0:
865  cKey = templateSources[0].getTable().getCentroidKey()
866  for source in templateSources:
867  offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
868  self.config.winter2013WcsRms*numpy.random.normal())
869  centroid = source.get(cKey)
870  source.set(cKey, centroid+offset)
871 
872  results = self.register.run(templateSources, templateExposure.getWcs(),
873  templateExposure.getBBox(), selectSources)
874  return results
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDataRef(self, sensorRef, templateIdList=None)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)