lsst.pipe.tasks  18.1.0-32-g274f93b6+3
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
28 import lsst.daf.base as dafBase
29 import lsst.geom as geom
30 import lsst.afw.math as afwMath
31 import lsst.afw.table as afwTable
32 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
33 from lsst.meas.base import ForcedMeasurementTask, EvaluateLocalCalibrationTask
34 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
35 from lsst.pipe.tasks.registerImage import RegisterTask
36 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
37 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
38 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
39  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
40  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
41  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
42 import lsst.ip.diffim.diffimTools as diffimTools
43 import lsst.ip.diffim.utils as diUtils
44 import lsst.afw.display as afwDisplay
45 
46 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
47 IqrToSigma = 0.741
48 
49 
50 class ImageDifferenceConfig(pexConfig.Config):
51  """Config for ImageDifferenceTask
52  """
53  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
54  doc="Add background to calexp before processing it. "
55  "Useful as ipDiffim does background matching.")
56  doUseRegister = pexConfig.Field(dtype=bool, default=True,
57  doc="Use image-to-image registration to align template with "
58  "science image")
59  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
60  doc="Writing debugging data for doUseRegister")
61  doSelectSources = pexConfig.Field(dtype=bool, default=True,
62  doc="Select stars to use for kernel fitting")
63  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
64  doc="Select stars of extreme color as part of the control sample")
65  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
66  doc="Select stars that are variable to be part "
67  "of the control sample")
68  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
69  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
70  doc="Convolve science image by its PSF before PSF-matching?")
71  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
72  doc="Scale variance of the template before PSF matching")
73  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
74  doc="Use a simple gaussian PSF model for pre-convolution "
75  "(else use fit PSF)? Ignored if doPreConvolve false.")
76  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
77  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
78  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
79  "kernel convolution? If True, also update the diffim PSF.")
80  doMerge = pexConfig.Field(dtype=bool, default=True,
81  doc="Merge positive and negative diaSources with grow radius "
82  "set by growFootprint")
83  doMatchSources = pexConfig.Field(dtype=bool, default=True,
84  doc="Match diaSources with input calexp sources and ref catalog sources")
85  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
86  doEvalLocCalibration = pexConfig.Field(
87  dtype=bool,
88  default=True,
89  doc="Store calibration products (local wcs and photoCalib) in the "
90  "output DiaSource catalog.")
91  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
92  doForcedMeasurement = pexConfig.Field(
93  dtype=bool,
94  default=True,
95  doc="Force photometer diaSource locations on PVI?")
96  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
97  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
98  doc="Write warped and PSF-matched template coadd exposure?")
99  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
100  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
101  doc="Add columns to the source table to hold analysis metrics?")
102 
103  coaddName = pexConfig.Field(
104  doc="coadd name: typically one of deep, goodSeeing, or dcr",
105  dtype=str,
106  default="deep",
107  )
108  convolveTemplate = pexConfig.Field(
109  doc="Which image gets convolved (default = template)",
110  dtype=bool,
111  default=True
112  )
113  refObjLoader = pexConfig.ConfigurableField(
114  target=LoadIndexedReferenceObjectsTask,
115  doc="reference object loader",
116  )
117  astrometer = pexConfig.ConfigurableField(
118  target=AstrometryTask,
119  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
120  )
121  sourceSelector = pexConfig.ConfigurableField(
122  target=ObjectSizeStarSelectorTask,
123  doc="Source selection algorithm",
124  )
125  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
126  decorrelate = pexConfig.ConfigurableField(
127  target=DecorrelateALKernelSpatialTask,
128  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
129  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
130  "default of 5.5).",
131  )
132  doSpatiallyVarying = pexConfig.Field(
133  dtype=bool,
134  default=False,
135  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
136  "image in order to allow for spatial variations"
137  )
138  detection = pexConfig.ConfigurableField(
139  target=SourceDetectionTask,
140  doc="Low-threshold detection for final measurement",
141  )
142  measurement = pexConfig.ConfigurableField(
143  target=DipoleFitTask,
144  doc="Enable updated dipole fitting method",
145  )
146  evalLocCalib = pexConfig.ConfigurableField(
147  target=EvaluateLocalCalibrationTask,
148  doc="Task to strip calibrations from an exposure and store their "
149  "local values in the output DiaSource catalog."
150  )
151  forcedMeasurement = pexConfig.ConfigurableField(
152  target=ForcedMeasurementTask,
153  doc="Subtask to force photometer PVI at diaSource location.",
154  )
155  getTemplate = pexConfig.ConfigurableField(
156  target=GetCoaddAsTemplateTask,
157  doc="Subtask to retrieve template exposure and sources",
158  )
159  scaleVariance = pexConfig.ConfigurableField(
160  target=ScaleVarianceTask,
161  doc="Subtask to rescale the variance of the template "
162  "to the statistically expected level"
163  )
164  controlStepSize = pexConfig.Field(
165  doc="What step size (every Nth one) to select a control sample from the kernelSources",
166  dtype=int,
167  default=5
168  )
169  controlRandomSeed = pexConfig.Field(
170  doc="Random seed for shuffing the control sample",
171  dtype=int,
172  default=10
173  )
174  register = pexConfig.ConfigurableField(
175  target=RegisterTask,
176  doc="Task to enable image-to-image image registration (warping)",
177  )
178  kernelSourcesFromRef = pexConfig.Field(
179  doc="Select sources to measure kernel from reference catalog if True, template if false",
180  dtype=bool,
181  default=False
182  )
183  templateSipOrder = pexConfig.Field(
184  dtype=int, default=2,
185  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
186  )
187  growFootprint = pexConfig.Field(
188  dtype=int, default=2,
189  doc="Grow positive and negative footprints by this amount before merging"
190  )
191  diaSourceMatchRadius = pexConfig.Field(
192  dtype=float, default=0.5,
193  doc="Match radius (in arcseconds) for DiaSource to Source association"
194  )
195 
196  def setDefaults(self):
197  # defaults are OK for catalog and diacatalog
198 
199  self.subtract['al'].kernel.name = "AL"
200  self.subtract['al'].kernel.active.fitForBackground = True
201  self.subtract['al'].kernel.active.spatialKernelOrder = 1
202  self.subtract['al'].kernel.active.spatialBgOrder = 2
203  self.doPreConvolve = False
204  self.doMatchSources = False
205  self.doAddMetrics = False
206  self.doUseRegister = False
207 
208  # DiaSource Detection
209  self.detection.thresholdPolarity = "both"
210  self.detection.thresholdValue = 5.5
211  self.detection.reEstimateBackground = False
212  self.detection.thresholdType = "pixel_stdev"
213 
214  # Add filtered flux measurement, the correct measurement for pre-convolved images.
215  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
216  # To change that you must modify algorithms.names in the task's applyOverrides method,
217  # after the user has set doPreConvolve.
218  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
219 
220  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
221  self.forcedMeasurement.copyColumns = {
222  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
223  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
224  self.forcedMeasurement.slots.shape = None
225 
226  # For shuffling the control sample
227  random.seed(self.controlRandomSeed)
228 
229  def validate(self):
230  pexConfig.Config.validate(self)
231  if self.doMeasurement and not self.doDetection:
232  raise ValueError("Cannot run source measurement without source detection.")
233  if self.doMerge and not self.doDetection:
234  raise ValueError("Cannot run source merging without source detection.")
235  if self.doUseRegister and not self.doSelectSources:
236  raise ValueError("doUseRegister=True and doSelectSources=False. "
237  "Cannot run RegisterTask without selecting sources.")
238  if self.doPreConvolve and self.doDecorrelation and not self.convolveTemplate:
239  raise ValueError("doPreConvolve=True and doDecorrelation=True and "
240  "convolveTemplate=False is not supported.")
241  if hasattr(self.getTemplate, "coaddName"):
242  if self.getTemplate.coaddName != self.coaddName:
243  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
244 
245 
246 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
247 
248  @staticmethod
249  def getTargetList(parsedCmd, **kwargs):
250  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
251  **kwargs)
252 
253 
254 class ImageDifferenceTask(pipeBase.CmdLineTask):
255  """Subtract an image from a template and measure the result
256  """
257  ConfigClass = ImageDifferenceConfig
258  RunnerClass = ImageDifferenceTaskRunner
259  _DefaultName = "imageDifference"
260 
261  def __init__(self, butler=None, **kwargs):
262  """!Construct an ImageDifference Task
263 
264  @param[in] butler Butler object to use in constructing reference object loaders
265  """
266  pipeBase.CmdLineTask.__init__(self, **kwargs)
267  self.makeSubtask("getTemplate")
268 
269  self.makeSubtask("subtract")
270 
271  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
272  self.makeSubtask("decorrelate")
273 
274  if self.config.doScaleTemplateVariance:
275  self.makeSubtask("scaleVariance")
276 
277  if self.config.doUseRegister:
278  self.makeSubtask("register")
279  self.schema = afwTable.SourceTable.makeMinimalSchema()
280 
281  if self.config.doSelectSources:
282  self.makeSubtask("sourceSelector")
283  if self.config.kernelSourcesFromRef:
284  self.makeSubtask('refObjLoader', butler=butler)
285  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
286 
287  self.algMetadata = dafBase.PropertyList()
288  if self.config.doDetection:
289  self.makeSubtask("detection", schema=self.schema)
290  if self.config.doMeasurement:
291  self.makeSubtask("measurement", schema=self.schema,
292  algMetadata=self.algMetadata)
293  if self.config.doEvalLocCalibration and self.config.doMeasurement:
294  self.makeSubtask("evalLocCalib", schema=self.schema)
295  if self.config.doForcedMeasurement:
296  self.schema.addField(
297  "ip_diffim_forced_PsfFlux_instFlux", "D",
298  "Forced PSF flux measured on the direct image.")
299  self.schema.addField(
300  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
301  "Forced PSF flux error measured on the direct image.")
302  self.schema.addField(
303  "ip_diffim_forced_PsfFlux_area", "F",
304  "Forced PSF flux effective area of PSF.",
305  units="pixel")
306  self.schema.addField(
307  "ip_diffim_forced_PsfFlux_flag", "Flag",
308  "Forced PSF flux general failure flag.")
309  self.schema.addField(
310  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
311  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
312  self.schema.addField(
313  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
314  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
315  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
316  if self.config.doMatchSources:
317  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
318  self.schema.addField("srcMatchId", "L", "unique id of source match")
319 
320  @pipeBase.timeMethod
321  def runDataRef(self, sensorRef, templateIdList=None):
322  """Subtract an image from a template coadd and measure the result
323 
324  Steps include:
325  - warp template coadd to match WCS of image
326  - PSF match image to warped template
327  - subtract image from PSF-matched, warped template
328  - persist difference image
329  - detect sources
330  - measure sources
331 
332  @param sensorRef: sensor-level butler data reference, used for the following data products:
333  Input only:
334  - calexp
335  - psf
336  - ccdExposureId
337  - ccdExposureId_bits
338  - self.config.coaddName + "Coadd_skyMap"
339  - self.config.coaddName + "Coadd"
340  Input or output, depending on config:
341  - self.config.coaddName + "Diff_subtractedExp"
342  Output, depending on config:
343  - self.config.coaddName + "Diff_matchedExp"
344  - self.config.coaddName + "Diff_src"
345 
346  @return pipe_base Struct containing these fields:
347  - subtractedExposure: exposure after subtracting template;
348  the unpersisted version if subtraction not run but detection run
349  None if neither subtraction nor detection run (i.e. nothing useful done)
350  - subtractRes: results of subtraction task; None if subtraction not run
351  - sources: detected and possibly measured sources; None if detection not run
352  """
353  self.log.info("Processing %s" % (sensorRef.dataId))
354 
355  # initialize outputs and some intermediate products
356  subtractedExposure = None
357  subtractRes = None
358  selectSources = None
359  kernelSources = None
360  controlSources = None
361  diaSources = None
362 
363  # We make one IdFactory that will be used by both icSrc and src datasets;
364  # I don't know if this is the way we ultimately want to do things, but at least
365  # this ensures the source IDs are fully unique.
366  expBits = sensorRef.get("ccdExposureId_bits")
367  expId = int(sensorRef.get("ccdExposureId"))
368  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
369 
370  # Retrieve the science image we wish to analyze
371  exposure = sensorRef.get("calexp", immediate=True)
372  if self.config.doAddCalexpBackground:
373  mi = exposure.getMaskedImage()
374  mi += sensorRef.get("calexpBackground").getImage()
375  if not exposure.hasPsf():
376  raise pipeBase.TaskError("Exposure has no psf")
377  sciencePsf = exposure.getPsf()
378 
379  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
380  templateExposure = None # Stitched coadd exposure
381  templateSources = None # Sources on the template image
382  if self.config.doSubtract:
383  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
384  templateExposure = template.exposure
385  templateSources = template.sources
386 
387  if self.config.doScaleTemplateVariance:
388  templateVarFactor = self.scaleVariance.run(
389  templateExposure.getMaskedImage())
390  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
391 
392  if self.config.subtract.name == 'zogy':
393  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
394  doWarping=True,
395  spatiallyVarying=self.config.doSpatiallyVarying,
396  doPreConvolve=self.config.doPreConvolve)
397  subtractedExposure = subtractRes.subtractedExposure
398 
399  elif self.config.subtract.name == 'al':
400  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
401  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
402 
403  # sigma of PSF of template image before warping
404  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
405 
406  # if requested, convolve the science exposure with its PSF
407  # (properly, this should be a cross-correlation, but our code does not yet support that)
408  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
409  # else sigma of original science exposure
410  preConvPsf = None
411  if self.config.doPreConvolve:
412  convControl = afwMath.ConvolutionControl()
413  # cannot convolve in place, so make a new MI to receive convolved image
414  srcMI = exposure.getMaskedImage()
415  destMI = srcMI.Factory(srcMI.getDimensions())
416  srcPsf = sciencePsf
417  if self.config.useGaussianForPreConvolution:
418  # convolve with a simplified PSF model: a double Gaussian
419  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
420  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
421  else:
422  # convolve with science exposure's PSF model
423  preConvPsf = srcPsf
424  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
425  exposure.setMaskedImage(destMI)
426  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
427  else:
428  scienceSigmaPost = scienceSigmaOrig
429 
430  # If requested, find sources in the image
431  if self.config.doSelectSources:
432  if not sensorRef.datasetExists("src"):
433  self.log.warn("Src product does not exist; running detection, measurement, selection")
434  # Run own detection and measurement; necessary in nightly processing
435  selectSources = self.subtract.getSelectSources(
436  exposure,
437  sigma=scienceSigmaPost,
438  doSmooth=not self.doPreConvolve,
439  idFactory=idFactory,
440  )
441  else:
442  self.log.info("Source selection via src product")
443  # Sources already exist; for data release processing
444  selectSources = sensorRef.get("src")
445 
446  # Number of basis functions
447  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
448  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
449  targetFwhmPix=templateSigma*FwhmPerSigma))
450 
451  if self.config.doAddMetrics:
452  # Modify the schema of all Sources
453  kcQa = KernelCandidateQa(nparam)
454  selectSources = kcQa.addToSchema(selectSources)
455 
456  if self.config.kernelSourcesFromRef:
457  # match exposure sources to reference catalog
458  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
459  matches = astromRet.matches
460  elif templateSources:
461  # match exposure sources to template sources
462  mc = afwTable.MatchControl()
463  mc.findOnlyClosest = False
464  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
465  mc)
466  else:
467  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
468  "but template sources not available. Cannot match science "
469  "sources with template sources. Run process* on data from "
470  "which templates are built.")
471 
472  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
473  matches=matches).sourceCat
474 
475  random.shuffle(kernelSources, random.random)
476  controlSources = kernelSources[::self.config.controlStepSize]
477  kernelSources = [k for i, k in enumerate(kernelSources)
478  if i % self.config.controlStepSize]
479 
480  if self.config.doSelectDcrCatalog:
481  redSelector = DiaCatalogSourceSelectorTask(
482  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
483  grMax=99.999))
484  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
485  controlSources.extend(redSources)
486 
487  blueSelector = DiaCatalogSourceSelectorTask(
488  DiaCatalogSourceSelectorConfig(grMin=-99.999,
489  grMax=self.sourceSelector.config.grMin))
490  blueSources = blueSelector.selectStars(exposure, selectSources,
491  matches=matches).starCat
492  controlSources.extend(blueSources)
493 
494  if self.config.doSelectVariableCatalog:
495  varSelector = DiaCatalogSourceSelectorTask(
496  DiaCatalogSourceSelectorConfig(includeVariable=True))
497  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
498  controlSources.extend(varSources)
499 
500  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
501  % (len(kernelSources), len(selectSources), len(controlSources)))
502  allresids = {}
503  if self.config.doUseRegister:
504  self.log.info("Registering images")
505 
506  if templateSources is None:
507  # Run detection on the template, which is
508  # temporarily background-subtracted
509  templateSources = self.subtract.getSelectSources(
510  templateExposure,
511  sigma=templateSigma,
512  doSmooth=True,
513  idFactory=idFactory
514  )
515 
516  # Third step: we need to fit the relative astrometry.
517  #
518  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
519  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
520  exposure.getWcs(), exposure.getBBox())
521  templateExposure = warpedExp
522 
523  # Create debugging outputs on the astrometric
524  # residuals as a function of position. Persistence
525  # not yet implemented; expected on (I believe) #2636.
526  if self.config.doDebugRegister:
527  # Grab matches to reference catalog
528  srcToMatch = {x.second.getId(): x.first for x in matches}
529 
530  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
531  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
532  sids = [m.first.getId() for m in wcsResults.matches]
533  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
534  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
535  m.second.get(inCentroidKey))) for m in wcsResults.matches]
536  allresids = dict(zip(sids, zip(positions, residuals)))
537 
538  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
539  wcsResults.wcs.pixelToSky(
540  m.second.get(inCentroidKey))) for m in wcsResults.matches]
541  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
542  2.5*numpy.log10(srcToMatch[x].get("r"))
543  for x in sids if x in srcToMatch.keys()])
544  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
545  if s in srcToMatch.keys()])
546  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
547  if s in srcToMatch.keys()])
548  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
549  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
550  (colors <= self.sourceSelector.config.grMax))
551  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
552  rms1Long = IqrToSigma*(
553  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
554  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
555  numpy.percentile(dlat[idx1], 25))
556  rms2Long = IqrToSigma*(
557  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
558  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
559  numpy.percentile(dlat[idx2], 25))
560  rms3Long = IqrToSigma*(
561  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
562  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
563  numpy.percentile(dlat[idx3], 25))
564  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
565  (numpy.median(dlong[idx1]), rms1Long,
566  numpy.median(dlat[idx1]), rms1Lat))
567  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
568  (numpy.median(dlong[idx2]), rms2Long,
569  numpy.median(dlat[idx2]), rms2Lat))
570  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
571  (numpy.median(dlong[idx3]), rms3Long,
572  numpy.median(dlat[idx3]), rms3Lat))
573 
574  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
575  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
576  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
577  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
578  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
579  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
580 
581  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
582  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
583  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
584  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
585  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
586  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
587 
588  # warp template exposure to match exposure,
589  # PSF match template exposure to exposure,
590  # then return the difference
591 
592  # Return warped template... Construct sourceKernelCand list after subtract
593  self.log.info("Subtracting images")
594  subtractRes = self.subtract.subtractExposures(
595  templateExposure=templateExposure,
596  scienceExposure=exposure,
597  candidateList=kernelSources,
598  convolveTemplate=self.config.convolveTemplate,
599  doWarping=not self.config.doUseRegister
600  )
601  subtractedExposure = subtractRes.subtractedExposure
602 
603  if self.config.doWriteMatchedExp:
604  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
605 
606  if self.config.doDetection:
607  self.log.info("Computing diffim PSF")
608  if subtractedExposure is None:
609  subtractedExposure = sensorRef.get(subtractedExposureName)
610 
611  # Get Psf from the appropriate input image if it doesn't exist
612  if not subtractedExposure.hasPsf():
613  if self.config.convolveTemplate:
614  subtractedExposure.setPsf(exposure.getPsf())
615  else:
616  if templateExposure is None:
617  template = self.getTemplate.run(exposure, sensorRef,
618  templateIdList=templateIdList)
619  subtractedExposure.setPsf(template.exposure.getPsf())
620 
621  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
622  # thus it may have already been decorrelated. Thus, we do not decorrelate if
623  # doSubtract is False.
624  if self.config.doDecorrelation and self.config.doSubtract:
625  preConvKernel = None
626  if preConvPsf is not None:
627  preConvKernel = preConvPsf.getLocalKernel()
628  if self.config.convolveTemplate:
629  self.log.info("Decorrelation after template image convolution")
630  decorrResult = self.decorrelate.run(exposure, templateExposure,
631  subtractedExposure,
632  subtractRes.psfMatchingKernel,
633  spatiallyVarying=self.config.doSpatiallyVarying,
634  preConvKernel=preConvKernel)
635  else:
636  self.log.info("Decorrelation after science image convolution")
637  decorrResult = self.decorrelate.run(templateExposure, exposure,
638  subtractedExposure,
639  subtractRes.psfMatchingKernel,
640  spatiallyVarying=self.config.doSpatiallyVarying,
641  preConvKernel=preConvKernel)
642  subtractedExposure = decorrResult.correctedExposure
643 
644  # END (if subtractAlgorithm == 'AL')
645 
646  if self.config.doDetection:
647  self.log.info("Running diaSource detection")
648  # Erase existing detection mask planes
649  mask = subtractedExposure.getMaskedImage().getMask()
650  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
651 
652  table = afwTable.SourceTable.make(self.schema, idFactory)
653  table.setMetadata(self.algMetadata)
654  results = self.detection.makeSourceCatalog(
655  table=table,
656  exposure=subtractedExposure,
657  doSmooth=not self.config.doPreConvolve
658  )
659 
660  if self.config.doMerge:
661  fpSet = results.fpSets.positive
662  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
663  self.config.growFootprint, False)
664  diaSources = afwTable.SourceCatalog(table)
665  fpSet.makeSources(diaSources)
666  self.log.info("Merging detections into %d sources" % (len(diaSources)))
667  else:
668  diaSources = results.sources
669 
670  if self.config.doMeasurement:
671  newDipoleFitting = self.config.doDipoleFitting
672  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
673  if not newDipoleFitting:
674  # Just fit dipole in diffim
675  self.measurement.run(diaSources, subtractedExposure)
676  else:
677  # Use (matched) template and science image (if avail.) to constrain dipole fitting
678  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
679  self.measurement.run(diaSources, subtractedExposure, exposure,
680  subtractRes.matchedExposure)
681  else:
682  self.measurement.run(diaSources, subtractedExposure, exposure)
683 
684  if self.config.doEvalLocCalibration and self.config.doMeasurement:
685  self.evalLocCalib.run(diaSources, subtractedExposure)
686 
687  if self.config.doForcedMeasurement:
688  # Run forced psf photometry on the PVI at the diaSource locations.
689  # Copy the measured flux and error into the diaSource.
690  forcedSources = self.forcedMeasurement.generateMeasCat(
691  exposure, diaSources, subtractedExposure.getWcs())
692  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
693  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
694  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
695  "ip_diffim_forced_PsfFlux_instFlux", True)
696  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
697  "ip_diffim_forced_PsfFlux_instFluxErr", True)
698  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
699  "ip_diffim_forced_PsfFlux_area", True)
700  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
701  "ip_diffim_forced_PsfFlux_flag", True)
702  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
703  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
704  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
705  "ip_diffim_forced_PsfFlux_flag_edge", True)
706  for diaSource, forcedSource in zip(diaSources, forcedSources):
707  diaSource.assign(forcedSource, mapper)
708 
709  # Match with the calexp sources if possible
710  if self.config.doMatchSources:
711  if sensorRef.datasetExists("src"):
712  # Create key,val pair where key=diaSourceId and val=sourceId
713  matchRadAsec = self.config.diaSourceMatchRadius
714  matchRadPixel = matchRadAsec/exposure.getWcs().pixelScale().asArcseconds()
715 
716  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
717  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
718  srcMatch in srcMatches])
719  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
720  len(diaSources)))
721  else:
722  self.log.warn("Src product does not exist; cannot match with diaSources")
723  srcMatchDict = {}
724 
725  # Create key,val pair where key=diaSourceId and val=refId
726  refAstromConfig = AstrometryConfig()
727  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
728  refAstrometer = AstrometryTask(refAstromConfig)
729  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
730  refMatches = astromRet.matches
731  if refMatches is None:
732  self.log.warn("No diaSource matches with reference catalog")
733  refMatchDict = {}
734  else:
735  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
736  len(diaSources)))
737  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
738  refMatch in refMatches])
739 
740  # Assign source Ids
741  for diaSource in diaSources:
742  sid = diaSource.getId()
743  if sid in srcMatchDict:
744  diaSource.set("srcMatchId", srcMatchDict[sid])
745  if sid in refMatchDict:
746  diaSource.set("refMatchId", refMatchDict[sid])
747 
748  if diaSources is not None and self.config.doWriteSources:
749  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
750 
751  if self.config.doAddMetrics and self.config.doSelectSources:
752  self.log.info("Evaluating metrics and control sample")
753 
754  kernelCandList = []
755  for cell in subtractRes.kernelCellSet.getCellList():
756  for cand in cell.begin(False): # include bad candidates
757  kernelCandList.append(cand)
758 
759  # Get basis list to build control sample kernels
760  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
761 
762  controlCandList = (
763  diffimTools.sourceTableToCandidateList(controlSources,
764  subtractRes.warpedExposure, exposure,
765  self.config.subtract.kernel.active,
766  self.config.subtract.kernel.active.detectionConfig,
767  self.log, doBuild=True, basisList=basisList))
768 
769  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
770  dof=nparam)
771  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
772 
773  if self.config.doDetection:
774  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
775  else:
776  kcQa.aggregate(selectSources, self.metadata, allresids)
777 
778  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
779 
780  if self.config.doWriteSubtractedExp:
781  sensorRef.put(subtractedExposure, subtractedExposureName)
782 
783  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
784  return pipeBase.Struct(
785  subtractedExposure=subtractedExposure,
786  subtractRes=subtractRes,
787  sources=diaSources,
788  )
789 
790  def fitAstrometry(self, templateSources, templateExposure, selectSources):
791  """Fit the relative astrometry between templateSources and selectSources
792 
793  @todo remove this method. It originally fit a new WCS to the template before calling register.run
794  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
795  It remains because a subtask overrides it.
796  """
797  results = self.register.run(templateSources, templateExposure.getWcs(),
798  templateExposure.getBBox(), selectSources)
799  return results
800 
801  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
802  """@todo Test and update for current debug display and slot names
803  """
804  import lsstDebug
805  display = lsstDebug.Info(__name__).display
806  showSubtracted = lsstDebug.Info(__name__).showSubtracted
807  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
808  showDiaSources = lsstDebug.Info(__name__).showDiaSources
809  showDipoles = lsstDebug.Info(__name__).showDipoles
810  maskTransparency = lsstDebug.Info(__name__).maskTransparency
811  if display:
812  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
813  if not maskTransparency:
814  maskTransparency = 0
815  disp.setMaskTransparency(maskTransparency)
816 
817  if display and showSubtracted:
818  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
819  mi = subtractRes.subtractedExposure.getMaskedImage()
820  x0, y0 = mi.getX0(), mi.getY0()
821  with disp.Buffering():
822  for s in diaSources:
823  x, y = s.getX() - x0, s.getY() - y0
824  ctype = "red" if s.get("flags_negative") else "yellow"
825  if (s.get("base_PixelFlags_flag_interpolatedCenter") or
826  s.get("base_PixelFlags_flag_saturatedCenter") or
827  s.get("base_PixelFlags_flag_crCenter")):
828  ptype = "x"
829  elif (s.get("base_PixelFlags_flag_interpolated") or
830  s.get("base_PixelFlags_flag_saturated") or
831  s.get("base_PixelFlags_flag_cr")):
832  ptype = "+"
833  else:
834  ptype = "o"
835  disp.dot(ptype, x, y, size=4, ctype=ctype)
836  lsstDebug.frame += 1
837 
838  if display and showPixelResiduals and selectSources:
839  nonKernelSources = []
840  for source in selectSources:
841  if source not in kernelSources:
842  nonKernelSources.append(source)
843 
844  diUtils.plotPixelResiduals(exposure,
845  subtractRes.warpedExposure,
846  subtractRes.subtractedExposure,
847  subtractRes.kernelCellSet,
848  subtractRes.psfMatchingKernel,
849  subtractRes.backgroundModel,
850  nonKernelSources,
851  self.subtract.config.kernel.active.detectionConfig,
852  origVariance=False)
853  diUtils.plotPixelResiduals(exposure,
854  subtractRes.warpedExposure,
855  subtractRes.subtractedExposure,
856  subtractRes.kernelCellSet,
857  subtractRes.psfMatchingKernel,
858  subtractRes.backgroundModel,
859  nonKernelSources,
860  self.subtract.config.kernel.active.detectionConfig,
861  origVariance=True)
862  if display and showDiaSources:
863  flagChecker = SourceFlagChecker(diaSources)
864  isFlagged = [flagChecker(x) for x in diaSources]
865  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
866  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
867  frame=lsstDebug.frame)
868  lsstDebug.frame += 1
869 
870  if display and showDipoles:
871  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
872  frame=lsstDebug.frame)
873  lsstDebug.frame += 1
874 
875  def _getConfigName(self):
876  """Return the name of the config dataset
877  """
878  return "%sDiff_config" % (self.config.coaddName,)
879 
880  def _getMetadataName(self):
881  """Return the name of the metadata dataset
882  """
883  return "%sDiff_metadata" % (self.config.coaddName,)
884 
885  def getSchemaCatalogs(self):
886  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
887  diaSrc = afwTable.SourceCatalog(self.schema)
888  diaSrc.getTable().setMetadata(self.algMetadata)
889  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
890 
891  @classmethod
892  def _makeArgumentParser(cls):
893  """Create an argument parser
894  """
895  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
896  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
897  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
898  help="Template data ID in case of calexp template,"
899  " e.g. --templateId visit=6789")
900  return parser
901 
902 
904  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
905  doc="Shift stars going into RegisterTask by this amount")
906  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
907  doc="Perturb stars going into RegisterTask by this amount")
908 
909  def setDefaults(self):
910  ImageDifferenceConfig.setDefaults(self)
911  self.getTemplate.retarget(GetCalexpAsTemplateTask)
912 
913 
915  """!Image difference Task used in the Winter 2013 data challege.
916  Enables testing the effects of registration shifts and scatter.
917 
918  For use with winter 2013 simulated images:
919  Use --templateId visit=88868666 for sparse data
920  --templateId visit=22222200 for dense data (g)
921  --templateId visit=11111100 for dense data (i)
922  """
923  ConfigClass = Winter2013ImageDifferenceConfig
924  _DefaultName = "winter2013ImageDifference"
925 
926  def __init__(self, **kwargs):
927  ImageDifferenceTask.__init__(self, **kwargs)
928 
929  def fitAstrometry(self, templateSources, templateExposure, selectSources):
930  """Fit the relative astrometry between templateSources and selectSources"""
931  if self.config.winter2013WcsShift > 0.0:
932  offset = geom.Extent2D(self.config.winter2013WcsShift,
933  self.config.winter2013WcsShift)
934  cKey = templateSources[0].getTable().getCentroidKey()
935  for source in templateSources:
936  centroid = source.get(cKey)
937  source.set(cKey, centroid + offset)
938  elif self.config.winter2013WcsRms > 0.0:
939  cKey = templateSources[0].getTable().getCentroidKey()
940  for source in templateSources:
941  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
942  self.config.winter2013WcsRms*numpy.random.normal())
943  centroid = source.get(cKey)
944  source.set(cKey, centroid + offset)
945 
946  results = self.register.run(templateSources, templateExposure.getWcs(),
947  templateExposure.getBBox(), selectSources)
948  return results
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDataRef(self, sensorRef, templateIdList=None)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def fitAstrometry(self, templateSources, templateExposure, selectSources)