lsst.pipe.tasks  17.0.1-11-gf64fdeb0
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
28 import lsst.daf.base as dafBase
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.math as afwMath
31 import lsst.afw.table as afwTable
32 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
33 from lsst.meas.base import ForcedMeasurementTask
34 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
35 from lsst.pipe.tasks.registerImage import RegisterTask
36 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
37 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
38  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
39  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
40  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
41 import lsst.ip.diffim.diffimTools as diffimTools
42 import lsst.ip.diffim.utils as diUtils
43 import lsst.afw.display as afwDisplay
44 
45 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
46 IqrToSigma = 0.741
47 
48 
49 class ImageDifferenceConfig(pexConfig.Config):
50  """Config for ImageDifferenceTask
51  """
52  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=True,
53  doc="Add background to calexp before processing it. "
54  "Useful as ipDiffim does background matching.")
55  doUseRegister = pexConfig.Field(dtype=bool, default=True,
56  doc="Use image-to-image registration to align template with "
57  "science image")
58  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
59  doc="Writing debugging data for doUseRegister")
60  doSelectSources = pexConfig.Field(dtype=bool, default=True,
61  doc="Select stars to use for kernel fitting")
62  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
63  doc="Select stars of extreme color as part of the control sample")
64  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
65  doc="Select stars that are variable to be part "
66  "of the control sample")
67  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
68  doPreConvolve = pexConfig.Field(dtype=bool, default=True,
69  doc="Convolve science image by its PSF before PSF-matching?")
70  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
71  doc="Use a simple gaussian PSF model for pre-convolution "
72  "(else use fit PSF)? Ignored if doPreConvolve false.")
73  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
74  doDecorrelation = pexConfig.Field(dtype=bool, default=False,
75  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
76  "kernel convolution? If True, also update the diffim PSF.")
77  doMerge = pexConfig.Field(dtype=bool, default=True,
78  doc="Merge positive and negative diaSources with grow radius "
79  "set by growFootprint")
80  doMatchSources = pexConfig.Field(dtype=bool, default=True,
81  doc="Match diaSources with input calexp sources and ref catalog sources")
82  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
83  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
84  doForcedMeasurement = pexConfig.Field(dtype=bool, default=True,
85  doc="Force photometer diaSource locations on PVI?")
86  doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=True, doc="Write difference exposure?")
87  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
88  doc="Write warped and PSF-matched template coadd exposure?")
89  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
90  doAddMetrics = pexConfig.Field(dtype=bool, default=True,
91  doc="Add columns to the source table to hold analysis metrics?")
92 
93  coaddName = pexConfig.Field(
94  doc="coadd name: typically one of deep, goodSeeing, or dcr",
95  dtype=str,
96  default="deep",
97  )
98  convolveTemplate = pexConfig.Field(
99  doc="Which image gets convolved (default = template)",
100  dtype=bool,
101  default=True
102  )
103  refObjLoader = pexConfig.ConfigurableField(
104  target=LoadIndexedReferenceObjectsTask,
105  doc="reference object loader",
106  )
107  astrometer = pexConfig.ConfigurableField(
108  target=AstrometryTask,
109  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
110  )
111  sourceSelector = pexConfig.ConfigurableField(
112  target=ObjectSizeStarSelectorTask,
113  doc="Source selection algorithm",
114  )
115  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
116  decorrelate = pexConfig.ConfigurableField(
117  target=DecorrelateALKernelSpatialTask,
118  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
119  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
120  "default of 5.5).",
121  )
122  doSpatiallyVarying = pexConfig.Field(
123  dtype=bool,
124  default=False,
125  doc="If using Zogy or A&L decorrelation, perform these on a grid across the "
126  "image in order to allow for spatial variations"
127  )
128  detection = pexConfig.ConfigurableField(
129  target=SourceDetectionTask,
130  doc="Low-threshold detection for final measurement",
131  )
132  measurement = pexConfig.ConfigurableField(
133  target=DipoleFitTask,
134  doc="Enable updated dipole fitting method",
135  )
136  forcedMeasurement = pexConfig.ConfigurableField(
137  target=ForcedMeasurementTask,
138  doc="Subtask to force photometer PVI at diaSource location.",
139  )
140  getTemplate = pexConfig.ConfigurableField(
141  target=GetCoaddAsTemplateTask,
142  doc="Subtask to retrieve template exposure and sources",
143  )
144  controlStepSize = pexConfig.Field(
145  doc="What step size (every Nth one) to select a control sample from the kernelSources",
146  dtype=int,
147  default=5
148  )
149  controlRandomSeed = pexConfig.Field(
150  doc="Random seed for shuffing the control sample",
151  dtype=int,
152  default=10
153  )
154  register = pexConfig.ConfigurableField(
155  target=RegisterTask,
156  doc="Task to enable image-to-image image registration (warping)",
157  )
158  kernelSourcesFromRef = pexConfig.Field(
159  doc="Select sources to measure kernel from reference catalog if True, template if false",
160  dtype=bool,
161  default=False
162  )
163  templateSipOrder = pexConfig.Field(dtype=int, default=2,
164  doc="Sip Order for fitting the Template Wcs "
165  "(default is too high, overfitting)")
166 
167  growFootprint = pexConfig.Field(dtype=int, default=2,
168  doc="Grow positive and negative footprints by this amount before merging")
169 
170  diaSourceMatchRadius = pexConfig.Field(dtype=float, default=0.5,
171  doc="Match radius (in arcseconds) "
172  "for DiaSource to Source association")
173 
174  def setDefaults(self):
175  # defaults are OK for catalog and diacatalog
176 
177  self.subtract['al'].kernel.name = "AL"
178  self.subtract['al'].kernel.active.fitForBackground = True
179  self.subtract['al'].kernel.active.spatialKernelOrder = 1
180  self.subtract['al'].kernel.active.spatialBgOrder = 0
181  self.doPreConvolve = False
182  self.doMatchSources = False
183  self.doAddMetrics = False
184  self.doUseRegister = False
185 
186  # DiaSource Detection
187  self.detection.thresholdPolarity = "both"
188  self.detection.thresholdValue = 5.5
189  self.detection.reEstimateBackground = False
190  self.detection.thresholdType = "pixel_stdev"
191 
192  # Add filtered flux measurement, the correct measurement for pre-convolved images.
193  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
194  # To change that you must modify algorithms.names in the task's applyOverrides method,
195  # after the user has set doPreConvolve.
196  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
197 
198  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
199  self.forcedMeasurement.copyColumns = {
200  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
201  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
202  self.forcedMeasurement.slots.shape = None
203 
204  # For shuffling the control sample
205  random.seed(self.controlRandomSeed)
206 
207  def validate(self):
208  pexConfig.Config.validate(self)
209  if self.doMeasurement and not self.doDetection:
210  raise ValueError("Cannot run source measurement without source detection.")
211  if self.doMerge and not self.doDetection:
212  raise ValueError("Cannot run source merging without source detection.")
213  if self.doUseRegister and not self.doSelectSources:
214  raise ValueError("doUseRegister=True and doSelectSources=False. "
215  "Cannot run RegisterTask without selecting sources.")
216  if hasattr(self.getTemplate, "coaddName"):
217  if self.getTemplate.coaddName != self.coaddName:
218  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
219 
220 
221 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
222 
223  @staticmethod
224  def getTargetList(parsedCmd, **kwargs):
225  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
226  **kwargs)
227 
228 
229 class ImageDifferenceTask(pipeBase.CmdLineTask):
230  """Subtract an image from a template and measure the result
231  """
232  ConfigClass = ImageDifferenceConfig
233  RunnerClass = ImageDifferenceTaskRunner
234  _DefaultName = "imageDifference"
235 
236  def __init__(self, butler=None, **kwargs):
237  """!Construct an ImageDifference Task
238 
239  @param[in] butler Butler object to use in constructing reference object loaders
240  """
241  pipeBase.CmdLineTask.__init__(self, **kwargs)
242  self.makeSubtask("getTemplate")
243 
244  self.makeSubtask("subtract")
245 
246  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
247  self.makeSubtask("decorrelate")
248 
249  if self.config.doUseRegister:
250  self.makeSubtask("register")
251  self.schema = afwTable.SourceTable.makeMinimalSchema()
252 
253  if self.config.doSelectSources:
254  self.makeSubtask("sourceSelector")
255  self.makeSubtask('refObjLoader', butler=butler)
256  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
257 
258  self.algMetadata = dafBase.PropertyList()
259  if self.config.doDetection:
260  self.makeSubtask("detection", schema=self.schema)
261  if self.config.doMeasurement:
262  self.makeSubtask("measurement", schema=self.schema,
263  algMetadata=self.algMetadata)
264  if self.config.doForcedMeasurement:
265  self.schema.addField(
266  "ip_diffim_forced_PsfFlux_instFlux", "D",
267  "Forced PSF flux measured on the direct image.")
268  self.schema.addField(
269  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
270  "Forced PSF flux error measured on the direct image.")
271  self.schema.addField(
272  "ip_diffim_forced_PsfFlux_area", "F",
273  "Forced PSF flux effective area of PSF.",
274  units="pixel")
275  self.schema.addField(
276  "ip_diffim_forced_PsfFlux_flag", "Flag",
277  "Forced PSF flux general failure flag.")
278  self.schema.addField(
279  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
280  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
281  self.schema.addField(
282  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
283  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
284  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
285  if self.config.doMatchSources:
286  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
287  self.schema.addField("srcMatchId", "L", "unique id of source match")
288 
289  @pipeBase.timeMethod
290  def runDataRef(self, sensorRef, templateIdList=None):
291  """Subtract an image from a template coadd and measure the result
292 
293  Steps include:
294  - warp template coadd to match WCS of image
295  - PSF match image to warped template
296  - subtract image from PSF-matched, warped template
297  - persist difference image
298  - detect sources
299  - measure sources
300 
301  @param sensorRef: sensor-level butler data reference, used for the following data products:
302  Input only:
303  - calexp
304  - psf
305  - ccdExposureId
306  - ccdExposureId_bits
307  - self.config.coaddName + "Coadd_skyMap"
308  - self.config.coaddName + "Coadd"
309  Input or output, depending on config:
310  - self.config.coaddName + "Diff_subtractedExp"
311  Output, depending on config:
312  - self.config.coaddName + "Diff_matchedExp"
313  - self.config.coaddName + "Diff_src"
314 
315  @return pipe_base Struct containing these fields:
316  - subtractedExposure: exposure after subtracting template;
317  the unpersisted version if subtraction not run but detection run
318  None if neither subtraction nor detection run (i.e. nothing useful done)
319  - subtractRes: results of subtraction task; None if subtraction not run
320  - sources: detected and possibly measured sources; None if detection not run
321  """
322  self.log.info("Processing %s" % (sensorRef.dataId))
323 
324  # initialize outputs and some intermediate products
325  subtractedExposure = None
326  subtractRes = None
327  selectSources = None
328  kernelSources = None
329  controlSources = None
330  diaSources = None
331 
332  # We make one IdFactory that will be used by both icSrc and src datasets;
333  # I don't know if this is the way we ultimately want to do things, but at least
334  # this ensures the source IDs are fully unique.
335  expBits = sensorRef.get("ccdExposureId_bits")
336  expId = int(sensorRef.get("ccdExposureId"))
337  idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
338 
339  # Retrieve the science image we wish to analyze
340  exposure = sensorRef.get("calexp", immediate=True)
341  if self.config.doAddCalexpBackground:
342  mi = exposure.getMaskedImage()
343  mi += sensorRef.get("calexpBackground").getImage()
344  if not exposure.hasPsf():
345  raise pipeBase.TaskError("Exposure has no psf")
346  sciencePsf = exposure.getPsf()
347 
348  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
349  templateExposure = None # Stitched coadd exposure
350  templateSources = None # Sources on the template image
351  if self.config.doSubtract:
352  template = self.getTemplate.run(exposure, sensorRef, templateIdList=templateIdList)
353  templateExposure = template.exposure
354  templateSources = template.sources
355 
356  if self.config.subtract.name == 'zogy':
357  subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
358  doWarping=True,
359  spatiallyVarying=self.config.doSpatiallyVarying,
360  doPreConvolve=self.config.doPreConvolve)
361  subtractedExposure = subtractRes.subtractedExposure
362 
363  elif self.config.subtract.name == 'al':
364  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
365  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
366 
367  # sigma of PSF of template image before warping
368  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
369 
370  # if requested, convolve the science exposure with its PSF
371  # (properly, this should be a cross-correlation, but our code does not yet support that)
372  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
373  # else sigma of original science exposure
374  preConvPsf = None
375  if self.config.doPreConvolve:
376  convControl = afwMath.ConvolutionControl()
377  # cannot convolve in place, so make a new MI to receive convolved image
378  srcMI = exposure.getMaskedImage()
379  destMI = srcMI.Factory(srcMI.getDimensions())
380  srcPsf = sciencePsf
381  if self.config.useGaussianForPreConvolution:
382  # convolve with a simplified PSF model: a double Gaussian
383  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
384  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
385  else:
386  # convolve with science exposure's PSF model
387  preConvPsf = srcPsf
388  afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
389  exposure.setMaskedImage(destMI)
390  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
391  else:
392  scienceSigmaPost = scienceSigmaOrig
393 
394  # If requested, find sources in the image
395  if self.config.doSelectSources:
396  if not sensorRef.datasetExists("src"):
397  self.log.warn("Src product does not exist; running detection, measurement, selection")
398  # Run own detection and measurement; necessary in nightly processing
399  selectSources = self.subtract.getSelectSources(
400  exposure,
401  sigma=scienceSigmaPost,
402  doSmooth=not self.doPreConvolve,
403  idFactory=idFactory,
404  )
405  else:
406  self.log.info("Source selection via src product")
407  # Sources already exist; for data release processing
408  selectSources = sensorRef.get("src")
409 
410  # Number of basis functions
411  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
412  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
413  targetFwhmPix=templateSigma*FwhmPerSigma))
414 
415  if self.config.doAddMetrics:
416  # Modify the schema of all Sources
417  kcQa = KernelCandidateQa(nparam)
418  selectSources = kcQa.addToSchema(selectSources)
419 
420  if self.config.kernelSourcesFromRef:
421  # match exposure sources to reference catalog
422  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
423  matches = astromRet.matches
424  elif templateSources:
425  # match exposure sources to template sources
426  mc = afwTable.MatchControl()
427  mc.findOnlyClosest = False
428  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*afwGeom.arcseconds,
429  mc)
430  else:
431  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
432  "but template sources not available. Cannot match science "
433  "sources with template sources. Run process* on data from "
434  "which templates are built.")
435 
436  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
437  matches=matches).sourceCat
438 
439  random.shuffle(kernelSources, random.random)
440  controlSources = kernelSources[::self.config.controlStepSize]
441  kernelSources = [k for i, k in enumerate(kernelSources)
442  if i % self.config.controlStepSize]
443 
444  if self.config.doSelectDcrCatalog:
445  redSelector = DiaCatalogSourceSelectorTask(
446  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
447  grMax=99.999))
448  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
449  controlSources.extend(redSources)
450 
451  blueSelector = DiaCatalogSourceSelectorTask(
452  DiaCatalogSourceSelectorConfig(grMin=-99.999,
453  grMax=self.sourceSelector.config.grMin))
454  blueSources = blueSelector.selectStars(exposure, selectSources,
455  matches=matches).starCat
456  controlSources.extend(blueSources)
457 
458  if self.config.doSelectVariableCatalog:
459  varSelector = DiaCatalogSourceSelectorTask(
460  DiaCatalogSourceSelectorConfig(includeVariable=True))
461  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
462  controlSources.extend(varSources)
463 
464  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
465  % (len(kernelSources), len(selectSources), len(controlSources)))
466  allresids = {}
467  if self.config.doUseRegister:
468  self.log.info("Registering images")
469 
470  if templateSources is None:
471  # Run detection on the template, which is
472  # temporarily background-subtracted
473  templateSources = self.subtract.getSelectSources(
474  templateExposure,
475  sigma=templateSigma,
476  doSmooth=True,
477  idFactory=idFactory
478  )
479 
480  # Third step: we need to fit the relative astrometry.
481  #
482  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
483  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
484  exposure.getWcs(), exposure.getBBox())
485  templateExposure = warpedExp
486 
487  # Create debugging outputs on the astrometric
488  # residuals as a function of position. Persistence
489  # not yet implemented; expected on (I believe) #2636.
490  if self.config.doDebugRegister:
491  # Grab matches to reference catalog
492  srcToMatch = {x.second.getId(): x.first for x in matches}
493 
494  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
495  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
496  sids = [m.first.getId() for m in wcsResults.matches]
497  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
498  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
499  m.second.get(inCentroidKey))) for m in wcsResults.matches]
500  allresids = dict(zip(sids, zip(positions, residuals)))
501 
502  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
503  wcsResults.wcs.pixelToSky(
504  m.second.get(inCentroidKey))) for m in wcsResults.matches]
505  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g")) +
506  2.5*numpy.log10(srcToMatch[x].get("r"))
507  for x in sids if x in srcToMatch.keys()])
508  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
509  if s in srcToMatch.keys()])
510  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
511  if s in srcToMatch.keys()])
512  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
513  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
514  (colors <= self.sourceSelector.config.grMax))
515  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
516  rms1Long = IqrToSigma*(
517  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
518  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
519  numpy.percentile(dlat[idx1], 25))
520  rms2Long = IqrToSigma*(
521  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
522  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
523  numpy.percentile(dlat[idx2], 25))
524  rms3Long = IqrToSigma*(
525  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
526  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
527  numpy.percentile(dlat[idx3], 25))
528  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
529  (numpy.median(dlong[idx1]), rms1Long,
530  numpy.median(dlat[idx1]), rms1Lat))
531  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
532  (numpy.median(dlong[idx2]), rms2Long,
533  numpy.median(dlat[idx2]), rms2Lat))
534  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
535  (numpy.median(dlong[idx3]), rms3Long,
536  numpy.median(dlat[idx3]), rms3Lat))
537 
538  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
539  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
540  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
541  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
542  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
543  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
544 
545  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
546  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
547  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
548  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
549  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
550  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
551 
552  # warp template exposure to match exposure,
553  # PSF match template exposure to exposure,
554  # then return the difference
555 
556  # Return warped template... Construct sourceKernelCand list after subtract
557  self.log.info("Subtracting images")
558  subtractRes = self.subtract.subtractExposures(
559  templateExposure=templateExposure,
560  scienceExposure=exposure,
561  candidateList=kernelSources,
562  convolveTemplate=self.config.convolveTemplate,
563  doWarping=not self.config.doUseRegister
564  )
565  subtractedExposure = subtractRes.subtractedExposure
566 
567  if self.config.doWriteMatchedExp:
568  sensorRef.put(subtractRes.matchedExposure, self.config.coaddName + "Diff_matchedExp")
569 
570  if self.config.doDetection:
571  self.log.info("Computing diffim PSF")
572  if subtractedExposure is None:
573  subtractedExposure = sensorRef.get(subtractedExposureName)
574 
575  # Get Psf from the appropriate input image if it doesn't exist
576  if not subtractedExposure.hasPsf():
577  if self.config.convolveTemplate:
578  subtractedExposure.setPsf(exposure.getPsf())
579  else:
580  if templateExposure is None:
581  template = self.getTemplate.run(exposure, sensorRef,
582  templateIdList=templateIdList)
583  subtractedExposure.setPsf(template.exposure.getPsf())
584 
585  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
586  # thus it may have already been decorrelated. Thus, we do not decorrelate if
587  # doSubtract is False.
588  if self.config.doDecorrelation and self.config.doSubtract:
589  preConvKernel = None
590  if preConvPsf is not None:
591  preConvKernel = preConvPsf.getLocalKernel()
592  decorrResult = self.decorrelate.run(exposure, templateExposure,
593  subtractedExposure,
594  subtractRes.psfMatchingKernel,
595  spatiallyVarying=self.config.doSpatiallyVarying,
596  preConvKernel=preConvKernel)
597  subtractedExposure = decorrResult.correctedExposure
598 
599  # END (if subtractAlgorithm == 'AL')
600 
601  if self.config.doDetection:
602  self.log.info("Running diaSource detection")
603  # Erase existing detection mask planes
604  mask = subtractedExposure.getMaskedImage().getMask()
605  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
606 
607  table = afwTable.SourceTable.make(self.schema, idFactory)
608  table.setMetadata(self.algMetadata)
609  results = self.detection.makeSourceCatalog(
610  table=table,
611  exposure=subtractedExposure,
612  doSmooth=not self.config.doPreConvolve
613  )
614 
615  if self.config.doMerge:
616  fpSet = results.fpSets.positive
617  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
618  self.config.growFootprint, False)
619  diaSources = afwTable.SourceCatalog(table)
620  fpSet.makeSources(diaSources)
621  self.log.info("Merging detections into %d sources" % (len(diaSources)))
622  else:
623  diaSources = results.sources
624 
625  if self.config.doMeasurement:
626  newDipoleFitting = self.config.doDipoleFitting
627  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
628  if not newDipoleFitting:
629  # Just fit dipole in diffim
630  self.measurement.run(diaSources, subtractedExposure)
631  else:
632  # Use (matched) template and science image (if avail.) to constrain dipole fitting
633  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
634  self.measurement.run(diaSources, subtractedExposure, exposure,
635  subtractRes.matchedExposure)
636  else:
637  self.measurement.run(diaSources, subtractedExposure, exposure)
638 
639  if self.config.doForcedMeasurement:
640  # Run forced psf photometry on the PVI at the diaSource locations.
641  # Copy the measured flux and error into the diaSource.
642  forcedSources = self.forcedMeasurement.generateMeasCat(
643  exposure, diaSources, subtractedExposure.getWcs())
644  self.forcedMeasurement.run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
645  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
646  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
647  "ip_diffim_forced_PsfFlux_instFlux", True)
648  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
649  "ip_diffim_forced_PsfFlux_instFluxErr", True)
650  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
651  "ip_diffim_forced_PsfFlux_area", True)
652  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
653  "ip_diffim_forced_PsfFlux_flag", True)
654  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
655  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
656  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
657  "ip_diffim_forced_PsfFlux_flag_edge", True)
658  for diaSource, forcedSource in zip(diaSources, forcedSources):
659  diaSource.assign(forcedSource, mapper)
660 
661  # Match with the calexp sources if possible
662  if self.config.doMatchSources:
663  if sensorRef.datasetExists("src"):
664  # Create key,val pair where key=diaSourceId and val=sourceId
665  matchRadAsec = self.config.diaSourceMatchRadius
666  matchRadPixel = matchRadAsec/exposure.getWcs().pixelScale().asArcseconds()
667 
668  srcMatches = afwTable.matchXy(sensorRef.get("src"), diaSources, matchRadPixel)
669  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
670  srcMatch in srcMatches])
671  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
672  len(diaSources)))
673  else:
674  self.log.warn("Src product does not exist; cannot match with diaSources")
675  srcMatchDict = {}
676 
677  # Create key,val pair where key=diaSourceId and val=refId
678  refAstromConfig = AstrometryConfig()
679  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
680  refAstrometer = AstrometryTask(refAstromConfig)
681  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
682  refMatches = astromRet.matches
683  if refMatches is None:
684  self.log.warn("No diaSource matches with reference catalog")
685  refMatchDict = {}
686  else:
687  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
688  len(diaSources)))
689  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
690  refMatch in refMatches])
691 
692  # Assign source Ids
693  for diaSource in diaSources:
694  sid = diaSource.getId()
695  if sid in srcMatchDict:
696  diaSource.set("srcMatchId", srcMatchDict[sid])
697  if sid in refMatchDict:
698  diaSource.set("refMatchId", refMatchDict[sid])
699 
700  if diaSources is not None and self.config.doWriteSources:
701  sensorRef.put(diaSources, self.config.coaddName + "Diff_diaSrc")
702 
703  if self.config.doAddMetrics and self.config.doSelectSources:
704  self.log.info("Evaluating metrics and control sample")
705 
706  kernelCandList = []
707  for cell in subtractRes.kernelCellSet.getCellList():
708  for cand in cell.begin(False): # include bad candidates
709  kernelCandList.append(cand)
710 
711  # Get basis list to build control sample kernels
712  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
713 
714  controlCandList = (
715  diffimTools.sourceTableToCandidateList(controlSources,
716  subtractRes.warpedExposure, exposure,
717  self.config.subtract.kernel.active,
718  self.config.subtract.kernel.active.detectionConfig,
719  self.log, doBuild=True, basisList=basisList))
720 
721  kcQa.apply(kernelCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel,
722  dof=nparam)
723  kcQa.apply(controlCandList, subtractRes.psfMatchingKernel, subtractRes.backgroundModel)
724 
725  if self.config.doDetection:
726  kcQa.aggregate(selectSources, self.metadata, allresids, diaSources)
727  else:
728  kcQa.aggregate(selectSources, self.metadata, allresids)
729 
730  sensorRef.put(selectSources, self.config.coaddName + "Diff_kernelSrc")
731 
732  if self.config.doWriteSubtractedExp:
733  sensorRef.put(subtractedExposure, subtractedExposureName)
734 
735  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
736  return pipeBase.Struct(
737  subtractedExposure=subtractedExposure,
738  subtractRes=subtractRes,
739  sources=diaSources,
740  )
741 
742  def fitAstrometry(self, templateSources, templateExposure, selectSources):
743  """Fit the relative astrometry between templateSources and selectSources
744 
745  @todo remove this method. It originally fit a new WCS to the template before calling register.run
746  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
747  It remains because a subtask overrides it.
748  """
749  results = self.register.run(templateSources, templateExposure.getWcs(),
750  templateExposure.getBBox(), selectSources)
751  return results
752 
753  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
754  """@todo Test and update for current debug display and slot names
755  """
756  import lsstDebug
757  display = lsstDebug.Info(__name__).display
758  showSubtracted = lsstDebug.Info(__name__).showSubtracted
759  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
760  showDiaSources = lsstDebug.Info(__name__).showDiaSources
761  showDipoles = lsstDebug.Info(__name__).showDipoles
762  maskTransparency = lsstDebug.Info(__name__).maskTransparency
763  if display:
764  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
765  if not maskTransparency:
766  maskTransparency = 0
767  disp.setMaskTransparency(maskTransparency)
768 
769  if display and showSubtracted:
770  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
771  mi = subtractRes.subtractedExposure.getMaskedImage()
772  x0, y0 = mi.getX0(), mi.getY0()
773  with disp.Buffering():
774  for s in diaSources:
775  x, y = s.getX() - x0, s.getY() - y0
776  ctype = "red" if s.get("flags_negative") else "yellow"
777  if (s.get("base_PixelFlags_flag_interpolatedCenter") or
778  s.get("base_PixelFlags_flag_saturatedCenter") or
779  s.get("base_PixelFlags_flag_crCenter")):
780  ptype = "x"
781  elif (s.get("base_PixelFlags_flag_interpolated") or
782  s.get("base_PixelFlags_flag_saturated") or
783  s.get("base_PixelFlags_flag_cr")):
784  ptype = "+"
785  else:
786  ptype = "o"
787  disp.dot(ptype, x, y, size=4, ctype=ctype)
788  lsstDebug.frame += 1
789 
790  if display and showPixelResiduals and selectSources:
791  nonKernelSources = []
792  for source in selectSources:
793  if source not in kernelSources:
794  nonKernelSources.append(source)
795 
796  diUtils.plotPixelResiduals(exposure,
797  subtractRes.warpedExposure,
798  subtractRes.subtractedExposure,
799  subtractRes.kernelCellSet,
800  subtractRes.psfMatchingKernel,
801  subtractRes.backgroundModel,
802  nonKernelSources,
803  self.subtract.config.kernel.active.detectionConfig,
804  origVariance=False)
805  diUtils.plotPixelResiduals(exposure,
806  subtractRes.warpedExposure,
807  subtractRes.subtractedExposure,
808  subtractRes.kernelCellSet,
809  subtractRes.psfMatchingKernel,
810  subtractRes.backgroundModel,
811  nonKernelSources,
812  self.subtract.config.kernel.active.detectionConfig,
813  origVariance=True)
814  if display and showDiaSources:
815  flagChecker = SourceFlagChecker(diaSources)
816  isFlagged = [flagChecker(x) for x in diaSources]
817  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
818  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
819  frame=lsstDebug.frame)
820  lsstDebug.frame += 1
821 
822  if display and showDipoles:
823  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
824  frame=lsstDebug.frame)
825  lsstDebug.frame += 1
826 
827  def _getConfigName(self):
828  """Return the name of the config dataset
829  """
830  return "%sDiff_config" % (self.config.coaddName,)
831 
832  def _getMetadataName(self):
833  """Return the name of the metadata dataset
834  """
835  return "%sDiff_metadata" % (self.config.coaddName,)
836 
837  def getSchemaCatalogs(self):
838  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
839  diaSrc = afwTable.SourceCatalog(self.schema)
840  diaSrc.getTable().setMetadata(self.algMetadata)
841  return {self.config.coaddName + "Diff_diaSrc": diaSrc}
842 
843  @classmethod
844  def _makeArgumentParser(cls):
845  """Create an argument parser
846  """
847  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
848  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
849  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
850  help="Template data ID in case of calexp template,"
851  " e.g. --templateId visit=6789")
852  return parser
853 
854 
856  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
857  doc="Shift stars going into RegisterTask by this amount")
858  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
859  doc="Perturb stars going into RegisterTask by this amount")
860 
861  def setDefaults(self):
862  ImageDifferenceConfig.setDefaults(self)
863  self.getTemplate.retarget(GetCalexpAsTemplateTask)
864 
865 
867  """!Image difference Task used in the Winter 2013 data challege.
868  Enables testing the effects of registration shifts and scatter.
869 
870  For use with winter 2013 simulated images:
871  Use --templateId visit=88868666 for sparse data
872  --templateId visit=22222200 for dense data (g)
873  --templateId visit=11111100 for dense data (i)
874  """
875  ConfigClass = Winter2013ImageDifferenceConfig
876  _DefaultName = "winter2013ImageDifference"
877 
878  def __init__(self, **kwargs):
879  ImageDifferenceTask.__init__(self, **kwargs)
880 
881  def fitAstrometry(self, templateSources, templateExposure, selectSources):
882  """Fit the relative astrometry between templateSources and selectSources"""
883  if self.config.winter2013WcsShift > 0.0:
884  offset = afwGeom.Extent2D(self.config.winter2013WcsShift,
885  self.config.winter2013WcsShift)
886  cKey = templateSources[0].getTable().getCentroidKey()
887  for source in templateSources:
888  centroid = source.get(cKey)
889  source.set(cKey, centroid + offset)
890  elif self.config.winter2013WcsRms > 0.0:
891  cKey = templateSources[0].getTable().getCentroidKey()
892  for source in templateSources:
893  offset = afwGeom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
894  self.config.winter2013WcsRms*numpy.random.normal())
895  centroid = source.get(cKey)
896  source.set(cKey, centroid + offset)
897 
898  results = self.register.run(templateSources, templateExposure.getWcs(),
899  templateExposure.getBBox(), selectSources)
900  return results
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDataRef(self, sensorRef, templateIdList=None)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)