lsst.pipe.tasks  21.0.0-95-g1e789892+a35ce0983c
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 from lsst.skymap import BaseSkyMap
47 from lsst.obs.base import ExposureIdInfo
48 
49 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
50 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
51 IqrToSigma = 0.741
52 
53 
54 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
55  dimensions=("instrument", "visit", "detector", "skymap"),
56  defaultTemplates={"coaddName": "deep",
57  "skyMapName": "deep",
58  "warpTypeSuffix": "",
59  "fakesType": ""}):
60 
61  exposure = pipeBase.connectionTypes.Input(
62  doc="Input science exposure to subtract from.",
63  dimensions=("instrument", "visit", "detector"),
64  storageClass="ExposureF",
65  name="{fakesType}calexp"
66  )
67 
68  # TODO DM-22953
69  # kernelSources = pipeBase.connectionTypes.Input(
70  # doc="Source catalog produced in calibrate task for kernel candidate sources",
71  # name="src",
72  # storageClass="SourceCatalog",
73  # dimensions=("instrument", "visit", "detector"),
74  # )
75 
76  skyMap = pipeBase.connectionTypes.Input(
77  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
78  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
79  dimensions=("skymap", ),
80  storageClass="SkyMap",
81  )
82  coaddExposures = pipeBase.connectionTypes.Input(
83  doc="Input template to match and subtract from the exposure",
84  dimensions=("tract", "patch", "skymap", "band"),
85  storageClass="ExposureF",
86  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
87  multiple=True,
88  deferLoad=True
89  )
90  dcrCoadds = pipeBase.connectionTypes.Input(
91  doc="Input DCR template to match and subtract from the exposure",
92  name="{fakesType}dcrCoadd{warpTypeSuffix}",
93  storageClass="ExposureF",
94  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
95  multiple=True,
96  deferLoad=True
97  )
98  outputSchema = pipeBase.connectionTypes.InitOutput(
99  doc="Schema (as an example catalog) for output DIASource catalog.",
100  storageClass="SourceCatalog",
101  name="{fakesType}{coaddName}Diff_diaSrc_schema",
102  )
103  # TODO DM-29965: Currently the AL likelihood image is not a separate data product
104  subtractedExposure = pipeBase.connectionTypes.Output(
105  doc="Output AL difference or Zogy proper difference image",
106  dimensions=("instrument", "visit", "detector"),
107  storageClass="ExposureF",
108  name="{fakesType}{coaddName}Diff_differenceExp",
109  )
110  scoreExposure = pipeBase.connectionTypes.Output(
111  doc="Output Zogy score (likelihood) image",
112  dimensions=("instrument", "visit", "detector"),
113  storageClass="ExposureF",
114  name="{fakesType}{coaddName}Diff_scoreExp",
115  )
116  warpedExposure = pipeBase.connectionTypes.Output(
117  doc="Warped template used to create `subtractedExposure`.",
118  dimensions=("instrument", "visit", "detector"),
119  storageClass="ExposureF",
120  name="{fakesType}{coaddName}Diff_warpedExp",
121  )
122  matchedExposure = pipeBase.connectionTypes.Output(
123  doc="Warped template used to create `subtractedExposure`.",
124  dimensions=("instrument", "visit", "detector"),
125  storageClass="ExposureF",
126  name="{fakesType}{coaddName}Diff_matchedExp",
127  )
128  diaSources = pipeBase.connectionTypes.Output(
129  doc="Output detected diaSources on the difference image",
130  dimensions=("instrument", "visit", "detector"),
131  storageClass="SourceCatalog",
132  name="{fakesType}{coaddName}Diff_diaSrc",
133  )
134 
135  def __init__(self, *, config=None):
136  super().__init__(config=config)
137  if config.coaddName == 'dcr':
138  self.inputs.remove("coaddExposures")
139  else:
140  self.inputs.remove("dcrCoadds")
141  if not config.doWriteSubtractedExp:
142  self.outputs.remove("subtractedExposure")
143  if not config.doWriteScoreExp:
144  self.outputs.remove("scoreExposure")
145  if not config.doWriteWarpedExp:
146  self.outputs.remove("warpedExposure")
147  if not config.doWriteMatchedExp:
148  self.outputs.remove("matchedExposure")
149  if not config.doWriteSources:
150  self.outputs.remove("diaSources")
151 
152  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
153  # Make kernelSources optional
154 
155 
156 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
157  pipelineConnections=ImageDifferenceTaskConnections):
158  """Config for ImageDifferenceTask
159  """
160  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
161  doc="Add background to calexp before processing it. "
162  "Useful as ipDiffim does background matching.")
163  doUseRegister = pexConfig.Field(dtype=bool, default=False,
164  doc="Re-compute astrometry on the template. "
165  "Use image-to-image registration to align template with "
166  "science image (AL only).")
167  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
168  doc="Writing debugging data for doUseRegister")
169  doSelectSources = pexConfig.Field(dtype=bool, default=False,
170  doc="Select stars to use for kernel fitting (AL only)")
171  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
172  doc="Select stars of extreme color as part "
173  "of the control sample (AL only)")
174  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
175  doc="Select stars that are variable to be part "
176  "of the control sample (AL only)")
177  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
178  doPreConvolve = pexConfig.Field(dtype=bool, default=False,
179  doc="Convolve science image by its PSF before PSF-matching (AL only)."
180  " The difference image becomes a likelihood image.")
181  useScoreImageDetection = pexConfig.Field(
182  dtype=bool, default=False, doc="Calculate and detect sources on the Zogy score image (Zogy only).")
183  doWriteScoreExp = pexConfig.Field(
184  dtype=bool, default=False, doc="Write score exposure (Zogy only) ?")
185  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
186  doc="Scale variance of the template before PSF matching")
187  doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
188  doc="Scale variance of the diffim before PSF matching. "
189  "You may do either this or template variance scaling, "
190  "or neither. (Doing both is a waste of CPU.)")
191  useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=True,
192  doc="Use a simple gaussian PSF model for pre-convolution "
193  "(else use fit PSF)? Ignored if doPreConvolve false.")
194  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
195  doDecorrelation = pexConfig.Field(dtype=bool, default=True,
196  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
197  "kernel convolution (AL only)? If True, also update the diffim PSF.")
198  doMerge = pexConfig.Field(dtype=bool, default=True,
199  doc="Merge positive and negative diaSources with grow radius "
200  "set by growFootprint")
201  doMatchSources = pexConfig.Field(dtype=bool, default=False,
202  doc="Match diaSources with input calexp sources and ref catalog sources")
203  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
204  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
205  doForcedMeasurement = pexConfig.Field(
206  dtype=bool,
207  default=True,
208  doc="Force photometer diaSource locations on PVI?")
209  doWriteSubtractedExp = pexConfig.Field(
210  dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
211  doWriteWarpedExp = pexConfig.Field(
212  dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
213  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
214  doc="Write warped and PSF-matched template coadd exposure?")
215  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
216  doAddMetrics = pexConfig.Field(dtype=bool, default=False,
217  doc="Add columns to the source table to hold analysis metrics?")
218 
219  coaddName = pexConfig.Field(
220  doc="coadd name: typically one of deep, goodSeeing, or dcr",
221  dtype=str,
222  default="deep",
223  )
224  convolveTemplate = pexConfig.Field(
225  doc="Which image gets convolved (default = template)",
226  dtype=bool,
227  default=True
228  )
229  refObjLoader = pexConfig.ConfigurableField(
230  target=LoadIndexedReferenceObjectsTask,
231  doc="reference object loader",
232  )
233  astrometer = pexConfig.ConfigurableField(
234  target=AstrometryTask,
235  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
236  )
237  sourceSelector = pexConfig.ConfigurableField(
238  target=ObjectSizeStarSelectorTask,
239  doc="Source selection algorithm",
240  )
241  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
242  decorrelate = pexConfig.ConfigurableField(
243  target=DecorrelateALKernelSpatialTask,
244  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
245  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
246  "default of 5.5).",
247  )
248  # Old style ImageMapper grid. ZogyTask has its own grid option
249  doSpatiallyVarying = pexConfig.Field(
250  dtype=bool,
251  default=False,
252  doc="Perform A&L decorrelation on a grid across the "
253  "image in order to allow for spatial variations. Zogy does not use this option."
254  )
255  detection = pexConfig.ConfigurableField(
256  target=SourceDetectionTask,
257  doc="Low-threshold detection for final measurement",
258  )
259  measurement = pexConfig.ConfigurableField(
260  target=DipoleFitTask,
261  doc="Enable updated dipole fitting method",
262  )
263  doApCorr = lsst.pex.config.Field(
264  dtype=bool,
265  default=True,
266  doc="Run subtask to apply aperture corrections"
267  )
268  applyApCorr = lsst.pex.config.ConfigurableField(
269  target=ApplyApCorrTask,
270  doc="Subtask to apply aperture corrections"
271  )
272  forcedMeasurement = pexConfig.ConfigurableField(
273  target=ForcedMeasurementTask,
274  doc="Subtask to force photometer PVI at diaSource location.",
275  )
276  getTemplate = pexConfig.ConfigurableField(
277  target=GetCoaddAsTemplateTask,
278  doc="Subtask to retrieve template exposure and sources",
279  )
280  scaleVariance = pexConfig.ConfigurableField(
281  target=ScaleVarianceTask,
282  doc="Subtask to rescale the variance of the template "
283  "to the statistically expected level"
284  )
285  controlStepSize = pexConfig.Field(
286  doc="What step size (every Nth one) to select a control sample from the kernelSources",
287  dtype=int,
288  default=5
289  )
290  controlRandomSeed = pexConfig.Field(
291  doc="Random seed for shuffing the control sample",
292  dtype=int,
293  default=10
294  )
295  register = pexConfig.ConfigurableField(
296  target=RegisterTask,
297  doc="Task to enable image-to-image image registration (warping)",
298  )
299  kernelSourcesFromRef = pexConfig.Field(
300  doc="Select sources to measure kernel from reference catalog if True, template if false",
301  dtype=bool,
302  default=False
303  )
304  templateSipOrder = pexConfig.Field(
305  dtype=int, default=2,
306  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
307  )
308  growFootprint = pexConfig.Field(
309  dtype=int, default=2,
310  doc="Grow positive and negative footprints by this amount before merging"
311  )
312  diaSourceMatchRadius = pexConfig.Field(
313  dtype=float, default=0.5,
314  doc="Match radius (in arcseconds) for DiaSource to Source association"
315  )
316 
317  def setDefaults(self):
318  # defaults are OK for catalog and diacatalog
319 
320  self.subtract['al'].kernel.name = "AL"
321  self.subtract['al'].kernel.active.fitForBackground = True
322  self.subtract['al'].kernel.active.spatialKernelOrder = 1
323  self.subtract['al'].kernel.active.spatialBgOrder = 2
324 
325  # DiaSource Detection
326  self.detection.thresholdPolarity = "both"
327  self.detection.thresholdValue = 5.0
328  self.detection.reEstimateBackground = False
329  self.detection.thresholdType = "pixel_stdev"
330 
331  # Add filtered flux measurement, the correct measurement for pre-convolved images.
332  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
333  # To change that you must modify algorithms.names in the task's applyOverrides method,
334  # after the user has set doPreConvolve.
335  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
336  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
337  'base_LocalWcs']
338 
339  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
340  self.forcedMeasurement.copyColumns = {
341  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
342  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
343  self.forcedMeasurement.slots.shape = None
344 
345  # For shuffling the control sample
346  random.seed(self.controlRandomSeed)
347 
348  def validate(self):
349  pexConfig.Config.validate(self)
350  if not self.doSubtract and not self.doDetection:
351  raise ValueError("Either doSubtract or doDetection must be enabled.")
352  if self.doMeasurement and not self.doDetection:
353  raise ValueError("Cannot run source measurement without source detection.")
354  if self.doMerge and not self.doDetection:
355  raise ValueError("Cannot run source merging without source detection.")
356  if self.doUseRegister and not self.doSelectSources:
357  raise ValueError("doUseRegister=True and doSelectSources=False. "
358  "Cannot run RegisterTask without selecting sources.")
359  if hasattr(self.getTemplate, "coaddName"):
360  if self.getTemplate.coaddName != self.coaddName:
361  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
362  if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
363  raise ValueError("Scaling the diffim variance and scaling the template variance "
364  "are both set. Please choose one or the other.")
365  # We cannot allow inconsistencies that would lead to None or not available output products
366  if self.subtract.name == 'zogy':
367  if self.doWriteScoreExp and not self.useScoreImageDetection:
368  raise ValueError("doWriteScoreExp=True and useScoreImageDetection=False "
369  "is not supported. Score image is not calculated.")
370  if self.doWriteMatchedExp:
371  raise ValueError("doWriteMatchedExp=True Matched exposure is not "
372  "calculated in zogy subtraction.")
373  if self.doAddMetrics:
374  raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
375  if self.doDecorrelation:
376  raise ValueError(
377  "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
378  if self.doPreConvolve:
379  raise ValueError(
380  "doPreConvolve=True Pre-convolution is not a zogy option.")
381  if self.doSelectSources:
382  raise ValueError(
383  "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
384  else:
385  if self.useScoreImageDetection:
386  raise ValueError("useScoreImageDetection=True Score exposure does not "
387  "exist in AL subtraction.")
388  if self.doWriteScoreExp: # Ensure output is not None
389  raise ValueError("doWriteScoreExp=True Score exposure does not exist in AL subtraction.")
390  if self.doAddMetrics and not self.doSubtract:
391  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
392  if self.doPreConvolve and self.doDecorrelation:
393  raise NotImplementedError(
394  "doPreConvolve=True and doDecorrelation=True "
395  "The decorrelation afterburner cannot handle pre-convolved exposures.")
396 
397 
398 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
399 
400  @staticmethod
401  def getTargetList(parsedCmd, **kwargs):
402  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
403  **kwargs)
404 
405 
406 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
407  """Subtract an image from a template and measure the result
408  """
409  ConfigClass = ImageDifferenceConfig
410  RunnerClass = ImageDifferenceTaskRunner
411  _DefaultName = "imageDifference"
412 
413  def __init__(self, butler=None, **kwargs):
414  """!Construct an ImageDifference Task
415 
416  @param[in] butler Butler object to use in constructing reference object loaders
417  """
418  super().__init__(**kwargs)
419  self.makeSubtask("getTemplate")
420 
421  self.makeSubtask("subtract")
422 
423  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
424  self.makeSubtask("decorrelate")
425 
426  if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
427  self.makeSubtask("scaleVariance")
428 
429  if self.config.doUseRegister:
430  self.makeSubtask("register")
431  self.schema = afwTable.SourceTable.makeMinimalSchema()
432 
433  if self.config.doSelectSources:
434  self.makeSubtask("sourceSelector")
435  if self.config.kernelSourcesFromRef:
436  self.makeSubtask('refObjLoader', butler=butler)
437  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
438 
439  self.algMetadata = dafBase.PropertyList()
440  if self.config.doDetection:
441  self.makeSubtask("detection", schema=self.schema)
442  if self.config.doMeasurement:
443  self.makeSubtask("measurement", schema=self.schema,
444  algMetadata=self.algMetadata)
445  if self.config.doApCorr:
446  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
447  if self.config.doForcedMeasurement:
448  self.schema.addField(
449  "ip_diffim_forced_PsfFlux_instFlux", "D",
450  "Forced PSF flux measured on the direct image.",
451  units="count")
452  self.schema.addField(
453  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
454  "Forced PSF flux error measured on the direct image.",
455  units="count")
456  self.schema.addField(
457  "ip_diffim_forced_PsfFlux_area", "F",
458  "Forced PSF flux effective area of PSF.",
459  units="pixel")
460  self.schema.addField(
461  "ip_diffim_forced_PsfFlux_flag", "Flag",
462  "Forced PSF flux general failure flag.")
463  self.schema.addField(
464  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
465  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
466  self.schema.addField(
467  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
468  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
469  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
470  if self.config.doMatchSources:
471  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
472  self.schema.addField("srcMatchId", "L", "unique id of source match")
473 
474  # initialize InitOutputs
475  self.outputSchema = afwTable.SourceCatalog(self.schema)
476  self.outputSchema.getTable().setMetadata(self.algMetadata)
477 
478  @staticmethod
479  def makeIdFactory(expId, expBits):
480  """Create IdFactory instance for unique 64 bit diaSource id-s.
481 
482  Parameters
483  ----------
484  expId : `int`
485  Exposure id.
486 
487  expBits: `int`
488  Number of used bits in ``expId``.
489 
490  Note
491  ----
492  The diasource id-s consists of the ``expId`` stored fixed in the highest value
493  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
494  low value end of the integer.
495 
496  Returns
497  -------
498  idFactory: `lsst.afw.table.IdFactory`
499  """
500  return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
501 
502  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
503  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
504  inputRefs: pipeBase.InputQuantizedConnection,
505  outputRefs: pipeBase.OutputQuantizedConnection):
506  inputs = butlerQC.get(inputRefs)
507  self.log.info(f"Processing {butlerQC.quantum.dataId}")
508  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
509  returnMaxBits=True)
510  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
511  if self.config.coaddName == 'dcr':
512  templateExposures = inputRefs.dcrCoadds
513  else:
514  templateExposures = inputRefs.coaddExposures
515  templateStruct = self.getTemplate.runQuantum(
516  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
517  )
518 
519  outputs = self.run(exposure=inputs['exposure'],
520  templateExposure=templateStruct.exposure,
521  idFactory=idFactory)
522  # Consistency with runDataref gen2 handling
523  if outputs.diaSources is None:
524  del outputs.diaSources
525  butlerQC.put(outputs, outputRefs)
526 
527  @pipeBase.timeMethod
528  def runDataRef(self, sensorRef, templateIdList=None):
529  """Subtract an image from a template coadd and measure the result.
530 
531  Data I/O wrapper around `run` using the butler in Gen2.
532 
533  Parameters
534  ----------
535  sensorRef : `lsst.daf.persistence.ButlerDataRef`
536  Sensor-level butler data reference, used for the following data products:
537 
538  Input only:
539  - calexp
540  - psf
541  - ccdExposureId
542  - ccdExposureId_bits
543  - self.config.coaddName + "Coadd_skyMap"
544  - self.config.coaddName + "Coadd"
545  Input or output, depending on config:
546  - self.config.coaddName + "Diff_subtractedExp"
547  Output, depending on config:
548  - self.config.coaddName + "Diff_matchedExp"
549  - self.config.coaddName + "Diff_src"
550 
551  Returns
552  -------
553  results : `lsst.pipe.base.Struct`
554  Returns the Struct by `run`.
555  """
556  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
557  subtractedExposure = None
558  selectSources = None
559  calexpBackgroundExposure = None
560  self.log.info("Processing %s" % (sensorRef.dataId))
561 
562  # We make one IdFactory that will be used by both icSrc and src datasets;
563  # I don't know if this is the way we ultimately want to do things, but at least
564  # this ensures the source IDs are fully unique.
565  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
566  expBits=sensorRef.get("ccdExposureId_bits"))
567  if self.config.doAddCalexpBackground:
568  calexpBackgroundExposure = sensorRef.get("calexpBackground")
569 
570  # Retrieve the science image we wish to analyze
571  exposure = sensorRef.get("calexp", immediate=True)
572 
573  # Retrieve the template image
574  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
575 
576  if sensorRef.datasetExists("src"):
577  self.log.info("Source selection via src product")
578  # Sources already exist; for data release processing
579  selectSources = sensorRef.get("src")
580 
581  if not self.config.doSubtract and self.config.doDetection:
582  # If we don't do subtraction, we need the subtracted exposure from the repo
583  subtractedExposure = sensorRef.get(subtractedExposureName)
584  # Both doSubtract and doDetection cannot be False
585 
586  results = self.run(exposure=exposure,
587  selectSources=selectSources,
588  templateExposure=template.exposure,
589  templateSources=template.sources,
590  idFactory=idFactory,
591  calexpBackgroundExposure=calexpBackgroundExposure,
592  subtractedExposure=subtractedExposure)
593 
594  if self.config.doWriteSources and results.diaSources is not None:
595  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
596  if self.config.doWriteWarpedExp:
597  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
598  if self.config.doWriteMatchedExp:
599  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
600  if self.config.doAddMetrics and self.config.doSelectSources:
601  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
602  if self.config.doWriteSubtractedExp:
603  sensorRef.put(results.subtractedExposure, subtractedExposureName)
604  if self.config.doWriteScoreExp:
605  sensorRef.put(results.scoreExposure, self.config.coaddName + "Diff_scoreExp")
606  return results
607 
608  @pipeBase.timeMethod
609  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
610  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
611  """PSF matches, subtract two images and perform detection on the difference image.
612 
613  Parameters
614  ----------
615  exposure : `lsst.afw.image.ExposureF`, optional
616  The science exposure, the minuend in the image subtraction.
617  Can be None only if ``config.doSubtract==False``.
618  selectSources : `lsst.afw.table.SourceCatalog`, optional
619  Identified sources on the science exposure. This catalog is used to
620  select sources in order to perform the AL PSF matching on stamp images
621  around them. The selection steps depend on config options and whether
622  ``templateSources`` and ``matchingSources`` specified.
623  templateExposure : `lsst.afw.image.ExposureF`, optional
624  The template to be subtracted from ``exposure`` in the image subtraction.
625  ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
626  The template exposure should cover the same sky area as the science exposure.
627  It is either a stich of patches of a coadd skymap image or a calexp
628  of the same pointing as the science exposure. Can be None only
629  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
630  templateSources : `lsst.afw.table.SourceCatalog`, optional
631  Identified sources on the template exposure.
632  idFactory : `lsst.afw.table.IdFactory`
633  Generator object to assign ids to detected sources in the difference image.
634  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
635  Background exposure to be added back to the science exposure
636  if ``config.doAddCalexpBackground==True``
637  subtractedExposure : `lsst.afw.image.ExposureF`, optional
638  If ``config.doSubtract==False`` and ``config.doDetection==True``,
639  performs the post subtraction source detection only on this exposure.
640  Otherwise should be None.
641 
642  Returns
643  -------
644  results : `lsst.pipe.base.Struct`
645  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
646  Difference image.
647  ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
648  The zogy score exposure, if calculated.
649  ``matchedExposure`` : `lsst.afw.image.ExposureF`
650  The matched PSF exposure.
651  ``subtractRes`` : `lsst.pipe.base.Struct`
652  The returned result structure of the ImagePsfMatchTask subtask.
653  ``diaSources`` : `lsst.afw.table.SourceCatalog`
654  The catalog of detected sources.
655  ``selectSources`` : `lsst.afw.table.SourceCatalog`
656  The input source catalog with optionally added Qa information.
657 
658  Notes
659  -----
660  The following major steps are included:
661 
662  - warp template coadd to match WCS of image
663  - PSF match image to warped template
664  - subtract image from PSF-matched, warped template
665  - detect sources
666  - measure sources
667 
668  For details about the image subtraction configuration modes
669  see `lsst.ip.diffim`.
670  """
671  subtractRes = None
672  controlSources = None
673  scoreExposure = None
674  diaSources = None
675  kernelSources = None
676  # We'll clone exposure if modified but will still need the original
677  exposureOrig = exposure
678 
679  if self.config.doAddCalexpBackground:
680  mi = exposure.getMaskedImage()
681  mi += calexpBackgroundExposure.getImage()
682 
683  if not exposure.hasPsf():
684  raise pipeBase.TaskError("Exposure has no psf")
685  sciencePsf = exposure.getPsf()
686 
687  if self.config.doSubtract:
688  if self.config.doScaleTemplateVariance:
689  self.log.info("Rescaling template variance")
690  templateVarFactor = self.scaleVariance.run(
691  templateExposure.getMaskedImage())
692  self.log.info("Template variance scaling factor: %.2f" % templateVarFactor)
693  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
694 
695  if self.config.subtract.name == 'zogy':
696  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
697  scoreExposure = subtractRes.scoreExp
698  subtractedExposure = subtractRes.diffExp
699  subtractRes.subtractedExposure = subtractedExposure
700  subtractRes.matchedExposure = None
701 
702  elif self.config.subtract.name == 'al':
703  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
704  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
705  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
706 
707  # if requested, convolve the science exposure with its PSF
708  # (properly, this should be a cross-correlation, but our code does not yet support that)
709  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
710  # else sigma of original science exposure
711  # TODO: DM-22762 This functional block should be moved into its own method
712  preConvPsf = None
713  if self.config.doPreConvolve:
714  convControl = afwMath.ConvolutionControl()
715  # cannot convolve in place, so need a new image anyway
716  srcMI = exposure.maskedImage
717  exposure = exposure.clone() # New deep copy
718  srcPsf = sciencePsf
719  if self.config.useGaussianForPreConvolution:
720  # convolve with a simplified PSF model: a double Gaussian
721  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
722  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
723  else:
724  # convolve with science exposure's PSF model
725  preConvPsf = srcPsf
726  afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
727  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
728  else:
729  scienceSigmaPost = scienceSigmaOrig
730 
731  # If requested, find and select sources from the image
732  # else, AL subtraction will do its own source detection
733  # TODO: DM-22762 This functional block should be moved into its own method
734  if self.config.doSelectSources:
735  if selectSources is None:
736  self.log.warn("Src product does not exist; running detection, measurement, selection")
737  # Run own detection and measurement; necessary in nightly processing
738  selectSources = self.subtract.getSelectSources(
739  exposure,
740  sigma=scienceSigmaPost,
741  doSmooth=not self.config.doPreConvolve,
742  idFactory=idFactory,
743  )
744 
745  if self.config.doAddMetrics:
746  # Number of basis functions
747 
748  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
749  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
750  targetFwhmPix=templateSigma*FwhmPerSigma))
751  # Modify the schema of all Sources
752  # DEPRECATED: This is a data dependent (nparam) output product schema
753  # outside the task constructor.
754  # NOTE: The pre-determination of nparam at this point
755  # may be incorrect as the template psf is warped later in
756  # ImagePsfMatchTask.matchExposures()
757  kcQa = KernelCandidateQa(nparam)
758  selectSources = kcQa.addToSchema(selectSources)
759  if self.config.kernelSourcesFromRef:
760  # match exposure sources to reference catalog
761  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
762  matches = astromRet.matches
763  elif templateSources:
764  # match exposure sources to template sources
765  mc = afwTable.MatchControl()
766  mc.findOnlyClosest = False
767  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
768  mc)
769  else:
770  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
771  "but template sources not available. Cannot match science "
772  "sources with template sources. Run process* on data from "
773  "which templates are built.")
774 
775  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
776  matches=matches).sourceCat
777  random.shuffle(kernelSources, random.random)
778  controlSources = kernelSources[::self.config.controlStepSize]
779  kernelSources = [k for i, k in enumerate(kernelSources)
780  if i % self.config.controlStepSize]
781 
782  if self.config.doSelectDcrCatalog:
783  redSelector = DiaCatalogSourceSelectorTask(
784  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
785  grMax=99.999))
786  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
787  controlSources.extend(redSources)
788 
789  blueSelector = DiaCatalogSourceSelectorTask(
790  DiaCatalogSourceSelectorConfig(grMin=-99.999,
791  grMax=self.sourceSelector.config.grMin))
792  blueSources = blueSelector.selectStars(exposure, selectSources,
793  matches=matches).starCat
794  controlSources.extend(blueSources)
795 
796  if self.config.doSelectVariableCatalog:
797  varSelector = DiaCatalogSourceSelectorTask(
798  DiaCatalogSourceSelectorConfig(includeVariable=True))
799  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
800  controlSources.extend(varSources)
801 
802  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)"
803  % (len(kernelSources), len(selectSources), len(controlSources)))
804 
805  allresids = {}
806  # TODO: DM-22762 This functional block should be moved into its own method
807  if self.config.doUseRegister:
808  self.log.info("Registering images")
809 
810  if templateSources is None:
811  # Run detection on the template, which is
812  # temporarily background-subtracted
813  # sigma of PSF of template image before warping
814  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
815  templateSources = self.subtract.getSelectSources(
816  templateExposure,
817  sigma=templateSigma,
818  doSmooth=True,
819  idFactory=idFactory
820  )
821 
822  # Third step: we need to fit the relative astrometry.
823  #
824  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
825  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
826  exposure.getWcs(), exposure.getBBox())
827  templateExposure = warpedExp
828 
829  # Create debugging outputs on the astrometric
830  # residuals as a function of position. Persistence
831  # not yet implemented; expected on (I believe) #2636.
832  if self.config.doDebugRegister:
833  # Grab matches to reference catalog
834  srcToMatch = {x.second.getId(): x.first for x in matches}
835 
836  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
837  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
838  sids = [m.first.getId() for m in wcsResults.matches]
839  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
840  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
841  m.second.get(inCentroidKey))) for m in wcsResults.matches]
842  allresids = dict(zip(sids, zip(positions, residuals)))
843 
844  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
845  wcsResults.wcs.pixelToSky(
846  m.second.get(inCentroidKey))) for m in wcsResults.matches]
847  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
848  + 2.5*numpy.log10(srcToMatch[x].get("r"))
849  for x in sids if x in srcToMatch.keys()])
850  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
851  if s in srcToMatch.keys()])
852  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
853  if s in srcToMatch.keys()])
854  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
855  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
856  & (colors <= self.sourceSelector.config.grMax))
857  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
858  rms1Long = IqrToSigma*(
859  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
860  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
861  - numpy.percentile(dlat[idx1], 25))
862  rms2Long = IqrToSigma*(
863  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
864  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
865  - numpy.percentile(dlat[idx2], 25))
866  rms3Long = IqrToSigma*(
867  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
868  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
869  - numpy.percentile(dlat[idx3], 25))
870  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
871  (numpy.median(dlong[idx1]), rms1Long,
872  numpy.median(dlat[idx1]), rms1Lat))
873  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f" %
874  (numpy.median(dlong[idx2]), rms2Long,
875  numpy.median(dlat[idx2]), rms2Lat))
876  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
877  (numpy.median(dlong[idx3]), rms3Long,
878  numpy.median(dlat[idx3]), rms3Lat))
879 
880  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
881  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
882  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
883  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
884  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
885  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
886 
887  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
888  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
889  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
890  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
891  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
892  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
893 
894  # warp template exposure to match exposure,
895  # PSF match template exposure to exposure,
896  # then return the difference
897 
898  # Return warped template... Construct sourceKernelCand list after subtract
899  self.log.info("Subtracting images")
900  subtractRes = self.subtract.subtractExposures(
901  templateExposure=templateExposure,
902  scienceExposure=exposure,
903  candidateList=kernelSources,
904  convolveTemplate=self.config.convolveTemplate,
905  doWarping=not self.config.doUseRegister
906  )
907  subtractedExposure = subtractRes.subtractedExposure
908 
909  if self.config.doDetection:
910  self.log.info("Computing diffim PSF")
911 
912  # Get Psf from the appropriate input image if it doesn't exist
913  if not subtractedExposure.hasPsf():
914  if self.config.convolveTemplate:
915  subtractedExposure.setPsf(exposure.getPsf())
916  else:
917  subtractedExposure.setPsf(templateExposure.getPsf())
918 
919  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
920  # thus it may have already been decorrelated. Thus, we do not decorrelate if
921  # doSubtract is False.
922 
923  # NOTE: At this point doSubtract == True
924  if self.config.doDecorrelation and self.config.doSubtract:
925  preConvKernel = None
926  if preConvPsf is not None:
927  preConvKernel = preConvPsf.getLocalKernel()
928  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
929  subtractedExposure,
930  subtractRes.psfMatchingKernel,
931  spatiallyVarying=self.config.doSpatiallyVarying,
932  preConvKernel=preConvKernel,
933  templateMatched=self.config.convolveTemplate)
934  subtractedExposure = decorrResult.correctedExposure
935 
936  # END (if subtractAlgorithm == 'AL')
937  # END (if self.config.doSubtract)
938  if self.config.doDetection:
939  self.log.info("Running diaSource detection")
940 
941  # subtractedExposure - reserved for task return value
942  # in zogy, it is always the proper difference image
943  # in AL, it may be (yet) pre-convolved and/or decorrelated
944  #
945  # detectionExposure - controls which exposure to use for detection
946  # in-place modifications will appear in task return
947  if self.config.useScoreImageDetection:
948  # zogy with score image detection enabled
949  self.log.info("Detection, diffim rescaling and measurements are on Zogy score image.")
950  detectionExposure = scoreExposure
951  detectOnLikelihood = True
952  else:
953  # AL or zogy with no score image detection
954  detectionExposure = subtractedExposure
955  detectOnLikelihood = False
956  if self.config.doPreConvolve:
957  # In AL, the likelihood image is not a separate product, yet
958  self.log.info("Detection, diffim rescaling and measurements are on AL pre-convolved "
959  "difference (likelihood) image.")
960  detectOnLikelihood = True
961  else:
962  self.log.info("Detection, diffim rescaling and measurements are on "
963  "(proper) difference image.")
964 
965  # Rescale difference image variance plane
966  if self.config.doScaleDiffimVariance:
967  self.log.info("Rescaling diffim variance")
968  diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
969  self.log.info("Diffim variance scaling factor: %.2f" % diffimVarFactor)
970  self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
971 
972  # Erase existing detection mask planes
973  mask = detectionExposure.getMaskedImage().getMask()
974  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
975 
976  table = afwTable.SourceTable.make(self.schema, idFactory)
977  table.setMetadata(self.algMetadata)
978  results = self.detection.run(
979  table=table,
980  exposure=detectionExposure,
981  doSmooth=not detectOnLikelihood
982  )
983 
984  if self.config.doMerge:
985  fpSet = results.fpSets.positive
986  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
987  self.config.growFootprint, False)
988  diaSources = afwTable.SourceCatalog(table)
989  fpSet.makeSources(diaSources)
990  self.log.info("Merging detections into %d sources" % (len(diaSources)))
991  else:
992  diaSources = results.sources
993 
994  if self.config.doMeasurement:
995  newDipoleFitting = self.config.doDipoleFitting
996  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
997  if not newDipoleFitting:
998  # Just fit dipole in diffim
999  self.measurement.run(diaSources, detectionExposure)
1000  else:
1001  # Use (matched) template and science image (if avail.) to constrain dipole fitting
1002  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1003  self.measurement.run(diaSources, detectionExposure, exposure,
1004  subtractRes.matchedExposure)
1005  else:
1006  self.measurement.run(diaSources, detectionExposure, exposure)
1007  if self.config.doApCorr:
1008  self.applyApCorr.run(
1009  catalog=diaSources,
1010  apCorrMap=detectionExposure.getInfo().getApCorrMap()
1011  )
1012 
1013  if self.config.doForcedMeasurement:
1014  # Run forced psf photometry on the PVI at the diaSource locations.
1015  # Copy the measured flux and error into the diaSource.
1016  forcedSources = self.forcedMeasurement.generateMeasCat(
1017  exposure, diaSources, detectionExposure.getWcs())
1018  self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1019  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1020  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1021  "ip_diffim_forced_PsfFlux_instFlux", True)
1022  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1023  "ip_diffim_forced_PsfFlux_instFluxErr", True)
1024  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1025  "ip_diffim_forced_PsfFlux_area", True)
1026  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1027  "ip_diffim_forced_PsfFlux_flag", True)
1028  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1029  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1030  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1031  "ip_diffim_forced_PsfFlux_flag_edge", True)
1032  for diaSource, forcedSource in zip(diaSources, forcedSources):
1033  diaSource.assign(forcedSource, mapper)
1034 
1035  # Match with the calexp sources if possible
1036  if self.config.doMatchSources:
1037  if selectSources is not None:
1038  # Create key,val pair where key=diaSourceId and val=sourceId
1039  matchRadAsec = self.config.diaSourceMatchRadius
1040  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1041 
1042  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1043  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1044  srcMatch in srcMatches])
1045  self.log.info("Matched %d / %d diaSources to sources" % (len(srcMatchDict),
1046  len(diaSources)))
1047  else:
1048  self.log.warn("Src product does not exist; cannot match with diaSources")
1049  srcMatchDict = {}
1050 
1051  # Create key,val pair where key=diaSourceId and val=refId
1052  refAstromConfig = AstrometryConfig()
1053  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1054  refAstrometer = AstrometryTask(refAstromConfig)
1055  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1056  refMatches = astromRet.matches
1057  if refMatches is None:
1058  self.log.warn("No diaSource matches with reference catalog")
1059  refMatchDict = {}
1060  else:
1061  self.log.info("Matched %d / %d diaSources to reference catalog" % (len(refMatches),
1062  len(diaSources)))
1063  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1064  refMatch in refMatches])
1065 
1066  # Assign source Ids
1067  for diaSource in diaSources:
1068  sid = diaSource.getId()
1069  if sid in srcMatchDict:
1070  diaSource.set("srcMatchId", srcMatchDict[sid])
1071  if sid in refMatchDict:
1072  diaSource.set("refMatchId", refMatchDict[sid])
1073 
1074  if self.config.doAddMetrics and self.config.doSelectSources:
1075  self.log.info("Evaluating metrics and control sample")
1076 
1077  kernelCandList = []
1078  for cell in subtractRes.kernelCellSet.getCellList():
1079  for cand in cell.begin(False): # include bad candidates
1080  kernelCandList.append(cand)
1081 
1082  # Get basis list to build control sample kernels
1083  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1084  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1085 
1086  controlCandList = (
1087  diffimTools.sourceTableToCandidateList(controlSources,
1088  subtractRes.warpedExposure, exposure,
1089  self.config.subtract.kernel.active,
1090  self.config.subtract.kernel.active.detectionConfig,
1091  self.log, doBuild=True, basisList=basisList))
1092 
1093  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1094  subtractRes.backgroundModel, dof=nparam)
1095  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1096  subtractRes.backgroundModel)
1097 
1098  if self.config.doDetection:
1099  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1100  else:
1101  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1102 
1103  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1104  return pipeBase.Struct(
1105  subtractedExposure=subtractedExposure,
1106  scoreExposure=scoreExposure,
1107  warpedExposure=subtractRes.warpedExposure,
1108  matchedExposure=subtractRes.matchedExposure,
1109  subtractRes=subtractRes,
1110  diaSources=diaSources,
1111  selectSources=selectSources
1112  )
1113 
1114  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1115  """Fit the relative astrometry between templateSources and selectSources
1116 
1117  Todo
1118  ----
1119 
1120  Remove this method. It originally fit a new WCS to the template before calling register.run
1121  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1122  It remains because a subtask overrides it.
1123  """
1124  results = self.register.run(templateSources, templateExposure.getWcs(),
1125  templateExposure.getBBox(), selectSources)
1126  return results
1127 
1128  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1129  """Make debug plots and displays.
1130 
1131  Todo
1132  ----
1133  Test and update for current debug display and slot names
1134  """
1135  import lsstDebug
1136  display = lsstDebug.Info(__name__).display
1137  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1138  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1139  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1140  showDipoles = lsstDebug.Info(__name__).showDipoles
1141  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1142  if display:
1143  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1144  if not maskTransparency:
1145  maskTransparency = 0
1146  disp.setMaskTransparency(maskTransparency)
1147 
1148  if display and showSubtracted:
1149  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1150  mi = subtractRes.subtractedExposure.getMaskedImage()
1151  x0, y0 = mi.getX0(), mi.getY0()
1152  with disp.Buffering():
1153  for s in diaSources:
1154  x, y = s.getX() - x0, s.getY() - y0
1155  ctype = "red" if s.get("flags_negative") else "yellow"
1156  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1157  or s.get("base_PixelFlags_flag_saturatedCenter")
1158  or s.get("base_PixelFlags_flag_crCenter")):
1159  ptype = "x"
1160  elif (s.get("base_PixelFlags_flag_interpolated")
1161  or s.get("base_PixelFlags_flag_saturated")
1162  or s.get("base_PixelFlags_flag_cr")):
1163  ptype = "+"
1164  else:
1165  ptype = "o"
1166  disp.dot(ptype, x, y, size=4, ctype=ctype)
1167  lsstDebug.frame += 1
1168 
1169  if display and showPixelResiduals and selectSources:
1170  nonKernelSources = []
1171  for source in selectSources:
1172  if source not in kernelSources:
1173  nonKernelSources.append(source)
1174 
1175  diUtils.plotPixelResiduals(exposure,
1176  subtractRes.warpedExposure,
1177  subtractRes.subtractedExposure,
1178  subtractRes.kernelCellSet,
1179  subtractRes.psfMatchingKernel,
1180  subtractRes.backgroundModel,
1181  nonKernelSources,
1182  self.subtract.config.kernel.active.detectionConfig,
1183  origVariance=False)
1184  diUtils.plotPixelResiduals(exposure,
1185  subtractRes.warpedExposure,
1186  subtractRes.subtractedExposure,
1187  subtractRes.kernelCellSet,
1188  subtractRes.psfMatchingKernel,
1189  subtractRes.backgroundModel,
1190  nonKernelSources,
1191  self.subtract.config.kernel.active.detectionConfig,
1192  origVariance=True)
1193  if display and showDiaSources:
1194  flagChecker = SourceFlagChecker(diaSources)
1195  isFlagged = [flagChecker(x) for x in diaSources]
1196  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1197  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1198  frame=lsstDebug.frame)
1199  lsstDebug.frame += 1
1200 
1201  if display and showDipoles:
1202  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1203  frame=lsstDebug.frame)
1204  lsstDebug.frame += 1
1205 
1206  def _getConfigName(self):
1207  """Return the name of the config dataset
1208  """
1209  return "%sDiff_config" % (self.config.coaddName,)
1210 
1211  def _getMetadataName(self):
1212  """Return the name of the metadata dataset
1213  """
1214  return "%sDiff_metadata" % (self.config.coaddName,)
1215 
1216  def getSchemaCatalogs(self):
1217  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1218  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1219 
1220  @classmethod
1221  def _makeArgumentParser(cls):
1222  """Create an argument parser
1223  """
1224  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1225  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1226  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1227  help="Template data ID in case of calexp template,"
1228  " e.g. --templateId visit=6789")
1229  return parser
1230 
1231 
1232 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1233  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1234  doc="Shift stars going into RegisterTask by this amount")
1235  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1236  doc="Perturb stars going into RegisterTask by this amount")
1237 
1238  def setDefaults(self):
1239  ImageDifferenceConfig.setDefaults(self)
1240  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1241 
1242 
1243 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1244  """!Image difference Task used in the Winter 2013 data challege.
1245  Enables testing the effects of registration shifts and scatter.
1246 
1247  For use with winter 2013 simulated images:
1248  Use --templateId visit=88868666 for sparse data
1249  --templateId visit=22222200 for dense data (g)
1250  --templateId visit=11111100 for dense data (i)
1251  """
1252  ConfigClass = Winter2013ImageDifferenceConfig
1253  _DefaultName = "winter2013ImageDifference"
1254 
1255  def __init__(self, **kwargs):
1256  ImageDifferenceTask.__init__(self, **kwargs)
1257 
1258  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1259  """Fit the relative astrometry between templateSources and selectSources"""
1260  if self.config.winter2013WcsShift > 0.0:
1261  offset = geom.Extent2D(self.config.winter2013WcsShift,
1262  self.config.winter2013WcsShift)
1263  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1264  for source in templateSources:
1265  centroid = source.get(cKey)
1266  source.set(cKey, centroid + offset)
1267  elif self.config.winter2013WcsRms > 0.0:
1268  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1269  for source in templateSources:
1270  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1271  self.config.winter2013WcsRms*numpy.random.normal())
1272  centroid = source.get(cKey)
1273  source.set(cKey, centroid + offset)
1274 
1275  results = self.register.run(templateSources, templateExposure.getWcs(),
1276  templateExposure.getBBox(), selectSources)
1277  return results
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)