lsst.pipe.tasks  21.0.0-142-gef555c1e+42c9bccae2
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 from lsst.skymap import BaseSkyMap
47 from lsst.obs.base import ExposureIdInfo
48 
49 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
50 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
51 IqrToSigma = 0.741
52 
53 
54 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
55  dimensions=("instrument", "visit", "detector", "skymap"),
56  defaultTemplates={"coaddName": "deep",
57  "skyMapName": "deep",
58  "warpTypeSuffix": "",
59  "fakesType": ""}):
60 
61  exposure = pipeBase.connectionTypes.Input(
62  doc="Input science exposure to subtract from.",
63  dimensions=("instrument", "visit", "detector"),
64  storageClass="ExposureF",
65  name="{fakesType}calexp"
66  )
67 
68  # TODO DM-22953
69  # kernelSources = pipeBase.connectionTypes.Input(
70  # doc="Source catalog produced in calibrate task for kernel candidate sources",
71  # name="src",
72  # storageClass="SourceCatalog",
73  # dimensions=("instrument", "visit", "detector"),
74  # )
75 
76  skyMap = pipeBase.connectionTypes.Input(
77  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
78  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
79  dimensions=("skymap", ),
80  storageClass="SkyMap",
81  )
82  coaddExposures = pipeBase.connectionTypes.Input(
83  doc="Input template to match and subtract from the exposure",
84  dimensions=("tract", "patch", "skymap", "band"),
85  storageClass="ExposureF",
86  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
87  multiple=True,
88  deferLoad=True
89  )
90  dcrCoadds = pipeBase.connectionTypes.Input(
91  doc="Input DCR template to match and subtract from the exposure",
92  name="{fakesType}dcrCoadd{warpTypeSuffix}",
93  storageClass="ExposureF",
94  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
95  multiple=True,
96  deferLoad=True
97  )
98  outputSchema = pipeBase.connectionTypes.InitOutput(
99  doc="Schema (as an example catalog) for output DIASource catalog.",
100  storageClass="SourceCatalog",
101  name="{fakesType}{coaddName}Diff_diaSrc_schema",
102  )
103  subtractedExposure = pipeBase.connectionTypes.Output(
104  doc="Output AL difference or Zogy proper difference image",
105  dimensions=("instrument", "visit", "detector"),
106  storageClass="ExposureF",
107  name="{fakesType}{coaddName}Diff_differenceExp",
108  )
109  scoreExposure = pipeBase.connectionTypes.Output(
110  doc="Output AL likelihood or Zogy score image",
111  dimensions=("instrument", "visit", "detector"),
112  storageClass="ExposureF",
113  name="{fakesType}{coaddName}Diff_scoreExp",
114  )
115  warpedExposure = pipeBase.connectionTypes.Output(
116  doc="Warped template used to create `subtractedExposure`.",
117  dimensions=("instrument", "visit", "detector"),
118  storageClass="ExposureF",
119  name="{fakesType}{coaddName}Diff_warpedExp",
120  )
121  matchedExposure = pipeBase.connectionTypes.Output(
122  doc="Warped template used to create `subtractedExposure`.",
123  dimensions=("instrument", "visit", "detector"),
124  storageClass="ExposureF",
125  name="{fakesType}{coaddName}Diff_matchedExp",
126  )
127  diaSources = pipeBase.connectionTypes.Output(
128  doc="Output detected diaSources on the difference image",
129  dimensions=("instrument", "visit", "detector"),
130  storageClass="SourceCatalog",
131  name="{fakesType}{coaddName}Diff_diaSrc",
132  )
133 
134  def __init__(self, *, config=None):
135  super().__init__(config=config)
136  if config.coaddName == 'dcr':
137  self.inputs.remove("coaddExposures")
138  else:
139  self.inputs.remove("dcrCoadds")
140  if not config.doWriteSubtractedExp:
141  self.outputs.remove("subtractedExposure")
142  if not config.doWriteScoreExp:
143  self.outputs.remove("scoreExposure")
144  if not config.doWriteWarpedExp:
145  self.outputs.remove("warpedExposure")
146  if not config.doWriteMatchedExp:
147  self.outputs.remove("matchedExposure")
148  if not config.doWriteSources:
149  self.outputs.remove("diaSources")
150 
151  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
152  # Make kernelSources optional
153 
154 
155 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
156  pipelineConnections=ImageDifferenceTaskConnections):
157  """Config for ImageDifferenceTask
158  """
159  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
160  doc="Add background to calexp before processing it. "
161  "Useful as ipDiffim does background matching.")
162  doUseRegister = pexConfig.Field(dtype=bool, default=False,
163  doc="Re-compute astrometry on the template. "
164  "Use image-to-image registration to align template with "
165  "science image (AL only).")
166  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
167  doc="Writing debugging data for doUseRegister")
168  doSelectSources = pexConfig.Field(dtype=bool, default=False,
169  doc="Select stars to use for kernel fitting (AL only)")
170  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
171  doc="Select stars of extreme color as part "
172  "of the control sample (AL only)")
173  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
174  doc="Select stars that are variable to be part "
175  "of the control sample (AL only)")
176  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
177  doPreConvolve = pexConfig.Field(dtype=bool, default=False,
178  doc="Not in use. Superseded by useScoreImageDetection.",
179  deprecated="This option superseded by useScoreImageDetection."
180  " Will be removed after v22.")
181  useScoreImageDetection = pexConfig.Field(
182  dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or "
183  "the Zogy score image. Use it for source detection (if doDetection=True).")
184  doWriteScoreExp = pexConfig.Field(
185  dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?")
186  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
187  doc="Scale variance of the template before PSF matching")
188  doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
189  doc="Scale variance of the diffim before PSF matching. "
190  "You may do either this or template variance scaling, "
191  "or neither. (Doing both is a waste of CPU.)")
192  useGaussianForPreConvolution = pexConfig.Field(
193  dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution "
194  "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)")
195  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
196  doDecorrelation = pexConfig.Field(dtype=bool, default=True,
197  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
198  "kernel convolution (AL only)? If True, also update the diffim PSF.")
199  doMerge = pexConfig.Field(dtype=bool, default=True,
200  doc="Merge positive and negative diaSources with grow radius "
201  "set by growFootprint")
202  doMatchSources = pexConfig.Field(dtype=bool, default=False,
203  doc="Match diaSources with input calexp sources and ref catalog sources")
204  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
205  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
206  doForcedMeasurement = pexConfig.Field(
207  dtype=bool,
208  default=True,
209  doc="Force photometer diaSource locations on PVI?")
210  doWriteSubtractedExp = pexConfig.Field(
211  dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
212  doWriteWarpedExp = pexConfig.Field(
213  dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
214  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
215  doc="Write warped and PSF-matched template coadd exposure?")
216  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
217  doAddMetrics = pexConfig.Field(dtype=bool, default=False,
218  doc="Add columns to the source table to hold analysis metrics?")
219 
220  coaddName = pexConfig.Field(
221  doc="coadd name: typically one of deep, goodSeeing, or dcr",
222  dtype=str,
223  default="deep",
224  )
225  convolveTemplate = pexConfig.Field(
226  doc="Which image gets convolved (default = template)",
227  dtype=bool,
228  default=True
229  )
230  refObjLoader = pexConfig.ConfigurableField(
231  target=LoadIndexedReferenceObjectsTask,
232  doc="reference object loader",
233  )
234  astrometer = pexConfig.ConfigurableField(
235  target=AstrometryTask,
236  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
237  )
238  sourceSelector = pexConfig.ConfigurableField(
239  target=ObjectSizeStarSelectorTask,
240  doc="Source selection algorithm",
241  )
242  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
243  decorrelate = pexConfig.ConfigurableField(
244  target=DecorrelateALKernelSpatialTask,
245  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
246  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
247  "default of 5.5).",
248  )
249  # Old style ImageMapper grid. ZogyTask has its own grid option
250  doSpatiallyVarying = pexConfig.Field(
251  dtype=bool,
252  default=False,
253  doc="Perform A&L decorrelation on a grid across the "
254  "image in order to allow for spatial variations. Zogy does not use this option."
255  )
256  detection = pexConfig.ConfigurableField(
257  target=SourceDetectionTask,
258  doc="Low-threshold detection for final measurement",
259  )
260  measurement = pexConfig.ConfigurableField(
261  target=DipoleFitTask,
262  doc="Enable updated dipole fitting method",
263  )
264  doApCorr = lsst.pex.config.Field(
265  dtype=bool,
266  default=True,
267  doc="Run subtask to apply aperture corrections"
268  )
269  applyApCorr = lsst.pex.config.ConfigurableField(
270  target=ApplyApCorrTask,
271  doc="Subtask to apply aperture corrections"
272  )
273  forcedMeasurement = pexConfig.ConfigurableField(
274  target=ForcedMeasurementTask,
275  doc="Subtask to force photometer PVI at diaSource location.",
276  )
277  getTemplate = pexConfig.ConfigurableField(
278  target=GetCoaddAsTemplateTask,
279  doc="Subtask to retrieve template exposure and sources",
280  )
281  scaleVariance = pexConfig.ConfigurableField(
282  target=ScaleVarianceTask,
283  doc="Subtask to rescale the variance of the template "
284  "to the statistically expected level"
285  )
286  controlStepSize = pexConfig.Field(
287  doc="What step size (every Nth one) to select a control sample from the kernelSources",
288  dtype=int,
289  default=5
290  )
291  controlRandomSeed = pexConfig.Field(
292  doc="Random seed for shuffing the control sample",
293  dtype=int,
294  default=10
295  )
296  register = pexConfig.ConfigurableField(
297  target=RegisterTask,
298  doc="Task to enable image-to-image image registration (warping)",
299  )
300  kernelSourcesFromRef = pexConfig.Field(
301  doc="Select sources to measure kernel from reference catalog if True, template if false",
302  dtype=bool,
303  default=False
304  )
305  templateSipOrder = pexConfig.Field(
306  dtype=int, default=2,
307  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
308  )
309  growFootprint = pexConfig.Field(
310  dtype=int, default=2,
311  doc="Grow positive and negative footprints by this amount before merging"
312  )
313  diaSourceMatchRadius = pexConfig.Field(
314  dtype=float, default=0.5,
315  doc="Match radius (in arcseconds) for DiaSource to Source association"
316  )
317  requiredTemplateFraction = pexConfig.Field(
318  dtype=float, default=0.1,
319  doc="Do not attempt to run task if template covers less than this fraction of pixels."
320  "Setting to 0 will always attempt image subtraction"
321  )
322  doSkySources = pexConfig.Field(
323  dtype=bool,
324  default=False,
325  doc="Generate sky sources?",
326  )
327  skySources = pexConfig.ConfigurableField(
328  target=SkyObjectsTask,
329  doc="Generate sky sources",
330  )
331 
332  def setDefaults(self):
333  # defaults are OK for catalog and diacatalog
334 
335  self.subtract['al'].kernel.name = "AL"
336  self.subtract['al'].kernel.active.fitForBackground = True
337  self.subtract['al'].kernel.active.spatialKernelOrder = 1
338  self.subtract['al'].kernel.active.spatialBgOrder = 2
339 
340  # DiaSource Detection
341  self.detection.thresholdPolarity = "both"
342  self.detection.thresholdValue = 5.0
343  self.detection.reEstimateBackground = False
344  self.detection.thresholdType = "pixel_stdev"
345 
346  # Add filtered flux measurement, the correct measurement for pre-convolved images.
347  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
348  # To change that you must modify algorithms.names in the task's applyOverrides method,
349  # after the user has set doPreConvolve.
350  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
351  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
352  'base_LocalWcs']
353 
354  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
355  self.forcedMeasurement.copyColumns = {
356  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
357  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
358  self.forcedMeasurement.slots.shape = None
359 
360  # For shuffling the control sample
361  random.seed(self.controlRandomSeed)
362 
363  def validate(self):
364  pexConfig.Config.validate(self)
365  if not self.doSubtract and not self.doDetection:
366  raise ValueError("Either doSubtract or doDetection must be enabled.")
367  if self.doMeasurement and not self.doDetection:
368  raise ValueError("Cannot run source measurement without source detection.")
369  if self.doMerge and not self.doDetection:
370  raise ValueError("Cannot run source merging without source detection.")
371  if self.doSkySources and not self.doDetection:
372  raise ValueError("Cannot run sky source creation without source detection.")
373  if self.doUseRegister and not self.doSelectSources:
374  raise ValueError("doUseRegister=True and doSelectSources=False. "
375  "Cannot run RegisterTask without selecting sources.")
376  if hasattr(self.getTemplate, "coaddName"):
377  if self.getTemplate.coaddName != self.coaddName:
378  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
379  if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
380  raise ValueError("Scaling the diffim variance and scaling the template variance "
381  "are both set. Please choose one or the other.")
382  # We cannot allow inconsistencies that would lead to None or not available output products
383  if self.subtract.name == 'zogy':
384  if self.doWriteMatchedExp:
385  raise ValueError("doWriteMatchedExp=True Matched exposure is not "
386  "calculated in zogy subtraction.")
387  if self.doAddMetrics:
388  raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
389  if self.doDecorrelation:
390  raise ValueError(
391  "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
392  if self.doSelectSources:
393  raise ValueError(
394  "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
395  if self.useGaussianForPreConvolution:
396  raise ValueError(
397  "useGaussianForPreConvolution=True This is an AL subtraction only option.")
398  else:
399  # AL only consistency checks
400  if self.doWriteSubtractedExp and self.useScoreImageDetection:
401  raise ValueError(
402  "doWriteSubtractedExp=True and useScoreImageDetection=True "
403  "Regular difference image is not calculated. "
404  "AL subtraction calculates either the regular difference image or the score image.")
405  if self.doWriteScoreExp and not self.useScoreImageDetection:
406  raise ValueError(
407  "doWriteScoreExp=True and useScoreImageDetection=False "
408  "Score image is not calculated. "
409  "AL subtraction calculates either the regular difference image or the score image.")
410  if self.doAddMetrics and not self.doSubtract:
411  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
412  if self.useScoreImageDetection and self.doDecorrelation:
413  raise NotImplementedError(
414  "doDecorrelation=True and useScoreImageDetection=True "
415  "The decorrelation afterburner for AL likelihood images is not implemented.")
416  if self.useGaussianForPreConvolution and not self.useScoreImageDetection:
417  raise ValueError(
418  "useGaussianForPreConvolution=True and useScoreImageDetection=False "
419  "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.")
420 
421 
422 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
423 
424  @staticmethod
425  def getTargetList(parsedCmd, **kwargs):
426  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
427  **kwargs)
428 
429 
430 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
431  """Subtract an image from a template and measure the result
432  """
433  ConfigClass = ImageDifferenceConfig
434  RunnerClass = ImageDifferenceTaskRunner
435  _DefaultName = "imageDifference"
436 
437  def __init__(self, butler=None, **kwargs):
438  """!Construct an ImageDifference Task
439 
440  @param[in] butler Butler object to use in constructing reference object loaders
441  """
442  super().__init__(**kwargs)
443  self.makeSubtask("getTemplate")
444 
445  self.makeSubtask("subtract")
446 
447  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
448  self.makeSubtask("decorrelate")
449 
450  if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
451  self.makeSubtask("scaleVariance")
452 
453  if self.config.doUseRegister:
454  self.makeSubtask("register")
455  self.schema = afwTable.SourceTable.makeMinimalSchema()
456 
457  if self.config.doSelectSources:
458  self.makeSubtask("sourceSelector")
459  if self.config.kernelSourcesFromRef:
460  self.makeSubtask('refObjLoader', butler=butler)
461  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
462 
463  self.algMetadata = dafBase.PropertyList()
464  if self.config.doDetection:
465  self.makeSubtask("detection", schema=self.schema)
466  if self.config.doMeasurement:
467  self.makeSubtask("measurement", schema=self.schema,
468  algMetadata=self.algMetadata)
469  if self.config.doApCorr:
470  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
471  if self.config.doForcedMeasurement:
472  self.schema.addField(
473  "ip_diffim_forced_PsfFlux_instFlux", "D",
474  "Forced PSF flux measured on the direct image.",
475  units="count")
476  self.schema.addField(
477  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
478  "Forced PSF flux error measured on the direct image.",
479  units="count")
480  self.schema.addField(
481  "ip_diffim_forced_PsfFlux_area", "F",
482  "Forced PSF flux effective area of PSF.",
483  units="pixel")
484  self.schema.addField(
485  "ip_diffim_forced_PsfFlux_flag", "Flag",
486  "Forced PSF flux general failure flag.")
487  self.schema.addField(
488  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
489  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
490  self.schema.addField(
491  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
492  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
493  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
494  if self.config.doMatchSources:
495  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
496  self.schema.addField("srcMatchId", "L", "unique id of source match")
497  if self.config.doSkySources:
498  self.makeSubtask("skySources")
499  self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
500 
501  # initialize InitOutputs
502  self.outputSchema = afwTable.SourceCatalog(self.schema)
503  self.outputSchema.getTable().setMetadata(self.algMetadata)
504 
505  @staticmethod
506  def makeIdFactory(expId, expBits):
507  """Create IdFactory instance for unique 64 bit diaSource id-s.
508 
509  Parameters
510  ----------
511  expId : `int`
512  Exposure id.
513 
514  expBits: `int`
515  Number of used bits in ``expId``.
516 
517  Note
518  ----
519  The diasource id-s consists of the ``expId`` stored fixed in the highest value
520  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
521  low value end of the integer.
522 
523  Returns
524  -------
525  idFactory: `lsst.afw.table.IdFactory`
526  """
527  return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
528 
529  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
530  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
531  inputRefs: pipeBase.InputQuantizedConnection,
532  outputRefs: pipeBase.OutputQuantizedConnection):
533  inputs = butlerQC.get(inputRefs)
534  self.log.info("Processing %s", butlerQC.quantum.dataId)
535  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
536  returnMaxBits=True)
537  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
538  if self.config.coaddName == 'dcr':
539  templateExposures = inputRefs.dcrCoadds
540  else:
541  templateExposures = inputRefs.coaddExposures
542  templateStruct = self.getTemplate.runQuantum(
543  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
544  )
545 
546  self.checkTemplateIsSufficient(templateStruct.exposure)
547 
548  outputs = self.run(exposure=inputs['exposure'],
549  templateExposure=templateStruct.exposure,
550  idFactory=idFactory)
551  # Consistency with runDataref gen2 handling
552  if outputs.diaSources is None:
553  del outputs.diaSources
554  butlerQC.put(outputs, outputRefs)
555 
556  @pipeBase.timeMethod
557  def runDataRef(self, sensorRef, templateIdList=None):
558  """Subtract an image from a template coadd and measure the result.
559 
560  Data I/O wrapper around `run` using the butler in Gen2.
561 
562  Parameters
563  ----------
564  sensorRef : `lsst.daf.persistence.ButlerDataRef`
565  Sensor-level butler data reference, used for the following data products:
566 
567  Input only:
568  - calexp
569  - psf
570  - ccdExposureId
571  - ccdExposureId_bits
572  - self.config.coaddName + "Coadd_skyMap"
573  - self.config.coaddName + "Coadd"
574  Input or output, depending on config:
575  - self.config.coaddName + "Diff_subtractedExp"
576  Output, depending on config:
577  - self.config.coaddName + "Diff_matchedExp"
578  - self.config.coaddName + "Diff_src"
579 
580  Returns
581  -------
582  results : `lsst.pipe.base.Struct`
583  Returns the Struct by `run`.
584  """
585  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
586  subtractedExposure = None
587  selectSources = None
588  calexpBackgroundExposure = None
589  self.log.info("Processing %s", sensorRef.dataId)
590 
591  # We make one IdFactory that will be used by both icSrc and src datasets;
592  # I don't know if this is the way we ultimately want to do things, but at least
593  # this ensures the source IDs are fully unique.
594  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
595  expBits=sensorRef.get("ccdExposureId_bits"))
596  if self.config.doAddCalexpBackground:
597  calexpBackgroundExposure = sensorRef.get("calexpBackground")
598 
599  # Retrieve the science image we wish to analyze
600  exposure = sensorRef.get("calexp", immediate=True)
601 
602  # Retrieve the template image
603  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
604 
605  if sensorRef.datasetExists("src"):
606  self.log.info("Source selection via src product")
607  # Sources already exist; for data release processing
608  selectSources = sensorRef.get("src")
609 
610  if not self.config.doSubtract and self.config.doDetection:
611  # If we don't do subtraction, we need the subtracted exposure from the repo
612  subtractedExposure = sensorRef.get(subtractedExposureName)
613  # Both doSubtract and doDetection cannot be False
614 
615  results = self.run(exposure=exposure,
616  selectSources=selectSources,
617  templateExposure=template.exposure,
618  templateSources=template.sources,
619  idFactory=idFactory,
620  calexpBackgroundExposure=calexpBackgroundExposure,
621  subtractedExposure=subtractedExposure)
622 
623  if self.config.doWriteSources and results.diaSources is not None:
624  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
625  if self.config.doWriteWarpedExp:
626  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
627  if self.config.doWriteMatchedExp:
628  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
629  if self.config.doAddMetrics and self.config.doSelectSources:
630  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
631  if self.config.doWriteSubtractedExp:
632  sensorRef.put(results.subtractedExposure, subtractedExposureName)
633  if self.config.doWriteScoreExp:
634  sensorRef.put(results.scoreExposure, self.config.coaddName + "Diff_scoreExp")
635  return results
636 
637  @pipeBase.timeMethod
638  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
639  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
640  """PSF matches, subtract two images and perform detection on the difference image.
641 
642  Parameters
643  ----------
644  exposure : `lsst.afw.image.ExposureF`, optional
645  The science exposure, the minuend in the image subtraction.
646  Can be None only if ``config.doSubtract==False``.
647  selectSources : `lsst.afw.table.SourceCatalog`, optional
648  Identified sources on the science exposure. This catalog is used to
649  select sources in order to perform the AL PSF matching on stamp images
650  around them. The selection steps depend on config options and whether
651  ``templateSources`` and ``matchingSources`` specified.
652  templateExposure : `lsst.afw.image.ExposureF`, optional
653  The template to be subtracted from ``exposure`` in the image subtraction.
654  ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
655  The template exposure should cover the same sky area as the science exposure.
656  It is either a stich of patches of a coadd skymap image or a calexp
657  of the same pointing as the science exposure. Can be None only
658  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
659  templateSources : `lsst.afw.table.SourceCatalog`, optional
660  Identified sources on the template exposure.
661  idFactory : `lsst.afw.table.IdFactory`
662  Generator object to assign ids to detected sources in the difference image.
663  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
664  Background exposure to be added back to the science exposure
665  if ``config.doAddCalexpBackground==True``
666  subtractedExposure : `lsst.afw.image.ExposureF`, optional
667  If ``config.doSubtract==False`` and ``config.doDetection==True``,
668  performs the post subtraction source detection only on this exposure.
669  Otherwise should be None.
670 
671  Returns
672  -------
673  results : `lsst.pipe.base.Struct`
674  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
675  Difference image.
676  ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
677  The zogy score exposure, if calculated.
678  ``matchedExposure`` : `lsst.afw.image.ExposureF`
679  The matched PSF exposure.
680  ``subtractRes`` : `lsst.pipe.base.Struct`
681  The returned result structure of the ImagePsfMatchTask subtask.
682  ``diaSources`` : `lsst.afw.table.SourceCatalog`
683  The catalog of detected sources.
684  ``selectSources`` : `lsst.afw.table.SourceCatalog`
685  The input source catalog with optionally added Qa information.
686 
687  Notes
688  -----
689  The following major steps are included:
690 
691  - warp template coadd to match WCS of image
692  - PSF match image to warped template
693  - subtract image from PSF-matched, warped template
694  - detect sources
695  - measure sources
696 
697  For details about the image subtraction configuration modes
698  see `lsst.ip.diffim`.
699  """
700  subtractRes = None
701  controlSources = None
702  subtractedExposure = None
703  scoreExposure = None
704  diaSources = None
705  kernelSources = None
706  # We'll clone exposure if modified but will still need the original
707  exposureOrig = exposure
708 
709  if self.config.doAddCalexpBackground:
710  mi = exposure.getMaskedImage()
711  mi += calexpBackgroundExposure.getImage()
712 
713  if not exposure.hasPsf():
714  raise pipeBase.TaskError("Exposure has no psf")
715  sciencePsf = exposure.getPsf()
716 
717  if self.config.doSubtract:
718  if self.config.doScaleTemplateVariance:
719  self.log.info("Rescaling template variance")
720  templateVarFactor = self.scaleVariance.run(
721  templateExposure.getMaskedImage())
722  self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
723  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
724  self.metadata.add("psfMatchingAlgorithm", self.config.subtract.name)
725 
726  if self.config.subtract.name == 'zogy':
727  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
728  scoreExposure = subtractRes.scoreExp
729  subtractedExposure = subtractRes.diffExp
730  subtractRes.subtractedExposure = subtractedExposure
731  subtractRes.matchedExposure = None
732 
733  elif self.config.subtract.name == 'al':
734  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
735  scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
736  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
737 
738  # if requested, convolve the science exposure with its PSF
739  # (properly, this should be a cross-correlation, but our code does not yet support that)
740  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
741  # else sigma of original science exposure
742  # TODO: DM-22762 This functional block should be moved into its own method
743  preConvPsf = None
744  if self.config.useScoreImageDetection:
745  self.log.warn("AL likelihood image: pre-convolution of PSF is not implemented.")
746  convControl = afwMath.ConvolutionControl()
747  # cannot convolve in place, so need a new image anyway
748  srcMI = exposure.maskedImage
749  exposure = exposure.clone() # New deep copy
750  srcPsf = sciencePsf
751  if self.config.useGaussianForPreConvolution:
752  self.log.infof(
753  "AL likelihood image: Using Gaussian (sigma={:.2f}) PSF estimation "
754  "for science image pre-convolution", scienceSigmaOrig)
755  # convolve with a simplified PSF model: a double Gaussian
756  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
757  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
758  else:
759  # convolve with science exposure's PSF model
760  self.log.infof(
761  "AL likelihood image: Using the science image PSF for pre-convolution.")
762  preConvPsf = srcPsf
763  afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
764  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
765  else:
766  scienceSigmaPost = scienceSigmaOrig
767 
768  # If requested, find and select sources from the image
769  # else, AL subtraction will do its own source detection
770  # TODO: DM-22762 This functional block should be moved into its own method
771  if self.config.doSelectSources:
772  if selectSources is None:
773  self.log.warning("Src product does not exist; running detection, measurement,"
774  " selection")
775  # Run own detection and measurement; necessary in nightly processing
776  selectSources = self.subtract.getSelectSources(
777  exposure,
778  sigma=scienceSigmaPost,
779  doSmooth=not self.config.useScoreImageDetection,
780  idFactory=idFactory,
781  )
782 
783  if self.config.doAddMetrics:
784  # Number of basis functions
785 
786  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
787  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
788  targetFwhmPix=templateSigma*FwhmPerSigma))
789  # Modify the schema of all Sources
790  # DEPRECATED: This is a data dependent (nparam) output product schema
791  # outside the task constructor.
792  # NOTE: The pre-determination of nparam at this point
793  # may be incorrect as the template psf is warped later in
794  # ImagePsfMatchTask.matchExposures()
795  kcQa = KernelCandidateQa(nparam)
796  selectSources = kcQa.addToSchema(selectSources)
797  if self.config.kernelSourcesFromRef:
798  # match exposure sources to reference catalog
799  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
800  matches = astromRet.matches
801  elif templateSources:
802  # match exposure sources to template sources
803  mc = afwTable.MatchControl()
804  mc.findOnlyClosest = False
805  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
806  mc)
807  else:
808  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
809  "but template sources not available. Cannot match science "
810  "sources with template sources. Run process* on data from "
811  "which templates are built.")
812 
813  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
814  matches=matches).sourceCat
815  random.shuffle(kernelSources, random.random)
816  controlSources = kernelSources[::self.config.controlStepSize]
817  kernelSources = [k for i, k in enumerate(kernelSources)
818  if i % self.config.controlStepSize]
819 
820  if self.config.doSelectDcrCatalog:
821  redSelector = DiaCatalogSourceSelectorTask(
822  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
823  grMax=99.999))
824  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
825  controlSources.extend(redSources)
826 
827  blueSelector = DiaCatalogSourceSelectorTask(
828  DiaCatalogSourceSelectorConfig(grMin=-99.999,
829  grMax=self.sourceSelector.config.grMin))
830  blueSources = blueSelector.selectStars(exposure, selectSources,
831  matches=matches).starCat
832  controlSources.extend(blueSources)
833 
834  if self.config.doSelectVariableCatalog:
835  varSelector = DiaCatalogSourceSelectorTask(
836  DiaCatalogSourceSelectorConfig(includeVariable=True))
837  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
838  controlSources.extend(varSources)
839 
840  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)",
841  len(kernelSources), len(selectSources), len(controlSources))
842 
843  allresids = {}
844  # TODO: DM-22762 This functional block should be moved into its own method
845  if self.config.doUseRegister:
846  self.log.info("Registering images")
847 
848  if templateSources is None:
849  # Run detection on the template, which is
850  # temporarily background-subtracted
851  # sigma of PSF of template image before warping
852  templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
853  templateSources = self.subtract.getSelectSources(
854  templateExposure,
855  sigma=templateSigma,
856  doSmooth=True,
857  idFactory=idFactory
858  )
859 
860  # Third step: we need to fit the relative astrometry.
861  #
862  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
863  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
864  exposure.getWcs(), exposure.getBBox())
865  templateExposure = warpedExp
866 
867  # Create debugging outputs on the astrometric
868  # residuals as a function of position. Persistence
869  # not yet implemented; expected on (I believe) #2636.
870  if self.config.doDebugRegister:
871  # Grab matches to reference catalog
872  srcToMatch = {x.second.getId(): x.first for x in matches}
873 
874  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
875  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
876  sids = [m.first.getId() for m in wcsResults.matches]
877  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
878  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
879  m.second.get(inCentroidKey))) for m in wcsResults.matches]
880  allresids = dict(zip(sids, zip(positions, residuals)))
881 
882  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
883  wcsResults.wcs.pixelToSky(
884  m.second.get(inCentroidKey))) for m in wcsResults.matches]
885  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
886  + 2.5*numpy.log10(srcToMatch[x].get("r"))
887  for x in sids if x in srcToMatch.keys()])
888  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
889  if s in srcToMatch.keys()])
890  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
891  if s in srcToMatch.keys()])
892  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
893  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
894  & (colors <= self.sourceSelector.config.grMax))
895  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
896  rms1Long = IqrToSigma*(
897  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
898  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
899  - numpy.percentile(dlat[idx1], 25))
900  rms2Long = IqrToSigma*(
901  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
902  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
903  - numpy.percentile(dlat[idx2], 25))
904  rms3Long = IqrToSigma*(
905  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
906  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
907  - numpy.percentile(dlat[idx3], 25))
908  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f",
909  numpy.median(dlong[idx1]), rms1Long,
910  numpy.median(dlat[idx1]), rms1Lat)
911  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f",
912  numpy.median(dlong[idx2]), rms2Long,
913  numpy.median(dlat[idx2]), rms2Lat)
914  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f",
915  numpy.median(dlong[idx3]), rms3Long,
916  numpy.median(dlat[idx3]), rms3Lat)
917 
918  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
919  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
920  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
921  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
922  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
923  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
924 
925  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
926  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
927  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
928  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
929  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
930  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
931 
932  # warp template exposure to match exposure,
933  # PSF match template exposure to exposure,
934  # then return the difference
935 
936  # Return warped template... Construct sourceKernelCand list after subtract
937  self.log.info("Subtracting images")
938  subtractRes = self.subtract.subtractExposures(
939  templateExposure=templateExposure,
940  scienceExposure=exposure,
941  candidateList=kernelSources,
942  convolveTemplate=self.config.convolveTemplate,
943  doWarping=not self.config.doUseRegister
944  )
945  if self.config.useScoreImageDetection:
946  scoreExposure = subtractRes.subtractedExposure
947  else:
948  subtractedExposure = subtractRes.subtractedExposure
949 
950  if self.config.doDetection:
951  self.log.info("Computing diffim PSF")
952 
953  # Get Psf from the appropriate input image if it doesn't exist
954  if subtractedExposure is not None and not subtractedExposure.hasPsf():
955  if self.config.convolveTemplate:
956  subtractedExposure.setPsf(exposure.getPsf())
957  else:
958  subtractedExposure.setPsf(templateExposure.getPsf())
959 
960  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
961  # thus it may have already been decorrelated. Thus, we do not decorrelate if
962  # doSubtract is False.
963 
964  # NOTE: At this point doSubtract == True
965  # useScoreImageDetection=True and doDecorrelation=True is not allowed in the config
966  if self.config.doDecorrelation and self.config.doSubtract:
967  preConvKernel = None
968  if preConvPsf is not None:
969  preConvKernel = preConvPsf.getLocalKernel()
970  decorrResult = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
971  subtractedExposure,
972  subtractRes.psfMatchingKernel,
973  spatiallyVarying=self.config.doSpatiallyVarying,
974  preConvKernel=preConvKernel,
975  templateMatched=self.config.convolveTemplate)
976  subtractedExposure = decorrResult.correctedExposure
977 
978  # END (if subtractAlgorithm == 'AL')
979  # END (if self.config.doSubtract)
980  if self.config.doDetection:
981  self.log.info("Running diaSource detection")
982 
983  # subtractedExposure - reserved for task return value
984  # in zogy, it is always the proper difference image
985  # in AL, it may be (yet) pre-convolved and/or decorrelated
986  #
987  # detectionExposure - controls which exposure to use for detection
988  # in-place modifications will appear in task return
989  if self.config.useScoreImageDetection:
990  # zogy with score image detection enabled
991  self.log.info("Detection, diffim rescaling and measurements are "
992  "on AL likelihood or Zogy score image.")
993  detectionExposure = scoreExposure
994  else:
995  # AL or zogy with no score image detection
996  detectionExposure = subtractedExposure
997 
998  # Rescale difference image variance plane
999  if self.config.doScaleDiffimVariance:
1000  self.log.info("Rescaling diffim variance")
1001  diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
1002  self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor)
1003  self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
1004 
1005  # Erase existing detection mask planes
1006  mask = detectionExposure.getMaskedImage().getMask()
1007  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
1008 
1009  table = afwTable.SourceTable.make(self.schema, idFactory)
1010  table.setMetadata(self.algMetadata)
1011  results = self.detection.run(
1012  table=table,
1013  exposure=detectionExposure,
1014  doSmooth=not self.config.useScoreImageDetection
1015  )
1016 
1017  if self.config.doMerge:
1018  fpSet = results.fpSets.positive
1019  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1020  self.config.growFootprint, False)
1021  diaSources = afwTable.SourceCatalog(table)
1022  fpSet.makeSources(diaSources)
1023  self.log.info("Merging detections into %d sources", len(diaSources))
1024  else:
1025  diaSources = results.sources
1026  # Inject skySources before measurement.
1027  if self.config.doSkySources:
1028  skySourceFootprints = self.skySources.run(
1029  mask=detectionExposure.mask,
1030  seed=detectionExposure.getInfo().getVisitInfo().getExposureId())
1031  if skySourceFootprints:
1032  for foot in skySourceFootprints:
1033  s = diaSources.addNew()
1034  s.setFootprint(foot)
1035  s.set(self.skySourceKey, True)
1036 
1037  if self.config.doMeasurement:
1038  newDipoleFitting = self.config.doDipoleFitting
1039  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1040  if not newDipoleFitting:
1041  # Just fit dipole in diffim
1042  self.measurement.run(diaSources, detectionExposure)
1043  else:
1044  # Use (matched) template and science image (if avail.) to constrain dipole fitting
1045  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1046  self.measurement.run(diaSources, detectionExposure, exposure,
1047  subtractRes.matchedExposure)
1048  else:
1049  self.measurement.run(diaSources, detectionExposure, exposure)
1050  if self.config.doApCorr:
1051  self.applyApCorr.run(
1052  catalog=diaSources,
1053  apCorrMap=detectionExposure.getInfo().getApCorrMap()
1054  )
1055 
1056  if self.config.doForcedMeasurement:
1057  # Run forced psf photometry on the PVI at the diaSource locations.
1058  # Copy the measured flux and error into the diaSource.
1059  forcedSources = self.forcedMeasurement.generateMeasCat(
1060  exposure, diaSources, detectionExposure.getWcs())
1061  self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1062  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1063  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1064  "ip_diffim_forced_PsfFlux_instFlux", True)
1065  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1066  "ip_diffim_forced_PsfFlux_instFluxErr", True)
1067  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1068  "ip_diffim_forced_PsfFlux_area", True)
1069  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1070  "ip_diffim_forced_PsfFlux_flag", True)
1071  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1072  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1073  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1074  "ip_diffim_forced_PsfFlux_flag_edge", True)
1075  for diaSource, forcedSource in zip(diaSources, forcedSources):
1076  diaSource.assign(forcedSource, mapper)
1077 
1078  # Match with the calexp sources if possible
1079  if self.config.doMatchSources:
1080  if selectSources is not None:
1081  # Create key,val pair where key=diaSourceId and val=sourceId
1082  matchRadAsec = self.config.diaSourceMatchRadius
1083  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1084 
1085  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1086  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1087  srcMatch in srcMatches])
1088  self.log.info("Matched %d / %d diaSources to sources",
1089  len(srcMatchDict), len(diaSources))
1090  else:
1091  self.log.warning("Src product does not exist; cannot match with diaSources")
1092  srcMatchDict = {}
1093 
1094  # Create key,val pair where key=diaSourceId and val=refId
1095  refAstromConfig = AstrometryConfig()
1096  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1097  refAstrometer = AstrometryTask(refAstromConfig)
1098  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1099  refMatches = astromRet.matches
1100  if refMatches is None:
1101  self.log.warning("No diaSource matches with reference catalog")
1102  refMatchDict = {}
1103  else:
1104  self.log.info("Matched %d / %d diaSources to reference catalog",
1105  len(refMatches), len(diaSources))
1106  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1107  refMatch in refMatches])
1108 
1109  # Assign source Ids
1110  for diaSource in diaSources:
1111  sid = diaSource.getId()
1112  if sid in srcMatchDict:
1113  diaSource.set("srcMatchId", srcMatchDict[sid])
1114  if sid in refMatchDict:
1115  diaSource.set("refMatchId", refMatchDict[sid])
1116 
1117  if self.config.doAddMetrics and self.config.doSelectSources:
1118  self.log.info("Evaluating metrics and control sample")
1119 
1120  kernelCandList = []
1121  for cell in subtractRes.kernelCellSet.getCellList():
1122  for cand in cell.begin(False): # include bad candidates
1123  kernelCandList.append(cand)
1124 
1125  # Get basis list to build control sample kernels
1126  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1127  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1128 
1129  controlCandList = (
1130  diffimTools.sourceTableToCandidateList(controlSources,
1131  subtractRes.warpedExposure, exposure,
1132  self.config.subtract.kernel.active,
1133  self.config.subtract.kernel.active.detectionConfig,
1134  self.log, doBuild=True, basisList=basisList))
1135 
1136  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1137  subtractRes.backgroundModel, dof=nparam)
1138  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1139  subtractRes.backgroundModel)
1140 
1141  if self.config.doDetection:
1142  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1143  else:
1144  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1145 
1146  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1147  return pipeBase.Struct(
1148  subtractedExposure=subtractedExposure,
1149  scoreExposure=scoreExposure,
1150  warpedExposure=subtractRes.warpedExposure,
1151  matchedExposure=subtractRes.matchedExposure,
1152  subtractRes=subtractRes,
1153  diaSources=diaSources,
1154  selectSources=selectSources
1155  )
1156 
1157  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1158  """Fit the relative astrometry between templateSources and selectSources
1159 
1160  Todo
1161  ----
1162 
1163  Remove this method. It originally fit a new WCS to the template before calling register.run
1164  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1165  It remains because a subtask overrides it.
1166  """
1167  results = self.register.run(templateSources, templateExposure.getWcs(),
1168  templateExposure.getBBox(), selectSources)
1169  return results
1170 
1171  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1172  """Make debug plots and displays.
1173 
1174  Todo
1175  ----
1176  Test and update for current debug display and slot names
1177  """
1178  import lsstDebug
1179  display = lsstDebug.Info(__name__).display
1180  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1181  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1182  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1183  showDipoles = lsstDebug.Info(__name__).showDipoles
1184  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1185  if display:
1186  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1187  if not maskTransparency:
1188  maskTransparency = 0
1189  disp.setMaskTransparency(maskTransparency)
1190 
1191  if display and showSubtracted:
1192  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1193  mi = subtractRes.subtractedExposure.getMaskedImage()
1194  x0, y0 = mi.getX0(), mi.getY0()
1195  with disp.Buffering():
1196  for s in diaSources:
1197  x, y = s.getX() - x0, s.getY() - y0
1198  ctype = "red" if s.get("flags_negative") else "yellow"
1199  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1200  or s.get("base_PixelFlags_flag_saturatedCenter")
1201  or s.get("base_PixelFlags_flag_crCenter")):
1202  ptype = "x"
1203  elif (s.get("base_PixelFlags_flag_interpolated")
1204  or s.get("base_PixelFlags_flag_saturated")
1205  or s.get("base_PixelFlags_flag_cr")):
1206  ptype = "+"
1207  else:
1208  ptype = "o"
1209  disp.dot(ptype, x, y, size=4, ctype=ctype)
1210  lsstDebug.frame += 1
1211 
1212  if display and showPixelResiduals and selectSources:
1213  nonKernelSources = []
1214  for source in selectSources:
1215  if source not in kernelSources:
1216  nonKernelSources.append(source)
1217 
1218  diUtils.plotPixelResiduals(exposure,
1219  subtractRes.warpedExposure,
1220  subtractRes.subtractedExposure,
1221  subtractRes.kernelCellSet,
1222  subtractRes.psfMatchingKernel,
1223  subtractRes.backgroundModel,
1224  nonKernelSources,
1225  self.subtract.config.kernel.active.detectionConfig,
1226  origVariance=False)
1227  diUtils.plotPixelResiduals(exposure,
1228  subtractRes.warpedExposure,
1229  subtractRes.subtractedExposure,
1230  subtractRes.kernelCellSet,
1231  subtractRes.psfMatchingKernel,
1232  subtractRes.backgroundModel,
1233  nonKernelSources,
1234  self.subtract.config.kernel.active.detectionConfig,
1235  origVariance=True)
1236  if display and showDiaSources:
1237  flagChecker = SourceFlagChecker(diaSources)
1238  isFlagged = [flagChecker(x) for x in diaSources]
1239  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1240  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1241  frame=lsstDebug.frame)
1242  lsstDebug.frame += 1
1243 
1244  if display and showDipoles:
1245  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1246  frame=lsstDebug.frame)
1247  lsstDebug.frame += 1
1248 
1249  def checkTemplateIsSufficient(self, templateExposure):
1250  """Raise NoWorkFound if template coverage < requiredTemplateFraction
1251 
1252  Parameters
1253  ----------
1254  templateExposure : `lsst.afw.image.ExposureF`
1255  The template exposure to check
1256 
1257  Raises
1258  ------
1259  NoWorkFound
1260  Raised if fraction of good pixels, defined as not having NO_DATA
1261  set, is less then the configured requiredTemplateFraction
1262  """
1263  # Count the number of pixels with the NO_DATA mask bit set
1264  # counting NaN pixels is insufficient because pixels without data are often intepolated over)
1265  pixNoData = numpy.count_nonzero(templateExposure.mask.array
1266  & templateExposure.mask.getPlaneBitMask('NO_DATA'))
1267  pixGood = templateExposure.getBBox().getArea() - pixNoData
1268  self.log.info("template has %d good pixels (%.1f%%)", pixGood,
1269  100*pixGood/templateExposure.getBBox().getArea())
1270 
1271  if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction:
1272  message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
1273  "To force subtraction, set config requiredTemplateFraction=0." % (
1274  100*pixGood/templateExposure.getBBox().getArea(),
1275  100*self.config.requiredTemplateFraction))
1276  raise pipeBase.NoWorkFound(message)
1277 
1278  def _getConfigName(self):
1279  """Return the name of the config dataset
1280  """
1281  return "%sDiff_config" % (self.config.coaddName,)
1282 
1283  def _getMetadataName(self):
1284  """Return the name of the metadata dataset
1285  """
1286  return "%sDiff_metadata" % (self.config.coaddName,)
1287 
1288  def getSchemaCatalogs(self):
1289  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1290  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1291 
1292  @classmethod
1293  def _makeArgumentParser(cls):
1294  """Create an argument parser
1295  """
1296  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1297  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1298  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1299  help="Template data ID in case of calexp template,"
1300  " e.g. --templateId visit=6789")
1301  return parser
1302 
1303 
1304 class ImageDifferenceFromTemplateConnections(ImageDifferenceTaskConnections,
1305  defaultTemplates={"coaddName": "goodSeeing"}
1306  ):
1307  inputTemplate = pipeBase.connectionTypes.Input(
1308  doc=("Warped template produced by GetMultiTractCoaddTemplate"),
1309  dimensions=("instrument", "visit", "detector"),
1310  storageClass="ExposureF",
1311  name="{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}",
1312  )
1313 
1314  def __init__(self, *, config=None):
1315  super().__init__(config=config)
1316  # ImageDifferenceConnections will have removed one of these.
1317  # Make sure they're both gone, because no coadds are needed.
1318  if "coaddExposures" in self.inputs:
1319  self.inputs.remove("coaddExposures")
1320  if "dcrCoadds" in self.inputs:
1321  self.inputs.remove("dcrCoadds")
1322 
1323 
1324 class ImageDifferenceFromTemplateConfig(ImageDifferenceConfig,
1325  pipelineConnections=ImageDifferenceFromTemplateConnections):
1326  pass
1327 
1328 
1329 class ImageDifferenceFromTemplateTask(ImageDifferenceTask):
1330  ConfigClass = ImageDifferenceFromTemplateConfig
1331  _DefaultName = "imageDifference"
1332 
1333  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
1334  def runQuantum(self, butlerQC, inputRefs, outputRefs):
1335  inputs = butlerQC.get(inputRefs)
1336  self.log.info("Processing %s", butlerQC.quantum.dataId)
1337  self.checkTemplateIsSufficient(inputs['inputTemplate'])
1338  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
1339  returnMaxBits=True)
1340  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
1341 
1342  outputs = self.run(exposure=inputs['exposure'],
1343  templateExposure=inputs['inputTemplate'],
1344  idFactory=idFactory)
1345 
1346  # Consistency with runDataref gen2 handling
1347  if outputs.diaSources is None:
1348  del outputs.diaSources
1349  butlerQC.put(outputs, outputRefs)
1350 
1351 
1352 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1353  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1354  doc="Shift stars going into RegisterTask by this amount")
1355  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1356  doc="Perturb stars going into RegisterTask by this amount")
1357 
1358  def setDefaults(self):
1359  ImageDifferenceConfig.setDefaults(self)
1360  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1361 
1362 
1363 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1364  """!Image difference Task used in the Winter 2013 data challege.
1365  Enables testing the effects of registration shifts and scatter.
1366 
1367  For use with winter 2013 simulated images:
1368  Use --templateId visit=88868666 for sparse data
1369  --templateId visit=22222200 for dense data (g)
1370  --templateId visit=11111100 for dense data (i)
1371  """
1372  ConfigClass = Winter2013ImageDifferenceConfig
1373  _DefaultName = "winter2013ImageDifference"
1374 
1375  def __init__(self, **kwargs):
1376  ImageDifferenceTask.__init__(self, **kwargs)
1377 
1378  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1379  """Fit the relative astrometry between templateSources and selectSources"""
1380  if self.config.winter2013WcsShift > 0.0:
1381  offset = geom.Extent2D(self.config.winter2013WcsShift,
1382  self.config.winter2013WcsShift)
1383  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1384  for source in templateSources:
1385  centroid = source.get(cKey)
1386  source.set(cKey, centroid + offset)
1387  elif self.config.winter2013WcsRms > 0.0:
1388  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1389  for source in templateSources:
1390  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1391  self.config.winter2013WcsRms*numpy.random.normal())
1392  centroid = source.get(cKey)
1393  source.set(cKey, centroid + offset)
1394 
1395  results = self.register.run(templateSources, templateExposure.getWcs(),
1396  templateExposure.getBBox(), selectSources)
1397  return results
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)