lsst.pipe.tasks  v23.0.x-g0162f352a6+f2fbba1123
imageDifference.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import random
24 import numpy
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 import lsst.geom as geom
31 import lsst.afw.math as afwMath
32 import lsst.afw.table as afwTable
33 from lsst.meas.astrom import AstrometryConfig, AstrometryTask
34 from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
36 from lsst.pipe.tasks.registerImage import RegisterTask
37 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40  KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41  GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42  DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
43 import lsst.ip.diffim.diffimTools as diffimTools
44 import lsst.ip.diffim.utils as diUtils
45 import lsst.afw.display as afwDisplay
46 from lsst.skymap import BaseSkyMap
47 from lsst.obs.base import ExposureIdInfo
48 
49 __all__ = ["ImageDifferenceConfig", "ImageDifferenceTask"]
50 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
51 IqrToSigma = 0.741
52 
53 
54 class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
55  dimensions=("instrument", "visit", "detector", "skymap"),
56  defaultTemplates={"coaddName": "deep",
57  "skyMapName": "deep",
58  "warpTypeSuffix": "",
59  "fakesType": ""}):
60 
61  exposure = pipeBase.connectionTypes.Input(
62  doc="Input science exposure to subtract from.",
63  dimensions=("instrument", "visit", "detector"),
64  storageClass="ExposureF",
65  name="{fakesType}calexp"
66  )
67 
68  # TODO DM-22953
69  # kernelSources = pipeBase.connectionTypes.Input(
70  # doc="Source catalog produced in calibrate task for kernel candidate sources",
71  # name="src",
72  # storageClass="SourceCatalog",
73  # dimensions=("instrument", "visit", "detector"),
74  # )
75 
76  skyMap = pipeBase.connectionTypes.Input(
77  doc="Input definition of geometry/bbox and projection/wcs for template exposures",
78  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
79  dimensions=("skymap", ),
80  storageClass="SkyMap",
81  )
82  coaddExposures = pipeBase.connectionTypes.Input(
83  doc="Input template to match and subtract from the exposure",
84  dimensions=("tract", "patch", "skymap", "band"),
85  storageClass="ExposureF",
86  name="{fakesType}{coaddName}Coadd{warpTypeSuffix}",
87  multiple=True,
88  deferLoad=True
89  )
90  dcrCoadds = pipeBase.connectionTypes.Input(
91  doc="Input DCR template to match and subtract from the exposure",
92  name="{fakesType}dcrCoadd{warpTypeSuffix}",
93  storageClass="ExposureF",
94  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
95  multiple=True,
96  deferLoad=True
97  )
98  outputSchema = pipeBase.connectionTypes.InitOutput(
99  doc="Schema (as an example catalog) for output DIASource catalog.",
100  storageClass="SourceCatalog",
101  name="{fakesType}{coaddName}Diff_diaSrc_schema",
102  )
103  subtractedExposure = pipeBase.connectionTypes.Output(
104  doc="Output AL difference or Zogy proper difference image",
105  dimensions=("instrument", "visit", "detector"),
106  storageClass="ExposureF",
107  name="{fakesType}{coaddName}Diff_differenceExp",
108  )
109  scoreExposure = pipeBase.connectionTypes.Output(
110  doc="Output AL likelihood or Zogy score image",
111  dimensions=("instrument", "visit", "detector"),
112  storageClass="ExposureF",
113  name="{fakesType}{coaddName}Diff_scoreExp",
114  )
115  warpedExposure = pipeBase.connectionTypes.Output(
116  doc="Warped template used to create `subtractedExposure`.",
117  dimensions=("instrument", "visit", "detector"),
118  storageClass="ExposureF",
119  name="{fakesType}{coaddName}Diff_warpedExp",
120  )
121  matchedExposure = pipeBase.connectionTypes.Output(
122  doc="Warped template used to create `subtractedExposure`.",
123  dimensions=("instrument", "visit", "detector"),
124  storageClass="ExposureF",
125  name="{fakesType}{coaddName}Diff_matchedExp",
126  )
127  diaSources = pipeBase.connectionTypes.Output(
128  doc="Output detected diaSources on the difference image",
129  dimensions=("instrument", "visit", "detector"),
130  storageClass="SourceCatalog",
131  name="{fakesType}{coaddName}Diff_diaSrc",
132  )
133 
134  def __init__(self, *, config=None):
135  super().__init__(config=config)
136  if config.coaddName == 'dcr':
137  self.inputs.remove("coaddExposures")
138  else:
139  self.inputs.remove("dcrCoadds")
140  if not config.doWriteSubtractedExp:
141  self.outputs.remove("subtractedExposure")
142  if not config.doWriteScoreExp:
143  self.outputs.remove("scoreExposure")
144  if not config.doWriteWarpedExp:
145  self.outputs.remove("warpedExposure")
146  if not config.doWriteMatchedExp:
147  self.outputs.remove("matchedExposure")
148  if not config.doWriteSources:
149  self.outputs.remove("diaSources")
150 
151  # TODO DM-22953: Add support for refObjLoader (kernelSourcesFromRef)
152  # Make kernelSources optional
153 
154 
155 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
156  pipelineConnections=ImageDifferenceTaskConnections):
157  """Config for ImageDifferenceTask
158  """
159  doAddCalexpBackground = pexConfig.Field(dtype=bool, default=False,
160  doc="Add background to calexp before processing it. "
161  "Useful as ipDiffim does background matching.")
162  doUseRegister = pexConfig.Field(dtype=bool, default=False,
163  doc="Re-compute astrometry on the template. "
164  "Use image-to-image registration to align template with "
165  "science image (AL only).")
166  doDebugRegister = pexConfig.Field(dtype=bool, default=False,
167  doc="Writing debugging data for doUseRegister")
168  doSelectSources = pexConfig.Field(dtype=bool, default=False,
169  doc="Select stars to use for kernel fitting (AL only)")
170  doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=False,
171  doc="Select stars of extreme color as part "
172  "of the control sample (AL only)")
173  doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=False,
174  doc="Select stars that are variable to be part "
175  "of the control sample (AL only)")
176  doSubtract = pexConfig.Field(dtype=bool, default=True, doc="Compute subtracted exposure?")
177  doPreConvolve = pexConfig.Field(dtype=bool, default=False,
178  doc="Not in use. Superseded by useScoreImageDetection.",
179  deprecated="This option superseded by useScoreImageDetection."
180  " Will be removed after v22.")
181  useScoreImageDetection = pexConfig.Field(
182  dtype=bool, default=False, doc="Calculate the pre-convolved AL likelihood or "
183  "the Zogy score image. Use it for source detection (if doDetection=True).")
184  doWriteScoreExp = pexConfig.Field(
185  dtype=bool, default=False, doc="Write AL likelihood or Zogy score exposure?")
186  doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=False,
187  doc="Scale variance of the template before PSF matching")
188  doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=True,
189  doc="Scale variance of the diffim before PSF matching. "
190  "You may do either this or template variance scaling, "
191  "or neither. (Doing both is a waste of CPU.)")
192  useGaussianForPreConvolution = pexConfig.Field(
193  dtype=bool, default=False, doc="Use a simple gaussian PSF model for pre-convolution "
194  "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)")
195  doDetection = pexConfig.Field(dtype=bool, default=True, doc="Detect sources?")
196  doDecorrelation = pexConfig.Field(dtype=bool, default=True,
197  doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
198  "kernel convolution (AL only)? If True, also update the diffim PSF.")
199  doMerge = pexConfig.Field(dtype=bool, default=True,
200  doc="Merge positive and negative diaSources with grow radius "
201  "set by growFootprint")
202  doMatchSources = pexConfig.Field(dtype=bool, default=False,
203  doc="Match diaSources with input calexp sources and ref catalog sources")
204  doMeasurement = pexConfig.Field(dtype=bool, default=True, doc="Measure diaSources?")
205  doDipoleFitting = pexConfig.Field(dtype=bool, default=True, doc="Measure dipoles using new algorithm?")
206  doForcedMeasurement = pexConfig.Field(
207  dtype=bool,
208  default=True,
209  doc="Force photometer diaSource locations on PVI?")
210  doWriteSubtractedExp = pexConfig.Field(
211  dtype=bool, default=True, doc="Write difference exposure (AL and Zogy) ?")
212  doWriteWarpedExp = pexConfig.Field(
213  dtype=bool, default=False, doc="Write WCS, warped template coadd exposure?")
214  doWriteMatchedExp = pexConfig.Field(dtype=bool, default=False,
215  doc="Write warped and PSF-matched template coadd exposure?")
216  doWriteSources = pexConfig.Field(dtype=bool, default=True, doc="Write sources?")
217  doAddMetrics = pexConfig.Field(dtype=bool, default=False,
218  doc="Add columns to the source table to hold analysis metrics?")
219 
220  coaddName = pexConfig.Field(
221  doc="coadd name: typically one of deep, goodSeeing, or dcr",
222  dtype=str,
223  default="deep",
224  )
225  convolveTemplate = pexConfig.Field(
226  doc="Which image gets convolved (default = template)",
227  dtype=bool,
228  default=True
229  )
230  refObjLoader = pexConfig.ConfigurableField(
231  target=LoadIndexedReferenceObjectsTask,
232  doc="reference object loader",
233  )
234  astrometer = pexConfig.ConfigurableField(
235  target=AstrometryTask,
236  doc="astrometry task; used to match sources to reference objects, but not to fit a WCS",
237  )
238  sourceSelector = pexConfig.ConfigurableField(
239  target=ObjectSizeStarSelectorTask,
240  doc="Source selection algorithm",
241  )
242  subtract = subtractAlgorithmRegistry.makeField("Subtraction Algorithm", default="al")
243  decorrelate = pexConfig.ConfigurableField(
244  target=DecorrelateALKernelSpatialTask,
245  doc="Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
246  "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
247  "default of 5.5).",
248  )
249  # Old style ImageMapper grid. ZogyTask has its own grid option
250  doSpatiallyVarying = pexConfig.Field(
251  dtype=bool,
252  default=False,
253  doc="Perform A&L decorrelation on a grid across the "
254  "image in order to allow for spatial variations. Zogy does not use this option."
255  )
256  detection = pexConfig.ConfigurableField(
257  target=SourceDetectionTask,
258  doc="Low-threshold detection for final measurement",
259  )
260  measurement = pexConfig.ConfigurableField(
261  target=DipoleFitTask,
262  doc="Enable updated dipole fitting method",
263  )
264  doApCorr = lsst.pex.config.Field(
265  dtype=bool,
266  default=True,
267  doc="Run subtask to apply aperture corrections"
268  )
269  applyApCorr = lsst.pex.config.ConfigurableField(
270  target=ApplyApCorrTask,
271  doc="Subtask to apply aperture corrections"
272  )
273  forcedMeasurement = pexConfig.ConfigurableField(
274  target=ForcedMeasurementTask,
275  doc="Subtask to force photometer PVI at diaSource location.",
276  )
277  getTemplate = pexConfig.ConfigurableField(
278  target=GetCoaddAsTemplateTask,
279  doc="Subtask to retrieve template exposure and sources",
280  )
281  scaleVariance = pexConfig.ConfigurableField(
282  target=ScaleVarianceTask,
283  doc="Subtask to rescale the variance of the template "
284  "to the statistically expected level"
285  )
286  controlStepSize = pexConfig.Field(
287  doc="What step size (every Nth one) to select a control sample from the kernelSources",
288  dtype=int,
289  default=5
290  )
291  controlRandomSeed = pexConfig.Field(
292  doc="Random seed for shuffing the control sample",
293  dtype=int,
294  default=10
295  )
296  register = pexConfig.ConfigurableField(
297  target=RegisterTask,
298  doc="Task to enable image-to-image image registration (warping)",
299  )
300  kernelSourcesFromRef = pexConfig.Field(
301  doc="Select sources to measure kernel from reference catalog if True, template if false",
302  dtype=bool,
303  default=False
304  )
305  templateSipOrder = pexConfig.Field(
306  dtype=int, default=2,
307  doc="Sip Order for fitting the Template Wcs (default is too high, overfitting)"
308  )
309  growFootprint = pexConfig.Field(
310  dtype=int, default=2,
311  doc="Grow positive and negative footprints by this amount before merging"
312  )
313  diaSourceMatchRadius = pexConfig.Field(
314  dtype=float, default=0.5,
315  doc="Match radius (in arcseconds) for DiaSource to Source association"
316  )
317  requiredTemplateFraction = pexConfig.Field(
318  dtype=float, default=0.1,
319  doc="Do not attempt to run task if template covers less than this fraction of pixels."
320  "Setting to 0 will always attempt image subtraction"
321  )
322  doSkySources = pexConfig.Field(
323  dtype=bool,
324  default=False,
325  doc="Generate sky sources?",
326  )
327  skySources = pexConfig.ConfigurableField(
328  target=SkyObjectsTask,
329  doc="Generate sky sources",
330  )
331 
332  def setDefaults(self):
333  # defaults are OK for catalog and diacatalog
334 
335  self.subtract['al'].kernel.name = "AL"
336  self.subtract['al'].kernel.active.fitForBackground = True
337  self.subtract['al'].kernel.active.spatialKernelOrder = 1
338  self.subtract['al'].kernel.active.spatialBgOrder = 2
339 
340  # DiaSource Detection
341  self.detection.thresholdPolarity = "both"
342  self.detection.thresholdValue = 5.0
343  self.detection.reEstimateBackground = False
344  self.detection.thresholdType = "pixel_stdev"
345 
346  # Add filtered flux measurement, the correct measurement for pre-convolved images.
347  # Enable all measurements, regardless of doPreConvolve, as it makes data harvesting easier.
348  # To change that you must modify algorithms.names in the task's applyOverrides method,
349  # after the user has set doPreConvolve.
350  self.measurement.algorithms.names.add('base_PeakLikelihoodFlux')
351  self.measurement.plugins.names |= ['base_LocalPhotoCalib',
352  'base_LocalWcs']
353 
354  self.forcedMeasurement.plugins = ["base_TransformedCentroid", "base_PsfFlux"]
355  self.forcedMeasurement.copyColumns = {
356  "id": "objectId", "parent": "parentObjectId", "coord_ra": "coord_ra", "coord_dec": "coord_dec"}
357  self.forcedMeasurement.slots.centroid = "base_TransformedCentroid"
358  self.forcedMeasurement.slots.shape = None
359 
360  # For shuffling the control sample
361  random.seed(self.controlRandomSeed)
362 
363  def validate(self):
364  pexConfig.Config.validate(self)
365  if not self.doSubtract and not self.doDetection:
366  raise ValueError("Either doSubtract or doDetection must be enabled.")
367  if self.doMeasurement and not self.doDetection:
368  raise ValueError("Cannot run source measurement without source detection.")
369  if self.doMerge and not self.doDetection:
370  raise ValueError("Cannot run source merging without source detection.")
371  if self.doSkySources and not self.doDetection:
372  raise ValueError("Cannot run sky source creation without source detection.")
373  if self.doUseRegister and not self.doSelectSources:
374  raise ValueError("doUseRegister=True and doSelectSources=False. "
375  "Cannot run RegisterTask without selecting sources.")
376  if hasattr(self.getTemplate, "coaddName"):
377  if self.getTemplate.coaddName != self.coaddName:
378  raise ValueError("Mis-matched coaddName and getTemplate.coaddName in the config.")
379  if self.doScaleDiffimVariance and self.doScaleTemplateVariance:
380  raise ValueError("Scaling the diffim variance and scaling the template variance "
381  "are both set. Please choose one or the other.")
382  # We cannot allow inconsistencies that would lead to None or not available output products
383  if self.subtract.name == 'zogy':
384  if self.doWriteMatchedExp:
385  raise ValueError("doWriteMatchedExp=True Matched exposure is not "
386  "calculated in zogy subtraction.")
387  if self.doAddMetrics:
388  raise ValueError("doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
389  if self.doDecorrelation:
390  raise ValueError(
391  "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
392  if self.doSelectSources:
393  raise ValueError(
394  "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
395  if self.useGaussianForPreConvolution:
396  raise ValueError(
397  "useGaussianForPreConvolution=True This is an AL subtraction only option.")
398  else:
399  # AL only consistency checks
400  if self.useScoreImageDetection and not self.convolveTemplate:
401  raise ValueError(
402  "convolveTemplate=False and useScoreImageDetection=True "
403  "Pre-convolution and matching of the science image is not a supported operation.")
404  if self.doWriteSubtractedExp and self.useScoreImageDetection:
405  raise ValueError(
406  "doWriteSubtractedExp=True and useScoreImageDetection=True "
407  "Regular difference image is not calculated. "
408  "AL subtraction calculates either the regular difference image or the score image.")
409  if self.doWriteScoreExp and not self.useScoreImageDetection:
410  raise ValueError(
411  "doWriteScoreExp=True and useScoreImageDetection=False "
412  "Score image is not calculated. "
413  "AL subtraction calculates either the regular difference image or the score image.")
414  if self.doAddMetrics and not self.doSubtract:
415  raise ValueError("Subtraction must be enabled for kernel metrics calculation.")
416  if self.useGaussianForPreConvolution and not self.useScoreImageDetection:
417  raise ValueError(
418  "useGaussianForPreConvolution=True and useScoreImageDetection=False "
419  "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.")
420 
421 
422 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
423 
424  @staticmethod
425  def getTargetList(parsedCmd, **kwargs):
426  return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
427  **kwargs)
428 
429 
430 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
431  """Subtract an image from a template and measure the result
432  """
433  ConfigClass = ImageDifferenceConfig
434  RunnerClass = ImageDifferenceTaskRunner
435  _DefaultName = "imageDifference"
436 
437  def __init__(self, butler=None, **kwargs):
438  """!Construct an ImageDifference Task
439 
440  @param[in] butler Butler object to use in constructing reference object loaders
441  """
442  super().__init__(**kwargs)
443  self.makeSubtask("getTemplate")
444 
445  self.makeSubtask("subtract")
446 
447  if self.config.subtract.name == 'al' and self.config.doDecorrelation:
448  self.makeSubtask("decorrelate")
449 
450  if self.config.doScaleTemplateVariance or self.config.doScaleDiffimVariance:
451  self.makeSubtask("scaleVariance")
452 
453  if self.config.doUseRegister:
454  self.makeSubtask("register")
455  self.schema = afwTable.SourceTable.makeMinimalSchema()
456 
457  if self.config.doSelectSources:
458  self.makeSubtask("sourceSelector")
459  if self.config.kernelSourcesFromRef:
460  self.makeSubtask('refObjLoader', butler=butler)
461  self.makeSubtask("astrometer", refObjLoader=self.refObjLoader)
462 
463  self.algMetadata = dafBase.PropertyList()
464  if self.config.doDetection:
465  self.makeSubtask("detection", schema=self.schema)
466  if self.config.doMeasurement:
467  self.makeSubtask("measurement", schema=self.schema,
468  algMetadata=self.algMetadata)
469  if self.config.doApCorr:
470  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
471  if self.config.doForcedMeasurement:
472  self.schema.addField(
473  "ip_diffim_forced_PsfFlux_instFlux", "D",
474  "Forced PSF flux measured on the direct image.",
475  units="count")
476  self.schema.addField(
477  "ip_diffim_forced_PsfFlux_instFluxErr", "D",
478  "Forced PSF flux error measured on the direct image.",
479  units="count")
480  self.schema.addField(
481  "ip_diffim_forced_PsfFlux_area", "F",
482  "Forced PSF flux effective area of PSF.",
483  units="pixel")
484  self.schema.addField(
485  "ip_diffim_forced_PsfFlux_flag", "Flag",
486  "Forced PSF flux general failure flag.")
487  self.schema.addField(
488  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", "Flag",
489  "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
490  self.schema.addField(
491  "ip_diffim_forced_PsfFlux_flag_edge", "Flag",
492  "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
493  self.makeSubtask("forcedMeasurement", refSchema=self.schema)
494  if self.config.doMatchSources:
495  self.schema.addField("refMatchId", "L", "unique id of reference catalog match")
496  self.schema.addField("srcMatchId", "L", "unique id of source match")
497  if self.config.doSkySources:
498  self.makeSubtask("skySources")
499  self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
500 
501  # initialize InitOutputs
502  self.outputSchema = afwTable.SourceCatalog(self.schema)
503  self.outputSchema.getTable().setMetadata(self.algMetadata)
504 
505  @staticmethod
506  def makeIdFactory(expId, expBits):
507  """Create IdFactory instance for unique 64 bit diaSource id-s.
508 
509  Parameters
510  ----------
511  expId : `int`
512  Exposure id.
513 
514  expBits: `int`
515  Number of used bits in ``expId``.
516 
517  Note
518  ----
519  The diasource id-s consists of the ``expId`` stored fixed in the highest value
520  ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
521  low value end of the integer.
522 
523  Returns
524  -------
525  idFactory: `lsst.afw.table.IdFactory`
526  """
527  return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
528 
529  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
530  def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
531  inputRefs: pipeBase.InputQuantizedConnection,
532  outputRefs: pipeBase.OutputQuantizedConnection):
533  inputs = butlerQC.get(inputRefs)
534  self.log.info("Processing %s", butlerQC.quantum.dataId)
535  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
536  returnMaxBits=True)
537  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
538  if self.config.coaddName == 'dcr':
539  templateExposures = inputRefs.dcrCoadds
540  else:
541  templateExposures = inputRefs.coaddExposures
542  templateStruct = self.getTemplate.runQuantum(
543  inputs['exposure'], butlerQC, inputRefs.skyMap, templateExposures
544  )
545 
546  self.checkTemplateIsSufficient(templateStruct.exposure)
547 
548  outputs = self.run(exposure=inputs['exposure'],
549  templateExposure=templateStruct.exposure,
550  idFactory=idFactory)
551  # Consistency with runDataref gen2 handling
552  if outputs.diaSources is None:
553  del outputs.diaSources
554  butlerQC.put(outputs, outputRefs)
555 
556  @pipeBase.timeMethod
557  def runDataRef(self, sensorRef, templateIdList=None):
558  """Subtract an image from a template coadd and measure the result.
559 
560  Data I/O wrapper around `run` using the butler in Gen2.
561 
562  Parameters
563  ----------
564  sensorRef : `lsst.daf.persistence.ButlerDataRef`
565  Sensor-level butler data reference, used for the following data products:
566 
567  Input only:
568  - calexp
569  - psf
570  - ccdExposureId
571  - ccdExposureId_bits
572  - self.config.coaddName + "Coadd_skyMap"
573  - self.config.coaddName + "Coadd"
574  Input or output, depending on config:
575  - self.config.coaddName + "Diff_subtractedExp"
576  Output, depending on config:
577  - self.config.coaddName + "Diff_matchedExp"
578  - self.config.coaddName + "Diff_src"
579 
580  Returns
581  -------
582  results : `lsst.pipe.base.Struct`
583  Returns the Struct by `run`.
584  """
585  subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
586  subtractedExposure = None
587  selectSources = None
588  calexpBackgroundExposure = None
589  self.log.info("Processing %s", sensorRef.dataId)
590 
591  # We make one IdFactory that will be used by both icSrc and src datasets;
592  # I don't know if this is the way we ultimately want to do things, but at least
593  # this ensures the source IDs are fully unique.
594  idFactory = self.makeIdFactory(expId=int(sensorRef.get("ccdExposureId")),
595  expBits=sensorRef.get("ccdExposureId_bits"))
596  if self.config.doAddCalexpBackground:
597  calexpBackgroundExposure = sensorRef.get("calexpBackground")
598 
599  # Retrieve the science image we wish to analyze
600  exposure = sensorRef.get("calexp", immediate=True)
601 
602  # Retrieve the template image
603  template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
604 
605  if sensorRef.datasetExists("src"):
606  self.log.info("Source selection via src product")
607  # Sources already exist; for data release processing
608  selectSources = sensorRef.get("src")
609 
610  if not self.config.doSubtract and self.config.doDetection:
611  # If we don't do subtraction, we need the subtracted exposure from the repo
612  subtractedExposure = sensorRef.get(subtractedExposureName)
613  # Both doSubtract and doDetection cannot be False
614 
615  results = self.run(exposure=exposure,
616  selectSources=selectSources,
617  templateExposure=template.exposure,
618  templateSources=template.sources,
619  idFactory=idFactory,
620  calexpBackgroundExposure=calexpBackgroundExposure,
621  subtractedExposure=subtractedExposure)
622 
623  if self.config.doWriteSources and results.diaSources is not None:
624  sensorRef.put(results.diaSources, self.config.coaddName + "Diff_diaSrc")
625  if self.config.doWriteWarpedExp:
626  sensorRef.put(results.warpedExposure, self.config.coaddName + "Diff_warpedExp")
627  if self.config.doWriteMatchedExp:
628  sensorRef.put(results.matchedExposure, self.config.coaddName + "Diff_matchedExp")
629  if self.config.doAddMetrics and self.config.doSelectSources:
630  sensorRef.put(results.selectSources, self.config.coaddName + "Diff_kernelSrc")
631  if self.config.doWriteSubtractedExp:
632  sensorRef.put(results.subtractedExposure, subtractedExposureName)
633  if self.config.doWriteScoreExp:
634  sensorRef.put(results.scoreExposure, self.config.coaddName + "Diff_scoreExp")
635  return results
636 
637  @pipeBase.timeMethod
638  def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
639  idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
640  """PSF matches, subtract two images and perform detection on the difference image.
641 
642  Parameters
643  ----------
644  exposure : `lsst.afw.image.ExposureF`, optional
645  The science exposure, the minuend in the image subtraction.
646  Can be None only if ``config.doSubtract==False``.
647  selectSources : `lsst.afw.table.SourceCatalog`, optional
648  Identified sources on the science exposure. This catalog is used to
649  select sources in order to perform the AL PSF matching on stamp images
650  around them. The selection steps depend on config options and whether
651  ``templateSources`` and ``matchingSources`` specified.
652  templateExposure : `lsst.afw.image.ExposureF`, optional
653  The template to be subtracted from ``exposure`` in the image subtraction.
654  ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
655  The template exposure should cover the same sky area as the science exposure.
656  It is either a stich of patches of a coadd skymap image or a calexp
657  of the same pointing as the science exposure. Can be None only
658  if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
659  templateSources : `lsst.afw.table.SourceCatalog`, optional
660  Identified sources on the template exposure.
661  idFactory : `lsst.afw.table.IdFactory`
662  Generator object to assign ids to detected sources in the difference image.
663  calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
664  Background exposure to be added back to the science exposure
665  if ``config.doAddCalexpBackground==True``
666  subtractedExposure : `lsst.afw.image.ExposureF`, optional
667  If ``config.doSubtract==False`` and ``config.doDetection==True``,
668  performs the post subtraction source detection only on this exposure.
669  Otherwise should be None.
670 
671  Returns
672  -------
673  results : `lsst.pipe.base.Struct`
674  ``subtractedExposure`` : `lsst.afw.image.ExposureF`
675  Difference image.
676  ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
677  The zogy score exposure, if calculated.
678  ``matchedExposure`` : `lsst.afw.image.ExposureF`
679  The matched PSF exposure.
680  ``subtractRes`` : `lsst.pipe.base.Struct`
681  The returned result structure of the ImagePsfMatchTask subtask.
682  ``diaSources`` : `lsst.afw.table.SourceCatalog`
683  The catalog of detected sources.
684  ``selectSources`` : `lsst.afw.table.SourceCatalog`
685  The input source catalog with optionally added Qa information.
686 
687  Notes
688  -----
689  The following major steps are included:
690 
691  - warp template coadd to match WCS of image
692  - PSF match image to warped template
693  - subtract image from PSF-matched, warped template
694  - detect sources
695  - measure sources
696 
697  For details about the image subtraction configuration modes
698  see `lsst.ip.diffim`.
699  """
700  subtractRes = None
701  controlSources = None
702  subtractedExposure = None
703  scoreExposure = None
704  diaSources = None
705  kernelSources = None
706  # We'll clone exposure if modified but will still need the original
707  exposureOrig = exposure
708 
709  if self.config.doAddCalexpBackground:
710  mi = exposure.getMaskedImage()
711  mi += calexpBackgroundExposure.getImage()
712 
713  if not exposure.hasPsf():
714  raise pipeBase.TaskError("Exposure has no psf")
715  sciencePsf = exposure.getPsf()
716 
717  if self.config.doSubtract:
718  if self.config.doScaleTemplateVariance:
719  self.log.info("Rescaling template variance")
720  templateVarFactor = self.scaleVariance.run(
721  templateExposure.getMaskedImage())
722  self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
723  self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
724  self.metadata.add("psfMatchingAlgorithm", self.config.subtract.name)
725 
726  if self.config.subtract.name == 'zogy':
727  subtractRes = self.subtract.run(exposure, templateExposure, doWarping=True)
728  scoreExposure = subtractRes.scoreExp
729  subtractedExposure = subtractRes.diffExp
730  subtractRes.subtractedExposure = subtractedExposure
731  subtractRes.matchedExposure = None
732 
733  elif self.config.subtract.name == 'al':
734  # compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
735  # Just need a rough estimate; average positions are fine
736  sciAvgPos = sciencePsf.getAveragePosition()
737  scienceSigmaOrig = sciencePsf.computeShape(sciAvgPos).getDeterminantRadius()
738 
739  templatePsf = templateExposure.getPsf()
740  templateAvgPos = templatePsf.getAveragePosition()
741  templateSigma = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()
742 
743  # if requested, convolve the science exposure with its PSF
744  # (properly, this should be a cross-correlation, but our code does not yet support that)
745  # compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
746  # else sigma of original science exposure
747  # TODO: DM-22762 This functional block should be moved into its own method
748  preConvPsf = None
749  if self.config.useScoreImageDetection:
750  self.log.warn("AL likelihood image: pre-convolution of PSF is not implemented.")
751  convControl = afwMath.ConvolutionControl()
752  # cannot convolve in place, so need a new image anyway
753  srcMI = exposure.maskedImage
754  exposure = exposure.clone() # New deep copy
755  srcPsf = sciencePsf
756  if self.config.useGaussianForPreConvolution:
757  self.log.infof(
758  "AL likelihood image: Using Gaussian (sigma={:.2f}) PSF estimation "
759  "for science image pre-convolution", scienceSigmaOrig)
760  # convolve with a simplified PSF model: a double Gaussian
761  kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
762  preConvPsf = SingleGaussianPsf(kWidth, kHeight, scienceSigmaOrig)
763  else:
764  # convolve with science exposure's PSF model
765  self.log.infof(
766  "AL likelihood image: Using the science image PSF for pre-convolution.")
767  preConvPsf = srcPsf
768  afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
769  scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
770  else:
771  scienceSigmaPost = scienceSigmaOrig
772 
773  # If requested, find and select sources from the image
774  # else, AL subtraction will do its own source detection
775  # TODO: DM-22762 This functional block should be moved into its own method
776  if self.config.doSelectSources:
777  if selectSources is None:
778  self.log.warning("Src product does not exist; running detection, measurement,"
779  " selection")
780  # Run own detection and measurement; necessary in nightly processing
781  selectSources = self.subtract.getSelectSources(
782  exposure,
783  sigma=scienceSigmaPost,
784  doSmooth=not self.config.useScoreImageDetection,
785  idFactory=idFactory,
786  )
787 
788  if self.config.doAddMetrics:
789  # Number of basis functions
790 
791  nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
792  referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
793  targetFwhmPix=templateSigma*FwhmPerSigma))
794  # Modify the schema of all Sources
795  # DEPRECATED: This is a data dependent (nparam) output product schema
796  # outside the task constructor.
797  # NOTE: The pre-determination of nparam at this point
798  # may be incorrect as the template psf is warped later in
799  # ImagePsfMatchTask.matchExposures()
800  kcQa = KernelCandidateQa(nparam)
801  selectSources = kcQa.addToSchema(selectSources)
802  if self.config.kernelSourcesFromRef:
803  # match exposure sources to reference catalog
804  astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
805  matches = astromRet.matches
806  elif templateSources:
807  # match exposure sources to template sources
808  mc = afwTable.MatchControl()
809  mc.findOnlyClosest = False
810  matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
811  mc)
812  else:
813  raise RuntimeError("doSelectSources=True and kernelSourcesFromRef=False,"
814  "but template sources not available. Cannot match science "
815  "sources with template sources. Run process* on data from "
816  "which templates are built.")
817 
818  kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
819  matches=matches).sourceCat
820  random.shuffle(kernelSources, random.random)
821  controlSources = kernelSources[::self.config.controlStepSize]
822  kernelSources = [k for i, k in enumerate(kernelSources)
823  if i % self.config.controlStepSize]
824 
825  if self.config.doSelectDcrCatalog:
826  redSelector = DiaCatalogSourceSelectorTask(
827  DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
828  grMax=99.999))
829  redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
830  controlSources.extend(redSources)
831 
832  blueSelector = DiaCatalogSourceSelectorTask(
833  DiaCatalogSourceSelectorConfig(grMin=-99.999,
834  grMax=self.sourceSelector.config.grMin))
835  blueSources = blueSelector.selectStars(exposure, selectSources,
836  matches=matches).starCat
837  controlSources.extend(blueSources)
838 
839  if self.config.doSelectVariableCatalog:
840  varSelector = DiaCatalogSourceSelectorTask(
841  DiaCatalogSourceSelectorConfig(includeVariable=True))
842  varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
843  controlSources.extend(varSources)
844 
845  self.log.info("Selected %d / %d sources for Psf matching (%d for control sample)",
846  len(kernelSources), len(selectSources), len(controlSources))
847 
848  allresids = {}
849  # TODO: DM-22762 This functional block should be moved into its own method
850  if self.config.doUseRegister:
851  self.log.info("Registering images")
852 
853  if templateSources is None:
854  # Run detection on the template, which is
855  # temporarily background-subtracted
856  # sigma of PSF of template image before warping
857  templateSources = self.subtract.getSelectSources(
858  templateExposure,
859  sigma=templateSigma,
860  doSmooth=True,
861  idFactory=idFactory
862  )
863 
864  # Third step: we need to fit the relative astrometry.
865  #
866  wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
867  warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
868  exposure.getWcs(), exposure.getBBox())
869  templateExposure = warpedExp
870 
871  # Create debugging outputs on the astrometric
872  # residuals as a function of position. Persistence
873  # not yet implemented; expected on (I believe) #2636.
874  if self.config.doDebugRegister:
875  # Grab matches to reference catalog
876  srcToMatch = {x.second.getId(): x.first for x in matches}
877 
878  refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
879  inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
880  sids = [m.first.getId() for m in wcsResults.matches]
881  positions = [m.first.get(refCoordKey) for m in wcsResults.matches]
882  residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
883  m.second.get(inCentroidKey))) for m in wcsResults.matches]
884  allresids = dict(zip(sids, zip(positions, residuals)))
885 
886  cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
887  wcsResults.wcs.pixelToSky(
888  m.second.get(inCentroidKey))) for m in wcsResults.matches]
889  colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get("g"))
890  + 2.5*numpy.log10(srcToMatch[x].get("r"))
891  for x in sids if x in srcToMatch.keys()])
892  dlong = numpy.array([r[0].asArcseconds() for s, r in zip(sids, cresiduals)
893  if s in srcToMatch.keys()])
894  dlat = numpy.array([r[1].asArcseconds() for s, r in zip(sids, cresiduals)
895  if s in srcToMatch.keys()])
896  idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
897  idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
898  & (colors <= self.sourceSelector.config.grMax))
899  idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
900  rms1Long = IqrToSigma*(
901  (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
902  rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
903  - numpy.percentile(dlat[idx1], 25))
904  rms2Long = IqrToSigma*(
905  (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
906  rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
907  - numpy.percentile(dlat[idx2], 25))
908  rms3Long = IqrToSigma*(
909  (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
910  rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
911  - numpy.percentile(dlat[idx3], 25))
912  self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f",
913  numpy.median(dlong[idx1]), rms1Long,
914  numpy.median(dlat[idx1]), rms1Lat)
915  self.log.info("Green star offsets'': %.3f %.3f, %.3f %.3f",
916  numpy.median(dlong[idx2]), rms2Long,
917  numpy.median(dlat[idx2]), rms2Lat)
918  self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f",
919  numpy.median(dlong[idx3]), rms3Long,
920  numpy.median(dlat[idx3]), rms3Lat)
921 
922  self.metadata.add("RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
923  self.metadata.add("RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
924  self.metadata.add("RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
925  self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
926  self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
927  self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
928 
929  self.metadata.add("RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
930  self.metadata.add("RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
931  self.metadata.add("RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
932  self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
933  self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
934  self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
935 
936  # warp template exposure to match exposure,
937  # PSF match template exposure to exposure,
938  # then return the difference
939 
940  # Return warped template... Construct sourceKernelCand list after subtract
941  self.log.info("Subtracting images")
942  subtractRes = self.subtract.subtractExposures(
943  templateExposure=templateExposure,
944  scienceExposure=exposure,
945  candidateList=kernelSources,
946  convolveTemplate=self.config.convolveTemplate,
947  doWarping=not self.config.doUseRegister
948  )
949  if self.config.useScoreImageDetection:
950  scoreExposure = subtractRes.subtractedExposure
951  else:
952  subtractedExposure = subtractRes.subtractedExposure
953 
954  if self.config.doDetection:
955  self.log.info("Computing diffim PSF")
956 
957  # Get Psf from the appropriate input image if it doesn't exist
958  if subtractedExposure is not None and not subtractedExposure.hasPsf():
959  if self.config.convolveTemplate:
960  subtractedExposure.setPsf(exposure.getPsf())
961  else:
962  subtractedExposure.setPsf(templateExposure.getPsf())
963 
964  # If doSubtract is False, then subtractedExposure was fetched from disk (above),
965  # thus it may have already been decorrelated. Thus, we do not decorrelate if
966  # doSubtract is False.
967 
968  # NOTE: At this point doSubtract == True
969  if self.config.doDecorrelation and self.config.doSubtract:
970  preConvKernel = None
971  if self.config.useGaussianForPreConvolution:
972  preConvKernel = preConvPsf.getLocalKernel()
973  if self.config.useScoreImageDetection:
974  scoreExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
975  scoreExposure,
976  subtractRes.psfMatchingKernel,
977  spatiallyVarying=self.config.doSpatiallyVarying,
978  preConvKernel=preConvKernel,
979  templateMatched=True,
980  preConvMode=True).correctedExposure
981  # Note that the subtracted exposure is always decorrelated,
982  # even if the score image is used for detection
983  subtractedExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
984  subtractedExposure,
985  subtractRes.psfMatchingKernel,
986  spatiallyVarying=self.config.doSpatiallyVarying,
987  preConvKernel=None,
988  templateMatched=self.config.convolveTemplate,
989  preConvMode=False).correctedExposure
990  # END (if subtractAlgorithm == 'AL')
991  # END (if self.config.doSubtract)
992  if self.config.doDetection:
993  self.log.info("Running diaSource detection")
994 
995  # subtractedExposure - reserved for task return value
996  # in zogy, it is always the proper difference image
997  # in AL, it may be (yet) pre-convolved and/or decorrelated
998  #
999  # detectionExposure - controls which exposure to use for detection
1000  # in-place modifications will appear in task return
1001  if self.config.useScoreImageDetection:
1002  # zogy with score image detection enabled
1003  self.log.info("Detection, diffim rescaling and measurements are "
1004  "on AL likelihood or Zogy score image.")
1005  detectionExposure = scoreExposure
1006  else:
1007  # AL or zogy with no score image detection
1008  detectionExposure = subtractedExposure
1009 
1010  # Rescale difference image variance plane
1011  if self.config.doScaleDiffimVariance:
1012  self.log.info("Rescaling diffim variance")
1013  diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
1014  self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor)
1015  self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
1016 
1017  # Erase existing detection mask planes
1018  mask = detectionExposure.getMaskedImage().getMask()
1019  mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
1020 
1021  table = afwTable.SourceTable.make(self.schema, idFactory)
1022  table.setMetadata(self.algMetadata)
1023  results = self.detection.run(
1024  table=table,
1025  exposure=detectionExposure,
1026  doSmooth=not self.config.useScoreImageDetection
1027  )
1028 
1029  if self.config.doMerge:
1030  fpSet = results.fpSets.positive
1031  fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1032  self.config.growFootprint, False)
1033  diaSources = afwTable.SourceCatalog(table)
1034  fpSet.makeSources(diaSources)
1035  self.log.info("Merging detections into %d sources", len(diaSources))
1036  else:
1037  diaSources = results.sources
1038  # Inject skySources before measurement.
1039  if self.config.doSkySources:
1040  skySourceFootprints = self.skySources.run(
1041  mask=detectionExposure.mask,
1042  seed=detectionExposure.getInfo().getVisitInfo().getExposureId())
1043  if skySourceFootprints:
1044  for foot in skySourceFootprints:
1045  s = diaSources.addNew()
1046  s.setFootprint(foot)
1047  s.set(self.skySourceKey, True)
1048 
1049  if self.config.doMeasurement:
1050  newDipoleFitting = self.config.doDipoleFitting
1051  self.log.info("Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1052  if not newDipoleFitting:
1053  # Just fit dipole in diffim
1054  self.measurement.run(diaSources, detectionExposure)
1055  else:
1056  # Use (matched) template and science image (if avail.) to constrain dipole fitting
1057  if self.config.doSubtract and 'matchedExposure' in subtractRes.getDict():
1058  self.measurement.run(diaSources, detectionExposure, exposure,
1059  subtractRes.matchedExposure)
1060  else:
1061  self.measurement.run(diaSources, detectionExposure, exposure)
1062  if self.config.doApCorr:
1063  self.applyApCorr.run(
1064  catalog=diaSources,
1065  apCorrMap=detectionExposure.getInfo().getApCorrMap()
1066  )
1067 
1068  if self.config.doForcedMeasurement:
1069  # Run forced psf photometry on the PVI at the diaSource locations.
1070  # Copy the measured flux and error into the diaSource.
1071  forcedSources = self.forcedMeasurement.generateMeasCat(
1072  exposure, diaSources, detectionExposure.getWcs())
1073  self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1074  mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1075  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFlux")[0],
1076  "ip_diffim_forced_PsfFlux_instFlux", True)
1077  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_instFluxErr")[0],
1078  "ip_diffim_forced_PsfFlux_instFluxErr", True)
1079  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_area")[0],
1080  "ip_diffim_forced_PsfFlux_area", True)
1081  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag")[0],
1082  "ip_diffim_forced_PsfFlux_flag", True)
1083  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_noGoodPixels")[0],
1084  "ip_diffim_forced_PsfFlux_flag_noGoodPixels", True)
1085  mapper.addMapping(forcedSources.schema.find("base_PsfFlux_flag_edge")[0],
1086  "ip_diffim_forced_PsfFlux_flag_edge", True)
1087  for diaSource, forcedSource in zip(diaSources, forcedSources):
1088  diaSource.assign(forcedSource, mapper)
1089 
1090  # Match with the calexp sources if possible
1091  if self.config.doMatchSources:
1092  if selectSources is not None:
1093  # Create key,val pair where key=diaSourceId and val=sourceId
1094  matchRadAsec = self.config.diaSourceMatchRadius
1095  matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1096 
1097  srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1098  srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId()) for
1099  srcMatch in srcMatches])
1100  self.log.info("Matched %d / %d diaSources to sources",
1101  len(srcMatchDict), len(diaSources))
1102  else:
1103  self.log.warning("Src product does not exist; cannot match with diaSources")
1104  srcMatchDict = {}
1105 
1106  # Create key,val pair where key=diaSourceId and val=refId
1107  refAstromConfig = AstrometryConfig()
1108  refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1109  refAstrometer = AstrometryTask(refAstromConfig)
1110  astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1111  refMatches = astromRet.matches
1112  if refMatches is None:
1113  self.log.warning("No diaSource matches with reference catalog")
1114  refMatchDict = {}
1115  else:
1116  self.log.info("Matched %d / %d diaSources to reference catalog",
1117  len(refMatches), len(diaSources))
1118  refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId()) for
1119  refMatch in refMatches])
1120 
1121  # Assign source Ids
1122  for diaSource in diaSources:
1123  sid = diaSource.getId()
1124  if sid in srcMatchDict:
1125  diaSource.set("srcMatchId", srcMatchDict[sid])
1126  if sid in refMatchDict:
1127  diaSource.set("refMatchId", refMatchDict[sid])
1128 
1129  if self.config.doAddMetrics and self.config.doSelectSources:
1130  self.log.info("Evaluating metrics and control sample")
1131 
1132  kernelCandList = []
1133  for cell in subtractRes.kernelCellSet.getCellList():
1134  for cand in cell.begin(False): # include bad candidates
1135  kernelCandList.append(cand)
1136 
1137  # Get basis list to build control sample kernels
1138  basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1139  nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1140 
1141  controlCandList = (
1142  diffimTools.sourceTableToCandidateList(controlSources,
1143  subtractRes.warpedExposure, exposure,
1144  self.config.subtract.kernel.active,
1145  self.config.subtract.kernel.active.detectionConfig,
1146  self.log, doBuild=True, basisList=basisList))
1147 
1148  KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1149  subtractRes.backgroundModel, dof=nparam)
1150  KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1151  subtractRes.backgroundModel)
1152 
1153  if self.config.doDetection:
1154  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1155  else:
1156  KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1157 
1158  self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1159  return pipeBase.Struct(
1160  subtractedExposure=subtractedExposure,
1161  scoreExposure=scoreExposure,
1162  warpedExposure=subtractRes.warpedExposure,
1163  matchedExposure=subtractRes.matchedExposure,
1164  subtractRes=subtractRes,
1165  diaSources=diaSources,
1166  selectSources=selectSources
1167  )
1168 
1169  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1170  """Fit the relative astrometry between templateSources and selectSources
1171 
1172  Todo
1173  ----
1174 
1175  Remove this method. It originally fit a new WCS to the template before calling register.run
1176  because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1177  It remains because a subtask overrides it.
1178  """
1179  results = self.register.run(templateSources, templateExposure.getWcs(),
1180  templateExposure.getBBox(), selectSources)
1181  return results
1182 
1183  def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1184  """Make debug plots and displays.
1185 
1186  Todo
1187  ----
1188  Test and update for current debug display and slot names
1189  """
1190  import lsstDebug
1191  display = lsstDebug.Info(__name__).display
1192  showSubtracted = lsstDebug.Info(__name__).showSubtracted
1193  showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
1194  showDiaSources = lsstDebug.Info(__name__).showDiaSources
1195  showDipoles = lsstDebug.Info(__name__).showDipoles
1196  maskTransparency = lsstDebug.Info(__name__).maskTransparency
1197  if display:
1198  disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1199  if not maskTransparency:
1200  maskTransparency = 0
1201  disp.setMaskTransparency(maskTransparency)
1202 
1203  if display and showSubtracted:
1204  disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
1205  mi = subtractRes.subtractedExposure.getMaskedImage()
1206  x0, y0 = mi.getX0(), mi.getY0()
1207  with disp.Buffering():
1208  for s in diaSources:
1209  x, y = s.getX() - x0, s.getY() - y0
1210  ctype = "red" if s.get("flags_negative") else "yellow"
1211  if (s.get("base_PixelFlags_flag_interpolatedCenter")
1212  or s.get("base_PixelFlags_flag_saturatedCenter")
1213  or s.get("base_PixelFlags_flag_crCenter")):
1214  ptype = "x"
1215  elif (s.get("base_PixelFlags_flag_interpolated")
1216  or s.get("base_PixelFlags_flag_saturated")
1217  or s.get("base_PixelFlags_flag_cr")):
1218  ptype = "+"
1219  else:
1220  ptype = "o"
1221  disp.dot(ptype, x, y, size=4, ctype=ctype)
1222  lsstDebug.frame += 1
1223 
1224  if display and showPixelResiduals and selectSources:
1225  nonKernelSources = []
1226  for source in selectSources:
1227  if source not in kernelSources:
1228  nonKernelSources.append(source)
1229 
1230  diUtils.plotPixelResiduals(exposure,
1231  subtractRes.warpedExposure,
1232  subtractRes.subtractedExposure,
1233  subtractRes.kernelCellSet,
1234  subtractRes.psfMatchingKernel,
1235  subtractRes.backgroundModel,
1236  nonKernelSources,
1237  self.subtract.config.kernel.active.detectionConfig,
1238  origVariance=False)
1239  diUtils.plotPixelResiduals(exposure,
1240  subtractRes.warpedExposure,
1241  subtractRes.subtractedExposure,
1242  subtractRes.kernelCellSet,
1243  subtractRes.psfMatchingKernel,
1244  subtractRes.backgroundModel,
1245  nonKernelSources,
1246  self.subtract.config.kernel.active.detectionConfig,
1247  origVariance=True)
1248  if display and showDiaSources:
1249  flagChecker = SourceFlagChecker(diaSources)
1250  isFlagged = [flagChecker(x) for x in diaSources]
1251  isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
1252  diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1253  frame=lsstDebug.frame)
1254  lsstDebug.frame += 1
1255 
1256  if display and showDipoles:
1257  DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1258  frame=lsstDebug.frame)
1259  lsstDebug.frame += 1
1260 
1261  def checkTemplateIsSufficient(self, templateExposure):
1262  """Raise NoWorkFound if template coverage < requiredTemplateFraction
1263 
1264  Parameters
1265  ----------
1266  templateExposure : `lsst.afw.image.ExposureF`
1267  The template exposure to check
1268 
1269  Raises
1270  ------
1271  NoWorkFound
1272  Raised if fraction of good pixels, defined as not having NO_DATA
1273  set, is less then the configured requiredTemplateFraction
1274  """
1275  # Count the number of pixels with the NO_DATA mask bit set
1276  # counting NaN pixels is insufficient because pixels without data are often intepolated over)
1277  pixNoData = numpy.count_nonzero(templateExposure.mask.array
1278  & templateExposure.mask.getPlaneBitMask('NO_DATA'))
1279  pixGood = templateExposure.getBBox().getArea() - pixNoData
1280  self.log.info("template has %d good pixels (%.1f%%)", pixGood,
1281  100*pixGood/templateExposure.getBBox().getArea())
1282 
1283  if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction:
1284  message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
1285  "To force subtraction, set config requiredTemplateFraction=0." % (
1286  100*pixGood/templateExposure.getBBox().getArea(),
1287  100*self.config.requiredTemplateFraction))
1288  raise pipeBase.NoWorkFound(message)
1289 
1290  def _getConfigName(self):
1291  """Return the name of the config dataset
1292  """
1293  return "%sDiff_config" % (self.config.coaddName,)
1294 
1295  def _getMetadataName(self):
1296  """Return the name of the metadata dataset
1297  """
1298  return "%sDiff_metadata" % (self.config.coaddName,)
1299 
1300  def getSchemaCatalogs(self):
1301  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1302  return {self.config.coaddName + "Diff_diaSrc": self.outputSchema}
1303 
1304  @classmethod
1305  def _makeArgumentParser(cls):
1306  """Create an argument parser
1307  """
1308  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1309  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=12345 ccd=1,2")
1310  parser.add_id_argument("--templateId", "calexp", doMakeDataRefList=True,
1311  help="Template data ID in case of calexp template,"
1312  " e.g. --templateId visit=6789")
1313  return parser
1314 
1315 
1316 class ImageDifferenceFromTemplateConnections(ImageDifferenceTaskConnections,
1317  defaultTemplates={"coaddName": "goodSeeing"}
1318  ):
1319  inputTemplate = pipeBase.connectionTypes.Input(
1320  doc=("Warped template produced by GetMultiTractCoaddTemplate"),
1321  dimensions=("instrument", "visit", "detector"),
1322  storageClass="ExposureF",
1323  name="{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}",
1324  )
1325 
1326  def __init__(self, *, config=None):
1327  super().__init__(config=config)
1328  # ImageDifferenceConnections will have removed one of these.
1329  # Make sure they're both gone, because no coadds are needed.
1330  if "coaddExposures" in self.inputs:
1331  self.inputs.remove("coaddExposures")
1332  if "dcrCoadds" in self.inputs:
1333  self.inputs.remove("dcrCoadds")
1334 
1335 
1336 class ImageDifferenceFromTemplateConfig(ImageDifferenceConfig,
1337  pipelineConnections=ImageDifferenceFromTemplateConnections):
1338  pass
1339 
1340 
1341 class ImageDifferenceFromTemplateTask(ImageDifferenceTask):
1342  ConfigClass = ImageDifferenceFromTemplateConfig
1343  _DefaultName = "imageDifference"
1344 
1345  @lsst.utils.inheritDoc(pipeBase.PipelineTask)
1346  def runQuantum(self, butlerQC, inputRefs, outputRefs):
1347  inputs = butlerQC.get(inputRefs)
1348  self.log.info("Processing %s", butlerQC.quantum.dataId)
1349  self.checkTemplateIsSufficient(inputs['inputTemplate'])
1350  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
1351  returnMaxBits=True)
1352  idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
1353 
1354  outputs = self.run(exposure=inputs['exposure'],
1355  templateExposure=inputs['inputTemplate'],
1356  idFactory=idFactory)
1357 
1358  # Consistency with runDataref gen2 handling
1359  if outputs.diaSources is None:
1360  del outputs.diaSources
1361  butlerQC.put(outputs, outputRefs)
1362 
1363 
1364 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1365  winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1366  doc="Shift stars going into RegisterTask by this amount")
1367  winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1368  doc="Perturb stars going into RegisterTask by this amount")
1369 
1370  def setDefaults(self):
1371  ImageDifferenceConfig.setDefaults(self)
1372  self.getTemplate.retarget(GetCalexpAsTemplateTask)
1373 
1374 
1375 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1376  """!Image difference Task used in the Winter 2013 data challege.
1377  Enables testing the effects of registration shifts and scatter.
1378 
1379  For use with winter 2013 simulated images:
1380  Use --templateId visit=88868666 for sparse data
1381  --templateId visit=22222200 for dense data (g)
1382  --templateId visit=11111100 for dense data (i)
1383  """
1384  ConfigClass = Winter2013ImageDifferenceConfig
1385  _DefaultName = "winter2013ImageDifference"
1386 
1387  def __init__(self, **kwargs):
1388  ImageDifferenceTask.__init__(self, **kwargs)
1389 
1390  def fitAstrometry(self, templateSources, templateExposure, selectSources):
1391  """Fit the relative astrometry between templateSources and selectSources"""
1392  if self.config.winter2013WcsShift > 0.0:
1393  offset = geom.Extent2D(self.config.winter2013WcsShift,
1394  self.config.winter2013WcsShift)
1395  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1396  for source in templateSources:
1397  centroid = source.get(cKey)
1398  source.set(cKey, centroid + offset)
1399  elif self.config.winter2013WcsRms > 0.0:
1400  cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1401  for source in templateSources:
1402  offset = geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1403  self.config.winter2013WcsRms*numpy.random.normal())
1404  centroid = source.get(cKey)
1405  source.set(cKey, centroid + offset)
1406 
1407  results = self.register.run(templateSources, templateExposure.getWcs(),
1408  templateExposure.getBBox(), selectSources)
1409  return results
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Image difference Task used in the Winter 2013 data challege.
def fitAstrometry(self, templateSources, templateExposure, selectSources)