33 import lsst.meas.extensions.trailedSources
39 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
40 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
41 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
42 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
43 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
48 from lsst.obs.base
import ExposureIdInfo
49 from lsst.utils.timer
import timeMethod
51 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
52 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
57 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
58 defaultTemplates={
"coaddName":
"deep",
63 exposure = pipeBase.connectionTypes.Input(
64 doc=
"Input science exposure to subtract from.",
65 dimensions=(
"instrument",
"visit",
"detector"),
66 storageClass=
"ExposureF",
67 name=
"{fakesType}calexp"
78 skyMap = pipeBase.connectionTypes.Input(
79 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
80 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
81 dimensions=(
"skymap", ),
82 storageClass=
"SkyMap",
84 coaddExposures = pipeBase.connectionTypes.Input(
85 doc=
"Input template to match and subtract from the exposure",
86 dimensions=(
"tract",
"patch",
"skymap",
"band"),
87 storageClass=
"ExposureF",
88 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
92 dcrCoadds = pipeBase.connectionTypes.Input(
93 doc=
"Input DCR template to match and subtract from the exposure",
94 name=
"{fakesType}dcrCoadd{warpTypeSuffix}",
95 storageClass=
"ExposureF",
96 dimensions=(
"tract",
"patch",
"skymap",
"band",
"subfilter"),
100 outputSchema = pipeBase.connectionTypes.InitOutput(
101 doc=
"Schema (as an example catalog) for output DIASource catalog.",
102 storageClass=
"SourceCatalog",
103 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
105 subtractedExposure = pipeBase.connectionTypes.Output(
106 doc=
"Output AL difference or Zogy proper difference image",
107 dimensions=(
"instrument",
"visit",
"detector"),
108 storageClass=
"ExposureF",
109 name=
"{fakesType}{coaddName}Diff_differenceExp",
111 scoreExposure = pipeBase.connectionTypes.Output(
112 doc=
"Output AL likelihood or Zogy score image",
113 dimensions=(
"instrument",
"visit",
"detector"),
114 storageClass=
"ExposureF",
115 name=
"{fakesType}{coaddName}Diff_scoreExp",
117 warpedExposure = pipeBase.connectionTypes.Output(
118 doc=
"Warped template used to create `subtractedExposure`.",
119 dimensions=(
"instrument",
"visit",
"detector"),
120 storageClass=
"ExposureF",
121 name=
"{fakesType}{coaddName}Diff_warpedExp",
123 matchedExposure = pipeBase.connectionTypes.Output(
124 doc=
"Warped template used to create `subtractedExposure`.",
125 dimensions=(
"instrument",
"visit",
"detector"),
126 storageClass=
"ExposureF",
127 name=
"{fakesType}{coaddName}Diff_matchedExp",
129 diaSources = pipeBase.connectionTypes.Output(
130 doc=
"Output detected diaSources on the difference image",
131 dimensions=(
"instrument",
"visit",
"detector"),
132 storageClass=
"SourceCatalog",
133 name=
"{fakesType}{coaddName}Diff_diaSrc",
136 def __init__(self, *, config=None):
137 super().__init__(config=config)
138 if config.coaddName ==
'dcr':
139 self.inputs.remove(
"coaddExposures")
141 self.inputs.remove(
"dcrCoadds")
142 if not config.doWriteSubtractedExp:
143 self.outputs.remove(
"subtractedExposure")
144 if not config.doWriteScoreExp:
145 self.outputs.remove(
"scoreExposure")
146 if not config.doWriteWarpedExp:
147 self.outputs.remove(
"warpedExposure")
148 if not config.doWriteMatchedExp:
149 self.outputs.remove(
"matchedExposure")
150 if not config.doWriteSources:
151 self.outputs.remove(
"diaSources")
157 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
158 pipelineConnections=ImageDifferenceTaskConnections):
159 """Config for ImageDifferenceTask
161 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
162 doc=
"Add background to calexp before processing it. "
163 "Useful as ipDiffim does background matching.")
164 doUseRegister = pexConfig.Field(dtype=bool, default=
False,
165 doc=
"Re-compute astrometry on the template. "
166 "Use image-to-image registration to align template with "
167 "science image (AL only).")
168 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
169 doc=
"Writing debugging data for doUseRegister")
170 doSelectSources = pexConfig.Field(dtype=bool, default=
False,
171 doc=
"Select stars to use for kernel fitting (AL only)")
172 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
173 doc=
"Select stars of extreme color as part "
174 "of the control sample (AL only)")
175 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
176 doc=
"Select stars that are variable to be part "
177 "of the control sample (AL only)")
178 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
179 doPreConvolve = pexConfig.Field(dtype=bool, default=
False,
180 doc=
"Not in use. Superseded by useScoreImageDetection.",
181 deprecated=
"This option superseded by useScoreImageDetection."
182 " Will be removed after v22.")
183 useScoreImageDetection = pexConfig.Field(
184 dtype=bool, default=
False, doc=
"Calculate the pre-convolved AL likelihood or "
185 "the Zogy score image. Use it for source detection (if doDetection=True).")
186 doWriteScoreExp = pexConfig.Field(
187 dtype=bool, default=
False, doc=
"Write AL likelihood or Zogy score exposure?")
188 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
189 doc=
"Scale variance of the template before PSF matching")
190 doScaleDiffimVariance = pexConfig.Field(dtype=bool, default=
True,
191 doc=
"Scale variance of the diffim before PSF matching. "
192 "You may do either this or template variance scaling, "
193 "or neither. (Doing both is a waste of CPU.)")
194 useGaussianForPreConvolution = pexConfig.Field(
195 dtype=bool, default=
False, doc=
"Use a simple gaussian PSF model for pre-convolution "
196 "(oherwise use exposure PSF)? (AL and if useScoreImageDetection=True only)")
197 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
198 doDecorrelation = pexConfig.Field(dtype=bool, default=
True,
199 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
200 "kernel convolution (AL only)? If True, also update the diffim PSF.")
201 doMerge = pexConfig.Field(dtype=bool, default=
True,
202 doc=
"Merge positive and negative diaSources with grow radius "
203 "set by growFootprint")
204 doMatchSources = pexConfig.Field(dtype=bool, default=
False,
205 doc=
"Match diaSources with input calexp sources and ref catalog sources")
206 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
207 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
208 doForcedMeasurement = pexConfig.Field(
211 doc=
"Force photometer diaSource locations on PVI?")
212 doWriteSubtractedExp = pexConfig.Field(
213 dtype=bool, default=
True, doc=
"Write difference exposure (AL and Zogy) ?")
214 doWriteWarpedExp = pexConfig.Field(
215 dtype=bool, default=
False, doc=
"Write WCS, warped template coadd exposure?")
216 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
217 doc=
"Write warped and PSF-matched template coadd exposure?")
218 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
219 doAddMetrics = pexConfig.Field(dtype=bool, default=
False,
220 doc=
"Add columns to the source table to hold analysis metrics?")
222 coaddName = pexConfig.Field(
223 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
227 convolveTemplate = pexConfig.Field(
228 doc=
"Which image gets convolved (default = template)",
232 refObjLoader = pexConfig.ConfigurableField(
233 target=LoadIndexedReferenceObjectsTask,
234 doc=
"reference object loader",
236 astrometer = pexConfig.ConfigurableField(
237 target=AstrometryTask,
238 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
240 sourceSelector = pexConfig.ConfigurableField(
241 target=ObjectSizeStarSelectorTask,
242 doc=
"Source selection algorithm",
244 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
245 decorrelate = pexConfig.ConfigurableField(
246 target=DecorrelateALKernelSpatialTask,
247 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
248 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
252 doSpatiallyVarying = pexConfig.Field(
255 doc=
"Perform A&L decorrelation on a grid across the "
256 "image in order to allow for spatial variations. Zogy does not use this option."
258 detection = pexConfig.ConfigurableField(
259 target=SourceDetectionTask,
260 doc=
"Low-threshold detection for final measurement",
262 measurement = pexConfig.ConfigurableField(
263 target=DipoleFitTask,
264 doc=
"Enable updated dipole fitting method",
266 doApCorr = lsst.pex.config.Field(
269 doc=
"Run subtask to apply aperture corrections"
271 applyApCorr = lsst.pex.config.ConfigurableField(
272 target=ApplyApCorrTask,
273 doc=
"Subtask to apply aperture corrections"
275 forcedMeasurement = pexConfig.ConfigurableField(
276 target=ForcedMeasurementTask,
277 doc=
"Subtask to force photometer PVI at diaSource location.",
279 getTemplate = pexConfig.ConfigurableField(
280 target=GetCoaddAsTemplateTask,
281 doc=
"Subtask to retrieve template exposure and sources",
283 scaleVariance = pexConfig.ConfigurableField(
284 target=ScaleVarianceTask,
285 doc=
"Subtask to rescale the variance of the template "
286 "to the statistically expected level"
288 controlStepSize = pexConfig.Field(
289 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
293 controlRandomSeed = pexConfig.Field(
294 doc=
"Random seed for shuffing the control sample",
298 register = pexConfig.ConfigurableField(
300 doc=
"Task to enable image-to-image image registration (warping)",
302 kernelSourcesFromRef = pexConfig.Field(
303 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
307 templateSipOrder = pexConfig.Field(
308 dtype=int, default=2,
309 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)"
311 growFootprint = pexConfig.Field(
312 dtype=int, default=2,
313 doc=
"Grow positive and negative footprints by this amount before merging"
315 diaSourceMatchRadius = pexConfig.Field(
316 dtype=float, default=0.5,
317 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
319 requiredTemplateFraction = pexConfig.Field(
320 dtype=float, default=0.1,
321 doc=
"Do not attempt to run task if template covers less than this fraction of pixels."
322 "Setting to 0 will always attempt image subtraction"
324 doSkySources = pexConfig.Field(
327 doc=
"Generate sky sources?",
329 skySources = pexConfig.ConfigurableField(
330 target=SkyObjectsTask,
331 doc=
"Generate sky sources",
334 def setDefaults(self):
337 self.subtract[
'al'].kernel.name =
"AL"
338 self.subtract[
'al'].kernel.active.fitForBackground =
True
339 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
340 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
343 self.detection.thresholdPolarity =
"both"
344 self.detection.thresholdValue = 5.0
345 self.detection.reEstimateBackground =
False
346 self.detection.thresholdType =
"pixel_stdev"
352 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
353 self.measurement.plugins.names |= [
'ext_trailedSources_Naive',
354 'base_LocalPhotoCalib',
357 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
358 self.forcedMeasurement.copyColumns = {
359 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
360 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
361 self.forcedMeasurement.slots.shape =
None
364 random.seed(self.controlRandomSeed)
367 pexConfig.Config.validate(self)
368 if not self.doSubtract
and not self.doDetection:
369 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
370 if self.doMeasurement
and not self.doDetection:
371 raise ValueError(
"Cannot run source measurement without source detection.")
372 if self.doMerge
and not self.doDetection:
373 raise ValueError(
"Cannot run source merging without source detection.")
374 if self.doSkySources
and not self.doDetection:
375 raise ValueError(
"Cannot run sky source creation without source detection.")
376 if self.doUseRegister
and not self.doSelectSources:
377 raise ValueError(
"doUseRegister=True and doSelectSources=False. "
378 "Cannot run RegisterTask without selecting sources.")
379 if hasattr(self.getTemplate,
"coaddName"):
380 if self.getTemplate.coaddName != self.coaddName:
381 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
382 if self.doScaleDiffimVariance
and self.doScaleTemplateVariance:
383 raise ValueError(
"Scaling the diffim variance and scaling the template variance "
384 "are both set. Please choose one or the other.")
386 if self.subtract.name ==
'zogy':
387 if self.doWriteMatchedExp:
388 raise ValueError(
"doWriteMatchedExp=True Matched exposure is not "
389 "calculated in zogy subtraction.")
390 if self.doAddMetrics:
391 raise ValueError(
"doAddMetrics=True Kernel metrics does not exist in zogy subtraction.")
392 if self.doDecorrelation:
394 "doDecorrelation=True The decorrelation afterburner does not exist in zogy subtraction.")
395 if self.doSelectSources:
397 "doSelectSources=True Selecting sources for PSF matching is not a zogy option.")
398 if self.useGaussianForPreConvolution:
400 "useGaussianForPreConvolution=True This is an AL subtraction only option.")
403 if self.useScoreImageDetection
and not self.convolveTemplate:
405 "convolveTemplate=False and useScoreImageDetection=True "
406 "Pre-convolution and matching of the science image is not a supported operation.")
407 if self.doWriteSubtractedExp
and self.useScoreImageDetection:
409 "doWriteSubtractedExp=True and useScoreImageDetection=True "
410 "Regular difference image is not calculated. "
411 "AL subtraction calculates either the regular difference image or the score image.")
412 if self.doWriteScoreExp
and not self.useScoreImageDetection:
414 "doWriteScoreExp=True and useScoreImageDetection=False "
415 "Score image is not calculated. "
416 "AL subtraction calculates either the regular difference image or the score image.")
417 if self.doAddMetrics
and not self.doSubtract:
418 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
419 if self.useGaussianForPreConvolution
and not self.useScoreImageDetection:
421 "useGaussianForPreConvolution=True and useScoreImageDetection=False "
422 "Gaussian PSF approximation exists only for AL subtraction w/ pre-convolution.")
425 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
428 def getTargetList(parsedCmd, **kwargs):
429 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
433 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
434 """Subtract an image from a template and measure the result
436 ConfigClass = ImageDifferenceConfig
437 RunnerClass = ImageDifferenceTaskRunner
438 _DefaultName =
"imageDifference"
440 def __init__(self, butler=None, **kwargs):
441 """!Construct an ImageDifference Task
443 @param[in] butler Butler object to use in constructing reference object loaders
445 super().__init__(**kwargs)
446 self.makeSubtask(
"getTemplate")
448 self.makeSubtask(
"subtract")
450 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
451 self.makeSubtask(
"decorrelate")
453 if self.config.doScaleTemplateVariance
or self.config.doScaleDiffimVariance:
454 self.makeSubtask(
"scaleVariance")
456 if self.config.doUseRegister:
457 self.makeSubtask(
"register")
458 self.schema = afwTable.SourceTable.makeMinimalSchema()
460 if self.config.doSelectSources:
461 self.makeSubtask(
"sourceSelector")
462 if self.config.kernelSourcesFromRef:
463 self.makeSubtask(
'refObjLoader', butler=butler)
464 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
466 self.algMetadata = dafBase.PropertyList()
467 if self.config.doDetection:
468 self.makeSubtask(
"detection", schema=self.schema)
469 if self.config.doMeasurement:
470 self.makeSubtask(
"measurement", schema=self.schema,
471 algMetadata=self.algMetadata)
472 if self.config.doApCorr:
473 self.makeSubtask(
"applyApCorr", schema=self.measurement.schema)
474 if self.config.doForcedMeasurement:
475 self.schema.addField(
476 "ip_diffim_forced_PsfFlux_instFlux",
"D",
477 "Forced PSF flux measured on the direct image.",
479 self.schema.addField(
480 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
481 "Forced PSF flux error measured on the direct image.",
483 self.schema.addField(
484 "ip_diffim_forced_PsfFlux_area",
"F",
485 "Forced PSF flux effective area of PSF.",
487 self.schema.addField(
488 "ip_diffim_forced_PsfFlux_flag",
"Flag",
489 "Forced PSF flux general failure flag.")
490 self.schema.addField(
491 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
492 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
493 self.schema.addField(
494 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
495 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
496 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
497 if self.config.doMatchSources:
498 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
499 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
500 if self.config.doSkySources:
501 self.makeSubtask(
"skySources")
502 self.skySourceKey = self.schema.addField(
"sky_source", type=
"Flag", doc=
"Sky objects.")
505 self.outputSchema = afwTable.SourceCatalog(self.schema)
506 self.outputSchema.getTable().setMetadata(self.algMetadata)
509 def makeIdFactory(expId, expBits):
510 """Create IdFactory instance for unique 64 bit diaSource id-s.
518 Number of used bits in ``expId``.
522 The diasource id-s consists of the ``expId`` stored fixed in the highest value
523 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
524 low value end of the integer.
528 idFactory: `lsst.afw.table.IdFactory`
530 return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
532 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
533 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
534 inputRefs: pipeBase.InputQuantizedConnection,
535 outputRefs: pipeBase.OutputQuantizedConnection):
536 inputs = butlerQC.get(inputRefs)
537 self.log.info(
"Processing %s", butlerQC.quantum.dataId)
538 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
540 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
541 if self.config.coaddName ==
'dcr':
542 templateExposures = inputRefs.dcrCoadds
544 templateExposures = inputRefs.coaddExposures
545 templateStruct = self.getTemplate.runQuantum(
546 inputs[
'exposure'], butlerQC, inputRefs.skyMap, templateExposures
549 self.checkTemplateIsSufficient(templateStruct.exposure)
551 outputs = self.run(exposure=inputs[
'exposure'],
552 templateExposure=templateStruct.exposure,
555 if outputs.diaSources
is None:
556 del outputs.diaSources
557 butlerQC.put(outputs, outputRefs)
560 def runDataRef(self, sensorRef, templateIdList=None):
561 """Subtract an image from a template coadd and measure the result.
563 Data I/O wrapper around `run` using the butler in Gen2.
567 sensorRef : `lsst.daf.persistence.ButlerDataRef`
568 Sensor-level butler data reference, used for the following data products:
575 - self.config.coaddName + "Coadd_skyMap"
576 - self.config.coaddName + "Coadd"
577 Input or output, depending on config:
578 - self.config.coaddName + "Diff_subtractedExp"
579 Output, depending on config:
580 - self.config.coaddName + "Diff_matchedExp"
581 - self.config.coaddName + "Diff_src"
585 results : `lsst.pipe.base.Struct`
586 Returns the Struct by `run`.
588 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp"
589 subtractedExposure =
None
591 calexpBackgroundExposure =
None
592 self.log.info(
"Processing %s", sensorRef.dataId)
597 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
598 expBits=sensorRef.get(
"ccdExposureId_bits"))
599 if self.config.doAddCalexpBackground:
600 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
603 exposure = sensorRef.get(
"calexp", immediate=
True)
606 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
608 if sensorRef.datasetExists(
"src"):
609 self.log.info(
"Source selection via src product")
611 selectSources = sensorRef.get(
"src")
613 if not self.config.doSubtract
and self.config.doDetection:
615 subtractedExposure = sensorRef.get(subtractedExposureName)
618 results = self.run(exposure=exposure,
619 selectSources=selectSources,
620 templateExposure=template.exposure,
621 templateSources=template.sources,
623 calexpBackgroundExposure=calexpBackgroundExposure,
624 subtractedExposure=subtractedExposure)
626 if self.config.doWriteSources
and results.diaSources
is not None:
627 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
628 if self.config.doWriteWarpedExp:
629 sensorRef.put(results.warpedExposure, self.config.coaddName +
"Diff_warpedExp")
630 if self.config.doWriteMatchedExp:
631 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
632 if self.config.doAddMetrics
and self.config.doSelectSources:
633 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
634 if self.config.doWriteSubtractedExp:
635 sensorRef.put(results.subtractedExposure, subtractedExposureName)
636 if self.config.doWriteScoreExp:
637 sensorRef.put(results.scoreExposure, self.config.coaddName +
"Diff_scoreExp")
641 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
642 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
643 """PSF matches, subtract two images and perform detection on the difference image.
647 exposure : `lsst.afw.image.ExposureF`, optional
648 The science exposure, the minuend in the image subtraction.
649 Can be None only if ``config.doSubtract==False``.
650 selectSources : `lsst.afw.table.SourceCatalog`, optional
651 Identified sources on the science exposure. This catalog is used to
652 select sources in order to perform the AL PSF matching on stamp images
653 around them. The selection steps depend on config options and whether
654 ``templateSources`` and ``matchingSources`` specified.
655 templateExposure : `lsst.afw.image.ExposureF`, optional
656 The template to be subtracted from ``exposure`` in the image subtraction.
657 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``.
658 The template exposure should cover the same sky area as the science exposure.
659 It is either a stich of patches of a coadd skymap image or a calexp
660 of the same pointing as the science exposure. Can be None only
661 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
662 templateSources : `lsst.afw.table.SourceCatalog`, optional
663 Identified sources on the template exposure.
664 idFactory : `lsst.afw.table.IdFactory`
665 Generator object to assign ids to detected sources in the difference image.
666 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
667 Background exposure to be added back to the science exposure
668 if ``config.doAddCalexpBackground==True``
669 subtractedExposure : `lsst.afw.image.ExposureF`, optional
670 If ``config.doSubtract==False`` and ``config.doDetection==True``,
671 performs the post subtraction source detection only on this exposure.
672 Otherwise should be None.
676 results : `lsst.pipe.base.Struct`
677 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
679 ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None`
680 The zogy score exposure, if calculated.
681 ``matchedExposure`` : `lsst.afw.image.ExposureF`
682 The matched PSF exposure.
683 ``subtractRes`` : `lsst.pipe.base.Struct`
684 The returned result structure of the ImagePsfMatchTask subtask.
685 ``diaSources`` : `lsst.afw.table.SourceCatalog`
686 The catalog of detected sources.
687 ``selectSources`` : `lsst.afw.table.SourceCatalog`
688 The input source catalog with optionally added Qa information.
692 The following major steps are included:
694 - warp template coadd to match WCS of image
695 - PSF match image to warped template
696 - subtract image from PSF-matched, warped template
700 For details about the image subtraction configuration modes
701 see `lsst.ip.diffim`.
704 controlSources =
None
705 subtractedExposure =
None
710 exposureOrig = exposure
712 if self.config.doAddCalexpBackground:
713 mi = exposure.getMaskedImage()
714 mi += calexpBackgroundExposure.getImage()
716 if not exposure.hasPsf():
717 raise pipeBase.TaskError(
"Exposure has no psf")
718 sciencePsf = exposure.getPsf()
720 if self.config.doSubtract:
721 if self.config.doScaleTemplateVariance:
722 self.log.info(
"Rescaling template variance")
723 templateVarFactor = self.scaleVariance.run(
724 templateExposure.getMaskedImage())
725 self.log.info(
"Template variance scaling factor: %.2f", templateVarFactor)
726 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
727 self.metadata.add(
"psfMatchingAlgorithm", self.config.subtract.name)
729 if self.config.subtract.name ==
'zogy':
730 subtractRes = self.subtract.run(exposure, templateExposure, doWarping=
True)
731 scoreExposure = subtractRes.scoreExp
732 subtractedExposure = subtractRes.diffExp
733 subtractRes.subtractedExposure = subtractedExposure
734 subtractRes.matchedExposure =
None
736 elif self.config.subtract.name ==
'al':
738 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
739 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
747 if self.config.useScoreImageDetection:
748 self.log.warn(
"AL likelihood image: pre-convolution of PSF is not implemented.")
749 convControl = afwMath.ConvolutionControl()
751 srcMI = exposure.maskedImage
752 exposure = exposure.clone()
754 if self.config.useGaussianForPreConvolution:
756 "AL likelihood image: Using Gaussian (sigma={:.2f}) PSF estimation "
757 "for science image pre-convolution", scienceSigmaOrig)
759 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
764 "AL likelihood image: Using the science image PSF for pre-convolution.")
766 afwMath.convolve(exposure.maskedImage, srcMI, preConvPsf.getLocalKernel(), convControl)
767 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
769 scienceSigmaPost = scienceSigmaOrig
774 if self.config.doSelectSources:
775 if selectSources
is None:
776 self.log.warning(
"Src product does not exist; running detection, measurement,"
779 selectSources = self.subtract.getSelectSources(
781 sigma=scienceSigmaPost,
782 doSmooth=
not self.config.useScoreImageDetection,
786 if self.config.doAddMetrics:
789 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
790 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
791 targetFwhmPix=templateSigma*FwhmPerSigma))
798 kcQa = KernelCandidateQa(nparam)
799 selectSources = kcQa.addToSchema(selectSources)
800 if self.config.kernelSourcesFromRef:
802 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
803 matches = astromRet.matches
804 elif templateSources:
806 mc = afwTable.MatchControl()
807 mc.findOnlyClosest =
False
808 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
811 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
812 "but template sources not available. Cannot match science "
813 "sources with template sources. Run process* on data from "
814 "which templates are built.")
816 kernelSources = self.sourceSelector.run(selectSources, exposure=exposure,
817 matches=matches).sourceCat
818 random.shuffle(kernelSources, random.random)
819 controlSources = kernelSources[::self.config.controlStepSize]
820 kernelSources = [k
for i, k
in enumerate(kernelSources)
821 if i % self.config.controlStepSize]
823 if self.config.doSelectDcrCatalog:
824 redSelector = DiaCatalogSourceSelectorTask(
825 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
827 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
828 controlSources.extend(redSources)
830 blueSelector = DiaCatalogSourceSelectorTask(
831 DiaCatalogSourceSelectorConfig(grMin=-99.999,
832 grMax=self.sourceSelector.config.grMin))
833 blueSources = blueSelector.selectStars(exposure, selectSources,
834 matches=matches).starCat
835 controlSources.extend(blueSources)
837 if self.config.doSelectVariableCatalog:
838 varSelector = DiaCatalogSourceSelectorTask(
839 DiaCatalogSourceSelectorConfig(includeVariable=
True))
840 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
841 controlSources.extend(varSources)
843 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)",
844 len(kernelSources), len(selectSources), len(controlSources))
848 if self.config.doUseRegister:
849 self.log.info(
"Registering images")
851 if templateSources
is None:
855 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
856 templateSources = self.subtract.getSelectSources(
865 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
866 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
867 exposure.getWcs(), exposure.getBBox())
868 templateExposure = warpedExp
873 if self.config.doDebugRegister:
875 srcToMatch = {x.second.getId(): x.first
for x
in matches}
877 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
878 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidSlot().getMeasKey()
879 sids = [m.first.getId()
for m
in wcsResults.matches]
880 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
881 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
882 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
883 allresids = dict(zip(sids, zip(positions, residuals)))
885 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
886 wcsResults.wcs.pixelToSky(
887 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
888 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g"))
889 + 2.5*numpy.log10(srcToMatch[x].get(
"r"))
890 for x
in sids
if x
in srcToMatch.keys()])
891 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
892 if s
in srcToMatch.keys()])
893 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
894 if s
in srcToMatch.keys()])
895 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
896 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
897 & (colors <= self.sourceSelector.config.grMax))
898 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
899 rms1Long = IqrToSigma*(
900 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
901 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
902 - numpy.percentile(dlat[idx1], 25))
903 rms2Long = IqrToSigma*(
904 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
905 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
906 - numpy.percentile(dlat[idx2], 25))
907 rms3Long = IqrToSigma*(
908 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
909 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
910 - numpy.percentile(dlat[idx3], 25))
911 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f",
912 numpy.median(dlong[idx1]), rms1Long,
913 numpy.median(dlat[idx1]), rms1Lat)
914 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f",
915 numpy.median(dlong[idx2]), rms2Long,
916 numpy.median(dlat[idx2]), rms2Lat)
917 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f",
918 numpy.median(dlong[idx3]), rms3Long,
919 numpy.median(dlat[idx3]), rms3Lat)
921 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
922 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
923 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
924 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
925 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
926 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
928 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
929 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
930 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
931 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
932 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
933 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
940 self.log.info(
"Subtracting images")
941 subtractRes = self.subtract.subtractExposures(
942 templateExposure=templateExposure,
943 scienceExposure=exposure,
944 candidateList=kernelSources,
945 convolveTemplate=self.config.convolveTemplate,
946 doWarping=
not self.config.doUseRegister
948 if self.config.useScoreImageDetection:
949 scoreExposure = subtractRes.subtractedExposure
951 subtractedExposure = subtractRes.subtractedExposure
953 if self.config.doDetection:
954 self.log.info(
"Computing diffim PSF")
957 if subtractedExposure
is not None and not subtractedExposure.hasPsf():
958 if self.config.convolveTemplate:
959 subtractedExposure.setPsf(exposure.getPsf())
961 subtractedExposure.setPsf(templateExposure.getPsf())
968 if self.config.doDecorrelation
and self.config.doSubtract:
970 if self.config.useGaussianForPreConvolution:
971 preConvKernel = preConvPsf.getLocalKernel()
972 if self.config.useScoreImageDetection:
973 scoreExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
975 subtractRes.psfMatchingKernel,
976 spatiallyVarying=self.config.doSpatiallyVarying,
977 preConvKernel=preConvKernel,
978 templateMatched=
True,
979 preConvMode=
True).correctedExposure
982 subtractedExposure = self.decorrelate.run(exposureOrig, subtractRes.warpedExposure,
984 subtractRes.psfMatchingKernel,
985 spatiallyVarying=self.config.doSpatiallyVarying,
987 templateMatched=self.config.convolveTemplate,
988 preConvMode=
False).correctedExposure
991 if self.config.doDetection:
992 self.log.info(
"Running diaSource detection")
1000 if self.config.useScoreImageDetection:
1002 self.log.info(
"Detection, diffim rescaling and measurements are "
1003 "on AL likelihood or Zogy score image.")
1004 detectionExposure = scoreExposure
1007 detectionExposure = subtractedExposure
1010 if self.config.doScaleDiffimVariance:
1011 self.log.info(
"Rescaling diffim variance")
1012 diffimVarFactor = self.scaleVariance.run(detectionExposure.getMaskedImage())
1013 self.log.info(
"Diffim variance scaling factor: %.2f", diffimVarFactor)
1014 self.metadata.add(
"scaleDiffimVarianceFactor", diffimVarFactor)
1017 mask = detectionExposure.getMaskedImage().getMask()
1018 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
1020 table = afwTable.SourceTable.make(self.schema, idFactory)
1021 table.setMetadata(self.algMetadata)
1022 results = self.detection.run(
1024 exposure=detectionExposure,
1025 doSmooth=
not self.config.useScoreImageDetection
1028 if self.config.doMerge:
1029 fpSet = results.fpSets.positive
1030 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
1031 self.config.growFootprint,
False)
1032 diaSources = afwTable.SourceCatalog(table)
1033 fpSet.makeSources(diaSources)
1034 self.log.info(
"Merging detections into %d sources", len(diaSources))
1036 diaSources = results.sources
1038 if self.config.doSkySources:
1039 skySourceFootprints = self.skySources.run(
1040 mask=detectionExposure.mask,
1041 seed=detectionExposure.info.id)
1042 if skySourceFootprints:
1043 for foot
in skySourceFootprints:
1044 s = diaSources.addNew()
1045 s.setFootprint(foot)
1046 s.set(self.skySourceKey,
True)
1048 if self.config.doMeasurement:
1049 newDipoleFitting = self.config.doDipoleFitting
1050 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
1051 if not newDipoleFitting:
1053 self.measurement.run(diaSources, detectionExposure)
1056 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
1057 self.measurement.run(diaSources, detectionExposure, exposure,
1058 subtractRes.matchedExposure)
1060 self.measurement.run(diaSources, detectionExposure, exposure)
1061 if self.config.doApCorr:
1062 self.applyApCorr.run(
1064 apCorrMap=detectionExposure.getInfo().getApCorrMap()
1067 if self.config.doForcedMeasurement:
1070 forcedSources = self.forcedMeasurement.generateMeasCat(
1071 exposure, diaSources, detectionExposure.getWcs())
1072 self.forcedMeasurement.run(forcedSources, exposure, diaSources, detectionExposure.getWcs())
1073 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
1074 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
1075 "ip_diffim_forced_PsfFlux_instFlux",
True)
1076 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
1077 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
1078 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
1079 "ip_diffim_forced_PsfFlux_area",
True)
1080 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
1081 "ip_diffim_forced_PsfFlux_flag",
True)
1082 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
1083 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
1084 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
1085 "ip_diffim_forced_PsfFlux_flag_edge",
True)
1086 for diaSource, forcedSource
in zip(diaSources, forcedSources):
1087 diaSource.assign(forcedSource, mapper)
1090 if self.config.doMatchSources:
1091 if selectSources
is not None:
1093 matchRadAsec = self.config.diaSourceMatchRadius
1094 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
1096 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
1097 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for
1098 srcMatch
in srcMatches])
1099 self.log.info(
"Matched %d / %d diaSources to sources",
1100 len(srcMatchDict), len(diaSources))
1102 self.log.warning(
"Src product does not exist; cannot match with diaSources")
1106 refAstromConfig = AstrometryConfig()
1107 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
1108 refAstrometer = AstrometryTask(refAstromConfig)
1109 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
1110 refMatches = astromRet.matches
1111 if refMatches
is None:
1112 self.log.warning(
"No diaSource matches with reference catalog")
1115 self.log.info(
"Matched %d / %d diaSources to reference catalog",
1116 len(refMatches), len(diaSources))
1117 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for
1118 refMatch
in refMatches])
1121 for diaSource
in diaSources:
1122 sid = diaSource.getId()
1123 if sid
in srcMatchDict:
1124 diaSource.set(
"srcMatchId", srcMatchDict[sid])
1125 if sid
in refMatchDict:
1126 diaSource.set(
"refMatchId", refMatchDict[sid])
1128 if self.config.doAddMetrics
and self.config.doSelectSources:
1129 self.log.info(
"Evaluating metrics and control sample")
1132 for cell
in subtractRes.kernelCellSet.getCellList():
1133 for cand
in cell.begin(
False):
1134 kernelCandList.append(cand)
1137 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
1138 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
1141 diffimTools.sourceTableToCandidateList(controlSources,
1142 subtractRes.warpedExposure, exposure,
1143 self.config.subtract.kernel.active,
1144 self.config.subtract.kernel.active.detectionConfig,
1145 self.log, doBuild=
True, basisList=basisList))
1147 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
1148 subtractRes.backgroundModel, dof=nparam)
1149 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
1150 subtractRes.backgroundModel)
1152 if self.config.doDetection:
1153 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
1155 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
1157 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
1158 return pipeBase.Struct(
1159 subtractedExposure=subtractedExposure,
1160 scoreExposure=scoreExposure,
1161 warpedExposure=subtractRes.warpedExposure,
1162 matchedExposure=subtractRes.matchedExposure,
1163 subtractRes=subtractRes,
1164 diaSources=diaSources,
1165 selectSources=selectSources
1168 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1169 """Fit the relative astrometry between templateSources and selectSources
1174 Remove this method. It originally fit a new WCS to the template before calling register.run
1175 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
1176 It remains because a subtask overrides it.
1178 results = self.register.run(templateSources, templateExposure.getWcs(),
1179 templateExposure.getBBox(), selectSources)
1182 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1183 """Make debug plots and displays.
1187 Test and update for current debug display and slot names
1197 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1198 if not maskTransparency:
1199 maskTransparency = 0
1200 disp.setMaskTransparency(maskTransparency)
1202 if display
and showSubtracted:
1203 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
1204 mi = subtractRes.subtractedExposure.getMaskedImage()
1205 x0, y0 = mi.getX0(), mi.getY0()
1206 with disp.Buffering():
1207 for s
in diaSources:
1208 x, y = s.getX() - x0, s.getY() - y0
1209 ctype =
"red" if s.get(
"flags_negative")
else "yellow"
1210 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
1211 or s.get(
"base_PixelFlags_flag_saturatedCenter")
1212 or s.get(
"base_PixelFlags_flag_crCenter")):
1214 elif (s.get(
"base_PixelFlags_flag_interpolated")
1215 or s.get(
"base_PixelFlags_flag_saturated")
1216 or s.get(
"base_PixelFlags_flag_cr")):
1220 disp.dot(ptype, x, y, size=4, ctype=ctype)
1221 lsstDebug.frame += 1
1223 if display
and showPixelResiduals
and selectSources:
1224 nonKernelSources = []
1225 for source
in selectSources:
1226 if source
not in kernelSources:
1227 nonKernelSources.append(source)
1229 diUtils.plotPixelResiduals(exposure,
1230 subtractRes.warpedExposure,
1231 subtractRes.subtractedExposure,
1232 subtractRes.kernelCellSet,
1233 subtractRes.psfMatchingKernel,
1234 subtractRes.backgroundModel,
1236 self.subtract.config.kernel.active.detectionConfig,
1238 diUtils.plotPixelResiduals(exposure,
1239 subtractRes.warpedExposure,
1240 subtractRes.subtractedExposure,
1241 subtractRes.kernelCellSet,
1242 subtractRes.psfMatchingKernel,
1243 subtractRes.backgroundModel,
1245 self.subtract.config.kernel.active.detectionConfig,
1247 if display
and showDiaSources:
1248 flagChecker = SourceFlagChecker(diaSources)
1249 isFlagged = [flagChecker(x)
for x
in diaSources]
1250 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1251 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1252 frame=lsstDebug.frame)
1253 lsstDebug.frame += 1
1255 if display
and showDipoles:
1256 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1257 frame=lsstDebug.frame)
1258 lsstDebug.frame += 1
1260 def checkTemplateIsSufficient(self, templateExposure):
1261 """Raise NoWorkFound if template coverage < requiredTemplateFraction
1265 templateExposure : `lsst.afw.image.ExposureF`
1266 The template exposure to check
1271 Raised if fraction of good pixels, defined as not having NO_DATA
1272 set, is less then the configured requiredTemplateFraction
1276 pixNoData = numpy.count_nonzero(templateExposure.mask.array
1277 & templateExposure.mask.getPlaneBitMask(
'NO_DATA'))
1278 pixGood = templateExposure.getBBox().getArea() - pixNoData
1279 self.log.info(
"template has %d good pixels (%.1f%%)", pixGood,
1280 100*pixGood/templateExposure.getBBox().getArea())
1282 if pixGood/templateExposure.getBBox().getArea() < self.config.requiredTemplateFraction:
1283 message = (
"Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
1284 "To force subtraction, set config requiredTemplateFraction=0." % (
1285 100*pixGood/templateExposure.getBBox().getArea(),
1286 100*self.config.requiredTemplateFraction))
1287 raise pipeBase.NoWorkFound(message)
1289 def _getConfigName(self):
1290 """Return the name of the config dataset
1292 return "%sDiff_config" % (self.config.coaddName,)
1294 def _getMetadataName(self):
1295 """Return the name of the metadata dataset
1297 return "%sDiff_metadata" % (self.config.coaddName,)
1299 def getSchemaCatalogs(self):
1300 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1301 return {self.config.coaddName +
"Diff_diaSrc": self.outputSchema}
1304 def _makeArgumentParser(cls):
1305 """Create an argument parser
1307 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1308 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1309 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1310 help=
"Template data ID in case of calexp template,"
1311 " e.g. --templateId visit=6789")
1316 defaultTemplates={
"coaddName":
"goodSeeing"}
1318 inputTemplate = pipeBase.connectionTypes.Input(
1319 doc=(
"Warped template produced by GetMultiTractCoaddTemplate"),
1320 dimensions=(
"instrument",
"visit",
"detector"),
1321 storageClass=
"ExposureF",
1322 name=
"{fakesType}{coaddName}Diff_templateExp{warpTypeSuffix}",
1325 def __init__(self, *, config=None):
1326 super().__init__(config=config)
1329 if "coaddExposures" in self.inputs:
1330 self.inputs.remove(
"coaddExposures")
1331 if "dcrCoadds" in self.inputs:
1332 self.inputs.remove(
"dcrCoadds")
1336 pipelineConnections=ImageDifferenceFromTemplateConnections):
1341 ConfigClass = ImageDifferenceFromTemplateConfig
1342 _DefaultName =
"imageDifference"
1344 @lsst.utils.inheritDoc(pipeBase.PipelineTask)
1346 inputs = butlerQC.get(inputRefs)
1347 self.log.info(
"Processing %s", butlerQC.quantum.dataId)
1348 self.checkTemplateIsSufficient(inputs[
'inputTemplate'])
1349 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
1351 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
1353 outputs = self.run(exposure=inputs[
'exposure'],
1354 templateExposure=inputs[
'inputTemplate'],
1355 idFactory=idFactory)
1358 if outputs.diaSources
is None:
1359 del outputs.diaSources
1360 butlerQC.put(outputs, outputRefs)
1364 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1365 doc=
"Shift stars going into RegisterTask by this amount")
1366 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1367 doc=
"Perturb stars going into RegisterTask by this amount")
1370 ImageDifferenceConfig.setDefaults(self)
1371 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1375 """!Image difference Task used in the Winter 2013 data challege.
1376 Enables testing the effects of registration shifts and scatter.
1378 For use with winter 2013 simulated images:
1379 Use --templateId visit=88868666 for sparse data
1380 --templateId visit=22222200 for dense data (g)
1381 --templateId visit=11111100 for dense data (i)
1383 ConfigClass = Winter2013ImageDifferenceConfig
1384 _DefaultName =
"winter2013ImageDifference"
1387 ImageDifferenceTask.__init__(self, **kwargs)
1390 """Fit the relative astrometry between templateSources and selectSources"""
1391 if self.config.winter2013WcsShift > 0.0:
1393 self.config.winter2013WcsShift)
1394 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1395 for source
in templateSources:
1396 centroid = source.get(cKey)
1397 source.set(cKey, centroid + offset)
1398 elif self.config.winter2013WcsRms > 0.0:
1399 cKey = templateSources[0].getTable().getCentroidSlot().getMeasKey()
1400 for source
in templateSources:
1401 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1402 self.config.winter2013WcsRms*numpy.random.normal())
1403 centroid = source.get(cKey)
1404 source.set(cKey, centroid + offset)
1406 results = self.register.run(templateSources, templateExposure.getWcs(),
1407 templateExposure.getBBox(), selectSources)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Image difference Task used in the Winter 2013 data challege.
def __init__(self, **kwargs)
def fitAstrometry(self, templateSources, templateExposure, selectSources)