27 import lsst.pex.config
as pexConfig
38 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
39 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
40 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
41 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
42 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
47 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
48 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
53 dimensions=(
"instrument",
"visit",
"detector",
"skymap"),
54 defaultTemplates={
"coaddName":
"deep",
59 exposure = pipeBase.connectionTypes.Input(
60 doc=
"Input science exposure to subtract from.",
61 dimensions=(
"instrument",
"visit",
"detector"),
62 storageClass=
"ExposureF",
74 skyMap = pipeBase.connectionTypes.Input(
75 doc=
"Input definition of geometry/bbox and projection/wcs for template exposures",
76 name=
"{skyMapName}Coadd_skyMap",
77 dimensions=(
"skymap", ),
78 storageClass=
"SkyMap",
80 coaddExposures = pipeBase.connectionTypes.Input(
81 doc=
"Input template to match and subtract from the exposure",
82 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter"),
83 storageClass=
"ExposureF",
84 name=
"{fakesType}{coaddName}Coadd{warpTypeSuffix}",
88 dcrCoadds = pipeBase.connectionTypes.Input(
89 doc=
"Input DCR template to match and subtract from the exposure",
90 name=
"{fakesType}dcrCoadd{warpTypeSuffix}",
91 storageClass=
"ExposureF",
92 dimensions=(
"tract",
"patch",
"skymap",
"abstract_filter",
"subfilter"),
96 outputSchema = pipeBase.connectionTypes.InitOutput(
97 doc=
"Schema (as an example catalog) for output DIASource catalog.",
98 storageClass=
"SourceCatalog",
99 name=
"{fakesType}{coaddName}Diff_diaSrc_schema",
101 subtractedExposure = pipeBase.connectionTypes.Output(
102 doc=
"Output difference image",
103 dimensions=(
"instrument",
"visit",
"detector"),
104 storageClass=
"ExposureF",
105 name=
"{fakesType}{coaddName}Diff_differenceExp",
107 diaSources = pipeBase.connectionTypes.Output(
108 doc=
"Output detected diaSources on the difference image",
109 dimensions=(
"instrument",
"visit",
"detector"),
110 storageClass=
"SourceCatalog",
111 name=
"{fakesType}{coaddName}Diff_diaSrc",
114 def __init__(self, *, config=None):
115 super().__init__(config=config)
116 if config.coaddName ==
'dcr':
117 self.inputs.remove(
"coaddExposures")
119 self.inputs.remove(
"dcrCoadds")
125 class ImageDifferenceConfig(pipeBase.PipelineTaskConfig,
126 pipelineConnections=ImageDifferenceTaskConnections):
127 """Config for ImageDifferenceTask
129 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
130 doc=
"Add background to calexp before processing it. "
131 "Useful as ipDiffim does background matching.")
132 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
133 doc=
"Use image-to-image registration to align template with "
135 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
136 doc=
"Writing debugging data for doUseRegister")
137 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
138 doc=
"Select stars to use for kernel fitting")
139 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
140 doc=
"Select stars of extreme color as part of the control sample")
141 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
142 doc=
"Select stars that are variable to be part "
143 "of the control sample")
144 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
145 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
146 doc=
"Convolve science image by its PSF before PSF-matching?")
147 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
148 doc=
"Scale variance of the template before PSF matching")
149 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
150 doc=
"Use a simple gaussian PSF model for pre-convolution "
151 "(else use fit PSF)? Ignored if doPreConvolve false.")
152 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
153 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
154 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
155 "kernel convolution? If True, also update the diffim PSF.")
156 doMerge = pexConfig.Field(dtype=bool, default=
True,
157 doc=
"Merge positive and negative diaSources with grow radius "
158 "set by growFootprint")
159 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
160 doc=
"Match diaSources with input calexp sources and ref catalog sources")
161 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
162 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
163 doForcedMeasurement = pexConfig.Field(
166 doc=
"Force photometer diaSource locations on PVI?")
167 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
168 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
169 doc=
"Write warped and PSF-matched template coadd exposure?")
170 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
171 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
172 doc=
"Add columns to the source table to hold analysis metrics?")
174 coaddName = pexConfig.Field(
175 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
179 convolveTemplate = pexConfig.Field(
180 doc=
"Which image gets convolved (default = template)",
184 refObjLoader = pexConfig.ConfigurableField(
185 target=LoadIndexedReferenceObjectsTask,
186 doc=
"reference object loader",
188 astrometer = pexConfig.ConfigurableField(
189 target=AstrometryTask,
190 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
192 sourceSelector = pexConfig.ConfigurableField(
193 target=ObjectSizeStarSelectorTask,
194 doc=
"Source selection algorithm",
196 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
197 decorrelate = pexConfig.ConfigurableField(
198 target=DecorrelateALKernelSpatialTask,
199 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. "
200 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the "
203 doSpatiallyVarying = pexConfig.Field(
206 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the "
207 "image in order to allow for spatial variations"
209 detection = pexConfig.ConfigurableField(
210 target=SourceDetectionTask,
211 doc=
"Low-threshold detection for final measurement",
213 measurement = pexConfig.ConfigurableField(
214 target=DipoleFitTask,
215 doc=
"Enable updated dipole fitting method",
217 forcedMeasurement = pexConfig.ConfigurableField(
218 target=ForcedMeasurementTask,
219 doc=
"Subtask to force photometer PVI at diaSource location.",
221 getTemplate = pexConfig.ConfigurableField(
222 target=GetCoaddAsTemplateTask,
223 doc=
"Subtask to retrieve template exposure and sources",
225 scaleVariance = pexConfig.ConfigurableField(
226 target=ScaleVarianceTask,
227 doc=
"Subtask to rescale the variance of the template "
228 "to the statistically expected level"
230 controlStepSize = pexConfig.Field(
231 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
235 controlRandomSeed = pexConfig.Field(
236 doc=
"Random seed for shuffing the control sample",
240 register = pexConfig.ConfigurableField(
242 doc=
"Task to enable image-to-image image registration (warping)",
244 kernelSourcesFromRef = pexConfig.Field(
245 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
249 templateSipOrder = pexConfig.Field(
250 dtype=int, default=2,
251 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)"
253 growFootprint = pexConfig.Field(
254 dtype=int, default=2,
255 doc=
"Grow positive and negative footprints by this amount before merging"
257 diaSourceMatchRadius = pexConfig.Field(
258 dtype=float, default=0.5,
259 doc=
"Match radius (in arcseconds) for DiaSource to Source association"
262 def setDefaults(self):
265 self.subtract[
'al'].kernel.name =
"AL"
266 self.subtract[
'al'].kernel.active.fitForBackground =
True
267 self.subtract[
'al'].kernel.active.spatialKernelOrder = 1
268 self.subtract[
'al'].kernel.active.spatialBgOrder = 2
269 self.doPreConvolve =
False
270 self.doMatchSources =
False
271 self.doAddMetrics =
False
272 self.doUseRegister =
False
275 self.detection.thresholdPolarity =
"both"
276 self.detection.thresholdValue = 5.5
277 self.detection.reEstimateBackground =
False
278 self.detection.thresholdType =
"pixel_stdev"
284 self.measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
285 self.measurement.plugins.names |= [
'base_LocalPhotoCalib',
288 self.forcedMeasurement.plugins = [
"base_TransformedCentroid",
"base_PsfFlux"]
289 self.forcedMeasurement.copyColumns = {
290 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
291 self.forcedMeasurement.slots.centroid =
"base_TransformedCentroid"
292 self.forcedMeasurement.slots.shape =
None
295 random.seed(self.controlRandomSeed)
298 pexConfig.Config.validate(self)
299 if self.doAddMetrics
and not self.doSubtract:
300 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
301 if not self.doSubtract
and not self.doDetection:
302 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
303 if self.subtract.name ==
'zogy' and self.doAddMetrics:
304 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
305 if self.doMeasurement
and not self.doDetection:
306 raise ValueError(
"Cannot run source measurement without source detection.")
307 if self.doMerge
and not self.doDetection:
308 raise ValueError(
"Cannot run source merging without source detection.")
309 if self.doUseRegister
and not self.doSelectSources:
310 raise ValueError(
"doUseRegister=True and doSelectSources=False. "
311 "Cannot run RegisterTask without selecting sources.")
312 if self.doPreConvolve
and self.doDecorrelation
and not self.convolveTemplate:
313 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and "
314 "convolveTemplate=False is not supported.")
315 if hasattr(self.getTemplate,
"coaddName"):
316 if self.getTemplate.coaddName != self.coaddName:
317 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
320 class ImageDifferenceTaskRunner(pipeBase.ButlerInitializedTaskRunner):
323 def getTargetList(parsedCmd, **kwargs):
324 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
328 class ImageDifferenceTask(pipeBase.CmdLineTask, pipeBase.PipelineTask):
329 """Subtract an image from a template and measure the result
331 ConfigClass = ImageDifferenceConfig
332 RunnerClass = ImageDifferenceTaskRunner
333 _DefaultName =
"imageDifference"
335 def __init__(self, butler=None, **kwargs):
336 """!Construct an ImageDifference Task
338 @param[in] butler Butler object to use in constructing reference object loaders
340 super().__init__(**kwargs)
341 self.makeSubtask(
"getTemplate")
343 self.makeSubtask(
"subtract")
345 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
346 self.makeSubtask(
"decorrelate")
348 if self.config.doScaleTemplateVariance:
349 self.makeSubtask(
"scaleVariance")
351 if self.config.doUseRegister:
352 self.makeSubtask(
"register")
353 self.schema = afwTable.SourceTable.makeMinimalSchema()
355 if self.config.doSelectSources:
356 self.makeSubtask(
"sourceSelector")
357 if self.config.kernelSourcesFromRef:
358 self.makeSubtask(
'refObjLoader', butler=butler)
359 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
361 self.algMetadata = dafBase.PropertyList()
362 if self.config.doDetection:
363 self.makeSubtask(
"detection", schema=self.schema)
364 if self.config.doMeasurement:
365 self.makeSubtask(
"measurement", schema=self.schema,
366 algMetadata=self.algMetadata)
367 if self.config.doForcedMeasurement:
368 self.schema.addField(
369 "ip_diffim_forced_PsfFlux_instFlux",
"D",
370 "Forced PSF flux measured on the direct image.")
371 self.schema.addField(
372 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
373 "Forced PSF flux error measured on the direct image.")
374 self.schema.addField(
375 "ip_diffim_forced_PsfFlux_area",
"F",
376 "Forced PSF flux effective area of PSF.",
378 self.schema.addField(
379 "ip_diffim_forced_PsfFlux_flag",
"Flag",
380 "Forced PSF flux general failure flag.")
381 self.schema.addField(
382 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
383 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
384 self.schema.addField(
385 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
386 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
387 self.makeSubtask(
"forcedMeasurement", refSchema=self.schema)
388 if self.config.doMatchSources:
389 self.schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
390 self.schema.addField(
"srcMatchId",
"L",
"unique id of source match")
393 self.outputSchema = afwTable.SourceCatalog(self.schema)
394 self.outputSchema.getTable().setMetadata(self.algMetadata)
397 def makeIdFactory(expId, expBits):
398 """Create IdFactory instance for unique 64 bit diaSource id-s.
406 Number of used bits in ``expId``.
410 The diasource id-s consists of the ``expId`` stored fixed in the highest value
411 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
412 low value end of the integer.
416 idFactory: `lsst.afw.table.IdFactory`
418 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
421 def runQuantum(self, butlerQC: pipeBase.ButlerQuantumContext,
422 inputRefs: pipeBase.InputQuantizedConnection,
423 outputRefs: pipeBase.OutputQuantizedConnection):
424 inputs = butlerQC.get(inputRefs)
425 self.log.info(f
"Processing {butlerQC.quantum.dataId}")
426 expId, expBits = butlerQC.quantum.dataId.pack(
"visit_detector",
428 idFactory = self.makeIdFactory(expId=expId, expBits=expBits)
429 if self.config.coaddName ==
'dcr':
430 templateExposures = inputRefs.dcrCoadds
432 templateExposures = inputRefs.coaddExposures
433 templateStruct = self.getTemplate.runQuantum(
434 inputs[
'exposure'], butlerQC, inputRefs.skyMap, templateExposures
437 outputs = self.run(exposure=inputs[
'exposure'],
438 templateExposure=templateStruct.exposure,
440 butlerQC.put(outputs, outputRefs)
443 def runDataRef(self, sensorRef, templateIdList=None):
444 """Subtract an image from a template coadd and measure the result.
446 Data I/O wrapper around `run` using the butler in Gen2.
450 sensorRef : `lsst.daf.persistence.ButlerDataRef`
451 Sensor-level butler data reference, used for the following data products:
458 - self.config.coaddName + "Coadd_skyMap"
459 - self.config.coaddName + "Coadd"
460 Input or output, depending on config:
461 - self.config.coaddName + "Diff_subtractedExp"
462 Output, depending on config:
463 - self.config.coaddName + "Diff_matchedExp"
464 - self.config.coaddName + "Diff_src"
468 results : `lsst.pipe.base.Struct`
469 Returns the Struct by `run`.
471 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp"
472 subtractedExposure =
None
474 calexpBackgroundExposure =
None
475 self.log.info(
"Processing %s" % (sensorRef.dataId))
480 idFactory = self.makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
481 expBits=sensorRef.get(
"ccdExposureId_bits"))
482 if self.config.doAddCalexpBackground:
483 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
486 exposure = sensorRef.get(
"calexp", immediate=
True)
489 template = self.getTemplate.runDataRef(exposure, sensorRef, templateIdList=templateIdList)
491 if sensorRef.datasetExists(
"src"):
492 self.log.info(
"Source selection via src product")
494 selectSources = sensorRef.get(
"src")
496 if not self.config.doSubtract
and self.config.doDetection:
498 subtractedExposure = sensorRef.get(subtractedExposureName)
501 results = self.run(exposure=exposure,
502 selectSources=selectSources,
503 templateExposure=template.exposure,
504 templateSources=template.sources,
506 calexpBackgroundExposure=calexpBackgroundExposure,
507 subtractedExposure=subtractedExposure)
509 if self.config.doWriteSources
and results.diaSources
is not None:
510 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
511 if self.config.doWriteMatchedExp:
512 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
513 if self.config.doAddMetrics
and self.config.doSelectSources:
514 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
515 if self.config.doWriteSubtractedExp:
516 sensorRef.put(results.subtractedExposure, subtractedExposureName)
519 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
520 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
521 """PSF matches, subtract two images and perform detection on the difference image.
525 exposure : `lsst.afw.image.ExposureF`, optional
526 The science exposure, the minuend in the image subtraction.
527 Can be None only if ``config.doSubtract==False``.
528 selectSources : `lsst.afw.table.SourceCatalog`, optional
529 Identified sources on the science exposure. This catalog is used to
530 select sources in order to perform the AL PSF matching on stamp images
531 around them. The selection steps depend on config options and whether
532 ``templateSources`` and ``matchingSources`` specified.
533 templateExposure : `lsst.afw.image.ExposureF`, optional
534 The template to be subtracted from ``exposure`` in the image subtraction.
535 The template exposure should cover the same sky area as the science exposure.
536 It is either a stich of patches of a coadd skymap image or a calexp
537 of the same pointing as the science exposure. Can be None only
538 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None.
539 templateSources : `lsst.afw.table.SourceCatalog`, optional
540 Identified sources on the template exposure.
541 idFactory : `lsst.afw.table.IdFactory`
542 Generator object to assign ids to detected sources in the difference image.
543 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional
544 Background exposure to be added back to the science exposure
545 if ``config.doAddCalexpBackground==True``
546 subtractedExposure : `lsst.afw.image.ExposureF`, optional
547 If ``config.doSubtract==False`` and ``config.doDetection==True``,
548 performs the post subtraction source detection only on this exposure.
549 Otherwise should be None.
553 results : `lsst.pipe.base.Struct`
554 ``subtractedExposure`` : `lsst.afw.image.ExposureF`
556 ``matchedExposure`` : `lsst.afw.image.ExposureF`
557 The matched PSF exposure.
558 ``subtractRes`` : `lsst.pipe.base.Struct`
559 The returned result structure of the ImagePsfMatchTask subtask.
560 ``diaSources`` : `lsst.afw.table.SourceCatalog`
561 The catalog of detected sources.
562 ``selectSources`` : `lsst.afw.table.SourceCatalog`
563 The input source catalog with optionally added Qa information.
567 The following major steps are included:
569 - warp template coadd to match WCS of image
570 - PSF match image to warped template
571 - subtract image from PSF-matched, warped template
575 For details about the image subtraction configuration modes
576 see `lsst.ip.diffim`.
579 controlSources =
None
583 if self.config.doAddCalexpBackground:
584 mi = exposure.getMaskedImage()
585 mi += calexpBackgroundExposure.getImage()
587 if not exposure.hasPsf():
588 raise pipeBase.TaskError(
"Exposure has no psf")
589 sciencePsf = exposure.getPsf()
591 if self.config.doSubtract:
592 if self.config.doScaleTemplateVariance:
593 templateVarFactor = self.scaleVariance.
run(
594 templateExposure.getMaskedImage())
595 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
597 if self.config.subtract.name ==
'zogy':
598 subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
600 spatiallyVarying=self.config.doSpatiallyVarying,
601 doPreConvolve=self.config.doPreConvolve)
602 subtractedExposure = subtractRes.subtractedExposure
604 elif self.config.subtract.name ==
'al':
606 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
607 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
615 if self.config.doPreConvolve:
616 convControl = afwMath.ConvolutionControl()
618 srcMI = exposure.getMaskedImage()
619 exposureOrig = exposure.clone()
620 destMI = srcMI.Factory(srcMI.getDimensions())
622 if self.config.useGaussianForPreConvolution:
624 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
629 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
630 exposure.setMaskedImage(destMI)
631 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
633 scienceSigmaPost = scienceSigmaOrig
634 exposureOrig = exposure
639 if self.config.doSelectSources:
640 if selectSources
is None:
641 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
643 selectSources = self.subtract.getSelectSources(
645 sigma=scienceSigmaPost,
646 doSmooth=
not self.config.doPreConvolve,
650 if self.config.doAddMetrics:
653 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
654 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
655 targetFwhmPix=templateSigma*FwhmPerSigma))
662 kcQa = KernelCandidateQa(nparam)
663 selectSources = kcQa.addToSchema(selectSources)
664 if self.config.kernelSourcesFromRef:
666 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
667 matches = astromRet.matches
668 elif templateSources:
670 mc = afwTable.MatchControl()
671 mc.findOnlyClosest =
False
672 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
675 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
676 "but template sources not available. Cannot match science "
677 "sources with template sources. Run process* on data from "
678 "which templates are built.")
680 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
681 matches=matches).sourceCat
682 random.shuffle(kernelSources, random.random)
683 controlSources = kernelSources[::self.config.controlStepSize]
684 kernelSources = [k
for i, k
in enumerate(kernelSources)
685 if i % self.config.controlStepSize]
687 if self.config.doSelectDcrCatalog:
688 redSelector = DiaCatalogSourceSelectorTask(
689 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
691 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
692 controlSources.extend(redSources)
694 blueSelector = DiaCatalogSourceSelectorTask(
695 DiaCatalogSourceSelectorConfig(grMin=-99.999,
696 grMax=self.sourceSelector.config.grMin))
697 blueSources = blueSelector.selectStars(exposure, selectSources,
698 matches=matches).starCat
699 controlSources.extend(blueSources)
701 if self.config.doSelectVariableCatalog:
702 varSelector = DiaCatalogSourceSelectorTask(
703 DiaCatalogSourceSelectorConfig(includeVariable=
True))
704 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
705 controlSources.extend(varSources)
707 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
708 % (len(kernelSources), len(selectSources), len(controlSources)))
712 if self.config.doUseRegister:
713 self.log.info(
"Registering images")
715 if templateSources
is None:
719 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
720 templateSources = self.subtract.getSelectSources(
729 wcsResults = self.fitAstrometry(templateSources, templateExposure, selectSources)
730 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
731 exposure.getWcs(), exposure.getBBox())
732 templateExposure = warpedExp
737 if self.config.doDebugRegister:
739 srcToMatch = {x.second.getId(): x.first
for x
in matches}
741 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
742 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
743 sids = [m.first.getId()
for m
in wcsResults.matches]
744 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
745 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
746 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
747 allresids = dict(zip(sids, zip(positions, residuals)))
749 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
750 wcsResults.wcs.pixelToSky(
751 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
752 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g"))
753 + 2.5*numpy.log10(srcToMatch[x].get(
"r"))
754 for x
in sids
if x
in srcToMatch.keys()])
755 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
756 if s
in srcToMatch.keys()])
757 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
758 if s
in srcToMatch.keys()])
759 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
760 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin)
761 & (colors <= self.sourceSelector.config.grMax))
762 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
763 rms1Long = IqrToSigma*(
764 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
765 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75)
766 - numpy.percentile(dlat[idx1], 25))
767 rms2Long = IqrToSigma*(
768 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
769 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75)
770 - numpy.percentile(dlat[idx2], 25))
771 rms3Long = IqrToSigma*(
772 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
773 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75)
774 - numpy.percentile(dlat[idx3], 25))
775 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
776 (numpy.median(dlong[idx1]), rms1Long,
777 numpy.median(dlat[idx1]), rms1Lat))
778 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
779 (numpy.median(dlong[idx2]), rms2Long,
780 numpy.median(dlat[idx2]), rms2Lat))
781 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
782 (numpy.median(dlong[idx3]), rms3Long,
783 numpy.median(dlat[idx3]), rms3Lat))
785 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
786 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
787 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
788 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
789 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
790 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
792 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
793 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
794 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
795 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
796 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
797 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
804 self.log.info(
"Subtracting images")
805 subtractRes = self.subtract.subtractExposures(
806 templateExposure=templateExposure,
807 scienceExposure=exposure,
808 candidateList=kernelSources,
809 convolveTemplate=self.config.convolveTemplate,
810 doWarping=
not self.config.doUseRegister
812 subtractedExposure = subtractRes.subtractedExposure
814 if self.config.doDetection:
815 self.log.info(
"Computing diffim PSF")
818 if not subtractedExposure.hasPsf():
819 if self.config.convolveTemplate:
820 subtractedExposure.setPsf(exposure.getPsf())
822 subtractedExposure.setPsf(templateExposure.getPsf())
829 if self.config.doDecorrelation
and self.config.doSubtract:
831 if preConvPsf
is not None:
832 preConvKernel = preConvPsf.getLocalKernel()
833 if self.config.convolveTemplate:
834 self.log.info(
"Decorrelation after template image convolution")
835 decorrResult = self.decorrelate.
run(exposureOrig, subtractRes.warpedExposure,
837 subtractRes.psfMatchingKernel,
838 spatiallyVarying=self.config.doSpatiallyVarying,
839 preConvKernel=preConvKernel)
841 self.log.info(
"Decorrelation after science image convolution")
842 decorrResult = self.decorrelate.
run(subtractRes.warpedExposure, exposureOrig,
844 subtractRes.psfMatchingKernel,
845 spatiallyVarying=self.config.doSpatiallyVarying,
846 preConvKernel=preConvKernel)
847 subtractedExposure = decorrResult.correctedExposure
851 if self.config.doDetection:
852 self.log.info(
"Running diaSource detection")
854 mask = subtractedExposure.getMaskedImage().getMask()
855 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
857 table = afwTable.SourceTable.make(self.schema, idFactory)
858 table.setMetadata(self.algMetadata)
859 results = self.detection.
run(
861 exposure=subtractedExposure,
862 doSmooth=
not self.config.doPreConvolve
865 if self.config.doMerge:
866 fpSet = results.fpSets.positive
867 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
868 self.config.growFootprint,
False)
869 diaSources = afwTable.SourceCatalog(table)
870 fpSet.makeSources(diaSources)
871 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
873 diaSources = results.sources
875 if self.config.doMeasurement:
876 newDipoleFitting = self.config.doDipoleFitting
877 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
878 if not newDipoleFitting:
880 self.measurement.
run(diaSources, subtractedExposure)
883 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
884 self.measurement.
run(diaSources, subtractedExposure, exposure,
885 subtractRes.matchedExposure)
887 self.measurement.
run(diaSources, subtractedExposure, exposure)
889 if self.config.doForcedMeasurement:
892 forcedSources = self.forcedMeasurement.generateMeasCat(
893 exposure, diaSources, subtractedExposure.getWcs())
894 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
895 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
896 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
897 "ip_diffim_forced_PsfFlux_instFlux",
True)
898 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
899 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
900 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
901 "ip_diffim_forced_PsfFlux_area",
True)
902 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
903 "ip_diffim_forced_PsfFlux_flag",
True)
904 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
905 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
906 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
907 "ip_diffim_forced_PsfFlux_flag_edge",
True)
908 for diaSource, forcedSource
in zip(diaSources, forcedSources):
909 diaSource.assign(forcedSource, mapper)
912 if self.config.doMatchSources:
913 if selectSources
is not None:
915 matchRadAsec = self.config.diaSourceMatchRadius
916 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
918 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
919 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for
920 srcMatch
in srcMatches])
921 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
924 self.log.warn(
"Src product does not exist; cannot match with diaSources")
928 refAstromConfig = AstrometryConfig()
929 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
930 refAstrometer = AstrometryTask(refAstromConfig)
931 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
932 refMatches = astromRet.matches
933 if refMatches
is None:
934 self.log.warn(
"No diaSource matches with reference catalog")
937 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
939 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for
940 refMatch
in refMatches])
943 for diaSource
in diaSources:
944 sid = diaSource.getId()
945 if sid
in srcMatchDict:
946 diaSource.set(
"srcMatchId", srcMatchDict[sid])
947 if sid
in refMatchDict:
948 diaSource.set(
"refMatchId", refMatchDict[sid])
950 if self.config.doAddMetrics
and self.config.doSelectSources:
951 self.log.info(
"Evaluating metrics and control sample")
954 for cell
in subtractRes.kernelCellSet.getCellList():
955 for cand
in cell.begin(
False):
956 kernelCandList.append(cand)
959 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
960 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
963 diffimTools.sourceTableToCandidateList(controlSources,
964 subtractRes.warpedExposure, exposure,
965 self.config.subtract.kernel.active,
966 self.config.subtract.kernel.active.detectionConfig,
967 self.log, doBuild=
True, basisList=basisList))
969 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
970 subtractRes.backgroundModel, dof=nparam)
971 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
972 subtractRes.backgroundModel)
974 if self.config.doDetection:
975 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
977 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
979 self.runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
980 return pipeBase.Struct(
981 subtractedExposure=subtractedExposure,
982 matchedExposure=subtractRes.matchedExposure,
983 subtractRes=subtractRes,
984 diaSources=diaSources,
985 selectSources=selectSources
988 def fitAstrometry(self, templateSources, templateExposure, selectSources):
989 """Fit the relative astrometry between templateSources and selectSources
994 Remove this method. It originally fit a new WCS to the template before calling register.run
995 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed.
996 It remains because a subtask overrides it.
998 results = self.register.
run(templateSources, templateExposure.getWcs(),
999 templateExposure.getBBox(), selectSources)
1002 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
1003 """Make debug plots and displays.
1007 Test and update for current debug display and slot names
1017 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
1018 if not maskTransparency:
1019 maskTransparency = 0
1020 disp.setMaskTransparency(maskTransparency)
1022 if display
and showSubtracted:
1023 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
1024 mi = subtractRes.subtractedExposure.getMaskedImage()
1025 x0, y0 = mi.getX0(), mi.getY0()
1026 with disp.Buffering():
1027 for s
in diaSources:
1028 x, y = s.getX() - x0, s.getY() - y0
1029 ctype =
"red" if s.get(
"flags_negative")
else "yellow"
1030 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
1031 or s.get(
"base_PixelFlags_flag_saturatedCenter")
1032 or s.get(
"base_PixelFlags_flag_crCenter")):
1034 elif (s.get(
"base_PixelFlags_flag_interpolated")
1035 or s.get(
"base_PixelFlags_flag_saturated")
1036 or s.get(
"base_PixelFlags_flag_cr")):
1040 disp.dot(ptype, x, y, size=4, ctype=ctype)
1041 lsstDebug.frame += 1
1043 if display
and showPixelResiduals
and selectSources:
1044 nonKernelSources = []
1045 for source
in selectSources:
1046 if source
not in kernelSources:
1047 nonKernelSources.append(source)
1049 diUtils.plotPixelResiduals(exposure,
1050 subtractRes.warpedExposure,
1051 subtractRes.subtractedExposure,
1052 subtractRes.kernelCellSet,
1053 subtractRes.psfMatchingKernel,
1054 subtractRes.backgroundModel,
1056 self.subtract.config.kernel.active.detectionConfig,
1058 diUtils.plotPixelResiduals(exposure,
1059 subtractRes.warpedExposure,
1060 subtractRes.subtractedExposure,
1061 subtractRes.kernelCellSet,
1062 subtractRes.psfMatchingKernel,
1063 subtractRes.backgroundModel,
1065 self.subtract.config.kernel.active.detectionConfig,
1067 if display
and showDiaSources:
1068 flagChecker = SourceFlagChecker(diaSources)
1069 isFlagged = [flagChecker(x)
for x
in diaSources]
1070 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
1071 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
1072 frame=lsstDebug.frame)
1073 lsstDebug.frame += 1
1075 if display
and showDipoles:
1076 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
1077 frame=lsstDebug.frame)
1078 lsstDebug.frame += 1
1080 def _getConfigName(self):
1081 """Return the name of the config dataset
1083 return "%sDiff_config" % (self.config.coaddName,)
1085 def _getMetadataName(self):
1086 """Return the name of the metadata dataset
1088 return "%sDiff_metadata" % (self.config.coaddName,)
1090 def getSchemaCatalogs(self):
1091 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
1092 return {self.config.coaddName +
"Diff_diaSrc": self.outputSchema}
1095 def _makeArgumentParser(cls):
1096 """Create an argument parser
1098 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1099 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
1100 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
1101 help=
"Template data ID in case of calexp template,"
1102 " e.g. --templateId visit=6789")
1106 class Winter2013ImageDifferenceConfig(ImageDifferenceConfig):
1107 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1108 doc=
"Shift stars going into RegisterTask by this amount")
1109 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1110 doc=
"Perturb stars going into RegisterTask by this amount")
1112 def setDefaults(self):
1113 ImageDifferenceConfig.setDefaults(self)
1114 self.getTemplate.retarget(GetCalexpAsTemplateTask)
1117 class Winter2013ImageDifferenceTask(ImageDifferenceTask):
1118 """!Image difference Task used in the Winter 2013 data challege.
1119 Enables testing the effects of registration shifts and scatter.
1121 For use with winter 2013 simulated images:
1122 Use --templateId visit=88868666 for sparse data
1123 --templateId visit=22222200 for dense data (g)
1124 --templateId visit=11111100 for dense data (i)
1126 ConfigClass = Winter2013ImageDifferenceConfig
1127 _DefaultName =
"winter2013ImageDifference"
1129 def __init__(self, **kwargs):
1130 ImageDifferenceTask.__init__(self, **kwargs)
1132 def fitAstrometry(self, templateSources, templateExposure, selectSources):
1133 """Fit the relative astrometry between templateSources and selectSources"""
1134 if self.config.winter2013WcsShift > 0.0:
1136 self.config.winter2013WcsShift)
1137 cKey = templateSources[0].getTable().getCentroidKey()
1138 for source
in templateSources:
1139 centroid = source.get(cKey)
1140 source.set(cKey, centroid + offset)
1141 elif self.config.winter2013WcsRms > 0.0:
1142 cKey = templateSources[0].getTable().getCentroidKey()
1143 for source
in templateSources:
1144 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1145 self.config.winter2013WcsRms*numpy.random.normal())
1146 centroid = source.get(cKey)
1147 source.set(cKey, centroid + offset)
1149 results = self.register.
run(templateSources, templateExposure.getWcs(),
1150 templateExposure.getBBox(), selectSources)