22from deprecated.sphinx
import deprecated
26import lsst.pipe.base
as pipeBase
28from lsst.daf.butler
import DimensionGraph
29from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
31 FocalPlaneBackgroundConfig, MaskObjectsTask)
33import lsst.pipe.base.connectionTypes
as cT
36 from lsst.pipe.base
import ArgumentParser
37 from lsst.pipe.base
import ConfigDatasetType
40 from argparse
import ArgumentParser
44 raise NotImplementedError()
55 from lsst.pipe.base
import Task
as BatchPoolTask
57__all__ = [
"SkyCorrectionConfig",
"SkyCorrectionTask"]
63 """Match the order of one list to another, padding if necessary
68 List to be reordered and padded. Elements can be any type.
70 Iterable of values to be compared
with outputKeys.
71 Length must match `inputList`
73 Iterable of values to be compared
with inputKeys.
75 Any value to be inserted where inputKey
not in outputKeys
80 Copy of inputList reordered per outputKeys
and padded
with `padWith`
81 so that the length matches length of outputKeys.
86 outputList.append(inputList[inputKeys.index(d)])
88 outputList.append(padWith)
93 """Make and write an image of an entire focal plane
100 List of detector ID
and CCD exposures (binned by `binning`).
101 filename : `str`, optional
104 Binning size that has been applied to images.
106 image = visualizeVisit.makeCameraImage(camera, dict(exp for exp
in exposures
if exp
is not None), binning)
107 if filename
is not None:
108 image.writeFits(filename)
112def _skyLookup(datasetType, registry, quantumDataId, collections):
113 """Lookup function to identify sky frames
117 datasetType : `lsst.daf.butler.DatasetType`
119 registry : `lsst.daf.butler.Registry`
120 Butler registry to query.
121 quantumDataId : `lsst.daf.butler.DataCoordinate`
122 Data id to transform to find sky frames.
123 The ``detector`` entry will be stripped.
124 collections : `lsst.daf.butler.CollectionSearch`
125 Collections to search through.
129 results : `list` [`lsst.daf.butler.DatasetRef`]
130 List of datasets that will be used as sky calibration frames
132 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"visit"]))
134 for dataId
in registry.queryDataIds([
"visit",
"detector"], dataId=newDataId).expanded():
135 skyFrame = registry.findDataset(datasetType, dataId, collections=collections,
136 timespan=dataId.timespan)
137 skyFrames.append(skyFrame)
143 rawLinker = cT.Input(
144 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
148 storageClass=
"Exposure",
149 dimensions=[
"instrument",
"exposure",
"detector"],
151 calExpArray = cT.Input(
152 doc=
"Input exposures to process",
155 storageClass=
"ExposureF",
156 dimensions=[
"instrument",
"visit",
"detector"],
158 calBkgArray = cT.Input(
159 doc=
"Input background files to use",
161 name=
"calexpBackground",
162 storageClass=
"Background",
163 dimensions=[
"instrument",
"visit",
"detector"],
165 camera = cT.PrerequisiteInput(
166 doc=
"Input camera to use.",
168 storageClass=
"Camera",
169 dimensions=[
"instrument"],
172 skyCalibs = cT.PrerequisiteInput(
173 doc=
"Input sky calibrations to use.",
176 storageClass=
"ExposureF",
177 dimensions=[
"instrument",
"physical_filter",
"detector"],
179 lookupFunction=_skyLookup,
181 calExpCamera = cT.Output(
182 doc=
"Output camera image.",
183 name=
'calexp_camera',
184 storageClass=
"ImageF",
185 dimensions=[
"instrument",
"visit"],
188 doc=
"Output sky corrected images.",
191 storageClass=
"Background",
192 dimensions=[
"instrument",
"visit",
"detector"],
197 """Configuration for SkyCorrectionTask"""
198 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"Background model")
199 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"2nd Background model")
200 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
201 maskObjects = ConfigurableField(target=MaskObjectsTask, doc=
"Mask Objects")
202 doMaskObjects = Field(dtype=bool, default=
True, doc=
"Mask objects to find good sky?")
203 doBgModel = Field(dtype=bool, default=
True, doc=
"Do background model subtraction?")
204 doBgModel2 = Field(dtype=bool, default=
True, doc=
"Do cleanup background model subtraction?")
205 doSky = Field(dtype=bool, default=
True, doc=
"Do sky frame subtraction?")
206 binning = Field(dtype=int, default=8, doc=
"Binning factor for constructing focal-plane images")
207 calexpType = Field(dtype=str, default=
"calexp",
208 doc=
"Should be set to fakes_calexp if you want to process calexps with fakes in.")
211 Config.setDefaults(self)
220 reason=
"pipe_drivers is deprecated. It will be removed after v25. "
221 "Please use lsst.pipe.tasks.skyCorrection.SkyCorrectionTask instead.",
223 category=FutureWarning,
226 """Correct sky over entire focal plane"""
227 ConfigClass = SkyCorrectionConfig
228 _DefaultName =
"skyCorr"
234 detectorOrder = [ref.dataId[
'detector']
for ref
in inputRefs.calExpArray]
237 [ref.dataId[
'detector']
for ref
in inputRefs.calExpArray],
240 [ref.dataId[
'detector']
for ref
in inputRefs.skyCalibs],
243 [ref.dataId[
'detector']
for ref
in inputRefs.calBkgArray],
246 [ref.dataId[
'detector']
for ref
in outputRefs.skyCorr],
248 inputs = butlerQC.get(inputRefs)
249 inputs.pop(
"rawLinker",
None)
250 outputs = self.
run(**inputs)
251 butlerQC.put(outputs, outputRefs)
256 self.makeSubtask(
"sky")
257 self.makeSubtask(
"maskObjects")
260 def _makeArgumentParser(cls, *args, **kwargs):
261 kwargs.pop(
"doBatch",
False)
263 parser = ArgumentParser(name=
"skyCorr", *args, **kwargs)
264 parser.add_id_argument(
"--id", datasetType=datasetType, level=
"visit",
265 help=
"data ID, e.g. --id visit=12345")
270 """Return walltime request for batch job
272 Subclasses should override if the walltime should be calculated
273 differently (e.g., addition of some serial time).
278 Requested time per iteration.
279 parsedCmd : `argparse.Namespace`
280 Results of argument parsing.
284 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
285 return time*numTargets
288 """Perform sky correction on an exposure
290 We restore the original sky, and remove it again using multiple
291 algorithms. We optionally apply:
293 1. A large-scale background model.
294 This step removes very-large-scale sky such
as moonlight.
296 3. A medium-scale background model.
297 This step removes residual sky (This
is smooth on the focal plane).
299 Only the master node executes this method. The data
is held on
300 the slave nodes, which do all the hard work.
304 expRef : `lsst.daf.persistence.ButlerDataRef`
305 Data reference
for exposure.
309 ~lsst.pipe.drivers.SkyCorrectionTask.run
312 extension =
"-%(visit)d.fits" % expRef.dataId
314 with self.
logOperation(
"processing %s" % (expRef.dataId,)):
317 pool.storeSet(butler=expRef.getButler())
318 camera = expRef.get(
"camera")
320 dataIdList = [ccdRef.dataId
for ccdRef
in expRef.subItems(
"ccd")
if
321 ccdRef.datasetExists(self.config.calexpType)]
322 ccdList = [dataId[
"ccd"]
for dataId
in dataIdList]
324 dataIdList = [dataId
for _, dataId
in sorted(zip(ccdList, dataIdList))]
326 exposures = pool.map(self.
loadImage, dataIdList)
331 exposures = pool.mapToPrevious(self.
collectMask, dataIdList)
334 if self.config.doBgModel:
337 if self.config.doSky:
339 scale = self.sky.solveScales(measScales)
340 self.log.info(
"Sky frame scale: %s" % (scale,))
345 calibs = pool.mapToPrevious(self.
collectSky, dataIdList)
348 if self.config.doBgModel2:
353 expRef.put(image,
"calexp_camera")
355 pool.mapToPrevious(self.
write, dataIdList)
358 """Perform full focal-plane background subtraction
360 This method runs on the master node.
366 pool : `lsst.ctrl.pool.Pool`
368 dataIdList : iterable of `dict`
369 List of data identifiers for the CCDs.
371 Configuration to use
for background subtraction.
376 List of binned images,
for creating focal plane image.
378 bgModel = FocalPlaneBackground.fromCamera(config, camera)
379 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId
in dataIdList]
381 for ii, bg
in enumerate(bgModelList):
382 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.array.sum())
384 return pool.mapToPrevious(self.
subtractModel, dataIdList, bgModel)
387 """Perform full focal-plane background subtraction
389 This method runs on the master node.
395 cacheExposures : `list` of `lsst.afw.image.Exposures`
396 List of loaded and processed input calExp.
397 idList : `list` of `int`
398 List of detector ids to iterate over.
400 Configuration to use
for background subtraction.
405 List of binned images,
for creating focal plane image.
406 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
407 Background lists generated.
408 cacheBgModel : `FocalPlaneBackground`
409 Full focal plane background model.
411 bgModel = FocalPlaneBackground.fromCamera(config, camera)
412 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone()) for id
in idList]
415 for nodeData, cacheExp
in zip(data, cacheExposures):
416 nodeData.bgModel.addCcd(cacheExp)
417 bgModelList.append(nodeData.bgModel)
419 for ii, bg
in enumerate(bgModelList):
420 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.getArray().sum())
426 for cacheExp
in cacheExposures:
428 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
429 cacheBgModel.append(nodeBgModel)
430 newCacheBgList.append(nodeBgList)
432 return exposures, newCacheBgList, cacheBgModel
434 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
435 """Duplicate runDataRef method without ctrl_pool for Gen3.
440 Array of detector input calExp images for the exposure to
442 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
443 Array of detector input background lists matching the
446 Array of SKY calibrations
for the input detectors to be
449 Camera matching the input data to process.
453 results : `pipeBase.Struct` containing
455 Full camera image of the sky-corrected data.
456 skyCorr : `list` of `lsst.afw.math.BackgroundList`
457 Detector-level sky-corrected background lists.
461 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
478 idList = [exp.getDetector().getId()
for exp
in calExpArray]
485 for calExp, calBgModel
in zip(calExpArray, calBkgArray):
486 nodeExp, nodeBgList = self.
loadImageRun(calExp, calBgModel)
487 cacheExposures.append(nodeExp)
488 cacheBgList.append(nodeBgList)
489 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
491 if self.config.doBgModel:
494 camera, cacheExposures, idList, self.config.bgModel
496 for cacheBg, newBg
in zip(cacheBgList, newCacheBgList):
497 cacheBg.append(newBg)
499 if self.config.doSky:
505 for cacheExp, skyCalib
in zip(cacheExposures, skyCalibs):
506 skyExp = self.sky.exposureToBackground(skyCalib)
507 cacheSky.append(skyExp)
508 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
509 measScales.append(scale)
511 scale = self.sky.solveScales(measScales)
512 self.log.info(
"Sky frame scale: %s" % (scale, ))
518 for cacheExp, nodeSky, nodeBgList
in zip(cacheExposures, cacheSky, cacheBgList):
519 self.sky.
subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
520 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
522 if self.config.doBgModel2:
526 camera, cacheExposures, idList, self.config.bgModel2
528 for cacheBg, newBg
in zip(cacheBgList, newBgList):
529 cacheBg.append(newBg)
535 return pipeBase.Struct(
541 """Load original image and restore the sky
543 This method runs on the slave nodes.
547 cache : `lsst.pipe.base.Struct`
557 cache.dataId = dataId
558 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=True).clone()
559 bgOld = cache.butler.get(
"calexpBackground", dataId, immediate=
True)
560 image = cache.exposure.getMaskedImage()
564 statsImage = bgData[0].getStatsImage()
567 image -= bgOld.getImage()
568 cache.bgList = afwMath.BackgroundList()
570 cache.bgList.append(bgData)
572 if self.config.doMaskObjects:
573 self.maskObjects.findObjects(cache.exposure)
578 """Serial implementation of self.loadImage() for Gen3.
580 Load and restore background to calExp
and calExpBkg.
585 Detector level calExp image to process.
586 calExpBkg : `lsst.afw.math.BackgroundList`
587 Detector level background list associated
with the calExp.
592 Background restored calExp.
593 bgList : `lsst.afw.math.BackgroundList`
594 New background list containing the restoration background.
596 image = calExp.getMaskedImage()
598 for bgOld
in calExpBkg:
599 statsImage = bgOld[0].getStatsImage()
602 image -= calExpBkg.getImage()
603 bgList = afwMath.BackgroundList()
604 for bgData
in calExpBkg:
605 bgList.append(bgData)
607 if self.config.doMaskObjects:
608 self.maskObjects.findObjects(calExp)
610 return (calExp, bgList)
613 """Measure scale for sky frame
615 This method runs on the slave nodes.
619 cache : `lsst.pipe.base.Struct`
629 assert cache.dataId == dataId
630 cache.sky = self.sky.getSkyData(cache.butler, dataId)
631 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
635 """Subtract sky frame
637 This method runs on the slave nodes.
641 cache : `lsst.pipe.base.Struct`
653 assert cache.dataId == dataId
654 self.sky.
subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
658 """Fit background model for CCD
660 This method runs on the slave nodes.
664 cache : `lsst.pipe.base.Struct`
666 data : `lsst.pipe.base.Struct`
667 Data identifier, with `dataId` (data identifier)
and `bgModel`
668 (background model) elements.
675 assert cache.dataId == data.dataId
676 data.bgModel.addCcd(cache.exposure)
680 """Subtract background model
682 This method runs on the slave nodes.
686 cache : `lsst.pipe.base.Struct`
690 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
698 assert cache.dataId == dataId
699 exposure = cache.exposure
700 image = exposure.getMaskedImage()
701 detector = exposure.getDetector()
702 bbox = image.getBBox()
704 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
705 image -= cache.bgModel.getImage()
707 self.log.error(f
"There was an error processing {dataId}, no calib file produced")
709 cache.bgList.append(cache.bgModel[0])
713 """Serial implementation of self.subtractModel() for Gen3.
715 Load and restore background to calExp
and calExpBkg.
720 Exposure to subtract the background model
from.
722 Full camera level background model.
727 Background subtracted input exposure.
728 bgModelCcd : `lsst.afw.math.BackgroundList`
729 Detector level realization of the full background model.
731 Background model
from the bgModelCcd realization.
733 image = exposure.getMaskedImage()
734 detector = exposure.getDetector()
735 bbox = image.getBBox()
736 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
737 image -= bgModelCcd.getImage()
739 return (exposure, bgModelCcd, bgModelCcd[0])
742 """Generate an image of the background model for visualisation
744 Useful for debugging.
748 cache : `lsst.pipe.base.Struct`
752 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
760 Binned background model image.
762 assert cache.dataId == dataId
763 exposure = cache.exposure
764 detector = exposure.getDetector()
765 bbox = exposure.getMaskedImage().getBBox()
766 image = bgModel.toCcdBackground(detector, bbox).getImage()
770 """Return the binned image required for visualization
772 This method just helps to cut down on boilerplate.
777 Image to go into visualisation.
786 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
789 """Collect exposure for potential visualisation
791 This method runs on the slave nodes.
795 cache : `lsst.pipe.base.Struct`
808 """Collect original image for visualisation
810 This method runs on the slave nodes.
814 cache : `lsst.pipe.base.Struct`
826 exposure = cache.butler.get("calexp", dataId, immediate=
True)
830 """Collect original image for visualisation
832 This method runs on the slave nodes.
836 cache : `lsst.pipe.base.Struct`
851 """Collect mask for visualisation
853 This method runs on the slave nodes.
857 cache : `lsst.pipe.base.Struct`
870 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
871 image.array[:] = cache.exposure.maskedImage.mask.array
875 """Write resultant background list
877 This method runs on the slave nodes.
881 cache : `lsst.pipe.base.Struct`
886 cache.butler.put(cache.bgList, "skyCorr", dataId)
888 def _getMetadataName(self):
889 """There's no metadata to write out"""
def logOperation(self, operation, catch=False, trace=True)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def subtractModel(self, cache, dataId, bgModel)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def subtractModelRun(self, exposure, bgModel)
def __init__(self, *args, **kwargs)
def accumulateModel(self, cache, data)
def subtractSkyFrame(self, cache, dataId, scale)
def measureSkyFrame(self, cache, dataId)
def collectBinnedImage(self, exposure, image)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def loadImage(self, cache, dataId)
def collectMask(self, cache, dataId)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def batchWallTime(cls, time, parsedCmd, numCores)
def collectOriginal(self, cache, dataId)
def runDataRef(self, expRef)
def realiseModel(self, cache, dataId, bgModel)
def write(self, cache, dataId)
def collectSky(self, cache, dataId)
def loadImageRun(self, calExp, calExpBkg)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)