22from deprecated.sphinx
import deprecated
26import lsst.pipe.base
as pipeBase
28from lsst.pipe.base
import ArgumentParser, ConfigDatasetType
29from lsst.daf.butler
import DimensionGraph
30from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
34 FocalPlaneBackgroundConfig, MaskObjectsTask)
36import lsst.pipe.base.connectionTypes
as cT
38__all__ = [
"SkyCorrectionConfig",
"SkyCorrectionTask"]
44 """Match the order of one list to another, padding if necessary
49 List to be reordered and padded. Elements can be any type.
51 Iterable of values to be compared
with outputKeys.
52 Length must match `inputList`
54 Iterable of values to be compared
with inputKeys.
56 Any value to be inserted where inputKey
not in outputKeys
61 Copy of inputList reordered per outputKeys
and padded
with `padWith`
62 so that the length matches length of outputKeys.
67 outputList.append(inputList[inputKeys.index(d)])
69 outputList.append(padWith)
73def makeCameraImage(camera, exposures, filename=None, binning=8):
74 """Make and write an image of an entire focal plane
81 List of detector ID
and CCD exposures (binned by `binning`).
82 filename : `str`, optional
85 Binning size that has been applied to images.
87 image = visualizeVisit.makeCameraImage(camera, dict(exp for exp
in exposures
if exp
is not None), binning)
88 if filename
is not None:
89 image.writeFits(filename)
93def _skyLookup(datasetType, registry, quantumDataId, collections):
94 """Lookup function to identify sky frames
98 datasetType : `lsst.daf.butler.DatasetType`
100 registry : `lsst.daf.butler.Registry`
101 Butler registry to query.
102 quantumDataId : `lsst.daf.butler.DataCoordinate`
103 Data id to transform to find sky frames.
104 The ``detector`` entry will be stripped.
105 collections : `lsst.daf.butler.CollectionSearch`
106 Collections to search through.
110 results : `list` [`lsst.daf.butler.DatasetRef`]
111 List of datasets that will be used as sky calibration frames
113 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"visit"]))
115 for dataId
in registry.queryDataIds([
"visit",
"detector"], dataId=newDataId).expanded():
116 skyFrame = registry.findDataset(datasetType, dataId, collections=collections,
117 timespan=dataId.timespan)
118 skyFrames.append(skyFrame)
124 rawLinker = cT.Input(
125 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
129 storageClass=
"Exposure",
130 dimensions=[
"instrument",
"exposure",
"detector"],
132 calExpArray = cT.Input(
133 doc=
"Input exposures to process",
136 storageClass=
"ExposureF",
137 dimensions=[
"instrument",
"visit",
"detector"],
139 calBkgArray = cT.Input(
140 doc=
"Input background files to use",
142 name=
"calexpBackground",
143 storageClass=
"Background",
144 dimensions=[
"instrument",
"visit",
"detector"],
146 camera = cT.PrerequisiteInput(
147 doc=
"Input camera to use.",
149 storageClass=
"Camera",
150 dimensions=[
"instrument"],
153 skyCalibs = cT.PrerequisiteInput(
154 doc=
"Input sky calibrations to use.",
157 storageClass=
"ExposureF",
158 dimensions=[
"instrument",
"physical_filter",
"detector"],
160 lookupFunction=_skyLookup,
162 calExpCamera = cT.Output(
163 doc=
"Output camera image.",
164 name=
'calexp_camera',
165 storageClass=
"ImageF",
166 dimensions=[
"instrument",
"visit"],
169 doc=
"Output sky corrected images.",
172 storageClass=
"Background",
173 dimensions=[
"instrument",
"visit",
"detector"],
178 """Configuration for SkyCorrectionTask"""
179 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"Background model")
180 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"2nd Background model")
181 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
182 maskObjects = ConfigurableField(target=MaskObjectsTask, doc=
"Mask Objects")
183 doMaskObjects = Field(dtype=bool, default=
True, doc=
"Mask objects to find good sky?")
184 doBgModel = Field(dtype=bool, default=
True, doc=
"Do background model subtraction?")
185 doBgModel2 = Field(dtype=bool, default=
True, doc=
"Do cleanup background model subtraction?")
186 doSky = Field(dtype=bool, default=
True, doc=
"Do sky frame subtraction?")
187 binning = Field(dtype=int, default=8, doc=
"Binning factor for constructing focal-plane images")
188 calexpType = Field(dtype=str, default=
"calexp",
189 doc=
"Should be set to fakes_calexp if you want to process calexps with fakes in.")
192 Config.setDefaults(self)
201 reason=
"pipe_drivers is deprecated. It will be removed after v25. "
202 "Please use lsst.pipe.tasks.skyCorrection.SkyCorrectionTask instead.",
204 category=FutureWarning,
207 """Correct sky over entire focal plane"""
208 ConfigClass = SkyCorrectionConfig
209 _DefaultName =
"skyCorr"
215 detectorOrder = [ref.dataId[
'detector']
for ref
in inputRefs.calExpArray]
218 [ref.dataId[
'detector']
for ref
in inputRefs.calExpArray],
221 [ref.dataId[
'detector']
for ref
in inputRefs.skyCalibs],
224 [ref.dataId[
'detector']
for ref
in inputRefs.calBkgArray],
227 [ref.dataId[
'detector']
for ref
in outputRefs.skyCorr],
229 inputs = butlerQC.get(inputRefs)
230 inputs.pop(
"rawLinker",
None)
231 outputs = self.
run(**inputs)
232 butlerQC.put(outputs, outputRefs)
237 self.makeSubtask(
"sky")
238 self.makeSubtask(
"maskObjects")
241 def _makeArgumentParser(cls, *args, **kwargs):
242 kwargs.pop(
"doBatch",
False)
243 datasetType = ConfigDatasetType(name=
"calexpType")
244 parser = ArgumentParser(name=
"skyCorr", *args, **kwargs)
245 parser.add_id_argument(
"--id", datasetType=datasetType, level=
"visit",
246 help=
"data ID, e.g. --id visit=12345")
251 """Return walltime request for batch job
253 Subclasses should override if the walltime should be calculated
254 differently (e.g., addition of some serial time).
259 Requested time per iteration.
260 parsedCmd : `argparse.Namespace`
261 Results of argument parsing.
265 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
266 return time*numTargets
269 """Perform sky correction on an exposure
271 We restore the original sky, and remove it again using multiple
272 algorithms. We optionally apply:
274 1. A large-scale background model.
275 This step removes very-large-scale sky such
as moonlight.
277 3. A medium-scale background model.
278 This step removes residual sky (This
is smooth on the focal plane).
280 Only the master node executes this method. The data
is held on
281 the slave nodes, which do all the hard work.
285 expRef : `lsst.daf.persistence.ButlerDataRef`
286 Data reference
for exposure.
290 ~lsst.pipe.drivers.SkyCorrectionTask.run
293 extension =
"-%(visit)d.fits" % expRef.dataId
295 with self.
logOperation(
"processing %s" % (expRef.dataId,)):
298 pool.storeSet(butler=expRef.getButler())
299 camera = expRef.get(
"camera")
301 dataIdList = [ccdRef.dataId
for ccdRef
in expRef.subItems(
"ccd")
if
302 ccdRef.datasetExists(self.config.calexpType)]
303 ccdList = [dataId[
"ccd"]
for dataId
in dataIdList]
305 dataIdList = [dataId
for _, dataId
in sorted(zip(ccdList, dataIdList))]
307 exposures = pool.map(self.
loadImage, dataIdList)
309 makeCameraImage(camera, exposures,
"restored" + extension)
311 makeCameraImage(camera, exposures,
"original" + extension)
312 exposures = pool.mapToPrevious(self.
collectMask, dataIdList)
313 makeCameraImage(camera, exposures,
"mask" + extension)
315 if self.config.doBgModel:
318 if self.config.doSky:
320 scale = self.sky.solveScales(measScales)
321 self.log.info(
"Sky frame scale: %s" % (scale,))
325 makeCameraImage(camera, exposures,
"skysub" + extension)
326 calibs = pool.mapToPrevious(self.
collectSky, dataIdList)
327 makeCameraImage(camera, calibs,
"sky" + extension)
329 if self.config.doBgModel2:
333 image = makeCameraImage(camera, exposures)
334 expRef.put(image,
"calexp_camera")
336 pool.mapToPrevious(self.
write, dataIdList)
339 """Perform full focal-plane background subtraction
341 This method runs on the master node.
347 pool : `lsst.ctrl.pool.Pool`
349 dataIdList : iterable of `dict`
350 List of data identifiers for the CCDs.
352 Configuration to use
for background subtraction.
357 List of binned images,
for creating focal plane image.
359 bgModel = FocalPlaneBackground.fromCamera(config, camera)
360 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId
in dataIdList]
362 for ii, bg
in enumerate(bgModelList):
363 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.array.sum())
365 return pool.mapToPrevious(self.
subtractModel, dataIdList, bgModel)
368 """Perform full focal-plane background subtraction
370 This method runs on the master node.
376 cacheExposures : `list` of `lsst.afw.image.Exposures`
377 List of loaded and processed input calExp.
378 idList : `list` of `int`
379 List of detector ids to iterate over.
381 Configuration to use
for background subtraction.
386 List of binned images,
for creating focal plane image.
387 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
388 Background lists generated.
389 cacheBgModel : `FocalPlaneBackground`
390 Full focal plane background model.
392 bgModel = FocalPlaneBackground.fromCamera(config, camera)
393 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone()) for id
in idList]
396 for nodeData, cacheExp
in zip(data, cacheExposures):
397 nodeData.bgModel.addCcd(cacheExp)
398 bgModelList.append(nodeData.bgModel)
400 for ii, bg
in enumerate(bgModelList):
401 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.getArray().sum())
407 for cacheExp
in cacheExposures:
409 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
410 cacheBgModel.append(nodeBgModel)
411 newCacheBgList.append(nodeBgList)
413 return exposures, newCacheBgList, cacheBgModel
415 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
416 """Duplicate runDataRef method without ctrl_pool for Gen3.
421 Array of detector input calExp images for the exposure to
423 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
424 Array of detector input background lists matching the
427 Array of SKY calibrations
for the input detectors to be
430 Camera matching the input data to process.
434 results : `pipeBase.Struct` containing
436 Full camera image of the sky-corrected data.
437 skyCorr : `list` of `lsst.afw.math.BackgroundList`
438 Detector-level sky-corrected background lists.
442 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
459 idList = [exp.getDetector().getId()
for exp
in calExpArray]
466 for calExp, calBgModel
in zip(calExpArray, calBkgArray):
467 nodeExp, nodeBgList = self.
loadImageRun(calExp, calBgModel)
468 cacheExposures.append(nodeExp)
469 cacheBgList.append(nodeBgList)
470 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
472 if self.config.doBgModel:
475 camera, cacheExposures, idList, self.config.bgModel
477 for cacheBg, newBg
in zip(cacheBgList, newCacheBgList):
478 cacheBg.append(newBg)
480 if self.config.doSky:
486 for cacheExp, skyCalib
in zip(cacheExposures, skyCalibs):
487 skyExp = self.sky.exposureToBackground(skyCalib)
488 cacheSky.append(skyExp)
489 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
490 measScales.append(scale)
492 scale = self.sky.solveScales(measScales)
493 self.log.info(
"Sky frame scale: %s" % (scale, ))
499 for cacheExp, nodeSky, nodeBgList
in zip(cacheExposures, cacheSky, cacheBgList):
500 self.sky.
subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
501 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
503 if self.config.doBgModel2:
507 camera, cacheExposures, idList, self.config.bgModel2
509 for cacheBg, newBg
in zip(cacheBgList, newBgList):
510 cacheBg.append(newBg)
514 image = makeCameraImage(camera, zip(idList, exposures))
516 return pipeBase.Struct(
522 """Load original image and restore the sky
524 This method runs on the slave nodes.
528 cache : `lsst.pipe.base.Struct`
538 cache.dataId = dataId
539 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=True).clone()
540 bgOld = cache.butler.get(
"calexpBackground", dataId, immediate=
True)
541 image = cache.exposure.getMaskedImage()
545 statsImage = bgData[0].getStatsImage()
548 image -= bgOld.getImage()
549 cache.bgList = afwMath.BackgroundList()
551 cache.bgList.append(bgData)
553 if self.config.doMaskObjects:
554 self.maskObjects.findObjects(cache.exposure)
559 """Serial implementation of self.loadImage() for Gen3.
561 Load and restore background to calExp
and calExpBkg.
566 Detector level calExp image to process.
567 calExpBkg : `lsst.afw.math.BackgroundList`
568 Detector level background list associated
with the calExp.
573 Background restored calExp.
574 bgList : `lsst.afw.math.BackgroundList`
575 New background list containing the restoration background.
577 image = calExp.getMaskedImage()
579 for bgOld
in calExpBkg:
580 statsImage = bgOld[0].getStatsImage()
583 image -= calExpBkg.getImage()
584 bgList = afwMath.BackgroundList()
585 for bgData
in calExpBkg:
586 bgList.append(bgData)
588 if self.config.doMaskObjects:
589 self.maskObjects.findObjects(calExp)
591 return (calExp, bgList)
594 """Measure scale for sky frame
596 This method runs on the slave nodes.
600 cache : `lsst.pipe.base.Struct`
610 assert cache.dataId == dataId
611 cache.sky = self.sky.getSkyData(cache.butler, dataId)
612 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
616 """Subtract sky frame
618 This method runs on the slave nodes.
622 cache : `lsst.pipe.base.Struct`
634 assert cache.dataId == dataId
635 self.sky.
subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
639 """Fit background model for CCD
641 This method runs on the slave nodes.
645 cache : `lsst.pipe.base.Struct`
647 data : `lsst.pipe.base.Struct`
648 Data identifier, with `dataId` (data identifier)
and `bgModel`
649 (background model) elements.
656 assert cache.dataId == data.dataId
657 data.bgModel.addCcd(cache.exposure)
661 """Subtract background model
663 This method runs on the slave nodes.
667 cache : `lsst.pipe.base.Struct`
671 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
679 assert cache.dataId == dataId
680 exposure = cache.exposure
681 image = exposure.getMaskedImage()
682 detector = exposure.getDetector()
683 bbox = image.getBBox()
685 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
686 image -= cache.bgModel.getImage()
688 self.log.error(f
"There was an error processing {dataId}, no calib file produced")
690 cache.bgList.append(cache.bgModel[0])
694 """Serial implementation of self.subtractModel() for Gen3.
696 Load and restore background to calExp
and calExpBkg.
701 Exposure to subtract the background model
from.
703 Full camera level background model.
708 Background subtracted input exposure.
709 bgModelCcd : `lsst.afw.math.BackgroundList`
710 Detector level realization of the full background model.
712 Background model
from the bgModelCcd realization.
714 image = exposure.getMaskedImage()
715 detector = exposure.getDetector()
716 bbox = image.getBBox()
717 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
718 image -= bgModelCcd.getImage()
720 return (exposure, bgModelCcd, bgModelCcd[0])
723 """Generate an image of the background model for visualisation
725 Useful for debugging.
729 cache : `lsst.pipe.base.Struct`
733 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
741 Binned background model image.
743 assert cache.dataId == dataId
744 exposure = cache.exposure
745 detector = exposure.getDetector()
746 bbox = exposure.getMaskedImage().getBBox()
747 image = bgModel.toCcdBackground(detector, bbox).getImage()
751 """Return the binned image required for visualization
753 This method just helps to cut down on boilerplate.
758 Image to go into visualisation.
767 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
770 """Collect exposure for potential visualisation
772 This method runs on the slave nodes.
776 cache : `lsst.pipe.base.Struct`
789 """Collect original image for visualisation
791 This method runs on the slave nodes.
795 cache : `lsst.pipe.base.Struct`
807 exposure = cache.butler.get("calexp", dataId, immediate=
True)
811 """Collect original image for visualisation
813 This method runs on the slave nodes.
817 cache : `lsst.pipe.base.Struct`
832 """Collect mask for visualisation
834 This method runs on the slave nodes.
838 cache : `lsst.pipe.base.Struct`
851 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
852 image.array[:] = cache.exposure.maskedImage.mask.array
856 """Write resultant background list
858 This method runs on the slave nodes.
862 cache : `lsst.pipe.base.Struct`
867 cache.butler.put(cache.bgList, "skyCorr", dataId)
869 def _getMetadataName(self):
870 """There's no metadata to write out"""
def logOperation(self, operation, catch=False, trace=True)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def subtractModel(self, cache, dataId, bgModel)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def subtractModelRun(self, exposure, bgModel)
def __init__(self, *args, **kwargs)
def accumulateModel(self, cache, data)
def subtractSkyFrame(self, cache, dataId, scale)
def measureSkyFrame(self, cache, dataId)
def collectBinnedImage(self, exposure, image)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def loadImage(self, cache, dataId)
def collectMask(self, cache, dataId)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def batchWallTime(cls, time, parsedCmd, numCores)
def collectOriginal(self, cache, dataId)
def runDataRef(self, expRef)
def realiseModel(self, cache, dataId, bgModel)
def write(self, cache, dataId)
def collectSky(self, cache, dataId)
def loadImageRun(self, calExp, calExpBkg)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)