23import lsst.pipe.base
as pipeBase
25from lsst.pipe.base
import ArgumentParser, ConfigDatasetType
26from lsst.daf.butler
import DimensionGraph
27from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
31 FocalPlaneBackgroundConfig, MaskObjectsTask)
33import lsst.pipe.base.connectionTypes
as cT
35__all__ = [
"SkyCorrectionConfig",
"SkyCorrectionTask"]
41 """Match the order of one list to another, padding if necessary
46 List to be reordered and padded. Elements can be any type.
48 Iterable of values to be compared
with outputKeys.
49 Length must match `inputList`
51 Iterable of values to be compared
with inputKeys.
53 Any value to be inserted where inputKey
not in outputKeys
58 Copy of inputList reordered per outputKeys
and padded
with `padWith`
59 so that the length matches length of outputKeys.
64 outputList.append(inputList[inputKeys.index(d)])
66 outputList.append(padWith)
71 """Make and write an image of an entire focal plane
78 List of detector ID
and CCD exposures (binned by `binning`).
79 filename : `str`, optional
82 Binning size that has been applied to images.
84 image = visualizeVisit.makeCameraImage(camera, dict(exp for exp
in exposures
if exp
is not None), binning)
85 if filename
is not None:
86 image.writeFits(filename)
90def _skyLookup(datasetType, registry, quantumDataId, collections):
91 """Lookup function to identify sky frames
95 datasetType : `lsst.daf.butler.DatasetType`
97 registry : `lsst.daf.butler.Registry`
98 Butler registry to query.
99 quantumDataId : `lsst.daf.butler.DataCoordinate`
100 Data id to transform to find sky frames.
101 The ``detector`` entry will be stripped.
102 collections : `lsst.daf.butler.CollectionSearch`
103 Collections to search through.
107 results : `list` [`lsst.daf.butler.DatasetRef`]
108 List of datasets that will be used as sky calibration frames
110 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"visit"]))
112 for dataId
in registry.queryDataIds([
"visit",
"detector"], dataId=newDataId).expanded():
113 skyFrame = registry.findDataset(datasetType, dataId, collections=collections,
114 timespan=dataId.timespan)
115 skyFrames.append(skyFrame)
121 rawLinker = cT.Input(
122 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
126 storageClass=
"Exposure",
127 dimensions=[
"instrument",
"exposure",
"detector"],
129 calExpArray = cT.Input(
130 doc=
"Input exposures to process",
133 storageClass=
"ExposureF",
134 dimensions=[
"instrument",
"visit",
"detector"],
136 calBkgArray = cT.Input(
137 doc=
"Input background files to use",
139 name=
"calexpBackground",
140 storageClass=
"Background",
141 dimensions=[
"instrument",
"visit",
"detector"],
143 camera = cT.PrerequisiteInput(
144 doc=
"Input camera to use.",
146 storageClass=
"Camera",
147 dimensions=[
"instrument"],
150 skyCalibs = cT.PrerequisiteInput(
151 doc=
"Input sky calibrations to use.",
154 storageClass=
"ExposureF",
155 dimensions=[
"instrument",
"physical_filter",
"detector"],
157 lookupFunction=_skyLookup,
159 calExpCamera = cT.Output(
160 doc=
"Output camera image.",
161 name=
'calexp_camera',
162 storageClass=
"ImageF",
163 dimensions=[
"instrument",
"visit"],
166 doc=
"Output sky corrected images.",
169 storageClass=
"Background",
170 dimensions=[
"instrument",
"visit",
"detector"],
175 """Configuration for SkyCorrectionTask"""
176 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"Background model")
177 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"2nd Background model")
178 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
179 maskObjects = ConfigurableField(target=MaskObjectsTask, doc=
"Mask Objects")
180 doMaskObjects = Field(dtype=bool, default=
True, doc=
"Mask objects to find good sky?")
181 doBgModel = Field(dtype=bool, default=
True, doc=
"Do background model subtraction?")
182 doBgModel2 = Field(dtype=bool, default=
True, doc=
"Do cleanup background model subtraction?")
183 doSky = Field(dtype=bool, default=
True, doc=
"Do sky frame subtraction?")
184 binning = Field(dtype=int, default=8, doc=
"Binning factor for constructing focal-plane images")
185 calexpType = Field(dtype=str, default=
"calexp",
186 doc=
"Should be set to fakes_calexp if you want to process calexps with fakes in.")
189 Config.setDefaults(self)
190 self.
bgModel2bgModel2.doSmooth =
True
194 self.
bgModel2bgModel2.smoothScale = 1.0
198 """Correct sky over entire focal plane"""
199 ConfigClass = SkyCorrectionConfig
200 _DefaultName =
"skyCorr"
206 detectorOrder = [ref.dataId[
'detector']
for ref
in inputRefs.calExpArray]
209 [ref.dataId[
'detector']
for ref
in inputRefs.calExpArray],
212 [ref.dataId[
'detector']
for ref
in inputRefs.skyCalibs],
215 [ref.dataId[
'detector']
for ref
in inputRefs.calBkgArray],
218 [ref.dataId[
'detector']
for ref
in outputRefs.skyCorr],
220 inputs = butlerQC.get(inputRefs)
221 inputs.pop(
"rawLinker",
None)
222 outputs = self.
runrun(**inputs)
223 butlerQC.put(outputs, outputRefs)
228 self.makeSubtask(
"sky")
229 self.makeSubtask(
"maskObjects")
232 def _makeArgumentParser(cls, *args, **kwargs):
233 kwargs.pop(
"doBatch",
False)
234 datasetType = ConfigDatasetType(name=
"calexpType")
235 parser = ArgumentParser(name=
"skyCorr", *args, **kwargs)
236 parser.add_id_argument(
"--id", datasetType=datasetType, level=
"visit",
237 help=
"data ID, e.g. --id visit=12345")
242 """Return walltime request for batch job
244 Subclasses should override if the walltime should be calculated
245 differently (e.g., addition of some serial time).
250 Requested time per iteration.
251 parsedCmd : `argparse.Namespace`
252 Results of argument parsing.
256 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
257 return time*numTargets
260 """Perform sky correction on an exposure
262 We restore the original sky, and remove it again using multiple
263 algorithms. We optionally apply:
265 1. A large-scale background model.
266 This step removes very-large-scale sky such
as moonlight.
268 3. A medium-scale background model.
269 This step removes residual sky (This
is smooth on the focal plane).
271 Only the master node executes this method. The data
is held on
272 the slave nodes, which do all the hard work.
276 expRef : `lsst.daf.persistence.ButlerDataRef`
277 Data reference
for exposure.
281 ~lsst.pipe.drivers.SkyCorrectionTask.run
284 extension =
"-%(visit)d.fits" % expRef.dataId
286 with self.
logOperationlogOperation(
"processing %s" % (expRef.dataId,)):
289 pool.storeSet(butler=expRef.getButler())
290 camera = expRef.get(
"camera")
292 dataIdList = [ccdRef.dataId
for ccdRef
in expRef.subItems(
"ccd")
if
293 ccdRef.datasetExists(self.config.calexpType)]
294 ccdList = [dataId[
"ccd"]
for dataId
in dataIdList]
296 dataIdList = [dataId
for _, dataId
in sorted(zip(ccdList, dataIdList))]
298 exposures = pool.map(self.
loadImageloadImage, dataIdList)
301 exposures = pool.mapToPrevious(self.
collectOriginalcollectOriginal, dataIdList)
303 exposures = pool.mapToPrevious(self.
collectMaskcollectMask, dataIdList)
306 if self.config.doBgModel:
307 exposures = self.
focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel)
309 if self.config.doSky:
310 measScales = pool.mapToPrevious(self.
measureSkyFramemeasureSkyFrame, dataIdList)
311 scale = self.sky.solveScales(measScales)
312 self.log.info(
"Sky frame scale: %s" % (scale,))
314 exposures = pool.mapToPrevious(self.
subtractSkyFramesubtractSkyFrame, dataIdList, scale)
317 calibs = pool.mapToPrevious(self.
collectSkycollectSky, dataIdList)
320 if self.config.doBgModel2:
321 exposures = self.
focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel2)
325 expRef.put(image,
"calexp_camera")
327 pool.mapToPrevious(self.
writewrite, dataIdList)
330 """Perform full focal-plane background subtraction
332 This method runs on the master node.
338 pool : `lsst.ctrl.pool.Pool`
340 dataIdList : iterable of `dict`
341 List of data identifiers for the CCDs.
343 Configuration to use
for background subtraction.
348 List of binned images,
for creating focal plane image.
350 bgModel = FocalPlaneBackground.fromCamera(config, camera)
351 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId
in dataIdList]
352 bgModelList = pool.mapToPrevious(self.
accumulateModelaccumulateModel, data)
353 for ii, bg
in enumerate(bgModelList):
354 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.array.sum())
356 return pool.mapToPrevious(self.
subtractModelsubtractModel, dataIdList, bgModel)
359 """Perform full focal-plane background subtraction
361 This method runs on the master node.
367 cacheExposures : `list` of `lsst.afw.image.Exposures`
368 List of loaded and processed input calExp.
369 idList : `list` of `int`
370 List of detector ids to iterate over.
372 Configuration to use
for background subtraction.
377 List of binned images,
for creating focal plane image.
378 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
379 Background lists generated.
380 cacheBgModel : `FocalPlaneBackground`
381 Full focal plane background model.
383 bgModel = FocalPlaneBackground.fromCamera(config, camera)
384 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone()) for id
in idList]
387 for nodeData, cacheExp
in zip(data, cacheExposures):
388 nodeData.bgModel.addCcd(cacheExp)
389 bgModelList.append(nodeData.bgModel)
391 for ii, bg
in enumerate(bgModelList):
392 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.getArray().sum())
398 for cacheExp
in cacheExposures:
399 nodeExp, nodeBgModel, nodeBgList = self.
subtractModelRunsubtractModelRun(cacheExp, bgModel)
400 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
401 cacheBgModel.append(nodeBgModel)
402 newCacheBgList.append(nodeBgList)
404 return exposures, newCacheBgList, cacheBgModel
406 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
407 """Duplicate runDataRef method without ctrl_pool for Gen3.
412 Array of detector input calExp images for the exposure to
414 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
415 Array of detector input background lists matching the
418 Array of SKY calibrations
for the input detectors to be
421 Camera matching the input data to process.
425 results : `pipeBase.Struct` containing
427 Full camera image of the sky-corrected data.
428 skyCorr : `list` of `lsst.afw.math.BackgroundList`
429 Detector-level sky-corrected background lists.
433 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
450 idList = [exp.getDetector().getId()
for exp
in calExpArray]
457 for calExp, calBgModel
in zip(calExpArray, calBkgArray):
458 nodeExp, nodeBgList = self.
loadImageRunloadImageRun(calExp, calBgModel)
459 cacheExposures.append(nodeExp)
460 cacheBgList.append(nodeBgList)
461 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
463 if self.config.doBgModel:
466 camera, cacheExposures, idList, self.config.bgModel
468 for cacheBg, newBg
in zip(cacheBgList, newCacheBgList):
469 cacheBg.append(newBg)
471 if self.config.doSky:
477 for cacheExp, skyCalib
in zip(cacheExposures, skyCalibs):
478 skyExp = self.sky.exposureToBackground(skyCalib)
479 cacheSky.append(skyExp)
480 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
481 measScales.append(scale)
483 scale = self.sky.solveScales(measScales)
484 self.log.info(
"Sky frame scale: %s" % (scale, ))
490 for cacheExp, nodeSky, nodeBgList
in zip(cacheExposures, cacheSky, cacheBgList):
491 self.sky.
subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
492 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
494 if self.config.doBgModel2:
498 camera, cacheExposures, idList, self.config.bgModel2
500 for cacheBg, newBg
in zip(cacheBgList, newBgList):
501 cacheBg.append(newBg)
507 return pipeBase.Struct(
513 """Load original image and restore the sky
515 This method runs on the slave nodes.
519 cache : `lsst.pipe.base.Struct`
529 cache.dataId = dataId
530 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=True).clone()
531 bgOld = cache.butler.get(
"calexpBackground", dataId, immediate=
True)
532 image = cache.exposure.getMaskedImage()
536 statsImage = bgData[0].getStatsImage()
539 image -= bgOld.getImage()
540 cache.bgList = afwMath.BackgroundList()
542 cache.bgList.append(bgData)
544 if self.config.doMaskObjects:
545 self.maskObjects.findObjects(cache.exposure)
547 return self.
collectcollect(cache)
550 """Serial implementation of self.loadImage() for Gen3.
552 Load and restore background to calExp
and calExpBkg.
557 Detector level calExp image to process.
558 calExpBkg : `lsst.afw.math.BackgroundList`
559 Detector level background list associated
with the calExp.
564 Background restored calExp.
565 bgList : `lsst.afw.math.BackgroundList`
566 New background list containing the restoration background.
568 image = calExp.getMaskedImage()
570 for bgOld
in calExpBkg:
571 statsImage = bgOld[0].getStatsImage()
574 image -= calExpBkg.getImage()
575 bgList = afwMath.BackgroundList()
576 for bgData
in calExpBkg:
577 bgList.append(bgData)
579 if self.config.doMaskObjects:
580 self.maskObjects.findObjects(calExp)
582 return (calExp, bgList)
585 """Measure scale for sky frame
587 This method runs on the slave nodes.
591 cache : `lsst.pipe.base.Struct`
601 assert cache.dataId == dataId
602 cache.sky = self.sky.getSkyData(cache.butler, dataId)
603 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
607 """Subtract sky frame
609 This method runs on the slave nodes.
613 cache : `lsst.pipe.base.Struct`
625 assert cache.dataId == dataId
626 self.sky.
subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
627 return self.
collectcollect(cache)
630 """Fit background model for CCD
632 This method runs on the slave nodes.
636 cache : `lsst.pipe.base.Struct`
638 data : `lsst.pipe.base.Struct`
639 Data identifier, with `dataId` (data identifier)
and `bgModel`
640 (background model) elements.
647 assert cache.dataId == data.dataId
648 data.bgModel.addCcd(cache.exposure)
652 """Subtract background model
654 This method runs on the slave nodes.
658 cache : `lsst.pipe.base.Struct`
662 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
670 assert cache.dataId == dataId
671 exposure = cache.exposure
672 image = exposure.getMaskedImage()
673 detector = exposure.getDetector()
674 bbox = image.getBBox()
676 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
677 image -= cache.bgModel.getImage()
679 self.log.error(f
"There was an error processing {dataId}, no calib file produced")
681 cache.bgList.append(cache.bgModel[0])
682 return self.
collectcollect(cache)
685 """Serial implementation of self.subtractModel() for Gen3.
687 Load and restore background to calExp
and calExpBkg.
692 Exposure to subtract the background model
from.
694 Full camera level background model.
699 Background subtracted input exposure.
700 bgModelCcd : `lsst.afw.math.BackgroundList`
701 Detector level realization of the full background model.
703 Background model
from the bgModelCcd realization.
705 image = exposure.getMaskedImage()
706 detector = exposure.getDetector()
707 bbox = image.getBBox()
708 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
709 image -= bgModelCcd.getImage()
711 return (exposure, bgModelCcd, bgModelCcd[0])
714 """Generate an image of the background model for visualisation
716 Useful for debugging.
720 cache : `lsst.pipe.base.Struct`
724 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
732 Binned background model image.
734 assert cache.dataId == dataId
735 exposure = cache.exposure
736 detector = exposure.getDetector()
737 bbox = exposure.getMaskedImage().getBBox()
738 image = bgModel.toCcdBackground(detector, bbox).getImage()
742 """Return the binned image required for visualization
744 This method just helps to cut down on boilerplate.
749 Image to go into visualisation.
758 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
761 """Collect exposure for potential visualisation
763 This method runs on the slave nodes.
767 cache : `lsst.pipe.base.Struct`
777 return self.
collectBinnedImagecollectBinnedImage(cache.exposure, cache.exposure.maskedImage)
780 """Collect original image for visualisation
782 This method runs on the slave nodes.
786 cache : `lsst.pipe.base.Struct`
798 exposure = cache.butler.get("calexp", dataId, immediate=
True)
802 """Collect original image for visualisation
804 This method runs on the slave nodes.
808 cache : `lsst.pipe.base.Struct`
820 return self.
collectBinnedImagecollectBinnedImage(cache.exposure, cache.sky.getImage())
823 """Collect mask for visualisation
825 This method runs on the slave nodes.
829 cache : `lsst.pipe.base.Struct`
842 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
843 image.array[:] = cache.exposure.maskedImage.mask.array
847 """Write resultant background list
849 This method runs on the slave nodes.
853 cache : `lsst.pipe.base.Struct`
858 cache.butler.put(cache.bgList, "skyCorr", dataId)
860 def _getMetadataName(self):
861 """There's no metadata to write out"""
def logOperation(self, operation, catch=False, trace=True)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def subtractModel(self, cache, dataId, bgModel)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def subtractModelRun(self, exposure, bgModel)
def __init__(self, *args, **kwargs)
def accumulateModel(self, cache, data)
def subtractSkyFrame(self, cache, dataId, scale)
def measureSkyFrame(self, cache, dataId)
def collectBinnedImage(self, exposure, image)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def loadImage(self, cache, dataId)
def collectMask(self, cache, dataId)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def batchWallTime(cls, time, parsedCmd, numCores)
def collectOriginal(self, cache, dataId)
def runDataRef(self, expRef)
def realiseModel(self, cache, dataId, bgModel)
def write(self, cache, dataId)
def collectSky(self, cache, dataId)
def loadImageRun(self, calExp, calExpBkg)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)