23import lsst.pipe.base
as pipeBase
25from lsst.pipe.base
import ArgumentParser, ConfigDatasetType
26from lsst.daf.butler
import DimensionGraph
27from lsst.pex.config import Config, Field, ConfigurableField, ConfigField
31 FocalPlaneBackgroundConfig, MaskObjectsTask)
33import lsst.pipe.base.connectionTypes
as cT
35__all__ = [
"SkyCorrectionConfig",
"SkyCorrectionTask"]
41 """Match the order of one list to another, padding if necessary
46 List to be reordered and padded. Elements can be any type.
48 Iterable of values to be compared
with outputKeys.
49 Length must match `inputList`
51 Iterable of values to be compared
with inputKeys.
53 Any value to be inserted where inputKey
not in outputKeys
58 Copy of inputList reordered per outputKeys
and padded
with `padWith`
59 so that the length matches length of outputKeys.
64 outputList.append(inputList[inputKeys.index(d)])
66 outputList.append(padWith)
71 """Make and write an image of an entire focal plane
78 List of detector ID
and CCD exposures (binned by `binning`).
79 filename : `str`, optional
82 Binning size that has been applied to images.
84 image = visualizeVisit.makeCameraImage(camera, dict(exp for exp
in exposures
if exp
is not None), binning)
85 if filename
is not None:
86 image.writeFits(filename)
90def _skyLookup(datasetType, registry, quantumDataId, collections):
91 """Lookup function to identify sky frames
95 datasetType : `lsst.daf.butler.DatasetType`
97 registry : `lsst.daf.butler.Registry`
98 Butler registry to query.
99 quantumDataId : `lsst.daf.butler.DataCoordinate`
100 Data id to transform to find sky frames.
101 The ``detector`` entry will be stripped.
102 collections : `lsst.daf.butler.CollectionSearch`
103 Collections to search through.
107 results : `list` [`lsst.daf.butler.DatasetRef`]
108 List of datasets that will be used as sky calibration frames
110 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"visit"]))
112 for dataId
in registry.queryDataIds([
"visit",
"detector"], dataId=newDataId).expanded():
113 skyFrame = registry.findDataset(datasetType, dataId, collections=collections,
114 timespan=dataId.timespan)
115 skyFrames.append(skyFrame)
121 rawLinker = cT.Input(
122 doc=
"Raw data to provide exp-visit linkage to connect calExp inputs to camera/sky calibs.",
126 storageClass=
"Exposure",
127 dimensions=[
"instrument",
"exposure",
"detector"],
129 calExpArray = cT.Input(
130 doc=
"Input exposures to process",
133 storageClass=
"ExposureF",
134 dimensions=[
"instrument",
"visit",
"detector"],
136 calBkgArray = cT.Input(
137 doc=
"Input background files to use",
139 name=
"calexpBackground",
140 storageClass=
"Background",
141 dimensions=[
"instrument",
"visit",
"detector"],
143 camera = cT.PrerequisiteInput(
144 doc=
"Input camera to use.",
146 storageClass=
"Camera",
147 dimensions=[
"instrument"],
150 skyCalibs = cT.PrerequisiteInput(
151 doc=
"Input sky calibrations to use.",
154 storageClass=
"ExposureF",
155 dimensions=[
"instrument",
"physical_filter",
"detector"],
157 lookupFunction=_skyLookup,
159 calExpCamera = cT.Output(
160 doc=
"Output camera image.",
161 name=
'calexp_camera',
162 storageClass=
"ImageF",
163 dimensions=[
"instrument",
"visit"],
166 doc=
"Output sky corrected images.",
169 storageClass=
"Background",
170 dimensions=[
"instrument",
"visit",
"detector"],
175 """Configuration for SkyCorrectionTask"""
176 bgModel = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"Background model")
177 bgModel2 = ConfigField(dtype=FocalPlaneBackgroundConfig, doc=
"2nd Background model")
178 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
179 maskObjects = ConfigurableField(target=MaskObjectsTask, doc=
"Mask Objects")
180 doMaskObjects = Field(dtype=bool, default=
True, doc=
"Mask objects to find good sky?")
181 doBgModel = Field(dtype=bool, default=
True, doc=
"Do background model subtraction?")
182 doBgModel2 = Field(dtype=bool, default=
True, doc=
"Do cleanup background model subtraction?")
183 doSky = Field(dtype=bool, default=
True, doc=
"Do sky frame subtraction?")
184 binning = Field(dtype=int, default=8, doc=
"Binning factor for constructing focal-plane images")
185 calexpType = Field(dtype=str, default=
"calexp",
186 doc=
"Should be set to fakes_calexp if you want to process calexps with fakes in.")
189 Config.setDefaults(self)
190 self.
bgModel2bgModel2.doSmooth =
True
194 self.
bgModel2bgModel2.smoothScale = 1.0
198 """Correct sky over entire focal plane"""
199 ConfigClass = SkyCorrectionConfig
200 _DefaultName =
"skyCorr"
205 detectorOrder = [ref.dataId[
'detector']
for ref
in inputRefs.calExpArray]
207 [ref.dataId[
'detector']
for ref
in inputRefs.skyCalibs],
210 [ref.dataId[
'detector']
for ref
in inputRefs.calBkgArray],
213 [ref.dataId[
'detector']
for ref
in outputRefs.skyCorr],
215 inputs = butlerQC.get(inputRefs)
216 inputs.pop(
"rawLinker",
None)
217 outputs = self.
runrun(**inputs)
218 butlerQC.put(outputs, outputRefs)
223 self.makeSubtask(
"sky")
224 self.makeSubtask(
"maskObjects")
227 def _makeArgumentParser(cls, *args, **kwargs):
228 kwargs.pop(
"doBatch",
False)
229 datasetType = ConfigDatasetType(name=
"calexpType")
230 parser = ArgumentParser(name=
"skyCorr", *args, **kwargs)
231 parser.add_id_argument(
"--id", datasetType=datasetType, level=
"visit",
232 help=
"data ID, e.g. --id visit=12345")
237 """Return walltime request for batch job
239 Subclasses should override if the walltime should be calculated
240 differently (e.g., addition of some serial time).
245 Requested time per iteration.
246 parsedCmd : `argparse.Namespace`
247 Results of argument parsing.
251 numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
252 return time*numTargets
255 """Perform sky correction on an exposure
257 We restore the original sky, and remove it again using multiple
258 algorithms. We optionally apply:
260 1. A large-scale background model.
261 This step removes very-large-scale sky such
as moonlight.
263 3. A medium-scale background model.
264 This step removes residual sky (This
is smooth on the focal plane).
266 Only the master node executes this method. The data
is held on
267 the slave nodes, which do all the hard work.
271 expRef : `lsst.daf.persistence.ButlerDataRef`
272 Data reference
for exposure.
276 ~lsst.pipe.drivers.SkyCorrectionTask.run
279 extension =
"-%(visit)d.fits" % expRef.dataId
281 with self.
logOperationlogOperation(
"processing %s" % (expRef.dataId,)):
284 pool.storeSet(butler=expRef.getButler())
285 camera = expRef.get(
"camera")
287 dataIdList = [ccdRef.dataId
for ccdRef
in expRef.subItems(
"ccd")
if
288 ccdRef.datasetExists(self.config.calexpType)]
290 exposures = pool.map(self.
loadImageloadImage, dataIdList)
293 exposures = pool.mapToPrevious(self.
collectOriginalcollectOriginal, dataIdList)
295 exposures = pool.mapToPrevious(self.
collectMaskcollectMask, dataIdList)
298 if self.config.doBgModel:
299 exposures = self.
focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel)
301 if self.config.doSky:
302 measScales = pool.mapToPrevious(self.
measureSkyFramemeasureSkyFrame, dataIdList)
303 scale = self.sky.solveScales(measScales)
304 self.log.info(
"Sky frame scale: %s" % (scale,))
306 exposures = pool.mapToPrevious(self.
subtractSkyFramesubtractSkyFrame, dataIdList, scale)
309 calibs = pool.mapToPrevious(self.
collectSkycollectSky, dataIdList)
312 if self.config.doBgModel2:
313 exposures = self.
focalPlaneBackgroundfocalPlaneBackground(camera, pool, dataIdList, self.config.bgModel2)
317 expRef.put(image,
"calexp_camera")
319 pool.mapToPrevious(self.
writewrite, dataIdList)
322 """Perform full focal-plane background subtraction
324 This method runs on the master node.
330 pool : `lsst.ctrl.pool.Pool`
332 dataIdList : iterable of `dict`
333 List of data identifiers for the CCDs.
335 Configuration to use
for background subtraction.
340 List of binned images,
for creating focal plane image.
342 bgModel = FocalPlaneBackground.fromCamera(config, camera)
343 data = [pipeBase.Struct(dataId=dataId, bgModel=bgModel.clone()) for dataId
in dataIdList]
344 bgModelList = pool.mapToPrevious(self.
accumulateModelaccumulateModel, data)
345 for ii, bg
in enumerate(bgModelList):
346 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.array.sum())
348 return pool.mapToPrevious(self.
subtractModelsubtractModel, dataIdList, bgModel)
351 """Perform full focal-plane background subtraction
353 This method runs on the master node.
359 cacheExposures : `list` of `lsst.afw.image.Exposures`
360 List of loaded and processed input calExp.
361 idList : `list` of `int`
362 List of detector ids to iterate over.
364 Configuration to use
for background subtraction.
369 List of binned images,
for creating focal plane image.
370 newCacheBgList : `list` of `lsst.afwMath.backgroundList`
371 Background lists generated.
372 cacheBgModel : `FocalPlaneBackground`
373 Full focal plane background model.
375 bgModel = FocalPlaneBackground.fromCamera(config, camera)
376 data = [pipeBase.Struct(id=id, bgModel=bgModel.clone()) for id
in idList]
379 for nodeData, cacheExp
in zip(data, cacheExposures):
380 nodeData.bgModel.addCcd(cacheExp)
381 bgModelList.append(nodeData.bgModel)
383 for ii, bg
in enumerate(bgModelList):
384 self.log.info(
"Background %d: %d pixels", ii, bg._numbers.getArray().sum())
390 for cacheExp
in cacheExposures:
391 nodeExp, nodeBgModel, nodeBgList = self.
subtractModelRunsubtractModelRun(cacheExp, bgModel)
392 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
393 cacheBgModel.append(nodeBgModel)
394 newCacheBgList.append(nodeBgList)
396 return exposures, newCacheBgList, cacheBgModel
398 def run(self, calExpArray, calBkgArray, skyCalibs, camera):
399 """Duplicate runDataRef method without ctrl_pool for Gen3.
404 Array of detector input calExp images for the exposure to
406 calBkgArray : `list` of `lsst.afw.math.BackgroundList`
407 Array of detector input background lists matching the
410 Array of SKY calibrations
for the input detectors to be
413 Camera matching the input data to process.
417 results : `pipeBase.Struct` containing
419 Full camera image of the sky-corrected data.
420 skyCorr : `list` of `lsst.afw.math.BackgroundList`
421 Detector-level sky-corrected background lists.
425 ~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
442 idList = [exp.getDetector().getId()
for exp
in calExpArray]
449 for calExp, calBgModel
in zip(calExpArray, calBkgArray):
450 nodeExp, nodeBgList = self.
loadImageRunloadImageRun(calExp, calBgModel)
451 cacheExposures.append(nodeExp)
452 cacheBgList.append(nodeBgList)
453 exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
455 if self.config.doBgModel:
458 camera, cacheExposures, idList, self.config.bgModel
460 for cacheBg, newBg
in zip(cacheBgList, newCacheBgList):
461 cacheBg.append(newBg)
463 if self.config.doSky:
469 for cacheExp, skyCalib
in zip(cacheExposures, skyCalibs):
470 skyExp = self.sky.exposureToBackground(skyCalib)
471 cacheSky.append(skyExp)
472 scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
473 measScales.append(scale)
475 scale = self.sky.solveScales(measScales)
476 self.log.info(
"Sky frame scale: %s" % (scale, ))
482 for cacheExp, nodeSky, nodeBgList
in zip(cacheExposures, cacheSky, cacheBgList):
483 self.sky.
subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
484 exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
486 if self.config.doBgModel2:
490 camera, cacheExposures, idList, self.config.bgModel2
492 for cacheBg, newBg
in zip(cacheBgList, newBgList):
493 cacheBg.append(newBg)
499 return pipeBase.Struct(
505 """Load original image and restore the sky
507 This method runs on the slave nodes.
511 cache : `lsst.pipe.base.Struct`
521 cache.dataId = dataId
522 cache.exposure = cache.butler.get(self.config.calexpType, dataId, immediate=True).clone()
523 bgOld = cache.butler.get(
"calexpBackground", dataId, immediate=
True)
524 image = cache.exposure.getMaskedImage()
528 statsImage = bgData[0].getStatsImage()
531 image -= bgOld.getImage()
532 cache.bgList = afwMath.BackgroundList()
534 cache.bgList.append(bgData)
536 if self.config.doMaskObjects:
537 self.maskObjects.findObjects(cache.exposure)
539 return self.
collectcollect(cache)
542 """Serial implementation of self.loadImage() for Gen3.
544 Load and restore background to calExp
and calExpBkg.
549 Detector level calExp image to process.
550 calExpBkg : `lsst.afw.math.BackgroundList`
551 Detector level background list associated
with the calExp.
556 Background restored calExp.
557 bgList : `lsst.afw.math.BackgroundList`
558 New background list containing the restoration background.
560 image = calExp.getMaskedImage()
562 for bgOld
in calExpBkg:
563 statsImage = bgOld[0].getStatsImage()
566 image -= calExpBkg.getImage()
567 bgList = afwMath.BackgroundList()
568 for bgData
in calExpBkg:
569 bgList.append(bgData)
571 if self.config.doMaskObjects:
572 self.maskObjects.findObjects(calExp)
574 return (calExp, bgList)
577 """Measure scale for sky frame
579 This method runs on the slave nodes.
583 cache : `lsst.pipe.base.Struct`
593 assert cache.dataId == dataId
594 cache.sky = self.sky.getSkyData(cache.butler, dataId)
595 scale = self.sky.measureScale(cache.exposure.getMaskedImage(), cache.sky)
599 """Subtract sky frame
601 This method runs on the slave nodes.
605 cache : `lsst.pipe.base.Struct`
617 assert cache.dataId == dataId
618 self.sky.
subtractSkyFrame(cache.exposure.getMaskedImage(), cache.sky, scale, cache.bgList)
619 return self.
collectcollect(cache)
622 """Fit background model for CCD
624 This method runs on the slave nodes.
628 cache : `lsst.pipe.base.Struct`
630 data : `lsst.pipe.base.Struct`
631 Data identifier, with `dataId` (data identifier)
and `bgModel`
632 (background model) elements.
639 assert cache.dataId == data.dataId
640 data.bgModel.addCcd(cache.exposure)
644 """Subtract background model
646 This method runs on the slave nodes.
650 cache : `lsst.pipe.base.Struct`
654 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
662 assert cache.dataId == dataId
663 exposure = cache.exposure
664 image = exposure.getMaskedImage()
665 detector = exposure.getDetector()
666 bbox = image.getBBox()
668 cache.bgModel = bgModel.toCcdBackground(detector, bbox)
669 image -= cache.bgModel.getImage()
671 self.log.error(f
"There was an error processing {dataId}, no calib file produced")
673 cache.bgList.append(cache.bgModel[0])
674 return self.
collectcollect(cache)
677 """Serial implementation of self.subtractModel() for Gen3.
679 Load and restore background to calExp
and calExpBkg.
684 Exposure to subtract the background model
from.
686 Full camera level background model.
691 Background subtracted input exposure.
692 bgModelCcd : `lsst.afw.math.BackgroundList`
693 Detector level realization of the full background model.
695 Background model
from the bgModelCcd realization.
697 image = exposure.getMaskedImage()
698 detector = exposure.getDetector()
699 bbox = image.getBBox()
700 bgModelCcd = bgModel.toCcdBackground(detector, bbox)
701 image -= bgModelCcd.getImage()
703 return (exposure, bgModelCcd, bgModelCcd[0])
706 """Generate an image of the background model for visualisation
708 Useful for debugging.
712 cache : `lsst.pipe.base.Struct`
716 bgModel : `lsst.pipe.drivers.background.FocalPlaneBackround`
724 Binned background model image.
726 assert cache.dataId == dataId
727 exposure = cache.exposure
728 detector = exposure.getDetector()
729 bbox = exposure.getMaskedImage().getBBox()
730 image = bgModel.toCcdBackground(detector, bbox).getImage()
734 """Return the binned image required for visualization
736 This method just helps to cut down on boilerplate.
741 Image to go into visualisation.
750 return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
753 """Collect exposure for potential visualisation
755 This method runs on the slave nodes.
759 cache : `lsst.pipe.base.Struct`
769 return self.
collectBinnedImagecollectBinnedImage(cache.exposure, cache.exposure.maskedImage)
772 """Collect original image for visualisation
774 This method runs on the slave nodes.
778 cache : `lsst.pipe.base.Struct`
790 exposure = cache.butler.get("calexp", dataId, immediate=
True)
794 """Collect original image for visualisation
796 This method runs on the slave nodes.
800 cache : `lsst.pipe.base.Struct`
812 return self.
collectBinnedImagecollectBinnedImage(cache.exposure, cache.sky.getImage())
815 """Collect mask for visualisation
817 This method runs on the slave nodes.
821 cache : `lsst.pipe.base.Struct`
834 image = afwImage.ImageF(cache.exposure.maskedImage.getBBox())
835 image.array[:] = cache.exposure.maskedImage.mask.array
839 """Write resultant background list
841 This method runs on the slave nodes.
845 cache : `lsst.pipe.base.Struct`
850 cache.butler.put(cache.bgList, "skyCorr", dataId)
852 def _getMetadataName(self):
853 """There's no metadata to write out"""
def logOperation(self, operation, catch=False, trace=True)
def run(self, calExpArray, calBkgArray, skyCalibs, camera)
def subtractModel(self, cache, dataId, bgModel)
def focalPlaneBackground(self, camera, pool, dataIdList, config)
def subtractModelRun(self, exposure, bgModel)
def __init__(self, *args, **kwargs)
def accumulateModel(self, cache, data)
def subtractSkyFrame(self, cache, dataId, scale)
def measureSkyFrame(self, cache, dataId)
def collectBinnedImage(self, exposure, image)
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config)
def loadImage(self, cache, dataId)
def collectMask(self, cache, dataId)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def batchWallTime(cls, time, parsedCmd, numCores)
def collectOriginal(self, cache, dataId)
def runDataRef(self, expRef)
def realiseModel(self, cache, dataId, bgModel)
def write(self, cache, dataId)
def collectSky(self, cache, dataId)
def loadImageRun(self, calExp, calExpBkg)
def makeCameraImage(camera, exposures, filename=None, binning=8)
def reorderAndPadList(inputList, inputKeys, outputKeys, padWith=None)