Coverage for python/lsst/summit/utils/bestEffort.py: 17%
86 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-26 11:03 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-26 11:03 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from sqlite3 import OperationalError
24import logging
25from lsst.ip.isr import IsrTask
26import lsst.daf.butler as dafButler
27from lsst.daf.butler.registry import ConflictingDefinitionError
29from lsst.summit.utils.quickLook import QuickLookIsrTask
30from lsst.summit.utils.butlerUtils import getLatissDefaultCollections
32# TODO: add attempt for fringe once registry & templates are fixed
34CURRENT_RUN = "LATISS/runs/quickLook/1"
35ALLOWED_REPOS = ['/repo/main', '/repo/LATISS', '/readonly/repo/main']
38class BestEffortIsr():
39 """Class for getting an assembled image with the maximum amount of isr.
41 BestEffortIsr.getExposure(dataId) returns an assembled image with as much
42 isr performed as possible, dictated by the calibration products available,
43 and optionally interpolates over cosmic rays. If an image image already
44 exists in the butler, it is returned (for the sake of speed), otherwise it
45 is generated and put(). Calibration products are loaded and cached to
46 improve performance.
48 This class uses the ``quickLookIsrTask``, see docs there for details.
50 Acceptable repodir values are currently listed in ALLOWED_REPOS. This will
51 be updated (removed) once DM-33849 is done.
53 defaultExtraIsrOptions is a dict of options applied to all images.
55 Parameters
56 ----------
57 repoDir : `str`
58 The repo root. Will be removed after DM-33849.
59 extraCollections : `list` of `str`, optional
60 Extra collections to add to the butler init. Collections are prepended.
61 defaultExtraIsrOptions : `dict`, optional
62 A dict of extra isr config options to apply. Each key should be an
63 attribute of an isrTaskConfigClass.
64 doRepairCosmics : `bool`, optional
65 Repair cosmic ray hits?
66 doWrite : `bool`, optional
67 Write the outputs to the quickLook rerun/collection?
69 Raises
70 ------
71 FileNotFoundError:
72 Raised when a butler cannot be automatically instantiated using
73 the DAF_BUTLER_REPOSITORY_INDEX environment variable.
74 """
75 _datasetName = 'quickLookExp'
77 def __init__(self, *,
78 extraCollections=[], defaultExtraIsrOptions={}, doRepairCosmics=True, doWrite=True):
79 self.log = logging.getLogger(__name__)
81 collections = getLatissDefaultCollections()
82 self.collections = extraCollections + collections
83 self.log.info(f'Instantiating butler with collections={self.collections}')
84 try:
85 self.butler = dafButler.Butler('LATISS', collections=self.collections,
86 instrument='LATISS',
87 run=CURRENT_RUN if doWrite else None)
88 except(FileNotFoundError, RuntimeError):
89 # Depending on the value of DAF_BUTLER_REPOSITORY_INDEX and whether
90 # it is present and blank, or just not set, both these exception
91 # types can be raised, see
92 # tests/test_butlerUtils.py:ButlerInitTestCase
93 # for details and tests which confirm these have not changed
94 raise FileNotFoundError # unify exception type
96 quickLookIsrConfig = QuickLookIsrTask.ConfigClass()
97 quickLookIsrConfig.doRepairCosmics = doRepairCosmics
98 self.doWrite = doWrite # the task, as run by run() method, can't do the write, so we handle in here
99 self.quickLookIsrTask = QuickLookIsrTask(config=quickLookIsrConfig)
101 self.defaultExtraIsrOptions = defaultExtraIsrOptions
103 self._cache = {}
105 def _applyConfigOverrides(self, config, overrides):
106 """Update a config class with a dict of options.
108 Parameters
109 ----------
110 config : `lsst.pex.config.Config`
111 The config class to update.
112 overrides : `dict`
113 The override options as a dict.
115 Raises
116 ------
117 ValueError
118 Raised if the override option isn't found in the config.
119 """
120 for option, value in overrides.items():
121 if hasattr(config, option):
122 setattr(config, option, value)
123 self.log.info(f"Set isr config override {option} to {value}")
124 else:
125 raise ValueError(f"Override option {option} not found in isrConfig")
127 @staticmethod
128 def _parseExpIdOrDataId(expIdOrDataId, **kwargs):
129 """Sanitize the expIdOrDataId to allow support both expIds and dataIds
131 Supports expId as an integer, or a complete or partial dict. The dict
132 is updated with the supplied kwargs.
134 Parameters
135 ----------
136 expIdOrDataId : `int` or `dict
137 The exposure id as an int or the dataId as as dict.
139 Returns
140 -------
141 dataId : `dict`
142 The sanitized dataId.
143 """
144 if type(expIdOrDataId) == int:
145 _dataId = {'expId': expIdOrDataId}
146 elif type(expIdOrDataId) == dict:
147 _dataId = expIdOrDataId
148 _dataId.update(kwargs)
149 else:
150 raise RuntimeError(f"Invalid expId or dataId type {expIdOrDataId}")
151 return _dataId
153 def clearCache(self):
154 """Clear the internal cache of loaded calibration products.
156 Only necessary if you want to use an existing bestEffortIsr object
157 after adding new calibration products to the calibration collection.
158 """
159 self._cache = {}
161 def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, forceRemake=False,
162 **kwargs):
163 """Get the postIsr and cosmic-repaired image for this dataId.
165 Note that when using the forceRemake option the image will not be
166 written to the repo for reuse.
168 Parameters
169 ----------
170 expIdOrDataId : `dict`
171 The dataId
172 extraIsrOptions : `dict`, optional
173 extraIsrOptions is a dict of extra isr options applied to this
174 image only.
175 skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK!
176 Skip doing cosmic ray repair for this image?
177 forceRemake : `bool`
178 Remake the exposure even if there is a pre-existing one in the
179 repo. Images that are force-remade are never written, as this is
180 assumed to be used for testing/debug purposes, as opposed to normal
181 operation. For updating individual images, removal from the
182 registry can be used, and for bulk-updates the overall run number
183 can be incremented.
185 Returns
186 -------
187 exp : `lsst.afw.image.Exposure`
188 The postIsr exposure
189 """
190 dataId = self._parseExpIdOrDataId(expIdOrDataId, **kwargs)
192 if not forceRemake:
193 try:
194 exp = self.butler.get(self._datasetName, dataId=dataId)
195 self.log.info("Found a ready-made quickLookExp in the repo. Returning that.")
196 return exp
197 except LookupError:
198 pass
200 try:
201 raw = self.butler.get('raw', dataId=dataId)
202 except LookupError:
203 raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None
205 # default options that are probably good for most engineering time
206 isrConfig = IsrTask.ConfigClass()
207 isrConfig.doWrite = False # this task writes separately, no need for this
208 isrConfig.doSaturation = True # saturation very important for roundness measurement in qfm
209 isrConfig.doSaturationInterpolation = True
210 isrConfig.overscan.leadingColumnsToSkip = 5
211 isrConfig.overscan.fitType = 'MEDIAN_PER_ROW'
213 # apply general overrides
214 self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions)
215 # apply per-image overrides
216 self._applyConfigOverrides(isrConfig, extraIsrOptions)
218 isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel',
219 'bfGains', 'ptc']
221 isrDict = {}
222 # we build a cache of all the isr components which will be used to save
223 # the IO time on subsequent calls. This assumes people will not update
224 # calibration products while this object lives, but this is a fringe
225 # use case, and if they do, all they would need to do would be call
226 # .clearCache() and this will rebuild with the new products.
227 for component in isrParts:
228 if component in self._cache and component != 'flat':
229 self.log.info(f"Using {component} from cache...")
230 isrDict[component] = self._cache[component]
231 continue
232 try:
233 # TODO: add caching for flats
234 item = self.butler.get(component, dataId=dataId)
235 self._cache[component] = item
236 isrDict[component] = self._cache[component]
237 except (RuntimeError, LookupError, OperationalError):
238 pass
240 quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure
242 if self.doWrite and not forceRemake:
243 try:
244 self.butler.put(quickLookExp, self._datasetName, dataId)
245 self.log.info(f'Put {self._datasetName} for {dataId}')
246 except ConflictingDefinitionError:
247 # TODO: DM-34302 fix this message so that it's less scary for
248 # users. Do this by having daemons know they're daemons.
249 self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)')
250 pass
252 return quickLookExp