Coverage for python/lsst/summit/utils/bestEffort.py: 17%
87 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-06 03:25 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-06 03:25 -0800
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from sqlite3 import OperationalError
24import logging
25from lsst.ip.isr import IsrTask
26import lsst.daf.butler as dafButler
27from lsst.daf.butler.registry import ConflictingDefinitionError
29from lsst.summit.utils.quickLook import QuickLookIsrTask
30from lsst.summit.utils.butlerUtils import getLatissDefaultCollections
32# TODO: add attempt for fringe once registry & templates are fixed
34CURRENT_RUN = "LATISS/runs/quickLook/1"
35ALLOWED_REPOS = ['/repo/main', '/repo/LATISS', '/readonly/repo/main']
38class BestEffortIsr():
39 """Class for getting an assembled image with the maximum amount of isr.
41 BestEffortIsr.getExposure(dataId) returns an assembled image with as much
42 isr performed as possible, dictated by the calibration products available,
43 and optionally interpolates over cosmic rays. If an image image already
44 exists in the butler, it is returned (for the sake of speed), otherwise it
45 is generated and put(). Calibration products are loaded and cached to
46 improve performance.
48 This class uses the ``quickLookIsrTask``, see docs there for details.
50 Acceptable repodir values are currently listed in ALLOWED_REPOS. This will
51 be updated (removed) once DM-33849 is done.
53 defaultExtraIsrOptions is a dict of options applied to all images.
55 Parameters
56 ----------
57 repoDir : `str`
58 The repo root. Will be removed after DM-33849.
59 extraCollections : `list` of `str`, optional
60 Extra collections to add to the butler init. Collections are prepended.
61 defaultExtraIsrOptions : `dict`, optional
62 A dict of extra isr config options to apply. Each key should be an
63 attribute of an isrTaskConfigClass.
64 doRepairCosmics : `bool`, optional
65 Repair cosmic ray hits?
66 doWrite : `bool`, optional
67 Write the outputs to the quickLook rerun/collection?
69 Raises
70 ------
71 FileNotFoundError:
72 Raised when a butler cannot be automatically instantiated using
73 the DAF_BUTLER_REPOSITORY_INDEX environment variable.
74 """
75 _datasetName = 'quickLookExp'
77 def __init__(self, *,
78 extraCollections=[],
79 defaultExtraIsrOptions={},
80 doRepairCosmics=True,
81 doWrite=True,
82 embargo=False):
83 self.log = logging.getLogger(__name__)
85 collections = getLatissDefaultCollections()
86 self.collections = extraCollections + collections
87 self.log.info(f'Instantiating butler with collections={self.collections}')
88 try:
89 repoString = "LATISS" if not embargo else "/repo/embargo"
90 self.butler = dafButler.Butler(repoString, collections=self.collections,
91 instrument='LATISS',
92 run=CURRENT_RUN if doWrite else None)
93 except(FileNotFoundError, RuntimeError):
94 # Depending on the value of DAF_BUTLER_REPOSITORY_INDEX and whether
95 # it is present and blank, or just not set, both these exception
96 # types can be raised, see
97 # tests/test_butlerUtils.py:ButlerInitTestCase
98 # for details and tests which confirm these have not changed
99 raise FileNotFoundError # unify exception type
101 quickLookIsrConfig = QuickLookIsrTask.ConfigClass()
102 quickLookIsrConfig.doRepairCosmics = doRepairCosmics
103 self.doWrite = doWrite # the task, as run by run() method, can't do the write, so we handle in here
104 self.quickLookIsrTask = QuickLookIsrTask(config=quickLookIsrConfig)
106 self.defaultExtraIsrOptions = defaultExtraIsrOptions
108 self._cache = {}
110 def _applyConfigOverrides(self, config, overrides):
111 """Update a config class with a dict of options.
113 Parameters
114 ----------
115 config : `lsst.pex.config.Config`
116 The config class to update.
117 overrides : `dict`
118 The override options as a dict.
120 Raises
121 ------
122 ValueError
123 Raised if the override option isn't found in the config.
124 """
125 for option, value in overrides.items():
126 if hasattr(config, option):
127 setattr(config, option, value)
128 self.log.info(f"Set isr config override {option} to {value}")
129 else:
130 raise ValueError(f"Override option {option} not found in isrConfig")
132 @staticmethod
133 def _parseExpIdOrDataId(expIdOrDataId, **kwargs):
134 """Sanitize the expIdOrDataId to allow support both expIds and dataIds
136 Supports expId as an integer, or a complete or partial dict. The dict
137 is updated with the supplied kwargs.
139 Parameters
140 ----------
141 expIdOrDataId : `int` or `dict
142 The exposure id as an int or the dataId as as dict.
144 Returns
145 -------
146 dataId : `dict`
147 The sanitized dataId.
148 """
149 if type(expIdOrDataId) == int:
150 _dataId = {'expId': expIdOrDataId}
151 elif type(expIdOrDataId) == dict:
152 _dataId = expIdOrDataId
153 _dataId.update(kwargs)
154 else:
155 raise RuntimeError(f"Invalid expId or dataId type {expIdOrDataId}")
156 return _dataId
158 def clearCache(self):
159 """Clear the internal cache of loaded calibration products.
161 Only necessary if you want to use an existing bestEffortIsr object
162 after adding new calibration products to the calibration collection.
163 """
164 self._cache = {}
166 def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, forceRemake=False,
167 **kwargs):
168 """Get the postIsr and cosmic-repaired image for this dataId.
170 Note that when using the forceRemake option the image will not be
171 written to the repo for reuse.
173 Parameters
174 ----------
175 expIdOrDataId : `dict`
176 The dataId
177 extraIsrOptions : `dict`, optional
178 extraIsrOptions is a dict of extra isr options applied to this
179 image only.
180 skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK!
181 Skip doing cosmic ray repair for this image?
182 forceRemake : `bool`
183 Remake the exposure even if there is a pre-existing one in the
184 repo. Images that are force-remade are never written, as this is
185 assumed to be used for testing/debug purposes, as opposed to normal
186 operation. For updating individual images, removal from the
187 registry can be used, and for bulk-updates the overall run number
188 can be incremented.
190 Returns
191 -------
192 exp : `lsst.afw.image.Exposure`
193 The postIsr exposure
194 """
195 dataId = self._parseExpIdOrDataId(expIdOrDataId, **kwargs)
197 if not forceRemake:
198 try:
199 exp = self.butler.get(self._datasetName, dataId=dataId)
200 self.log.info("Found a ready-made quickLookExp in the repo. Returning that.")
201 return exp
202 except LookupError:
203 pass
205 try:
206 raw = self.butler.get('raw', dataId=dataId)
207 except LookupError:
208 raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None
210 # default options that are probably good for most engineering time
211 isrConfig = IsrTask.ConfigClass()
212 isrConfig.doWrite = False # this task writes separately, no need for this
213 isrConfig.doSaturation = True # saturation very important for roundness measurement in qfm
214 isrConfig.doSaturationInterpolation = True
215 isrConfig.overscan.leadingColumnsToSkip = 5
216 isrConfig.overscan.fitType = 'MEDIAN_PER_ROW'
218 # apply general overrides
219 self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions)
220 # apply per-image overrides
221 self._applyConfigOverrides(isrConfig, extraIsrOptions)
223 isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel',
224 'bfGains', 'ptc']
226 isrDict = {}
227 # we build a cache of all the isr components which will be used to save
228 # the IO time on subsequent calls. This assumes people will not update
229 # calibration products while this object lives, but this is a fringe
230 # use case, and if they do, all they would need to do would be call
231 # .clearCache() and this will rebuild with the new products.
232 for component in isrParts:
233 if component in self._cache and component != 'flat':
234 self.log.info(f"Using {component} from cache...")
235 isrDict[component] = self._cache[component]
236 continue
237 try:
238 # TODO: add caching for flats
239 item = self.butler.get(component, dataId=dataId)
240 self._cache[component] = item
241 isrDict[component] = self._cache[component]
242 except (RuntimeError, LookupError, OperationalError):
243 pass
245 quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure
247 if self.doWrite and not forceRemake:
248 try:
249 self.butler.put(quickLookExp, self._datasetName, dataId)
250 self.log.info(f'Put {self._datasetName} for {dataId}')
251 except ConflictingDefinitionError:
252 # TODO: DM-34302 fix this message so that it's less scary for
253 # users. Do this by having daemons know they're daemons.
254 self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)')
255 pass
257 return quickLookExp