Coverage for python/lsst/summit/utils/bestEffort.py: 17%
85 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-18 02:50 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-18 02:50 -0700
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from sqlite3 import OperationalError
24import logging
25from lsst.ip.isr import IsrTask
26import lsst.daf.butler as dafButler
27from lsst.daf.butler.registry import ConflictingDefinitionError
29from lsst.summit.utils.quickLook import QuickLookIsrTask
30from lsst.summit.utils.butlerUtils import getLatissDefaultCollections
32# TODO: add attempt for fringe once registry & templates are fixed
34CURRENT_RUN = "LATISS/runs/quickLook/1"
35ALLOWED_REPOS = ['/repo/main', '/repo/LATISS', '/readonly/repo/main']
38class BestEffortIsr():
39 """Class for getting an assembled image with the maximum amount of isr.
41 BestEffortIsr.getExposure(dataId) returns an assembled image with as much
42 isr performed as possible, dictated by the calibration products available,
43 and optionally interpolates over cosmic rays. If an image image already
44 exists in the butler, it is returned (for the sake of speed), otherwise it
45 is generated and put(). Calibration products are loaded and cached to
46 improve performance.
48 This class uses the ``quickLookIsrTask``, see docs there for details.
50 Acceptable repodir values are currently listed in ALLOWED_REPOS. This will
51 be updated (removed) once DM-33849 is done.
53 defaultExtraIsrOptions is a dict of options applied to all images.
55 Parameters
56 ----------
57 repoDir : `str`
58 The repo root. Will be removed after DM-33849.
59 extraCollections : `list` of `str`, optional
60 Extra collections to add to the butler init. Collections are prepended.
61 defaultExtraIsrOptions : `dict`, optional
62 A dict of extra isr config options to apply. Each key should be an
63 attribute of an isrTaskConfigClass.
64 doRepairCosmics : `bool`, optional
65 Repair cosmic ray hits?
66 doWrite : `bool`, optional
67 Write the outputs to the quickLook rerun/collection?
69 Raises
70 ------
71 FileNotFoundError:
72 Raised when a butler cannot be automatically instantiated using
73 the DAF_BUTLER_REPOSITORY_INDEX environment variable.
74 """
75 _datasetName = 'quickLookExp'
77 def __init__(self, *,
78 extraCollections=[], defaultExtraIsrOptions={}, doRepairCosmics=True, doWrite=True):
79 self.log = logging.getLogger(__name__)
81 collections = getLatissDefaultCollections()
82 self.collections = extraCollections + collections
83 self.log.info(f'Instantiating butler with collections={self.collections}')
84 try:
85 self.butler = dafButler.Butler('LATISS', collections=self.collections,
86 instrument='LATISS',
87 run=CURRENT_RUN if doWrite else None)
88 except(FileNotFoundError, RuntimeError):
89 # Depending on the value of DAF_BUTLER_REPOSITORY_INDEX and whether
90 # it is present and blank, or just not set, both these exception
91 # types can be raised, see
92 # tests/test_butlerUtils.py:ButlerInitTestCase
93 # for details and tests which confirm these have not changed
94 raise FileNotFoundError # unify exception type
96 quickLookIsrConfig = QuickLookIsrTask.ConfigClass()
97 quickLookIsrConfig.doRepairCosmics = doRepairCosmics
98 self.doWrite = doWrite # the task, as run by run() method, can't do the write, so we handle in here
99 self.quickLookIsrTask = QuickLookIsrTask(config=quickLookIsrConfig)
101 self.defaultExtraIsrOptions = defaultExtraIsrOptions
103 self._cache = {}
105 def _applyConfigOverrides(self, config, overrides):
106 """Update a config class with a dict of options.
108 Parameters
109 ----------
110 config : `lsst.pex.config.Config`
111 The config class to update.
112 overrides : `dict`
113 The override options as a dict.
115 Raises
116 ------
117 ValueError
118 Raised if the override option isn't found in the config.
119 """
120 for option, value in overrides.items():
121 if hasattr(config, option):
122 setattr(config, option, value)
123 self.log.info(f"Set isr config override {option} to {value}")
124 else:
125 raise ValueError(f"Override option {option} not found in isrConfig")
127 @staticmethod
128 def _parseExpIdOrDataId(expIdOrDataId, **kwargs):
129 """Sanitize the expIdOrDataId to allow support both expIds and dataIds
131 Supports expId as an integer, or a complete or partial dict. The dict
132 is updated with the supplied kwargs.
134 Parameters
135 ----------
136 expIdOrDataId : `int` or `dict
137 The exposure id as an int or the dataId as as dict.
139 Returns
140 -------
141 dataId : `dict`
142 The sanitized dataId.
143 """
144 if type(expIdOrDataId) == int:
145 _dataId = {'expId': expIdOrDataId}
146 elif type(expIdOrDataId) == dict:
147 _dataId = expIdOrDataId
148 _dataId.update(kwargs)
149 else:
150 raise RuntimeError(f"Invalid expId or dataId type {expIdOrDataId}")
151 return _dataId
153 def clearCache(self):
154 """Clear the internal cache of loaded calibration products.
156 Only necessary if you want to use an existing bestEffortIsr object
157 after adding new calibration products to the calibration collection.
158 """
159 self._cache = {}
161 def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, **kwargs):
162 """Get the postIsr and cosmic-repaired image for this dataId.
164 Parameters
165 ----------
166 expIdOrDataId : `dict`
167 The dataId
168 extraIsrOptions : `dict`, optional
169 extraIsrOptions is a dict of extra isr options applied to this
170 image only.
171 skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK!
172 Skip doing cosmic ray repair for this image?
174 Returns
175 -------
176 exp : `lsst.afw.image.Exposure`
177 The postIsr exposure
178 """
179 dataId = self._parseExpIdOrDataId(expIdOrDataId, **kwargs)
181 try:
182 exp = self.butler.get(self._datasetName, dataId=dataId)
183 self.log.info("Found a ready-made quickLookExp in the repo. Returning that.")
184 return exp
185 except LookupError:
186 pass
188 try:
189 raw = self.butler.get('raw', dataId=dataId)
190 except LookupError:
191 raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None
193 # default options that are probably good for most engineering time
194 isrConfig = IsrTask.ConfigClass()
195 isrConfig.doWrite = False # this task writes separately, no need for this
196 isrConfig.doSaturation = True # saturation very important for roundness measurement in qfm
197 isrConfig.doSaturationInterpolation = True
198 isrConfig.overscan.leadingColumnsToSkip = 5
199 isrConfig.overscan.fitType = 'MEDIAN_PER_ROW'
201 # apply general overrides
202 self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions)
203 # apply per-image overrides
204 self._applyConfigOverrides(isrConfig, extraIsrOptions)
206 isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel',
207 'bfGains', 'ptc']
209 isrDict = {}
210 # we build a cache of all the isr components which will be used to save
211 # the IO time on subsequent calls. This assumes people will not update
212 # calibration products while this object lives, but this is a fringe
213 # use case, and if they do, all they would need to do would be call
214 # .clearCache() and this will rebuild with the new products.
215 for component in isrParts:
216 if component in self._cache and component != 'flat':
217 self.log.info(f"Using {component} from cache...")
218 isrDict[component] = self._cache[component]
219 continue
220 try:
221 # TODO: add caching for flats
222 item = self.butler.get(component, dataId=dataId)
223 self._cache[component] = item
224 isrDict[component] = self._cache[component]
225 except (RuntimeError, LookupError, OperationalError):
226 pass
228 quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure
230 if self.doWrite:
231 try:
232 self.butler.put(quickLookExp, self._datasetName, dataId)
233 self.log.info(f'Put {self._datasetName} for {dataId}')
234 except ConflictingDefinitionError:
235 # TODO: DM-34302 fix this message so that it's less scary for
236 # users. Do this by having daemons know they're daemons.
237 self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)')
238 pass
240 return quickLookExp