Coverage for python/lsst/summit/utils/bestEffort.py: 14%
93 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-28 11:53 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-28 11:53 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import logging
23from lsst.ip.isr import IsrTask
24import lsst.daf.butler as dafButler
25from lsst.daf.butler.registry import ConflictingDefinitionError
27from lsst.summit.utils.quickLook import QuickLookIsrTask
28from lsst.summit.utils.butlerUtils import getLatissDefaultCollections, datasetExists
30# TODO: add attempt for fringe once registry & templates are fixed
32CURRENT_RUN = "LATISS/runs/quickLook/1"
33ALLOWED_REPOS = ['/repo/main', '/repo/LATISS', '/readonly/repo/main']
36class BestEffortIsr():
37 """Class for getting an assembled image with the maximum amount of isr.
39 BestEffortIsr.getExposure(dataId) returns an assembled image with as much
40 isr performed as possible, dictated by the calibration products available,
41 and optionally interpolates over cosmic rays. If an image image already
42 exists in the butler, it is returned (for the sake of speed), otherwise it
43 is generated and put(). Calibration products are loaded and cached to
44 improve performance.
46 This class uses the ``quickLookIsrTask``, see docs there for details.
48 Acceptable repodir values are currently listed in ALLOWED_REPOS. This will
49 be updated (removed) once DM-33849 is done.
51 defaultExtraIsrOptions is a dict of options applied to all images.
53 Parameters
54 ----------
55 repoDir : `str`
56 The repo root. Will be removed after DM-33849.
57 extraCollections : `list` of `str`, optional
58 Extra collections to add to the butler init. Collections are prepended.
59 defaultExtraIsrOptions : `dict`, optional
60 A dict of extra isr config options to apply. Each key should be an
61 attribute of an isrTaskConfigClass.
62 doRepairCosmics : `bool`, optional
63 Repair cosmic ray hits?
64 doWrite : `bool`, optional
65 Write the outputs to the quickLook rerun/collection?
67 Raises
68 ------
69 FileNotFoundError:
70 Raised when a butler cannot be automatically instantiated using
71 the DAF_BUTLER_REPOSITORY_INDEX environment variable.
72 """
73 _datasetName = 'quickLookExp'
75 def __init__(self, *,
76 extraCollections=[],
77 defaultExtraIsrOptions={},
78 doRepairCosmics=True,
79 doWrite=True,
80 embargo=False):
81 self.log = logging.getLogger(__name__)
83 collections = getLatissDefaultCollections()
84 self.collections = extraCollections + collections
85 self.log.info(f'Instantiating butler with collections={self.collections}')
86 try:
87 repoString = "LATISS" if not embargo else "/repo/embargo"
88 self.butler = dafButler.Butler(repoString, collections=self.collections,
89 instrument='LATISS',
90 run=CURRENT_RUN if doWrite else None)
91 except(FileNotFoundError, RuntimeError):
92 # Depending on the value of DAF_BUTLER_REPOSITORY_INDEX and whether
93 # it is present and blank, or just not set, both these exception
94 # types can be raised, see
95 # tests/test_butlerUtils.py:ButlerInitTestCase
96 # for details and tests which confirm these have not changed
97 raise FileNotFoundError # unify exception type
99 quickLookIsrConfig = QuickLookIsrTask.ConfigClass()
100 quickLookIsrConfig.doRepairCosmics = doRepairCosmics
101 self.doWrite = doWrite # the task, as run by run() method, can't do the write, so we handle in here
102 self.quickLookIsrTask = QuickLookIsrTask(config=quickLookIsrConfig)
104 self.defaultExtraIsrOptions = defaultExtraIsrOptions
106 self._cache = {}
108 def _applyConfigOverrides(self, config, overrides):
109 """Update a config class with a dict of options.
111 Parameters
112 ----------
113 config : `lsst.pex.config.Config`
114 The config class to update.
115 overrides : `dict`
116 The override options as a dict.
118 Raises
119 ------
120 ValueError
121 Raised if the override option isn't found in the config.
122 """
123 for option, value in overrides.items():
124 if hasattr(config, option):
125 setattr(config, option, value)
126 self.log.info(f"Set isr config override {option} to {value}")
127 else:
128 raise ValueError(f"Override option {option} not found in isrConfig")
130 @staticmethod
131 def _parseExpIdOrDataId(expIdOrDataId, **kwargs):
132 """Sanitize the expIdOrDataId to allow support both expIds and dataIds
134 Supports expId as an integer, or a complete or partial dict. The dict
135 is updated with the supplied kwargs.
137 Parameters
138 ----------
139 expIdOrDataId : `int` or `dict
140 The exposure id as an int or the dataId as as dict, or an expRecord
141 or a dataCoordinate.
143 Returns
144 -------
145 dataId : `dict`
146 The sanitized dataId.
147 """
148 match expIdOrDataId:
149 case int() as expId:
150 return {"expId": expId}
151 case dafButler.DataCoordinate() as dataId:
152 return dataId
153 case dafButler.DimensionRecord() as record:
154 return record.dataId
155 case dict() as dataId:
156 return dataId
157 raise RuntimeError(f"Invalid expId or dataId type {expIdOrDataId}: {type(expIdOrDataId)}")
159 def clearCache(self):
160 """Clear the internal cache of loaded calibration products.
162 Only necessary if you want to use an existing bestEffortIsr object
163 after adding new calibration products to the calibration collection.
164 """
165 self._cache = {}
167 def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, forceRemake=False,
168 **kwargs):
169 """Get the postIsr and cosmic-repaired image for this dataId.
171 Note that when using the forceRemake option the image will not be
172 written to the repo for reuse.
174 Parameters
175 ----------
176 expIdOrDataId : `dict`
177 The dataId
178 extraIsrOptions : `dict`, optional
179 extraIsrOptions is a dict of extra isr options applied to this
180 image only.
181 skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK!
182 Skip doing cosmic ray repair for this image?
183 forceRemake : `bool`
184 Remake the exposure even if there is a pre-existing one in the
185 repo. Images that are force-remade are never written, as this is
186 assumed to be used for testing/debug purposes, as opposed to normal
187 operation. For updating individual images, removal from the
188 registry can be used, and for bulk-updates the overall run number
189 can be incremented.
191 Returns
192 -------
193 exp : `lsst.afw.image.Exposure`
194 The postIsr exposure
195 """
196 dataId = self._parseExpIdOrDataId(expIdOrDataId)
198 if not forceRemake:
199 try:
200 exp = self.butler.get(self._datasetName, dataId, **kwargs)
201 self.log.info("Found a ready-made quickLookExp in the repo. Returning that.")
202 return exp
203 except LookupError:
204 pass
206 try:
207 raw = self.butler.get('raw', dataId, **kwargs)
208 except LookupError:
209 raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None
211 # default options that are probably good for most engineering time
212 isrConfig = IsrTask.ConfigClass()
213 isrConfig.doWrite = False # this task writes separately, no need for this
214 isrConfig.doSaturation = True # saturation very important for roundness measurement in qfm
215 isrConfig.doSaturationInterpolation = True
216 isrConfig.overscan.leadingColumnsToSkip = 5
217 isrConfig.overscan.fitType = 'MEDIAN_PER_ROW'
219 # apply general overrides
220 self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions)
221 # apply per-image overrides
222 self._applyConfigOverrides(isrConfig, extraIsrOptions)
224 isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel',
225 'bfGains', 'ptc']
227 isrDict = {}
228 # we build a cache of all the isr components which will be used to save
229 # the IO time on subsequent calls. This assumes people will not update
230 # calibration products while this object lives, but this is a fringe
231 # use case, and if they do, all they would need to do would be call
232 # .clearCache() and this will rebuild with the new products.
233 for component in isrParts:
234 if component in self._cache and component != 'flat':
235 self.log.info(f"Using {component} from cache...")
236 isrDict[component] = self._cache[component]
237 continue
238 if datasetExists(self.butler, component, dataId):
239 try:
240 # TODO: add caching for flats
241 item = self.butler.get(component, dataId=dataId)
242 self._cache[component] = item
243 isrDict[component] = self._cache[component]
244 self.log.info(f"Loaded {component} to cache")
245 except Exception: # now that we log the exception, we can catch all errors
246 # the product *should* exist but the get() failed, so log
247 # a very loud warning inc. the traceback as this is a sign
248 # of butler/database failures or something like that.
249 self.log.critical(f'Failed to find expected data product {component}!')
250 self.log.exception(f'Finding failure for {component}:')
251 else:
252 self.log.debug('No %s found for %s', component, dataId)
254 quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure
256 if self.doWrite and not forceRemake:
257 try:
258 self.butler.put(quickLookExp, self._datasetName, dataId, **kwargs)
259 self.log.info(f'Put {self._datasetName} for {dataId}')
260 except ConflictingDefinitionError:
261 # TODO: DM-34302 fix this message so that it's less scary for
262 # users. Do this by having daemons know they're daemons.
263 self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)')
264 pass
266 return quickLookExp