Coverage for python/lsst/ctrl/mpexec/preExecInit.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ['PreExecInit']
24# -------------------------------
25# Imports of standard modules --
26# -------------------------------
27import logging
28import itertools
30# -----------------------------
31# Imports for other modules --
32# -----------------------------
33from lsst.base import Packages
34from lsst.daf.butler import DatasetType
35from lsst.pipe.base import PipelineDatasetTypes
37_LOG = logging.getLogger(__name__.partition(".")[2])
40class PreExecInit:
41 """Initialization of registry for QuantumGraph execution.
43 This class encapsulates all necessary operations that have to be performed
44 on butler and registry to prepare them for QuantumGraph execution.
46 Parameters
47 ----------
48 butler : `~lsst.daf.butler.Butler`
49 Data butler instance.
50 taskFactory : `~lsst.pipe.base.TaskFactory`
51 Task factory.
52 skipExisting : `bool`, optional
53 If `True` then do not try to overwrite any datasets that might exist
54 in ``butler.run``; instead compare them when appropriate/possible. If
55 `False`, then any existing conflicting dataset will cause a butler
56 exception to be raised.
57 """
58 def __init__(self, butler, taskFactory, skipExisting=False):
59 self.butler = butler
60 self.taskFactory = taskFactory
61 self.skipExisting = skipExisting
62 if self.skipExisting and self.butler.run is None:
63 raise RuntimeError(
64 "Cannot perform skipExisting logic unless butler is initialized "
65 "with a default output RUN collection."
66 )
68 def initialize(self, graph, saveInitOutputs=True, registerDatasetTypes=False, saveVersions=True):
69 """Perform all initialization steps.
71 Convenience method to execute all initialization steps. Instead of
72 calling this method and providing all options it is also possible to
73 call methods individually.
75 Parameters
76 ----------
77 graph : `~lsst.pipe.base.QuantumGraph`
78 Execution graph.
79 saveInitOutputs : `bool`, optional
80 If ``True`` (default) then save "init outputs", configurations,
81 and package versions to butler.
82 registerDatasetTypes : `bool`, optional
83 If ``True`` then register dataset types in registry, otherwise
84 they must be already registered.
85 saveVersions : `bool`, optional
86 If ``False`` then do not save package versions even if
87 ``saveInitOutputs`` is set to ``True``.
88 """
89 # register dataset types or check consistency
90 self.initializeDatasetTypes(graph, registerDatasetTypes)
92 # Save task initialization data or check that saved data
93 # is consistent with what tasks would save
94 if saveInitOutputs:
95 self.saveInitOutputs(graph)
96 self.saveConfigs(graph)
97 if saveVersions:
98 self.savePackageVersions(graph)
100 def initializeDatasetTypes(self, graph, registerDatasetTypes=False):
101 """Save or check DatasetTypes output by the tasks in a graph.
103 Iterates over all DatasetTypes for all tasks in a graph and either
104 tries to add them to registry or compares them to exising ones.
106 Parameters
107 ----------
108 graph : `~lsst.pipe.base.QuantumGraph`
109 Execution graph.
110 registerDatasetTypes : `bool`, optional
111 If ``True`` then register dataset types in registry, otherwise
112 they must be already registered.
114 Raises
115 ------
116 ValueError
117 Raised if existing DatasetType is different from DatasetType
118 in a graph.
119 KeyError
120 Raised if ``registerDatasetTypes`` is ``False`` and DatasetType
121 does not exist in registry.
122 """
123 pipeline = graph.taskGraph
124 datasetTypes = PipelineDatasetTypes.fromPipeline(pipeline, registry=self.butler.registry,
125 include_configs=True, include_packages=True)
126 for datasetType in itertools.chain(datasetTypes.initIntermediates, datasetTypes.initOutputs,
127 datasetTypes.intermediates, datasetTypes.outputs):
128 # Only composites are registered, no components, and by this point
129 # the composite should already exist.
130 if registerDatasetTypes and not datasetType.isComponent():
131 _LOG.debug("Registering DatasetType %s with registry", datasetType)
132 # this is a no-op if it already exists and is consistent,
133 # and it raises if it is inconsistent.
134 self.butler.registry.registerDatasetType(datasetType)
135 else:
136 _LOG.debug("Checking DatasetType %s against registry", datasetType)
137 expected = self.butler.registry.getDatasetType(datasetType.name)
138 if datasetType.isComponent() \
139 and datasetType.parentStorageClass == DatasetType.PlaceholderParentStorageClass:
140 # Force the parent storage classes to match since we
141 # are using a placeholder
142 datasetType.finalizeParentStorageClass(expected.parentStorageClass)
143 if expected != datasetType:
144 raise ValueError(f"DatasetType configuration does not match Registry: "
145 f"{datasetType} != {expected}")
147 def saveInitOutputs(self, graph):
148 """Write any datasets produced by initializing tasks in a graph.
150 Parameters
151 ----------
152 graph : `~lsst.pipe.base.QuantumGraph`
153 Execution graph.
155 Raises
156 ------
157 Exception
158 Raised if ``skipExisting`` is `False` and datasets already
159 exists. Content of a butler collection may be changed if
160 exception is raised.
162 Notes
163 -----
164 If ``skipExisting`` is `True` then existing datasets are not
165 overwritten, instead we should check that their stored object is
166 exactly the same as what we would save at this time. Comparing
167 arbitrary types of object is, of course, non-trivial. Current
168 implementation only checks the existence of the datasets and their
169 types against the types of objects produced by tasks. Ideally we
170 would like to check that object data is identical too but presently
171 there is no generic way to compare objects. In the future we can
172 potentially introduce some extensible mechanism for that.
173 """
174 _LOG.debug("Will save InitOutputs for all tasks")
175 for taskDef in graph.iterTaskGraph():
176 task = self.taskFactory.makeTask(taskDef.taskClass,
177 taskDef.label,
178 taskDef.config,
179 None,
180 self.butler)
181 for name in taskDef.connections.initOutputs:
182 attribute = getattr(taskDef.connections, name)
183 initOutputVar = getattr(task, name)
184 objFromStore = None
185 if self.skipExisting:
186 # check if it is there already
187 _LOG.debug("Retrieving InitOutputs for task=%s key=%s dsTypeName=%s",
188 task, name, attribute.name)
189 try:
190 objFromStore = self.butler.get(attribute.name, {}, collections=[self.butler.run])
191 # Types are supposed to be identical.
192 # TODO: Check that object contents is identical too.
193 if type(objFromStore) is not type(initOutputVar):
194 raise TypeError(f"Stored initOutput object type {type(objFromStore)} "
195 f"is different from task-generated type "
196 f"{type(initOutputVar)} for task {taskDef}")
197 except LookupError:
198 pass
199 if objFromStore is None:
200 # butler will raise exception if dataset is already there
201 _LOG.debug("Saving InitOutputs for task=%s key=%s", task, name)
202 self.butler.put(initOutputVar, attribute.name, {})
204 def saveConfigs(self, graph):
205 """Write configurations for pipeline tasks to butler or check that
206 existing configurations are equal to the new ones.
208 Parameters
209 ----------
210 graph : `~lsst.pipe.base.QuantumGraph`
211 Execution graph.
213 Raises
214 ------
215 Exception
216 Raised if ``skipExisting`` is `False` and datasets already exists.
217 Content of a butler collection should not be changed if exception
218 is raised.
219 """
220 def logConfigMismatch(msg):
221 """Log messages about configuration mismatch.
222 """
223 _LOG.fatal("Comparing configuration: %s", msg)
225 _LOG.debug("Will save Configs for all tasks")
226 # start transaction to rollback any changes on exceptions
227 with self.butler.transaction():
228 for taskDef in graph.taskGraph:
229 configName = taskDef.configDatasetName
231 oldConfig = None
232 if self.skipExisting:
233 try:
234 oldConfig = self.butler.get(configName, {}, collections=[self.butler.run])
235 if not taskDef.config.compare(oldConfig, shortcut=False, output=logConfigMismatch):
236 raise TypeError(
237 f"Config does not match existing task config {configName!r} in butler; "
238 "tasks configurations must be consistent within the same run collection")
239 except LookupError:
240 pass
241 if oldConfig is None:
242 # butler will raise exception if dataset is already there
243 _LOG.debug("Saving Config for task=%s dataset type=%s", taskDef.label, configName)
244 self.butler.put(taskDef.config, configName, {})
246 def savePackageVersions(self, graph):
247 """Write versions of software packages to butler.
249 Parameters
250 ----------
251 graph : `~lsst.pipe.base.QuantumGraph`
252 Execution graph.
254 Raises
255 ------
256 Exception
257 Raised if ``checkExisting`` is ``True`` but versions are not
258 compatible.
259 """
260 packages = Packages.fromSystem()
261 _LOG.debug("want to save packages: %s", packages)
262 datasetType = "packages"
263 dataId = {}
264 oldPackages = None
265 # start transaction to rollback any changes on exceptions
266 with self.butler.transaction():
267 if self.skipExisting:
268 try:
269 oldPackages = self.butler.get(datasetType, dataId, collections=[self.butler.run])
270 _LOG.debug("old packages: %s", oldPackages)
271 except LookupError:
272 pass
273 if oldPackages is not None:
274 # Note that because we can only detect python modules that have
275 # been imported, the stored list of products may be more or
276 # less complete than what we have now. What's important is
277 # that the products that are in common have the same version.
278 diff = packages.difference(oldPackages)
279 if diff:
280 versions_str = "; ".join(f"{pkg}: {diff[pkg][1]} vs {diff[pkg][0]}" for pkg in diff)
281 raise TypeError(f"Package versions mismatch: ({versions_str})")
282 else:
283 _LOG.debug("new packages are consistent with old")
284 # Update the old set of packages in case we have more packages
285 # that haven't been persisted.
286 extra = packages.extra(oldPackages)
287 if extra:
288 _LOG.debug("extra packages: %s", extra)
289 oldPackages.update(packages)
290 # have to remove existing dataset first, butler has no
291 # replace option.
292 ref = self.butler.registry.findDataset(datasetType, dataId, collections=[self.butler.run])
293 self.butler.pruneDatasets([ref], unstore=True, purge=True)
294 self.butler.put(oldPackages, datasetType, dataId)
295 else:
296 self.butler.put(packages, datasetType, dataId)