Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ['PreExecInit'] 

23 

24# ------------------------------- 

25# Imports of standard modules -- 

26# ------------------------------- 

27import logging 

28import itertools 

29 

30# ----------------------------- 

31# Imports for other modules -- 

32# ----------------------------- 

33from lsst.base import Packages 

34from lsst.daf.butler import DatasetType 

35from lsst.pipe.base import PipelineDatasetTypes 

36 

37_LOG = logging.getLogger(__name__.partition(".")[2]) 

38 

39 

40class PreExecInit: 

41 """Initialization of registry for QuantumGraph execution. 

42 

43 This class encapsulates all necessary operations that have to be performed 

44 on butler and registry to prepare them for QuantumGraph execution. 

45 

46 Parameters 

47 ---------- 

48 butler : `~lsst.daf.butler.Butler` 

49 Data butler instance. 

50 taskFactory : `~lsst.pipe.base.TaskFactory` 

51 Task factory. 

52 skipExisting : `bool`, optional 

53 If `True` then do not try to overwrite any datasets that might exist 

54 in ``butler.run``; instead compare them when appropriate/possible. If 

55 `False`, then any existing conflicting dataset will cause a butler 

56 exception to be raised. 

57 """ 

58 def __init__(self, butler, taskFactory, skipExisting=False): 

59 self.butler = butler 

60 self.taskFactory = taskFactory 

61 self.skipExisting = skipExisting 

62 if self.skipExisting and self.butler.run is None: 

63 raise RuntimeError( 

64 "Cannot perform skipExisting logic unless butler is initialized " 

65 "with a default output RUN collection." 

66 ) 

67 

68 def initialize(self, graph, saveInitOutputs=True, registerDatasetTypes=False, saveVersions=True): 

69 """Perform all initialization steps. 

70 

71 Convenience method to execute all initialization steps. Instead of 

72 calling this method and providing all options it is also possible to 

73 call methods individually. 

74 

75 Parameters 

76 ---------- 

77 graph : `~lsst.pipe.base.QuantumGraph` 

78 Execution graph. 

79 saveInitOutputs : `bool`, optional 

80 If ``True`` (default) then save "init outputs", configurations, 

81 and package versions to butler. 

82 registerDatasetTypes : `bool`, optional 

83 If ``True`` then register dataset types in registry, otherwise 

84 they must be already registered. 

85 saveVersions : `bool`, optional 

86 If ``False`` then do not save package versions even if 

87 ``saveInitOutputs`` is set to ``True``. 

88 """ 

89 # register dataset types or check consistency 

90 self.initializeDatasetTypes(graph, registerDatasetTypes) 

91 

92 # Save task initialization data or check that saved data 

93 # is consistent with what tasks would save 

94 if saveInitOutputs: 

95 self.saveInitOutputs(graph) 

96 self.saveConfigs(graph) 

97 if saveVersions: 

98 self.savePackageVersions(graph) 

99 

100 def initializeDatasetTypes(self, graph, registerDatasetTypes=False): 

101 """Save or check DatasetTypes output by the tasks in a graph. 

102 

103 Iterates over all DatasetTypes for all tasks in a graph and either 

104 tries to add them to registry or compares them to exising ones. 

105 

106 Parameters 

107 ---------- 

108 graph : `~lsst.pipe.base.QuantumGraph` 

109 Execution graph. 

110 registerDatasetTypes : `bool`, optional 

111 If ``True`` then register dataset types in registry, otherwise 

112 they must be already registered. 

113 

114 Raises 

115 ------ 

116 ValueError 

117 Raised if existing DatasetType is different from DatasetType 

118 in a graph. 

119 KeyError 

120 Raised if ``registerDatasetTypes`` is ``False`` and DatasetType 

121 does not exist in registry. 

122 """ 

123 pipeline = graph.taskGraph 

124 datasetTypes = PipelineDatasetTypes.fromPipeline(pipeline, registry=self.butler.registry, 

125 include_configs=True, include_packages=True) 

126 for datasetType in itertools.chain(datasetTypes.initIntermediates, datasetTypes.initOutputs, 

127 datasetTypes.intermediates, datasetTypes.outputs): 

128 # Only composites are registered, no components, and by this point 

129 # the composite should already exist. 

130 if registerDatasetTypes and not datasetType.isComponent(): 

131 _LOG.debug("Registering DatasetType %s with registry", datasetType) 

132 # this is a no-op if it already exists and is consistent, 

133 # and it raises if it is inconsistent. 

134 self.butler.registry.registerDatasetType(datasetType) 

135 else: 

136 _LOG.debug("Checking DatasetType %s against registry", datasetType) 

137 expected = self.butler.registry.getDatasetType(datasetType.name) 

138 if datasetType.isComponent() \ 

139 and datasetType.parentStorageClass == DatasetType.PlaceholderParentStorageClass: 

140 # Force the parent storage classes to match since we 

141 # are using a placeholder 

142 datasetType.finalizeParentStorageClass(expected.parentStorageClass) 

143 if expected != datasetType: 

144 raise ValueError(f"DatasetType configuration does not match Registry: " 

145 f"{datasetType} != {expected}") 

146 

147 def saveInitOutputs(self, graph): 

148 """Write any datasets produced by initializing tasks in a graph. 

149 

150 Parameters 

151 ---------- 

152 graph : `~lsst.pipe.base.QuantumGraph` 

153 Execution graph. 

154 

155 Raises 

156 ------ 

157 Exception 

158 Raised if ``skipExisting`` is `False` and datasets already 

159 exists. Content of a butler collection may be changed if 

160 exception is raised. 

161 

162 Notes 

163 ----- 

164 If ``skipExisting`` is `True` then existing datasets are not 

165 overwritten, instead we should check that their stored object is 

166 exactly the same as what we would save at this time. Comparing 

167 arbitrary types of object is, of course, non-trivial. Current 

168 implementation only checks the existence of the datasets and their 

169 types against the types of objects produced by tasks. Ideally we 

170 would like to check that object data is identical too but presently 

171 there is no generic way to compare objects. In the future we can 

172 potentially introduce some extensible mechanism for that. 

173 """ 

174 _LOG.debug("Will save InitOutputs for all tasks") 

175 for taskDef in graph.iterTaskGraph(): 

176 task = self.taskFactory.makeTask(taskDef.taskClass, 

177 taskDef.label, 

178 taskDef.config, 

179 None, 

180 self.butler) 

181 for name in taskDef.connections.initOutputs: 

182 attribute = getattr(taskDef.connections, name) 

183 initOutputVar = getattr(task, name) 

184 objFromStore = None 

185 if self.skipExisting: 

186 # check if it is there already 

187 _LOG.debug("Retrieving InitOutputs for task=%s key=%s dsTypeName=%s", 

188 task, name, attribute.name) 

189 try: 

190 objFromStore = self.butler.get(attribute.name, {}, collections=[self.butler.run]) 

191 # Types are supposed to be identical. 

192 # TODO: Check that object contents is identical too. 

193 if type(objFromStore) is not type(initOutputVar): 

194 raise TypeError(f"Stored initOutput object type {type(objFromStore)} " 

195 f"is different from task-generated type " 

196 f"{type(initOutputVar)} for task {taskDef}") 

197 except LookupError: 

198 pass 

199 if objFromStore is None: 

200 # butler will raise exception if dataset is already there 

201 _LOG.debug("Saving InitOutputs for task=%s key=%s", task, name) 

202 self.butler.put(initOutputVar, attribute.name, {}) 

203 

204 def saveConfigs(self, graph): 

205 """Write configurations for pipeline tasks to butler or check that 

206 existing configurations are equal to the new ones. 

207 

208 Parameters 

209 ---------- 

210 graph : `~lsst.pipe.base.QuantumGraph` 

211 Execution graph. 

212 

213 Raises 

214 ------ 

215 Exception 

216 Raised if ``skipExisting`` is `False` and datasets already exists. 

217 Content of a butler collection should not be changed if exception 

218 is raised. 

219 """ 

220 def logConfigMismatch(msg): 

221 """Log messages about configuration mismatch. 

222 """ 

223 _LOG.fatal("Comparing configuration: %s", msg) 

224 

225 _LOG.debug("Will save Configs for all tasks") 

226 # start transaction to rollback any changes on exceptions 

227 with self.butler.transaction(): 

228 for taskDef in graph.taskGraph: 

229 configName = taskDef.configDatasetName 

230 

231 oldConfig = None 

232 if self.skipExisting: 

233 try: 

234 oldConfig = self.butler.get(configName, {}, collections=[self.butler.run]) 

235 if not taskDef.config.compare(oldConfig, shortcut=False, output=logConfigMismatch): 

236 raise TypeError( 

237 f"Config does not match existing task config {configName!r} in butler; " 

238 "tasks configurations must be consistent within the same run collection") 

239 except LookupError: 

240 pass 

241 if oldConfig is None: 

242 # butler will raise exception if dataset is already there 

243 _LOG.debug("Saving Config for task=%s dataset type=%s", taskDef.label, configName) 

244 self.butler.put(taskDef.config, configName, {}) 

245 

246 def savePackageVersions(self, graph): 

247 """Write versions of software packages to butler. 

248 

249 Parameters 

250 ---------- 

251 graph : `~lsst.pipe.base.QuantumGraph` 

252 Execution graph. 

253 

254 Raises 

255 ------ 

256 Exception 

257 Raised if ``checkExisting`` is ``True`` but versions are not 

258 compatible. 

259 """ 

260 packages = Packages.fromSystem() 

261 _LOG.debug("want to save packages: %s", packages) 

262 datasetType = "packages" 

263 dataId = {} 

264 oldPackages = None 

265 # start transaction to rollback any changes on exceptions 

266 with self.butler.transaction(): 

267 if self.skipExisting: 

268 try: 

269 oldPackages = self.butler.get(datasetType, dataId, collections=[self.butler.run]) 

270 _LOG.debug("old packages: %s", oldPackages) 

271 except LookupError: 

272 pass 

273 if oldPackages is not None: 

274 # Note that because we can only detect python modules that have been imported, the stored 

275 # list of products may be more or less complete than what we have now. What's important is 

276 # that the products that are in common have the same version. 

277 diff = packages.difference(oldPackages) 

278 if diff: 

279 versions_str = "; ".join(f"{pkg}: {diff[pkg][1]} vs {diff[pkg][0]}" for pkg in diff) 

280 raise TypeError(f"Package versions mismatch: ({versions_str})") 

281 else: 

282 _LOG.debug("new packages are consistent with old") 

283 # Update the old set of packages in case we have more packages that haven't been persisted. 

284 extra = packages.extra(oldPackages) 

285 if extra: 

286 _LOG.debug("extra packages: %s", extra) 

287 oldPackages.update(packages) 

288 # have to remove existing dataset first, butler nas no replace option 

289 ref = self.butler.registry.findDataset(datasetType, dataId, collections=[self.butler.run]) 

290 self.butler.pruneDatasets([ref], unstore=True, purge=True) 

291 self.butler.put(oldPackages, datasetType, dataId) 

292 else: 

293 self.butler.put(packages, datasetType, dataId)