Coverage for python/lsst/ctrl/mpexec/cli/script/cleanup.py: 34%

64 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-07 09:59 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28 

29import re 

30from typing import Any 

31 

32from lsst.daf.butler import Butler, CollectionType 

33from lsst.daf.butler.registry import CollectionTypeError, MissingCollectionError 

34 

35from .confirmable import ConfirmableResult 

36 

37 

38class NoSuchCollectionFailure: 

39 """Failure when there is no such collection. 

40 

41 Parameters 

42 ---------- 

43 collection : `str` 

44 Name of collection. 

45 """ 

46 

47 def __init__(self, collection: str): 

48 self.collection = collection 

49 

50 def __str__(self) -> str: 

51 return f'Did not find a collection named "{self.collection}"' 

52 

53 

54class NotChainedCollectionFailure: 

55 """Failure when this is not a chained collection. 

56 

57 Parameters 

58 ---------- 

59 collection : `str` 

60 Name of collection. 

61 type : `str` 

62 Type of collection. 

63 """ 

64 

65 def __init__(self, collection: str, type: str): 

66 self.collection = collection 

67 self.type = type 

68 

69 def __str__(self) -> str: 

70 return f'COLLECTION must be a CHAINED collection, "{self.collection}" is a "{self.type}" collection.' 

71 

72 

73class CleanupResult(ConfirmableResult): 

74 """Information containing the result of the cleanup request. 

75 

76 Parameters 

77 ---------- 

78 butler_config : `str` 

79 Butler configuration URI. 

80 """ 

81 

82 def __init__(self, butler_config: str): 

83 self.butler_config = butler_config 

84 self.runs_to_remove: list[str] = [] 

85 self.others_to_remove: list[str] = [] 

86 self.failure: Any = None 

87 

88 def describe(self, will: bool) -> str: 

89 if self.can_continue: 

90 msg = "Will remove:" if will else "Removed:" 

91 msg += "\n" 

92 msg += f" runs: {', '.join(self.runs_to_remove)}\n" 

93 msg += f" others: {', '.join(self.others_to_remove)}" 

94 else: 

95 msg = "Did not find any collections to remove." 

96 return msg 

97 

98 def on_confirmation(self) -> None: 

99 butler = Butler.from_config(self.butler_config, writeable=True) 

100 with butler.transaction(): 

101 for collection in self.others_to_remove: 

102 butler.registry.removeCollection(collection) 

103 butler.removeRuns(self.runs_to_remove) 

104 

105 @property 

106 def failed(self) -> bool: 

107 return self.failure is not None 

108 

109 @property 

110 def describe_failure(self) -> str: 

111 return str(self.failure) 

112 

113 @property 

114 def can_continue(self) -> bool: 

115 return bool(self.runs_to_remove) or bool(self.others_to_remove) 

116 

117 

118def cleanup( 

119 butler_config: str, 

120 collection: str, 

121) -> CleanupResult: 

122 """Remove collections that start with the same name as a CHAINED 

123 collection but are not members of that collection. 

124 

125 Parameters 

126 ---------- 

127 butler_config : str 

128 The path location of the gen3 butler/registry config file. 

129 collection : str 

130 The name of the chained collection. 

131 """ 

132 butler = Butler.from_config(butler_config) 

133 result = CleanupResult(butler_config) 

134 try: 

135 to_keep = set(butler.registry.getCollectionChain(collection)) 

136 except MissingCollectionError: 

137 result.failure = NoSuchCollectionFailure(collection) 

138 return result 

139 except CollectionTypeError: 

140 result.failure = NotChainedCollectionFailure( 

141 collection, butler.registry.getCollectionType(collection).name 

142 ) 

143 return result 

144 regex = re.compile(collection + ".+") 

145 to_consider = set(butler.registry.queryCollections(regex)) 

146 to_remove = to_consider - to_keep 

147 for r in to_remove: 

148 if butler.registry.getCollectionType(r) == CollectionType.RUN: 

149 result.runs_to_remove.append(r) 

150 else: 

151 result.others_to_remove.append(r) 

152 return result