Coverage for python/lsst/ctrl/mpexec/cli/script/cleanup.py: 29%

64 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-11 09:04 +0000

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22 

23import re 

24from typing import Any 

25 

26from lsst.daf.butler import Butler, CollectionType 

27from lsst.daf.butler.registry import CollectionTypeError, MissingCollectionError 

28 

29from .confirmable import ConfirmableResult 

30 

31 

32class NoSuchCollectionFailure: 

33 """Failure when there is no such collection.""" 

34 

35 def __init__(self, collection: str): 

36 self.collection = collection 

37 

38 def __str__(self) -> str: 

39 return f'Did not find a collection named "{self.collection}"' 

40 

41 

42class NotChainedCollectionFailure: 

43 """Failure when this is not a chained collection.""" 

44 

45 def __init__(self, collection: str, type: str): 

46 self.collection = collection 

47 self.type = type 

48 

49 def __str__(self) -> str: 

50 return f'COLLECTION must be a CHAINED collection, "{self.collection}" is a "{self.type}" collection.' 

51 

52 

53class CleanupResult(ConfirmableResult): 

54 """Information containing the result of the cleanup request.""" 

55 

56 def __init__(self, butler_config: str): 

57 self.butler_config = butler_config 

58 self.runs_to_remove: list[str] = [] 

59 self.others_to_remove: list[str] = [] 

60 self.failure: Any = None 

61 

62 def describe(self, will: bool) -> str: 

63 if self.can_continue: 

64 msg = "Will remove:" if will else "Removed:" 

65 msg += "\n" 

66 msg += f" runs: {', '.join(self.runs_to_remove)}\n" 

67 msg += f" others: {', '.join(self.others_to_remove)}" 

68 else: 

69 msg = "Did not find any collections to remove." 

70 return msg 

71 

72 def on_confirmation(self) -> None: 

73 butler = Butler(self.butler_config, writeable=True) 

74 with butler.transaction(): 

75 for collection in self.others_to_remove: 

76 butler.registry.removeCollection(collection) 

77 butler.removeRuns(self.runs_to_remove) 

78 

79 @property 

80 def failed(self) -> bool: 

81 return self.failure is not None 

82 

83 @property 

84 def describe_failure(self) -> str: 

85 return str(self.failure) 

86 

87 @property 

88 def can_continue(self) -> bool: 

89 return bool(self.runs_to_remove) or bool(self.others_to_remove) 

90 

91 

92def cleanup( 

93 butler_config: str, 

94 collection: str, 

95) -> CleanupResult: 

96 """Remove collections that start with the same name as a CHAINED 

97 collection but are not members of that collection. 

98 

99 Parameters 

100 ---------- 

101 butler_config : str 

102 The path location of the gen3 butler/registry config file. 

103 collection : str 

104 The name of the chained collection. 

105 """ 

106 butler = Butler(butler_config) 

107 result = CleanupResult(butler_config) 

108 try: 

109 to_keep = set(butler.registry.getCollectionChain(collection)) 

110 except MissingCollectionError: 

111 result.failure = NoSuchCollectionFailure(collection) 

112 return result 

113 except CollectionTypeError: 

114 result.failure = NotChainedCollectionFailure( 

115 collection, butler.registry.getCollectionType(collection).name 

116 ) 

117 return result 

118 regex = re.compile(collection + ".+") 

119 to_consider = set(butler.registry.queryCollections(regex)) 

120 to_remove = to_consider - to_keep 

121 for r in to_remove: 

122 if butler.registry.getCollectionType(r) == CollectionType.RUN: 

123 result.runs_to_remove.append(r) 

124 else: 

125 result.others_to_remove.append(r) 

126 return result