Coverage for python/lsst/daf/butler/script/queryCollections.py: 8%

68 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-27 09:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30from collections.abc import Iterable 

31 

32from astropy.table import Table 

33 

34from .._butler import Butler 

35from ..registry import CollectionType 

36 

37 

38def _getTable( 

39 repo: str, 

40 glob: Iterable[str], 

41 collection_type: Iterable[CollectionType], 

42 inverse: bool, 

43) -> Table: 

44 """Run queryCollections and return the results in Table form. 

45 

46 Only lists the first child (or parent if `inverse` is `True`) in the 

47 description column. 

48 

49 Parameters 

50 ---------- 

51 repo 

52 glob 

53 collection_type 

54 Same as `queryCollections` 

55 inverse : `bool` 

56 True if parent CHAINED datasets of each dataset should be listed in the 

57 description column, False if children of CHAINED datasets should be 

58 listed. 

59 

60 Returns 

61 ------- 

62 collections : `astropy.table.Table` 

63 Same as `queryCollections` 

64 """ 

65 typeCol = "Type" 

66 descriptionCol = "Parents" if inverse else "Children" 

67 table = Table( 

68 names=("Name", typeCol, descriptionCol), 

69 dtype=(str, str, str), 

70 ) 

71 butler = Butler.from_config(repo) 

72 names = sorted( 

73 butler.registry.queryCollections(collectionTypes=frozenset(collection_type), expression=glob or ...) 

74 ) 

75 if inverse: 

76 for name in names: 

77 type = butler.registry.getCollectionType(name) 

78 parentNames = butler.registry.getCollectionParentChains(name) 

79 if parentNames: 

80 first = True 

81 for parentName in sorted(parentNames): 

82 table.add_row((name if first else "", type.name if first else "", parentName)) 

83 first = False 

84 else: 

85 table.add_row((name, type.name, "")) 

86 # If none of the datasets has a parent dataset then remove the 

87 # description column. 

88 if not any(c for c in table[descriptionCol]): 

89 del table[descriptionCol] 

90 else: 

91 for name in names: 

92 type = butler.registry.getCollectionType(name) 

93 if type == CollectionType.CHAINED: 

94 children = butler.registry.getCollectionChain(name) 

95 if children: 

96 first = True 

97 for child in sorted(children): 

98 table.add_row((name if first else "", type.name if first else "", child)) 

99 first = False 

100 else: 

101 table.add_row((name, type.name, "")) 

102 else: 

103 table.add_row((name, type.name, "")) 

104 # If there aren't any CHAINED datasets in the results then remove the 

105 # description column. 

106 if not any(columnVal == CollectionType.CHAINED.name for columnVal in table[typeCol]): 

107 del table[descriptionCol] 

108 

109 return table 

110 

111 

112def _getTree( 

113 repo: str, 

114 glob: Iterable[str], 

115 collection_type: Iterable[CollectionType], 

116 inverse: bool, 

117) -> Table: 

118 """Run queryCollections and return the results in a table representing tree 

119 form. 

120 

121 Recursively lists children (or parents if `inverse` is `True`) 

122 

123 Parameters 

124 ---------- 

125 repo 

126 glob 

127 collection_type 

128 Same as `queryCollections` 

129 inverse : `bool` 

130 True if parent CHAINED datasets of each dataset should be listed in the 

131 description column, False if children of CHAINED datasets should be 

132 listed. 

133 

134 Returns 

135 ------- 

136 collections : `astropy.table.Table` 

137 Same as `queryCollections` 

138 """ 

139 table = Table( 

140 names=("Name", "Type"), 

141 dtype=(str, str), 

142 ) 

143 butler = Butler.from_config(repo, without_datastore=True) 

144 

145 def addCollection(name: str, level: int = 0) -> None: 

146 collectionType = butler.registry.getCollectionType(name) 

147 table.add_row((" " * level + name, collectionType.name)) 

148 if inverse: 

149 parentNames = butler.registry.getCollectionParentChains(name) 

150 for pname in sorted(parentNames): 

151 addCollection(pname, level + 1) 

152 else: 

153 if collectionType == CollectionType.CHAINED: 

154 childNames = butler.registry.getCollectionChain(name) 

155 for name in childNames: 

156 addCollection(name, level + 1) 

157 

158 collections = butler.registry.queryCollections( 

159 collectionTypes=frozenset(collection_type), expression=glob or ... 

160 ) 

161 for collection in sorted(collections): 

162 addCollection(collection) 

163 return table 

164 

165 

166def _getFlatten( 

167 repo: str, 

168 glob: Iterable[str], 

169 collection_type: Iterable[CollectionType], 

170) -> Table: 

171 butler = Butler.from_config(repo) 

172 collectionNames = list( 

173 butler.registry.queryCollections( 

174 collectionTypes=frozenset(collection_type), flattenChains=True, expression=glob or ... 

175 ) 

176 ) 

177 

178 collectionTypes = [butler.registry.getCollectionType(c).name for c in collectionNames] 

179 return Table((collectionNames, collectionTypes), names=("Name", "Type")) 

180 

181 

182def queryCollections( 

183 repo: str, 

184 glob: Iterable[str], 

185 collection_type: Iterable[CollectionType], 

186 chains: str, 

187) -> Table: 

188 """Get the collections whose names match an expression. 

189 

190 Parameters 

191 ---------- 

192 repo : `str` 

193 URI to the location of the repo or URI to a config file describing the 

194 repo and its location. 

195 glob : `~collections.abc.Iterable` [`str`] 

196 A list of glob-style search string that fully or partially identify 

197 the dataset type names to search for. 

198 collection_type : `~collections.abc.Iterable` [ `CollectionType` ], \ 

199 optional 

200 If provided, only return collections of these types. 

201 chains : `str` 

202 Must be one of "FLATTEN", "TABLE", or "TREE" (case sensitive). 

203 Affects contents and formatting of results, see 

204 ``cli.commands.query_collections``. 

205 inverse : `bool` 

206 If true, show what CHAINED collections the dataset is a member of 

207 (instead of what datasets any CHAINED collection contains) 

208 

209 Returns 

210 ------- 

211 collections : `astropy.table.Table` 

212 A table containing information about collections. 

213 """ 

214 if (inverse := chains == "INVERSE-TABLE") or chains == "TABLE": 

215 return _getTable(repo, glob, collection_type, inverse) 

216 elif (inverse := chains == "INVERSE-TREE") or chains == "TREE": 

217 return _getTree(repo, glob, collection_type, inverse) 

218 elif chains == "FLATTEN": 

219 return _getFlatten(repo, glob, collection_type) 

220 raise RuntimeError(f"Value for --chains not recognized: {chains}")