Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23import yaml 

24 

25from ..opt import (collection_type_option, dataset_type_option, directory_argument, options_file_option, 

26 glob_argument, repo_argument, transfer_option, verbose_option) 

27from ..utils import cli_handle_exception, split_commas, typeStrAcceptsMultiple, unwrap 

28from ... import script 

29 

30 

31willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist." 

32existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file." 

33 

34 

35# The conversion from the import command name to the butler_import function 

36# name for subcommand lookup is implemented in the cli/butler.py, in 

37# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they 

38# must be reflected in that location. If this becomes a common pattern a better 

39# mechanism should be implemented. 

40@click.command("import") 

41@repo_argument(required=True, help=willCreateRepoHelp) 

42@directory_argument(required=True) 

43@transfer_option() 

44@click.option("--export-file", 

45 help="Name for the file that contains database information associated with the exported " 

46 "datasets. If this is not an absolute path, does not exist in the current working " 

47 "directory, and --dir is provided, it is assumed to be in that directory. Defaults " 

48 "to \"export.yaml\".", 

49 type=click.File("r")) 

50@click.option("--skip-dimensions", "-s", type=str, multiple=True, callback=split_commas, 

51 metavar=typeStrAcceptsMultiple, 

52 help="Dimensions that should be skipped during import") 

53@options_file_option() 

54def butler_import(*args, **kwargs): 

55 """Import data into a butler repository.""" 

56 cli_handle_exception(script.butlerImport, *args, **kwargs) 

57 

58 

59@click.command() 

60@repo_argument(required=True, help=willCreateRepoHelp) 

61@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).") 

62@click.option("--standalone", is_flag=True, help="Include all defaults in the config file in the repo, " 

63 "insulating the repo from changes in package defaults.") 

64@click.option("--override", is_flag=True, help="Allow values in the supplied config to override all " 

65 "repo settings.") 

66@click.option("--outfile", "-f", default=None, type=str, help="Name of output file to receive repository " 

67 "configuration. Default is to write butler.yaml into the specified repo.") 

68@options_file_option() 

69def create(*args, **kwargs): 

70 """Create an empty Gen3 Butler repository.""" 

71 cli_handle_exception(script.createRepo, *args, **kwargs) 

72 

73 

74@click.command(short_help="Dump butler config to stdout.") 

75@repo_argument(required=True, help=existingRepoHelp) 

76@click.option("--subset", "-s", type=str, 

77 help="Subset of a configuration to report. This can be any key in the hierarchy such as " 

78 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.") 

79@click.option("--searchpath", "-p", type=str, multiple=True, callback=split_commas, 

80 metavar=typeStrAcceptsMultiple, 

81 help="Additional search paths to use for configuration overrides") 

82@click.option("--file", "outfile", type=click.File("w"), default="-", 

83 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout " 

84 "by default.") 

85@options_file_option() 

86def config_dump(*args, **kwargs): 

87 """Dump either a subset or full Butler configuration to standard output.""" 

88 cli_handle_exception(script.configDump, *args, **kwargs) 

89 

90 

91@click.command(short_help="Validate the configuration files.") 

92@repo_argument(required=True, help=existingRepoHelp) 

93@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.") 

94@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True) 

95@click.option("--ignore", "-i", type=str, multiple=True, callback=split_commas, 

96 metavar=typeStrAcceptsMultiple, 

97 help="DatasetType(s) to ignore for validation.") 

98@options_file_option() 

99def config_validate(*args, **kwargs): 

100 """Validate the configuration files for a Gen3 Butler repository.""" 

101 is_good = cli_handle_exception(script.configValidate, *args, **kwargs) 

102 if not is_good: 

103 raise click.exceptions.Exit(1) 

104 

105 

106@click.command() 

107@repo_argument(required=True) 

108@click.option("--collection", 

109 help=unwrap("""Name of the collection to remove. If this is a TAGGED or CHAINED collection, 

110 datasets within the collection are not modified unless --unstore is passed. If this 

111 is a RUN collection, --purge and --unstore must be passed, and all datasets in it 

112 are fully removed from the data repository. """)) 

113@click.option("--purge", 

114 help=unwrap("""Permit RUN collections to be removed, fully removing datasets within them. 

115 Requires --unstore as an added precaution against accidental deletion. Must not be 

116 passed if the collection is not a RUN."""), 

117 is_flag=True) 

118@click.option("--unstore", 

119 help=("""Remove all datasets in the collection from all datastores in which they appear."""), 

120 is_flag=True) 

121@options_file_option() 

122def prune_collection(**kwargs): 

123 """Remove a collection and possibly prune datasets within it.""" 

124 cli_handle_exception(script.pruneCollection, **kwargs) 

125 

126 

127@click.command(short_help="Search for collections.") 

128@repo_argument(required=True) 

129@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

130 "collections to return.") 

131@collection_type_option() 

132@click.option("--flatten-chains/--no-flatten-chains", 

133 help="Recursively get the child collections of matching CHAINED collections. Default is " 

134 "--no-flatten-chains.") 

135@click.option("--include-chains/--no-include-chains", 

136 default=None, 

137 help="For --include-chains, return records for matching CHAINED collections. For " 

138 "--no-include-chains do not return records for CHAINED collections. Default is the " 

139 "opposite of --flatten-chains: include either CHAINED collections or their children, but " 

140 "not both.") 

141@options_file_option() 

142def query_collections(*args, **kwargs): 

143 """Get the collections whose names match an expression.""" 

144 print(yaml.dump(cli_handle_exception(script.queryCollections, *args, **kwargs))) 

145 

146 

147@click.command() 

148@repo_argument(required=True) 

149@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

150 "dataset types to return.") 

151@verbose_option(help="Include dataset type name, dimensions, and storage class in output.") 

152@click.option("--components/--no-components", 

153 default=None, 

154 help="For --components, apply all expression patterns to component dataset type names as well. " 

155 "For --no-components, never apply patterns to components. Default (where neither is " 

156 "specified) is to apply patterns to components only if their parent datasets were not " 

157 "matched by the expression. Fully-specified component datasets (`str` or `DatasetType` " 

158 "instances) are always included.") 

159@options_file_option() 

160def query_dataset_types(*args, **kwargs): 

161 """Get the dataset types in a repository.""" 

162 print(yaml.dump(cli_handle_exception(script.queryDatasetTypes, *args, **kwargs), sort_keys=False)) 

163 

164 

165@click.command() 

166@repo_argument(required=True) 

167@click.argument('dataset-type-name', nargs=1) 

168def remove_dataset_type(*args, **kwargs): 

169 """Remove a dataset type definition from a repository.""" 

170 cli_handle_exception(script.removeDatasetType, *args, **kwargs) 

171 

172 

173@click.command() 

174@repo_argument(required=True) 

175@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

176 "dataset types to be queried.") 

177@click.option("--collections", 

178 help=unwrap("""One or more expressions that fully or partially identify the collections to 

179 search for datasets.If not provided all datasets are returned."""), 

180 multiple=True, 

181 metavar=typeStrAcceptsMultiple, 

182 callback=split_commas) 

183@click.option("--where", 

184 help=unwrap("""A string expression similar to a SQL WHERE clause. May involve any column of a 

185 dimension table or a dimension name as a shortcut for the primary key column of a 

186 dimension table.""")) 

187@click.option("--find-first", 

188 is_flag=True, 

189 help=unwrap("""For each result data ID, only yield one DatasetRef of each DatasetType, from the 

190 first collection in which a dataset of that dataset type appears (according to the 

191 order of 'collections' passed in). If used, 'collections' must specify at least one 

192 expression and must not contain wildcards.""")) 

193@click.option("--show-uri", 

194 is_flag=True, 

195 help="Show the dataset URI in results.") 

196@options_file_option() 

197def query_datasets(**kwargs): 

198 """List the datasets in a repository.""" 

199 tables = cli_handle_exception(script.queryDatasets, **kwargs) 

200 

201 for table in tables: 

202 print("") 

203 table.pprint_all() 

204 print("")