Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23import yaml 

24 

25from ..opt import (collection_type_option, dataset_type_option, directory_argument, options_file_option, 

26 glob_argument, repo_argument, transfer_option, verbose_option) 

27from ..utils import cli_handle_exception, split_commas, typeStrAcceptsMultiple, unwrap 

28from ...script import (butlerImport, createRepo, configDump, configValidate, pruneCollection, 

29 queryCollections, queryDatasetTypes, removeDatasetType) 

30 

31willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist." 

32existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file." 

33 

34 

35# The conversion from the import command name to the butler_import function 

36# name for subcommand lookup is implemented in the cli/butler.py, in 

37# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they 

38# must be reflected in that location. If this becomes a common pattern a better 

39# mechanism should be implemented. 

40@click.command("import") 

41@repo_argument(required=True, help=willCreateRepoHelp) 

42@directory_argument(required=True) 

43@transfer_option() 

44@click.option("--export-file", 

45 help="Name for the file that contains database information associated with the exported " 

46 "datasets. If this is not an absolute path, does not exist in the current working " 

47 "directory, and --dir is provided, it is assumed to be in that directory. Defaults " 

48 "to \"export.yaml\".", 

49 type=click.File("r")) 

50@click.option("--skip-dimensions", "-s", type=str, multiple=True, callback=split_commas, 

51 metavar=typeStrAcceptsMultiple, 

52 help="Dimensions that should be skipped during import") 

53@options_file_option() 

54def butler_import(*args, **kwargs): 

55 """Import data into a butler repository.""" 

56 cli_handle_exception(butlerImport, *args, **kwargs) 

57 

58 

59@click.command() 

60@repo_argument(required=True, help=willCreateRepoHelp) 

61@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).") 

62@click.option("--standalone", is_flag=True, help="Include all defaults in the config file in the repo, " 

63 "insulating the repo from changes in package defaults.") 

64@click.option("--override", is_flag=True, help="Allow values in the supplied config to override all " 

65 "repo settings.") 

66@click.option("--outfile", "-f", default=None, type=str, help="Name of output file to receive repository " 

67 "configuration. Default is to write butler.yaml into the specified repo.") 

68@options_file_option() 

69def create(*args, **kwargs): 

70 """Create an empty Gen3 Butler repository.""" 

71 cli_handle_exception(createRepo, *args, **kwargs) 

72 

73 

74@click.command(short_help="Dump butler config to stdout.") 

75@repo_argument(required=True, help=existingRepoHelp) 

76@click.option("--subset", "-s", type=str, 

77 help="Subset of a configuration to report. This can be any key in the hierarchy such as " 

78 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.") 

79@click.option("--searchpath", "-p", type=str, multiple=True, callback=split_commas, 

80 metavar=typeStrAcceptsMultiple, 

81 help="Additional search paths to use for configuration overrides") 

82@click.option("--file", "outfile", type=click.File("w"), default="-", 

83 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout " 

84 "by default.") 

85@options_file_option() 

86def config_dump(*args, **kwargs): 

87 """Dump either a subset or full Butler configuration to standard output.""" 

88 cli_handle_exception(configDump, *args, **kwargs) 

89 

90 

91@click.command(short_help="Validate the configuration files.") 

92@repo_argument(required=True, help=existingRepoHelp) 

93@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.") 

94@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True) 

95@click.option("--ignore", "-i", type=str, multiple=True, callback=split_commas, 

96 metavar=typeStrAcceptsMultiple, 

97 help="DatasetType(s) to ignore for validation.") 

98@options_file_option() 

99def config_validate(*args, **kwargs): 

100 """Validate the configuration files for a Gen3 Butler repository.""" 

101 is_good = cli_handle_exception(configValidate, *args, **kwargs) 

102 if not is_good: 

103 raise click.exceptions.Exit(1) 

104 

105 

106@click.command() 

107@repo_argument(required=True) 

108@click.option("--collection", 

109 help=unwrap("""Name of the collection to remove. If this is a TAGGED or CHAINED collection, 

110 datasets within the collection are not modified unless --unstore is passed. If this 

111 is a RUN collection, --purge and --unstore must be passed, and all datasets in it 

112 are fully removed from the data repository. """)) 

113@click.option("--purge", 

114 help=unwrap("""Permit RUN collections to be removed, fully removing datasets within them. 

115 Requires --unstore as an added precaution against accidental deletion. Must not be 

116 passed if the collection is not a RUN."""), 

117 is_flag=True) 

118@click.option("--unstore", 

119 help=("""Remove all datasets in the collection from all datastores in which they appear."""), 

120 is_flag=True) 

121@options_file_option() 

122def prune_collection(**kwargs): 

123 """Remove a collection and possibly prune datasets within it.""" 

124 cli_handle_exception(pruneCollection, **kwargs) 

125 

126 

127@click.command(short_help="Search for collections.") 

128@repo_argument(required=True) 

129@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

130 "collections to return.") 

131@collection_type_option() 

132@click.option("--flatten-chains/--no-flatten-chains", 

133 help="Recursively get the child collections of matching CHAINED collections. Default is " 

134 "--no-flatten-chains.") 

135@click.option("--include-chains/--no-include-chains", 

136 default=None, 

137 help="For --include-chains, return records for matching CHAINED collections. For " 

138 "--no-include-chains do not return records for CHAINED collections. Default is the " 

139 "opposite of --flatten-chains: include either CHAINED collections or their children, but " 

140 "not both.") 

141@options_file_option() 

142def query_collections(*args, **kwargs): 

143 """Get the collections whose names match an expression.""" 

144 print(yaml.dump(cli_handle_exception(queryCollections, *args, **kwargs))) 

145 

146 

147@click.command() 

148@repo_argument(required=True) 

149@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

150 "dataset types to return.") 

151@verbose_option(help="Include dataset type name, dimensions, and storage class in output.") 

152@click.option("--components/--no-components", 

153 default=None, 

154 help="For --components, apply all expression patterns to component dataset type names as well. " 

155 "For --no-components, never apply patterns to components. Default (where neither is " 

156 "specified) is to apply patterns to components only if their parent datasets were not " 

157 "matched by the expression. Fully-specified component datasets (`str` or `DatasetType` " 

158 "instances) are always included.") 

159@options_file_option() 

160def query_dataset_types(*args, **kwargs): 

161 """Get the dataset types in a repository.""" 

162 print(yaml.dump(cli_handle_exception(queryDatasetTypes, *args, **kwargs), sort_keys=False)) 

163 

164 

165@click.command() 

166@repo_argument(required=True) 

167@click.argument('dataset-type-name', nargs=1) 

168def remove_dataset_type(*args, **kwargs): 

169 """Remove a dataset type definition from a reopsitory.""" 

170 cli_handle_exception(removeDatasetType, *args, **kwargs)