Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23import yaml 

24 

25from ..opt import (collection_type_option, dataset_type_option, directory_argument, glob_argument, 

26 repo_argument, run_option, transfer_option, verbose_option) 

27from ..utils import split_commas, cli_handle_exception, typeStrAcceptsMultiple 

28from ...script import (butlerImport, createRepo, configDump, configValidate, queryCollections, 

29 queryDatasetTypes) 

30 

31willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist." 

32existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file." 

33 

34 

35# The conversion from the import command name to the butler_import function 

36# name for subcommand lookup is implemented in the cli/butler.py, in 

37# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they 

38# must be reflected in that location. If this becomes a common pattern a better 

39# mechanism should be implemented. 

40@click.command("import") 

41@repo_argument(required=True, help=willCreateRepoHelp) 

42@directory_argument(required=True) 

43@transfer_option() 

44@run_option(required=True) 

45@click.option("--export-file", 

46 help="Name for the file that contains database information associated with the exported " 

47 "datasets. If this is not an absolute path, does not exist in the current working " 

48 "directory, and --dir is provided, it is assumed to be in that directory. Defaults " 

49 "to \"export.yaml\".", 

50 type=click.File("r")) 

51@click.option("--skip-dimensions", "-s", type=str, multiple=True, callback=split_commas, 

52 metavar=typeStrAcceptsMultiple, 

53 help="Dimensions that should be skipped during import") 

54def butler_import(*args, **kwargs): 

55 """Import data into a butler repository.""" 

56 cli_handle_exception(butlerImport, *args, **kwargs) 

57 

58 

59@click.command() 

60@repo_argument(required=True, help=willCreateRepoHelp) 

61@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).") 

62@click.option("--standalone", is_flag=True, help="Include all defaults in the config file in the repo, " 

63 "insulating the repo from changes in package defaults.") 

64@click.option("--override", is_flag=True, help="Allow values in the supplied config to override all " 

65 "repo settings.") 

66@click.option("--outfile", "-f", default=None, type=str, help="Name of output file to receive repository " 

67 "configuration. Default is to write butler.yaml into the specified repo.") 

68def create(*args, **kwargs): 

69 """Create an empty Gen3 Butler repository.""" 

70 cli_handle_exception(createRepo, *args, **kwargs) 

71 

72 

73@click.command(short_help="Dump butler config to stdout.") 

74@repo_argument(required=True, help=existingRepoHelp) 

75@click.option("--subset", "-s", type=str, 

76 help="Subset of a configuration to report. This can be any key in the hierarchy such as " 

77 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.") 

78@click.option("--searchpath", "-p", type=str, multiple=True, callback=split_commas, 

79 metavar=typeStrAcceptsMultiple, 

80 help="Additional search paths to use for configuration overrides") 

81@click.option("--file", "outfile", type=click.File("w"), default="-", 

82 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout " 

83 "by default.") 

84def config_dump(*args, **kwargs): 

85 """Dump either a subset or full Butler configuration to standard output.""" 

86 cli_handle_exception(configDump, *args, **kwargs) 

87 

88 

89@click.command(short_help="Validate the configuration files.") 

90@repo_argument(required=True, help=existingRepoHelp) 

91@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.") 

92@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True) 

93@click.option("--ignore", "-i", type=str, multiple=True, callback=split_commas, 

94 metavar=typeStrAcceptsMultiple, 

95 help="DatasetType(s) to ignore for validation.") 

96def config_validate(*args, **kwargs): 

97 """Validate the configuration files for a Gen3 Butler repository.""" 

98 is_good = cli_handle_exception(configValidate, *args, **kwargs) 

99 if not is_good: 

100 raise click.exceptions.Exit(1) 

101 

102 

103@click.command(short_help="Search for collections.") 

104@repo_argument(required=True) 

105@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

106 "collections to return.") 

107@collection_type_option() 

108@click.option("--flatten-chains/--no-flatten-chains", 

109 help="Recursively get the child collections of matching CHAINED collections. Default is " 

110 "--no-flatten-chains.") 

111@click.option("--include-chains/--no-include-chains", 

112 default=None, 

113 help="For --include-chains, return records for matching CHAINED collections. For " 

114 "--no-include-chains do not return records for CHAINED collections. Default is the " 

115 "opposite of --flatten-chains: include either CHAINED collections or their children, but " 

116 "not both.") 

117def query_collections(*args, **kwargs): 

118 """Get the collections whose names match an expression.""" 

119 print(yaml.dump(cli_handle_exception(queryCollections, *args, **kwargs))) 

120 

121 

122@click.command() 

123@repo_argument(required=True) 

124@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

125 "dataset types to return.") 

126@verbose_option(help="Include dataset type name, dimensions, and storage class in output.") 

127@click.option("--components/--no-components", 

128 default=None, 

129 help="For --components, apply all expression patterns to component dataset type names as well. " 

130 "For --no-components, never apply patterns to components. Default (where neither is " 

131 "specified) is to apply patterns to components only if their parent datasets were not " 

132 "matched by the expression. Fully-specified component datasets (`str` or `DatasetType` " 

133 "instances) are always included.") 

134def query_dataset_types(*args, **kwargs): 

135 """Get the dataset types in a repository.""" 

136 print(yaml.dump(cli_handle_exception(queryDatasetTypes, *args, **kwargs), sort_keys=False))