Coverage for python/lsst/daf/butler/cli/cmd/commands.py : 70%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
23import yaml
25from ..opt import (
26 collection_type_option,
27 collection_argument,
28 collections_option,
29 dataset_type_option,
30 datasets_option,
31 dimensions_argument,
32 directory_argument,
33 glob_argument,
34 options_file_option,
35 repo_argument,
36 transfer_option,
37 verbose_option,
38 where_option,
39)
41from ..utils import (
42 cli_handle_exception,
43 ButlerCommand,
44 split_commas,
45 to_upper,
46 typeStrAcceptsMultiple,
47 unwrap,
48)
49from ... import script
52willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist."
53existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file."
54whereHelp = unwrap("""A string expression similar to a SQL WHERE clause. May involve any column of a dimension
55 table or a dimension name as a shortcut for the primary key column of a dimension
56 table.""")
59# The conversion from the import command name to the butler_import function
60# name for subcommand lookup is implemented in the cli/butler.py, in
61# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they
62# must be reflected in that location. If this becomes a common pattern a better
63# mechanism should be implemented.
64@click.command("import", cls=ButlerCommand)
65@repo_argument(required=True, help=willCreateRepoHelp)
66@directory_argument(required=True)
67@transfer_option()
68@click.option("--export-file",
69 help="Name for the file that contains database information associated with the exported "
70 "datasets. If this is not an absolute path, does not exist in the current working "
71 "directory, and --dir is provided, it is assumed to be in that directory. Defaults "
72 "to \"export.yaml\".",
73 type=click.File("r"))
74@click.option("--skip-dimensions", "-s", type=str, multiple=True, callback=split_commas,
75 metavar=typeStrAcceptsMultiple,
76 help="Dimensions that should be skipped during import")
77@options_file_option()
78def butler_import(*args, **kwargs):
79 """Import data into a butler repository."""
80 cli_handle_exception(script.butlerImport, *args, **kwargs)
83@click.command(cls=ButlerCommand)
84@repo_argument(required=True, help=willCreateRepoHelp)
85@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).")
86@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.")
87@click.option("--standalone", is_flag=True, help="Include all defaults in the config file in the repo, "
88 "insulating the repo from changes in package defaults.")
89@click.option("--override", is_flag=True, help="Allow values in the supplied config to override all "
90 "repo settings.")
91@click.option("--outfile", "-f", default=None, type=str, help="Name of output file to receive repository "
92 "configuration. Default is to write butler.yaml into the specified repo.")
93@options_file_option()
94def create(*args, **kwargs):
95 """Create an empty Gen3 Butler repository."""
96 cli_handle_exception(script.createRepo, *args, **kwargs)
99@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand)
100@repo_argument(required=True, help=existingRepoHelp)
101@click.option("--subset", "-s", type=str,
102 help="Subset of a configuration to report. This can be any key in the hierarchy such as "
103 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.")
104@click.option("--searchpath", "-p", type=str, multiple=True, callback=split_commas,
105 metavar=typeStrAcceptsMultiple,
106 help="Additional search paths to use for configuration overrides")
107@click.option("--file", "outfile", type=click.File("w"), default="-",
108 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout "
109 "by default.")
110@options_file_option()
111def config_dump(*args, **kwargs):
112 """Dump either a subset or full Butler configuration to standard output."""
113 cli_handle_exception(script.configDump, *args, **kwargs)
116@click.command(short_help="Validate the configuration files.", cls=ButlerCommand)
117@repo_argument(required=True, help=existingRepoHelp)
118@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.")
119@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True)
120@click.option("--ignore", "-i", type=str, multiple=True, callback=split_commas,
121 metavar=typeStrAcceptsMultiple,
122 help="DatasetType(s) to ignore for validation.")
123@options_file_option()
124def config_validate(*args, **kwargs):
125 """Validate the configuration files for a Gen3 Butler repository."""
126 is_good = cli_handle_exception(script.configValidate, *args, **kwargs)
127 if not is_good:
128 raise click.exceptions.Exit(1)
131@click.command(cls=ButlerCommand)
132@repo_argument(required=True)
133@collection_argument(help=unwrap("""COLLECTION is the Name of the collection to remove. If this is a tagged or
134 chained collection, datasets within the collection are not modified unless --unstore
135 is passed. If this is a run collection, --purge and --unstore must be passed, and
136 all datasets in it are fully removed from the data repository."""))
137@click.option("--purge",
138 help=unwrap("""Permit RUN collections to be removed, fully removing datasets within them.
139 Requires --unstore as an added precaution against accidental deletion. Must not be
140 passed if the collection is not a RUN."""),
141 is_flag=True)
142@click.option("--unstore",
143 help=("""Remove all datasets in the collection from all datastores in which they appear."""),
144 is_flag=True)
145@options_file_option()
146def prune_collection(**kwargs):
147 """Remove a collection and possibly prune datasets within it."""
148 cli_handle_exception(script.pruneCollection, **kwargs)
151@click.command(short_help="Search for collections.", cls=ButlerCommand)
152@repo_argument(required=True)
153@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the "
154 "collections to return.")
155@collection_type_option()
156@click.option("--chains",
157 default="table",
158 help=unwrap("""Affects how results are presented. TABLE lists each dataset in a row with
159 chained datasets' children listed in a Definition column. TREE lists children below
160 their parent in tree form. FLATTEN lists all datasets, including child datasets in
161 one list.Defaults to TABLE. """),
162 callback=to_upper,
163 type=click.Choice(("TABLE", "TREE", "FLATTEN"), case_sensitive=False))
164@options_file_option()
165def query_collections(*args, **kwargs):
166 """Get the collections whose names match an expression."""
167 table = cli_handle_exception(script.queryCollections, *args, **kwargs)
168 # The unit test that mocks script.queryCollections does not return a table
169 # so we need the following `if`.
170 if table:
171 # When chains==TREE, the children of chained datasets are indented
172 # relative to their parents. For this to work properly the table must
173 # be left-aligned.
174 table.pprint_all(align="<")
177@click.command(cls=ButlerCommand)
178@repo_argument(required=True)
179@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the "
180 "dataset types to return.")
181@verbose_option(help="Include dataset type name, dimensions, and storage class in output.")
182@click.option("--components/--no-components",
183 default=None,
184 help="For --components, apply all expression patterns to component dataset type names as well. "
185 "For --no-components, never apply patterns to components. Default (where neither is "
186 "specified) is to apply patterns to components only if their parent datasets were not "
187 "matched by the expression. Fully-specified component datasets (`str` or `DatasetType` "
188 "instances) are always included.")
189@options_file_option()
190def query_dataset_types(*args, **kwargs):
191 """Get the dataset types in a repository."""
192 print(yaml.dump(cli_handle_exception(script.queryDatasetTypes, *args, **kwargs), sort_keys=False))
195@click.command(cls=ButlerCommand)
196@repo_argument(required=True)
197@click.argument('dataset-type-name', nargs=1)
198def remove_dataset_type(*args, **kwargs):
199 """Remove a dataset type definition from a repository."""
200 cli_handle_exception(script.removeDatasetType, *args, **kwargs)
203@click.command(cls=ButlerCommand)
204@repo_argument(required=True)
205@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the "
206 "dataset types to be queried.")
207@collections_option()
208@where_option(help=whereHelp)
209@click.option("--find-first",
210 is_flag=True,
211 help=unwrap("""For each result data ID, only yield one DatasetRef of each DatasetType, from the
212 first collection in which a dataset of that dataset type appears (according to the
213 order of 'collections' passed in). If used, 'collections' must specify at least one
214 expression and must not contain wildcards."""))
215@click.option("--show-uri",
216 is_flag=True,
217 help="Show the dataset URI in results.")
218@options_file_option()
219def query_datasets(**kwargs):
220 """List the datasets in a repository."""
221 tables = cli_handle_exception(script.queryDatasets, **kwargs)
223 for table in tables:
224 print("")
225 table.pprint_all()
226 print("")
229@click.command(cls=ButlerCommand)
230@repo_argument(required=True)
231@click.argument('input-collection')
232@click.argument('output-collection')
233@click.argument('dataset-type-name')
234@click.option("--begin-date", type=str, default=None,
235 help=unwrap("""ISO-8601 datetime (TAI) of the beginning of the validity range for the
236 certified calibrations."""))
237@click.option("--end-date", type=str, default=None,
238 help=unwrap("""ISO-8601 datetime (TAI) of the end of the validity range for the
239 certified calibrations."""))
240@click.option("--search-all-inputs", is_flag=True, default=False,
241 help=unwrap("""Search all children of the inputCollection if it is a CHAINED collection,
242 instead of just the most recent one."""))
243def certify_calibrations(*args, **kwargs):
244 """Certify calibrations in a repository.
245 """
246 cli_handle_exception(script.certifyCalibrations, *args, **kwargs)
249@click.command(cls=ButlerCommand)
250@repo_argument(required=True)
251@dimensions_argument(help=unwrap("""DIMENSIONS are the keys of the data IDs to yield, such as exposure,
252 instrument, or tract. Will be expanded to include any dependencies."""))
253@collections_option()
254@datasets_option(help=unwrap("""An expression that fully or partially identifies dataset types that should
255 constrain the yielded data IDs. For example, including "raw" here would
256 constrain the yielded "instrument", "exposure", "detector", and
257 "physical_filter" values to only those for which at least one "raw" dataset
258 exists in "collections"."""))
259@where_option(help=whereHelp)
260@options_file_option()
261def query_data_ids(**kwargs):
262 """List the data IDs in a repository.
263 """
264 table = cli_handle_exception(script.queryDataIds, **kwargs)
265 if table:
266 table.pprint_all()
267 else:
268 if not kwargs.get("dimensions") and not kwargs.get("datasets"):
269 print("No results. Try requesting some dimensions or datasets, see --help for more information.")
270 else:
271 print("No results. Try --help for more information.")