Coverage for python/lsst/daf/butler/cli/cmd/commands.py : 66%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
24from ..opt import (
25 collection_type_option,
26 collection_argument,
27 collections_option,
28 components_option,
29 dataset_type_option,
30 datasets_option,
31 dimensions_argument,
32 directory_argument,
33 element_argument,
34 glob_argument,
35 options_file_option,
36 repo_argument,
37 transfer_option,
38 verbose_option,
39 where_option,
40)
42from ..utils import (
43 cli_handle_exception,
44 ButlerCommand,
45 split_commas,
46 to_upper,
47 typeStrAcceptsMultiple,
48 unwrap,
49)
50from ... import script
53willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist."
54existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file."
55whereHelp = unwrap("""A string expression similar to a SQL WHERE clause. May involve any column of a dimension
56 table or a dimension name as a shortcut for the primary key column of a dimension
57 table.""")
60# The conversion from the import command name to the butler_import function
61# name for subcommand lookup is implemented in the cli/butler.py, in
62# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they
63# must be reflected in that location. If this becomes a common pattern a better
64# mechanism should be implemented.
65@click.command("import", cls=ButlerCommand)
66@repo_argument(required=True, help=willCreateRepoHelp)
67@directory_argument(required=True)
68@transfer_option()
69@click.option("--export-file",
70 help="Name for the file that contains database information associated with the exported "
71 "datasets. If this is not an absolute path, does not exist in the current working "
72 "directory, and --dir is provided, it is assumed to be in that directory. Defaults "
73 "to \"export.yaml\".",
74 type=click.File("r"))
75@click.option("--skip-dimensions", "-s", type=str, multiple=True, callback=split_commas,
76 metavar=typeStrAcceptsMultiple,
77 help="Dimensions that should be skipped during import")
78@options_file_option()
79def butler_import(*args, **kwargs):
80 """Import data into a butler repository."""
81 cli_handle_exception(script.butlerImport, *args, **kwargs)
84@click.command(cls=ButlerCommand)
85@repo_argument(required=True, help=willCreateRepoHelp)
86@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).")
87@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.")
88@click.option("--standalone", is_flag=True, help="Include all defaults in the config file in the repo, "
89 "insulating the repo from changes in package defaults.")
90@click.option("--override", is_flag=True, help="Allow values in the supplied config to override all "
91 "repo settings.")
92@click.option("--outfile", "-f", default=None, type=str, help="Name of output file to receive repository "
93 "configuration. Default is to write butler.yaml into the specified repo.")
94@options_file_option()
95def create(*args, **kwargs):
96 """Create an empty Gen3 Butler repository."""
97 cli_handle_exception(script.createRepo, *args, **kwargs)
100@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand)
101@repo_argument(required=True, help=existingRepoHelp)
102@click.option("--subset", "-s", type=str,
103 help="Subset of a configuration to report. This can be any key in the hierarchy such as "
104 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.")
105@click.option("--searchpath", "-p", type=str, multiple=True, callback=split_commas,
106 metavar=typeStrAcceptsMultiple,
107 help="Additional search paths to use for configuration overrides")
108@click.option("--file", "outfile", type=click.File("w"), default="-",
109 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout "
110 "by default.")
111@options_file_option()
112def config_dump(*args, **kwargs):
113 """Dump either a subset or full Butler configuration to standard output."""
114 cli_handle_exception(script.configDump, *args, **kwargs)
117@click.command(short_help="Validate the configuration files.", cls=ButlerCommand)
118@repo_argument(required=True, help=existingRepoHelp)
119@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.")
120@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True)
121@click.option("--ignore", "-i", type=str, multiple=True, callback=split_commas,
122 metavar=typeStrAcceptsMultiple,
123 help="DatasetType(s) to ignore for validation.")
124@options_file_option()
125def config_validate(*args, **kwargs):
126 """Validate the configuration files for a Gen3 Butler repository."""
127 is_good = cli_handle_exception(script.configValidate, *args, **kwargs)
128 if not is_good:
129 raise click.exceptions.Exit(1)
132@click.command(cls=ButlerCommand)
133@repo_argument(required=True)
134@collection_argument(help=unwrap("""COLLECTION is the Name of the collection to remove. If this is a tagged or
135 chained collection, datasets within the collection are not modified unless --unstore
136 is passed. If this is a run collection, --purge and --unstore must be passed, and
137 all datasets in it are fully removed from the data repository."""))
138@click.option("--purge",
139 help=unwrap("""Permit RUN collections to be removed, fully removing datasets within them.
140 Requires --unstore as an added precaution against accidental deletion. Must not be
141 passed if the collection is not a RUN."""),
142 is_flag=True)
143@click.option("--unstore",
144 help=("""Remove all datasets in the collection from all datastores in which they appear."""),
145 is_flag=True)
146@options_file_option()
147def prune_collection(**kwargs):
148 """Remove a collection and possibly prune datasets within it."""
149 cli_handle_exception(script.pruneCollection, **kwargs)
152@click.command(short_help="Search for collections.", cls=ButlerCommand)
153@repo_argument(required=True)
154@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the "
155 "collections to return.")
156@collection_type_option()
157@click.option("--chains",
158 default="table",
159 help=unwrap("""Affects how results are presented. TABLE lists each dataset in a row with
160 chained datasets' children listed in a Definition column. TREE lists children below
161 their parent in tree form. FLATTEN lists all datasets, including child datasets in
162 one list.Defaults to TABLE. """),
163 callback=to_upper,
164 type=click.Choice(("TABLE", "TREE", "FLATTEN"), case_sensitive=False))
165@options_file_option()
166def query_collections(*args, **kwargs):
167 """Get the collections whose names match an expression."""
168 table = cli_handle_exception(script.queryCollections, *args, **kwargs)
169 # The unit test that mocks script.queryCollections does not return a table
170 # so we need the following `if`.
171 if table:
172 # When chains==TREE, the children of chained datasets are indented
173 # relative to their parents. For this to work properly the table must
174 # be left-aligned.
175 table.pprint_all(align="<")
178@click.command(cls=ButlerCommand)
179@repo_argument(required=True)
180@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the "
181 "dataset types to return.")
182@verbose_option(help="Include dataset type name, dimensions, and storage class in output.")
183@components_option()
184@options_file_option()
185def query_dataset_types(*args, **kwargs):
186 """Get the dataset types in a repository."""
187 table = cli_handle_exception(script.queryDatasetTypes, *args, **kwargs)
188 if table:
189 table.pprint_all()
190 else:
191 print("No results. Try --help for more information.")
194@click.command(cls=ButlerCommand)
195@repo_argument(required=True)
196@click.argument('dataset-type-name', nargs=1)
197def remove_dataset_type(*args, **kwargs):
198 """Remove a dataset type definition from a repository."""
199 cli_handle_exception(script.removeDatasetType, *args, **kwargs)
202@click.command(cls=ButlerCommand)
203@repo_argument(required=True)
204@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the "
205 "dataset types to be queried.")
206@collections_option()
207@where_option(help=whereHelp)
208@click.option("--find-first",
209 is_flag=True,
210 help=unwrap("""For each result data ID, only yield one DatasetRef of each DatasetType, from the
211 first collection in which a dataset of that dataset type appears (according to the
212 order of 'collections' passed in). If used, 'collections' must specify at least one
213 expression and must not contain wildcards."""))
214@click.option("--show-uri",
215 is_flag=True,
216 help="Show the dataset URI in results.")
217@options_file_option()
218def query_datasets(**kwargs):
219 """List the datasets in a repository."""
220 tables = cli_handle_exception(script.queryDatasets, **kwargs)
222 for table in tables:
223 print("")
224 table.pprint_all()
225 print("")
228@click.command(cls=ButlerCommand)
229@repo_argument(required=True)
230@click.argument('input-collection')
231@click.argument('output-collection')
232@click.argument('dataset-type-name')
233@click.option("--begin-date", type=str, default=None,
234 help=unwrap("""ISO-8601 datetime (TAI) of the beginning of the validity range for the
235 certified calibrations."""))
236@click.option("--end-date", type=str, default=None,
237 help=unwrap("""ISO-8601 datetime (TAI) of the end of the validity range for the
238 certified calibrations."""))
239@click.option("--search-all-inputs", is_flag=True, default=False,
240 help=unwrap("""Search all children of the inputCollection if it is a CHAINED collection,
241 instead of just the most recent one."""))
242def certify_calibrations(*args, **kwargs):
243 """Certify calibrations in a repository.
244 """
245 cli_handle_exception(script.certifyCalibrations, *args, **kwargs)
248@click.command(cls=ButlerCommand)
249@repo_argument(required=True)
250@dimensions_argument(help=unwrap("""DIMENSIONS are the keys of the data IDs to yield, such as exposure,
251 instrument, or tract. Will be expanded to include any dependencies."""))
252@collections_option()
253@datasets_option(help=unwrap("""An expression that fully or partially identifies dataset types that should
254 constrain the yielded data IDs. For example, including "raw" here would
255 constrain the yielded "instrument", "exposure", "detector", and
256 "physical_filter" values to only those for which at least one "raw" dataset
257 exists in "collections"."""))
258@where_option(help=whereHelp)
259@options_file_option()
260def query_data_ids(**kwargs):
261 """List the data IDs in a repository.
262 """
263 table = cli_handle_exception(script.queryDataIds, **kwargs)
264 if table:
265 table.pprint_all()
266 else:
267 if not kwargs.get("dimensions") and not kwargs.get("datasets"):
268 print("No results. Try requesting some dimensions or datasets, see --help for more information.")
269 else:
270 print("No results. Try --help for more information.")
273@click.command(cls=ButlerCommand)
274@repo_argument(required=True)
275@element_argument(required=True)
276@datasets_option(help=unwrap("""An expression that fully or partially identifies dataset types that should
277 constrain the yielded records. Only affects results when used with
278 --collections."""))
279@collections_option(help=collections_option.help + " Only affects results when used with --datasets.")
280@where_option(help=whereHelp)
281@click.option("--no-check", is_flag=True,
282 help=unwrap("""Don't check the query before execution. By default the query is checked before it
283 executed, this may reject some valid queries that resemble common mistakes."""))
284@options_file_option()
285def query_dimension_records(**kwargs):
286 """Query for dimension information."""
287 table = cli_handle_exception(script.queryDimensionRecords, **kwargs)
288 if table:
289 table.pprint_all()
290 else:
291 print("No results. Try --help for more information.")