Coverage for python/lsst/daf/butler/cli/cmd/commands.py : 55%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
24from ..opt import (
25 collection_type_option,
26 collection_argument,
27 collections_argument,
28 collections_option,
29 components_option,
30 dataset_type_option,
31 datasets_option,
32 dimensions_argument,
33 directory_argument,
34 element_argument,
35 glob_argument,
36 options_file_option,
37 query_datasets_options,
38 repo_argument,
39 transfer_option,
40 verbose_option,
41 where_option,
42)
44from ..utils import (
45 ButlerCommand,
46 MWOptionDecorator,
47 option_section,
48 printAstropyTables,
49 split_commas,
50 to_upper,
51 typeStrAcceptsMultiple,
52 unwrap,
53 where_help,
54)
56from ... import script
59willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist."
60existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file."
63@click.command(cls=ButlerCommand, short_help="Add existing datasets to a tagged collection.")
64@repo_argument(required=True)
65@collection_argument(help="COLLECTION is the collection the datasets should be associated with.")
66@query_datasets_options(repo=False, showUri=False, useArguments=False)
67@options_file_option()
68def associate(**kwargs):
69 """Add existing datasets to a tagged collection; searches for datasets with
70 the options and adds them to the named COLLECTION.
71 """
72 script.associate(**kwargs)
75# The conversion from the import command name to the butler_import function
76# name for subcommand lookup is implemented in the cli/butler.py, in
77# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they
78# must be reflected in that location. If this becomes a common pattern a better
79# mechanism should be implemented.
80@click.command("import", cls=ButlerCommand)
81@repo_argument(required=True, help=willCreateRepoHelp)
82@directory_argument(required=True)
83@transfer_option()
84@click.option("--export-file",
85 help="Name for the file that contains database information associated with the exported "
86 "datasets. If this is not an absolute path, does not exist in the current working "
87 "directory, and --dir is provided, it is assumed to be in that directory. Defaults "
88 "to \"export.yaml\".",
89 type=click.File("r"))
90@click.option("--skip-dimensions", "-s", type=str, multiple=True, callback=split_commas,
91 metavar=typeStrAcceptsMultiple,
92 help="Dimensions that should be skipped during import")
93@click.option("--reuse-ids", is_flag=True, help="Force re-use of imported dataset IDs for integer IDs.")
94@options_file_option()
95def butler_import(*args, **kwargs):
96 """Import data into a butler repository."""
97 script.butlerImport(*args, **kwargs)
100@click.command(cls=ButlerCommand)
101@repo_argument(required=True, help=willCreateRepoHelp)
102@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).")
103@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.")
104@click.option("--standalone", is_flag=True, help="Include all defaults in the config file in the repo, "
105 "insulating the repo from changes in package defaults.")
106@click.option("--override", is_flag=True, help="Allow values in the supplied config to override all "
107 "repo settings.")
108@click.option("--outfile", "-f", default=None, type=str, help="Name of output file to receive repository "
109 "configuration. Default is to write butler.yaml into the specified repo.")
110@options_file_option()
111def create(*args, **kwargs):
112 """Create an empty Gen3 Butler repository."""
113 script.createRepo(*args, **kwargs)
116@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand)
117@repo_argument(required=True, help=existingRepoHelp)
118@click.option("--subset", "-s", type=str,
119 help="Subset of a configuration to report. This can be any key in the hierarchy such as "
120 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.")
121@click.option("--searchpath", "-p", type=str, multiple=True, callback=split_commas,
122 metavar=typeStrAcceptsMultiple,
123 help="Additional search paths to use for configuration overrides")
124@click.option("--file", "outfile", type=click.File("w"), default="-",
125 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout "
126 "by default.")
127@options_file_option()
128def config_dump(*args, **kwargs):
129 """Dump either a subset or full Butler configuration to standard output."""
130 script.configDump(*args, **kwargs)
133@click.command(short_help="Validate the configuration files.", cls=ButlerCommand)
134@repo_argument(required=True, help=existingRepoHelp)
135@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.")
136@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True)
137@click.option("--ignore", "-i", type=str, multiple=True, callback=split_commas,
138 metavar=typeStrAcceptsMultiple,
139 help="DatasetType(s) to ignore for validation.")
140@options_file_option()
141def config_validate(*args, **kwargs):
142 """Validate the configuration files for a Gen3 Butler repository."""
143 is_good = script.configValidate(*args, **kwargs)
144 if not is_good:
145 raise click.exceptions.Exit(1)
148@click.command(cls=ButlerCommand)
149@repo_argument(required=True)
150@collection_argument(help=unwrap("""COLLECTION is the Name of the collection to remove. If this is a tagged or
151 chained collection, datasets within the collection are not modified unless --unstore
152 is passed. If this is a run collection, --purge and --unstore must be passed, and
153 all datasets in it are fully removed from the data repository."""))
154@click.option("--purge",
155 help=unwrap("""Permit RUN collections to be removed, fully removing datasets within them.
156 Requires --unstore as an added precaution against accidental deletion. Must not be
157 passed if the collection is not a RUN."""),
158 is_flag=True)
159@click.option("--unstore",
160 help=("""Remove all datasets in the collection from all datastores in which they appear."""),
161 is_flag=True)
162@click.option("--unlink",
163 help="Before removing the given `collection` unlink it from from this parent collection.",
164 multiple=True,
165 callback=split_commas)
166@options_file_option()
167def prune_collection(**kwargs):
168 """Remove a collection and possibly prune datasets within it."""
169 script.pruneCollection(**kwargs)
172pruneDatasets_wouldRemoveMsg = unwrap("""The following datasets will be removed from any datastores in which
173 they are present:""")
174pruneDatasets_wouldDisassociateMsg = unwrap("""The following datasets will be disassociated from {collections}
175 if they are currently present in it (which is not checked):""")
176pruneDatasets_wouldDisassociateAndRemoveMsg = unwrap("""The following datasets will be disassociated from
177 {collections} if they are currently present in it (which is
178 not checked), and removed from any datastores in which they
179 are present.""")
180pruneDatasets_willRemoveMsg = "The following datasets will be removed:"
181pruneDatasets_askContinueMsg = "Continue?"
182pruneDatasets_didRemoveAforementioned = "The datasets were removed."
183pruneDatasets_didNotRemoveAforementioned = "Did not remove the datasets."
184pruneDatasets_didRemoveMsg = "Removed the following datasets:"
185pruneDatasets_noDatasetsFound = "Did not find any datasets."
186pruneDatasets_errPurgeAndDisassociate = unwrap(
187 """"--disassociate and --purge may not be used together: --disassociate purges from just the passed TAGged
188 collections, but --purge forces disassociation from all of them. """
189)
190pruneDatasets_errQuietWithDryRun = "Can not use --quiet and --dry-run together."
191pruneDatasets_errNoCollectionRestriction = unwrap(
192 """Must indicate collections from which to prune datasets by passing COLLETION arguments (select all
193 collections by passing '*', or consider using 'butler prune-collections'), by using --purge to pass a run
194 collection, or by using --disassociate to select a tagged collection.""")
195pruneDatasets_errPruneOnNotRun = "Can not prune a collection that is not a RUN collection: {collection}"
196pruneDatasets_errNoOp = "No operation: one of --purge, --unstore, or --disassociate must be provided."
198disassociate_option = MWOptionDecorator(
199 "--disassociate", "disassociate_tags",
200 help=unwrap("""Disassociate pruned datasets from the given tagged collections. May not be used with
201 --purge."""),
202 multiple=True,
203 callback=split_commas,
204 metavar="TAG"
205)
208purge_option = MWOptionDecorator(
209 "--purge", "purge_run",
210 help=unwrap("""Completely remove the dataset from the given RUN in the Registry. May not be used with
211 --disassociate. Note, this may remove provenance information from datasets other than those
212 provided, and should be used with extreme care."""),
213 metavar="RUN"
214)
217find_all_option = MWOptionDecorator(
218 "--find-all", is_flag=True,
219 help=unwrap("""Purge the dataset results from all of the collections in which a dataset of that dataset
220 type + data id combination appear. (By default only the first found dataset type + data id is
221 purged, according to the order of COLLECTIONS passed in).""")
222)
225unstore_option = MWOptionDecorator(
226 "--unstore",
227 is_flag=True,
228 help=unwrap("""Remove these datasets from all datastores configured with this data repository. If
229 --disassociate and --purge are not used then --unstore will be used by default. Note that
230 --unstore will make it impossible to retrieve these datasets even via other collections.
231 Datasets that are already not stored are ignored by this option.""")
232)
235dry_run_option = MWOptionDecorator(
236 "--dry-run",
237 is_flag=True,
238 help=unwrap("""Display the datasets that would be removed but do not remove them.
240 Note that a dataset can be in collections other than its RUN-type collection, and removing it
241 will remove it from all of them, even though the only one this will show is its RUN
242 collection.""")
243)
246confirm_option = MWOptionDecorator(
247 "--confirm/--no-confirm",
248 default=True,
249 help="Print expected action and a confirmation prompt before executing. Default is --confirm."
250)
253quiet_option = MWOptionDecorator(
254 "--quiet",
255 is_flag=True,
256 help=unwrap("""Makes output quiet. Implies --no-confirm. Requires --dry-run not be passed.""")
257)
260@click.command(cls=ButlerCommand, short_help="Remove datasets.")
261@repo_argument(required=True)
262@collections_argument(help=unwrap("""COLLECTIONS is or more expressions that identify the collections to
263 search for datasets. Glob-style expressions may be used but only if the
264 --find-all flag is also passed."""))
265@option_section("Query Datasets Options:")
266@datasets_option(help="One or more glob-style expressions that identify the dataset types to be pruned.",
267 multiple=True,
268 callback=split_commas)
269@find_all_option()
270@where_option(help=where_help)
271@option_section("Prune Options:")
272@disassociate_option()
273@purge_option()
274@unstore_option()
275@option_section("Execution Options:")
276@dry_run_option()
277@confirm_option()
278@quiet_option()
279@option_section("Other Options:")
280@options_file_option()
281def prune_datasets(**kwargs):
282 """Query for and remove one or more datasets from a collection and/or
283 storage.
284 """
285 quiet = kwargs.pop("quiet", False)
286 if quiet:
287 if kwargs["dry_run"]:
288 raise click.ClickException(pruneDatasets_errQuietWithDryRun)
289 kwargs["confirm"] = False
291 result = script.pruneDatasets(**kwargs)
293 if result.errPurgeAndDisassociate:
294 raise click.ClickException(pruneDatasets_errPurgeAndDisassociate)
295 return
296 if result.errNoCollectionRestriction:
297 raise click.ClickException(pruneDatasets_errNoCollectionRestriction)
298 if result.errPruneOnNotRun:
299 raise click.ClickException(pruneDatasets_errPruneOnNotRun.format(**result.errDict))
300 if result.errNoOp:
301 raise click.ClickException(pruneDatasets_errNoOp)
302 if result.dryRun:
303 if result.action["disassociate"] and result.action["unstore"]:
304 msg = pruneDatasets_wouldDisassociateAndRemoveMsg
305 elif result.action["disassociate"]:
306 msg = pruneDatasets_wouldDisassociateMsg
307 else:
308 msg = pruneDatasets_wouldRemoveMsg
309 print(msg.format(**result.action))
310 printAstropyTables(result.tables)
311 return
312 if result.confirm:
313 if not result.tables:
314 print(pruneDatasets_noDatasetsFound)
315 return
316 print(pruneDatasets_willRemoveMsg)
317 printAstropyTables(result.tables)
318 doContinue = click.confirm(pruneDatasets_askContinueMsg, default=False)
319 if doContinue:
320 result.onConfirmation()
321 print(pruneDatasets_didRemoveAforementioned)
322 else:
323 print(pruneDatasets_didNotRemoveAforementioned)
324 return
325 if result.finished:
326 if not quiet:
327 print(pruneDatasets_didRemoveMsg)
328 printAstropyTables(result.tables)
329 return
332@click.command(short_help="Search for collections.", cls=ButlerCommand)
333@repo_argument(required=True)
334@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the "
335 "collections to return.")
336@collection_type_option()
337@click.option("--chains",
338 default="table",
339 help=unwrap("""Affects how results are presented. TABLE lists each dataset in a row with
340 chained datasets' children listed in a Definition column. TREE lists children below
341 their parent in tree form. FLATTEN lists all datasets, including child datasets in
342 one list.Defaults to TABLE. """),
343 callback=to_upper,
344 type=click.Choice(("TABLE", "TREE", "FLATTEN"), case_sensitive=False))
345@options_file_option()
346def query_collections(*args, **kwargs):
347 """Get the collections whose names match an expression."""
348 table = script.queryCollections(*args, **kwargs)
349 # The unit test that mocks script.queryCollections does not return a table
350 # so we need the following `if`.
351 if table:
352 # When chains==TREE, the children of chained datasets are indented
353 # relative to their parents. For this to work properly the table must
354 # be left-aligned.
355 table.pprint_all(align="<")
358@click.command(cls=ButlerCommand)
359@repo_argument(required=True)
360@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the "
361 "dataset types to return.")
362@verbose_option(help="Include dataset type name, dimensions, and storage class in output.")
363@components_option()
364@options_file_option()
365def query_dataset_types(*args, **kwargs):
366 """Get the dataset types in a repository."""
367 table = script.queryDatasetTypes(*args, **kwargs)
368 if table:
369 table.pprint_all()
370 else:
371 print("No results. Try --help for more information.")
374@click.command(cls=ButlerCommand)
375@repo_argument(required=True)
376@click.argument('dataset-type-name', nargs=1)
377def remove_dataset_type(*args, **kwargs):
378 """Remove a dataset type definition from a repository."""
379 script.removeDatasetType(*args, **kwargs)
382@click.command(cls=ButlerCommand)
383@query_datasets_options()
384@options_file_option()
385def query_datasets(**kwargs):
386 """List the datasets in a repository."""
387 for table in script.QueryDatasets(**kwargs).getTables():
388 print("")
389 table.pprint_all()
390 print("")
393@click.command(cls=ButlerCommand)
394@repo_argument(required=True)
395@click.argument('input-collection')
396@click.argument('output-collection')
397@click.argument('dataset-type-name')
398@click.option("--begin-date", type=str, default=None,
399 help=unwrap("""ISO-8601 datetime (TAI) of the beginning of the validity range for the
400 certified calibrations."""))
401@click.option("--end-date", type=str, default=None,
402 help=unwrap("""ISO-8601 datetime (TAI) of the end of the validity range for the
403 certified calibrations."""))
404@click.option("--search-all-inputs", is_flag=True, default=False,
405 help=unwrap("""Search all children of the inputCollection if it is a CHAINED collection,
406 instead of just the most recent one."""))
407@options_file_option()
408def certify_calibrations(*args, **kwargs):
409 """Certify calibrations in a repository.
410 """
411 script.certifyCalibrations(*args, **kwargs)
414@click.command(cls=ButlerCommand)
415@repo_argument(required=True)
416@dimensions_argument(help=unwrap("""DIMENSIONS are the keys of the data IDs to yield, such as exposure,
417 instrument, or tract. Will be expanded to include any dependencies."""))
418@collections_option()
419@datasets_option(help=unwrap("""An expression that fully or partially identifies dataset types that should
420 constrain the yielded data IDs. For example, including "raw" here would
421 constrain the yielded "instrument", "exposure", "detector", and
422 "physical_filter" values to only those for which at least one "raw" dataset
423 exists in "collections"."""))
424@where_option(help=where_help)
425@options_file_option()
426def query_data_ids(**kwargs):
427 """List the data IDs in a repository.
428 """
429 table = script.queryDataIds(**kwargs)
430 if table:
431 table.pprint_all()
432 else:
433 if not kwargs.get("dimensions") and not kwargs.get("datasets"):
434 print("No results. Try requesting some dimensions or datasets, see --help for more information.")
435 else:
436 print("No results. Try --help for more information.")
439@click.command(cls=ButlerCommand)
440@repo_argument(required=True)
441@element_argument(required=True)
442@datasets_option(help=unwrap("""An expression that fully or partially identifies dataset types that should
443 constrain the yielded records. Only affects results when used with
444 --collections."""))
445@collections_option(help=collections_option.help + " Only affects results when used with --datasets.")
446@where_option(help=where_help)
447@click.option("--no-check", is_flag=True,
448 help=unwrap("""Don't check the query before execution. By default the query is checked before it
449 executed, this may reject some valid queries that resemble common mistakes."""))
450@options_file_option()
451def query_dimension_records(**kwargs):
452 """Query for dimension information."""
453 table = script.queryDimensionRecords(**kwargs)
454 if table:
455 table.pprint_all()
456 else:
457 print("No results. Try --help for more information.")