Coverage for python/lsst/daf/butler/cli/cmd/commands.py: 56%
305 statements
« prev ^ index » next coverage.py v7.11.0, created at 2025-11-20 01:50 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = ()
31from typing import Any
33import click
35from ... import script
36from ..opt import (
37 collection_argument,
38 collection_type_option,
39 collections_argument,
40 collections_option,
41 confirm_option,
42 dataset_type_option,
43 datasets_option,
44 destination_argument,
45 dimensions_argument,
46 directory_argument,
47 element_argument,
48 glob_argument,
49 limit_option,
50 offset_option,
51 options_file_option,
52 order_by_option,
53 query_datasets_options,
54 register_dataset_types_option,
55 repo_argument,
56 track_file_attrs_option,
57 transfer_dimensions_option,
58 transfer_option,
59 verbose_option,
60 where_option,
61)
62from ..utils import (
63 ButlerCommand,
64 MWOptionDecorator,
65 option_section,
66 printAstropyTables,
67 split_commas,
68 to_upper,
69 typeStrAcceptsMultiple,
70 unwrap,
71 where_help,
72)
74willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist."
75existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file."
78@click.command(cls=ButlerCommand, short_help="Add existing datasets to a tagged collection.")
79@repo_argument(required=True)
80@collection_argument(help="COLLECTION is the collection the datasets should be associated with.")
81@query_datasets_options(repo=False, showUri=False, useArguments=False, default_limit=0, use_order_by=False)
82@options_file_option()
83def associate(**kwargs: Any) -> None:
84 """Add existing datasets to a tagged collection; searches for datasets with
85 the options and adds them to the named COLLECTION.
86 """
87 script.associate(**kwargs)
90# The conversion from the import command name to the butler_import function
91# name for subcommand lookup is implemented in the cli/butler.py, in
92# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they
93# must be reflected in that location. If this becomes a common pattern a better
94# mechanism should be implemented.
95@click.command("import", cls=ButlerCommand)
96@repo_argument(required=True, help=willCreateRepoHelp)
97@directory_argument(required=True)
98@transfer_option()
99@click.option(
100 "--export-file",
101 help="Name for the file that contains database information associated with the exported "
102 "datasets. If this is not an absolute path, does not exist in the current working "
103 "directory, and --dir is provided, it is assumed to be in that directory. Defaults "
104 'to "export.yaml".',
105 type=str,
106)
107@click.option(
108 "--skip-dimensions",
109 "-s",
110 type=str,
111 multiple=True,
112 callback=split_commas,
113 metavar=typeStrAcceptsMultiple,
114 help="Dimensions that should be skipped during import",
115)
116@track_file_attrs_option()
117@options_file_option()
118def butler_import(*args: Any, **kwargs: Any) -> None:
119 """Import data into a butler repository."""
120 script.butlerImport(*args, **kwargs)
123@click.command(cls=ButlerCommand)
124@repo_argument(required=True, help=willCreateRepoHelp)
125@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).")
126@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.")
127@click.option(
128 "--standalone",
129 is_flag=True,
130 help="Include all defaults in the config file in the repo, "
131 "insulating the repo from changes in package defaults.",
132)
133@click.option(
134 "--override", is_flag=True, help="Allow values in the supplied config to override all repo settings."
135)
136@click.option(
137 "--outfile",
138 "-f",
139 default=None,
140 type=str,
141 help="Name of output file to receive repository "
142 "configuration. Default is to write butler.yaml into the specified repo.",
143)
144@options_file_option()
145def create(*args: Any, **kwargs: Any) -> None:
146 """Create an empty Gen3 Butler repository."""
147 script.createRepo(*args, **kwargs)
150@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand)
151@repo_argument(required=True, help=existingRepoHelp)
152@click.option(
153 "--subset",
154 "-s",
155 type=str,
156 help="Subset of a configuration to report. This can be any key in the hierarchy such as "
157 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.",
158)
159@click.option(
160 "--searchpath",
161 "-p",
162 type=str,
163 multiple=True,
164 callback=split_commas,
165 metavar=typeStrAcceptsMultiple,
166 help="Additional search paths to use for configuration overrides",
167)
168@click.option(
169 "--file",
170 "outfile",
171 type=click.File(mode="w"),
172 default="-",
173 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout by default.",
174)
175@options_file_option()
176def config_dump(*args: Any, **kwargs: Any) -> None:
177 """Dump either a subset or full Butler configuration to standard output."""
178 script.configDump(*args, **kwargs)
181@click.command(short_help="Validate the configuration files.", cls=ButlerCommand)
182@repo_argument(required=True, help=existingRepoHelp)
183@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.")
184@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True)
185@click.option(
186 "--ignore",
187 "-i",
188 type=str,
189 multiple=True,
190 callback=split_commas,
191 metavar=typeStrAcceptsMultiple,
192 help="DatasetType(s) to ignore for validation.",
193)
194@options_file_option()
195def config_validate(*args: Any, **kwargs: Any) -> None:
196 """Validate the configuration files for a Gen3 Butler repository."""
197 is_good = script.configValidate(*args, **kwargs)
198 if not is_good:
199 raise click.exceptions.Exit(1)
202pruneDatasets_wouldRemoveMsg = unwrap(
203 """The following datasets will be removed from any datastores in which
204 they are present:"""
205)
206pruneDatasets_wouldDisassociateMsg = unwrap(
207 """The following datasets will be disassociated from {collections}
208 if they are currently present in it (which is not checked):"""
209)
210pruneDatasets_wouldDisassociateAndRemoveMsg = unwrap(
211 """The following datasets will be disassociated from
212 {collections} if they are currently present in it (which is
213 not checked), and removed from any datastores in which they
214 are present."""
215)
216pruneDatasets_willRemoveMsg = "The following datasets will be removed:"
217pruneDatasets_askContinueMsg = "Continue?"
218pruneDatasets_didRemoveAforementioned = "The datasets were removed."
219pruneDatasets_didNotRemoveAforementioned = "Did not remove the datasets."
220pruneDatasets_didRemoveMsg = "Removed the following datasets:"
221pruneDatasets_noDatasetsFound = "Did not find any datasets."
222pruneDatasets_errPurgeAndDisassociate = unwrap(
223 """"--disassociate and --purge may not be used together: --disassociate purges from just the passed TAGged
224 collections, but --purge forces disassociation from all of them. """
225)
226pruneDatasets_errQuietWithDryRun = "Can not use --quiet and --dry-run together."
227pruneDatasets_errNoCollectionRestriction = unwrap(
228 """Must indicate collections from which to prune datasets by passing COLLECTION arguments (select all
229 collections by passing '*', or consider using 'butler prune-collections'), by using --purge to pass a run
230 collection, or by using --disassociate to select a tagged collection."""
231)
232pruneDatasets_errPruneOnNotRun = "Can not prune a collection that is not a RUN collection: {collection}"
233pruneDatasets_errNoOp = "No operation: one of --purge, --unstore, or --disassociate must be provided."
235disassociate_option = MWOptionDecorator(
236 "--disassociate",
237 "disassociate_tags",
238 help=unwrap(
239 """Disassociate pruned datasets from the given tagged collections. May not be used with
240 --purge."""
241 ),
242 multiple=True,
243 callback=split_commas,
244 metavar="TAG",
245)
248purge_option = MWOptionDecorator(
249 "--purge",
250 "purge_run",
251 help=unwrap(
252 """Completely remove the dataset from the given RUN in the Registry. May not be used with
253 --disassociate. Implies --unstore. Note, this may remove provenance information from
254 datasets other than those provided, and should be used with extreme care.
255 RUN has to be provided for backward compatibility, but is used only if COLLECTIONS is
256 not provided. Otherwise, datasets will be removed from
257 any RUN-type collections in COLLECTIONS."""
258 ),
259 metavar="RUN",
260)
263find_all_option = MWOptionDecorator(
264 "--find-all",
265 is_flag=True,
266 help=unwrap(
267 """Purge the dataset results from all of the collections in which a dataset of that dataset
268 type + data id combination appear. (By default only the first found dataset type + data id is
269 purged, according to the order of COLLECTIONS passed in)."""
270 ),
271)
274unstore_option = MWOptionDecorator(
275 "--unstore",
276 is_flag=True,
277 help=unwrap(
278 """Remove these datasets from all datastores configured with this data repository. If
279 --disassociate and --purge are not used then --unstore will be used by default. Note that
280 --unstore will make it impossible to retrieve these datasets even via other collections.
281 Datasets that are already not stored are ignored by this option."""
282 ),
283)
286dry_run_option = MWOptionDecorator(
287 "--dry-run",
288 is_flag=True,
289 help=unwrap(
290 """Display the datasets that would be removed but do not remove them.
292 Note that a dataset can be in collections other than its RUN-type collection, and removing it
293 will remove it from all of them, even though the only one this will show is its RUN
294 collection."""
295 ),
296)
299quiet_option = MWOptionDecorator(
300 "--quiet",
301 is_flag=True,
302 help=unwrap("""Makes output quiet. Implies --no-confirm. Requires --dry-run not be passed."""),
303)
306@click.command(cls=ButlerCommand, short_help="Remove datasets.")
307@repo_argument(required=True)
308@collections_argument(
309 help=unwrap(
310 """COLLECTIONS is or more expressions that identify the collections to
311 search for datasets. Glob-style expressions may be used but only if the
312 --find-all flag is also passed."""
313 )
314)
315@option_section("Query Datasets Options:")
316@datasets_option(
317 help="One or more glob-style expressions that identify the dataset types to be pruned.",
318 multiple=True,
319 callback=split_commas,
320)
321@find_all_option()
322@where_option(help=where_help)
323@option_section("Prune Options:")
324@disassociate_option()
325@purge_option()
326@unstore_option()
327@option_section("Execution Options:")
328@dry_run_option()
329@confirm_option()
330@quiet_option()
331@option_section("Other Options:")
332@options_file_option()
333def prune_datasets(**kwargs: Any) -> None:
334 """Query for and remove one or more datasets from a collection and/or
335 storage.
336 """
337 quiet = kwargs.pop("quiet", False)
338 if quiet:
339 if kwargs["dry_run"]:
340 raise click.ClickException(message=pruneDatasets_errQuietWithDryRun)
341 kwargs["confirm"] = False
343 result = script.pruneDatasets(**kwargs)
345 if result.errPurgeAndDisassociate:
346 raise click.ClickException(message=pruneDatasets_errPurgeAndDisassociate)
347 if result.errNoCollectionRestriction:
348 raise click.ClickException(message=pruneDatasets_errNoCollectionRestriction)
349 if result.errPruneOnNotRun:
350 raise click.ClickException(message=pruneDatasets_errPruneOnNotRun.format(**result.errDict))
351 if result.errNoOp:
352 raise click.ClickException(message=pruneDatasets_errNoOp)
353 if result.dryRun:
354 assert result.action is not None, "Dry run results have not been set up properly."
355 if result.action["disassociate"] and result.action["unstore"]:
356 msg = pruneDatasets_wouldDisassociateAndRemoveMsg
357 elif result.action["disassociate"]:
358 msg = pruneDatasets_wouldDisassociateMsg
359 else:
360 msg = pruneDatasets_wouldRemoveMsg
361 print(msg.format(**result.action))
362 printAstropyTables(result.tables)
363 return
364 if result.confirm:
365 if not result.tables:
366 print(pruneDatasets_noDatasetsFound)
367 return
368 print(pruneDatasets_willRemoveMsg)
369 printAstropyTables(result.tables)
370 doContinue = click.confirm(text=pruneDatasets_askContinueMsg, default=False)
371 if doContinue:
372 if result.onConfirmation:
373 result.onConfirmation()
374 print(pruneDatasets_didRemoveAforementioned)
375 else:
376 print(pruneDatasets_didNotRemoveAforementioned)
377 return
378 if result.finished:
379 if not quiet:
380 print(pruneDatasets_didRemoveMsg)
381 printAstropyTables(result.tables)
382 return
385@click.command(short_help="Search for collections.", cls=ButlerCommand)
386@repo_argument(required=True)
387@glob_argument(
388 help="GLOB is one or more glob-style expressions that fully or partially identify the "
389 "collections to return."
390)
391@collection_type_option()
392@click.option(
393 "--chains",
394 default="TREE",
395 help="""Affects how results are presented:
397 TABLE lists each collection in table form, with columns for collection
398 name and type, and a column that lists children of CHAINED collections
399 (if any CHAINED collections are found).
401 INVERSE-TABLE is like TABLE but instead of a column listing CHAINED
402 collection children, it lists the parents of the collection if it is
403 contained in any CHAINED collections.
405 TREE recursively lists children below each CHAINED collection in tree
406 form.
408 INVERSE-TREE recursively lists parent collections below each collection
409 in tree form.
411 FLATTEN lists all collections, including children of CHAINED
412 collections, in one list.
414 NO-CHILDREN lists all collections in one list. CHAINED collections are
415 included, but they are not expanded to include their children.
417 [default: TREE]""",
418 # above, the default value is included, instead of using show_default, so
419 # that the default is printed on its own line instead of coming right after
420 # the FLATTEN text.
421 callback=to_upper,
422 type=click.Choice(
423 choices=("TABLE", "INVERSE-TABLE", "TREE", "INVERSE-TREE", "FLATTEN", "NO-CHILDREN"),
424 case_sensitive=False,
425 ),
426)
427@click.option(
428 "-t",
429 "--show-dataset-types",
430 is_flag=True,
431 help="Also show the dataset types registered within each collection.",
432)
433@click.option(
434 "--exclude-dataset-types",
435 type=click.STRING,
436 multiple=True,
437 default=["*_config,*_log,*_metadata,packages"],
438 callback=split_commas,
439 show_default=True,
440 help="Dataset types (comma-separated) to exclude. Only valid with --show-dataset-types.",
441)
442@options_file_option()
443def query_collections(*args: Any, **kwargs: Any) -> None:
444 """Get the collections whose names match an expression."""
445 table = script.queryCollections(*args, **kwargs)
446 # The unit test that mocks script.queryCollections does not return a table
447 # so we need the following `if`.
448 if table:
449 # When chains==TREE, the children of chained datasets are indented
450 # relative to their parents. For this to work properly the table must
451 # be left-aligned.
452 table.pprint_all(align="<")
455@click.command(cls=ButlerCommand)
456@repo_argument(required=True)
457@glob_argument(
458 help="GLOB is one or more glob-style expressions that fully or partially identify the "
459 "dataset types to return."
460)
461@verbose_option(help="Include dataset type name, dimensions, and storage class in output.")
462@collections_option(
463 help="Constrain the resulting dataset types by these collections. "
464 "This constraint does not say that a dataset of this type is definitely present, "
465 "solely that one may have been present at some point."
466)
467@options_file_option()
468def query_dataset_types(*args: Any, **kwargs: Any) -> None:
469 """Get the dataset types in a repository."""
470 table = script.queryDatasetTypes(*args, **kwargs)
471 if table:
472 table.pprint_all(align="<")
473 else:
474 print("No results. Try --help for more information.")
477@click.command(cls=ButlerCommand)
478@repo_argument(required=True)
479@click.argument("dataset-type-name", nargs=-1)
480def remove_dataset_type(*args: Any, **kwargs: Any) -> None:
481 """Remove the dataset type definitions from a repository."""
482 script.removeDatasetType(*args, **kwargs)
485@click.command(cls=ButlerCommand)
486@query_datasets_options()
487@options_file_option()
488def query_datasets(**kwargs: Any) -> None:
489 """List the datasets in a repository."""
490 for table in script.QueryDatasets(**kwargs).getTables():
491 print("")
492 table.pprint_all()
493 print("")
496@click.command(cls=ButlerCommand)
497@repo_argument(required=True)
498@click.argument("input-collection")
499@click.argument("output-collection")
500@click.argument("dataset-type-name")
501@click.option(
502 "--begin-date",
503 type=str,
504 default=None,
505 help=unwrap(
506 """ISO-8601 datetime (TAI) of the beginning of the validity range for the
507 certified calibrations."""
508 ),
509)
510@click.option(
511 "--end-date",
512 type=str,
513 default=None,
514 help=unwrap(
515 """ISO-8601 datetime (TAI) of the end of the validity range for the
516 certified calibrations."""
517 ),
518)
519@click.option(
520 "--search-all-inputs",
521 is_flag=True,
522 default=False,
523 help=unwrap(
524 """Search all children of the inputCollection if it is a CHAINED collection,
525 instead of just the most recent one."""
526 ),
527)
528@options_file_option()
529def certify_calibrations(*args: Any, **kwargs: Any) -> None:
530 """Certify calibrations in a repository."""
531 script.certifyCalibrations(*args, **kwargs)
534@click.command(cls=ButlerCommand)
535@repo_argument(required=True)
536@dimensions_argument(
537 help=unwrap(
538 """DIMENSIONS are the keys of the data IDs to yield, such as exposure,
539 instrument, or tract. Will be expanded to include any dependencies."""
540 )
541)
542@collections_option(help=collections_option.help + " May only be used with --datasets.")
543@datasets_option(
544 help=unwrap(
545 """An expression that fully or partially identifies dataset types that should
546 constrain the yielded data IDs. For example, including "raw" here would
547 constrain the yielded "instrument", "exposure", "detector", and
548 "physical_filter" values to only those for which at least one "raw" dataset
549 exists in "collections". Requires --collections."""
550 )
551)
552@where_option(help=where_help)
553@order_by_option()
554@limit_option()
555@offset_option()
556@options_file_option()
557def query_data_ids(**kwargs: Any) -> None:
558 """List the data IDs in a repository."""
559 table, reason = script.queryDataIds(**kwargs)
560 if table:
561 table.pprint_all()
562 else:
563 if reason:
564 print(reason)
565 if not kwargs.get("dimensions") and not kwargs.get("datasets"):
566 print("No results. Try requesting some dimensions or datasets, see --help for more information.")
567 else:
568 print("No results. Try --help for more information.")
571@click.command(cls=ButlerCommand)
572@repo_argument(required=True)
573@element_argument(required=True)
574@datasets_option(
575 help=unwrap(
576 """An expression that fully or partially identifies dataset types that should
577 constrain the yielded records. May only be used with
578 --collections."""
579 )
580)
581@collections_option(help=collections_option.help + " May only be used with --datasets.")
582@where_option(help=where_help)
583@order_by_option()
584@limit_option()
585@offset_option()
586@click.option(
587 "--no-check",
588 is_flag=True,
589 default=None,
590 help=unwrap(
591 """Don't check the query before execution. By default the query is checked before it
592 executed, this may reject some valid queries that resemble common mistakes."""
593 ),
594)
595@options_file_option()
596def query_dimension_records(**kwargs: Any) -> None:
597 """Query for dimension information."""
598 if kwargs.pop("no_check") is not None:
599 click.echo("WARNING: --no-check option has no effect and will be removed after v28.")
600 table = script.queryDimensionRecords(**kwargs)
601 if table:
602 table.pprint_all()
603 else:
604 print("No results. Try --help for more information.")
607@click.command(cls=ButlerCommand)
608@repo_argument(required=True)
609@query_datasets_options(showUri=False, useArguments=False, repo=False)
610@destination_argument(help="Destination URI of folder to receive file artifacts.")
611@transfer_option()
612@verbose_option(help="Report destination location of all transferred artifacts.")
613@click.option(
614 "--preserve-path/--no-preserve-path",
615 is_flag=True,
616 default=True,
617 help="Preserve the datastore path to the artifact at the destination.",
618)
619@click.option(
620 "--clobber/--no-clobber",
621 is_flag=True,
622 default=False,
623 help="If clobber, overwrite files if they exist locally.",
624)
625@click.option(
626 "--zip/--no-zip",
627 is_flag=True,
628 default=False,
629 help="Retrieve artifacts and place in a Zip file.",
630)
631@options_file_option()
632def retrieve_artifacts(**kwargs: Any) -> None:
633 """Retrieve file artifacts associated with datasets in a repository."""
634 verbose = kwargs.pop("verbose")
635 transferred = script.retrieveArtifacts(**kwargs)
636 if not transferred:
637 print("No datasets matched query.")
638 elif kwargs["zip"]:
639 print(f"Zip file written to {transferred[0]}")
640 else:
641 if verbose:
642 print(f"Transferred the following to {kwargs['destination']}:")
643 for uri in transferred:
644 print(uri)
645 print()
646 print(f"Number of artifacts retrieved into destination {kwargs['destination']}: {len(transferred)}")
649@click.command(cls=ButlerCommand)
650@click.argument("source", required=True)
651@click.argument("dest", required=True)
652@query_datasets_options(showUri=False, useArguments=False, repo=False)
653@transfer_option()
654@register_dataset_types_option()
655@transfer_dimensions_option()
656@click.option(
657 "--dry-run/--no-dry-run", default=False, help="Enable dry run mode and do not transfer any datasets."
658)
659@options_file_option()
660def transfer_datasets(**kwargs: Any) -> None:
661 """Transfer datasets from a source butler to a destination butler.
663 SOURCE is a URI to the Butler repository containing the RUN dataset.
665 DEST is a URI to the Butler repository that will receive copies of the
666 datasets.
667 """
668 number = script.transferDatasets(**kwargs)
669 print(f"Number of datasets transferred: {number}")
672@click.command(cls=ButlerCommand)
673@repo_argument(required=True)
674@click.argument("parent", required=True, nargs=1)
675@click.argument("children", required=False, nargs=-1, callback=split_commas)
676@click.option(
677 "--doc",
678 default="",
679 help="Documentation string associated with this collection. "
680 "Only relevant if the collection is newly created.",
681)
682@click.option(
683 "--flatten/--no-flatten",
684 default=False,
685 help="If `True` recursively flatten out any nested chained collections in children first.",
686)
687@click.option(
688 "--mode",
689 type=click.Choice(["redefine", "extend", "remove", "prepend", "pop"]),
690 default="redefine",
691 help="Update mode: "
692 "'redefine': Create new chain or redefine existing chain with the supplied CHILDREN. "
693 "'remove': Modify existing chain to remove the supplied CHILDREN. "
694 "'pop': Pop a numbered element off the chain. Defaults to popping "
695 "the first element (0). ``children`` must be integers if given. "
696 "'prepend': Modify existing chain to prepend the supplied CHILDREN to the front. "
697 "'extend': Modify existing chain to extend it with the supplied CHILDREN.",
698)
699def collection_chain(**kwargs: Any) -> None:
700 """Define a collection chain.
702 PARENT is the name of the chained collection to create or modify. If the
703 collection already exists the chain associated with it will be updated.
705 CHILDREN are the collections to be used to modify the chain. The supplied
706 values will be split on comma. The exact usage depends on the MODE option.
707 For example,
709 $ butler collection-chain REPO PARENT child1,child2 child3
711 will result in three children being included in the chain.
713 When the MODE is 'pop' the CHILDREN should be integer indices indicating
714 collections to be removed from the current chain.
715 MODE 'pop' can take negative integers to indicate removal relative to the
716 end of the chain, but when doing that '--' must be given to indicate the
717 end of the options specification.
719 $ butler collection-chain REPO --mode=pop PARENT -- -1
721 Will remove the final collection from the chain.
722 """
723 chain = script.collectionChain(**kwargs)
724 print(f"[{', '.join(chain)}]")
727@click.command(cls=ButlerCommand)
728@repo_argument(required=True)
729@click.argument("dataset_type", required=True)
730@click.argument("run", required=True)
731@click.argument("table_file", required=True)
732@click.option(
733 "--formatter",
734 type=str,
735 help="Fully-qualified python class to use as the Formatter. If not specified the formatter"
736 " will be determined from the dataset type and datastore configuration.",
737)
738@click.option(
739 "--id-generation-mode",
740 default="UNIQUE",
741 help="Mode to use for generating dataset IDs. The default creates a unique ID. Other options"
742 " are: 'DATAID_TYPE' for creating a reproducible ID from the dataID and dataset type;"
743 " 'DATAID_TYPE_RUN' for creating a reproducible ID from the dataID, dataset type and run."
744 " The latter is usually used for 'raw'-type data that will be ingested in multiple."
745 " repositories.",
746 callback=to_upper,
747 type=click.Choice(("UNIQUE", "DATAID_TYPE", "DATAID_TYPE_RUN"), case_sensitive=False),
748)
749@click.option(
750 "--data-id",
751 type=str,
752 multiple=True,
753 callback=split_commas,
754 help="Keyword=value string with an additional dataId value that is fixed for all ingested"
755 " files. This can be used to simplify the table file by removing repeated entries that are"
756 " fixed for all files to be ingested. Multiple key/values can be given either by using"
757 " comma separation or multiple command line options.",
758)
759@click.option(
760 "--prefix",
761 type=str,
762 help="For relative paths in the table file, specify a prefix to use. The default is to"
763 " use the current working directory.",
764)
765@track_file_attrs_option()
766@transfer_option()
767def ingest_files(**kwargs: Any) -> None:
768 """Ingest files from table file.
770 DATASET_TYPE is the name of the dataset type to be associated with these
771 files. This dataset type must already exist and will not be created by
772 this command. There can only be one dataset type per invocation of this
773 command.
775 RUN is the run to use for the file ingest.
777 TABLE_FILE refers to a file that can be read by astropy.table with
778 columns of:
780 file URI, dimension1, dimension2, ..., dimensionN
782 where the first column is the URI to the file to be ingested and the
783 remaining columns define the dataId to associate with that file.
784 The column names should match the dimensions for the specified dataset
785 type. Relative file URI by default is assumed to be relative to the
786 current working directory but can be overridden using the ``--prefix``
787 option.
789 This command does not create dimension records and so any records must
790 be created by other means. This command should not be used to ingest
791 raw camera exposures.
792 """
793 script.ingest_files(**kwargs)
796@click.command(cls=ButlerCommand)
797@repo_argument(required=True)
798@click.argument("dataset_type", required=True)
799@click.argument("storage_class", required=True)
800@click.argument("dimensions", required=False, nargs=-1)
801@click.option(
802 "--is-calibration/--no-is-calibration",
803 is_flag=True,
804 default=False,
805 help="Indicate that this dataset type can be part of a calibration collection.",
806)
807def register_dataset_type(**kwargs: Any) -> None:
808 """Register a new dataset type with this butler repository.
810 DATASET_TYPE is the name of the dataset type.
812 STORAGE_CLASS is the name of the StorageClass to be associated with
813 this dataset type.
815 DIMENSIONS is a list of all the dimensions relevant to this
816 dataset type. It can be an empty list.
818 A component dataset type (such as "something.component") is not a
819 real dataset type and so can not be defined by this command. They are
820 automatically derived from the composite dataset type when a composite
821 storage class is specified.
822 """
823 inserted = script.register_dataset_type(**kwargs)
824 if inserted:
825 print("Dataset type successfully registered.")
826 else:
827 print("Dataset type already existed in identical form.")
830@click.command(cls=ButlerCommand)
831@repo_argument(required=True)
832@directory_argument(required=True, help="DIRECTORY is the folder to receive the exported calibrations.")
833@collections_argument(help="COLLECTIONS are the collection to export calibrations from.")
834@dataset_type_option(help="Specific DatasetType(s) to export.", multiple=True)
835@transfer_option()
836def export_calibs(*args: Any, **kwargs: Any) -> None:
837 """Export calibrations from the butler for import elsewhere."""
838 table = script.exportCalibs(*args, **kwargs)
839 if table:
840 table.pprint_all(align="<")
843@click.command(cls=ButlerCommand)
844@repo_argument(required=True)
845@click.argument("zip", required=True)
846@transfer_option()
847@transfer_dimensions_option(
848 default=False, help="Attempt to register missing dimension records during ingest."
849)
850@click.option(
851 "--dry-run/--no-dry-run", default=False, help="Enable dry run mode and do not ingest any datasets."
852)
853def ingest_zip(**kwargs: Any) -> None:
854 """Ingest a Zip file created by retrieve-artifacts.
856 ZIP is the URI to the Zip file that should be ingested.
858 This command does not create dimension records and so any records must
859 be created by other means.
860 """
861 script.ingest_zip(**kwargs)