Coverage for python / lsst / daf / butler / cli / cmd / commands.py: 56%
308 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-24 08:17 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = ()
31from typing import Any
33import click
35from ... import script
36from ..._butler import Butler
37from ..opt import (
38 collection_argument,
39 collection_type_option,
40 collections_argument,
41 collections_option,
42 confirm_option,
43 dataset_type_option,
44 datasets_option,
45 destination_argument,
46 dimensions_argument,
47 directory_argument,
48 element_argument,
49 glob_argument,
50 limit_option,
51 offset_option,
52 options_file_option,
53 order_by_option,
54 query_datasets_options,
55 register_dataset_types_option,
56 repo_argument,
57 track_file_attrs_option,
58 transfer_dimensions_option,
59 transfer_option,
60 verbose_option,
61 where_option,
62)
63from ..utils import (
64 ButlerCommand,
65 MWOptionDecorator,
66 option_section,
67 printAstropyTables,
68 split_commas,
69 to_upper,
70 typeStrAcceptsMultiple,
71 unwrap,
72 where_help,
73)
75willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist."
76existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file."
79@click.command(cls=ButlerCommand, short_help="Add existing datasets to a tagged collection.")
80@repo_argument(required=True)
81@collection_argument(help="COLLECTION is the collection the datasets should be associated with.")
82@query_datasets_options(repo=False, showUri=False, useArguments=False, default_limit=0, use_order_by=False)
83@options_file_option()
84def associate(**kwargs: Any) -> None:
85 """Add existing datasets to a tagged collection; searches for datasets with
86 the options and adds them to the named COLLECTION.
87 """
88 script.associate(**kwargs)
91# The conversion from the import command name to the butler_import function
92# name for subcommand lookup is implemented in the cli/butler.py, in
93# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they
94# must be reflected in that location. If this becomes a common pattern a better
95# mechanism should be implemented.
96@click.command("import", cls=ButlerCommand)
97@repo_argument(required=True, help=willCreateRepoHelp)
98@directory_argument(required=True)
99@transfer_option()
100@click.option(
101 "--export-file",
102 help="Name for the file that contains database information associated with the exported "
103 "datasets. If this is not an absolute path, does not exist in the current working "
104 "directory, and --dir is provided, it is assumed to be in that directory. Defaults "
105 'to "export.yaml".',
106 type=str,
107)
108@click.option(
109 "--skip-dimensions",
110 "-s",
111 type=str,
112 multiple=True,
113 callback=split_commas,
114 metavar=typeStrAcceptsMultiple,
115 help="Dimensions that should be skipped during import",
116)
117@track_file_attrs_option()
118@options_file_option()
119def butler_import(*args: Any, **kwargs: Any) -> None:
120 """Import data into a butler repository."""
121 script.butlerImport(*args, **kwargs)
124@click.command(cls=ButlerCommand)
125@repo_argument(required=True, help=willCreateRepoHelp)
126@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).")
127@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.")
128@click.option(
129 "--standalone",
130 is_flag=True,
131 help="Include all defaults in the config file in the repo, "
132 "insulating the repo from changes in package defaults.",
133)
134@click.option(
135 "--override", is_flag=True, help="Allow values in the supplied config to override all repo settings."
136)
137@click.option(
138 "--outfile",
139 "-f",
140 default=None,
141 type=str,
142 help="Name of output file to receive repository "
143 "configuration. Default is to write butler.yaml into the specified repo.",
144)
145@options_file_option()
146def create(*args: Any, **kwargs: Any) -> None:
147 """Create an empty Gen3 Butler repository."""
148 script.createRepo(*args, **kwargs)
151@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand)
152@repo_argument(required=True, help=existingRepoHelp)
153@click.option(
154 "--subset",
155 "-s",
156 type=str,
157 help="Subset of a configuration to report. This can be any key in the hierarchy such as "
158 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.",
159)
160@click.option(
161 "--searchpath",
162 "-p",
163 type=str,
164 multiple=True,
165 callback=split_commas,
166 metavar=typeStrAcceptsMultiple,
167 help="Additional search paths to use for configuration overrides",
168)
169@click.option(
170 "--file",
171 "outfile",
172 type=click.File(mode="w"),
173 default="-",
174 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout by default.",
175)
176@options_file_option()
177def config_dump(*args: Any, **kwargs: Any) -> None:
178 """Dump either a subset or full Butler configuration to standard output."""
179 script.configDump(*args, **kwargs)
182@click.command(short_help="Validate the configuration files.", cls=ButlerCommand)
183@repo_argument(required=True, help=existingRepoHelp)
184@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.")
185@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True)
186@click.option(
187 "--ignore",
188 "-i",
189 type=str,
190 multiple=True,
191 callback=split_commas,
192 metavar=typeStrAcceptsMultiple,
193 help="DatasetType(s) to ignore for validation.",
194)
195@options_file_option()
196def config_validate(*args: Any, **kwargs: Any) -> None:
197 """Validate the configuration files for a Gen3 Butler repository."""
198 is_good = script.configValidate(*args, **kwargs)
199 if not is_good:
200 raise click.exceptions.Exit(1)
203pruneDatasets_wouldRemoveMsg = unwrap(
204 """The following datasets will be removed from any datastores in which
205 they are present:"""
206)
207pruneDatasets_wouldDisassociateMsg = unwrap(
208 """The following datasets will be disassociated from {collections}
209 if they are currently present in it (which is not checked):"""
210)
211pruneDatasets_wouldDisassociateAndRemoveMsg = unwrap(
212 """The following datasets will be disassociated from
213 {collections} if they are currently present in it (which is
214 not checked), and removed from any datastores in which they
215 are present."""
216)
217pruneDatasets_willRemoveMsg = "The following datasets will be removed:"
218pruneDatasets_askContinueMsg = "Continue?"
219pruneDatasets_didRemoveAforementioned = "The datasets were removed."
220pruneDatasets_didNotRemoveAforementioned = "Did not remove the datasets."
221pruneDatasets_didRemoveMsg = "Removed the following datasets:"
222pruneDatasets_noDatasetsFound = "Did not find any datasets."
223pruneDatasets_errPurgeAndDisassociate = unwrap(
224 """"--disassociate and --purge may not be used together: --disassociate purges from just the passed TAGged
225 collections, but --purge forces disassociation from all of them. """
226)
227pruneDatasets_errQuietWithDryRun = "Can not use --quiet and --dry-run together."
228pruneDatasets_errNoCollectionRestriction = unwrap(
229 """Must indicate collections from which to prune datasets by passing COLLECTION arguments (select all
230 collections by passing '*', or consider using 'butler prune-collections'), by using --purge to pass a run
231 collection, or by using --disassociate to select a tagged collection."""
232)
233pruneDatasets_errPruneOnNotRun = "Can not prune a collection that is not a RUN collection: {collection}"
234pruneDatasets_errNoOp = "No operation: one of --purge, --unstore, or --disassociate must be provided."
236disassociate_option = MWOptionDecorator(
237 "--disassociate",
238 "disassociate_tags",
239 help=unwrap(
240 """Disassociate pruned datasets from the given tagged collections. May not be used with
241 --purge."""
242 ),
243 multiple=True,
244 callback=split_commas,
245 metavar="TAG",
246)
249purge_option = MWOptionDecorator(
250 "--purge",
251 "purge_run",
252 help=unwrap(
253 """Completely remove the dataset from the given RUN in the Registry. May not be used with
254 --disassociate. Implies --unstore. Note, this may remove provenance information from
255 datasets other than those provided, and should be used with extreme care.
256 RUN has to be provided for backward compatibility, but is used only if COLLECTIONS is
257 not provided. Otherwise, datasets will be removed from
258 any RUN-type collections in COLLECTIONS."""
259 ),
260 metavar="RUN",
261)
264find_all_option = MWOptionDecorator(
265 "--find-all",
266 is_flag=True,
267 help=unwrap(
268 """Purge the dataset results from all of the collections in which a dataset of that dataset
269 type + data id combination appear. (By default only the first found dataset type + data id is
270 purged, according to the order of COLLECTIONS passed in)."""
271 ),
272)
275unstore_option = MWOptionDecorator(
276 "--unstore",
277 is_flag=True,
278 help=unwrap(
279 """Remove these datasets from all datastores configured with this data repository. If
280 --disassociate and --purge are not used then --unstore will be used by default. Note that
281 --unstore will make it impossible to retrieve these datasets even via other collections.
282 Datasets that are already not stored are ignored by this option."""
283 ),
284)
287dry_run_option = MWOptionDecorator(
288 "--dry-run",
289 is_flag=True,
290 help=unwrap(
291 """Display the datasets that would be removed but do not remove them.
293 Note that a dataset can be in collections other than its RUN-type collection, and removing it
294 will remove it from all of them, even though the only one this will show is its RUN
295 collection."""
296 ),
297)
300quiet_option = MWOptionDecorator(
301 "--quiet",
302 is_flag=True,
303 help=unwrap("""Makes output quiet. Implies --no-confirm. Requires --dry-run not be passed."""),
304)
307@click.command(cls=ButlerCommand, short_help="Remove datasets.")
308@repo_argument(required=True)
309@collections_argument(
310 help=unwrap(
311 """COLLECTIONS is or more expressions that identify the collections to
312 search for datasets. Glob-style expressions may be used but only if the
313 --find-all flag is also passed."""
314 )
315)
316@option_section("Query Datasets Options:")
317@datasets_option(
318 help="One or more glob-style expressions that identify the dataset types to be pruned.",
319 multiple=True,
320 callback=split_commas,
321)
322@find_all_option()
323@where_option(help=where_help)
324@option_section("Prune Options:")
325@disassociate_option()
326@purge_option()
327@unstore_option()
328@option_section("Execution Options:")
329@dry_run_option()
330@confirm_option()
331@quiet_option()
332@option_section("Other Options:")
333@options_file_option()
334def prune_datasets(**kwargs: Any) -> None:
335 """Query for and remove one or more datasets from a collection and/or
336 storage.
337 """
338 quiet = kwargs.pop("quiet", False)
339 if quiet:
340 if kwargs["dry_run"]:
341 raise click.ClickException(message=pruneDatasets_errQuietWithDryRun)
342 kwargs["confirm"] = False
344 result = script.pruneDatasets(**kwargs)
346 if result.errPurgeAndDisassociate:
347 raise click.ClickException(message=pruneDatasets_errPurgeAndDisassociate)
348 if result.errNoCollectionRestriction:
349 raise click.ClickException(message=pruneDatasets_errNoCollectionRestriction)
350 if result.errPruneOnNotRun:
351 raise click.ClickException(message=pruneDatasets_errPruneOnNotRun.format(**result.errDict))
352 if result.errNoOp:
353 raise click.ClickException(message=pruneDatasets_errNoOp)
354 if result.dryRun:
355 assert result.action is not None, "Dry run results have not been set up properly."
356 if result.action["disassociate"] and result.action["unstore"]:
357 msg = pruneDatasets_wouldDisassociateAndRemoveMsg
358 elif result.action["disassociate"]:
359 msg = pruneDatasets_wouldDisassociateMsg
360 else:
361 msg = pruneDatasets_wouldRemoveMsg
362 print(msg.format(**result.action))
363 printAstropyTables(result.tables)
364 return
365 if result.confirm:
366 if not result.tables:
367 print(pruneDatasets_noDatasetsFound)
368 return
369 print(pruneDatasets_willRemoveMsg)
370 printAstropyTables(result.tables)
371 doContinue = click.confirm(text=pruneDatasets_askContinueMsg, default=False)
372 if doContinue:
373 if result.onConfirmation:
374 result.onConfirmation()
375 print(pruneDatasets_didRemoveAforementioned)
376 else:
377 print(pruneDatasets_didNotRemoveAforementioned)
378 return
379 if result.finished:
380 if not quiet:
381 print(pruneDatasets_didRemoveMsg)
382 printAstropyTables(result.tables)
383 return
386@click.command(short_help="Search for collections.", cls=ButlerCommand)
387@repo_argument(required=True)
388@glob_argument(
389 help="GLOB is one or more glob-style expressions that fully or partially identify the "
390 "collections to return."
391)
392@collection_type_option()
393@click.option(
394 "--chains",
395 default="TREE",
396 help="""Affects how results are presented:
398 TABLE lists each collection in table form, with columns for collection
399 name and type, and a column that lists children of CHAINED collections
400 (if any CHAINED collections are found).
402 INVERSE-TABLE is like TABLE but instead of a column listing CHAINED
403 collection children, it lists the parents of the collection if it is
404 contained in any CHAINED collections.
406 TREE recursively lists children below each CHAINED collection in tree
407 form.
409 INVERSE-TREE recursively lists parent collections below each collection
410 in tree form.
412 FLATTEN lists all collections, including children of CHAINED
413 collections, in one list.
415 NO-CHILDREN lists all collections in one list. CHAINED collections are
416 included, but they are not expanded to include their children.
418 [default: TREE]""",
419 # above, the default value is included, instead of using show_default, so
420 # that the default is printed on its own line instead of coming right after
421 # the FLATTEN text.
422 callback=to_upper,
423 type=click.Choice(
424 choices=("TABLE", "INVERSE-TABLE", "TREE", "INVERSE-TREE", "FLATTEN", "NO-CHILDREN"),
425 case_sensitive=False,
426 ),
427)
428@click.option(
429 "-t",
430 "--show-dataset-types",
431 is_flag=True,
432 help="Also show the dataset types registered within each collection.",
433)
434@click.option(
435 "--exclude-dataset-types",
436 type=click.STRING,
437 multiple=True,
438 default=["*_config,*_log,*_metadata,packages"],
439 callback=split_commas,
440 show_default=True,
441 help="Dataset types (comma-separated) to exclude. Only valid with --show-dataset-types.",
442)
443@options_file_option()
444def query_collections(*args: Any, **kwargs: Any) -> None:
445 """Get the collections whose names match an expression."""
446 table = script.queryCollections(*args, **kwargs)
447 # The unit test that mocks script.queryCollections does not return a table
448 # so we need the following `if`.
449 if table:
450 # When chains==TREE, the children of chained datasets are indented
451 # relative to their parents. For this to work properly the table must
452 # be left-aligned.
453 table.pprint_all(align="<")
456@click.command(cls=ButlerCommand)
457@repo_argument(required=True)
458@glob_argument(
459 help="GLOB is one or more glob-style expressions that fully or partially identify the "
460 "dataset types to return."
461)
462@verbose_option(help="Include dataset type name, dimensions, and storage class in output.")
463@collections_option(
464 help="Constrain the resulting dataset types by these collections. "
465 "This constraint does not say that a dataset of this type is definitely present, "
466 "solely that one may have been present at some point."
467)
468@options_file_option()
469def query_dataset_types(*args: Any, **kwargs: Any) -> None:
470 """Get the dataset types in a repository."""
471 table = script.queryDatasetTypes(*args, **kwargs)
472 if table:
473 table.pprint_all(align="<")
474 else:
475 print("No results. Try --help for more information.")
478@click.command(cls=ButlerCommand)
479@repo_argument(required=True)
480@click.argument("dataset-type-name", nargs=-1)
481def remove_dataset_type(*args: Any, **kwargs: Any) -> None:
482 """Remove the dataset type definitions from a repository."""
483 script.removeDatasetType(*args, **kwargs)
486@click.command(cls=ButlerCommand)
487@query_datasets_options()
488@options_file_option()
489def query_datasets(**kwargs: Any) -> None:
490 """List the datasets in a repository."""
491 repo = kwargs.pop("repo")
492 with Butler.from_config(repo, writeable=False) as butler:
493 for table in script.QueryDatasets(butler=butler, **kwargs).getTables():
494 print("")
495 table.pprint_all()
496 print("")
499@click.command(cls=ButlerCommand)
500@repo_argument(required=True)
501@click.argument("input-collection")
502@click.argument("output-collection")
503@click.argument("dataset-type-name")
504@click.option(
505 "--begin-date",
506 type=str,
507 default=None,
508 help=unwrap(
509 """ISO-8601 datetime (TAI) of the beginning of the validity range for the
510 certified calibrations."""
511 ),
512)
513@click.option(
514 "--end-date",
515 type=str,
516 default=None,
517 help=unwrap(
518 """ISO-8601 datetime (TAI) of the end of the validity range for the
519 certified calibrations."""
520 ),
521)
522@click.option(
523 "--search-all-inputs",
524 is_flag=True,
525 default=False,
526 help=unwrap(
527 """Search all children of the inputCollection if it is a CHAINED collection,
528 instead of just the most recent one."""
529 ),
530)
531@options_file_option()
532def certify_calibrations(*args: Any, **kwargs: Any) -> None:
533 """Certify calibrations in a repository."""
534 script.certifyCalibrations(*args, **kwargs)
537@click.command(cls=ButlerCommand)
538@repo_argument(required=True)
539@dimensions_argument(
540 help=unwrap(
541 """DIMENSIONS are the keys of the data IDs to yield, such as exposure,
542 instrument, or tract. Will be expanded to include any dependencies."""
543 )
544)
545@collections_option(help=collections_option.help + " May only be used with --datasets.")
546@datasets_option(
547 help=unwrap(
548 """An expression that fully or partially identifies dataset types that should
549 constrain the yielded data IDs. For example, including "raw" here would
550 constrain the yielded "instrument", "exposure", "detector", and
551 "physical_filter" values to only those for which at least one "raw" dataset
552 exists in "collections". Requires --collections."""
553 )
554)
555@where_option(help=where_help)
556@order_by_option()
557@limit_option()
558@offset_option()
559@options_file_option()
560def query_data_ids(**kwargs: Any) -> None:
561 """List the data IDs in a repository."""
562 table, reason = script.queryDataIds(**kwargs)
563 if table:
564 table.pprint_all()
565 else:
566 if reason:
567 print(reason)
568 if not kwargs.get("dimensions") and not kwargs.get("datasets"):
569 print("No results. Try requesting some dimensions or datasets, see --help for more information.")
570 else:
571 print("No results. Try --help for more information.")
574@click.command(cls=ButlerCommand)
575@repo_argument(required=True)
576@element_argument(required=True)
577@datasets_option(
578 help=unwrap(
579 """An expression that fully or partially identifies dataset types that should
580 constrain the yielded records. May only be used with
581 --collections."""
582 )
583)
584@collections_option(help=collections_option.help + " May only be used with --datasets.")
585@where_option(help=where_help)
586@order_by_option()
587@limit_option()
588@offset_option()
589@click.option(
590 "--no-check",
591 is_flag=True,
592 default=None,
593 help=unwrap(
594 """Don't check the query before execution. By default the query is checked before it
595 executed, this may reject some valid queries that resemble common mistakes."""
596 ),
597)
598@options_file_option()
599def query_dimension_records(**kwargs: Any) -> None:
600 """Query for dimension information."""
601 if kwargs.pop("no_check") is not None:
602 click.echo("WARNING: --no-check option has no effect and will be removed after v28.")
603 table = script.queryDimensionRecords(**kwargs)
604 if table:
605 table.pprint_all()
606 else:
607 print("No results. Try --help for more information.")
610@click.command(cls=ButlerCommand)
611@repo_argument(required=True)
612@query_datasets_options(showUri=False, useArguments=False, repo=False)
613@destination_argument(help="Destination URI of folder to receive file artifacts.")
614@transfer_option()
615@verbose_option(help="Report destination location of all transferred artifacts.")
616@click.option(
617 "--preserve-path/--no-preserve-path",
618 is_flag=True,
619 default=True,
620 help="Preserve the datastore path to the artifact at the destination.",
621)
622@click.option(
623 "--clobber/--no-clobber",
624 is_flag=True,
625 default=False,
626 help="If clobber, overwrite files if they exist locally.",
627)
628@click.option(
629 "--zip/--no-zip",
630 is_flag=True,
631 default=False,
632 help="Retrieve artifacts and place in a Zip file.",
633)
634@options_file_option()
635def retrieve_artifacts(**kwargs: Any) -> None:
636 """Retrieve file artifacts associated with datasets in a repository."""
637 verbose = kwargs.pop("verbose")
638 transferred = script.retrieveArtifacts(**kwargs)
639 if not transferred:
640 print("No datasets matched query.")
641 elif kwargs["zip"]:
642 print(f"Zip file written to {transferred[0]}")
643 else:
644 if verbose:
645 print(f"Transferred the following to {kwargs['destination']}:")
646 for uri in transferred:
647 print(uri)
648 print()
649 print(f"Number of artifacts retrieved into destination {kwargs['destination']}: {len(transferred)}")
652@click.command(cls=ButlerCommand)
653@click.argument("source", required=True)
654@click.argument("dest", required=True)
655@query_datasets_options(showUri=False, useArguments=False, repo=False)
656@transfer_option()
657@register_dataset_types_option()
658@transfer_dimensions_option()
659@click.option(
660 "--dry-run/--no-dry-run", default=False, help="Enable dry run mode and do not transfer any datasets."
661)
662@options_file_option()
663def transfer_datasets(**kwargs: Any) -> None:
664 """Transfer datasets from a source butler to a destination butler.
666 SOURCE is a URI to the Butler repository containing the RUN dataset.
668 DEST is a URI to the Butler repository that will receive copies of the
669 datasets.
670 """
671 number = script.transferDatasets(**kwargs)
672 print(f"Number of datasets transferred: {number}")
675@click.command(cls=ButlerCommand)
676@repo_argument(required=True)
677@click.argument("parent", required=True, nargs=1)
678@click.argument("children", required=False, nargs=-1, callback=split_commas)
679@click.option(
680 "--doc",
681 default="",
682 help="Documentation string associated with this collection. "
683 "Only relevant if the collection is newly created.",
684)
685@click.option(
686 "--flatten/--no-flatten",
687 default=False,
688 help="If `True` recursively flatten out any nested chained collections in children first.",
689)
690@click.option(
691 "--mode",
692 type=click.Choice(["redefine", "extend", "remove", "prepend", "pop"]),
693 default="redefine",
694 help="Update mode: "
695 "'redefine': Create new chain or redefine existing chain with the supplied CHILDREN. "
696 "'remove': Modify existing chain to remove the supplied CHILDREN. "
697 "'pop': Pop a numbered element off the chain. Defaults to popping "
698 "the first element (0). ``children`` must be integers if given. "
699 "'prepend': Modify existing chain to prepend the supplied CHILDREN to the front. "
700 "'extend': Modify existing chain to extend it with the supplied CHILDREN.",
701)
702def collection_chain(**kwargs: Any) -> None:
703 """Define a collection chain.
705 PARENT is the name of the chained collection to create or modify. If the
706 collection already exists the chain associated with it will be updated.
708 CHILDREN are the collections to be used to modify the chain. The supplied
709 values will be split on comma. The exact usage depends on the MODE option.
710 For example::
712 $ butler collection-chain REPO PARENT child1,child2 child3
714 will result in three children being included in the chain.
716 When the MODE is 'pop' the CHILDREN should be integer indices indicating
717 collections to be removed from the current chain.
718 MODE 'pop' can take negative integers to indicate removal relative to the
719 end of the chain, but when doing that '--' must be given to indicate the
720 end of the options specification::
722 $ butler collection-chain REPO --mode=pop PARENT -- -1
724 Will remove the final collection from the chain.
725 """
726 chain = script.collectionChain(**kwargs)
727 print(f"[{', '.join(chain)}]")
730@click.command(cls=ButlerCommand)
731@repo_argument(required=True)
732@click.argument("dataset_type", required=True)
733@click.argument("run", required=True)
734@click.argument("table_file", required=True)
735@click.option(
736 "--formatter",
737 type=str,
738 help="Fully-qualified python class to use as the Formatter. If not specified the formatter"
739 " will be determined from the dataset type and datastore configuration.",
740)
741@click.option(
742 "--id-generation-mode",
743 default="UNIQUE",
744 help="Mode to use for generating dataset IDs. The default creates a unique ID. Other options"
745 " are: 'DATAID_TYPE' for creating a reproducible ID from the dataID and dataset type;"
746 " 'DATAID_TYPE_RUN' for creating a reproducible ID from the dataID, dataset type and run."
747 " The latter is usually used for 'raw'-type data that will be ingested in multiple."
748 " repositories.",
749 callback=to_upper,
750 type=click.Choice(("UNIQUE", "DATAID_TYPE", "DATAID_TYPE_RUN"), case_sensitive=False),
751)
752@click.option(
753 "--data-id",
754 type=str,
755 multiple=True,
756 callback=split_commas,
757 help="Keyword=value string with an additional dataId value that is fixed for all ingested"
758 " files. This can be used to simplify the table file by removing repeated entries that are"
759 " fixed for all files to be ingested. Multiple key/values can be given either by using"
760 " comma separation or multiple command line options.",
761)
762@click.option(
763 "--prefix",
764 type=str,
765 help="For relative paths in the table file, specify a prefix to use. The default is to"
766 " use the current working directory.",
767)
768@track_file_attrs_option()
769@transfer_option()
770def ingest_files(**kwargs: Any) -> None:
771 """Ingest files from table file.
773 DATASET_TYPE is the name of the dataset type to be associated with these
774 files. This dataset type must already exist and will not be created by
775 this command. There can only be one dataset type per invocation of this
776 command.
778 RUN is the run to use for the file ingest.
780 TABLE_FILE refers to a file that can be read by astropy.table with
781 columns of:
783 file URI, dimension1, dimension2, ..., dimensionN
785 where the first column is the URI to the file to be ingested and the
786 remaining columns define the dataId to associate with that file.
787 The column names should match the dimensions for the specified dataset
788 type. Relative file URI by default is assumed to be relative to the
789 current working directory but can be overridden using the ``--prefix``
790 option.
792 This command does not create dimension records and so any records must
793 be created by other means. This command should not be used to ingest
794 raw camera exposures.
795 """
796 script.ingest_files(**kwargs)
799@click.command(cls=ButlerCommand)
800@repo_argument(required=True)
801@click.argument("dataset_type", required=True)
802@click.argument("storage_class", required=True)
803@click.argument("dimensions", required=False, nargs=-1)
804@click.option(
805 "--is-calibration/--no-is-calibration",
806 is_flag=True,
807 default=False,
808 help="Indicate that this dataset type can be part of a calibration collection.",
809)
810def register_dataset_type(**kwargs: Any) -> None:
811 """Register a new dataset type with this butler repository.
813 DATASET_TYPE is the name of the dataset type.
815 STORAGE_CLASS is the name of the StorageClass to be associated with
816 this dataset type.
818 DIMENSIONS is a list of all the dimensions relevant to this
819 dataset type. It can be an empty list.
821 A component dataset type (such as "something.component") is not a
822 real dataset type and so can not be defined by this command. They are
823 automatically derived from the composite dataset type when a composite
824 storage class is specified.
825 """
826 inserted = script.register_dataset_type(**kwargs)
827 if inserted:
828 print("Dataset type successfully registered.")
829 else:
830 print("Dataset type already existed in identical form.")
833@click.command(cls=ButlerCommand)
834@repo_argument(required=True)
835@directory_argument(required=True, help="DIRECTORY is the folder to receive the exported calibrations.")
836@collections_argument(help="COLLECTIONS are the collection to export calibrations from.")
837@dataset_type_option(help="Specific DatasetType(s) to export.", multiple=True)
838@transfer_option()
839def export_calibs(*args: Any, **kwargs: Any) -> None:
840 """Export calibrations from the butler for import elsewhere."""
841 table = script.exportCalibs(*args, **kwargs)
842 if table:
843 table.pprint_all(align="<")
846@click.command(cls=ButlerCommand)
847@repo_argument(required=True)
848@click.argument("zip", required=True)
849@transfer_option()
850@transfer_dimensions_option(
851 default=False, help="Attempt to register missing dimension records during ingest."
852)
853@click.option(
854 "--dry-run/--no-dry-run", default=False, help="Enable dry run mode and do not ingest any datasets."
855)
856def ingest_zip(**kwargs: Any) -> None:
857 """Ingest a Zip file created by retrieve-artifacts.
859 ZIP is the URI to the Zip file that should be ingested.
861 This command does not create dimension records and so any records must
862 be created by other means.
863 """
864 script.ingest_zip(**kwargs)