Coverage for python/lsst/daf/butler/cli/cmd/commands.py: 56%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ()
24import click
25from deprecated.sphinx import deprecated
27from ... import script
28from ..opt import (
29 collection_argument,
30 collection_type_option,
31 collections_argument,
32 collections_option,
33 components_option,
34 confirm_option,
35 dataset_type_option,
36 datasets_option,
37 destination_argument,
38 dimensions_argument,
39 directory_argument,
40 element_argument,
41 glob_argument,
42 limit_option,
43 offset_option,
44 options_file_option,
45 order_by_option,
46 query_datasets_options,
47 register_dataset_types_option,
48 repo_argument,
49 transfer_option,
50 verbose_option,
51 where_option,
52)
53from ..utils import (
54 ButlerCommand,
55 MWOptionDecorator,
56 option_section,
57 printAstropyTables,
58 split_commas,
59 to_upper,
60 typeStrAcceptsMultiple,
61 unwrap,
62 where_help,
63)
65willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist."
66existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file."
69@click.command(cls=ButlerCommand, short_help="Add existing datasets to a tagged collection.")
70@repo_argument(required=True)
71@collection_argument(help="COLLECTION is the collection the datasets should be associated with.")
72@query_datasets_options(repo=False, showUri=False, useArguments=False)
73@options_file_option()
74def associate(**kwargs):
75 """Add existing datasets to a tagged collection; searches for datasets with
76 the options and adds them to the named COLLECTION.
77 """
78 script.associate(**kwargs)
81# The conversion from the import command name to the butler_import function
82# name for subcommand lookup is implemented in the cli/butler.py, in
83# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they
84# must be reflected in that location. If this becomes a common pattern a better
85# mechanism should be implemented.
86@click.command("import", cls=ButlerCommand)
87@repo_argument(required=True, help=willCreateRepoHelp)
88@directory_argument(required=True)
89@transfer_option()
90@click.option(
91 "--export-file",
92 help="Name for the file that contains database information associated with the exported "
93 "datasets. If this is not an absolute path, does not exist in the current working "
94 "directory, and --dir is provided, it is assumed to be in that directory. Defaults "
95 'to "export.yaml".',
96 type=click.File(mode="r"),
97)
98@click.option(
99 "--skip-dimensions",
100 "-s",
101 type=str,
102 multiple=True,
103 callback=split_commas,
104 metavar=typeStrAcceptsMultiple,
105 help="Dimensions that should be skipped during import",
106)
107@click.option("--reuse-ids", is_flag=True, help="Force re-use of imported dataset IDs for integer IDs.")
108@options_file_option()
109def butler_import(*args, **kwargs):
110 """Import data into a butler repository."""
111 script.butlerImport(*args, **kwargs)
114@click.command(cls=ButlerCommand)
115@repo_argument(required=True, help=willCreateRepoHelp)
116@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).")
117@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.")
118@click.option(
119 "--standalone",
120 is_flag=True,
121 help="Include all defaults in the config file in the repo, "
122 "insulating the repo from changes in package defaults.",
123)
124@click.option(
125 "--override", is_flag=True, help="Allow values in the supplied config to override all repo settings."
126)
127@click.option(
128 "--outfile",
129 "-f",
130 default=None,
131 type=str,
132 help="Name of output file to receive repository "
133 "configuration. Default is to write butler.yaml into the specified repo.",
134)
135@options_file_option()
136def create(*args, **kwargs):
137 """Create an empty Gen3 Butler repository."""
138 script.createRepo(*args, **kwargs)
141@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand)
142@repo_argument(required=True, help=existingRepoHelp)
143@click.option(
144 "--subset",
145 "-s",
146 type=str,
147 help="Subset of a configuration to report. This can be any key in the hierarchy such as "
148 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.",
149)
150@click.option(
151 "--searchpath",
152 "-p",
153 type=str,
154 multiple=True,
155 callback=split_commas,
156 metavar=typeStrAcceptsMultiple,
157 help="Additional search paths to use for configuration overrides",
158)
159@click.option(
160 "--file",
161 "outfile",
162 type=click.File(mode="w"),
163 default="-",
164 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout "
165 "by default.",
166)
167@options_file_option()
168def config_dump(*args, **kwargs):
169 """Dump either a subset or full Butler configuration to standard output."""
170 script.configDump(*args, **kwargs)
173@click.command(short_help="Validate the configuration files.", cls=ButlerCommand)
174@repo_argument(required=True, help=existingRepoHelp)
175@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.")
176@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True)
177@click.option(
178 "--ignore",
179 "-i",
180 type=str,
181 multiple=True,
182 callback=split_commas,
183 metavar=typeStrAcceptsMultiple,
184 help="DatasetType(s) to ignore for validation.",
185)
186@options_file_option()
187def config_validate(*args, **kwargs):
188 """Validate the configuration files for a Gen3 Butler repository."""
189 is_good = script.configValidate(*args, **kwargs)
190 if not is_good:
191 raise click.exceptions.Exit(1)
194@click.command(cls=ButlerCommand)
195@repo_argument(required=True)
196@collection_argument(
197 help=unwrap(
198 """COLLECTION is the Name of the collection to remove. If this is a tagged or
199 chained collection, datasets within the collection are not modified unless --unstore
200 is passed. If this is a run collection, --purge and --unstore must be passed, and
201 all datasets in it are fully removed from the data repository."""
202 )
203)
204@click.option(
205 "--purge",
206 help=unwrap(
207 """Permit RUN collections to be removed, fully removing datasets within them.
208 Requires --unstore as an added precaution against accidental deletion. Must not be
209 passed if the collection is not a RUN."""
210 ),
211 is_flag=True,
212)
213@click.option(
214 "--unstore",
215 help=("""Remove all datasets in the collection from all datastores in which they appear."""),
216 is_flag=True,
217)
218@click.option(
219 "--unlink",
220 help="Before removing the given `collection` unlink it from from this parent collection.",
221 multiple=True,
222 callback=split_commas,
223)
224@confirm_option()
225@options_file_option()
226@deprecated(
227 reason="Please consider using remove-collections or remove-runs instead. Will be removed after v24.",
228 version="v24.0",
229 category=FutureWarning,
230)
231def prune_collection(**kwargs):
232 """Remove a collection and possibly prune datasets within it."""
233 result = script.pruneCollection(**kwargs)
234 if result.confirm:
235 print("The following collections will be removed:")
236 result.removeTable.pprint_all(align="<")
237 doContinue = click.confirm(text="Continue?", default=False)
238 else:
239 doContinue = True
240 if doContinue:
241 result.onConfirmation()
242 print("Removed collections.")
243 else:
244 print("Aborted.")
247pruneDatasets_wouldRemoveMsg = unwrap(
248 """The following datasets will be removed from any datastores in which
249 they are present:"""
250)
251pruneDatasets_wouldDisassociateMsg = unwrap(
252 """The following datasets will be disassociated from {collections}
253 if they are currently present in it (which is not checked):"""
254)
255pruneDatasets_wouldDisassociateAndRemoveMsg = unwrap(
256 """The following datasets will be disassociated from
257 {collections} if they are currently present in it (which is
258 not checked), and removed from any datastores in which they
259 are present."""
260)
261pruneDatasets_willRemoveMsg = "The following datasets will be removed:"
262pruneDatasets_askContinueMsg = "Continue?"
263pruneDatasets_didRemoveAforementioned = "The datasets were removed."
264pruneDatasets_didNotRemoveAforementioned = "Did not remove the datasets."
265pruneDatasets_didRemoveMsg = "Removed the following datasets:"
266pruneDatasets_noDatasetsFound = "Did not find any datasets."
267pruneDatasets_errPurgeAndDisassociate = unwrap(
268 """"--disassociate and --purge may not be used together: --disassociate purges from just the passed TAGged
269 collections, but --purge forces disassociation from all of them. """
270)
271pruneDatasets_errQuietWithDryRun = "Can not use --quiet and --dry-run together."
272pruneDatasets_errNoCollectionRestriction = unwrap(
273 """Must indicate collections from which to prune datasets by passing COLLETION arguments (select all
274 collections by passing '*', or consider using 'butler prune-collections'), by using --purge to pass a run
275 collection, or by using --disassociate to select a tagged collection."""
276)
277pruneDatasets_errPruneOnNotRun = "Can not prune a collection that is not a RUN collection: {collection}"
278pruneDatasets_errNoOp = "No operation: one of --purge, --unstore, or --disassociate must be provided."
280disassociate_option = MWOptionDecorator(
281 "--disassociate",
282 "disassociate_tags",
283 help=unwrap(
284 """Disassociate pruned datasets from the given tagged collections. May not be used with
285 --purge."""
286 ),
287 multiple=True,
288 callback=split_commas,
289 metavar="TAG",
290)
293purge_option = MWOptionDecorator(
294 "--purge",
295 "purge_run",
296 help=unwrap(
297 """Completely remove the dataset from the given RUN in the Registry. May not be used with
298 --disassociate. Note, this may remove provenance information from datasets other than those
299 provided, and should be used with extreme care."""
300 ),
301 metavar="RUN",
302)
305find_all_option = MWOptionDecorator(
306 "--find-all",
307 is_flag=True,
308 help=unwrap(
309 """Purge the dataset results from all of the collections in which a dataset of that dataset
310 type + data id combination appear. (By default only the first found dataset type + data id is
311 purged, according to the order of COLLECTIONS passed in)."""
312 ),
313)
316unstore_option = MWOptionDecorator(
317 "--unstore",
318 is_flag=True,
319 help=unwrap(
320 """Remove these datasets from all datastores configured with this data repository. If
321 --disassociate and --purge are not used then --unstore will be used by default. Note that
322 --unstore will make it impossible to retrieve these datasets even via other collections.
323 Datasets that are already not stored are ignored by this option."""
324 ),
325)
328dry_run_option = MWOptionDecorator(
329 "--dry-run",
330 is_flag=True,
331 help=unwrap(
332 """Display the datasets that would be removed but do not remove them.
334 Note that a dataset can be in collections other than its RUN-type collection, and removing it
335 will remove it from all of them, even though the only one this will show is its RUN
336 collection."""
337 ),
338)
341quiet_option = MWOptionDecorator(
342 "--quiet",
343 is_flag=True,
344 help=unwrap("""Makes output quiet. Implies --no-confirm. Requires --dry-run not be passed."""),
345)
348@click.command(cls=ButlerCommand, short_help="Remove datasets.")
349@repo_argument(required=True)
350@collections_argument(
351 help=unwrap(
352 """COLLECTIONS is or more expressions that identify the collections to
353 search for datasets. Glob-style expressions may be used but only if the
354 --find-all flag is also passed."""
355 )
356)
357@option_section("Query Datasets Options:")
358@datasets_option(
359 help="One or more glob-style expressions that identify the dataset types to be pruned.",
360 multiple=True,
361 callback=split_commas,
362)
363@find_all_option()
364@where_option(help=where_help)
365@option_section("Prune Options:")
366@disassociate_option()
367@purge_option()
368@unstore_option()
369@option_section("Execution Options:")
370@dry_run_option()
371@confirm_option()
372@quiet_option()
373@option_section("Other Options:")
374@options_file_option()
375def prune_datasets(**kwargs):
376 """Query for and remove one or more datasets from a collection and/or
377 storage.
378 """
379 quiet = kwargs.pop("quiet", False)
380 if quiet:
381 if kwargs["dry_run"]:
382 raise click.ClickException(message=pruneDatasets_errQuietWithDryRun)
383 kwargs["confirm"] = False
385 result = script.pruneDatasets(**kwargs)
387 if result.errPurgeAndDisassociate:
388 raise click.ClickException(message=pruneDatasets_errPurgeAndDisassociate)
389 if result.errNoCollectionRestriction:
390 raise click.ClickException(message=pruneDatasets_errNoCollectionRestriction)
391 if result.errPruneOnNotRun:
392 raise click.ClickException(message=pruneDatasets_errPruneOnNotRun.format(**result.errDict))
393 if result.errNoOp:
394 raise click.ClickException(message=pruneDatasets_errNoOp)
395 if result.dryRun:
396 if result.action["disassociate"] and result.action["unstore"]:
397 msg = pruneDatasets_wouldDisassociateAndRemoveMsg
398 elif result.action["disassociate"]:
399 msg = pruneDatasets_wouldDisassociateMsg
400 else:
401 msg = pruneDatasets_wouldRemoveMsg
402 print(msg.format(**result.action))
403 printAstropyTables(result.tables)
404 return
405 if result.confirm:
406 if not result.tables:
407 print(pruneDatasets_noDatasetsFound)
408 return
409 print(pruneDatasets_willRemoveMsg)
410 printAstropyTables(result.tables)
411 doContinue = click.confirm(text=pruneDatasets_askContinueMsg, default=False)
412 if doContinue:
413 result.onConfirmation()
414 print(pruneDatasets_didRemoveAforementioned)
415 else:
416 print(pruneDatasets_didNotRemoveAforementioned)
417 return
418 if result.finished:
419 if not quiet:
420 print(pruneDatasets_didRemoveMsg)
421 printAstropyTables(result.tables)
422 return
425@click.command(short_help="Search for collections.", cls=ButlerCommand)
426@repo_argument(required=True)
427@glob_argument(
428 help="GLOB is one or more glob-style expressions that fully or partially identify the "
429 "collections to return."
430)
431@collection_type_option()
432@click.option(
433 "--chains",
434 default="TREE",
435 help="""Affects how results are presented:
437 TABLE lists each dataset in table form, with columns for dataset name
438 and type, and a column that lists children of CHAINED datasets (if any
439 CHAINED datasets are found).
441 INVERSE-TABLE is like TABLE but instead of a column listing CHAINED
442 dataset children, it lists the parents of the dataset if it is contained
443 in any CHAINED collections.
445 TREE recursively lists children below each CHAINED dataset in tree form.
447 INVERSE-TREE recursively lists parent datasets below each dataset in
448 tree form.
450 FLATTEN lists all datasets, including child datasets, in one list.
452 [default: TREE]""",
453 # above, the default value is included, instead of using show_default, so
454 # that the default is printed on its own line instead of coming right after
455 # the FLATTEN text.
456 callback=to_upper,
457 type=click.Choice(
458 choices=("TABLE", "INVERSE-TABLE", "TREE", "INVERSE-TREE", "FLATTEN"),
459 case_sensitive=False,
460 ),
461)
462@options_file_option()
463def query_collections(*args, **kwargs):
464 """Get the collections whose names match an expression."""
465 table = script.queryCollections(*args, **kwargs)
466 # The unit test that mocks script.queryCollections does not return a table
467 # so we need the following `if`.
468 if table:
469 # When chains==TREE, the children of chained datasets are indented
470 # relative to their parents. For this to work properly the table must
471 # be left-aligned.
472 table.pprint_all(align="<")
475@click.command(cls=ButlerCommand)
476@repo_argument(required=True)
477@glob_argument(
478 help="GLOB is one or more glob-style expressions that fully or partially identify the "
479 "dataset types to return."
480)
481@verbose_option(help="Include dataset type name, dimensions, and storage class in output.")
482@components_option()
483@options_file_option()
484def query_dataset_types(*args, **kwargs):
485 """Get the dataset types in a repository."""
486 table = script.queryDatasetTypes(*args, **kwargs)
487 if table:
488 table.pprint_all()
489 else:
490 print("No results. Try --help for more information.")
493@click.command(cls=ButlerCommand)
494@repo_argument(required=True)
495@click.argument("dataset-type-name", nargs=1)
496def remove_dataset_type(*args, **kwargs):
497 """Remove a dataset type definition from a repository."""
498 script.removeDatasetType(*args, **kwargs)
501@click.command(cls=ButlerCommand)
502@query_datasets_options()
503@options_file_option()
504def query_datasets(**kwargs):
505 """List the datasets in a repository."""
506 for table in script.QueryDatasets(**kwargs).getTables():
507 print("")
508 table.pprint_all()
509 print("")
512@click.command(cls=ButlerCommand)
513@repo_argument(required=True)
514@click.argument("input-collection")
515@click.argument("output-collection")
516@click.argument("dataset-type-name")
517@click.option(
518 "--begin-date",
519 type=str,
520 default=None,
521 help=unwrap(
522 """ISO-8601 datetime (TAI) of the beginning of the validity range for the
523 certified calibrations."""
524 ),
525)
526@click.option(
527 "--end-date",
528 type=str,
529 default=None,
530 help=unwrap(
531 """ISO-8601 datetime (TAI) of the end of the validity range for the
532 certified calibrations."""
533 ),
534)
535@click.option(
536 "--search-all-inputs",
537 is_flag=True,
538 default=False,
539 help=unwrap(
540 """Search all children of the inputCollection if it is a CHAINED collection,
541 instead of just the most recent one."""
542 ),
543)
544@options_file_option()
545def certify_calibrations(*args, **kwargs):
546 """Certify calibrations in a repository."""
547 script.certifyCalibrations(*args, **kwargs)
550@click.command(cls=ButlerCommand)
551@repo_argument(required=True)
552@dimensions_argument(
553 help=unwrap(
554 """DIMENSIONS are the keys of the data IDs to yield, such as exposure,
555 instrument, or tract. Will be expanded to include any dependencies."""
556 )
557)
558@collections_option(help=collections_option.help + " May only be used with --datasets.")
559@datasets_option(
560 help=unwrap(
561 """An expression that fully or partially identifies dataset types that should
562 constrain the yielded data IDs. For example, including "raw" here would
563 constrain the yielded "instrument", "exposure", "detector", and
564 "physical_filter" values to only those for which at least one "raw" dataset
565 exists in "collections". Requires --collections."""
566 )
567)
568@where_option(help=where_help)
569@order_by_option()
570@limit_option()
571@offset_option()
572@options_file_option()
573def query_data_ids(**kwargs):
574 """List the data IDs in a repository."""
575 table = script.queryDataIds(**kwargs)
576 if table:
577 table.pprint_all()
578 else:
579 if not kwargs.get("dimensions") and not kwargs.get("datasets"):
580 print("No results. Try requesting some dimensions or datasets, see --help for more information.")
581 else:
582 print("No results. Try --help for more information.")
585@click.command(cls=ButlerCommand)
586@repo_argument(required=True)
587@element_argument(required=True)
588@datasets_option(
589 help=unwrap(
590 """An expression that fully or partially identifies dataset types that should
591 constrain the yielded records. May only be used with
592 --collections."""
593 )
594)
595@collections_option(help=collections_option.help + " May only be used with --datasets.")
596@where_option(help=where_help)
597@order_by_option()
598@limit_option()
599@offset_option()
600@click.option(
601 "--no-check",
602 is_flag=True,
603 help=unwrap(
604 """Don't check the query before execution. By default the query is checked before it
605 executed, this may reject some valid queries that resemble common mistakes."""
606 ),
607)
608@options_file_option()
609def query_dimension_records(**kwargs):
610 """Query for dimension information."""
611 table = script.queryDimensionRecords(**kwargs)
612 if table:
613 table.pprint_all()
614 else:
615 print("No results. Try --help for more information.")
618@click.command(cls=ButlerCommand)
619@repo_argument(required=True)
620@query_datasets_options(showUri=False, useArguments=False, repo=False)
621@destination_argument(help="Destination URI of folder to receive file artifacts.")
622@transfer_option()
623@verbose_option(help="Report destination location of all transferred artifacts.")
624@click.option(
625 "--preserve-path/--no-preserve-path",
626 is_flag=True,
627 default=True,
628 help="Preserve the datastore path to the artifact at the destination.",
629)
630@click.option(
631 "--clobber/--no-clobber",
632 is_flag=True,
633 default=False,
634 help="If clobber, overwrite files if they exist locally.",
635)
636@options_file_option()
637def retrieve_artifacts(**kwargs):
638 """Retrieve file artifacts associated with datasets in a repository."""
639 verbose = kwargs.pop("verbose")
640 transferred = script.retrieveArtifacts(**kwargs)
641 if verbose and transferred:
642 print(f"Transferred the following to {kwargs['destination']}:")
643 for uri in transferred:
644 print(uri)
645 print()
646 print(f"Number of artifacts retrieved into destination {kwargs['destination']}: {len(transferred)}")
649@click.command(cls=ButlerCommand)
650@click.argument("source", required=True)
651@click.argument("dest", required=True)
652@query_datasets_options(showUri=False, useArguments=False, repo=False)
653@transfer_option()
654@register_dataset_types_option()
655@options_file_option()
656def transfer_datasets(**kwargs):
657 """Transfer datasets from a source butler to a destination butler.
659 SOURCE is a URI to the Butler repository containing the RUN dataset.
661 DEST is a URI to the Butler repository that will receive copies of the
662 datasets.
663 """
664 number = script.transferDatasets(**kwargs)
665 print(f"Number of datasets transferred: {number}")
668@click.command(cls=ButlerCommand)
669@repo_argument(required=True)
670@click.argument("parent", required=True, nargs=1)
671@click.argument("children", required=False, nargs=-1, callback=split_commas)
672@click.option(
673 "--doc",
674 default="",
675 help="Documentation string associated with this collection. "
676 "Only relevant if the collection is newly created.",
677)
678@click.option(
679 "--flatten/--no-flatten",
680 default=False,
681 help="If `True` recursively flatten out any nested chained collections in children first.",
682)
683@click.option(
684 "--mode",
685 type=click.Choice(["redefine", "extend", "remove", "prepend", "pop"]),
686 default="redefine",
687 help="Update mode: "
688 "'redefine': Create new chain or redefine existing chain with the supplied CHILDREN. "
689 "'remove': Modify existing chain to remove the supplied CHILDREN. "
690 "'pop': Pop a numbered element off the chain. Defaults to popping "
691 "the first element (0). ``children`` must be integers if given. "
692 "'prepend': Modify existing chain to prepend the supplied CHILDREN to the front. "
693 "'extend': Modify existing chain to extend it with the supplied CHILDREN.",
694)
695def collection_chain(**kwargs):
696 """Define a collection chain.
698 PARENT is the name of the chained collection to create or modify. If the
699 collection already exists the chain associated with it will be updated.
701 CHILDREN are the collections to be used to modify the chain. The supplied
702 values will be split on comma. The exact usage depends on the MODE option.
703 For example,
705 $ butler collection-chain REPO PARENT child1,child2 child3
707 will result in three children being included in the chain.
709 When the MODE is 'pop' the CHILDREN should be integer indices indicating
710 collections to be removed from the current chain.
711 MODE 'pop' can take negative integers to indicate removal relative to the
712 end of the chain, but when doing that '--' must be given to indicate the
713 end of the options specification.
715 $ butler collection-chain REPO --mode=pop PARENT -- -1
717 Will remove the final collection from the chain.
718 """
719 chain = script.collectionChain(**kwargs)
720 print(f"[{', '.join(chain)}]")
723@click.command(cls=ButlerCommand)
724@repo_argument(required=True)
725@click.argument("dataset_type", required=True)
726@click.argument("run", required=True)
727@click.argument("table_file", required=True)
728@click.option(
729 "--formatter",
730 type=str,
731 help="Fully-qualified python class to use as the Formatter. If not specified the formatter"
732 " will be determined from the dataset type and datastore configuration.",
733)
734@click.option(
735 "--id-generation-mode",
736 default="UNIQUE",
737 help="Mode to use for generating dataset IDs. The default creates a unique ID. Other options"
738 " are: 'DATAID_TYPE' for creating a reproducible ID from the dataID and dataset type;"
739 " 'DATAID_TYPE_RUN' for creating a reproducible ID from the dataID, dataset type and run."
740 " The latter is usually used for 'raw'-type data that will be ingested in multiple."
741 " repositories.",
742 callback=to_upper,
743 type=click.Choice(("UNIQUE", "DATAID_TYPE", "DATAID_TYPE_RUN"), case_sensitive=False),
744)
745@click.option(
746 "--data-id",
747 type=str,
748 multiple=True,
749 callback=split_commas,
750 help="Keyword=value string with an additional dataId value that is fixed for all ingested"
751 " files. This can be used to simplify the table file by removing repeated entries that are"
752 " fixed for all files to be ingested. Multiple key/values can be given either by using"
753 " comma separation or multiple command line options.",
754)
755@click.option(
756 "--prefix",
757 type=str,
758 help="For relative paths in the table file, specify a prefix to use. The default is to"
759 " use the current working directory.",
760)
761@transfer_option()
762def ingest_files(**kwargs):
763 """Ingest files from table file.
765 DATASET_TYPE is the name of the dataset type to be associated with these
766 files. This dataset type must already exist and will not be created by
767 this command. There can only be one dataset type per invocation of this
768 command.
770 RUN is the run to use for the file ingest.
772 TABLE_FILE refers to a file that can be read by astropy.table with
773 columns of:
775 file URI, dimension1, dimension2, ..., dimensionN
777 where the first column is the URI to the file to be ingested and the
778 remaining columns define the dataId to associate with that file.
779 The column names should match the dimensions for the specified dataset
780 type. Relative file URI by default is assumed to be relative to the
781 current working directory but can be overridden using the ``--prefix``
782 option.
784 This command does not create dimension records and so any records must
785 be created by other means. This command should not be used to ingest
786 raw camera exposures.
787 """
788 script.ingest_files(**kwargs)
791@click.command(cls=ButlerCommand)
792@repo_argument(required=True)
793@click.argument("dataset_type", required=True)
794@click.argument("storage_class", required=True)
795@click.argument("dimensions", required=False, nargs=-1)
796@click.option(
797 "--is-calibration/--no-is-calibration",
798 is_flag=True,
799 default=False,
800 help="Indicate that this dataset type can be part of a calibration collection.",
801)
802def register_dataset_type(**kwargs):
803 """Register a new dataset type with this butler repository.
805 DATASET_TYPE is the name of the dataset type.
807 STORAGE_CLASS is the name of the StorageClass to be associated with
808 this dataset type.
810 DIMENSIONS is a list of all the dimensions relevant to this
811 dataset type. It can be an empty list.
813 A component dataset type (such as "something.component") is not a
814 real dataset type and so can not be defined by this command. They are
815 automatically derived from the composite dataset type when a composite
816 storage class is specified.
817 """
818 inserted = script.register_dataset_type(**kwargs)
819 if inserted:
820 print("Dataset type successfully registered.")
821 else:
822 print("Dataset type already existed in identical form.")
825@click.command(cls=ButlerCommand)
826@repo_argument(required=True)
827@directory_argument(required=True)
828@collections_argument(help="COLLECTIONS are the collection to export calibrations from.")
829def export_calibs(*args, **kwargs):
830 """Export calibrations from the butler for import elsewhere."""
831 table = script.exportCalibs(*args, **kwargs)
832 if table:
833 table.pprint_all(align="<")