Coverage for python/lsst/daf/butler/cli/cmd/commands.py: 56%

290 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-06-06 09:38 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = () 

24 

25import warnings 

26from typing import Any 

27 

28import click 

29 

30from ... import script 

31from ..opt import ( 

32 collection_argument, 

33 collection_type_option, 

34 collections_argument, 

35 collections_option, 

36 components_option, 

37 confirm_option, 

38 dataset_type_option, 

39 datasets_option, 

40 destination_argument, 

41 dimensions_argument, 

42 directory_argument, 

43 element_argument, 

44 glob_argument, 

45 limit_option, 

46 offset_option, 

47 options_file_option, 

48 order_by_option, 

49 query_datasets_options, 

50 register_dataset_types_option, 

51 repo_argument, 

52 transfer_dimensions_option, 

53 transfer_option, 

54 verbose_option, 

55 where_option, 

56) 

57from ..utils import ( 

58 ButlerCommand, 

59 MWOptionDecorator, 

60 option_section, 

61 printAstropyTables, 

62 split_commas, 

63 to_upper, 

64 typeStrAcceptsMultiple, 

65 unwrap, 

66 where_help, 

67) 

68 

69willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist." 

70existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file." 

71 

72 

73@click.command(cls=ButlerCommand, short_help="Add existing datasets to a tagged collection.") 

74@repo_argument(required=True) 

75@collection_argument(help="COLLECTION is the collection the datasets should be associated with.") 

76@query_datasets_options(repo=False, showUri=False, useArguments=False) 

77@options_file_option() 

78def associate(**kwargs: Any) -> None: 

79 """Add existing datasets to a tagged collection; searches for datasets with 

80 the options and adds them to the named COLLECTION. 

81 """ 

82 script.associate(**kwargs) 

83 

84 

85# The conversion from the import command name to the butler_import function 

86# name for subcommand lookup is implemented in the cli/butler.py, in 

87# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they 

88# must be reflected in that location. If this becomes a common pattern a better 

89# mechanism should be implemented. 

90@click.command("import", cls=ButlerCommand) 

91@repo_argument(required=True, help=willCreateRepoHelp) 

92@directory_argument(required=True) 

93@transfer_option() 

94@click.option( 

95 "--export-file", 

96 help="Name for the file that contains database information associated with the exported " 

97 "datasets. If this is not an absolute path, does not exist in the current working " 

98 "directory, and --dir is provided, it is assumed to be in that directory. Defaults " 

99 'to "export.yaml".', 

100 type=str, 

101) 

102@click.option( 

103 "--skip-dimensions", 

104 "-s", 

105 type=str, 

106 multiple=True, 

107 callback=split_commas, 

108 metavar=typeStrAcceptsMultiple, 

109 help="Dimensions that should be skipped during import", 

110) 

111@click.option("--reuse-ids", is_flag=True, help="Force re-use of imported dataset IDs for integer IDs.") 

112@options_file_option() 

113def butler_import(*args: Any, **kwargs: Any) -> None: 

114 """Import data into a butler repository.""" 

115 

116 # `reuse_ids`` is not used by `butlerImport`. 

117 reuse_ids = kwargs.pop("reuse_ids", False) 

118 if reuse_ids: 

119 warnings.warn("--reuse-ids option is deprecated and will be removed after v26.", FutureWarning) 

120 

121 script.butlerImport(*args, **kwargs) 

122 

123 

124@click.command(cls=ButlerCommand) 

125@repo_argument(required=True, help=willCreateRepoHelp) 

126@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).") 

127@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.") 

128@click.option( 

129 "--standalone", 

130 is_flag=True, 

131 help="Include all defaults in the config file in the repo, " 

132 "insulating the repo from changes in package defaults.", 

133) 

134@click.option( 

135 "--override", is_flag=True, help="Allow values in the supplied config to override all repo settings." 

136) 

137@click.option( 

138 "--outfile", 

139 "-f", 

140 default=None, 

141 type=str, 

142 help="Name of output file to receive repository " 

143 "configuration. Default is to write butler.yaml into the specified repo.", 

144) 

145@options_file_option() 

146def create(*args: Any, **kwargs: Any) -> None: 

147 """Create an empty Gen3 Butler repository.""" 

148 script.createRepo(*args, **kwargs) 

149 

150 

151@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand) 

152@repo_argument(required=True, help=existingRepoHelp) 

153@click.option( 

154 "--subset", 

155 "-s", 

156 type=str, 

157 help="Subset of a configuration to report. This can be any key in the hierarchy such as " 

158 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.", 

159) 

160@click.option( 

161 "--searchpath", 

162 "-p", 

163 type=str, 

164 multiple=True, 

165 callback=split_commas, 

166 metavar=typeStrAcceptsMultiple, 

167 help="Additional search paths to use for configuration overrides", 

168) 

169@click.option( 

170 "--file", 

171 "outfile", 

172 type=click.File(mode="w"), 

173 default="-", 

174 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout by default.", 

175) 

176@options_file_option() 

177def config_dump(*args: Any, **kwargs: Any) -> None: 

178 """Dump either a subset or full Butler configuration to standard output.""" 

179 script.configDump(*args, **kwargs) 

180 

181 

182@click.command(short_help="Validate the configuration files.", cls=ButlerCommand) 

183@repo_argument(required=True, help=existingRepoHelp) 

184@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.") 

185@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True) 

186@click.option( 

187 "--ignore", 

188 "-i", 

189 type=str, 

190 multiple=True, 

191 callback=split_commas, 

192 metavar=typeStrAcceptsMultiple, 

193 help="DatasetType(s) to ignore for validation.", 

194) 

195@options_file_option() 

196def config_validate(*args: Any, **kwargs: Any) -> None: 

197 """Validate the configuration files for a Gen3 Butler repository.""" 

198 is_good = script.configValidate(*args, **kwargs) 

199 if not is_good: 

200 raise click.exceptions.Exit(1) 

201 

202 

203pruneDatasets_wouldRemoveMsg = unwrap( 

204 """The following datasets will be removed from any datastores in which 

205 they are present:""" 

206) 

207pruneDatasets_wouldDisassociateMsg = unwrap( 

208 """The following datasets will be disassociated from {collections} 

209 if they are currently present in it (which is not checked):""" 

210) 

211pruneDatasets_wouldDisassociateAndRemoveMsg = unwrap( 

212 """The following datasets will be disassociated from 

213 {collections} if they are currently present in it (which is 

214 not checked), and removed from any datastores in which they 

215 are present.""" 

216) 

217pruneDatasets_willRemoveMsg = "The following datasets will be removed:" 

218pruneDatasets_askContinueMsg = "Continue?" 

219pruneDatasets_didRemoveAforementioned = "The datasets were removed." 

220pruneDatasets_didNotRemoveAforementioned = "Did not remove the datasets." 

221pruneDatasets_didRemoveMsg = "Removed the following datasets:" 

222pruneDatasets_noDatasetsFound = "Did not find any datasets." 

223pruneDatasets_errPurgeAndDisassociate = unwrap( 

224 """"--disassociate and --purge may not be used together: --disassociate purges from just the passed TAGged 

225 collections, but --purge forces disassociation from all of them. """ 

226) 

227pruneDatasets_errQuietWithDryRun = "Can not use --quiet and --dry-run together." 

228pruneDatasets_errNoCollectionRestriction = unwrap( 

229 """Must indicate collections from which to prune datasets by passing COLLECTION arguments (select all 

230 collections by passing '*', or consider using 'butler prune-collections'), by using --purge to pass a run 

231 collection, or by using --disassociate to select a tagged collection.""" 

232) 

233pruneDatasets_errPruneOnNotRun = "Can not prune a collection that is not a RUN collection: {collection}" 

234pruneDatasets_errNoOp = "No operation: one of --purge, --unstore, or --disassociate must be provided." 

235 

236disassociate_option = MWOptionDecorator( 

237 "--disassociate", 

238 "disassociate_tags", 

239 help=unwrap( 

240 """Disassociate pruned datasets from the given tagged collections. May not be used with 

241 --purge.""" 

242 ), 

243 multiple=True, 

244 callback=split_commas, 

245 metavar="TAG", 

246) 

247 

248 

249purge_option = MWOptionDecorator( 

250 "--purge", 

251 "purge_run", 

252 help=unwrap( 

253 """Completely remove the dataset from the given RUN in the Registry. May not be used with 

254 --disassociate. Implies --unstore. Note, this may remove provenance information from 

255 datasets other than those provided, and should be used with extreme care. 

256 RUN has to be provided for backward compatibility, but is used only if COLLECTIONS is 

257 not provided. Otherwise, datasets will be removed from 

258 any RUN-type collections in COLLECTIONS.""" 

259 ), 

260 metavar="RUN", 

261) 

262 

263 

264find_all_option = MWOptionDecorator( 

265 "--find-all", 

266 is_flag=True, 

267 help=unwrap( 

268 """Purge the dataset results from all of the collections in which a dataset of that dataset 

269 type + data id combination appear. (By default only the first found dataset type + data id is 

270 purged, according to the order of COLLECTIONS passed in).""" 

271 ), 

272) 

273 

274 

275unstore_option = MWOptionDecorator( 

276 "--unstore", 

277 is_flag=True, 

278 help=unwrap( 

279 """Remove these datasets from all datastores configured with this data repository. If 

280 --disassociate and --purge are not used then --unstore will be used by default. Note that 

281 --unstore will make it impossible to retrieve these datasets even via other collections. 

282 Datasets that are already not stored are ignored by this option.""" 

283 ), 

284) 

285 

286 

287dry_run_option = MWOptionDecorator( 

288 "--dry-run", 

289 is_flag=True, 

290 help=unwrap( 

291 """Display the datasets that would be removed but do not remove them. 

292 

293 Note that a dataset can be in collections other than its RUN-type collection, and removing it 

294 will remove it from all of them, even though the only one this will show is its RUN 

295 collection.""" 

296 ), 

297) 

298 

299 

300quiet_option = MWOptionDecorator( 

301 "--quiet", 

302 is_flag=True, 

303 help=unwrap("""Makes output quiet. Implies --no-confirm. Requires --dry-run not be passed."""), 

304) 

305 

306 

307@click.command(cls=ButlerCommand, short_help="Remove datasets.") 

308@repo_argument(required=True) 

309@collections_argument( 

310 help=unwrap( 

311 """COLLECTIONS is or more expressions that identify the collections to 

312 search for datasets. Glob-style expressions may be used but only if the 

313 --find-all flag is also passed.""" 

314 ) 

315) 

316@option_section("Query Datasets Options:") 

317@datasets_option( 

318 help="One or more glob-style expressions that identify the dataset types to be pruned.", 

319 multiple=True, 

320 callback=split_commas, 

321) 

322@find_all_option() 

323@where_option(help=where_help) 

324@option_section("Prune Options:") 

325@disassociate_option() 

326@purge_option() 

327@unstore_option() 

328@option_section("Execution Options:") 

329@dry_run_option() 

330@confirm_option() 

331@quiet_option() 

332@option_section("Other Options:") 

333@options_file_option() 

334def prune_datasets(**kwargs: Any) -> None: 

335 """Query for and remove one or more datasets from a collection and/or 

336 storage. 

337 """ 

338 quiet = kwargs.pop("quiet", False) 

339 if quiet: 

340 if kwargs["dry_run"]: 

341 raise click.ClickException(message=pruneDatasets_errQuietWithDryRun) 

342 kwargs["confirm"] = False 

343 

344 result = script.pruneDatasets(**kwargs) 

345 

346 if result.errPurgeAndDisassociate: 

347 raise click.ClickException(message=pruneDatasets_errPurgeAndDisassociate) 

348 if result.errNoCollectionRestriction: 

349 raise click.ClickException(message=pruneDatasets_errNoCollectionRestriction) 

350 if result.errPruneOnNotRun: 

351 raise click.ClickException(message=pruneDatasets_errPruneOnNotRun.format(**result.errDict)) 

352 if result.errNoOp: 

353 raise click.ClickException(message=pruneDatasets_errNoOp) 

354 if result.dryRun: 

355 assert result.action is not None, "Dry run results have not been set up properly." 

356 if result.action["disassociate"] and result.action["unstore"]: 

357 msg = pruneDatasets_wouldDisassociateAndRemoveMsg 

358 elif result.action["disassociate"]: 

359 msg = pruneDatasets_wouldDisassociateMsg 

360 else: 

361 msg = pruneDatasets_wouldRemoveMsg 

362 print(msg.format(**result.action)) 

363 printAstropyTables(result.tables) 

364 return 

365 if result.confirm: 

366 if not result.tables: 

367 print(pruneDatasets_noDatasetsFound) 

368 return 

369 print(pruneDatasets_willRemoveMsg) 

370 printAstropyTables(result.tables) 

371 doContinue = click.confirm(text=pruneDatasets_askContinueMsg, default=False) 

372 if doContinue: 

373 if result.onConfirmation: 

374 result.onConfirmation() 

375 print(pruneDatasets_didRemoveAforementioned) 

376 else: 

377 print(pruneDatasets_didNotRemoveAforementioned) 

378 return 

379 if result.finished: 

380 if not quiet: 

381 print(pruneDatasets_didRemoveMsg) 

382 printAstropyTables(result.tables) 

383 return 

384 

385 

386@click.command(short_help="Search for collections.", cls=ButlerCommand) 

387@repo_argument(required=True) 

388@glob_argument( 

389 help="GLOB is one or more glob-style expressions that fully or partially identify the " 

390 "collections to return." 

391) 

392@collection_type_option() 

393@click.option( 

394 "--chains", 

395 default="TREE", 

396 help="""Affects how results are presented: 

397 

398 TABLE lists each dataset in table form, with columns for dataset name 

399 and type, and a column that lists children of CHAINED datasets (if any 

400 CHAINED datasets are found). 

401 

402 INVERSE-TABLE is like TABLE but instead of a column listing CHAINED 

403 dataset children, it lists the parents of the dataset if it is contained 

404 in any CHAINED collections. 

405 

406 TREE recursively lists children below each CHAINED dataset in tree form. 

407 

408 INVERSE-TREE recursively lists parent datasets below each dataset in 

409 tree form. 

410 

411 FLATTEN lists all datasets, including child datasets, in one list. 

412 

413 [default: TREE]""", 

414 # above, the default value is included, instead of using show_default, so 

415 # that the default is printed on its own line instead of coming right after 

416 # the FLATTEN text. 

417 callback=to_upper, 

418 type=click.Choice( 

419 choices=("TABLE", "INVERSE-TABLE", "TREE", "INVERSE-TREE", "FLATTEN"), 

420 case_sensitive=False, 

421 ), 

422) 

423@options_file_option() 

424def query_collections(*args: Any, **kwargs: Any) -> None: 

425 """Get the collections whose names match an expression.""" 

426 table = script.queryCollections(*args, **kwargs) 

427 # The unit test that mocks script.queryCollections does not return a table 

428 # so we need the following `if`. 

429 if table: 

430 # When chains==TREE, the children of chained datasets are indented 

431 # relative to their parents. For this to work properly the table must 

432 # be left-aligned. 

433 table.pprint_all(align="<") 

434 

435 

436@click.command(cls=ButlerCommand) 

437@repo_argument(required=True) 

438@glob_argument( 

439 help="GLOB is one or more glob-style expressions that fully or partially identify the " 

440 "dataset types to return." 

441) 

442@verbose_option(help="Include dataset type name, dimensions, and storage class in output.") 

443@components_option() 

444@options_file_option() 

445def query_dataset_types(*args: Any, **kwargs: Any) -> None: 

446 """Get the dataset types in a repository.""" 

447 table = script.queryDatasetTypes(*args, **kwargs) 

448 if table: 

449 table.pprint_all() 

450 else: 

451 print("No results. Try --help for more information.") 

452 

453 

454@click.command(cls=ButlerCommand) 

455@repo_argument(required=True) 

456@click.argument("dataset-type-name", nargs=-1) 

457def remove_dataset_type(*args: Any, **kwargs: Any) -> None: 

458 """Remove the dataset type definitions from a repository.""" 

459 script.removeDatasetType(*args, **kwargs) 

460 

461 

462@click.command(cls=ButlerCommand) 

463@query_datasets_options() 

464@options_file_option() 

465def query_datasets(**kwargs: Any) -> None: 

466 """List the datasets in a repository.""" 

467 for table in script.QueryDatasets(**kwargs).getTables(): 

468 print("") 

469 table.pprint_all() 

470 print("") 

471 

472 

473@click.command(cls=ButlerCommand) 

474@repo_argument(required=True) 

475@click.argument("input-collection") 

476@click.argument("output-collection") 

477@click.argument("dataset-type-name") 

478@click.option( 

479 "--begin-date", 

480 type=str, 

481 default=None, 

482 help=unwrap( 

483 """ISO-8601 datetime (TAI) of the beginning of the validity range for the 

484 certified calibrations.""" 

485 ), 

486) 

487@click.option( 

488 "--end-date", 

489 type=str, 

490 default=None, 

491 help=unwrap( 

492 """ISO-8601 datetime (TAI) of the end of the validity range for the 

493 certified calibrations.""" 

494 ), 

495) 

496@click.option( 

497 "--search-all-inputs", 

498 is_flag=True, 

499 default=False, 

500 help=unwrap( 

501 """Search all children of the inputCollection if it is a CHAINED collection, 

502 instead of just the most recent one.""" 

503 ), 

504) 

505@options_file_option() 

506def certify_calibrations(*args: Any, **kwargs: Any) -> None: 

507 """Certify calibrations in a repository.""" 

508 script.certifyCalibrations(*args, **kwargs) 

509 

510 

511@click.command(cls=ButlerCommand) 

512@repo_argument(required=True) 

513@dimensions_argument( 

514 help=unwrap( 

515 """DIMENSIONS are the keys of the data IDs to yield, such as exposure, 

516 instrument, or tract. Will be expanded to include any dependencies.""" 

517 ) 

518) 

519@collections_option(help=collections_option.help + " May only be used with --datasets.") 

520@datasets_option( 

521 help=unwrap( 

522 """An expression that fully or partially identifies dataset types that should 

523 constrain the yielded data IDs. For example, including "raw" here would 

524 constrain the yielded "instrument", "exposure", "detector", and 

525 "physical_filter" values to only those for which at least one "raw" dataset 

526 exists in "collections". Requires --collections.""" 

527 ) 

528) 

529@where_option(help=where_help) 

530@order_by_option() 

531@limit_option() 

532@offset_option() 

533@options_file_option() 

534def query_data_ids(**kwargs: Any) -> None: 

535 """List the data IDs in a repository.""" 

536 table, reason = script.queryDataIds(**kwargs) 

537 if table: 

538 table.pprint_all() 

539 else: 

540 if reason: 

541 print(reason) 

542 if not kwargs.get("dimensions") and not kwargs.get("datasets"): 

543 print("No results. Try requesting some dimensions or datasets, see --help for more information.") 

544 else: 

545 print("No results. Try --help for more information.") 

546 

547 

548@click.command(cls=ButlerCommand) 

549@repo_argument(required=True) 

550@element_argument(required=True) 

551@datasets_option( 

552 help=unwrap( 

553 """An expression that fully or partially identifies dataset types that should 

554 constrain the yielded records. May only be used with 

555 --collections.""" 

556 ) 

557) 

558@collections_option(help=collections_option.help + " May only be used with --datasets.") 

559@where_option(help=where_help) 

560@order_by_option() 

561@limit_option() 

562@offset_option() 

563@click.option( 

564 "--no-check", 

565 is_flag=True, 

566 help=unwrap( 

567 """Don't check the query before execution. By default the query is checked before it 

568 executed, this may reject some valid queries that resemble common mistakes.""" 

569 ), 

570) 

571@options_file_option() 

572def query_dimension_records(**kwargs: Any) -> None: 

573 """Query for dimension information.""" 

574 table = script.queryDimensionRecords(**kwargs) 

575 if table: 

576 table.pprint_all() 

577 else: 

578 print("No results. Try --help for more information.") 

579 

580 

581@click.command(cls=ButlerCommand) 

582@repo_argument(required=True) 

583@query_datasets_options(showUri=False, useArguments=False, repo=False) 

584@destination_argument(help="Destination URI of folder to receive file artifacts.") 

585@transfer_option() 

586@verbose_option(help="Report destination location of all transferred artifacts.") 

587@click.option( 

588 "--preserve-path/--no-preserve-path", 

589 is_flag=True, 

590 default=True, 

591 help="Preserve the datastore path to the artifact at the destination.", 

592) 

593@click.option( 

594 "--clobber/--no-clobber", 

595 is_flag=True, 

596 default=False, 

597 help="If clobber, overwrite files if they exist locally.", 

598) 

599@options_file_option() 

600def retrieve_artifacts(**kwargs: Any) -> None: 

601 """Retrieve file artifacts associated with datasets in a repository.""" 

602 verbose = kwargs.pop("verbose") 

603 transferred = script.retrieveArtifacts(**kwargs) 

604 if verbose and transferred: 

605 print(f"Transferred the following to {kwargs['destination']}:") 

606 for uri in transferred: 

607 print(uri) 

608 print() 

609 print(f"Number of artifacts retrieved into destination {kwargs['destination']}: {len(transferred)}") 

610 

611 

612@click.command(cls=ButlerCommand) 

613@click.argument("source", required=True) 

614@click.argument("dest", required=True) 

615@query_datasets_options(showUri=False, useArguments=False, repo=False) 

616@transfer_option() 

617@register_dataset_types_option() 

618@transfer_dimensions_option() 

619@options_file_option() 

620def transfer_datasets(**kwargs: Any) -> None: 

621 """Transfer datasets from a source butler to a destination butler. 

622 

623 SOURCE is a URI to the Butler repository containing the RUN dataset. 

624 

625 DEST is a URI to the Butler repository that will receive copies of the 

626 datasets. 

627 """ 

628 number = script.transferDatasets(**kwargs) 

629 print(f"Number of datasets transferred: {number}") 

630 

631 

632@click.command(cls=ButlerCommand) 

633@repo_argument(required=True) 

634@click.argument("parent", required=True, nargs=1) 

635@click.argument("children", required=False, nargs=-1, callback=split_commas) 

636@click.option( 

637 "--doc", 

638 default="", 

639 help="Documentation string associated with this collection. " 

640 "Only relevant if the collection is newly created.", 

641) 

642@click.option( 

643 "--flatten/--no-flatten", 

644 default=False, 

645 help="If `True` recursively flatten out any nested chained collections in children first.", 

646) 

647@click.option( 

648 "--mode", 

649 type=click.Choice(["redefine", "extend", "remove", "prepend", "pop"]), 

650 default="redefine", 

651 help="Update mode: " 

652 "'redefine': Create new chain or redefine existing chain with the supplied CHILDREN. " 

653 "'remove': Modify existing chain to remove the supplied CHILDREN. " 

654 "'pop': Pop a numbered element off the chain. Defaults to popping " 

655 "the first element (0). ``children`` must be integers if given. " 

656 "'prepend': Modify existing chain to prepend the supplied CHILDREN to the front. " 

657 "'extend': Modify existing chain to extend it with the supplied CHILDREN.", 

658) 

659def collection_chain(**kwargs: Any) -> None: 

660 """Define a collection chain. 

661 

662 PARENT is the name of the chained collection to create or modify. If the 

663 collection already exists the chain associated with it will be updated. 

664 

665 CHILDREN are the collections to be used to modify the chain. The supplied 

666 values will be split on comma. The exact usage depends on the MODE option. 

667 For example, 

668 

669 $ butler collection-chain REPO PARENT child1,child2 child3 

670 

671 will result in three children being included in the chain. 

672 

673 When the MODE is 'pop' the CHILDREN should be integer indices indicating 

674 collections to be removed from the current chain. 

675 MODE 'pop' can take negative integers to indicate removal relative to the 

676 end of the chain, but when doing that '--' must be given to indicate the 

677 end of the options specification. 

678 

679 $ butler collection-chain REPO --mode=pop PARENT -- -1 

680 

681 Will remove the final collection from the chain. 

682 """ 

683 chain = script.collectionChain(**kwargs) 

684 print(f"[{', '.join(chain)}]") 

685 

686 

687@click.command(cls=ButlerCommand) 

688@repo_argument(required=True) 

689@click.argument("dataset_type", required=True) 

690@click.argument("run", required=True) 

691@click.argument("table_file", required=True) 

692@click.option( 

693 "--formatter", 

694 type=str, 

695 help="Fully-qualified python class to use as the Formatter. If not specified the formatter" 

696 " will be determined from the dataset type and datastore configuration.", 

697) 

698@click.option( 

699 "--id-generation-mode", 

700 default="UNIQUE", 

701 help="Mode to use for generating dataset IDs. The default creates a unique ID. Other options" 

702 " are: 'DATAID_TYPE' for creating a reproducible ID from the dataID and dataset type;" 

703 " 'DATAID_TYPE_RUN' for creating a reproducible ID from the dataID, dataset type and run." 

704 " The latter is usually used for 'raw'-type data that will be ingested in multiple." 

705 " repositories.", 

706 callback=to_upper, 

707 type=click.Choice(("UNIQUE", "DATAID_TYPE", "DATAID_TYPE_RUN"), case_sensitive=False), 

708) 

709@click.option( 

710 "--data-id", 

711 type=str, 

712 multiple=True, 

713 callback=split_commas, 

714 help="Keyword=value string with an additional dataId value that is fixed for all ingested" 

715 " files. This can be used to simplify the table file by removing repeated entries that are" 

716 " fixed for all files to be ingested. Multiple key/values can be given either by using" 

717 " comma separation or multiple command line options.", 

718) 

719@click.option( 

720 "--prefix", 

721 type=str, 

722 help="For relative paths in the table file, specify a prefix to use. The default is to" 

723 " use the current working directory.", 

724) 

725@transfer_option() 

726def ingest_files(**kwargs: Any) -> None: 

727 """Ingest files from table file. 

728 

729 DATASET_TYPE is the name of the dataset type to be associated with these 

730 files. This dataset type must already exist and will not be created by 

731 this command. There can only be one dataset type per invocation of this 

732 command. 

733 

734 RUN is the run to use for the file ingest. 

735 

736 TABLE_FILE refers to a file that can be read by astropy.table with 

737 columns of: 

738 

739 file URI, dimension1, dimension2, ..., dimensionN 

740 

741 where the first column is the URI to the file to be ingested and the 

742 remaining columns define the dataId to associate with that file. 

743 The column names should match the dimensions for the specified dataset 

744 type. Relative file URI by default is assumed to be relative to the 

745 current working directory but can be overridden using the ``--prefix`` 

746 option. 

747 

748 This command does not create dimension records and so any records must 

749 be created by other means. This command should not be used to ingest 

750 raw camera exposures. 

751 """ 

752 script.ingest_files(**kwargs) 

753 

754 

755@click.command(cls=ButlerCommand) 

756@repo_argument(required=True) 

757@click.argument("dataset_type", required=True) 

758@click.argument("storage_class", required=True) 

759@click.argument("dimensions", required=False, nargs=-1) 

760@click.option( 

761 "--is-calibration/--no-is-calibration", 

762 is_flag=True, 

763 default=False, 

764 help="Indicate that this dataset type can be part of a calibration collection.", 

765) 

766def register_dataset_type(**kwargs: Any) -> None: 

767 """Register a new dataset type with this butler repository. 

768 

769 DATASET_TYPE is the name of the dataset type. 

770 

771 STORAGE_CLASS is the name of the StorageClass to be associated with 

772 this dataset type. 

773 

774 DIMENSIONS is a list of all the dimensions relevant to this 

775 dataset type. It can be an empty list. 

776 

777 A component dataset type (such as "something.component") is not a 

778 real dataset type and so can not be defined by this command. They are 

779 automatically derived from the composite dataset type when a composite 

780 storage class is specified. 

781 """ 

782 inserted = script.register_dataset_type(**kwargs) 

783 if inserted: 

784 print("Dataset type successfully registered.") 

785 else: 

786 print("Dataset type already existed in identical form.") 

787 

788 

789@click.command(cls=ButlerCommand) 

790@repo_argument(required=True) 

791@directory_argument(required=True, help="DIRECTORY is the folder to receive the exported calibrations.") 

792@collections_argument(help="COLLECTIONS are the collection to export calibrations from.") 

793@dataset_type_option(help="Specific DatasetType(s) to export.", multiple=True) 

794@transfer_option() 

795def export_calibs(*args: Any, **kwargs: Any) -> None: 

796 """Export calibrations from the butler for import elsewhere.""" 

797 table = script.exportCalibs(*args, **kwargs) 

798 if table: 

799 table.pprint_all(align="<")