Coverage for python/lsst/daf/butler/cli/cmd/commands.py: 77%

289 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-05 11:07 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = () 

30 

31from typing import Any 

32 

33import click 

34 

35from ... import script 

36from ..opt import ( 

37 collection_argument, 

38 collection_type_option, 

39 collections_argument, 

40 collections_option, 

41 components_option, 

42 confirm_option, 

43 dataset_type_option, 

44 datasets_option, 

45 destination_argument, 

46 dimensions_argument, 

47 directory_argument, 

48 element_argument, 

49 glob_argument, 

50 limit_option, 

51 offset_option, 

52 options_file_option, 

53 order_by_option, 

54 query_datasets_options, 

55 register_dataset_types_option, 

56 repo_argument, 

57 transfer_dimensions_option, 

58 transfer_option, 

59 verbose_option, 

60 where_option, 

61) 

62from ..utils import ( 

63 ButlerCommand, 

64 MWOptionDecorator, 

65 option_section, 

66 printAstropyTables, 

67 split_commas, 

68 to_upper, 

69 typeStrAcceptsMultiple, 

70 unwrap, 

71 where_help, 

72) 

73 

74willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist." 

75existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file." 

76 

77 

78@click.command(cls=ButlerCommand, short_help="Add existing datasets to a tagged collection.") 

79@repo_argument(required=True) 

80@collection_argument(help="COLLECTION is the collection the datasets should be associated with.") 

81@query_datasets_options(repo=False, showUri=False, useArguments=False) 

82@options_file_option() 

83def associate(**kwargs: Any) -> None: 

84 """Add existing datasets to a tagged collection; searches for datasets with 

85 the options and adds them to the named COLLECTION. 

86 """ 

87 script.associate(**kwargs) 

88 

89 

90# The conversion from the import command name to the butler_import function 

91# name for subcommand lookup is implemented in the cli/butler.py, in 

92# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they 

93# must be reflected in that location. If this becomes a common pattern a better 

94# mechanism should be implemented. 

95@click.command("import", cls=ButlerCommand) 

96@repo_argument(required=True, help=willCreateRepoHelp) 

97@directory_argument(required=True) 

98@transfer_option() 

99@click.option( 

100 "--export-file", 

101 help="Name for the file that contains database information associated with the exported " 

102 "datasets. If this is not an absolute path, does not exist in the current working " 

103 "directory, and --dir is provided, it is assumed to be in that directory. Defaults " 

104 'to "export.yaml".', 

105 type=str, 

106) 

107@click.option( 

108 "--skip-dimensions", 

109 "-s", 

110 type=str, 

111 multiple=True, 

112 callback=split_commas, 

113 metavar=typeStrAcceptsMultiple, 

114 help="Dimensions that should be skipped during import", 

115) 

116@click.option("--reuse-ids", is_flag=True, help="Force re-use of imported dataset IDs for integer IDs.") 

117@options_file_option() 

118def butler_import(*args: Any, **kwargs: Any) -> None: 

119 """Import data into a butler repository.""" 

120 # `reuse_ids`` is not used by `butlerImport`. 

121 reuse_ids = kwargs.pop("reuse_ids", False) 

122 if reuse_ids: 

123 click.echo("WARNING: --reuse-ids option is deprecated and will be removed after v26.") 

124 

125 script.butlerImport(*args, **kwargs) 

126 

127 

128@click.command(cls=ButlerCommand) 

129@repo_argument(required=True, help=willCreateRepoHelp) 

130@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).") 

131@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.") 

132@click.option( 

133 "--standalone", 

134 is_flag=True, 

135 help="Include all defaults in the config file in the repo, " 

136 "insulating the repo from changes in package defaults.", 

137) 

138@click.option( 

139 "--override", is_flag=True, help="Allow values in the supplied config to override all repo settings." 

140) 

141@click.option( 

142 "--outfile", 

143 "-f", 

144 default=None, 

145 type=str, 

146 help="Name of output file to receive repository " 

147 "configuration. Default is to write butler.yaml into the specified repo.", 

148) 

149@options_file_option() 

150def create(*args: Any, **kwargs: Any) -> None: 

151 """Create an empty Gen3 Butler repository.""" 

152 script.createRepo(*args, **kwargs) 

153 

154 

155@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand) 

156@repo_argument(required=True, help=existingRepoHelp) 

157@click.option( 

158 "--subset", 

159 "-s", 

160 type=str, 

161 help="Subset of a configuration to report. This can be any key in the hierarchy such as " 

162 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.", 

163) 

164@click.option( 

165 "--searchpath", 

166 "-p", 

167 type=str, 

168 multiple=True, 

169 callback=split_commas, 

170 metavar=typeStrAcceptsMultiple, 

171 help="Additional search paths to use for configuration overrides", 

172) 

173@click.option( 

174 "--file", 

175 "outfile", 

176 type=click.File(mode="w"), 

177 default="-", 

178 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout by default.", 

179) 

180@options_file_option() 

181def config_dump(*args: Any, **kwargs: Any) -> None: 

182 """Dump either a subset or full Butler configuration to standard output.""" 

183 script.configDump(*args, **kwargs) 

184 

185 

186@click.command(short_help="Validate the configuration files.", cls=ButlerCommand) 

187@repo_argument(required=True, help=existingRepoHelp) 

188@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.") 

189@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True) 

190@click.option( 

191 "--ignore", 

192 "-i", 

193 type=str, 

194 multiple=True, 

195 callback=split_commas, 

196 metavar=typeStrAcceptsMultiple, 

197 help="DatasetType(s) to ignore for validation.", 

198) 

199@options_file_option() 

200def config_validate(*args: Any, **kwargs: Any) -> None: 

201 """Validate the configuration files for a Gen3 Butler repository.""" 

202 is_good = script.configValidate(*args, **kwargs) 

203 if not is_good: 

204 raise click.exceptions.Exit(1) 

205 

206 

207pruneDatasets_wouldRemoveMsg = unwrap( 

208 """The following datasets will be removed from any datastores in which 

209 they are present:""" 

210) 

211pruneDatasets_wouldDisassociateMsg = unwrap( 

212 """The following datasets will be disassociated from {collections} 

213 if they are currently present in it (which is not checked):""" 

214) 

215pruneDatasets_wouldDisassociateAndRemoveMsg = unwrap( 

216 """The following datasets will be disassociated from 

217 {collections} if they are currently present in it (which is 

218 not checked), and removed from any datastores in which they 

219 are present.""" 

220) 

221pruneDatasets_willRemoveMsg = "The following datasets will be removed:" 

222pruneDatasets_askContinueMsg = "Continue?" 

223pruneDatasets_didRemoveAforementioned = "The datasets were removed." 

224pruneDatasets_didNotRemoveAforementioned = "Did not remove the datasets." 

225pruneDatasets_didRemoveMsg = "Removed the following datasets:" 

226pruneDatasets_noDatasetsFound = "Did not find any datasets." 

227pruneDatasets_errPurgeAndDisassociate = unwrap( 

228 """"--disassociate and --purge may not be used together: --disassociate purges from just the passed TAGged 

229 collections, but --purge forces disassociation from all of them. """ 

230) 

231pruneDatasets_errQuietWithDryRun = "Can not use --quiet and --dry-run together." 

232pruneDatasets_errNoCollectionRestriction = unwrap( 

233 """Must indicate collections from which to prune datasets by passing COLLECTION arguments (select all 

234 collections by passing '*', or consider using 'butler prune-collections'), by using --purge to pass a run 

235 collection, or by using --disassociate to select a tagged collection.""" 

236) 

237pruneDatasets_errPruneOnNotRun = "Can not prune a collection that is not a RUN collection: {collection}" 

238pruneDatasets_errNoOp = "No operation: one of --purge, --unstore, or --disassociate must be provided." 

239 

240disassociate_option = MWOptionDecorator( 

241 "--disassociate", 

242 "disassociate_tags", 

243 help=unwrap( 

244 """Disassociate pruned datasets from the given tagged collections. May not be used with 

245 --purge.""" 

246 ), 

247 multiple=True, 

248 callback=split_commas, 

249 metavar="TAG", 

250) 

251 

252 

253purge_option = MWOptionDecorator( 

254 "--purge", 

255 "purge_run", 

256 help=unwrap( 

257 """Completely remove the dataset from the given RUN in the Registry. May not be used with 

258 --disassociate. Implies --unstore. Note, this may remove provenance information from 

259 datasets other than those provided, and should be used with extreme care. 

260 RUN has to be provided for backward compatibility, but is used only if COLLECTIONS is 

261 not provided. Otherwise, datasets will be removed from 

262 any RUN-type collections in COLLECTIONS.""" 

263 ), 

264 metavar="RUN", 

265) 

266 

267 

268find_all_option = MWOptionDecorator( 

269 "--find-all", 

270 is_flag=True, 

271 help=unwrap( 

272 """Purge the dataset results from all of the collections in which a dataset of that dataset 

273 type + data id combination appear. (By default only the first found dataset type + data id is 

274 purged, according to the order of COLLECTIONS passed in).""" 

275 ), 

276) 

277 

278 

279unstore_option = MWOptionDecorator( 

280 "--unstore", 

281 is_flag=True, 

282 help=unwrap( 

283 """Remove these datasets from all datastores configured with this data repository. If 

284 --disassociate and --purge are not used then --unstore will be used by default. Note that 

285 --unstore will make it impossible to retrieve these datasets even via other collections. 

286 Datasets that are already not stored are ignored by this option.""" 

287 ), 

288) 

289 

290 

291dry_run_option = MWOptionDecorator( 

292 "--dry-run", 

293 is_flag=True, 

294 help=unwrap( 

295 """Display the datasets that would be removed but do not remove them. 

296 

297 Note that a dataset can be in collections other than its RUN-type collection, and removing it 

298 will remove it from all of them, even though the only one this will show is its RUN 

299 collection.""" 

300 ), 

301) 

302 

303 

304quiet_option = MWOptionDecorator( 

305 "--quiet", 

306 is_flag=True, 

307 help=unwrap("""Makes output quiet. Implies --no-confirm. Requires --dry-run not be passed."""), 

308) 

309 

310 

311@click.command(cls=ButlerCommand, short_help="Remove datasets.") 

312@repo_argument(required=True) 

313@collections_argument( 

314 help=unwrap( 

315 """COLLECTIONS is or more expressions that identify the collections to 

316 search for datasets. Glob-style expressions may be used but only if the 

317 --find-all flag is also passed.""" 

318 ) 

319) 

320@option_section("Query Datasets Options:") 

321@datasets_option( 

322 help="One or more glob-style expressions that identify the dataset types to be pruned.", 

323 multiple=True, 

324 callback=split_commas, 

325) 

326@find_all_option() 

327@where_option(help=where_help) 

328@option_section("Prune Options:") 

329@disassociate_option() 

330@purge_option() 

331@unstore_option() 

332@option_section("Execution Options:") 

333@dry_run_option() 

334@confirm_option() 

335@quiet_option() 

336@option_section("Other Options:") 

337@options_file_option() 

338def prune_datasets(**kwargs: Any) -> None: 

339 """Query for and remove one or more datasets from a collection and/or 

340 storage. 

341 """ 

342 quiet = kwargs.pop("quiet", False) 

343 if quiet: 

344 if kwargs["dry_run"]: 

345 raise click.ClickException(message=pruneDatasets_errQuietWithDryRun) 

346 kwargs["confirm"] = False 

347 

348 result = script.pruneDatasets(**kwargs) 

349 

350 if result.errPurgeAndDisassociate: 

351 raise click.ClickException(message=pruneDatasets_errPurgeAndDisassociate) 

352 if result.errNoCollectionRestriction: 

353 raise click.ClickException(message=pruneDatasets_errNoCollectionRestriction) 

354 if result.errPruneOnNotRun: 

355 raise click.ClickException(message=pruneDatasets_errPruneOnNotRun.format(**result.errDict)) 

356 if result.errNoOp: 

357 raise click.ClickException(message=pruneDatasets_errNoOp) 

358 if result.dryRun: 

359 assert result.action is not None, "Dry run results have not been set up properly." 

360 if result.action["disassociate"] and result.action["unstore"]: 

361 msg = pruneDatasets_wouldDisassociateAndRemoveMsg 

362 elif result.action["disassociate"]: 

363 msg = pruneDatasets_wouldDisassociateMsg 

364 else: 

365 msg = pruneDatasets_wouldRemoveMsg 

366 print(msg.format(**result.action)) 

367 printAstropyTables(result.tables) 

368 return 

369 if result.confirm: 

370 if not result.tables: 

371 print(pruneDatasets_noDatasetsFound) 

372 return 

373 print(pruneDatasets_willRemoveMsg) 

374 printAstropyTables(result.tables) 

375 doContinue = click.confirm(text=pruneDatasets_askContinueMsg, default=False) 

376 if doContinue: 

377 if result.onConfirmation: 

378 result.onConfirmation() 

379 print(pruneDatasets_didRemoveAforementioned) 

380 else: 

381 print(pruneDatasets_didNotRemoveAforementioned) 

382 return 

383 if result.finished: 

384 if not quiet: 

385 print(pruneDatasets_didRemoveMsg) 

386 printAstropyTables(result.tables) 

387 return 

388 

389 

390@click.command(short_help="Search for collections.", cls=ButlerCommand) 

391@repo_argument(required=True) 

392@glob_argument( 

393 help="GLOB is one or more glob-style expressions that fully or partially identify the " 

394 "collections to return." 

395) 

396@collection_type_option() 

397@click.option( 

398 "--chains", 

399 default="TREE", 

400 help="""Affects how results are presented: 

401 

402 TABLE lists each dataset in table form, with columns for dataset name 

403 and type, and a column that lists children of CHAINED datasets (if any 

404 CHAINED datasets are found). 

405 

406 INVERSE-TABLE is like TABLE but instead of a column listing CHAINED 

407 dataset children, it lists the parents of the dataset if it is contained 

408 in any CHAINED collections. 

409 

410 TREE recursively lists children below each CHAINED dataset in tree form. 

411 

412 INVERSE-TREE recursively lists parent datasets below each dataset in 

413 tree form. 

414 

415 FLATTEN lists all datasets, including child datasets, in one list. 

416 

417 [default: TREE]""", 

418 # above, the default value is included, instead of using show_default, so 

419 # that the default is printed on its own line instead of coming right after 

420 # the FLATTEN text. 

421 callback=to_upper, 

422 type=click.Choice( 

423 choices=("TABLE", "INVERSE-TABLE", "TREE", "INVERSE-TREE", "FLATTEN"), 

424 case_sensitive=False, 

425 ), 

426) 

427@options_file_option() 

428def query_collections(*args: Any, **kwargs: Any) -> None: 

429 """Get the collections whose names match an expression.""" 

430 table = script.queryCollections(*args, **kwargs) 

431 # The unit test that mocks script.queryCollections does not return a table 

432 # so we need the following `if`. 

433 if table: 

434 # When chains==TREE, the children of chained datasets are indented 

435 # relative to their parents. For this to work properly the table must 

436 # be left-aligned. 

437 table.pprint_all(align="<") 

438 

439 

440@click.command(cls=ButlerCommand) 

441@repo_argument(required=True) 

442@glob_argument( 

443 help="GLOB is one or more glob-style expressions that fully or partially identify the " 

444 "dataset types to return." 

445) 

446@verbose_option(help="Include dataset type name, dimensions, and storage class in output.") 

447@components_option() 

448@options_file_option() 

449def query_dataset_types(*args: Any, **kwargs: Any) -> None: 

450 """Get the dataset types in a repository.""" 

451 table = script.queryDatasetTypes(*args, **kwargs) 

452 if table: 

453 table.pprint_all() 

454 else: 

455 print("No results. Try --help for more information.") 

456 

457 

458@click.command(cls=ButlerCommand) 

459@repo_argument(required=True) 

460@click.argument("dataset-type-name", nargs=-1) 

461def remove_dataset_type(*args: Any, **kwargs: Any) -> None: 

462 """Remove the dataset type definitions from a repository.""" 

463 script.removeDatasetType(*args, **kwargs) 

464 

465 

466@click.command(cls=ButlerCommand) 

467@query_datasets_options() 

468@options_file_option() 

469def query_datasets(**kwargs: Any) -> None: 

470 """List the datasets in a repository.""" 

471 for table in script.QueryDatasets(**kwargs).getTables(): 

472 print("") 

473 table.pprint_all() 

474 print("") 

475 

476 

477@click.command(cls=ButlerCommand) 

478@repo_argument(required=True) 

479@click.argument("input-collection") 

480@click.argument("output-collection") 

481@click.argument("dataset-type-name") 

482@click.option( 

483 "--begin-date", 

484 type=str, 

485 default=None, 

486 help=unwrap( 

487 """ISO-8601 datetime (TAI) of the beginning of the validity range for the 

488 certified calibrations.""" 

489 ), 

490) 

491@click.option( 

492 "--end-date", 

493 type=str, 

494 default=None, 

495 help=unwrap( 

496 """ISO-8601 datetime (TAI) of the end of the validity range for the 

497 certified calibrations.""" 

498 ), 

499) 

500@click.option( 

501 "--search-all-inputs", 

502 is_flag=True, 

503 default=False, 

504 help=unwrap( 

505 """Search all children of the inputCollection if it is a CHAINED collection, 

506 instead of just the most recent one.""" 

507 ), 

508) 

509@options_file_option() 

510def certify_calibrations(*args: Any, **kwargs: Any) -> None: 

511 """Certify calibrations in a repository.""" 

512 script.certifyCalibrations(*args, **kwargs) 

513 

514 

515@click.command(cls=ButlerCommand) 

516@repo_argument(required=True) 

517@dimensions_argument( 

518 help=unwrap( 

519 """DIMENSIONS are the keys of the data IDs to yield, such as exposure, 

520 instrument, or tract. Will be expanded to include any dependencies.""" 

521 ) 

522) 

523@collections_option(help=collections_option.help + " May only be used with --datasets.") 

524@datasets_option( 

525 help=unwrap( 

526 """An expression that fully or partially identifies dataset types that should 

527 constrain the yielded data IDs. For example, including "raw" here would 

528 constrain the yielded "instrument", "exposure", "detector", and 

529 "physical_filter" values to only those for which at least one "raw" dataset 

530 exists in "collections". Requires --collections.""" 

531 ) 

532) 

533@where_option(help=where_help) 

534@order_by_option() 

535@limit_option() 

536@offset_option() 

537@options_file_option() 

538def query_data_ids(**kwargs: Any) -> None: 

539 """List the data IDs in a repository.""" 

540 table, reason = script.queryDataIds(**kwargs) 

541 if table: 

542 table.pprint_all() 

543 else: 

544 if reason: 

545 print(reason) 

546 if not kwargs.get("dimensions") and not kwargs.get("datasets"): 

547 print("No results. Try requesting some dimensions or datasets, see --help for more information.") 

548 else: 

549 print("No results. Try --help for more information.") 

550 

551 

552@click.command(cls=ButlerCommand) 

553@repo_argument(required=True) 

554@element_argument(required=True) 

555@datasets_option( 

556 help=unwrap( 

557 """An expression that fully or partially identifies dataset types that should 

558 constrain the yielded records. May only be used with 

559 --collections.""" 

560 ) 

561) 

562@collections_option(help=collections_option.help + " May only be used with --datasets.") 

563@where_option(help=where_help) 

564@order_by_option() 

565@limit_option() 

566@offset_option() 

567@click.option( 

568 "--no-check", 

569 is_flag=True, 

570 help=unwrap( 

571 """Don't check the query before execution. By default the query is checked before it 

572 executed, this may reject some valid queries that resemble common mistakes.""" 

573 ), 

574) 

575@options_file_option() 

576def query_dimension_records(**kwargs: Any) -> None: 

577 """Query for dimension information.""" 

578 table = script.queryDimensionRecords(**kwargs) 

579 if table: 

580 table.pprint_all() 

581 else: 

582 print("No results. Try --help for more information.") 

583 

584 

585@click.command(cls=ButlerCommand) 

586@repo_argument(required=True) 

587@query_datasets_options(showUri=False, useArguments=False, repo=False) 

588@destination_argument(help="Destination URI of folder to receive file artifacts.") 

589@transfer_option() 

590@verbose_option(help="Report destination location of all transferred artifacts.") 

591@click.option( 

592 "--preserve-path/--no-preserve-path", 

593 is_flag=True, 

594 default=True, 

595 help="Preserve the datastore path to the artifact at the destination.", 

596) 

597@click.option( 

598 "--clobber/--no-clobber", 

599 is_flag=True, 

600 default=False, 

601 help="If clobber, overwrite files if they exist locally.", 

602) 

603@options_file_option() 

604def retrieve_artifacts(**kwargs: Any) -> None: 

605 """Retrieve file artifacts associated with datasets in a repository.""" 

606 verbose = kwargs.pop("verbose") 

607 transferred = script.retrieveArtifacts(**kwargs) 

608 if verbose and transferred: 

609 print(f"Transferred the following to {kwargs['destination']}:") 

610 for uri in transferred: 

611 print(uri) 

612 print() 

613 print(f"Number of artifacts retrieved into destination {kwargs['destination']}: {len(transferred)}") 

614 

615 

616@click.command(cls=ButlerCommand) 

617@click.argument("source", required=True) 

618@click.argument("dest", required=True) 

619@query_datasets_options(showUri=False, useArguments=False, repo=False) 

620@transfer_option() 

621@register_dataset_types_option() 

622@transfer_dimensions_option() 

623@options_file_option() 

624def transfer_datasets(**kwargs: Any) -> None: 

625 """Transfer datasets from a source butler to a destination butler. 

626 

627 SOURCE is a URI to the Butler repository containing the RUN dataset. 

628 

629 DEST is a URI to the Butler repository that will receive copies of the 

630 datasets. 

631 """ 

632 number = script.transferDatasets(**kwargs) 

633 print(f"Number of datasets transferred: {number}") 

634 

635 

636@click.command(cls=ButlerCommand) 

637@repo_argument(required=True) 

638@click.argument("parent", required=True, nargs=1) 

639@click.argument("children", required=False, nargs=-1, callback=split_commas) 

640@click.option( 

641 "--doc", 

642 default="", 

643 help="Documentation string associated with this collection. " 

644 "Only relevant if the collection is newly created.", 

645) 

646@click.option( 

647 "--flatten/--no-flatten", 

648 default=False, 

649 help="If `True` recursively flatten out any nested chained collections in children first.", 

650) 

651@click.option( 

652 "--mode", 

653 type=click.Choice(["redefine", "extend", "remove", "prepend", "pop"]), 

654 default="redefine", 

655 help="Update mode: " 

656 "'redefine': Create new chain or redefine existing chain with the supplied CHILDREN. " 

657 "'remove': Modify existing chain to remove the supplied CHILDREN. " 

658 "'pop': Pop a numbered element off the chain. Defaults to popping " 

659 "the first element (0). ``children`` must be integers if given. " 

660 "'prepend': Modify existing chain to prepend the supplied CHILDREN to the front. " 

661 "'extend': Modify existing chain to extend it with the supplied CHILDREN.", 

662) 

663def collection_chain(**kwargs: Any) -> None: 

664 """Define a collection chain. 

665 

666 PARENT is the name of the chained collection to create or modify. If the 

667 collection already exists the chain associated with it will be updated. 

668 

669 CHILDREN are the collections to be used to modify the chain. The supplied 

670 values will be split on comma. The exact usage depends on the MODE option. 

671 For example, 

672 

673 $ butler collection-chain REPO PARENT child1,child2 child3 

674 

675 will result in three children being included in the chain. 

676 

677 When the MODE is 'pop' the CHILDREN should be integer indices indicating 

678 collections to be removed from the current chain. 

679 MODE 'pop' can take negative integers to indicate removal relative to the 

680 end of the chain, but when doing that '--' must be given to indicate the 

681 end of the options specification. 

682 

683 $ butler collection-chain REPO --mode=pop PARENT -- -1 

684 

685 Will remove the final collection from the chain. 

686 """ 

687 chain = script.collectionChain(**kwargs) 

688 print(f"[{', '.join(chain)}]") 

689 

690 

691@click.command(cls=ButlerCommand) 

692@repo_argument(required=True) 

693@click.argument("dataset_type", required=True) 

694@click.argument("run", required=True) 

695@click.argument("table_file", required=True) 

696@click.option( 

697 "--formatter", 

698 type=str, 

699 help="Fully-qualified python class to use as the Formatter. If not specified the formatter" 

700 " will be determined from the dataset type and datastore configuration.", 

701) 

702@click.option( 

703 "--id-generation-mode", 

704 default="UNIQUE", 

705 help="Mode to use for generating dataset IDs. The default creates a unique ID. Other options" 

706 " are: 'DATAID_TYPE' for creating a reproducible ID from the dataID and dataset type;" 

707 " 'DATAID_TYPE_RUN' for creating a reproducible ID from the dataID, dataset type and run." 

708 " The latter is usually used for 'raw'-type data that will be ingested in multiple." 

709 " repositories.", 

710 callback=to_upper, 

711 type=click.Choice(("UNIQUE", "DATAID_TYPE", "DATAID_TYPE_RUN"), case_sensitive=False), 

712) 

713@click.option( 

714 "--data-id", 

715 type=str, 

716 multiple=True, 

717 callback=split_commas, 

718 help="Keyword=value string with an additional dataId value that is fixed for all ingested" 

719 " files. This can be used to simplify the table file by removing repeated entries that are" 

720 " fixed for all files to be ingested. Multiple key/values can be given either by using" 

721 " comma separation or multiple command line options.", 

722) 

723@click.option( 

724 "--prefix", 

725 type=str, 

726 help="For relative paths in the table file, specify a prefix to use. The default is to" 

727 " use the current working directory.", 

728) 

729@transfer_option() 

730def ingest_files(**kwargs: Any) -> None: 

731 """Ingest files from table file. 

732 

733 DATASET_TYPE is the name of the dataset type to be associated with these 

734 files. This dataset type must already exist and will not be created by 

735 this command. There can only be one dataset type per invocation of this 

736 command. 

737 

738 RUN is the run to use for the file ingest. 

739 

740 TABLE_FILE refers to a file that can be read by astropy.table with 

741 columns of: 

742 

743 file URI, dimension1, dimension2, ..., dimensionN 

744 

745 where the first column is the URI to the file to be ingested and the 

746 remaining columns define the dataId to associate with that file. 

747 The column names should match the dimensions for the specified dataset 

748 type. Relative file URI by default is assumed to be relative to the 

749 current working directory but can be overridden using the ``--prefix`` 

750 option. 

751 

752 This command does not create dimension records and so any records must 

753 be created by other means. This command should not be used to ingest 

754 raw camera exposures. 

755 """ 

756 script.ingest_files(**kwargs) 

757 

758 

759@click.command(cls=ButlerCommand) 

760@repo_argument(required=True) 

761@click.argument("dataset_type", required=True) 

762@click.argument("storage_class", required=True) 

763@click.argument("dimensions", required=False, nargs=-1) 

764@click.option( 

765 "--is-calibration/--no-is-calibration", 

766 is_flag=True, 

767 default=False, 

768 help="Indicate that this dataset type can be part of a calibration collection.", 

769) 

770def register_dataset_type(**kwargs: Any) -> None: 

771 """Register a new dataset type with this butler repository. 

772 

773 DATASET_TYPE is the name of the dataset type. 

774 

775 STORAGE_CLASS is the name of the StorageClass to be associated with 

776 this dataset type. 

777 

778 DIMENSIONS is a list of all the dimensions relevant to this 

779 dataset type. It can be an empty list. 

780 

781 A component dataset type (such as "something.component") is not a 

782 real dataset type and so can not be defined by this command. They are 

783 automatically derived from the composite dataset type when a composite 

784 storage class is specified. 

785 """ 

786 inserted = script.register_dataset_type(**kwargs) 

787 if inserted: 

788 print("Dataset type successfully registered.") 

789 else: 

790 print("Dataset type already existed in identical form.") 

791 

792 

793@click.command(cls=ButlerCommand) 

794@repo_argument(required=True) 

795@directory_argument(required=True, help="DIRECTORY is the folder to receive the exported calibrations.") 

796@collections_argument(help="COLLECTIONS are the collection to export calibrations from.") 

797@dataset_type_option(help="Specific DatasetType(s) to export.", multiple=True) 

798@transfer_option() 

799def export_calibs(*args: Any, **kwargs: Any) -> None: 

800 """Export calibrations from the butler for import elsewhere.""" 

801 table = script.exportCalibs(*args, **kwargs) 

802 if table: 

803 table.pprint_all(align="<")