Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23 

24from ..opt import ( 

25 collection_type_option, 

26 collection_argument, 

27 collections_argument, 

28 collections_option, 

29 components_option, 

30 dataset_type_option, 

31 datasets_option, 

32 dimensions_argument, 

33 directory_argument, 

34 element_argument, 

35 glob_argument, 

36 options_file_option, 

37 query_datasets_options, 

38 repo_argument, 

39 transfer_option, 

40 verbose_option, 

41 where_option, 

42) 

43 

44from ..utils import ( 

45 ButlerCommand, 

46 MWOptionDecorator, 

47 option_section, 

48 printAstropyTables, 

49 split_commas, 

50 to_upper, 

51 typeStrAcceptsMultiple, 

52 unwrap, 

53 where_help, 

54) 

55 

56from ... import script 

57 

58 

59willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist." 

60existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file." 

61 

62 

63@click.command(cls=ButlerCommand, short_help="Add existing datasets to a tagged collection.") 

64@repo_argument(required=True) 

65@collection_argument(help="COLLECTION is the collection the datasets should be associated with.") 

66@query_datasets_options(repo=False, showUri=False, useArguments=False) 

67@options_file_option() 

68def associate(**kwargs): 

69 """Add existing datasets to a tagged collection; searches for datasets with 

70 the options and adds them to the named COLLECTION. 

71 """ 

72 script.associate(**kwargs) 

73 

74 

75# The conversion from the import command name to the butler_import function 

76# name for subcommand lookup is implemented in the cli/butler.py, in 

77# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they 

78# must be reflected in that location. If this becomes a common pattern a better 

79# mechanism should be implemented. 

80@click.command("import", cls=ButlerCommand) 

81@repo_argument(required=True, help=willCreateRepoHelp) 

82@directory_argument(required=True) 

83@transfer_option() 

84@click.option("--export-file", 

85 help="Name for the file that contains database information associated with the exported " 

86 "datasets. If this is not an absolute path, does not exist in the current working " 

87 "directory, and --dir is provided, it is assumed to be in that directory. Defaults " 

88 "to \"export.yaml\".", 

89 type=click.File("r")) 

90@click.option("--skip-dimensions", "-s", type=str, multiple=True, callback=split_commas, 

91 metavar=typeStrAcceptsMultiple, 

92 help="Dimensions that should be skipped during import") 

93@options_file_option() 

94def butler_import(*args, **kwargs): 

95 """Import data into a butler repository.""" 

96 script.butlerImport(*args, **kwargs) 

97 

98 

99@click.command(cls=ButlerCommand) 

100@repo_argument(required=True, help=willCreateRepoHelp) 

101@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).") 

102@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.") 

103@click.option("--standalone", is_flag=True, help="Include all defaults in the config file in the repo, " 

104 "insulating the repo from changes in package defaults.") 

105@click.option("--override", is_flag=True, help="Allow values in the supplied config to override all " 

106 "repo settings.") 

107@click.option("--outfile", "-f", default=None, type=str, help="Name of output file to receive repository " 

108 "configuration. Default is to write butler.yaml into the specified repo.") 

109@options_file_option() 

110def create(*args, **kwargs): 

111 """Create an empty Gen3 Butler repository.""" 

112 script.createRepo(*args, **kwargs) 

113 

114 

115@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand) 

116@repo_argument(required=True, help=existingRepoHelp) 

117@click.option("--subset", "-s", type=str, 

118 help="Subset of a configuration to report. This can be any key in the hierarchy such as " 

119 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.") 

120@click.option("--searchpath", "-p", type=str, multiple=True, callback=split_commas, 

121 metavar=typeStrAcceptsMultiple, 

122 help="Additional search paths to use for configuration overrides") 

123@click.option("--file", "outfile", type=click.File("w"), default="-", 

124 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout " 

125 "by default.") 

126@options_file_option() 

127def config_dump(*args, **kwargs): 

128 """Dump either a subset or full Butler configuration to standard output.""" 

129 script.configDump(*args, **kwargs) 

130 

131 

132@click.command(short_help="Validate the configuration files.", cls=ButlerCommand) 

133@repo_argument(required=True, help=existingRepoHelp) 

134@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.") 

135@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True) 

136@click.option("--ignore", "-i", type=str, multiple=True, callback=split_commas, 

137 metavar=typeStrAcceptsMultiple, 

138 help="DatasetType(s) to ignore for validation.") 

139@options_file_option() 

140def config_validate(*args, **kwargs): 

141 """Validate the configuration files for a Gen3 Butler repository.""" 

142 is_good = script.configValidate(*args, **kwargs) 

143 if not is_good: 

144 raise click.exceptions.Exit(1) 

145 

146 

147@click.command(cls=ButlerCommand) 

148@repo_argument(required=True) 

149@collection_argument(help=unwrap("""COLLECTION is the Name of the collection to remove. If this is a tagged or 

150 chained collection, datasets within the collection are not modified unless --unstore 

151 is passed. If this is a run collection, --purge and --unstore must be passed, and 

152 all datasets in it are fully removed from the data repository.""")) 

153@click.option("--purge", 

154 help=unwrap("""Permit RUN collections to be removed, fully removing datasets within them. 

155 Requires --unstore as an added precaution against accidental deletion. Must not be 

156 passed if the collection is not a RUN."""), 

157 is_flag=True) 

158@click.option("--unstore", 

159 help=("""Remove all datasets in the collection from all datastores in which they appear."""), 

160 is_flag=True) 

161@options_file_option() 

162def prune_collection(**kwargs): 

163 """Remove a collection and possibly prune datasets within it.""" 

164 script.pruneCollection(**kwargs) 

165 

166 

167pruneDatasets_wouldRemoveMsg = unwrap("""The following datasets will be removed from any datastores in which 

168 they are present:""") 

169pruneDatasets_wouldDisassociateMsg = unwrap("""The following datasets will be disassociated from {collections} 

170 if they are currently present in it (which is not checked):""") 

171pruneDatasets_wouldDisassociateAndRemoveMsg = unwrap("""The following datasets will be disassociated from 

172 {collections} if they are currently present in it (which is 

173 not checked), and removed from any datastores in which they 

174 are present.""") 

175pruneDatasets_willRemoveMsg = "The following datasets will be removed:" 

176pruneDatasets_askContinueMsg = "Continue?" 

177pruneDatasets_didRemoveAforementioned = "The datasets were removed." 

178pruneDatasets_didNotRemoveAforementioned = "Did not remove the datasets." 

179pruneDatasets_didRemoveMsg = "Removed the following datasets:" 

180pruneDatasets_noDatasetsFound = "Did not find any datasets." 

181pruneDatasets_errPurgeAndDisassociate = unwrap( 

182 """"--disassociate and --purge may not be used together: --disassociate purges from just the passed TAGged 

183 collections, but --purge forces disassociation from all of them. """ 

184) 

185pruneDatasets_errQuietWithDryRun = "Can not use --quiet and --dry-run together." 

186pruneDatasets_errNoCollectionRestriction = unwrap( 

187 """Must indicate collections from which to prune datasets by passing COLLETION arguments (select all 

188 collections by passing '*', or consider using 'butler prune-collections'), by using --purge to pass a run 

189 collection, or by using --disassociate to select a tagged collection.""") 

190pruneDatasets_errPruneOnNotRun = "Can not prune a collection that is not a RUN collection: {collection}" 

191pruneDatasets_errNoOp = "No operation: one of --purge, --unstore, or --disassociate must be provided." 

192 

193disassociate_option = MWOptionDecorator( 

194 "--disassociate", "disassociate_tags", 

195 help=unwrap("""Disassociate pruned datasets from the given tagged collections. May not be used with 

196 --purge."""), 

197 multiple=True, 

198 callback=split_commas, 

199 metavar="TAG" 

200) 

201 

202 

203purge_option = MWOptionDecorator( 

204 "--purge", "purge_run", 

205 help=unwrap("""Completely remove the dataset from the given RUN in the Registry. May not be used with 

206 --disassociate. Note, this may remove provenance information from datasets other than those 

207 provided, and should be used with extreme care."""), 

208 metavar="RUN" 

209) 

210 

211 

212find_all_option = MWOptionDecorator( 

213 "--find-all", is_flag=True, 

214 help=unwrap("""Purge the dataset results from all of the collections in which a dataset of that dataset 

215 type + data id combination appear. (By default only the first found dataset type + data id is 

216 purged, according to the order of COLLECTIONS passed in).""") 

217) 

218 

219 

220unstore_option = MWOptionDecorator( 

221 "--unstore", 

222 is_flag=True, 

223 help=unwrap("""Remove these datasets from all datastores configured with this data repository. If 

224 --disassociate and --purge are not used then --unstore will be used by default. Note that 

225 --unstore will make it impossible to retrieve these datasets even via other collections. 

226 Datasets that are already not stored are ignored by this option.""") 

227) 

228 

229 

230dry_run_option = MWOptionDecorator( 

231 "--dry-run", 

232 is_flag=True, 

233 help=unwrap("""Display the datasets that would be removed but do not remove them. 

234 

235 Note that a dataset can be in collections other than its RUN-type collection, and removing it 

236 will remove it from all of them, even though the only one this will show is its RUN 

237 collection.""") 

238) 

239 

240 

241confirm_option = MWOptionDecorator( 

242 "--confirm/--no-confirm", 

243 default=True, 

244 help="Print expected action and a confirmation prompt before executing. Default is --confirm." 

245) 

246 

247 

248quiet_option = MWOptionDecorator( 

249 "--quiet", 

250 is_flag=True, 

251 help=unwrap("""Makes output quiet. Implies --no-confirm. Requires --dry-run not be passed.""") 

252) 

253 

254 

255@click.command(cls=ButlerCommand, short_help="Remove datasets.") 

256@repo_argument(required=True) 

257@collections_argument(help=unwrap("""COLLECTIONS is or more expressions that identify the collections to 

258 search for datasets. Glob-style expressions may be used but only if the 

259 --find-all flag is also passed.""")) 

260@option_section("Query Datasets Options:") 

261@datasets_option(help="One or more glob-style expressions that identify the dataset types to be pruned.", 

262 multiple=True, 

263 callback=split_commas) 

264@find_all_option() 

265@where_option(help=where_help) 

266@option_section("Prune Options:") 

267@disassociate_option() 

268@purge_option() 

269@unstore_option() 

270@option_section("Execution Options:") 

271@dry_run_option() 

272@confirm_option() 

273@quiet_option() 

274@option_section("Other Options:") 

275@options_file_option() 

276def prune_datasets(**kwargs): 

277 """Query for and remove one or more datasets from a collection and/or 

278 storage. 

279 """ 

280 quiet = kwargs.pop("quiet", False) 

281 if quiet: 

282 if kwargs["dry_run"]: 

283 raise click.ClickException(pruneDatasets_errQuietWithDryRun) 

284 kwargs["confirm"] = False 

285 

286 result = script.pruneDatasets(**kwargs) 

287 

288 if result.errPurgeAndDisassociate: 

289 raise click.ClickException(pruneDatasets_errPurgeAndDisassociate) 

290 return 

291 if result.errNoCollectionRestriction: 

292 raise click.ClickException(pruneDatasets_errNoCollectionRestriction) 

293 if result.errPruneOnNotRun: 

294 raise click.ClickException(pruneDatasets_errPruneOnNotRun.format(**result.errDict)) 

295 if result.errNoOp: 

296 raise click.ClickException(pruneDatasets_errNoOp) 

297 if result.dryRun: 

298 if result.action["disassociate"] and result.action["unstore"]: 

299 msg = pruneDatasets_wouldDisassociateAndRemoveMsg 

300 elif result.action["disassociate"]: 

301 msg = pruneDatasets_wouldDisassociateMsg 

302 else: 

303 msg = pruneDatasets_wouldRemoveMsg 

304 print(msg.format(**result.action)) 

305 printAstropyTables(result.tables) 

306 return 

307 if result.confirm: 

308 if not result.tables: 

309 print(pruneDatasets_noDatasetsFound) 

310 return 

311 print(pruneDatasets_willRemoveMsg) 

312 printAstropyTables(result.tables) 

313 doContinue = click.confirm(pruneDatasets_askContinueMsg, default=False) 

314 if doContinue: 

315 result.onConfirmation() 

316 print(pruneDatasets_didRemoveAforementioned) 

317 else: 

318 print(pruneDatasets_didNotRemoveAforementioned) 

319 return 

320 if result.finished: 

321 if not quiet: 

322 print(pruneDatasets_didRemoveMsg) 

323 printAstropyTables(result.tables) 

324 return 

325 

326 

327@click.command(short_help="Search for collections.", cls=ButlerCommand) 

328@repo_argument(required=True) 

329@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

330 "collections to return.") 

331@collection_type_option() 

332@click.option("--chains", 

333 default="table", 

334 help=unwrap("""Affects how results are presented. TABLE lists each dataset in a row with 

335 chained datasets' children listed in a Definition column. TREE lists children below 

336 their parent in tree form. FLATTEN lists all datasets, including child datasets in 

337 one list.Defaults to TABLE. """), 

338 callback=to_upper, 

339 type=click.Choice(("TABLE", "TREE", "FLATTEN"), case_sensitive=False)) 

340@options_file_option() 

341def query_collections(*args, **kwargs): 

342 """Get the collections whose names match an expression.""" 

343 table = script.queryCollections(*args, **kwargs) 

344 # The unit test that mocks script.queryCollections does not return a table 

345 # so we need the following `if`. 

346 if table: 

347 # When chains==TREE, the children of chained datasets are indented 

348 # relative to their parents. For this to work properly the table must 

349 # be left-aligned. 

350 table.pprint_all(align="<") 

351 

352 

353@click.command(cls=ButlerCommand) 

354@repo_argument(required=True) 

355@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

356 "dataset types to return.") 

357@verbose_option(help="Include dataset type name, dimensions, and storage class in output.") 

358@components_option() 

359@options_file_option() 

360def query_dataset_types(*args, **kwargs): 

361 """Get the dataset types in a repository.""" 

362 table = script.queryDatasetTypes(*args, **kwargs) 

363 if table: 

364 table.pprint_all() 

365 else: 

366 print("No results. Try --help for more information.") 

367 

368 

369@click.command(cls=ButlerCommand) 

370@repo_argument(required=True) 

371@click.argument('dataset-type-name', nargs=1) 

372def remove_dataset_type(*args, **kwargs): 

373 """Remove a dataset type definition from a repository.""" 

374 script.removeDatasetType(*args, **kwargs) 

375 

376 

377@click.command(cls=ButlerCommand) 

378@query_datasets_options() 

379@options_file_option() 

380def query_datasets(**kwargs): 

381 """List the datasets in a repository.""" 

382 for table in script.QueryDatasets(**kwargs).getTables(): 

383 print("") 

384 table.pprint_all() 

385 print("") 

386 

387 

388@click.command(cls=ButlerCommand) 

389@repo_argument(required=True) 

390@click.argument('input-collection') 

391@click.argument('output-collection') 

392@click.argument('dataset-type-name') 

393@click.option("--begin-date", type=str, default=None, 

394 help=unwrap("""ISO-8601 datetime (TAI) of the beginning of the validity range for the 

395 certified calibrations.""")) 

396@click.option("--end-date", type=str, default=None, 

397 help=unwrap("""ISO-8601 datetime (TAI) of the end of the validity range for the 

398 certified calibrations.""")) 

399@click.option("--search-all-inputs", is_flag=True, default=False, 

400 help=unwrap("""Search all children of the inputCollection if it is a CHAINED collection, 

401 instead of just the most recent one.""")) 

402@options_file_option() 

403def certify_calibrations(*args, **kwargs): 

404 """Certify calibrations in a repository. 

405 """ 

406 script.certifyCalibrations(*args, **kwargs) 

407 

408 

409@click.command(cls=ButlerCommand) 

410@repo_argument(required=True) 

411@dimensions_argument(help=unwrap("""DIMENSIONS are the keys of the data IDs to yield, such as exposure, 

412 instrument, or tract. Will be expanded to include any dependencies.""")) 

413@collections_option() 

414@datasets_option(help=unwrap("""An expression that fully or partially identifies dataset types that should 

415 constrain the yielded data IDs. For example, including "raw" here would 

416 constrain the yielded "instrument", "exposure", "detector", and 

417 "physical_filter" values to only those for which at least one "raw" dataset 

418 exists in "collections".""")) 

419@where_option(help=where_help) 

420@options_file_option() 

421def query_data_ids(**kwargs): 

422 """List the data IDs in a repository. 

423 """ 

424 table = script.queryDataIds(**kwargs) 

425 if table: 

426 table.pprint_all() 

427 else: 

428 if not kwargs.get("dimensions") and not kwargs.get("datasets"): 

429 print("No results. Try requesting some dimensions or datasets, see --help for more information.") 

430 else: 

431 print("No results. Try --help for more information.") 

432 

433 

434@click.command(cls=ButlerCommand) 

435@repo_argument(required=True) 

436@element_argument(required=True) 

437@datasets_option(help=unwrap("""An expression that fully or partially identifies dataset types that should 

438 constrain the yielded records. Only affects results when used with 

439 --collections.""")) 

440@collections_option(help=collections_option.help + " Only affects results when used with --datasets.") 

441@where_option(help=where_help) 

442@click.option("--no-check", is_flag=True, 

443 help=unwrap("""Don't check the query before execution. By default the query is checked before it 

444 executed, this may reject some valid queries that resemble common mistakes.""")) 

445@options_file_option() 

446def query_dimension_records(**kwargs): 

447 """Query for dimension information.""" 

448 table = script.queryDimensionRecords(**kwargs) 

449 if table: 

450 table.pprint_all() 

451 else: 

452 print("No results. Try --help for more information.")