Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23 

24from ..opt import ( 

25 collection_type_option, 

26 collection_argument, 

27 collections_argument, 

28 collections_option, 

29 components_option, 

30 dataset_type_option, 

31 datasets_option, 

32 dimensions_argument, 

33 directory_argument, 

34 element_argument, 

35 glob_argument, 

36 options_file_option, 

37 repo_argument, 

38 transfer_option, 

39 verbose_option, 

40 where_option, 

41) 

42 

43from ..utils import ( 

44 ButlerCommand, 

45 MWOptionDecorator, 

46 option_section, 

47 printAstropyTables, 

48 split_commas, 

49 to_upper, 

50 typeStrAcceptsMultiple, 

51 unwrap, 

52) 

53 

54from ... import script 

55 

56 

57willCreateRepoHelp = "REPO is the URI or path to the new repository. Will be created if it does not exist." 

58existingRepoHelp = "REPO is the URI or path to an existing data repository root or configuration file." 

59whereHelp = unwrap("""A string expression similar to a SQL WHERE clause. May involve any column of a dimension 

60 table or a dimension name as a shortcut for the primary key column of a dimension 

61 table.""") 

62 

63 

64# The conversion from the import command name to the butler_import function 

65# name for subcommand lookup is implemented in the cli/butler.py, in 

66# funcNameToCmdName and cmdNameToFuncName. If name changes are made here they 

67# must be reflected in that location. If this becomes a common pattern a better 

68# mechanism should be implemented. 

69@click.command("import", cls=ButlerCommand) 

70@repo_argument(required=True, help=willCreateRepoHelp) 

71@directory_argument(required=True) 

72@transfer_option() 

73@click.option("--export-file", 

74 help="Name for the file that contains database information associated with the exported " 

75 "datasets. If this is not an absolute path, does not exist in the current working " 

76 "directory, and --dir is provided, it is assumed to be in that directory. Defaults " 

77 "to \"export.yaml\".", 

78 type=click.File("r")) 

79@click.option("--skip-dimensions", "-s", type=str, multiple=True, callback=split_commas, 

80 metavar=typeStrAcceptsMultiple, 

81 help="Dimensions that should be skipped during import") 

82@options_file_option() 

83def butler_import(*args, **kwargs): 

84 """Import data into a butler repository.""" 

85 script.butlerImport(*args, **kwargs) 

86 

87 

88@click.command(cls=ButlerCommand) 

89@repo_argument(required=True, help=willCreateRepoHelp) 

90@click.option("--seed-config", help="Path to an existing YAML config file to apply (on top of defaults).") 

91@click.option("--dimension-config", help="Path to an existing YAML config file with dimension configuration.") 

92@click.option("--standalone", is_flag=True, help="Include all defaults in the config file in the repo, " 

93 "insulating the repo from changes in package defaults.") 

94@click.option("--override", is_flag=True, help="Allow values in the supplied config to override all " 

95 "repo settings.") 

96@click.option("--outfile", "-f", default=None, type=str, help="Name of output file to receive repository " 

97 "configuration. Default is to write butler.yaml into the specified repo.") 

98@options_file_option() 

99def create(*args, **kwargs): 

100 """Create an empty Gen3 Butler repository.""" 

101 script.createRepo(*args, **kwargs) 

102 

103 

104@click.command(short_help="Dump butler config to stdout.", cls=ButlerCommand) 

105@repo_argument(required=True, help=existingRepoHelp) 

106@click.option("--subset", "-s", type=str, 

107 help="Subset of a configuration to report. This can be any key in the hierarchy such as " 

108 "'.datastore.root' where the leading '.' specified the delimiter for the hierarchy.") 

109@click.option("--searchpath", "-p", type=str, multiple=True, callback=split_commas, 

110 metavar=typeStrAcceptsMultiple, 

111 help="Additional search paths to use for configuration overrides") 

112@click.option("--file", "outfile", type=click.File("w"), default="-", 

113 help="Print the (possibly-expanded) configuration for a repository to a file, or to stdout " 

114 "by default.") 

115@options_file_option() 

116def config_dump(*args, **kwargs): 

117 """Dump either a subset or full Butler configuration to standard output.""" 

118 script.configDump(*args, **kwargs) 

119 

120 

121@click.command(short_help="Validate the configuration files.", cls=ButlerCommand) 

122@repo_argument(required=True, help=existingRepoHelp) 

123@click.option("--quiet", "-q", is_flag=True, help="Do not report individual failures.") 

124@dataset_type_option(help="Specific DatasetType(s) to validate.", multiple=True) 

125@click.option("--ignore", "-i", type=str, multiple=True, callback=split_commas, 

126 metavar=typeStrAcceptsMultiple, 

127 help="DatasetType(s) to ignore for validation.") 

128@options_file_option() 

129def config_validate(*args, **kwargs): 

130 """Validate the configuration files for a Gen3 Butler repository.""" 

131 is_good = script.configValidate(*args, **kwargs) 

132 if not is_good: 

133 raise click.exceptions.Exit(1) 

134 

135 

136@click.command(cls=ButlerCommand) 

137@repo_argument(required=True) 

138@collection_argument(help=unwrap("""COLLECTION is the Name of the collection to remove. If this is a tagged or 

139 chained collection, datasets within the collection are not modified unless --unstore 

140 is passed. If this is a run collection, --purge and --unstore must be passed, and 

141 all datasets in it are fully removed from the data repository.""")) 

142@click.option("--purge", 

143 help=unwrap("""Permit RUN collections to be removed, fully removing datasets within them. 

144 Requires --unstore as an added precaution against accidental deletion. Must not be 

145 passed if the collection is not a RUN."""), 

146 is_flag=True) 

147@click.option("--unstore", 

148 help=("""Remove all datasets in the collection from all datastores in which they appear."""), 

149 is_flag=True) 

150@options_file_option() 

151def prune_collection(**kwargs): 

152 """Remove a collection and possibly prune datasets within it.""" 

153 script.pruneCollection(**kwargs) 

154 

155 

156pruneDatasets_wouldRemoveMsg = unwrap("""The following datasets will be removed from any datastores in which 

157 they are present:""") 

158pruneDatasets_wouldDisassociateMsg = unwrap("""The following datasets will be disassociated from {collections} 

159 if they are currently present in it (which is not checked):""") 

160pruneDatasets_wouldDisassociateAndRemoveMsg = unwrap("""The following datasets will be disassociated from 

161 {collections} if they are currently present in it (which is 

162 not checked), and removed from any datastores in which they 

163 are present.""") 

164pruneDatasets_willRemoveMsg = "The following datasets will be removed:" 

165pruneDatasets_askContinueMsg = "Continue?" 

166pruneDatasets_didRemoveAforementioned = "The datasets were removed." 

167pruneDatasets_didNotRemoveAforementioned = "Did not remove the datasets." 

168pruneDatasets_didRemoveMsg = "Removed the following datasets:" 

169pruneDatasets_noDatasetsFound = "Did not find any datasets." 

170pruneDatasets_errPurgeAndDisassociate = unwrap( 

171 """"--disassociate and --purge may not be used together: --disassociate purges from just the passed TAGged 

172 collections, but --purge forces disassociation from all of them. """ 

173) 

174pruneDatasets_errQuietWithDryRun = "Can not use --quiet and --dry-run together." 

175pruneDatasets_errNoCollectionRestriction = unwrap( 

176 """Must indicate collections from which to prune datasets by passing COLLETION arguments (select all 

177 collections by passing '*', or consider using 'butler prune-collections'), by using --purge to pass a run 

178 collection, or by using --disassociate to select a tagged collection.""") 

179pruneDatasets_errPruneOnNotRun = "Can not prune a collection that is not a RUN collection: {collection}" 

180pruneDatasets_errNoOp = "No operation: one of --purge, --unstore, or --disassociate must be provided." 

181 

182disassociate_option = MWOptionDecorator( 

183 "--disassociate", "disassociate_tags", 

184 help=unwrap("""Disassociate pruned datasets from the given tagged collections. May not be used with 

185 --purge."""), 

186 multiple=True, 

187 callback=split_commas, 

188 metavar="TAG" 

189) 

190 

191 

192purge_option = MWOptionDecorator( 

193 "--purge", "purge_run", 

194 help=unwrap("""Completely remove the dataset from the given RUN in the Registry. May not be used with 

195 --disassociate. Note, this may remove provenance information from datasets other than those 

196 provided, and should be used with extreme care."""), 

197 metavar="RUN" 

198) 

199 

200 

201find_all_option = MWOptionDecorator( 

202 "--find-all", is_flag=True, 

203 help=unwrap("""Purge the dataset results from all of the collections in which a dataset of that dataset 

204 type + data id combination appear. (By default only the first found dataset type + data id is 

205 purged, according to the order of COLLECTIONS passed in).""") 

206) 

207 

208 

209unstore_option = MWOptionDecorator( 

210 "--unstore", 

211 is_flag=True, 

212 help=unwrap("""Remove these datasets from all datastores configured with this data repository. If 

213 --disassociate and --purge are not used then --unstore will be used by default. Note that 

214 --unstore will make it impossible to retrieve these datasets even via other collections. 

215 Datasets that are already not stored are ignored by this option.""") 

216) 

217 

218 

219dry_run_option = MWOptionDecorator( 

220 "--dry-run", 

221 is_flag=True, 

222 help=unwrap("""Display the datasets that would be removed but do not remove them. 

223 

224 Note that a dataset can be in collections other than its RUN-type collection, and removing it 

225 will remove it from all of them, even though the only one this will show is its RUN 

226 collection.""") 

227) 

228 

229 

230confirm_option = MWOptionDecorator( 

231 "--confirm/--no-confirm", 

232 default=True, 

233 help="Print expected action and a confirmation prompt before executing. Default is --confirm." 

234) 

235 

236 

237quiet_option = MWOptionDecorator( 

238 "--quiet", 

239 is_flag=True, 

240 help=unwrap("""Makes output quiet. Implies --no-confirm. Requires --dry-run not be passed.""") 

241) 

242 

243 

244@click.command(cls=ButlerCommand, short_help="Remove datasets.") 

245@repo_argument(required=True) 

246@collections_argument(help=unwrap("""COLLECTIONS is or more expressions that identify the collections to 

247 search for datasets. Glob-style expressions may be used but only if the 

248 --find-all flag is also passed.""")) 

249@option_section("Query Datasets Options:") 

250@datasets_option(help="One or more glob-style expressions that identify the dataset types to be pruned.", 

251 multiple=True, 

252 callback=split_commas) 

253@find_all_option() 

254@where_option(help=whereHelp) 

255@option_section("Prune Options:") 

256@disassociate_option() 

257@purge_option() 

258@unstore_option() 

259@option_section("Execution Options:") 

260@dry_run_option() 

261@confirm_option() 

262@quiet_option() 

263@option_section("Other Options:") 

264@options_file_option() 

265def prune_datasets(**kwargs): 

266 """Query for and remove one or more datasets from a collection and/or 

267 storage. 

268 """ 

269 quiet = kwargs.pop("quiet", False) 

270 if quiet: 

271 if kwargs["dry_run"]: 

272 raise click.ClickException(pruneDatasets_errQuietWithDryRun) 

273 kwargs["confirm"] = False 

274 

275 result = script.pruneDatasets(**kwargs) 

276 

277 if result.errPurgeAndDisassociate: 

278 raise click.ClickException(pruneDatasets_errPurgeAndDisassociate) 

279 return 

280 if result.errNoCollectionRestriction: 

281 raise click.ClickException(pruneDatasets_errNoCollectionRestriction) 

282 if result.errPruneOnNotRun: 

283 raise click.ClickException(pruneDatasets_errPruneOnNotRun.format(**result.errDict)) 

284 if result.errNoOp: 

285 raise click.ClickException(pruneDatasets_errNoOp) 

286 if result.dryRun: 

287 if result.action["disassociate"] and result.action["unstore"]: 

288 msg = pruneDatasets_wouldDisassociateAndRemoveMsg 

289 elif result.action["disassociate"]: 

290 msg = pruneDatasets_wouldDisassociateMsg 

291 else: 

292 msg = pruneDatasets_wouldRemoveMsg 

293 print(msg.format(**result.action)) 

294 printAstropyTables(result.tables) 

295 return 

296 if result.confirm: 

297 if not result.tables: 

298 print(pruneDatasets_noDatasetsFound) 

299 return 

300 print(pruneDatasets_willRemoveMsg) 

301 printAstropyTables(result.tables) 

302 doContinue = click.confirm(pruneDatasets_askContinueMsg, default=False) 

303 if doContinue: 

304 result.onConfirmation() 

305 print(pruneDatasets_didRemoveAforementioned) 

306 else: 

307 print(pruneDatasets_didNotRemoveAforementioned) 

308 return 

309 if result.finished: 

310 if not quiet: 

311 print(pruneDatasets_didRemoveMsg) 

312 printAstropyTables(result.tables) 

313 return 

314 

315 

316@click.command(short_help="Search for collections.", cls=ButlerCommand) 

317@repo_argument(required=True) 

318@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

319 "collections to return.") 

320@collection_type_option() 

321@click.option("--chains", 

322 default="table", 

323 help=unwrap("""Affects how results are presented. TABLE lists each dataset in a row with 

324 chained datasets' children listed in a Definition column. TREE lists children below 

325 their parent in tree form. FLATTEN lists all datasets, including child datasets in 

326 one list.Defaults to TABLE. """), 

327 callback=to_upper, 

328 type=click.Choice(("TABLE", "TREE", "FLATTEN"), case_sensitive=False)) 

329@options_file_option() 

330def query_collections(*args, **kwargs): 

331 """Get the collections whose names match an expression.""" 

332 table = script.queryCollections(*args, **kwargs) 

333 # The unit test that mocks script.queryCollections does not return a table 

334 # so we need the following `if`. 

335 if table: 

336 # When chains==TREE, the children of chained datasets are indented 

337 # relative to their parents. For this to work properly the table must 

338 # be left-aligned. 

339 table.pprint_all(align="<") 

340 

341 

342@click.command(cls=ButlerCommand) 

343@repo_argument(required=True) 

344@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

345 "dataset types to return.") 

346@verbose_option(help="Include dataset type name, dimensions, and storage class in output.") 

347@components_option() 

348@options_file_option() 

349def query_dataset_types(*args, **kwargs): 

350 """Get the dataset types in a repository.""" 

351 table = script.queryDatasetTypes(*args, **kwargs) 

352 if table: 

353 table.pprint_all() 

354 else: 

355 print("No results. Try --help for more information.") 

356 

357 

358@click.command(cls=ButlerCommand) 

359@repo_argument(required=True) 

360@click.argument('dataset-type-name', nargs=1) 

361def remove_dataset_type(*args, **kwargs): 

362 """Remove a dataset type definition from a repository.""" 

363 script.removeDatasetType(*args, **kwargs) 

364 

365 

366@click.command(cls=ButlerCommand) 

367@repo_argument(required=True) 

368@glob_argument(help="GLOB is one or more glob-style expressions that fully or partially identify the " 

369 "dataset types to be queried.") 

370@collections_option() 

371@where_option(help=whereHelp) 

372@click.option("--find-first", 

373 is_flag=True, 

374 help=unwrap("""For each result data ID, only yield one DatasetRef of each DatasetType, from the 

375 first collection in which a dataset of that dataset type appears (according to the 

376 order of 'collections' passed in). If used, 'collections' must specify at least one 

377 expression and must not contain wildcards.""")) 

378@click.option("--show-uri", 

379 is_flag=True, 

380 help="Show the dataset URI in results.") 

381@options_file_option() 

382def query_datasets(**kwargs): 

383 """List the datasets in a repository.""" 

384 for table in script.QueryDatasets(**kwargs).getTables(): 

385 print("") 

386 table.pprint_all() 

387 print("") 

388 

389 

390@click.command(cls=ButlerCommand) 

391@repo_argument(required=True) 

392@click.argument('input-collection') 

393@click.argument('output-collection') 

394@click.argument('dataset-type-name') 

395@click.option("--begin-date", type=str, default=None, 

396 help=unwrap("""ISO-8601 datetime (TAI) of the beginning of the validity range for the 

397 certified calibrations.""")) 

398@click.option("--end-date", type=str, default=None, 

399 help=unwrap("""ISO-8601 datetime (TAI) of the end of the validity range for the 

400 certified calibrations.""")) 

401@click.option("--search-all-inputs", is_flag=True, default=False, 

402 help=unwrap("""Search all children of the inputCollection if it is a CHAINED collection, 

403 instead of just the most recent one.""")) 

404def certify_calibrations(*args, **kwargs): 

405 """Certify calibrations in a repository. 

406 """ 

407 script.certifyCalibrations(*args, **kwargs) 

408 

409 

410@click.command(cls=ButlerCommand) 

411@repo_argument(required=True) 

412@dimensions_argument(help=unwrap("""DIMENSIONS are the keys of the data IDs to yield, such as exposure, 

413 instrument, or tract. Will be expanded to include any dependencies.""")) 

414@collections_option() 

415@datasets_option(help=unwrap("""An expression that fully or partially identifies dataset types that should 

416 constrain the yielded data IDs. For example, including "raw" here would 

417 constrain the yielded "instrument", "exposure", "detector", and 

418 "physical_filter" values to only those for which at least one "raw" dataset 

419 exists in "collections".""")) 

420@where_option(help=whereHelp) 

421@options_file_option() 

422def query_data_ids(**kwargs): 

423 """List the data IDs in a repository. 

424 """ 

425 table = script.queryDataIds(**kwargs) 

426 if table: 

427 table.pprint_all() 

428 else: 

429 if not kwargs.get("dimensions") and not kwargs.get("datasets"): 

430 print("No results. Try requesting some dimensions or datasets, see --help for more information.") 

431 else: 

432 print("No results. Try --help for more information.") 

433 

434 

435@click.command(cls=ButlerCommand) 

436@repo_argument(required=True) 

437@element_argument(required=True) 

438@datasets_option(help=unwrap("""An expression that fully or partially identifies dataset types that should 

439 constrain the yielded records. Only affects results when used with 

440 --collections.""")) 

441@collections_option(help=collections_option.help + " Only affects results when used with --datasets.") 

442@where_option(help=whereHelp) 

443@click.option("--no-check", is_flag=True, 

444 help=unwrap("""Don't check the query before execution. By default the query is checked before it 

445 executed, this may reject some valid queries that resemble common mistakes.""")) 

446@options_file_option() 

447def query_dimension_records(**kwargs): 

448 """Query for dimension information.""" 

449 table = script.queryDimensionRecords(**kwargs) 

450 if table: 

451 table.pprint_all() 

452 else: 

453 print("No results. Try --help for more information.")