Coverage for python / lsst / dax / apdb / cli / apdb_cli.py: 19%

1# This file is part of dax_apdb 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["main"] 

25 

26import argparse 

27from collections.abc import Sequence 

28 

29from .. import scripts 

30from . import options 

31from .logging_cli import LoggingCli 

32 

33 

34def main(args: Sequence[str] | None = None) -> int | None: 

35 """APDB command line tools.""" 

36 parser = argparse.ArgumentParser(description="APDB command line tools") 

37 log_cli = LoggingCli(parser) 

38 

39 subparsers = parser.add_subparsers(title="available subcommands", required=True) 

40 _create_sql_subcommand(subparsers) 

41 _create_cassandra_subcommand(subparsers) 

42 _list_cassandra_subcommand(subparsers) 

43 _delete_cassandra_subcommand(subparsers) 

44 _list_index_subcommand(subparsers) 

45 _metadata_subcommand(subparsers) 

46 _convert_legacy_config_subcommand(subparsers) 

47 _metrics_subcommand(subparsers) 

48 _replication_subcommand(subparsers) 

49 _partition_subcommand(subparsers) 

50 

51 parsed_args = parser.parse_args(args) 

52 log_cli.process_args(parsed_args) 

53 

54 kwargs = vars(parsed_args) 

55 # Strip keywords not understood by scripts. 

56 method = kwargs.pop("method") 

57 return method(**kwargs) 

58 

59 

60def _create_sql_subcommand(subparsers: argparse._SubParsersAction) -> None: 

61 parser = subparsers.add_parser("create-sql", help="Create new APDB instance in SQL database.") 

62 parser.add_argument("db_url", help="Database URL in SQLAlchemy format for APDB instance.") 

63 parser.add_argument("output_config", help="Name of the new configuration file for created APDB instance.") 

64 options.common_apdb_options(parser) 

65 options.sql_config_options(parser) 

66 parser.add_argument( 

67 "--drop", help="If True then drop existing tables.", default=False, action="store_true" 

68 ) 

69 parser.set_defaults(method=scripts.create_sql) 

70 

71 

72def _create_cassandra_subcommand(subparsers: argparse._SubParsersAction) -> None: 

73 parser = subparsers.add_parser("create-cassandra", help="Create new APDB instance in Cassandra cluster.") 

74 parser.add_argument("host", help="One or more host names for Cassandra cluster.", nargs="+") 

75 parser.add_argument( 

76 "keyspace", help="Cassandra keyspace name for APDB tables, will be created if does not exist." 

77 ) 

78 parser.add_argument("output_config", help="Name of the new configuration file for created APDB instance.") 

79 options.common_apdb_options(parser) 

80 options.cassandra_config_options(parser) 

81 parser.add_argument( 

82 "--drop", help="If True then drop existing tables.", default=False, action="store_true" 

83 ) 

84 parser.set_defaults(method=scripts.create_cassandra) 

85 

86 

87def _list_cassandra_subcommand(subparsers: argparse._SubParsersAction) -> None: 

88 parser = subparsers.add_parser("list-cassandra", help="List APDB instances in Cassandra cluster.") 

89 parser.add_argument("host", help="One of the host names for Cassandra cluster.") 

90 parser.add_argument( 

91 "-v", 

92 "--verbose", 

93 help="Provide full list of roles and associated permissions.", 

94 default=False, 

95 action="store_true", 

96 ) 

97 parser.set_defaults(method=scripts.list_cassandra) 

98 

99 

100def _delete_cassandra_subcommand(subparsers: argparse._SubParsersAction) -> None: 

101 parser = subparsers.add_parser("delete-cassandra", help="Delete APDB instance from Cassandra cluster.") 

102 parser.add_argument("host", help="One of the host names for Cassandra cluster.") 

103 parser.add_argument("keyspace", help="Cassandra keyspace name for APDB tables.") 

104 parser.add_argument( 

105 "-y", 

106 "--confirm", 

107 help="Assume 'yes' answer for confirmation.", 

108 default=False, 

109 action="store_true", 

110 ) 

111 parser.set_defaults(method=scripts.delete_cassandra) 

112 

113 

114def _list_index_subcommand(subparsers: argparse._SubParsersAction) -> None: 

115 parser = subparsers.add_parser("list-index", help="List contents of APDB index file.") 

116 parser.add_argument( 

117 "index_path", help="Location of index file, if missing then $DAX_APDB_INDEX_URI is used.", nargs="?" 

118 ) 

119 parser.set_defaults(method=scripts.list_index) 

120 

121 

122def _metadata_subcommand(subparsers: argparse._SubParsersAction) -> None: 

123 parser = subparsers.add_parser("metadata", help="Operations with APDB metadata table.") 

124 subparsers = parser.add_subparsers(title="available subcommands", required=True) 

125 _metadata_set_subcommand(subparsers) 

126 _metadata_get_subcommand(subparsers) 

127 _metadata_show_subcommand(subparsers) 

128 _metadata_delete_subcommand(subparsers) 

129 

130 

131def _metadata_show_subcommand(subparsers: argparse._SubParsersAction) -> None: 

132 parser = subparsers.add_parser("show", help="Show contents of APDB metadata table.") 

133 parser.add_argument( 

134 "-j", 

135 "--json", 

136 dest="use_json", 

137 help="Dump metadata in JSON format.", 

138 default=False, 

139 action="store_true", 

140 ) 

141 parser.add_argument("config", help="Path or URI of APDB configuration file.") 

142 parser.set_defaults(method=scripts.metadata_show) 

143 

144 

145def _metadata_get_subcommand(subparsers: argparse._SubParsersAction) -> None: 

146 parser = subparsers.add_parser("get", help="Print value of the metadata item.") 

147 parser.add_argument("config", help="Path or URI of APDB configuration file.") 

148 parser.add_argument("key", help="Metadata key, arbitrary string.") 

149 parser.set_defaults(method=scripts.metadata_get) 

150 

151 

152def _metadata_set_subcommand(subparsers: argparse._SubParsersAction) -> None: 

153 parser = subparsers.add_parser("set", help="Add or update metadata item.") 

154 parser.add_argument( 

155 "-f", 

156 "--force", 

157 help="Force update of the existing key.", 

158 default=False, 

159 action="store_true", 

160 ) 

161 parser.add_argument("config", help="Path or URI of APDB configuration file.") 

162 parser.add_argument("key", help="Metadata key, arbitrary string.") 

163 parser.add_argument("value", help="Corresponding metadata value.") 

164 parser.set_defaults(method=scripts.metadata_set) 

165 

166 

167def _metadata_delete_subcommand(subparsers: argparse._SubParsersAction) -> None: 

168 parser = subparsers.add_parser("delete", help="Delete metadata item.") 

169 parser.add_argument("config", help="Path or URI of APDB configuration file.") 

170 parser.add_argument("key", help="Metadata key, arbitrary string.") 

171 parser.set_defaults(method=scripts.metadata_delete) 

172 

173 

174def _convert_legacy_config_subcommand(subparsers: argparse._SubParsersAction) -> None: 

175 parser = subparsers.add_parser("convert-legacy-config", help="Convert legacy pex_config to YAML.") 

176 parser.add_argument("legacy_config", help="Path or URI of APDB legacy configuration file.") 

177 parser.add_argument("new_config", help="Path or URI to write new YAML configuration file.") 

178 parser.set_defaults(method=scripts.convert_legacy_config) 

179 

180 

181def _metrics_subcommand(subparsers: argparse._SubParsersAction) -> None: 

182 parser = subparsers.add_parser("metrics", help="Operations with metrics produced by APDB.") 

183 subparsers = parser.add_subparsers(title="available subcommands", required=True) 

184 _metrics_log_to_influx(subparsers) 

185 

186 

187def _metrics_log_to_influx(subparsers: argparse._SubParsersAction) -> None: 

188 parser = subparsers.add_parser( 

189 "log-to-infux", help="Extract metrics from log files and dump as InfluxDB data." 

190 ) 

191 parser.add_argument( 

192 "file", help="Name(s) of the log file to parse, '-' to read from standard input.", nargs="+" 

193 ) 

194 parser.add_argument( 

195 "-c", 

196 "--context-keys", 

197 help=("Names of keys to extract from message context, comma-separated."), 

198 default="", 

199 metavar="KEY[,KEY,...]", 

200 ) 

201 parser.add_argument( 

202 "-t", 

203 "--extra-tags", 

204 help=( 

205 "Extra tags and their values, comma-separated. " 

206 "If tag name is prefixed with minus it will be removed." 

207 ), 

208 default="", 

209 metavar="TAG=VALUE[,-TAG][,TAG=VALUE...]", 

210 ) 

211 parser.add_argument( 

212 "-m", 

213 "--mode", 

214 help="Source of log file.", 

215 choices=["ap_proto", "pipeline", "replication", "json_line"], 

216 default="ap_proto", 

217 ) 

218 parser.add_argument( 

219 "-p", 

220 "--prefix", 

221 help="Additional prefix for metrics names.", 

222 default="", 

223 ) 

224 parser.add_argument( 

225 "-H", 

226 "--no-header", 

227 help="Do not add DML header.", 

228 action="store_true", 

229 default=False, 

230 ) 

231 parser.add_argument( 

232 "-d", 

233 "--header-database", 

234 help="Database name to use for header, default: %(default)s.", 

235 default="telegraf", 

236 ) 

237 parser.add_argument( 

238 "--fix-row-count", 

239 help="Fix incorrect inserted row counts by parsing additional data from logs.", 

240 action="store_true", 

241 default=False, 

242 ) 

243 parser.set_defaults(method=scripts.metrics_log_to_influx) 

244 

245 

246def _replication_subcommand(subparsers: argparse._SubParsersAction) -> None: 

247 parser = subparsers.add_parser("replication", help="Operations with replication tables produced by APDB.") 

248 subparsers = parser.add_subparsers(title="available subcommands", required=True) 

249 _replication_list_chunks_subcommand(subparsers) 

250 _replication_delete_chunks_subcommand(subparsers) 

251 

252 

253def _replication_list_chunks_subcommand(subparsers: argparse._SubParsersAction) -> None: 

254 parser = subparsers.add_parser("list-chunks", help="Print full list of replication chunks in APDB.") 

255 parser.add_argument("apdb_config", help="Path to the APDB configuration.") 

256 parser.set_defaults(method=scripts.replication_list_chunks) 

257 

258 

259def _replication_delete_chunks_subcommand(subparsers: argparse._SubParsersAction) -> None: 

260 parser = subparsers.add_parser("delete-chunks", help="Delete replication chunks from APDB.") 

261 parser.add_argument("apdb_config", help="Path to the APDB configuration.") 

262 parser.add_argument( 

263 "chunk_id", type=int, help="Chunk ID to delete, all earlier chunks are deleted as well." 

264 ) 

265 parser.add_argument( 

266 "-p", 

267 "--print-only", 

268 help="Only print the list ochunks that will be deleted, but do not delete them.", 

269 action="store_true", 

270 default=False, 

271 ) 

272 parser.add_argument( 

273 "--force", 

274 help="Do not ask for confirmation before deleting chunks.", 

275 action="store_true", 

276 default=False, 

277 ) 

278 parser.set_defaults(method=scripts.replication_delete_chunks) 

279 

280 

281def _partition_subcommand(subparsers: argparse._SubParsersAction) -> None: 

282 parser = subparsers.add_parser("partition", help="Operations with APDB partitioning.") 

283 subparsers = parser.add_subparsers(title="available subcommands", required=True) 

284 _partition_show_temporal(subparsers) 

285 _partition_extend_temporal(subparsers) 

286 _partition_delete_temporal(subparsers) 

287 

288 

289def _partition_show_temporal(subparsers: argparse._SubParsersAction) -> None: 

290 parser = subparsers.add_parser("show-temporal", help="Print range of temporal partitions.") 

291 parser.add_argument("apdb_config", help="Path to the APDB configuration.") 

292 parser.set_defaults(method=scripts.partition_show_temporal) 

293 

294 

295def _partition_extend_temporal(subparsers: argparse._SubParsersAction) -> None: 

296 parser = subparsers.add_parser("extend-temporal", help="Extend the range of temporal partitions.") 

297 parser.add_argument("apdb_config", help="Path to the APDB configuration.") 

298 parser.add_argument("time", help="Timestamps in ISOT format and TAI scale (YYYY-MM-DDTHH:MM:SS).") 

299 parser.add_argument("--past", action="store_true", default=False, help="Extend the range in the past.") 

300 parser.add_argument( 

301 "--max-days", 

302 type=int, 

303 default=365, 

304 metavar="NUMBER", 

305 help="Max. number of days for extension, default: %(default)s.", 

306 ) 

307 parser.set_defaults(method=scripts.partition_extend_temporal) 

308 

309 

310def _partition_delete_temporal(subparsers: argparse._SubParsersAction) -> None: 

311 parser = subparsers.add_parser("delete-temporal", help="Delete temporal partitions.") 

312 parser.add_argument("apdb_config", help="Path to the APDB configuration.") 

313 parser.add_argument("time", help="Timestamps in ISOT format and TAI scale (YYYY-MM-DD[THH:MM:SS]).") 

314 parser.add_argument( 

315 "--after", 

316 action="store_true", 

317 default=False, 

318 help="Delete partitions after specified time. Default is to delete partitions before this time.", 

319 ) 

320 parser.add_argument("--force", action="store_true", default=False, help="Do not ask for confirmation.") 

321 parser.set_defaults(method=scripts.partition_delete_temporal)