Coverage for python/lsst/summit/utils/consdbClient.py: 30%

83 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-15 03:28 -0700

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import logging 

23import os 

24from dataclasses import dataclass 

25from typing import Any 

26from urllib.parse import quote 

27 

28import requests 

29from astropy.table import Table # type: ignore 

30 

31__all__ = ["ConsDbClient", "FlexibleMetadataInfo"] 

32 

33 

34logger = logging.getLogger(__name__) 

35 

36 

37def _urljoin(*args: str) -> str: 

38 """Join parts of a URL with slashes. 

39 

40 Does not do any quoting. Mostly to remove a level of list-making. 

41 

42 Parameters 

43 ---------- 

44 *args : `str` 

45 Each parameter is a URL part. 

46 

47 Returns 

48 ------- 

49 url : `str` 

50 The joined URL. 

51 """ 

52 return "/".join(args) 

53 

54 

55@dataclass 

56class FlexibleMetadataInfo: 

57 """Description of a flexible metadata value. 

58 

59 Parameters 

60 ---------- 

61 dtype : `str` 

62 Data type of the flexible metadata value. 

63 One of ``bool``, ``int``, ``float``, or ``str``. 

64 doc : `str` 

65 Documentation string. 

66 unit : `str`, optional 

67 Unit of the value. 

68 ucd : `str`, optional 

69 IVOA Unified Content Descriptor 

70 (https://www.ivoa.net/documents/UCD1+/). 

71 """ 

72 

73 dtype: str 

74 doc: str 

75 unit: str | None = None 

76 ucd: str | None = None 

77 

78 

79class ConsDbClient: 

80 """A client library for accessing the Consolidated Database. 

81 

82 This library provides a basic interface for using flexible metadata 

83 (key/value pairs associated with observation ids from an observation 

84 type table), determining the schema of ConsDB tables, querying the 

85 ConsDB using a general SQL SELECT statement, and inserting into 

86 ConsDB tables. 

87 

88 Parameters 

89 ---------- 

90 url : `str`, optional 

91 Base URL of the Web service, defaults to the value of environment 

92 variable ``LSST_CONSDB_PQ_URL`` (the location of the publish/query 

93 service). 

94 

95 Notes 

96 ----- 

97 This client is a thin layer over the publish/query Web service, which 

98 avoids having a dependency on database drivers. 

99 

100 It enforces the return of query results as Astropy Tables. 

101 """ 

102 

103 def __init__(self, url: str | None = None): 

104 self.session = requests.Session() 

105 if url is None: 

106 self.url = os.environ["LSST_CONSDB_PQ_URL"] 

107 else: 

108 self.url = url 

109 self.url = self.url.rstrip("/") 

110 

111 def _handle_get(self, url: str, query: dict[str, str | list[str]] | None = None) -> Any: 

112 """Submit GET requests to the server. 

113 

114 Parameters 

115 ---------- 

116 url : `str` 

117 URL to GET. 

118 query : `dict` [`str`, `str` | `list` [`str`]], optional 

119 Query parameters to attach to the URL. 

120 

121 Raises 

122 ------ 

123 requests.exceptions.RequestException 

124 Raised if any kind of connection error occurs. 

125 requests.exceptions.HTTPError 

126 Raised if a non-successful status is returned. 

127 requests.exceptions.JSONDecodeError 

128 Raised if the result does not decode as JSON. 

129 

130 Returns 

131 ------- 

132 result : `Any` 

133 Result of decoding the Web service result content as JSON. 

134 """ 

135 logger.debug(f"GET {url}") 

136 response = self.session.get(url, params=query) 

137 response.raise_for_status() 

138 return response.json() 

139 

140 def _handle_post(self, url: str, data: dict[str, Any]) -> requests.Response: 

141 """Submit POST requests to the server. 

142 

143 Parameters 

144 ---------- 

145 url : `str` 

146 URL to POST. 

147 data : `dict` [`str`, `Any`] 

148 Key/value pairs of data to POST. 

149 

150 Raises 

151 ------ 

152 requests.exceptions.RequestException 

153 Raised if any kind of connection error occurs. 

154 requests.exceptions.HTTPError 

155 Raised if a non-successful status is returned. 

156 

157 Returns 

158 ------- 

159 result : `requests.Response` 

160 The raw Web service result object. 

161 """ 

162 logger.debug(f"POST {url}: {data}") 

163 response = self.session.post(url, json=data) 

164 response.raise_for_status() 

165 return response 

166 

167 @staticmethod 

168 def compute_flexible_metadata_table_name(instrument: str, obs_type: str) -> str: 

169 """Compute the name of a flexible metadata table. 

170 

171 Each instrument and observation type made with that instrument can 

172 have a flexible metadata table. This function is useful when 

173 issuing SQL queries, and it avoids a round-trip to the server. 

174 

175 Parameters 

176 ---------- 

177 instrument : `str` 

178 Name of the instrument (e.g. ``LATISS``). 

179 obs_type : `str` 

180 Name of the observation type (e.g. ``Exposure``). 

181 

182 Returns 

183 ------- 

184 table_name : `str` 

185 Name of the appropriate flexible metadata table. 

186 """ 

187 return f"cdb_{instrument}.{obs_type}_flexdata" 

188 

189 def add_flexible_metadata_key( 

190 self, 

191 instrument: str, 

192 obs_type: str, 

193 key: str, 

194 dtype: str, 

195 doc: str, 

196 unit: str = None, 

197 ucd: str = None, 

198 ) -> requests.Response: 

199 """Add a key to a flexible metadata table. 

200 

201 Parameters 

202 ---------- 

203 instrument : `str` 

204 Name of the instrument (e.g. ``LATISS``). 

205 obs_type : `str` 

206 Name of the observation type (e.g. ``Exposure``). 

207 key : `str` 

208 Name of the key to be added (must not already exist). 

209 dtype : `str` 

210 One of ``bool``, ``int``, ``float``, or ``str``. 

211 doc : `str` 

212 Documentation string for the key. 

213 unit : `str`, optional 

214 Unit for the value. Should be from the IVOA 

215 (https://www.ivoa.net/documents/VOUnits/) or astropy. 

216 ucd : `str`, optional 

217 IVOA Unified Content Descriptor 

218 (https://www.ivoa.net/documents/UCD1+/). 

219 

220 Returns 

221 ------- 

222 response : `requests.Response` 

223 HTTP response from the server, with 200 status for success. 

224 

225 Raises 

226 ------ 

227 requests.exceptions.RequestException 

228 Raised if any kind of connection error occurs. 

229 requests.exceptions.HTTPError 

230 Raised if a non-successful status is returned. 

231 """ 

232 data = {"key": key, "dtype": dtype, "doc": doc} 

233 if unit is not None: 

234 data["unit"] = unit 

235 if ucd is not None: 

236 data["ucd"] = ucd 

237 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "addkey") 

238 return self._handle_post(url, data) 

239 

240 def get_flexible_metadata_keys(self, instrument: str, obs_type: str) -> dict[str, FlexibleMetadataInfo]: 

241 """Retrieve the valid keys for a flexible metadata table. 

242 

243 Parameters 

244 ---------- 

245 instrument : `str` 

246 Name of the instrument (e.g. ``LATISS``). 

247 obs_type : `str` 

248 Name of the observation type (e.g. ``Exposure``). 

249 

250 Returns 

251 ------- 

252 key_info : `dict` [ `str`, `FlexibleMetadataInfo` ] 

253 Dict of keys and information values. 

254 

255 Raises 

256 ------ 

257 requests.exceptions.RequestException 

258 Raised if any kind of connection error occurs. 

259 requests.exceptions.HTTPError 

260 Raised if a non-successful status is returned. 

261 """ 

262 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "schema") 

263 result = self._handle_get(url) 

264 return {key: FlexibleMetadataInfo(*value) for key, value in result.items()} 

265 

266 def get_flexible_metadata( 

267 self, instrument: str, obs_type: str, obs_id: int, keys: list[str] | None = None 

268 ) -> dict[str, Any]: 

269 """Get the flexible metadata for an observation. 

270 

271 Parameters 

272 ---------- 

273 instrument : `str` 

274 Name of the instrument (e.g. ``LATISS``). 

275 obs_type : `str` 

276 Name of the observation type (e.g. ``Exposure``). 

277 obs_id : `int` 

278 Unique observation id. 

279 keys : `list` [ `str` ], optional 

280 List of keys to be retrieved; all if not specified. 

281 

282 Returns 

283 ------- 

284 result_dict : `dict` [ `str`, `Any` ] 

285 Dictionary of key/value pairs for the observation. 

286 

287 Raises 

288 ------ 

289 requests.exceptions.RequestException 

290 Raised if any kind of connection error occurs. 

291 requests.exceptions.HTTPError 

292 Raised if a non-successful status is returned. 

293 """ 

294 url = _urljoin( 

295 self.url, 

296 "flex", 

297 quote(instrument), 

298 quote(obs_type), 

299 "obs", 

300 quote(str(obs_id)), 

301 ) 

302 return self._handle_get(url, {"k": keys} if keys else None) 

303 

304 def insert_flexible_metadata( 

305 self, 

306 instrument: str, 

307 obs_type: str, 

308 obs_id: int, 

309 values: dict[str, Any] | None = None, 

310 *, 

311 allow_update: bool = False, 

312 **kwargs, 

313 ) -> requests.Response: 

314 """Set flexible metadata values for an observation. 

315 

316 Parameters 

317 ---------- 

318 instrument : `str` 

319 Name of the instrument (e.g. ``LATISS``). 

320 obs_type : `str` 

321 Name of the observation type (e.g. ``Exposure``). 

322 obs_id : `int` 

323 Unique observation id. 

324 values : `dict` [ `str`, `Any` ], optional 

325 Dictionary of key/value pairs to add for the observation. 

326 allow_update : `bool`, optional 

327 If ``True``, allow replacement of values of existing keys. 

328 **kwargs : `dict` 

329 Additional key/value pairs, overriding ``values``. 

330 

331 Returns 

332 ------- 

333 response : `requests.Response` 

334 HTTP response from the server, with 200 status for success. 

335 

336 Raises 

337 ------ 

338 ValueError 

339 Raised if no values are provided in ``values`` or kwargs. 

340 requests.exceptions.RequestException 

341 Raised if any kind of connection error occurs. 

342 requests.exceptions.HTTPError 

343 Raised if a non-successful status is returned. 

344 """ 

345 if values: 

346 values.update(kwargs) 

347 else: 

348 values = kwargs 

349 if not values: 

350 raise ValueError(f"No values to set for {instrument} {obs_type} {obs_id}") 

351 data = {"values": values} 

352 url = _urljoin( 

353 self.url, 

354 "flex", 

355 quote(instrument), 

356 quote(obs_type), 

357 "obs", 

358 quote(str(obs_id)), 

359 ) 

360 if allow_update: 

361 url += "?u=1" 

362 return self._handle_post(url, data) 

363 

364 def insert( 

365 self, 

366 instrument: str, 

367 table: str, 

368 obs_id: int, 

369 values: dict[str, Any], 

370 *, 

371 allow_update=False, 

372 **kwargs, 

373 ) -> requests.Response: 

374 """Insert values into a single ConsDB fixed metadata table. 

375 

376 Parameters 

377 ---------- 

378 instrument : `str` 

379 Name of the instrument (e.g. ``LATISS``). 

380 table : `str` 

381 Name of the table to insert into. 

382 obs_id : `int` 

383 Unique observation id. 

384 values : `dict` [ `str`, `Any` ], optional 

385 Dictionary of column/value pairs to add for the observation. 

386 allow_update : `bool`, optional 

387 If ``True``, allow replacement of values of existing columns. 

388 **kwargs : `dict` 

389 Additional column/value pairs, overriding ``values``. 

390 

391 Returns 

392 ------- 

393 response : `requests.Response` 

394 HTTP response from the server, with 200 status for success. 

395 

396 Raises 

397 ------ 

398 ValueError 

399 Raised if no values are provided in ``values`` or kwargs. 

400 requests.exceptions.RequestException 

401 Raised if any kind of connection error occurs. 

402 requests.exceptions.HTTPError 

403 Raised if a non-successful status is returned. 

404 """ 

405 if values: 

406 values.update(kwargs) 

407 else: 

408 values = kwargs 

409 if not values: 

410 raise ValueError(f"No values to insert for {instrument} {table} {obs_id}") 

411 data = {"table": table, "obs_id": obs_id, "values": values} 

412 url = _urljoin( 

413 self.url, 

414 "insert", 

415 quote(instrument), 

416 quote(table), 

417 "obs", 

418 quote(str(obs_id)), 

419 ) 

420 if allow_update: 

421 url += "?u=1" 

422 return self._handle_post(url, data) 

423 

424 def query(self, query: str) -> Table: 

425 """Query the ConsDB database. 

426 

427 Parameters 

428 ---------- 

429 query : `str` 

430 A SQL query (currently) to the database. 

431 

432 Returns 

433 ------- 

434 result : `Table` 

435 An ``astropy.Table`` containing the query results. 

436 

437 Raises 

438 ------ 

439 requests.exceptions.RequestException 

440 Raised if any kind of connection error occurs. 

441 requests.exceptions.HTTPError 

442 Raised if a non-successful status is returned. 

443 

444 Notes 

445 ----- 

446 This is a very general query interface because it is expected that 

447 a wide variety of types of queries will be needed. If some types prove 

448 to be common, syntactic sugar could be added to make them simpler. 

449 """ 

450 url = _urljoin(self.url, "query") 

451 data = {"query": query} 

452 result = self._handle_post(url, data).json() 

453 return Table(rows=result["data"], names=result["columns"]) 

454 

455 def schema(self, instrument: str, table: str) -> dict[str, tuple[str, str]]: 

456 """Retrieve the schema of a fixed metadata table in ConsDB. 

457 

458 Parameters 

459 ---------- 

460 instrument : `str` 

461 Name of the instrument (e.g. ``LATISS``). 

462 table : `str` 

463 Name of the table to insert into. 

464 

465 Returns 

466 ------- 

467 column_dict : `dict` [ `str`, `tuple` [ `str`, `str` ] ] 

468 Dict of columns. Values are tuples containing a data type string 

469 and a documentation string. 

470 

471 Raises 

472 ------ 

473 requests.exceptions.RequestException 

474 Raised if any kind of connection error occurs. 

475 requests.exceptions.HTTPError 

476 Raised if a non-successful status is returned. 

477 

478 Notes 

479 ----- 

480 Fixed metadata data types may use the full database vocabulary, 

481 unlike flexible metadata data types. 

482 """ 

483 url = _urljoin(self.url, "schema", quote(instrument), quote(table)) 

484 result = self._handle_get(url) 

485 return {key: tuple(value) for key, value in result.items()}