Coverage for python/lsst/summit/utils/consdbClient.py: 29%
85 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-17 08:55 +0000
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-17 08:55 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23import os
24from dataclasses import dataclass
25from typing import Any
26from urllib.parse import quote
28import requests
29from astropy.table import Table # type: ignore
31__all__ = ["ConsDbClient", "FlexibleMetadataInfo"]
34logger = logging.getLogger(__name__)
37def _urljoin(*args: str) -> str:
38 """Join parts of a URL with slashes.
40 Does not do any quoting. Mostly to remove a level of list-making.
42 Parameters
43 ----------
44 *args : `str`
45 Each parameter is a URL part.
47 Returns
48 -------
49 url : `str`
50 The joined URL.
51 """
52 return "/".join(args)
55@dataclass
56class FlexibleMetadataInfo:
57 """Description of a flexible metadata value.
59 Parameters
60 ----------
61 dtype : `str`
62 Data type of the flexible metadata value.
63 One of ``bool``, ``int``, ``float``, or ``str``.
64 doc : `str`
65 Documentation string.
66 unit : `str`, optional
67 Unit of the value.
68 ucd : `str`, optional
69 IVOA Unified Content Descriptor
70 (https://www.ivoa.net/documents/UCD1+/).
71 """
73 dtype: str
74 doc: str
75 unit: str | None = None
76 ucd: str | None = None
79class ConsDbClient:
80 """A client library for accessing the Consolidated Database.
82 This library provides a basic interface for using flexible metadata
83 (key/value pairs associated with observation ids from an observation
84 type table), determining the schema of ConsDB tables, querying the
85 ConsDB using a general SQL SELECT statement, and inserting into
86 ConsDB tables.
88 Parameters
89 ----------
90 url : `str`, optional
91 Base URL of the Web service, defaults to the value of environment
92 variable ``LSST_CONSDB_PQ_URL`` (the location of the publish/query
93 service).
95 Notes
96 -----
97 This client is a thin layer over the publish/query Web service, which
98 avoids having a dependency on database drivers.
100 It enforces the return of query results as Astropy Tables.
101 """
103 def __init__(self, url: str | None = None):
104 self.session = requests.Session()
105 if url is None:
106 self.url = os.environ["LSST_CONSDB_PQ_URL"]
107 else:
108 self.url = url
109 self.url = self.url.rstrip("/")
111 def _handle_get(self, url: str, query: dict[str, str | list[str]] | None = None) -> Any:
112 """Submit GET requests to the server.
114 Parameters
115 ----------
116 url : `str`
117 URL to GET.
118 query : `dict` [`str`, `str` | `list` [`str`]], optional
119 Query parameters to attach to the URL.
121 Raises
122 ------
123 requests.exceptions.RequestException
124 Raised if any kind of connection error occurs.
125 requests.exceptions.HTTPError
126 Raised if a non-successful status is returned.
127 requests.exceptions.JSONDecodeError
128 Raised if the result does not decode as JSON.
130 Returns
131 -------
132 result : `Any`
133 Result of decoding the Web service result content as JSON.
134 """
135 logger.debug(f"GET {url}")
136 response = self.session.get(url, params=query)
137 response.raise_for_status()
138 return response.json()
140 def _handle_post(self, url: str, data: dict[str, Any]) -> requests.Response:
141 """Submit POST requests to the server.
143 Parameters
144 ----------
145 url : `str`
146 URL to POST.
147 data : `dict` [`str`, `Any`]
148 Key/value pairs of data to POST.
150 Raises
151 ------
152 requests.exceptions.RequestException
153 Raised if any kind of connection error occurs.
154 requests.exceptions.HTTPError
155 Raised if a non-successful status is returned.
157 Returns
158 -------
159 result : `requests.Response`
160 The raw Web service result object.
161 """
162 logger.debug(f"POST {url}: {data}")
163 response = self.session.post(url, json=data)
164 response.raise_for_status()
165 return response
167 @staticmethod
168 def compute_flexible_metadata_table_name(instrument: str, obs_type: str) -> str:
169 """Compute the name of a flexible metadata table.
171 Each instrument and observation type made with that instrument can
172 have a flexible metadata table. This function is useful when
173 issuing SQL queries, and it avoids a round-trip to the server.
175 Parameters
176 ----------
177 instrument : `str`
178 Name of the instrument (e.g. ``LATISS``).
179 obs_type : `str`
180 Name of the observation type (e.g. ``Exposure``).
182 Returns
183 -------
184 table_name : `str`
185 Name of the appropriate flexible metadata table.
186 """
187 return f"cdb_{instrument}.{obs_type}_flexdata"
189 def add_flexible_metadata_key(
190 self,
191 instrument: str,
192 obs_type: str,
193 key: str,
194 dtype: str,
195 doc: str,
196 unit: str = None,
197 ucd: str = None,
198 ) -> requests.Response:
199 """Add a key to a flexible metadata table.
201 Parameters
202 ----------
203 instrument : `str`
204 Name of the instrument (e.g. ``LATISS``).
205 obs_type : `str`
206 Name of the observation type (e.g. ``Exposure``).
207 key : `str`
208 Name of the key to be added (must not already exist).
209 dtype : `str`
210 One of ``bool``, ``int``, ``float``, or ``str``.
211 doc : `str`
212 Documentation string for the key.
213 unit : `str`, optional
214 Unit for the value. Should be from the IVOA
215 (https://www.ivoa.net/documents/VOUnits/) or astropy.
216 ucd : `str`, optional
217 IVOA Unified Content Descriptor
218 (https://www.ivoa.net/documents/UCD1+/).
220 Returns
221 -------
222 response : `requests.Response`
223 HTTP response from the server, with 200 status for success.
225 Raises
226 ------
227 requests.exceptions.RequestException
228 Raised if any kind of connection error occurs.
229 requests.exceptions.HTTPError
230 Raised if a non-successful status is returned.
231 """
232 data = {"key": key, "dtype": dtype, "doc": doc}
233 if unit is not None:
234 data["unit"] = unit
235 if ucd is not None:
236 data["ucd"] = ucd
237 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "addkey")
238 return self._handle_post(url, data)
240 def get_flexible_metadata_keys(self, instrument: str, obs_type: str) -> dict[str, FlexibleMetadataInfo]:
241 """Retrieve the valid keys for a flexible metadata table.
243 Parameters
244 ----------
245 instrument : `str`
246 Name of the instrument (e.g. ``LATISS``).
247 obs_type : `str`
248 Name of the observation type (e.g. ``Exposure``).
250 Returns
251 -------
252 key_info : `dict` [ `str`, `FlexibleMetadataInfo` ]
253 Dict of keys and information values.
255 Raises
256 ------
257 requests.exceptions.RequestException
258 Raised if any kind of connection error occurs.
259 requests.exceptions.HTTPError
260 Raised if a non-successful status is returned.
261 """
262 url = _urljoin(self.url, "flex", quote(instrument), quote(obs_type), "schema")
263 result = self._handle_get(url)
264 return {key: FlexibleMetadataInfo(*value) for key, value in result.items()}
266 def get_flexible_metadata(
267 self, instrument: str, obs_type: str, obs_id: int, keys: list[str] | None = None
268 ) -> dict[str, Any]:
269 """Get the flexible metadata for an observation.
271 Parameters
272 ----------
273 instrument : `str`
274 Name of the instrument (e.g. ``LATISS``).
275 obs_type : `str`
276 Name of the observation type (e.g. ``Exposure``).
277 obs_id : `int`
278 Unique observation id.
279 keys : `list` [ `str` ], optional
280 List of keys to be retrieved; all if not specified.
282 Returns
283 -------
284 result_dict : `dict` [ `str`, `Any` ]
285 Dictionary of key/value pairs for the observation.
287 Raises
288 ------
289 requests.exceptions.RequestException
290 Raised if any kind of connection error occurs.
291 requests.exceptions.HTTPError
292 Raised if a non-successful status is returned.
293 """
294 url = _urljoin(
295 self.url,
296 "flex",
297 quote(instrument),
298 quote(obs_type),
299 "obs",
300 quote(str(obs_id)),
301 )
302 return self._handle_get(url, {"k": keys} if keys else None)
304 def insert_flexible_metadata(
305 self,
306 instrument: str,
307 obs_type: str,
308 obs_id: int,
309 values: dict[str, Any] | None = None,
310 *,
311 allow_update: bool = False,
312 **kwargs,
313 ) -> requests.Response:
314 """Set flexible metadata values for an observation.
316 Parameters
317 ----------
318 instrument : `str`
319 Name of the instrument (e.g. ``LATISS``).
320 obs_type : `str`
321 Name of the observation type (e.g. ``Exposure``).
322 obs_id : `int`
323 Unique observation id.
324 values : `dict` [ `str`, `Any` ], optional
325 Dictionary of key/value pairs to add for the observation.
326 allow_update : `bool`, optional
327 If ``True``, allow replacement of values of existing keys.
328 **kwargs : `dict`
329 Additional key/value pairs, overriding ``values``.
331 Returns
332 -------
333 response : `requests.Response`
334 HTTP response from the server, with 200 status for success.
336 Raises
337 ------
338 ValueError
339 Raised if no values are provided in ``values`` or kwargs.
340 requests.exceptions.RequestException
341 Raised if any kind of connection error occurs.
342 requests.exceptions.HTTPError
343 Raised if a non-successful status is returned.
344 """
345 if values:
346 values.update(kwargs)
347 else:
348 values = kwargs
349 if not values:
350 raise ValueError(f"No values to set for {instrument} {obs_type} {obs_id}")
351 data = {"values": values}
352 url = _urljoin(
353 self.url,
354 "flex",
355 quote(instrument),
356 quote(obs_type),
357 "obs",
358 quote(str(obs_id)),
359 )
360 if allow_update:
361 url += "?u=1"
362 return self._handle_post(url, data)
364 def insert(
365 self,
366 instrument: str,
367 table: str,
368 obs_id: int,
369 values: dict[str, Any],
370 *,
371 allow_update=False,
372 **kwargs,
373 ) -> requests.Response:
374 """Insert values into a single ConsDB fixed metadata table.
376 Parameters
377 ----------
378 instrument : `str`
379 Name of the instrument (e.g. ``LATISS``).
380 table : `str`
381 Name of the table to insert into.
382 obs_id : `int`
383 Unique observation id.
384 values : `dict` [ `str`, `Any` ], optional
385 Dictionary of column/value pairs to add for the observation.
386 allow_update : `bool`, optional
387 If ``True``, allow replacement of values of existing columns.
388 **kwargs : `dict`
389 Additional column/value pairs, overriding ``values``.
391 Returns
392 -------
393 response : `requests.Response`
394 HTTP response from the server, with 200 status for success.
396 Raises
397 ------
398 ValueError
399 Raised if no values are provided in ``values`` or kwargs.
400 requests.exceptions.RequestException
401 Raised if any kind of connection error occurs.
402 requests.exceptions.HTTPError
403 Raised if a non-successful status is returned.
404 """
405 if values:
406 values.update(kwargs)
407 else:
408 values = kwargs
409 if not values:
410 raise ValueError(f"No values to insert for {instrument} {table} {obs_id}")
411 data = {"table": table, "obs_id": obs_id, "values": values}
412 url = _urljoin(
413 self.url,
414 "insert",
415 quote(instrument),
416 quote(table),
417 "obs",
418 quote(str(obs_id)),
419 )
420 if allow_update:
421 url += "?u=1"
422 return self._handle_post(url, data)
424 def query(self, query: str) -> Table:
425 """Query the ConsDB database.
427 Parameters
428 ----------
429 query : `str`
430 A SQL query (currently) to the database.
432 Returns
433 -------
434 result : `Table`
435 An ``astropy.Table`` containing the query results.
437 Raises
438 ------
439 requests.exceptions.RequestException
440 Raised if any kind of connection error occurs.
441 requests.exceptions.HTTPError
442 Raised if a non-successful status is returned.
444 Notes
445 -----
446 This is a very general query interface because it is expected that
447 a wide variety of types of queries will be needed. If some types prove
448 to be common, syntactic sugar could be added to make them simpler.
449 """
450 url = _urljoin(self.url, "query")
451 data = {"query": query}
452 result = self._handle_post(url, data).json()
453 if "columns" not in result:
454 # No result rows
455 return Table(rows=[])
456 return Table(rows=result["data"], names=result["columns"])
458 def schema(self, instrument: str, table: str) -> dict[str, tuple[str, str]]:
459 """Retrieve the schema of a fixed metadata table in ConsDB.
461 Parameters
462 ----------
463 instrument : `str`
464 Name of the instrument (e.g. ``LATISS``).
465 table : `str`
466 Name of the table to insert into.
468 Returns
469 -------
470 column_dict : `dict` [ `str`, `tuple` [ `str`, `str` ] ]
471 Dict of columns. Values are tuples containing a data type string
472 and a documentation string.
474 Raises
475 ------
476 requests.exceptions.RequestException
477 Raised if any kind of connection error occurs.
478 requests.exceptions.HTTPError
479 Raised if a non-successful status is returned.
481 Notes
482 -----
483 Fixed metadata data types may use the full database vocabulary,
484 unlike flexible metadata data types.
485 """
486 url = _urljoin(self.url, "schema", quote(instrument), quote(table))
487 result = self._handle_get(url)
488 return {key: tuple(value) for key, value in result.items()}