Coverage for python/lsst/summit/utils/packageVersions.py: 93%

1# This file is part of summit_utils.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (https://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This program is free software: you can redistribute it and/or modify

10# it under the terms of the GNU General Public License as published by

11# the Free Software Foundation, either version 3 of the License, or

12# (at your option) any later version.

13#

14# This program is distributed in the hope that it will be useful,

15# but WITHOUT ANY WARRANTY; without even the implied warranty of

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

17# GNU General Public License for more details.

18#

19# You should have received a copy of the GNU General Public License

20# along with this program. If not, see <https://www.gnu.org/licenses/>.

22"""The science-package version data model, and reading it back from ConsDB.

24The Rapid Analysis backend records, for every image it dispatches, the exact

25versions of the handful of science packages that determine the AOS results. A

26set of versions is identified by a content hash (a stable, order-independent

27SHA-256 of the versions) - two pods with the same versions hash identically.

29This module is the public home of that data model. It lives here, rather than

30in Rapid Analysis, because most users cannot import Rapid Analysis code but do

31need to consume the data: everything needed to read a record back is carried in

32ConsDB and this package, so a consumer can recover the full versions with only

33this module and a ConsDB client.

35The JSON blob stored in the ConsDB ``ra_package_versions`` column is::

37 {"hash": "<sha256hex>", "versions": {"ts_wep": "v17.6.1-alpha", ...}}

39``versions`` is inline (self-contained; no dereference needed) and ``hash`` is

40the stable identity, handy as a SQL-side join/dedup key. The blob shape is

41`PackageVersions.toDict` / `fromDict`, pinned by tests in both this package and

42Rapid Analysis; change it only with a migration plan for the rows already

43written.

45Scraping the versions and writing them to ConsDB both live in Rapid Analysis

46(the write goes through its ``ConsDBPopulator``), reusing `toDict` and the

47table/column constants here so the two sides cannot drift.

48"""

50from __future__ import annotations

52__all__ = [

53 "UNKNOWN_VERSION",

54 "PACKAGE_VERSIONS_TABLE",

55 "PACKAGE_VERSIONS_COLUMN",

56 "PackageVersions",

57 "readPackageVersionsFromConsDb",

58 "readPackageVersionsForExposure",

59 "readPackageVersionsByHash",

60]

62import hashlib

63import json

64import re

65from collections.abc import Mapping

66from dataclasses import dataclass

67from typing import TYPE_CHECKING, Any

69import numpy as np

71if TYPE_CHECKING:

72 from lsst.daf.butler import DimensionRecord

74 from .consdbClient import ConsDbClient

76# Sentinel recorded when a package's version couldn't be determined at

77# processing time. Readers must handle it, but it is not expected from a

78# healthy pod: Rapid Analysis' CI treats a missing package directory as a hard

79# failure, but it could show up if e.g. rows are written from a non-git

80# environment (e.g. a conda-installed stack, where the package directory isn't

81# a checkout) and in rows backfilled from older metadata. Note that it takes

82# part in `PackageVersions.versionHash`, so a set containing it is a distinct

83# identity from the same set with the real version, and the two will not join

84# or dedup.

85UNKNOWN_VERSION = "unknown"

87# The anticipated home of the package-versions JSONB column in ConsDB, as

88# ``cdb_<instrument>.<PACKAGE_VERSIONS_TABLE>.<PACKAGE_VERSIONS_COLUMN>``.

89# The column does not exist yet; these defaults are parameterised on every

90# function that uses them so they can be corrected without code changes if

91# the schema lands elsewhere.

92PACKAGE_VERSIONS_TABLE = "exposure_quicklook"

93PACKAGE_VERSIONS_COLUMN = "ra_package_versions"

95# Everything interpolated into raw SQL by readPackageVersionsFromConsDb must

96# look like a plain identifier; anything else is rejected.

97_SQL_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")

100def _checkIsSqlIdentifier(value: str, description: str) -> None:

101 """Raise if ``value`` is not usable as a bare SQL identifier.

102

103 Parameters

104 ----------

105 value : `str`

106 The value to check.

107 description : `str`

108 What the value is, for the error message, e.g. ``"table name"``.

109 """

110 if not _SQL_IDENTIFIER_RE.match(value): 110 ↛ 111line 110 didn't jump to line 111 because the condition on line 110 was never true

111 raise ValueError(f"Invalid {description}: {value!r}")

112

113

114@dataclass

115class PackageVersions:

116 """The versions of the tracked science packages at processing time.

117

118 The versions are held in a dict keyed by package name rather than as named

119 fields so that the set of recorded packages can grow without changing the

120 wire format. A set of versions is identified by its `versionHash`, which is

121 also the key under which Rapid Analysis stores it.

122

123 Parameters

124 ----------

125 versions : `dict` [`str`, `str`]

126 Mapping of package name to version (a git tag/SHA for git checkouts, an

127 installed distribution version for installed packages, or

128 `UNKNOWN_VERSION`).

129 """

130

131 versions: dict[str, str]

132

133 def versionHash(self) -> str:

134 """A stable, order-independent hash of the package versions.

135

136 This is the identity of a version set: two pods with the same package

137 versions hash identically regardless of the order the packages happen

138 to be recorded in, so the hash is a stable dedup/join key in ConsDB.

139

140 Returns

141 -------

142 hexdigest : `str`

143 The SHA-256 hex digest of the canonicalised versions.

144 """

145 canonical = json.dumps(self.versions, sort_keys=True)

146 return hashlib.sha256(canonical.encode("utf-8")).hexdigest()

147

148 def toDict(self) -> dict[str, Any]:

149 """Render as the dict that is stored as JSON in the ConsDB column.

150

151 Returns

152 -------

153 data : `dict` [`str`, `Any`]

154 ``{"hash": ..., "versions": {...}}``. ``versions`` is inline so

155 the record is self-contained; ``hash`` is included as a SQL-side

156 join/dedup key.

157 """

158 return {"hash": self.versionHash(), "versions": dict(self.versions)}

159

160 @classmethod

161 def fromDict(cls, data: Mapping[str, Any]) -> PackageVersions:

162 """Build from the dict form of the ConsDB column (see `toDict`).

163

164 The ``hash`` key, if present, is ignored: ``versions`` is the source of

165 truth and the hash is derived from it (recompute with `versionHash`).

166

167 Parameters

168 ----------

169 data : `Mapping` [`str`, `Any`]

170 The dict, as produced by `toDict`.

171

172 Returns

173 -------

174 packageVersions : `PackageVersions`

175 The parsed versions.

176

177 Raises

178 ------

179 ValueError

180 Raised if there is no ``versions`` key.

181 """

182 if "versions" not in data:

183 raise ValueError(f"Package-version dict has no 'versions' key: {dict(data)!r}")

184 return cls(versions=dict(data["versions"]))

185

186 def toJson(self) -> str:

187 """Render as a canonical JSON string.

188

189 Returns

190 -------

191 jsonString : `str`

192 The `toDict` dict serialised with sorted keys, so identical

193 version sets always serialise identically.

194 """

195 return json.dumps(self.toDict(), sort_keys=True)

196

197 @classmethod

198 def fromJson(cls, jsonString: str | bytes) -> PackageVersions:

199 """Build from a JSON string, as produced by `toJson`.

200

201 Parameters

202 ----------

203 jsonString : `str` or `bytes`

204 The JSON document to parse.

205

206 Returns

207 -------

208 packageVersions : `PackageVersions`

209 The parsed versions.

210

211 Raises

212 ------

213 ValueError

214 Raised if the document is not valid JSON, or parses to something

215 other than a JSON object with a ``versions`` key.

216 """

217 parsed = json.loads(jsonString)

218 if not isinstance(parsed, dict):

219 raise ValueError(f"Expected a JSON object for package versions, got {type(parsed)}")

220 return cls.fromDict(parsed)

221

222

223def readPackageVersionsFromConsDb(

224 client: ConsDbClient,

225 instrument: str,

226 dayObs: int,

227 seqNum: int,

228 table: str = PACKAGE_VERSIONS_TABLE,

229 column: str = PACKAGE_VERSIONS_COLUMN,

230) -> PackageVersions | None:

231 """Read the package versions for an exposure back from ConsDB.

232

233 Reads back the blob Rapid Analysis writes. Tolerant of how the JSONB column

234 is delivered over the query API: both an already-parsed JSON object and a

235 JSON string are accepted.

236

237 Parameters

238 ----------

239 client : `ConsDbClient`

240 The client used to talk to ConsDB.

241 instrument : `str`

242 The instrument name, e.g. ``"LSSTCam"``.

243 dayObs : `int`

244 The dayObs of the exposure.

245 seqNum : `int`

246 The seqNum of the exposure.

247 table : `str`, optional

248 The table name within the instrument's schema, without the

249 ``cdb_<instrument>.`` prefix. Defaults to `PACKAGE_VERSIONS_TABLE`.

250 column : `str`, optional

251 The JSONB column to read. Defaults to `PACKAGE_VERSIONS_COLUMN`.

252

253 Returns

254 -------

255 packageVersions : `PackageVersions` or `None`

256 The versions recorded for the exposure, or `None` if the exposure has

257 no row in the table or the column is null.

258

259 Raises

260 ------

261 RuntimeError

262 Raised if more than one row matches, which should be impossible.

263 TypeError

264 Raised if the column value is of an unexpected type.

265 """

266 _checkIsSqlIdentifier(instrument, "instrument name")

267 _checkIsSqlIdentifier(table, "table name")

268 _checkIsSqlIdentifier(column, "column name")

269 schemaName = f"cdb_{instrument.lower()}"

270 # Join via the exposure table rather than assuming day_obs/seq_num are

271 # denormalised onto the quicklook table.

272 query = (

273 f"SELECT q.{column} "

274 f"FROM {schemaName}.{table} q "

275 f"JOIN {schemaName}.exposure e ON e.exposure_id = q.exposure_id "

276 f"WHERE e.day_obs = {int(dayObs)} AND e.seq_num = {int(seqNum)}"

277 )

278 result = client.query(query)

279 if len(result) == 0:

280 return None

281 if len(result) > 1: 281 ↛ 282line 281 didn't jump to line 282 because the condition on line 281 was never true

282 raise RuntimeError(

283 f"Got {len(result)} package-version rows for {instrument} ({dayObs=}, {seqNum=});"

284 " expected at most one"

285 )

286 return _parsePackageVersions(result[column][0])

287

288

289def readPackageVersionsForExposure(

290 client: ConsDbClient,

291 expRecord: DimensionRecord,

292 table: str = PACKAGE_VERSIONS_TABLE,

293 column: str = PACKAGE_VERSIONS_COLUMN,

294) -> PackageVersions | None:

295 """Read the package versions for an exposure record back from ConsDB.

296

297 Convenience wrapper over `readPackageVersionsFromConsDb` that takes the

298 dataId straight off an exposure ``DimensionRecord`` instead of its unpacked

299 ``instrument``/``dayObs``/``seqNum``.

300

301 Parameters

302 ----------

303 client : `ConsDbClient`

304 The client used to talk to ConsDB.

305 expRecord : `lsst.daf.butler.DimensionRecord`

306 The exposure record whose versions to read.

307 table : `str`, optional

308 The table name within the instrument's schema, without the

309 ``cdb_<instrument>.`` prefix. Defaults to `PACKAGE_VERSIONS_TABLE`.

310 column : `str`, optional

311 The JSONB column to read. Defaults to `PACKAGE_VERSIONS_COLUMN`.

312

313 Returns

314 -------

315 packageVersions : `PackageVersions` or `None`

316 The versions recorded for the exposure, or `None` if there is no row or

317 the column is null.

318 """

319 return readPackageVersionsFromConsDb(

320 client,

321 expRecord.instrument,

322 expRecord.day_obs,

323 expRecord.seq_num,

324 table=table,

325 column=column,

326 )

327

328

329def readPackageVersionsByHash(

330 client: ConsDbClient,

331 instrument: str,

332 versionHash: str,

333 table: str = PACKAGE_VERSIONS_TABLE,

334 column: str = PACKAGE_VERSIONS_COLUMN,

335) -> PackageVersions | None:

336 """Read the package versions for a given version hash from ConsDB.

337

338 A hash identifies a version *set*, which can appear on many exposures, so

339 this returns the versions from an arbitrary matching row - they are

340 identical across every row that shares the hash - or `None` if no row

341 carries the hash yet. Matches on the ``hash`` key of the JSONB blob, which

342 `PackageVersions.toDict` writes alongside the inline versions.

343

344 Parameters

345 ----------

346 client : `ConsDbClient`

347 The client used to talk to ConsDB.

348 instrument : `str`

349 The instrument whose schema to search, e.g. ``"LSSTCam"``.

350 versionHash : `str`

351 The `PackageVersions.versionHash` to look up (a hex digest).

352 table : `str`, optional

353 The table name within the instrument's schema, without the

354 ``cdb_<instrument>.`` prefix. Defaults to `PACKAGE_VERSIONS_TABLE`.

355 column : `str`, optional

356 The JSONB column to read. Defaults to `PACKAGE_VERSIONS_COLUMN`.

357

358 Returns

359 -------

360 packageVersions : `PackageVersions` or `None`

361 The versions with that hash, or `None` if no row carries it.

362

363 Raises

364 ------

365 ValueError

366 Raised if ``versionHash`` is not a bare hex digest (it is interpolated

367 into the query as a string literal, so it is validated first).

368 TypeError

369 Raised if the column value is of an unexpected type.

370 """

371 _checkIsSqlIdentifier(instrument, "instrument name")

372 _checkIsSqlIdentifier(table, "table name")

373 _checkIsSqlIdentifier(column, "column name")

374 if not re.fullmatch(r"[0-9a-fA-F]+", versionHash):

375 raise ValueError(f"Invalid version hash: {versionHash!r}")

376 # versionHash() always produces lowercase hex and the SQL comparison is

377 # case-sensitive, so normalise: an uppercase paste of a valid hash must

378 # find the same row rather than silently returning None.

379 versionHash = versionHash.lower()

380 schemaName = f"cdb_{instrument.lower()}"

381 # ``->>'hash'`` extracts the blob's hash key (see toDict); LIMIT 1 because

382 # the versions are identical across every row that shares the hash.

383 query = (

384 f"SELECT q.{column} "

385 f"FROM {schemaName}.{table} q "

386 f"WHERE q.{column}->>'hash' = '{versionHash}' "

387 "LIMIT 1"

388 )

389 result = client.query(query)

390 if len(result) == 0:

391 return None

392 return _parsePackageVersions(result[column][0])

393

394

395def _parsePackageVersions(value: Any) -> PackageVersions | None:

396 """Parse a ``ra_package_versions`` column value into a `PackageVersions`.

397

398 Tolerant of how the JSONB column is delivered over the query API: a null

399 (or masked) value yields `None`, an already-parsed JSON object or a JSON

400 string both yield a `PackageVersions`.

401

402 Parameters

403 ----------

404 value : `Any`

405 The raw value from the query result.

406

407 Returns

408 -------

409 packageVersions : `PackageVersions` or `None`

410 The parsed versions, or `None` if the value is null.

411

412 Raises

413 ------

414 TypeError

415 Raised if the value is of an unexpected type.

416 """

417 if value is None or value is np.ma.masked:

418 return None

419 if isinstance(value, (str, bytes)):

420 return PackageVersions.fromJson(value)

421 if isinstance(value, Mapping): 421 ↛ 423line 421 didn't jump to line 423 because the condition on line 421 was always true

422 return PackageVersions.fromDict(value)

423 raise TypeError(f"Unexpected type for package versions: {type(value)}")