Coverage for python/lsst/analysis/ap/ppdb.py: 94%

1# This file is part of analysis_ap. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Load PPDB catalogs (DiaObjects, DiaSources, DiaForcedSources) through the 

23Rubin Science Platform TAP service. 

24 

25These tools deliberately use the scientist-facing TAP interface (ADQL over 

26``pyvo``) rather than a direct database connection, so that exercising them 

27also exercises the interface scientists will use once the PPDB goes live. 

28 

29The prototype PPDB currently lives at the ``data-int`` RSP environment 

30(``https://data-int.lsst.cloud/api/ppdbtap``). Tooling running elsewhere 

31(e.g. the USDF RSP) reaches it cross-environment with a ``data-int`` RSP 

32token carrying the ``read:tap`` scope, supplied via the ``RSP_TOKEN`` 

33environment variable. 

34 

35The PPDB ``DiaObject`` table holds only the latest version of each object, 

36so it has one row per ``diaObjectId``. ``DiaSource`` and ``DiaForcedSource`` 

37are append-only. 

38 

39The production PPDB will allow cone searches only on ``DiaObject``. Sources 

40and forced sources for a region must therefore be loaded object-first: 

41cone-search DiaObjects, then load the sources of those objects. The 

42``load_sources_for_region`` / ``load_forced_sources_for_region`` methods do 

43this for you and are the encouraged entry points. ``load_sources`` / 

44``load_forced_sources`` require a ``diaObjectId`` (scalar or iterable) and 

45take no spatial argument. Direct, object-unrestricted access to the source 

46tables is available only through the explicit, prototype-only 

47``load_sources_by_cone`` / ``load_forced_sources_by_cone`` (spatial) and 

48``load_sources_by_time_window`` / ``load_forced_sources_by_time_window`` 

49(temporal) methods, which warn when used. 

50 

51All loaders return `astropy.table.Table` objects. Their columns are returned 

52in SDM-schema order, since the TAP service returns ``SELECT *`` columns 

53alphabetically; passing an explicit ``columns`` list overrides this with the 

54order you request. 

55""" 

56 

57__all__ = [ 

58 "PpdbTap", 

59 "DiaObjectLightCurve", 

60 "region_from_exposure", 

61 "DEFAULT_PPDB_TAP_URL", 

62 "DEFAULT_PADDING_ARCSEC", 

63 "DEFAULT_ROW_LIMIT", 

64] 

65 

66import dataclasses 

67import functools 

68import logging 

69import numbers 

70import os 

71import time 

72from importlib.resources import files 

73 

74import astropy.table 

75import pyvo 

76import requests 

77 

78import lsst.geom 

79from lsst.pipe.tasks.schemaUtils import readSdmSchemaFile 

80 

81_log = logging.getLogger(__name__) 

82 

83# TAP endpoint of the prototype PPDB (the ``data-int`` RSP environment). 

84DEFAULT_PPDB_TAP_URL = "https://data-int.lsst.cloud/api/ppdbtap" 

85 

86# Default padding (arcseconds) loaded around an exposure footprint. 

87DEFAULT_PADDING_ARCSEC = 5.0 

88 

89# Default cap on rows returned by a single query, to guard against 

90# accidentally pulling the whole (multi-million row) PPDB. Pass ``limit=None`` 

91# to disable. 

92DEFAULT_ROW_LIMIT = 100000 

93 

94# Valid LSST band names; used to validate band filters before they are 

95# interpolated into an ADQL string. 

96_VALID_BANDS = frozenset("ugrizy") 

97 

98# The PPDB tables share the APDB SDM schema (``sdm_schemas/apdb.yaml``); its 

99# column order is the canonical, scientifically-grouped order. Loaders use it 

100# to reorder results, because the TAP service returns ``SELECT *`` columns 

101# alphabetically (its ``tap_schema.columns`` has no ``column_index``). Both 

102# helpers are memoized, so the schema file is parsed at most once. 

103 

104 

105@functools.cache 

106def _sdm_schema(): 

107 """Parse and return the APDB SDM schema (shared by the PPDB tables).""" 

108 path = os.fspath(files("lsst.sdm.schemas").joinpath("apdb.yaml")) 

109 return readSdmSchemaFile(path) 

110 

111 

112@functools.cache 

113def _sdm_column_order(table_name): 

114 """Return the SDM-schema column order for a PPDB table, or ``()``. 

115 

116 Parameters 

117 ---------- 

118 table_name : `str` 

119 Unqualified table name, e.g. ``"DiaSource"``. 

120 

121 Returns 

122 ------- 

123 columns : `tuple` [`str`] 

124 Column names in schema order, or an empty tuple if the table is not 

125 in the SDM schema. 

126 """ 

127 table_def = _sdm_schema().get(table_name) 

128 if table_def is None: 128 ↛ 129line 128 didn't jump to line 129 because the condition on line 128 was never true

129 return () 

130 return tuple(column.name for column in table_def.columns) 

131 

132 

133def region_from_exposure(exposure, padding=DEFAULT_PADDING_ARCSEC): 

134 """Compute a cone (ra, dec, radius) covering an exposure's sky footprint. 

135 

136 The cone is the smallest circle centered on the exposure's bounding-box 

137 center that contains all four corners, expanded by ``padding`` 

138 arcseconds. A circumscribing circle (rather than the exact polygon) keeps 

139 the resulting ADQL robust: ``CIRCLE`` is universally supported by TAP 

140 services, whereas ``POLYGON`` support and winding conventions vary. 

141 

142 Parameters 

143 ---------- 

144 exposure : `lsst.afw.image.Exposure` 

145 Exposure whose WCS and bounding box define the region. 

146 padding : `float`, optional 

147 Margin to add around the footprint, in arcseconds. 

148 

149 Returns 

150 ------- 

151 ra : `float` 

152 Cone center right ascension, in degrees (ICRS). 

153 dec : `float` 

154 Cone center declination, in degrees (ICRS). 

155 radius : `float` 

156 Cone radius, in degrees. 

157 

158 Raises 

159 ------ 

160 ValueError 

161 If the exposure has no WCS, an empty bounding box, or negative 

162 padding. 

163 """ 

164 if padding < 0: 

165 raise ValueError(f"padding must be non-negative, got {padding}") 

166 wcs = exposure.getWcs() 

167 if wcs is None: 

168 raise ValueError("Exposure has no WCS; cannot compute a sky region.") 

169 bbox = exposure.getBBox() 

170 if bbox.isEmpty(): 170 ↛ 171line 170 didn't jump to line 171 because the condition on line 170 was never true

171 raise ValueError("Exposure has an empty bounding box.") 

172 

173 center = wcs.pixelToSky(bbox.getCenter()) 

174 corners = [wcs.pixelToSky(lsst.geom.Point2D(corner)) 

175 for corner in bbox.getCorners()] 

176 radius = max(center.separation(corner) for corner in corners) 

177 radius = radius + padding * lsst.geom.arcseconds 

178 return (center.getRa().asDegrees(), 

179 center.getDec().asDegrees(), 

180 radius.asDegrees()) 

181 

182 

183@dataclasses.dataclass 

184class DiaObjectLightCurve: 

185 """The full PPDB record for a single diaObject. 

186 

187 Attributes 

188 ---------- 

189 diaObjectId : `int` 

190 Identifier of the diaObject. 

191 dia_object : `astropy.table.Row` 

192 The DiaObject row. 

193 dia_sources : `astropy.table.Table` 

194 All DiaSources associated with the object, time-ordered. 

195 dia_forced_sources : `astropy.table.Table` 

196 All DiaForcedSources for the object, time-ordered. 

197 """ 

198 

199 diaObjectId: int 

200 dia_object: "astropy.table.Row" 

201 dia_sources: astropy.table.Table 

202 dia_forced_sources: astropy.table.Table 

203 

204 @property 

205 def n_sources(self): 

206 """Number of associated DiaSources (`int`).""" 

207 return len(self.dia_sources) 

208 

209 @property 

210 def n_forced_sources(self): 

211 """Number of associated DiaForcedSources (`int`).""" 

212 return len(self.dia_forced_sources) 

213 

214 def __repr__(self): 

215 return (f"DiaObjectLightCurve(diaObjectId={self.diaObjectId}, " 

216 f"n_sources={self.n_sources}, " 

217 f"n_forced_sources={self.n_forced_sources})") 

218 

219 

220class PpdbTap: 

221 """Load PPDB catalogs through the RSP TAP service as astropy Tables. 

222 

223 Parameters 

224 ---------- 

225 url : `str`, optional 

226 TAP endpoint to query. Defaults to the prototype PPDB at ``data-int``. 

227 token : `str`, optional 

228 RSP token with the ``read:tap`` scope. If not given (and ``service`` 

229 is also not given), the ``RSP_TOKEN`` environment variable is used. 

230 service : `pyvo.dal.TAPService`, optional 

231 Pre-built TAP service. When supplied, ``url`` and ``token`` are 

232 ignored. Intended for use with unit tests. 

233 log : `logging.Logger`, optional 

234 Logger to use; defaults to the module logger. 

235 

236 Notes 

237 ----- 

238 The token is read only from the environment or the constructor argument 

239 and is never logged. 

240 """ 

241 

242 def __init__(self, url=DEFAULT_PPDB_TAP_URL, *, token=None, 

243 service=None, log=None): 

244 self.url = url 

245 self.log = log if log is not None else _log 

246 if service is not None: 

247 self._service = service 

248 return 

249 if token is None: 

250 token = os.environ.get("RSP_TOKEN") 

251 if not token: 

252 raise RuntimeError( 

253 "No RSP token available for the PPDB TAP service. Generate " 

254 "a token with the 'read:tap' scope on data-int " 

255 "(https://data-int.lsst.cloud) and set it in the RSP_TOKEN " 

256 "environment variable, or pass token=...") 

257 session = requests.Session() 

258 session.headers["Authorization"] = f"Bearer {token}" 

259 self._service = pyvo.dal.TAPService(url, session=session) 

260 

261 # ------------------------------------------------------------------ 

262 # Query execution 

263 # ------------------------------------------------------------------ 

264 def run_query(self, adql, *, maxrec=None, order_table=None): 

265 """Run an ADQL query and return the result as an astropy Table. 

266 

267 Parameters 

268 ---------- 

269 adql : `str` 

270 The ADQL query to execute. 

271 maxrec : `int`, optional 

272 Server-side cap on the number of records returned. 

273 order_table : `str`, optional 

274 If given (e.g. ``"DiaSource"``), reorder the result columns into 

275 that table's SDM-schema order. The loaders pass this so that 

276 ``SELECT *`` results, which the TAP service returns alphabetically, 

277 come back in the schema's logical column order. Columns not in the 

278 schema are appended in their original order. 

279 

280 Returns 

281 ------- 

282 table : `astropy.table.Table` 

283 The query result. 

284 """ 

285 self.log.debug("Executing ADQL query: %s", adql) 

286 start = time.perf_counter() 

287 results = self._service.run_async(adql, maxrec=maxrec) 

288 table = results.to_table() 

289 if order_table is not None: 

290 table = self._order_columns(table, order_table) 

291 elapsed = time.perf_counter() - start 

292 self.log.info("Query returned %d rows in %.3f s", len(table), elapsed) 

293 return table 

294 

295 @staticmethod 

296 def _order_columns(table, table_name): 

297 """Reorder a result table's columns into SDM-schema order. 

298 

299 Every result column is expected to be defined in the SDM schema for 

300 ``table_name`` (the PPDB does not add columns outside it), so columns 

301 are returned in schema order. Returns the table unchanged if the 

302 schema is unknown or the order already matches. 

303 

304 Raises 

305 ------ 

306 RuntimeError 

307 If the result contains a column not present in the SDM schema, 

308 which signals an unexpected schema mismatch. 

309 """ 

310 schema_order = _sdm_column_order(table_name) 

311 if not schema_order: 311 ↛ 312line 311 didn't jump to line 312 because the condition on line 311 was never true

312 return table 

313 schema_set = set(schema_order) 

314 unknown = [c for c in table.colnames if c not in schema_set] 

315 if unknown: 

316 raise RuntimeError( 

317 f"ppdb.{table_name} returned columns not in the SDM schema: " 

318 f"{unknown}") 

319 new_order = [c for c in schema_order if c in table.colnames] 

320 if new_order == table.colnames: 

321 return table 

322 return table[new_order] 

323 

324 # ------------------------------------------------------------------ 

325 # ADQL construction helpers 

326 # ------------------------------------------------------------------ 

327 @staticmethod 

328 def _select_clause(columns, limit): 

329 """Build the ``SELECT [TOP n] cols`` prefix of a query.""" 

330 cols = "*" if not columns else ", ".join(columns) 

331 top = f"TOP {int(limit)} " if limit is not None else "" 

332 return f"SELECT {top}{cols}" 

333 

334 @staticmethod 

335 def _cone_clause(ra, dec, radius): 

336 """Build an ADQL cone-search predicate (all angles in degrees).""" 

337 return (f"CONTAINS(POINT('ICRS', ra, dec), " 

338 f"CIRCLE('ICRS', {float(ra)}, {float(dec)}, " 

339 f"{float(radius)})) = 1") 

340 

341 def _resolve_cone(self, ra, dec, radius, exposure, padding): 

342 """Resolve spatial arguments into a (ra, dec, radius) cone or None. 

343 

344 Either an ``exposure`` or an explicit ``ra``/``dec``/``radius`` triple 

345 may be given, but not both. Returns None when no spatial constraint 

346 was requested. Explicit ``radius`` is in degrees; ``padding`` (used 

347 only with ``exposure``) is in arcseconds. 

348 """ 

349 if exposure is not None: 

350 if any(v is not None for v in (ra, dec, radius)): 

351 raise ValueError("Specify either exposure= or ra/dec/radius, " 

352 "not both.") 

353 return region_from_exposure(exposure, padding=padding) 

354 if ra is None and dec is None and radius is None: 

355 return None 

356 if None in (ra, dec, radius): 

357 raise ValueError("ra, dec, and radius must all be provided for a " 

358 "cone search.") 

359 return float(ra), float(dec), float(radius) 

360 

361 @staticmethod 

362 def _band_clause(bands): 

363 """Build an ADQL ``band IN (...)`` predicate, validating each band.""" 

364 quoted = [] 

365 for band in bands: 

366 if band not in _VALID_BANDS: 

367 raise ValueError(f"Unknown band {band!r}; expected one of " 

368 f"{sorted(_VALID_BANDS)}.") 

369 quoted.append(f"'{band}'") 

370 return f"band IN ({', '.join(quoted)})" 

371 

372 def _finish(self, table, limit, table_name): 

373 """Warn on probable truncation and return the table unchanged.""" 

374 if limit is not None and len(table) == limit: 

375 self.log.warning( 

376 "%s query returned exactly the row limit (%d); results are " 

377 "likely truncated. Narrow the query or raise limit.", 

378 table_name, limit) 

379 return table 

380 

381 # ------------------------------------------------------------------ 

382 # DiaObject loaders 

383 # ------------------------------------------------------------------ 

384 def load_objects(self, *, ra=None, dec=None, radius=None, exposure=None, 

385 padding=DEFAULT_PADDING_ARCSEC, columns=None, 

386 limit=DEFAULT_ROW_LIMIT): 

387 """Load DiaObjects, optionally within a spatial region. 

388 

389 Parameters 

390 ---------- 

391 ra, dec, radius : `float`, optional 

392 Cone-search center and radius, in degrees. All three must be 

393 given together; mutually exclusive with ``exposure``. 

394 exposure : `lsst.afw.image.Exposure`, optional 

395 Load objects covering this exposure's footprint (see 

396 `region_from_exposure`). Mutually exclusive with 

397 ``ra``/``dec``/``radius``. 

398 padding : `float`, optional 

399 Margin around the exposure footprint, in arcseconds (only used 

400 with ``exposure``). 

401 columns : `list` [`str`], optional 

402 Columns to select; defaults to all columns. 

403 limit : `int`, optional 

404 Maximum number of rows to return; None means no limit. 

405 

406 Returns 

407 ------- 

408 objects : `astropy.table.Table` 

409 The matching DiaObjects. 

410 """ 

411 cone = self._resolve_cone(ra, dec, radius, exposure, padding) 

412 where = f" WHERE {self._cone_clause(*cone)}" if cone is not None else "" 

413 adql = (f"{self._select_clause(columns, limit)} FROM ppdb.DiaObject" 

414 f"{where} ORDER BY diaObjectId") 

415 order_table = "DiaObject" if columns is None else None 

416 table = self._finish(self.run_query(adql, order_table=order_table), 

417 limit, "DiaObject") 

418 if len(table) and "diaObjectId" in table.colnames: 

419 n_unique = len(set(table["diaObjectId"].tolist())) 

420 if n_unique != len(table): 

421 self.log.warning( 

422 "DiaObject query returned %d rows but only %d unique " 

423 "diaObjectIds; the PPDB may contain duplicate current " 

424 "versions.", 

425 len(table), n_unique) 

426 return table 

427 

428 def load_object(self, diaObjectId, *, columns=None): 

429 """Load a single DiaObject. 

430 

431 Parameters 

432 ---------- 

433 diaObjectId : `int` 

434 Identifier of the object to load. 

435 columns : `list` [`str`], optional 

436 Columns to select; defaults to all columns. 

437 

438 Returns 

439 ------- 

440 object : `astropy.table.Row` 

441 The current DiaObject row. 

442 

443 Raises 

444 ------ 

445 RuntimeError 

446 If the object is not in the PPDB. 

447 """ 

448 adql = (f"{self._select_clause(columns, None)} FROM ppdb.DiaObject " 

449 f"WHERE diaObjectId = {int(diaObjectId)}") 

450 order_table = "DiaObject" if columns is None else None 

451 table = self.run_query(adql, order_table=order_table) 

452 if len(table) == 0: 

453 raise RuntimeError( 

454 f"diaObjectId={diaObjectId} not found in ppdb.DiaObject") 

455 if len(table) > 1: 455 ↛ 456line 455 didn't jump to line 456 because the condition on line 455 was never true

456 self.log.warning( 

457 "diaObjectId=%s has %d rows; returning the first.", 

458 diaObjectId, len(table)) 

459 return table[0] 

460 

461 # ------------------------------------------------------------------ 

462 # DiaSource / DiaForcedSource loaders 

463 # 

464 # The production PPDB will permit cone searches only on DiaObject. The 

465 # encouraged way to retrieve sources for a region is therefore object- 

466 # first: cone-search DiaObjects (`load_*_for_region`), then load their 

467 # sources by diaObjectId. `load_sources`/`load_forced_sources` require a 

468 # diaObjectId and take no spatial argument. Direct, object-unrestricted 

469 # source-table access is confined to the explicit, prototype-only 

470 # `load_*_by_cone` and `load_*_by_time_window` methods. 

471 # ------------------------------------------------------------------ 

472 def _source_filter_clauses(self, bands, mjd_begin, mjd_end): 

473 """Build the time/band filter clauses shared by the source loaders.""" 

474 clauses = [] 

475 if mjd_begin is not None: 

476 clauses.append(f"midpointMjdTai >= {float(mjd_begin)}") 

477 if mjd_end is not None: 

478 clauses.append(f"midpointMjdTai < {float(mjd_end)}") 

479 if bands: 

480 clauses.append(self._band_clause(bands)) 

481 return clauses 

482 

483 def _run_source_query(self, table_name, clauses, columns, limit): 

484 """Run a single time-ordered SELECT against a source table.""" 

485 where = (" WHERE " + " AND ".join(clauses)) if clauses else "" 

486 adql = (f"{self._select_clause(columns, limit)} FROM ppdb.{table_name}" 

487 f"{where} ORDER BY midpointMjdTai") 

488 order_table = table_name if columns is None else None 

489 return self._finish(self.run_query(adql, order_table=order_table), 

490 limit, table_name) 

491 

492 def _load_source_like(self, table_name, diaObjectId, extra_clauses, 

493 columns, limit, id_chunk_size): 

494 """Dispatch a source load by one id or many ids. 

495 

496 ``diaObjectId`` is required: source tables are loaded by object. An 

497 iterable (including the empty list a region search may produce) takes 

498 the chunked ``IN`` path; a scalar takes a single ``=`` query. 

499 """ 

500 if diaObjectId is None: 

501 raise ValueError( 

502 "diaObjectId is required; DiaSources and DiaForcedSources are " 

503 "loaded by object. Use load_*_for_region for a spatial " 

504 "region, or the prototype-only load_*_by_cone / " 

505 "load_*_by_time_window methods for direct source-table access.") 

506 if isinstance(diaObjectId, numbers.Integral): 

507 clauses = [f"diaObjectId = {int(diaObjectId)}"] + list(extra_clauses) 

508 return self._run_source_query(table_name, clauses, columns, limit) 

509 return self._load_sources_for_ids(table_name, diaObjectId, 

510 extra_clauses, columns, limit, 

511 id_chunk_size) 

512 

513 def _load_sources_for_ids(self, table_name, ids, extra_clauses, columns, 

514 limit, id_chunk_size): 

515 """Load sources for many diaObjectIds via chunked ``IN`` queries. 

516 

517 A cone search can return more diaObjectIds than fit in a single 

518 ``diaObjectId IN (...)`` clause, so the ids are split into chunks 

519 of ``id_chunk_size`` and the per-chunk results are concatenated and 

520 re-sorted by time. 

521 ``limit`` caps the total rows returned across all chunks. 

522 """ 

523 if id_chunk_size < 1: 523 ↛ 524line 523 didn't jump to line 524 because the condition on line 523 was never true

524 raise ValueError("id_chunk_size must be >= 1") 

525 ids = list(dict.fromkeys(int(i) for i in ids)) 

526 if not ids or (limit is not None and limit <= 0): 

527 # Empty result, but issue a query so columns/dtypes are populated. 

528 return self._run_source_query(table_name, ["1 = 0"], columns, None) 

529 

530 order_table = table_name if columns is None else None 

531 pages = [] 

532 total = 0 

533 truncated = False 

534 for start in range(0, len(ids), id_chunk_size): 

535 if limit is not None and total >= limit: 535 ↛ 536line 535 didn't jump to line 536 because the condition on line 535 was never true

536 truncated = True 

537 break 

538 chunk = ids[start:start + id_chunk_size] 

539 remaining = None if limit is None else limit - total 

540 id_list = ", ".join(str(i) for i in chunk) 

541 clauses = [f"diaObjectId IN ({id_list})"] + list(extra_clauses) 

542 where = " WHERE " + " AND ".join(clauses) 

543 adql = (f"{self._select_clause(columns, remaining)} " 

544 f"FROM ppdb.{table_name}{where} ORDER BY midpointMjdTai") 

545 page = self.run_query(adql, order_table=order_table) 

546 total += len(page) 

547 pages.append(page) 

548 

549 table = (pages[0] if len(pages) == 1 

550 else astropy.table.vstack(pages, metadata_conflicts="silent")) 

551 if len(table) and "midpointMjdTai" in table.colnames: 551 ↛ 553line 551 didn't jump to line 553 because the condition on line 551 was always true

552 table.sort("midpointMjdTai") 

553 if limit is not None and (truncated or total == limit): 553 ↛ 554line 553 didn't jump to line 554 because the condition on line 553 was never true

554 self.log.warning( 

555 "%s query reached the row limit (%d) while loading sources for " 

556 "%d objects; results are likely truncated. Raise limit or " 

557 "narrow the query.", table_name, limit, len(ids)) 

558 return table 

559 

560 def _direct_source_query(self, table_name, region_method, kind, clauses, 

561 columns, limit): 

562 """Warn, then run a prototype-only direct source-table query. 

563 

564 ``load_sources``/``load_forced_sources`` require a diaObjectId; these 

565 direct (object-unrestricted) source-table searches will not exist in 

566 the production PPDB, so callers reach them only through the explicit 

567 ``load_*_by_cone`` / ``load_*_by_time_window`` methods, which route 

568 here and emit this warning. 

569 """ 

570 self.log.warning( 

571 "Direct %s %s search is a prototype-only capability and will not " 

572 "be available in the production PPDB; prefer %s().", 

573 table_name, kind, region_method) 

574 return self._run_source_query(table_name, clauses, columns, limit) 

575 

576 def _load_by_cone(self, table_name, region_method, *, ra, dec, radius, 

577 exposure, padding, bands, mjd_begin, mjd_end, columns, 

578 limit): 

579 """Direct cone search on a source table (prototype-only).""" 

580 cone = self._resolve_cone(ra, dec, radius, exposure, padding) 

581 if cone is None: 

582 raise ValueError("A cone search requires ra/dec/radius or an " 

583 "exposure.") 

584 clauses = [self._cone_clause(*cone)] 

585 clauses += self._source_filter_clauses(bands, mjd_begin, mjd_end) 

586 return self._direct_source_query(table_name, region_method, "cone", 

587 clauses, columns, limit) 

588 

589 def _load_by_time_window(self, table_name, region_method, *, mjd_begin, 

590 mjd_end, bands, columns, limit): 

591 """Direct time-window scan of a source table (prototype-only).""" 

592 if mjd_begin is None and mjd_end is None: 

593 raise ValueError("A time-window search requires mjd_begin and/or " 

594 "mjd_end.") 

595 clauses = self._source_filter_clauses(bands, mjd_begin, mjd_end) 

596 return self._direct_source_query(table_name, region_method, 

597 "time-window", clauses, columns, 

598 limit) 

599 

600 def load_sources(self, *, diaObjectId, mjd_begin=None, mjd_end=None, 

601 bands=None, columns=None, limit=DEFAULT_ROW_LIMIT, 

602 id_chunk_size=1000): 

603 """Load DiaSources for a given object (or objects), by time and band. 

604 

605 DiaSources are loaded by object: ``diaObjectId`` is required. This 

606 loader has no spatial argument, because the production PPDB does not 

607 permit searches on the source tables. To load sources for a region, 

608 use `load_sources_for_region` (object-first); for direct, object- 

609 unrestricted prototype-only access, use `load_sources_by_cone` or 

610 `load_sources_by_time_window`. 

611 

612 Parameters 

613 ---------- 

614 diaObjectId : `int` or iterable of `int` 

615 Restrict to sources of this object, or of any of these objects 

616 (issued as chunked ``diaObjectId IN (...)`` queries). Required. 

617 mjd_begin, mjd_end : `float`, optional 

618 Restrict to ``mjd_begin <= midpointMjdTai < mjd_end`` (TAI). 

619 bands : `list` [`str`], optional 

620 Restrict to these bands (subset of ``ugrizy``). 

621 columns : `list` [`str`], optional 

622 Columns to select; defaults to all columns. 

623 limit : `int`, optional 

624 Maximum number of rows to return; None means no limit. 

625 id_chunk_size : `int`, optional 

626 Number of diaObjectIds per ``IN`` query when an iterable is given. 

627 

628 Returns 

629 ------- 

630 sources : `astropy.table.Table` 

631 The matching DiaSources, ordered by ``midpointMjdTai``. 

632 

633 Raises 

634 ------ 

635 ValueError 

636 If ``diaObjectId`` is None. 

637 """ 

638 extra = self._source_filter_clauses(bands, mjd_begin, mjd_end) 

639 return self._load_source_like("DiaSource", diaObjectId, extra, 

640 columns, limit, id_chunk_size) 

641 

642 def load_forced_sources(self, *, diaObjectId, mjd_begin=None, 

643 mjd_end=None, bands=None, columns=None, 

644 limit=DEFAULT_ROW_LIMIT, id_chunk_size=1000): 

645 """Load DiaForcedSources, with the same filters as `load_sources`. 

646 

647 Like `load_sources`, ``diaObjectId`` is required and there is no 

648 spatial argument; use `load_forced_sources_for_region`, or the 

649 prototype-only `load_forced_sources_by_cone` / 

650 `load_forced_sources_by_time_window`. 

651 

652 Returns 

653 ------- 

654 forced_sources : `astropy.table.Table` 

655 The matching DiaForcedSources, ordered by ``midpointMjdTai``. 

656 

657 Raises 

658 ------ 

659 ValueError 

660 If ``diaObjectId`` is None. 

661 """ 

662 extra = self._source_filter_clauses(bands, mjd_begin, mjd_end) 

663 return self._load_source_like("DiaForcedSource", diaObjectId, extra, 

664 columns, limit, id_chunk_size) 

665 

666 def load_sources_for_region(self, *, ra=None, dec=None, radius=None, 

667 exposure=None, padding=DEFAULT_PADDING_ARCSEC, 

668 mjd_begin=None, mjd_end=None, bands=None, 

669 columns=None, limit=DEFAULT_ROW_LIMIT, 

670 object_limit=DEFAULT_ROW_LIMIT, 

671 id_chunk_size=1000): 

672 """Load DiaSources for a region, the production-compatible way. 

673 

674 Performs the two-step workflow the production PPDB requires: cone- 

675 search DiaObjects (the only table that allows spatial search), then 

676 load the DiaSources of those objects. 

677 

678 Parameters 

679 ---------- 

680 ra, dec, radius : `float`, optional 

681 Cone center and radius, in degrees; mutually exclusive with 

682 ``exposure``. The radius is applied to *object* positions. 

683 exposure : `lsst.afw.image.Exposure`, optional 

684 Load sources for objects covering this exposure's footprint. 

685 padding : `float`, optional 

686 Margin around the exposure footprint, in arcseconds. 

687 mjd_begin, mjd_end, bands, columns : optional 

688 Passed through to the source query (see `load_sources`). 

689 limit : `int`, optional 

690 Cap on returned sources; None means no limit. 

691 object_limit : `int`, optional 

692 Cap on the DiaObjects found by the cone search. 

693 id_chunk_size : `int`, optional 

694 diaObjectIds per ``IN`` query when fetching sources. 

695 

696 Returns 

697 ------- 

698 sources : `astropy.table.Table` 

699 DiaSources of the objects in the region, ordered by time. 

700 

701 Notes 

702 ----- 

703 Because this is object-first, DiaSources not associated with a 

704 returned DiaObject (e.g. unassociated or ssObject-only detections) 

705 are not included. To find those, use `load_sources_by_cone`. 

706 """ 

707 ids = self._region_object_ids( 

708 ra, dec, radius, exposure, padding, object_limit) 

709 return self.load_sources(diaObjectId=ids, mjd_begin=mjd_begin, 

710 mjd_end=mjd_end, bands=bands, columns=columns, 

711 limit=limit, id_chunk_size=id_chunk_size) 

712 

713 def load_forced_sources_for_region(self, *, ra=None, dec=None, 

714 radius=None, exposure=None, 

715 padding=DEFAULT_PADDING_ARCSEC, 

716 mjd_begin=None, mjd_end=None, bands=None, 

717 columns=None, limit=DEFAULT_ROW_LIMIT, 

718 object_limit=DEFAULT_ROW_LIMIT, 

719 id_chunk_size=1000): 

720 """Load DiaForcedSources for a region (object-first). 

721 

722 Same two-step workflow and caveats as `load_sources_for_region`. 

723 

724 Returns 

725 ------- 

726 forced_sources : `astropy.table.Table` 

727 DiaForcedSources of the objects in the region, ordered by time. 

728 """ 

729 ids = self._region_object_ids( 

730 ra, dec, radius, exposure, padding, object_limit) 

731 return self.load_forced_sources( 

732 diaObjectId=ids, mjd_begin=mjd_begin, mjd_end=mjd_end, bands=bands, 

733 columns=columns, limit=limit, id_chunk_size=id_chunk_size) 

734 

735 def _region_object_ids(self, ra, dec, radius, exposure, padding, 

736 object_limit): 

737 """Cone-search DiaObjects and return their diaObjectIds.""" 

738 objects = self.load_objects(ra=ra, dec=dec, radius=radius, 

739 exposure=exposure, padding=padding, 

740 columns=["diaObjectId"], 

741 limit=object_limit) 

742 if not len(objects): 

743 return [] 

744 return objects["diaObjectId"].tolist() 

745 

746 def load_sources_by_cone(self, *, ra=None, dec=None, radius=None, 

747 exposure=None, padding=DEFAULT_PADDING_ARCSEC, 

748 mjd_begin=None, mjd_end=None, bands=None, 

749 columns=None, limit=DEFAULT_ROW_LIMIT): 

750 """Directly cone-search the DiaSource table (prototype-only). 

751 

752 .. warning:: 

753 

754 The production PPDB will not allow spatial searches on the source 

755 tables. This works only against the prototype and is intended for 

756 debugging/validation (e.g. finding DiaSources with no associated 

757 DiaObject). For the production-compatible region workflow use 

758 `load_sources_for_region`. 

759 

760 Parameters mirror `load_sources_for_region` minus ``object_limit``. 

761 

762 Returns 

763 ------- 

764 sources : `astropy.table.Table` 

765 DiaSources within the cone, ordered by ``midpointMjdTai``. 

766 """ 

767 return self._load_by_cone( 

768 "DiaSource", "load_sources_for_region", ra=ra, dec=dec, 

769 radius=radius, exposure=exposure, padding=padding, bands=bands, 

770 mjd_begin=mjd_begin, mjd_end=mjd_end, columns=columns, limit=limit) 

771 

772 def load_forced_sources_by_cone(self, *, ra=None, dec=None, radius=None, 

773 exposure=None, 

774 padding=DEFAULT_PADDING_ARCSEC, 

775 mjd_begin=None, mjd_end=None, bands=None, 

776 columns=None, limit=DEFAULT_ROW_LIMIT): 

777 """Directly cone-search the DiaForcedSource table (prototype-only). 

778 

779 .. warning:: 

780 

781 Not supported by the production PPDB; prototype/debugging only. 

782 Prefer `load_forced_sources_for_region`. 

783 

784 Returns 

785 ------- 

786 forced_sources : `astropy.table.Table` 

787 DiaForcedSources within the cone, ordered by ``midpointMjdTai``. 

788 """ 

789 return self._load_by_cone( 

790 "DiaForcedSource", "load_forced_sources_for_region", ra=ra, 

791 dec=dec, radius=radius, exposure=exposure, padding=padding, 

792 bands=bands, mjd_begin=mjd_begin, mjd_end=mjd_end, columns=columns, 

793 limit=limit) 

794 

795 def load_sources_by_time_window(self, *, mjd_begin=None, mjd_end=None, 

796 bands=None, columns=None, 

797 limit=DEFAULT_ROW_LIMIT): 

798 """Load DiaSources in a time window, any object (prototype-only). 

799 

800 .. warning:: 

801 

802 The production PPDB will not allow object-unrestricted searches on 

803 the source tables. This works only against the prototype and is 

804 intended for debugging (e.g. inspecting everything processed in an 

805 MJD range). For the production-compatible workflow use 

806 `load_sources_for_region`. 

807 

808 Parameters 

809 ---------- 

810 mjd_begin, mjd_end : `float`, optional 

811 Time window on ``midpointMjdTai`` (TAI), as 

812 ``mjd_begin <= midpointMjdTai < mjd_end``. At least one bound is 

813 required (an unbounded source-table scan is not permitted). 

814 bands : `list` [`str`], optional 

815 Restrict to these bands (subset of ``ugrizy``). 

816 columns : `list` [`str`], optional 

817 Columns to select; defaults to all columns. 

818 limit : `int`, optional 

819 Maximum number of rows to return; None means no limit. 

820 

821 Returns 

822 ------- 

823 sources : `astropy.table.Table` 

824 DiaSources in the window, ordered by ``midpointMjdTai``. 

825 

826 Raises 

827 ------ 

828 ValueError 

829 If neither ``mjd_begin`` nor ``mjd_end`` is given. 

830 """ 

831 return self._load_by_time_window( 

832 "DiaSource", "load_sources_for_region", mjd_begin=mjd_begin, 

833 mjd_end=mjd_end, bands=bands, columns=columns, limit=limit) 

834 

835 def load_forced_sources_by_time_window(self, *, mjd_begin=None, 

836 mjd_end=None, bands=None, 

837 columns=None, 

838 limit=DEFAULT_ROW_LIMIT): 

839 """Load DiaForcedSources in a time window, any object (prototype-only). 

840 

841 .. warning:: 

842 

843 Not supported by the production PPDB; prototype/debugging only. 

844 Prefer `load_forced_sources_for_region`. At least one of 

845 ``mjd_begin`` / ``mjd_end`` is required. 

846 

847 Returns 

848 ------- 

849 forced_sources : `astropy.table.Table` 

850 DiaForcedSources in the window, ordered by ``midpointMjdTai``. 

851 

852 Raises 

853 ------ 

854 ValueError 

855 If neither ``mjd_begin`` nor ``mjd_end`` is given. 

856 """ 

857 return self._load_by_time_window( 

858 "DiaForcedSource", "load_forced_sources_for_region", 

859 mjd_begin=mjd_begin, mjd_end=mjd_end, bands=bands, columns=columns, 

860 limit=limit) 

861 

862 def load_sources_for_object(self, diaObjectId, *, columns=None, 

863 limit=DEFAULT_ROW_LIMIT): 

864 """Load all DiaSources associated with one diaObject. 

865 

866 Returns 

867 ------- 

868 sources : `astropy.table.Table` 

869 The object's DiaSources, ordered by ``midpointMjdTai``. 

870 """ 

871 return self.load_sources(diaObjectId=diaObjectId, columns=columns, 

872 limit=limit) 

873 

874 def load_forced_sources_for_object(self, diaObjectId, *, columns=None, 

875 limit=DEFAULT_ROW_LIMIT): 

876 """Load all DiaForcedSources associated with one diaObject. 

877 

878 Returns 

879 ------- 

880 forced_sources : `astropy.table.Table` 

881 The object's DiaForcedSources, ordered by ``midpointMjdTai``. 

882 """ 

883 return self.load_forced_sources(diaObjectId=diaObjectId, 

884 columns=columns, limit=limit) 

885 

886 def load_source(self, diaSourceId, *, columns=None): 

887 """Load a single DiaSource by id. 

888 

889 Raises 

890 ------ 

891 RuntimeError 

892 If the source does not exist. 

893 """ 

894 return self._load_one("DiaSource", "diaSourceId", diaSourceId, columns) 

895 

896 def load_forced_source(self, diaForcedSourceId, *, columns=None): 

897 """Load a single DiaForcedSource by id. 

898 

899 Raises 

900 ------ 

901 RuntimeError 

902 If the forced source does not exist. 

903 """ 

904 return self._load_one("DiaForcedSource", "diaForcedSourceId", 

905 diaForcedSourceId, columns) 

906 

907 def _load_one(self, table_name, id_column, id_value, columns): 

908 """Load exactly one row from a table by id, raising if absent.""" 

909 adql = (f"{self._select_clause(columns, None)} FROM ppdb.{table_name} " 

910 f"WHERE {id_column} = {int(id_value)}") 

911 order_table = table_name if columns is None else None 

912 table = self.run_query(adql, order_table=order_table) 

913 if len(table) == 0: