Coverage for python/lsst/daf/butler/registry/queries/_query_backend.py: 31%

111 statements  

« prev     ^ index     » next       coverage.py v7.10.6, created at 2025-10-10 01:28 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29from ... import _timespan 

30 

31__all__ = ("QueryBackend",) 

32 

33from abc import abstractmethod 

34from collections.abc import Iterable, Mapping, Sequence, Set 

35from contextlib import AbstractContextManager 

36from typing import TYPE_CHECKING, Any, Generic, TypeVar 

37 

38from lsst.daf.relation import ( 

39 BinaryOperationRelation, 

40 ColumnExpression, 

41 ColumnTag, 

42 LeafRelation, 

43 MarkerRelation, 

44 Predicate, 

45 Relation, 

46 UnaryOperationRelation, 

47) 

48 

49from ..._collection_type import CollectionType 

50from ..._column_tags import DatasetColumnTag, DimensionKeyColumnTag 

51from ..._dataset_type import DatasetType 

52from ..._exceptions import MissingDatasetTypeError 

53from ..._exceptions_legacy import DatasetTypeError 

54from ...dimensions import DimensionGroup, DimensionRecordSet, DimensionUniverse 

55from ..wildcards import CollectionWildcard 

56from ._query_context import QueryContext 

57from .find_first_dataset import FindFirstDataset 

58 

59if TYPE_CHECKING: 

60 from ..interfaces import CollectionRecord 

61 

62 

63_C = TypeVar("_C", bound=QueryContext) 

64 

65 

66class QueryBackend(Generic[_C]): 

67 """An interface for constructing and evaluating the 

68 `~lsst.daf.relation.Relation` objects that comprise registry queries. 

69 

70 This ABC is expected to have a concrete subclass for each concrete registry 

71 type, and most subclasses will be paired with a `QueryContext` subclass. 

72 See `QueryContext` for the division of responsibilities between these two 

73 interfaces. 

74 """ 

75 

76 @property 

77 @abstractmethod 

78 def universe(self) -> DimensionUniverse: 

79 """Definition of all dimensions and dimension elements for this 

80 registry (`DimensionUniverse`). 

81 """ 

82 raise NotImplementedError() 

83 

84 @abstractmethod 

85 def caching_context(self) -> AbstractContextManager[None]: 

86 """Enable caching of collection records and summaries for the duration 

87 of the returned context manager. 

88 """ 

89 raise NotImplementedError() 

90 

91 def context(self) -> _C: 

92 """Return a context manager that can be used to execute queries with 

93 this backend. 

94 

95 Returns 

96 ------- 

97 context : `QueryContext` 

98 Context manager that manages state and connections needed to 

99 execute queries. 

100 """ 

101 raise NotImplementedError() 

102 

103 @abstractmethod 

104 def get_collection_name(self, key: Any) -> str: 

105 """Return the collection name associated with a collection primary key 

106 value. 

107 

108 Parameters 

109 ---------- 

110 key : `~typing.Any` 

111 Collection primary key value. 

112 

113 Returns 

114 ------- 

115 name : `str` 

116 Collection name. 

117 """ 

118 raise NotImplementedError() 

119 

120 @abstractmethod 

121 def resolve_collection_wildcard( 

122 self, 

123 expression: Any, 

124 *, 

125 collection_types: Set[CollectionType] = CollectionType.all(), 

126 flatten_chains: bool = True, 

127 include_chains: bool | None = None, 

128 ) -> list[CollectionRecord]: 

129 """Return the collection records that match a wildcard expression. 

130 

131 Parameters 

132 ---------- 

133 expression : `~typing.Any` 

134 Names and/or patterns for collections; will be passed to 

135 `CollectionWildcard.from_expression`. 

136 collection_types : `collections.abc.Set` [ `CollectionType` ], optional 

137 If provided, only yield collections of these types. 

138 flatten_chains : `bool`, optional 

139 If `True` (default) recursively yield the child collections of 

140 `~CollectionType.CHAINED` collections. 

141 include_chains : `bool`, optional 

142 If `False`, return records for `~CollectionType.CHAINED` 

143 collections themselves. The default is the opposite of 

144 ``flattenChains``: either return records for CHAINED collections or 

145 their children, but not both. 

146 

147 Returns 

148 ------- 

149 records : `list` [ `CollectionRecord` ] 

150 Matching collection records. 

151 """ 

152 raise NotImplementedError() 

153 

154 @abstractmethod 

155 def resolve_dataset_type_wildcard( 

156 self, 

157 expression: Any, 

158 missing: list[str] | None = None, 

159 explicit_only: bool = False, 

160 ) -> list[DatasetType]: 

161 """Return the dataset types that match a wildcard expression. 

162 

163 Parameters 

164 ---------- 

165 expression : `~typing.Any` 

166 Names and/or patterns for dataset types; will be passed to 

167 `DatasetTypeWildcard.from_expression`. 

168 missing : `list` of `str`, optional 

169 String dataset type names that were explicitly given (i.e. not 

170 regular expression patterns) but not found will be appended to this 

171 list, if it is provided. 

172 explicit_only : `bool`, optional 

173 If `True`, require explicit `DatasetType` instances or `str` names, 

174 with `re.Pattern` instances deprecated and ``...`` prohibited. 

175 

176 Returns 

177 ------- 

178 dataset_types : `list` [ `DatasetType` ] 

179 A list of resolved dataset types. 

180 """ 

181 raise NotImplementedError() 

182 

183 def resolve_single_dataset_type_wildcard( 

184 self, 

185 expression: Any, 

186 explicit_only: bool = False, 

187 ) -> DatasetType: 

188 """Return a single dataset type that matches a wildcard expression. 

189 

190 Parameters 

191 ---------- 

192 expression : `~typing.Any` 

193 Names and/or patterns for the dataset type; will be passed to 

194 `DatasetTypeWildcard.from_expression`. 

195 explicit_only : `bool`, optional 

196 If `True`, require explicit `DatasetType` instances or `str` names, 

197 with `re.Pattern` instances deprecated and ``...`` prohibited. 

198 

199 Returns 

200 ------- 

201 single : `DatasetType` 

202 The matched dataset type. 

203 """ 

204 missing: list[str] = [] 

205 matching = self.resolve_dataset_type_wildcard( 

206 expression, missing=missing, explicit_only=explicit_only 

207 ) 

208 if not matching: 

209 if missing: 

210 raise MissingDatasetTypeError( 

211 "\n".join( 

212 f"Dataset type {t!r} is not registered, so no instances of it can exist." 

213 for t in missing 

214 ) 

215 ) 

216 else: 

217 raise MissingDatasetTypeError( 

218 f"No registered dataset types matched expression {expression!r}, " 

219 "so no datasets will be found." 

220 ) 

221 if len(matching) > 1: 

222 raise DatasetTypeError( 

223 f"Expression {expression!r} matched multiple parent dataset types: " 

224 f"{[t.name for t in matching]}, but only one is allowed." 

225 ) 

226 (single_parent,) = matching 

227 if missing: 

228 raise DatasetTypeError( 

229 f"Expression {expression!r} appears to involve multiple dataset types, even though only " 

230 f"one ({single_parent.name}) is registered, and only one is allowed here." 

231 ) 

232 return single_parent 

233 

234 @abstractmethod 

235 def filter_dataset_collections( 

236 self, 

237 dataset_types: Iterable[DatasetType], 

238 collections: Sequence[CollectionRecord], 

239 *, 

240 governor_constraints: Mapping[str, Set[str]], 

241 rejections: list[str] | None = None, 

242 ) -> dict[DatasetType, list[CollectionRecord]]: 

243 """Filter a sequence of collections to those for which a dataset query 

244 might succeed. 

245 

246 Parameters 

247 ---------- 

248 dataset_types : `~collections.abc.Iterable` [ `DatasetType` ] 

249 Dataset types that are being queried. Must include only parent 

250 or standalone dataset types, not components. 

251 collections : `~collections.abc.Sequence` [ `CollectionRecord` ] 

252 Sequence of collections that will be searched. 

253 governor_constraints : `~collections.abc.Mapping` [ `str`, \ 

254 `~collections.abc.Set` [ `str` ] ], optional 

255 Constraints imposed by other aspects of the query on governor 

256 dimensions; collections inconsistent with these constraints will be 

257 skipped. 

258 rejections : `list` [ `str` ], optional 

259 If not `None`, a `list` that diagnostic messages will be appended 

260 to, for any collection that matches ``collections`` that is not 

261 returned. At least one message is guaranteed whenever the result 

262 is empty. 

263 

264 Returns 

265 ------- 

266 dataset_collections : `dict` [ `DatasetType`, \ 

267 `list` [ `CollectionRecord` ] ] 

268 The collections to search for each dataset. The dictionary's keys 

269 are always exactly ``dataset_types`` (in the same order), and each 

270 nested `list` of collections is ordered consistently with the 

271 given ``collections``. 

272 

273 Notes 

274 ----- 

275 This method accepts multiple dataset types and multiple collections at 

276 once to enable implementations to batch up the fetching of summary 

277 information needed to relate them. 

278 """ 

279 raise NotImplementedError() 

280 

281 def resolve_dataset_collections( 

282 self, 

283 dataset_type: DatasetType, 

284 collections: CollectionWildcard, 

285 *, 

286 governor_constraints: Mapping[str, Set[str]], 

287 rejections: list[str] | None = None, 

288 collection_types: Set[CollectionType] = CollectionType.all(), 

289 allow_calibration_collections: bool = False, 

290 ) -> list[CollectionRecord]: 

291 """Resolve the sequence of collections to query for a dataset type. 

292 

293 Parameters 

294 ---------- 

295 dataset_type : `DatasetType` 

296 Dataset type to be queried in the returned collections. 

297 collections : `CollectionWildcard` 

298 Expression for the collections to be queried. 

299 governor_constraints : `~collections.abc.Mapping` [ `str`, \ 

300 `~collections.abc.Set` ], optional 

301 Constraints imposed by other aspects of the query on governor 

302 dimensions; collections inconsistent with these constraints will be 

303 skipped. 

304 rejections : `list` [ `str` ], optional 

305 If not `None`, a `list` that diagnostic messages will be appended 

306 to, for any collection that matches ``collections`` that is not 

307 returned. At least one message is guaranteed whenever the result 

308 is empty. 

309 collection_types : `~collections.abc.Set` [ `CollectionType` ], \ 

310 optional 

311 Collection types to consider when resolving the collection 

312 expression. 

313 allow_calibration_collections : `bool`, optional 

314 If `False`, skip (with a ``rejections`` message) any calibration 

315 collections that match ``collections`` are not given explicitly by 

316 name, and raise `NotImplementedError` for any calibration 

317 collection that is given explicitly. This is a temporary option 

318 that will be removed when the query system can handle temporal 

319 joins involving calibration collections. 

320 

321 Returns 

322 ------- 

323 records : `list` [ `CollectionRecord` ] 

324 A new list of `CollectionRecord` instances, for collections that 

325 both match ``collections`` and may have datasets of the given type. 

326 

327 Notes 

328 ----- 

329 This is a higher-level driver for `resolve_collection_wildcard` and 

330 `filter_dataset_collections` that is mostly concerned with handling 

331 queries against `~Collection.Type.CALIBRATION` collections that aren't 

332 fully supported yet. Once that support improves, this method may be 

333 removed. 

334 """ 

335 if collections == CollectionWildcard() and collection_types == CollectionType.all(): 

336 collection_types = {CollectionType.RUN} 

337 explicit_collections = frozenset(collections.strings) 

338 matching_collection_records = self.resolve_collection_wildcard( 

339 collections, collection_types=collection_types 

340 ) 

341 ((_, filtered_collection_records),) = self.filter_dataset_collections( 

342 [dataset_type], 

343 matching_collection_records, 

344 governor_constraints=governor_constraints, 

345 rejections=rejections, 

346 ).items() 

347 if not allow_calibration_collections: 

348 supported_collection_records: list[CollectionRecord] = [] 

349 for record in filtered_collection_records: 

350 if record.type is CollectionType.CALIBRATION: 

351 # If collection name was provided explicitly then raise, 

352 # since this is a kind of query we don't support yet; 

353 # otherwise collection is a part of a chained one or regex 

354 # match, and we skip it to not break queries of other 

355 # included collections. 

356 if record.name in explicit_collections: 

357 raise NotImplementedError( 

358 f"Query for dataset type {dataset_type.name!r} in CALIBRATION-type " 

359 f"collection {record.name!r} is not yet supported." 

360 ) 

361 else: 

362 if rejections is not None: 

363 rejections.append( 

364 f"Not searching for dataset {dataset_type.name!r} in CALIBRATION " 

365 f"collection {record.name!r} because calibration queries aren't fully " 

366 "implemented; this is not an error only because the query structure " 

367 "implies that searching this collection may be incidental." 

368 ) 

369 supported_collection_records.append(record) 

370 else: 

371 supported_collection_records.append(record) 

372 else: 

373 supported_collection_records = filtered_collection_records 

374 if not supported_collection_records and rejections is not None and not rejections: 

375 rejections.append(f"No collections to search matching expression {collections!r}.") 

376 return supported_collection_records 

377 

378 @abstractmethod 

379 def _make_dataset_query_relation_impl( 

380 self, 

381 dataset_type: DatasetType, 

382 collections: Sequence[CollectionRecord], 

383 columns: Set[str], 

384 context: _C, 

385 ) -> Relation: 

386 """Construct a relation that represents an unordered query for datasets 

387 that returns matching results from all given collections. 

388 

389 Parameters 

390 ---------- 

391 dataset_type : `DatasetType` 

392 Type for the datasets being queried. 

393 collections : `~collections.abc.Sequence` [ `CollectionRecord` ] 

394 Records for collections to query. Should generally be the result 

395 of a call to `resolve_dataset_collections`, and must not be empty. 

396 context : `QueryContext` 

397 Context that manages per-query state. 

398 columns : `~collections.abc.Set` [ `str` ] 

399 Columns to include in the relation. See `Query.find_datasets` for 

400 details. 

401 

402 Returns 

403 ------- 

404 relation : `lsst.daf.relation.Relation` 

405 Relation representing a dataset query. 

406 

407 Notes 

408 ----- 

409 This method must be implemented by derived classes but is not 

410 responsible for joining the resulting relation to an existing relation. 

411 """ 

412 raise NotImplementedError() 

413 

414 def make_dataset_query_relation( 

415 self, 

416 dataset_type: DatasetType, 

417 collections: Sequence[CollectionRecord], 

418 columns: Set[str], 

419 context: _C, 

420 *, 

421 join_to: Relation | None = None, 

422 temporal_join_on: Set[ColumnTag] = frozenset(), 

423 ) -> Relation: 

424 """Construct a relation that represents an unordered query for datasets 

425 that returns matching results from all given collections. 

426 

427 Parameters 

428 ---------- 

429 dataset_type : `DatasetType` 

430 Type for the datasets being queried. 

431 collections : `~collections.abc.Sequence` [ `CollectionRecord` ] 

432 Records for collections to query. Should generally be the result 

433 of a call to `resolve_dataset_collections`, and must not be empty. 

434 columns : `~collections.abc.Set` [ `str` ] 

435 Columns to include in the relation. See `Query.find_datasets` for 

436 details. 

437 context : `QueryContext` 

438 Context that manages per-query state. 

439 join_to : `Relation`, optional 

440 Another relation to join with the query for datasets in all 

441 collections. 

442 temporal_join_on : `~collections.abc.Set` [ `ColumnTag` ], optional 

443 Timespan columns in ``join_to`` that calibration dataset timespans 

444 must overlap. Must already be present in ``join_to``. Ignored if 

445 ``join_to`` is `None` or if there are no calibration collections. 

446 

447 Returns 

448 ------- 

449 relation : `lsst.daf.relation.Relation` 

450 Relation representing a dataset query. 

451 """ 

452 # If we need to do a temporal join to a calibration collection, we need 

453 # to include the timespan column in the base query and prepare the join 

454 # predicate. 

455 join_predicates: list[Predicate] = [] 

456 base_timespan_tag: ColumnTag | None = None 

457 full_columns: set[str] = set(columns) 

458 if ( 

459 temporal_join_on 

460 and join_to is not None 

461 and any(r.type is CollectionType.CALIBRATION for r in collections) 

462 ): 

463 base_timespan_tag = DatasetColumnTag(dataset_type.name, "timespan") 

464 rhs = ColumnExpression.reference(base_timespan_tag, dtype=_timespan.Timespan) 

465 full_columns.add("timespan") 

466 for timespan_tag in temporal_join_on: 

467 lhs = ColumnExpression.reference(timespan_tag, dtype=_timespan.Timespan) 

468 join_predicates.append(lhs.predicate_method("overlaps", rhs)) 

469 # Delegate to the concrete QueryBackend subclass to do most of the 

470 # work. 

471 result = self._make_dataset_query_relation_impl( 

472 dataset_type, 

473 collections, 

474 full_columns, 

475 context=context, 

476 ) 

477 if join_to is not None: 

478 result = join_to.join( 

479 result, predicate=Predicate.logical_and(*join_predicates) if join_predicates else None 

480 ) 

481 if join_predicates and "timespan" not in columns: 

482 # Drop the timespan column we added for the join only if the 

483 # timespan wasn't requested in its own right. 

484 result = result.with_only_columns(result.columns - {base_timespan_tag}) 

485 return result 

486 

487 def make_dataset_search_relation( 

488 self, 

489 dataset_type: DatasetType, 

490 collections: Sequence[CollectionRecord], 

491 columns: Set[str], 

492 context: _C, 

493 *, 

494 join_to: Relation | None = None, 

495 temporal_join_on: Set[ColumnTag] = frozenset(), 

496 ) -> Relation: 

497 """Construct a relation that represents an order query for datasets 

498 that returns results from the first matching collection for each data 

499 ID. 

500 

501 Parameters 

502 ---------- 

503 dataset_type : `DatasetType` 

504 Type for the datasets being search. 

505 collections : `~collections.abc.Sequence` [ `CollectionRecord` ] 

506 Records for collections to search. Should generally be the result 

507 of a call to `resolve_dataset_collections`, and must not be empty. 

508 columns : `~collections.abc.Set` [ `str` ] 

509 Columns to include in the ``relation``. See 

510 `make_dataset_query_relation` for options. 

511 context : `QueryContext` 

512 Context that manages per-query state. 

513 join_to : `Relation`, optional 

514 Another relation to join with the query for datasets in all 

515 collections before filtering out out shadowed datasets. 

516 temporal_join_on : `~collections.abc.Set` [ `ColumnTag` ], optional 

517 Timespan columns in ``join_to`` that calibration dataset timespans 

518 must overlap. Must already be present in ``join_to``. Ignored if 

519 ``join_to`` is `None` or if there are no calibration collections. 

520 

521 Returns 

522 ------- 

523 relation : `lsst.daf.relation.Relation` 

524 Relation representing a find-first dataset search. 

525 """ 

526 base = self.make_dataset_query_relation( 

527 dataset_type, 

528 collections, 

529 columns | {"rank"}, 

530 context=context, 

531 join_to=join_to, 

532 temporal_join_on=temporal_join_on, 

533 ) 

534 # Query-simplification shortcut: if there is only one collection, a 

535 # find-first search is just a regular result subquery. Same if there 

536 # are no collections. 

537 if len(collections) <= 1: 

538 return base 

539 # We filter the dimension keys in the given relation through 

540 # DimensionGroup.required.names to minimize the set we partition on 

541 # and order it in a more index-friendly way. More precisely, any 

542 # index we define on dimensions will be consistent with this order, but 

543 # any particular index may not have the same dimension columns. 

544 dimensions = self.universe.conform( 

545 [tag.dimension for tag in DimensionKeyColumnTag.filter_from(base.columns)] 

546 ) 

547 find_first = FindFirstDataset( 

548 dimensions=DimensionKeyColumnTag.generate(dimensions.required), 

549 rank=DatasetColumnTag(dataset_type.name, "rank"), 

550 ) 

551 return find_first.apply( 

552 base, preferred_engine=context.preferred_engine, require_preferred_engine=True 

553 ).with_only_columns(base.columns - {find_first.rank}) 

554 

555 def make_doomed_dataset_relation( 

556 self, 

557 dataset_type: DatasetType, 

558 columns: Set[str], 

559 messages: Iterable[str], 

560 context: _C, 

561 ) -> Relation: 

562 """Construct a relation that represents a doomed query for datasets. 

563 

564 Parameters 

565 ---------- 

566 dataset_type : `DatasetType` 

567 Dataset type being queried. 

568 columns : `~collections.abc.Set` [ `str` ] 

569 Dataset columns to include (dimension key columns are always 

570 included). See `make_dataset_query_relation` for allowed values. 

571 messages : `~collections.abc.Iterable` [ `str` ] 

572 Diagnostic messages that explain why the query is doomed to yield 

573 no rows. 

574 context : `QueryContext` 

575 Context that manages per-query state. 

576 

577 Returns 

578 ------- 

579 relation : `lsst.daf.relation.Relation` 

580 Relation with the requested columns and no rows. 

581 """ 

582 column_tags: set[ColumnTag] = set(DimensionKeyColumnTag.generate(dataset_type.dimensions.required)) 

583 column_tags.update(DatasetColumnTag.generate(dataset_type.name, columns)) 

584 return context.preferred_engine.make_doomed_relation(columns=column_tags, messages=list(messages)) 

585 

586 @abstractmethod 

587 def resolve_governor_constraints( 

588 self, dimensions: DimensionGroup, constraints: Mapping[str, Set[str]] 

589 ) -> Mapping[str, Set[str]]: 

590 """Resolve governor dimension constraints provided by user input to 

591 a query against the content in the `Registry`. 

592 

593 Parameters 

594 ---------- 

595 dimensions : `DimensionGroup` 

596 Dimensions that bound the governor dimensions to consider (via 

597 ``dimensions.governors``, more specifically). 

598 constraints : `~collections.abc.Mapping` [ `str`, \ 

599 `~collections.abc.Set` [ `str` ] ] 

600 Constraints from user input to the query (e.g. from data IDs and 

601 string expression predicates). 

602 

603 Returns 

604 ------- 

605 resolved : `~collections.abc.Mapping` [ `str`, \ 

606 `~collections.abc.Set` [ `str` ] ] 

607 A shallow copy of ``constraints`` with keys equal to 

608 ``dimensions.governors.names`` and value sets constrained by the 

609 Registry content if they were not already in ``constraints``. 

610 

611 Raises 

612 ------ 

613 DataIdValueError 

614 Raised if ``constraints`` includes governor dimension values that 

615 are not present in the `Registry`. 

616 """ 

617 raise NotImplementedError() 

618 

619 @abstractmethod 

620 def get_dimension_record_cache(self, element_name: str) -> DimensionRecordSet | None: 

621 """Return a local cache of all `DimensionRecord` objects for a 

622 dimension element, fetching it if necessary. 

623 

624 Parameters 

625 ---------- 

626 element_name : `str` 

627 Name of the dimension element. 

628 

629 Returns 

630 ------- 

631 cache : `~collections.abc.Mapping` [ `DataCoordinate`, \ 

632 `DimensionRecord` ] or `None` 

633 Mapping from data ID to dimension record, or `None` if this 

634 element's records are never cached. 

635 """ 

636 raise NotImplementedError() 

637 

638 def extract_dimension_relationships(self, relation: Relation) -> set[frozenset[str]]: 

639 """Extract the dimension key relationships encoded in a relation tree. 

640 

641 Parameters 

642 ---------- 

643 relation : `Relation` 

644 Relation tree to process. 

645 

646 Returns 

647 ------- 

648 relationships : `set` [ `frozenset` [ `str` ] ] 

649 Set of sets of dimension names, where each inner set represents a 

650 relationship between dimensions. 

651 

652 Notes 

653 ----- 

654 Dimension relationships include both many-to-one implied dependencies 

655 and many-to-many joins backed by "always-join" dimension elements, and 

656 it's important to join in the dimension table that defines a 

657 relationship in any query involving dimensions that are a superset of 

658 that relationship. For example, let's consider a relation tree that 

659 joins dataset existence-check relations for two dataset types, with 

660 dimensions ``{instrument, exposure, detector}`` and ``{instrument, 

661 physical_filter}``. The joined relation appears to have all dimension 

662 keys in its expanded graph present except ``band``, and the system 

663 could easily correct this by joining that dimension in directly. But 

664 it's also missing the ``{instrument, exposure, physical_filter}`` 

665 relationship we'd get from the ``exposure`` dimension's own relation 

666 (``exposure`` implies ``physical_filter``) and the similar 

667 ``{instrument, physical_filter, band}`` relationship from the 

668 ``physical_filter`` dimension relation; we need the relationship logic 

669 to recognize that those dimensions need to be joined in as well in 

670 order for the full relation to have rows that represent valid data IDs. 

671 

672 The implementation of this method relies on the assumption that 

673 `LeafRelation` objects always have rows that are consistent with all 

674 defined relationships (i.e. are valid data IDs). This is true for not 

675 just dimension relations themselves, but anything created from queries 

676 based on them, including datasets and query results. It is possible to 

677 construct `LeafRelation` objects that don't satisfy this criteria (e.g. 

678 when accepting in user-provided data IDs), and in this case 

679 higher-level guards or warnings must be provided.`` 

680 """ 

681 return { 

682 frozenset( 

683 tag.dimension 

684 for tag in DimensionKeyColumnTag.filter_from(leaf_relation.columns & relation.columns) 

685 ) 

686 for leaf_relation in self._extract_leaf_relations(relation).values() 

687 } 

688 

689 def _extract_leaf_relations(self, relation: Relation) -> dict[str, LeafRelation]: 

690 """Recursively extract leaf relations from a relation tree. 

691 

692 Parameters 

693 ---------- 

694 relation : `Relation` 

695 Tree to process. 

696 

697 Returns 

698 ------- 

699 leaves : `dict` [ `str`, `LeafRelation` ] 

700 Leaf relations, keyed and deduplicated by name. 

701 """ 

702 match relation: 

703 case LeafRelation() as leaf: 

704 return {leaf.name: leaf} 

705 case UnaryOperationRelation(target=target): 

706 return self._extract_leaf_relations(target) 

707 case BinaryOperationRelation(lhs=lhs, rhs=rhs): 

708 return self._extract_leaf_relations(lhs) | self._extract_leaf_relations(rhs) 

709 case MarkerRelation(target=target): 

710 return self._extract_leaf_relations(target) 

711 raise AssertionError("Match should be exhaustive and all branches should return.")