Coverage for python/lsst/daf/butler/core/dimensions/_coordinate.py: 33%

362 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-14 19:21 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22# 

23# Design notes for this module are in 

24# doc/lsst.daf.butler/dev/dataCoordinate.py. 

25# 

26 

27from __future__ import annotations 

28 

29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue", "SerializedDataCoordinate") 

30 

31import numbers 

32from abc import abstractmethod 

33from collections.abc import Iterator, Mapping, Set 

34from typing import TYPE_CHECKING, Any, ClassVar, Literal, overload 

35 

36from deprecated.sphinx import deprecated 

37from lsst.sphgeom import IntersectionRegion, Region 

38 

39try: 

40 from pydantic.v1 import BaseModel 

41except ModuleNotFoundError: 

42 from pydantic import BaseModel # type: ignore 

43 

44from ..json import from_json_pydantic, to_json_pydantic 

45from ..named import NamedKeyDict, NamedKeyMapping, NamedValueAbstractSet, NameLookupMapping 

46from ..persistenceContext import PersistenceContextVars 

47from ..timespan import Timespan 

48from ._elements import Dimension, DimensionElement 

49from ._graph import DimensionGraph 

50from ._records import DimensionRecord, SerializedDimensionRecord 

51 

52if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

53 from ...registry import Registry 

54 from ._universe import DimensionUniverse 

55 

56DataIdKey = str | Dimension 

57"""Type annotation alias for the keys that can be used to index a 

58DataCoordinate. 

59""" 

60 

61# Pydantic will cast int to str if str is first in the Union. 

62DataIdValue = int | str | None 

63"""Type annotation alias for the values that can be present in a 

64DataCoordinate or other data ID. 

65""" 

66 

67 

68class SerializedDataCoordinate(BaseModel): 

69 """Simplified model for serializing a `DataCoordinate`.""" 

70 

71 dataId: dict[str, DataIdValue] 

72 records: dict[str, SerializedDimensionRecord] | None = None 

73 

74 @classmethod 

75 def direct(cls, *, dataId: dict[str, DataIdValue], records: dict[str, dict]) -> SerializedDataCoordinate: 

76 """Construct a `SerializedDataCoordinate` directly without validators. 

77 

78 This differs from the pydantic "construct" method in that the arguments 

79 are explicitly what the model requires, and it will recurse through 

80 members, constructing them from their corresponding `direct` methods. 

81 

82 This method should only be called when the inputs are trusted. 

83 """ 

84 key = (frozenset(dataId.items()), records is not None) 

85 cache = PersistenceContextVars.serializedDataCoordinateMapping.get() 

86 if cache is not None and (result := cache.get(key)) is not None: 

87 return result 

88 node = SerializedDataCoordinate.__new__(cls) 

89 setter = object.__setattr__ 

90 setter(node, "dataId", dataId) 

91 setter( 

92 node, 

93 "records", 

94 records 

95 if records is None 

96 else {k: SerializedDimensionRecord.direct(**v) for k, v in records.items()}, 

97 ) 

98 setter(node, "__fields_set__", {"dataId", "records"}) 

99 if cache is not None: 

100 cache[key] = node 

101 return node 

102 

103 

104def _intersectRegions(*args: Region) -> Region | None: 

105 """Return the intersection of several regions. 

106 

107 For internal use by `ExpandedDataCoordinate` only. 

108 

109 If no regions are provided, returns `None`. 

110 """ 

111 if len(args) == 0: 

112 return None 

113 else: 

114 result = args[0] 

115 for n in range(1, len(args)): 

116 result = IntersectionRegion(result, args[n]) 

117 return result 

118 

119 

120class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]): 

121 """Data ID dictionary. 

122 

123 An immutable data ID dictionary that guarantees that its key-value pairs 

124 identify at least all required dimensions in a `DimensionGraph`. 

125 

126 `DataCoordinate` itself is an ABC, but provides `staticmethod` factory 

127 functions for private concrete implementations that should be sufficient 

128 for most purposes. `standardize` is the most flexible and safe of these; 

129 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are 

130 more specialized and perform little or no checking of inputs. 

131 

132 Notes 

133 ----- 

134 Like any data ID class, `DataCoordinate` behaves like a dictionary, but 

135 with some subtleties: 

136 

137 - Both `Dimension` instances and `str` names thereof may be used as keys 

138 in lookup operations, but iteration (and `keys`) will yield `Dimension` 

139 instances. The `names` property can be used to obtain the corresponding 

140 `str` names. 

141 

142 - Lookups for implied dimensions (those in ``self.graph.implied``) are 

143 supported if and only if `hasFull` returns `True`, and are never 

144 included in iteration or `keys`. The `full` property may be used to 

145 obtain a mapping whose keys do include implied dimensions. 

146 

147 - Equality comparison with other mappings is supported, but it always 

148 considers only required dimensions (as well as requiring both operands 

149 to identify the same dimensions). This is not quite consistent with the 

150 way mappings usually work - normally differing keys imply unequal 

151 mappings - but it makes sense in this context because data IDs with the 

152 same values for required dimensions but different values for implied 

153 dimensions represent a serious problem with the data that 

154 `DataCoordinate` cannot generally recognize on its own, and a data ID 

155 that knows implied dimension values should still be able to compare as 

156 equal to one that does not. This is of course not the way comparisons 

157 between simple `dict` data IDs work, and hence using a `DataCoordinate` 

158 instance for at least one operand in any data ID comparison is strongly 

159 recommended. 

160 

161 See Also 

162 -------- 

163 :ref:`lsst.daf.butler-dimensions_data_ids` 

164 """ 

165 

166 __slots__ = () 

167 

168 _serializedType = SerializedDataCoordinate 

169 

170 @staticmethod 

171 def standardize( 

172 mapping: NameLookupMapping[Dimension, DataIdValue] | None = None, 

173 *, 

174 graph: DimensionGraph | None = None, 

175 universe: DimensionUniverse | None = None, 

176 defaults: DataCoordinate | None = None, 

177 **kwargs: Any, 

178 ) -> DataCoordinate: 

179 """Standardize the supplied dataId. 

180 

181 Adapts an arbitrary mapping and/or additional arguments into a true 

182 `DataCoordinate`, or augment an existing one. 

183 

184 Parameters 

185 ---------- 

186 mapping : `~collections.abc.Mapping`, optional 

187 An informal data ID that maps dimensions or dimension names to 

188 their primary key values (may also be a true `DataCoordinate`). 

189 graph : `DimensionGraph` 

190 The dimensions to be identified by the new `DataCoordinate`. 

191 If not provided, will be inferred from the keys of ``mapping`` and 

192 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

193 is already a `DataCoordinate`. 

194 universe : `DimensionUniverse` 

195 All known dimensions and their relationships; used to expand 

196 and validate dependencies when ``graph`` is not provided. 

197 defaults : `DataCoordinate`, optional 

198 Default dimension key-value pairs to use when needed. These are 

199 never used to infer ``graph``, and are ignored if a different value 

200 is provided for the same key in ``mapping`` or `**kwargs``. 

201 **kwargs 

202 Additional keyword arguments are treated like additional key-value 

203 pairs in ``mapping``. 

204 

205 Returns 

206 ------- 

207 coordinate : `DataCoordinate` 

208 A validated `DataCoordinate` instance. 

209 

210 Raises 

211 ------ 

212 TypeError 

213 Raised if the set of optional arguments provided is not supported. 

214 KeyError 

215 Raised if a key-value pair for a required dimension is missing. 

216 """ 

217 d: dict[str, DataIdValue] = {} 

218 if isinstance(mapping, DataCoordinate): 

219 if graph is None: 

220 if not kwargs: 

221 # Already standardized to exactly what we want. 

222 return mapping 

223 elif kwargs.keys().isdisjoint(graph.dimensions.names): 

224 # User provided kwargs, but told us not to use them by 

225 # passing in dimensions that are disjoint from those kwargs. 

226 # This is not necessarily user error - it's a useful pattern 

227 # to pass in all of the key-value pairs you have and let the 

228 # code here pull out only what it needs. 

229 return mapping.subset(graph) 

230 assert universe is None or universe == mapping.universe 

231 universe = mapping.universe 

232 d.update((name, mapping[name]) for name in mapping.graph.required.names) 

233 if mapping.hasFull(): 

234 d.update((name, mapping[name]) for name in mapping.graph.implied.names) 

235 elif isinstance(mapping, NamedKeyMapping): 

236 d.update(mapping.byName()) 

237 elif mapping is not None: 

238 d.update(mapping) 

239 d.update(kwargs) 

240 if graph is None: 

241 if defaults is not None: 

242 universe = defaults.universe 

243 elif universe is None: 

244 raise TypeError("universe must be provided if graph is not.") 

245 graph = DimensionGraph(universe, names=d.keys()) 

246 if not graph.dimensions: 

247 return DataCoordinate.makeEmpty(graph.universe) 

248 if defaults is not None: 

249 if defaults.hasFull(): 

250 for k, v in defaults.full.items(): 

251 d.setdefault(k.name, v) 

252 else: 

253 for k, v in defaults.items(): 

254 d.setdefault(k.name, v) 

255 if d.keys() >= graph.dimensions.names: 

256 values = tuple(d[name] for name in graph._dataCoordinateIndices.keys()) 

257 else: 

258 try: 

259 values = tuple(d[name] for name in graph.required.names) 

260 except KeyError as err: 

261 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err 

262 # Some backends cannot handle numpy.int64 type which is a subclass of 

263 # numbers.Integral; convert that to int. 

264 values = tuple( 

265 int(val) if isinstance(val, numbers.Integral) else val for val in values # type: ignore 

266 ) 

267 return _BasicTupleDataCoordinate(graph, values) 

268 

269 @staticmethod 

270 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate: 

271 """Return an empty `DataCoordinate`. 

272 

273 It identifies the null set of dimensions. 

274 

275 Parameters 

276 ---------- 

277 universe : `DimensionUniverse` 

278 Universe to which this null dimension set belongs. 

279 

280 Returns 

281 ------- 

282 dataId : `DataCoordinate` 

283 A data ID object that identifies no dimensions. `hasFull` and 

284 `hasRecords` are guaranteed to return `True`, because both `full` 

285 and `records` are just empty mappings. 

286 """ 

287 return _ExpandedTupleDataCoordinate(universe.empty, (), {}) 

288 

289 @staticmethod 

290 def fromRequiredValues(graph: DimensionGraph, values: tuple[DataIdValue, ...]) -> DataCoordinate: 

291 """Construct a `DataCoordinate` from required dimension values. 

292 

293 This is a low-level interface with at most assertion-level checking of 

294 inputs. Most callers should use `standardize` instead. 

295 

296 Parameters 

297 ---------- 

298 graph : `DimensionGraph` 

299 Dimensions this data ID will identify. 

300 values : `tuple` [ `int` or `str` ] 

301 Tuple of primary key values corresponding to ``graph.required``, 

302 in that order. 

303 

304 Returns 

305 ------- 

306 dataId : `DataCoordinate` 

307 A data ID object that identifies the given dimensions. 

308 ``dataId.hasFull()`` will return `True` if and only if 

309 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

310 return `True`. 

311 """ 

312 assert len(graph.required) == len( 

313 values 

314 ), f"Inconsistency between dimensions {graph.required} and required values {values}." 

315 return _BasicTupleDataCoordinate(graph, values) 

316 

317 @staticmethod 

318 def fromFullValues(graph: DimensionGraph, values: tuple[DataIdValue, ...]) -> DataCoordinate: 

319 """Construct a `DataCoordinate` from all dimension values. 

320 

321 This is a low-level interface with at most assertion-level checking of 

322 inputs. Most callers should use `standardize` instead. 

323 

324 Parameters 

325 ---------- 

326 graph : `DimensionGraph` 

327 Dimensions this data ID will identify. 

328 values : `tuple` [ `int` or `str` ] 

329 Tuple of primary key values corresponding to 

330 ``itertools.chain(graph.required, graph.implied)``, in that order. 

331 Note that this is _not_ the same order as ``graph.dimensions``, 

332 though these contain the same elements. 

333 

334 Returns 

335 ------- 

336 dataId : `DataCoordinate` 

337 A data ID object that identifies the given dimensions. 

338 ``dataId.hasFull()`` will return `True` if and only if 

339 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

340 return `True`. 

341 """ 

342 assert len(graph.dimensions) == len( 

343 values 

344 ), f"Inconsistency between dimensions {graph.dimensions} and full values {values}." 

345 return _BasicTupleDataCoordinate(graph, values) 

346 

347 def __hash__(self) -> int: 

348 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required)) 

349 

350 def __eq__(self, other: Any) -> bool: 

351 if not isinstance(other, DataCoordinate): 

352 other = DataCoordinate.standardize(other, universe=self.universe) 

353 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required) 

354 

355 def __repr__(self) -> str: 

356 # We can't make repr yield something that could be exec'd here without 

357 # printing out the whole DimensionUniverse the graph is derived from. 

358 # So we print something that mostly looks like a dict, but doesn't 

359 # quote its keys: that's both more compact and something that can't 

360 # be mistaken for an actual dict or something that could be exec'd. 

361 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names] 

362 if self.hasFull() and self.graph.required != self.graph.dimensions: 

363 terms.append("...") 

364 return "{{{}}}".format(", ".join(terms)) 

365 

366 def __lt__(self, other: Any) -> bool: 

367 # Allow DataCoordinate to be sorted 

368 if not isinstance(other, type(self)): 

369 return NotImplemented 

370 # Form tuple of tuples for each DataCoordinate: 

371 # Unlike repr() we only use required keys here to ensure that 

372 # __eq__ can not be true simultaneously with __lt__ being true. 

373 self_kv = tuple(self.items()) 

374 other_kv = tuple(other.items()) 

375 

376 return self_kv < other_kv 

377 

378 def __iter__(self) -> Iterator[Dimension]: 

379 return iter(self.keys()) 

380 

381 def __len__(self) -> int: 

382 return len(self.keys()) 

383 

384 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

385 return self.graph.required 

386 

387 @property 

388 def names(self) -> Set[str]: 

389 """Names of the required dimensions identified by this data ID. 

390 

391 They are returned in the same order as `keys` 

392 (`collections.abc.Set` [ `str` ]). 

393 """ 

394 return self.keys().names 

395 

396 @abstractmethod 

397 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

398 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``. 

399 

400 Parameters 

401 ---------- 

402 graph : `DimensionGraph` 

403 The dimensions identified by the returned `DataCoordinate`. 

404 

405 Returns 

406 ------- 

407 coordinate : `DataCoordinate` 

408 A `DataCoordinate` instance that identifies only the given 

409 dimensions. May be ``self`` if ``graph == self.graph``. 

410 

411 Raises 

412 ------ 

413 KeyError 

414 Raised if the primary key value for one or more required dimensions 

415 is unknown. This may happen if ``graph.issubset(self.graph)`` is 

416 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if 

417 ``self.hasFull()`` is `False` and 

418 ``graph.required.issubset(self.graph.required)`` is `False`. As 

419 an example of the latter case, consider trying to go from a data ID 

420 with dimensions {instrument, physical_filter, band} to 

421 just {instrument, band}; band is implied by 

422 physical_filter and hence would have no value in the original data 

423 ID if ``self.hasFull()`` is `False`. 

424 

425 Notes 

426 ----- 

427 If `hasFull` and `hasRecords` return `True` on ``self``, they will 

428 return `True` (respectively) on the returned `DataCoordinate` as well. 

429 The converse does not hold. 

430 """ 

431 raise NotImplementedError() 

432 

433 @abstractmethod 

434 def union(self, other: DataCoordinate) -> DataCoordinate: 

435 """Combine two data IDs. 

436 

437 Yields a new one that identifies all dimensions that either of them 

438 identify. 

439 

440 Parameters 

441 ---------- 

442 other : `DataCoordinate` 

443 Data ID to combine with ``self``. 

444 

445 Returns 

446 ------- 

447 unioned : `DataCoordinate` 

448 A `DataCoordinate` instance that satisfies 

449 ``unioned.graph == self.graph.union(other.graph)``. Will preserve 

450 ``hasFull`` and ``hasRecords`` whenever possible. 

451 

452 Notes 

453 ----- 

454 No checking for consistency is performed on values for keys that 

455 ``self`` and ``other`` have in common, and which value is included in 

456 the returned data ID is not specified. 

457 """ 

458 raise NotImplementedError() 

459 

460 @abstractmethod 

461 def expanded( 

462 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None] 

463 ) -> DataCoordinate: 

464 """Return a `DataCoordinate` that holds the given records. 

465 

466 Guarantees that `hasRecords` returns `True`. 

467 

468 This is a low-level interface with at most assertion-level checking of 

469 inputs. Most callers should use `Registry.expandDataId` instead. 

470 

471 Parameters 

472 ---------- 

473 records : `~collections.abc.Mapping` [ `str`, `DimensionRecord` or \ 

474 `None` ] 

475 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

476 `~collections.abc.Mapping` with `str` (`DimensionElement` name) 

477 keys and `DimensionRecord` values. Keys must cover all elements in 

478 ``self.graph.elements``. Values may be `None`, but only to reflect 

479 actual NULL values in the database, not just records that have not 

480 been fetched. 

481 """ 

482 raise NotImplementedError() 

483 

484 @property 

485 def universe(self) -> DimensionUniverse: 

486 """Universe that defines all known compatible dimensions. 

487 

488 The univers will be compatible with this coordinate 

489 (`DimensionUniverse`). 

490 """ 

491 return self.graph.universe 

492 

493 @property 

494 @abstractmethod 

495 def graph(self) -> DimensionGraph: 

496 """Dimensions identified by this data ID (`DimensionGraph`). 

497 

498 Note that values are only required to be present for dimensions in 

499 ``self.graph.required``; all others may be retrieved (from a 

500 `Registry`) given these. 

501 """ 

502 raise NotImplementedError() 

503 

504 @abstractmethod 

505 def hasFull(self) -> bool: 

506 """Whether this data ID contains implied and required values. 

507 

508 Returns 

509 ------- 

510 state : `bool` 

511 If `True`, `__getitem__`, `get`, and `__contains__` (but not 

512 `keys`!) will act as though the mapping includes key-value pairs 

513 for implied dimensions, and the `full` property may be used. If 

514 `False`, these operations only include key-value pairs for required 

515 dimensions, and accessing `full` is an error. Always `True` if 

516 there are no implied dimensions. 

517 """ 

518 raise NotImplementedError() 

519 

520 @property 

521 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]: 

522 """Return mapping for all dimensions in ``self.graph``. 

523 

524 The mapping includes key-value pairs for all dimensions in 

525 ``self.graph``, including implied (`NamedKeyMapping`). 

526 

527 Accessing this attribute if `hasFull` returns `False` is a logic error 

528 that may raise an exception of unspecified type either immediately or 

529 when implied keys are accessed via the returned mapping, depending on 

530 the implementation and whether assertions are enabled. 

531 """ 

532 assert self.hasFull(), "full may only be accessed if hasFull() returns True." 

533 return _DataCoordinateFullView(self) 

534 

535 @abstractmethod 

536 def hasRecords(self) -> bool: 

537 """Whether this data ID contains records. 

538 

539 These are the records for all of the dimension elements it identifies. 

540 

541 Returns 

542 ------- 

543 state : `bool` 

544 If `True`, the following attributes may be accessed: 

545 

546 - `records` 

547 - `region` 

548 - `timespan` 

549 - `pack` 

550 

551 If `False`, accessing any of these is considered a logic error. 

552 """ 

553 raise NotImplementedError() 

554 

555 @property 

556 def records(self) -> NamedKeyMapping[DimensionElement, DimensionRecord | None]: 

557 """Return the records. 

558 

559 Returns a mapping that contains `DimensionRecord` objects for all 

560 elements identified by this data ID (`NamedKeyMapping`). 

561 

562 The values of this mapping may be `None` if and only if there is no 

563 record for that element with these dimensions in the database (which 

564 means some foreign key field must have a NULL value). 

565 

566 Accessing this attribute if `hasRecords` returns `False` is a logic 

567 error that may raise an exception of unspecified type either 

568 immediately or when the returned mapping is used, depending on the 

569 implementation and whether assertions are enabled. 

570 """ 

571 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True." 

572 return _DataCoordinateRecordsView(self) 

573 

574 @abstractmethod 

575 def _record(self, name: str) -> DimensionRecord | None: 

576 """Protected implementation hook that backs the ``records`` attribute. 

577 

578 Parameters 

579 ---------- 

580 name : `str` 

581 The name of a `DimensionElement`, guaranteed to be in 

582 ``self.graph.elements.names``. 

583 

584 Returns 

585 ------- 

586 record : `DimensionRecord` or `None` 

587 The dimension record for the given element identified by this 

588 data ID, or `None` if there is no such record. 

589 """ 

590 raise NotImplementedError() 

591 

592 @property 

593 def region(self) -> Region | None: 

594 """Spatial region associated with this data ID. 

595 

596 (`lsst.sphgeom.Region` or `None`). 

597 

598 This is `None` if and only if ``self.graph.spatial`` is empty. 

599 

600 Accessing this attribute if `hasRecords` returns `False` is a logic 

601 error that may or may not raise an exception, depending on the 

602 implementation and whether assertions are enabled. 

603 """ 

604 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True." 

605 regions = [] 

606 for family in self.graph.spatial: 

607 element = family.choose(self.graph.elements) 

608 record = self._record(element.name) 

609 if record is None or record.region is None: 

610 return None 

611 else: 

612 regions.append(record.region) 

613 return _intersectRegions(*regions) 

614 

615 @property 

616 def timespan(self) -> Timespan | None: 

617 """Temporal interval associated with this data ID. 

618 

619 (`Timespan` or `None`). 

620 

621 This is `None` if and only if ``self.graph.timespan`` is empty. 

622 

623 Accessing this attribute if `hasRecords` returns `False` is a logic 

624 error that may or may not raise an exception, depending on the 

625 implementation and whether assertions are enabled. 

626 """ 

627 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True." 

628 timespans = [] 

629 for family in self.graph.temporal: 

630 element = family.choose(self.graph.elements) 

631 record = self._record(element.name) 

632 # DimensionRecord subclasses for temporal elements always have 

633 # .timespan, but they're dynamic so this can't be type-checked. 

634 if record is None or record.timespan is None: 

635 return None 

636 else: 

637 timespans.append(record.timespan) 

638 if not timespans: 

639 return None 

640 elif len(timespans) == 1: 

641 return timespans[0] 

642 else: 

643 return Timespan.intersection(*timespans) 

644 

645 @overload 

646 def pack(self, name: str, *, returnMaxBits: Literal[True]) -> tuple[int, int]: 

647 ... 

648 

649 @overload 

650 def pack(self, name: str, *, returnMaxBits: Literal[False]) -> int: 

651 ... 

652 

653 # TODO: Remove this method and its overloads above on DM-38687. 

654 @deprecated( 

655 "Deprecated in favor of configurable dimension packers. Will be removed after v27.", 

656 version="v26", 

657 category=FutureWarning, 

658 ) 

659 def pack(self, name: str, *, returnMaxBits: bool = False) -> tuple[int, int] | int: 

660 """Pack this data ID into an integer. 

661 

662 Parameters 

663 ---------- 

664 name : `str` 

665 Name of the `DimensionPacker` algorithm (as defined in the 

666 dimension configuration). 

667 returnMaxBits : `bool`, optional 

668 If `True` (`False` is default), return the maximum number of 

669 nonzero bits in the returned integer across all data IDs. 

670 

671 Returns 

672 ------- 

673 packed : `int` 

674 Integer ID. This ID is unique only across data IDs that have 

675 the same values for the packer's "fixed" dimensions. 

676 maxBits : `int`, optional 

677 Maximum number of nonzero bits in ``packed``. Not returned unless 

678 ``returnMaxBits`` is `True`. 

679 

680 Notes 

681 ----- 

682 Accessing this attribute if `hasRecords` returns `False` is a logic 

683 error that may or may not raise an exception, depending on the 

684 implementation and whether assertions are enabled. 

685 """ 

686 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." 

687 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) 

688 

689 def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate: 

690 """Convert this class to a simple python type. 

691 

692 This is suitable for serialization. 

693 

694 Parameters 

695 ---------- 

696 minimal : `bool`, optional 

697 Use minimal serialization. If set the records will not be attached. 

698 

699 Returns 

700 ------- 

701 simple : `SerializedDataCoordinate` 

702 The object converted to simple form. 

703 """ 

704 # Convert to a dict form 

705 if self.hasFull(): 

706 dataId = self.full.byName() 

707 else: 

708 dataId = self.byName() 

709 records: dict[str, SerializedDimensionRecord] | None 

710 if not minimal and self.hasRecords(): 

711 records = {k: v.to_simple() for k, v in self.records.byName().items() if v is not None} 

712 else: 

713 records = None 

714 

715 return SerializedDataCoordinate(dataId=dataId, records=records) 

716 

717 @classmethod 

718 def from_simple( 

719 cls, 

720 simple: SerializedDataCoordinate, 

721 universe: DimensionUniverse | None = None, 

722 registry: Registry | None = None, 

723 ) -> DataCoordinate: 

724 """Construct a new object from the simplified form. 

725 

726 The data is assumed to be of the form returned from the `to_simple` 

727 method. 

728 

729 Parameters 

730 ---------- 

731 simple : `dict` of [`str`, `Any`] 

732 The `dict` returned by `to_simple()`. 

733 universe : `DimensionUniverse` 

734 The special graph of all known dimensions. 

735 registry : `lsst.daf.butler.Registry`, optional 

736 Registry from which a universe can be extracted. Can be `None` 

737 if universe is provided explicitly. 

738 

739 Returns 

740 ------- 

741 dataId : `DataCoordinate` 

742 Newly-constructed object. 

743 """ 

744 key = (frozenset(simple.dataId.items()), simple.records is not None) 

745 cache = PersistenceContextVars.dataCoordinates.get() 

746 if cache is not None and (result := cache.get(key)) is not None: 

747 return result 

748 if universe is None and registry is None: 

749 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

750 if universe is None and registry is not None: 

751 universe = registry.dimensions 

752 if universe is None: 

753 # this is for mypy 

754 raise ValueError("Unable to determine a usable universe") 

755 

756 dataId = cls.standardize(simple.dataId, universe=universe) 

757 if simple.records: 

758 dataId = dataId.expanded( 

759 {k: DimensionRecord.from_simple(v, universe=universe) for k, v in simple.records.items()} 

760 ) 

761 if cache is not None: 

762 cache[key] = dataId 

763 return dataId 

764 

765 to_json = to_json_pydantic 

766 from_json: ClassVar = classmethod(from_json_pydantic) 

767 

768 

769DataId = DataCoordinate | Mapping[str, Any] 

770"""A type-annotation alias for signatures that accept both informal data ID 

771dictionaries and validated `DataCoordinate` instances. 

772""" 

773 

774 

775class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]): 

776 """View class for `DataCoordinate.full`. 

777 

778 Provides the default implementation for 

779 `DataCoordinate.full`. 

780 

781 Parameters 

782 ---------- 

783 target : `DataCoordinate` 

784 The `DataCoordinate` instance this object provides a view of. 

785 """ 

786 

787 def __init__(self, target: DataCoordinate): 

788 self._target = target 

789 

790 __slots__ = ("_target",) 

791 

792 def __repr__(self) -> str: 

793 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names] 

794 return "{{{}}}".format(", ".join(terms)) 

795 

796 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

797 return self._target[key] 

798 

799 def __iter__(self) -> Iterator[Dimension]: 

800 return iter(self.keys()) 

801 

802 def __len__(self) -> int: 

803 return len(self.keys()) 

804 

805 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

806 return self._target.graph.dimensions 

807 

808 @property 

809 def names(self) -> Set[str]: 

810 # Docstring inherited from `NamedKeyMapping`. 

811 return self.keys().names 

812 

813 

814class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, DimensionRecord | None]): 

815 """View class for `DataCoordinate.records`. 

816 

817 Provides the default implementation for 

818 `DataCoordinate.records`. 

819 

820 Parameters 

821 ---------- 

822 target : `DataCoordinate` 

823 The `DataCoordinate` instance this object provides a view of. 

824 """ 

825 

826 def __init__(self, target: DataCoordinate): 

827 self._target = target 

828 

829 __slots__ = ("_target",) 

830 

831 def __repr__(self) -> str: 

832 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names] 

833 return "{{{}}}".format(", ".join(terms)) 

834 

835 def __str__(self) -> str: 

836 return "\n".join(str(v) for v in self.values()) 

837 

838 def __getitem__(self, key: DimensionElement | str) -> DimensionRecord | None: 

839 if isinstance(key, DimensionElement): 

840 key = key.name 

841 return self._target._record(key) 

842 

843 def __iter__(self) -> Iterator[DimensionElement]: 

844 return iter(self.keys()) 

845 

846 def __len__(self) -> int: 

847 return len(self.keys()) 

848 

849 def keys(self) -> NamedValueAbstractSet[DimensionElement]: # type: ignore 

850 return self._target.graph.elements 

851 

852 @property 

853 def names(self) -> Set[str]: 

854 # Docstring inherited from `NamedKeyMapping`. 

855 return self.keys().names 

856 

857 

858class _BasicTupleDataCoordinate(DataCoordinate): 

859 """Standard implementation of `DataCoordinate`. 

860 

861 Backed by a tuple of values. 

862 

863 This class should only be accessed outside this module via the 

864 `DataCoordinate` interface, and should only be constructed via the static 

865 methods there. 

866 

867 Parameters 

868 ---------- 

869 graph : `DimensionGraph` 

870 The dimensions to be identified. 

871 values : `tuple` [ `int` or `str` ] 

872 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May 

873 include values for just required dimensions (which always come first) 

874 or all dimensions. 

875 """ 

876 

877 def __init__(self, graph: DimensionGraph, values: tuple[DataIdValue, ...]): 

878 self._graph = graph 

879 self._values = values 

880 

881 __slots__ = ("_graph", "_values") 

882 

883 @property 

884 def graph(self) -> DimensionGraph: 

885 # Docstring inherited from DataCoordinate. 

886 return self._graph 

887 

888 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

889 # Docstring inherited from DataCoordinate. 

890 if isinstance(key, Dimension): 

891 key = key.name 

892 index = self._graph._dataCoordinateIndices[key] 

893 try: 

894 return self._values[index] 

895 except IndexError: 

896 # Caller asked for an implied dimension, but this object only has 

897 # values for the required ones. 

898 raise KeyError(key) from None 

899 

900 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

901 # Docstring inherited from DataCoordinate. 

902 if self._graph == graph: 

903 return self 

904 elif self.hasFull() or self._graph.required >= graph.dimensions: 

905 return _BasicTupleDataCoordinate( 

906 graph, 

907 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

908 ) 

909 else: 

910 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names)) 

911 

912 def union(self, other: DataCoordinate) -> DataCoordinate: 

913 # Docstring inherited from DataCoordinate. 

914 graph = self.graph.union(other.graph) 

915 # See if one or both input data IDs is already what we want to return; 

916 # if so, return the most complete one we have. 

917 if other.graph == graph: 

918 if self.graph == graph: 

919 # Input data IDs have the same graph (which is also the result 

920 # graph), but may not have the same content. 

921 # other might have records; self does not, so try other first. 

922 # If it at least has full values, it's no worse than self. 

923 if other.hasFull(): 

924 return other 

925 else: 

926 return self 

927 elif other.hasFull(): 

928 return other 

929 # There's some chance that neither self nor other has full values, 

930 # but together provide enough to the union to. Let the general 

931 # case below handle that. 

932 elif self.graph == graph: 

933 # No chance at returning records. If self has full values, it's 

934 # the best we can do. 

935 if self.hasFull(): 

936 return self 

937 # General case with actual merging of dictionaries. 

938 values = self.full.byName() if self.hasFull() else self.byName() 

939 values.update(other.full.byName() if other.hasFull() else other.byName()) 

940 return DataCoordinate.standardize(values, graph=graph) 

941 

942 def expanded( 

943 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None] 

944 ) -> DataCoordinate: 

945 # Docstring inherited from DataCoordinate 

946 values = self._values 

947 if not self.hasFull(): 

948 # Extract a complete values tuple from the attributes of the given 

949 # records. It's possible for these to be inconsistent with 

950 # self._values (which is a serious problem, of course), but we've 

951 # documented this as a no-checking API. 

952 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied) 

953 return _ExpandedTupleDataCoordinate(self._graph, values, records) 

954 

955 def hasFull(self) -> bool: 

956 # Docstring inherited from DataCoordinate. 

957 return len(self._values) == len(self._graph._dataCoordinateIndices) 

958 

959 def hasRecords(self) -> bool: 

960 # Docstring inherited from DataCoordinate. 

961 return False 

962 

963 def _record(self, name: str) -> DimensionRecord | None: 

964 # Docstring inherited from DataCoordinate. 

965 assert False 

966 

967 def __reduce__(self) -> tuple[Any, ...]: 

968 return (_BasicTupleDataCoordinate, (self._graph, self._values)) 

969 

970 def __getattr__(self, name: str) -> Any: 

971 if name in self.graph.elements.names: 

972 raise AttributeError( 

973 f"Dimension record attribute {name!r} is only available on expanded DataCoordinates." 

974 ) 

975 raise AttributeError(name) 

976 

977 

978class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate): 

979 """A `DataCoordinate` implementation that can hold `DimensionRecord`. 

980 

981 This class should only be accessed outside this module via the 

982 `DataCoordinate` interface, and should only be constructed via calls to 

983 `DataCoordinate.expanded`. 

984 

985 Parameters 

986 ---------- 

987 graph : `DimensionGraph` 

988 The dimensions to be identified. 

989 values : `tuple` [ `int` or `str` ] 

990 Data ID values, ordered to match ``graph._dataCoordinateIndices``. 

991 May include values for just required dimensions (which always come 

992 first) or all dimensions. 

993 records : `~collections.abc.Mapping` [ `str`, `DimensionRecord` or `None` ] 

994 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

995 `~collections.abc.Mapping` with `str` (`DimensionElement` name) keys 

996 and `DimensionRecord` values. Keys must cover all elements in 

997 ``self.graph.elements``. Values may be `None`, but only to reflect 

998 actual NULL values in the database, not just records that have not 

999 been fetched. 

1000 """ 

1001 

1002 def __init__( 

1003 self, 

1004 graph: DimensionGraph, 

1005 values: tuple[DataIdValue, ...], 

1006 records: NameLookupMapping[DimensionElement, DimensionRecord | None], 

1007 ): 

1008 super().__init__(graph, values) 

1009 assert super().hasFull(), "This implementation requires full dimension records." 

1010 self._records = records 

1011 

1012 __slots__ = ("_records",) 

1013 

1014 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

1015 # Docstring inherited from DataCoordinate. 

1016 if self._graph == graph: 

1017 return self 

1018 return _ExpandedTupleDataCoordinate( 

1019 graph, tuple(self[k] for k in graph._dataCoordinateIndices.keys()), records=self._records 

1020 ) 

1021 

1022 def expanded( 

1023 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None] 

1024 ) -> DataCoordinate: 

1025 # Docstring inherited from DataCoordinate. 

1026 return self 

1027 

1028 def union(self, other: DataCoordinate) -> DataCoordinate: 

1029 # Docstring inherited from DataCoordinate. 

1030 graph = self.graph.union(other.graph) 

1031 # See if one or both input data IDs is already what we want to return; 

1032 # if so, return the most complete one we have. 

1033 if self.graph == graph: 

1034 # self has records, so even if other is also a valid result, it's 

1035 # no better. 

1036 return self 

1037 if other.graph == graph: 

1038 # If other has full values, and self does not identify some of 

1039 # those, it's the base we can do. It may have records, too. 

1040 if other.hasFull(): 

1041 return other 

1042 # If other does not have full values, there's a chance self may 

1043 # provide the values needed to complete it. For example, self 

1044 # could be {band} while other could be 

1045 # {instrument, physical_filter, band}, with band unknown. 

1046 # General case with actual merging of dictionaries. 

1047 values = self.full.byName() 

1048 values.update(other.full.byName() if other.hasFull() else other.byName()) 

1049 basic = DataCoordinate.standardize(values, graph=graph) 

1050 # See if we can add records. 

1051 if self.hasRecords() and other.hasRecords(): 

1052 # Sometimes the elements of a union of graphs can contain elements 

1053 # that weren't in either input graph (because graph unions are only 

1054 # on dimensions). e.g. {visit} | {detector} brings along 

1055 # visit_detector_region. 

1056 elements = set(graph.elements.names) 

1057 elements -= self.graph.elements.names 

1058 elements -= other.graph.elements.names 

1059 if not elements: 

1060 records = NamedKeyDict[DimensionElement, DimensionRecord | None](self.records) 

1061 records.update(other.records) 

1062 return basic.expanded(records.freeze()) 

1063 return basic 

1064 

1065 def hasFull(self) -> bool: 

1066 # Docstring inherited from DataCoordinate. 

1067 return True 

1068 

1069 def hasRecords(self) -> bool: 

1070 # Docstring inherited from DataCoordinate. 

1071 return True 

1072 

1073 def _record(self, name: str) -> DimensionRecord | None: 

1074 # Docstring inherited from DataCoordinate. 

1075 return self._records[name] 

1076 

1077 def __reduce__(self) -> tuple[Any, ...]: 

1078 return (_ExpandedTupleDataCoordinate, (self._graph, self._values, self._records)) 

1079 

1080 def __getattr__(self, name: str) -> Any: 

1081 try: 

1082 return self._record(name) 

1083 except KeyError: 

1084 raise AttributeError(name) from None 

1085 

1086 def __dir__(self) -> list[str]: 

1087 result = list(super().__dir__()) 

1088 result.extend(self.graph.elements.names) 

1089 return result