Coverage for python/lsst/daf/butler/core/dimensions/_coordinate.py: 33%

355 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-05 01:26 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22# 

23# Design notes for this module are in 

24# doc/lsst.daf.butler/dev/dataCoordinate.py. 

25# 

26 

27from __future__ import annotations 

28 

29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue", "SerializedDataCoordinate") 

30 

31import numbers 

32from abc import abstractmethod 

33from collections.abc import Iterator, Mapping, Set 

34from typing import TYPE_CHECKING, Any, ClassVar, Literal, overload 

35 

36from deprecated.sphinx import deprecated 

37from lsst.daf.butler._compat import _BaseModelCompat 

38from lsst.sphgeom import IntersectionRegion, Region 

39 

40from ..json import from_json_pydantic, to_json_pydantic 

41from ..named import NamedKeyDict, NamedKeyMapping, NamedValueAbstractSet, NameLookupMapping 

42from ..persistenceContext import PersistenceContextVars 

43from ..timespan import Timespan 

44from ._elements import Dimension, DimensionElement 

45from ._graph import DimensionGraph 

46from ._records import DimensionRecord, SerializedDimensionRecord 

47 

48if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

49 from ...registry import Registry 

50 from ._universe import DimensionUniverse 

51 

52DataIdKey = str | Dimension 

53"""Type annotation alias for the keys that can be used to index a 

54DataCoordinate. 

55""" 

56 

57# Pydantic will cast int to str if str is first in the Union. 

58DataIdValue = int | str | None 

59"""Type annotation alias for the values that can be present in a 

60DataCoordinate or other data ID. 

61""" 

62 

63 

64class SerializedDataCoordinate(_BaseModelCompat): 

65 """Simplified model for serializing a `DataCoordinate`.""" 

66 

67 dataId: dict[str, DataIdValue] 

68 records: dict[str, SerializedDimensionRecord] | None = None 

69 

70 @classmethod 

71 def direct( 

72 cls, *, dataId: dict[str, DataIdValue], records: dict[str, dict] | None 

73 ) -> SerializedDataCoordinate: 

74 """Construct a `SerializedDataCoordinate` directly without validators. 

75 

76 This differs from the pydantic "construct" method in that the arguments 

77 are explicitly what the model requires, and it will recurse through 

78 members, constructing them from their corresponding `direct` methods. 

79 

80 This method should only be called when the inputs are trusted. 

81 """ 

82 key = (frozenset(dataId.items()), records is not None) 

83 cache = PersistenceContextVars.serializedDataCoordinateMapping.get() 

84 if cache is not None and (result := cache.get(key)) is not None: 

85 return result 

86 

87 if records is None: 

88 serialized_records = None 

89 else: 

90 serialized_records = {k: SerializedDimensionRecord.direct(**v) for k, v in records.items()} 

91 

92 node = cls.model_construct(dataId=dataId, records=serialized_records) 

93 

94 if cache is not None: 

95 cache[key] = node 

96 return node 

97 

98 

99def _intersectRegions(*args: Region) -> Region | None: 

100 """Return the intersection of several regions. 

101 

102 For internal use by `ExpandedDataCoordinate` only. 

103 

104 If no regions are provided, returns `None`. 

105 """ 

106 if len(args) == 0: 

107 return None 

108 else: 

109 result = args[0] 

110 for n in range(1, len(args)): 

111 result = IntersectionRegion(result, args[n]) 

112 return result 

113 

114 

115class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]): 

116 """Data ID dictionary. 

117 

118 An immutable data ID dictionary that guarantees that its key-value pairs 

119 identify at least all required dimensions in a `DimensionGraph`. 

120 

121 `DataCoordinate` itself is an ABC, but provides `staticmethod` factory 

122 functions for private concrete implementations that should be sufficient 

123 for most purposes. `standardize` is the most flexible and safe of these; 

124 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are 

125 more specialized and perform little or no checking of inputs. 

126 

127 Notes 

128 ----- 

129 Like any data ID class, `DataCoordinate` behaves like a dictionary, but 

130 with some subtleties: 

131 

132 - Both `Dimension` instances and `str` names thereof may be used as keys 

133 in lookup operations, but iteration (and `keys`) will yield `Dimension` 

134 instances. The `names` property can be used to obtain the corresponding 

135 `str` names. 

136 

137 - Lookups for implied dimensions (those in ``self.graph.implied``) are 

138 supported if and only if `hasFull` returns `True`, and are never 

139 included in iteration or `keys`. The `full` property may be used to 

140 obtain a mapping whose keys do include implied dimensions. 

141 

142 - Equality comparison with other mappings is supported, but it always 

143 considers only required dimensions (as well as requiring both operands 

144 to identify the same dimensions). This is not quite consistent with the 

145 way mappings usually work - normally differing keys imply unequal 

146 mappings - but it makes sense in this context because data IDs with the 

147 same values for required dimensions but different values for implied 

148 dimensions represent a serious problem with the data that 

149 `DataCoordinate` cannot generally recognize on its own, and a data ID 

150 that knows implied dimension values should still be able to compare as 

151 equal to one that does not. This is of course not the way comparisons 

152 between simple `dict` data IDs work, and hence using a `DataCoordinate` 

153 instance for at least one operand in any data ID comparison is strongly 

154 recommended. 

155 

156 See Also 

157 -------- 

158 :ref:`lsst.daf.butler-dimensions_data_ids` 

159 """ 

160 

161 __slots__ = () 

162 

163 _serializedType = SerializedDataCoordinate 

164 

165 @staticmethod 

166 def standardize( 

167 mapping: NameLookupMapping[Dimension, DataIdValue] | None = None, 

168 *, 

169 graph: DimensionGraph | None = None, 

170 universe: DimensionUniverse | None = None, 

171 defaults: DataCoordinate | None = None, 

172 **kwargs: Any, 

173 ) -> DataCoordinate: 

174 """Standardize the supplied dataId. 

175 

176 Adapts an arbitrary mapping and/or additional arguments into a true 

177 `DataCoordinate`, or augment an existing one. 

178 

179 Parameters 

180 ---------- 

181 mapping : `~collections.abc.Mapping`, optional 

182 An informal data ID that maps dimensions or dimension names to 

183 their primary key values (may also be a true `DataCoordinate`). 

184 graph : `DimensionGraph` 

185 The dimensions to be identified by the new `DataCoordinate`. 

186 If not provided, will be inferred from the keys of ``mapping`` and 

187 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

188 is already a `DataCoordinate`. 

189 universe : `DimensionUniverse` 

190 All known dimensions and their relationships; used to expand 

191 and validate dependencies when ``graph`` is not provided. 

192 defaults : `DataCoordinate`, optional 

193 Default dimension key-value pairs to use when needed. These are 

194 never used to infer ``graph``, and are ignored if a different value 

195 is provided for the same key in ``mapping`` or `**kwargs``. 

196 **kwargs 

197 Additional keyword arguments are treated like additional key-value 

198 pairs in ``mapping``. 

199 

200 Returns 

201 ------- 

202 coordinate : `DataCoordinate` 

203 A validated `DataCoordinate` instance. 

204 

205 Raises 

206 ------ 

207 TypeError 

208 Raised if the set of optional arguments provided is not supported. 

209 KeyError 

210 Raised if a key-value pair for a required dimension is missing. 

211 """ 

212 d: dict[str, DataIdValue] = {} 

213 if isinstance(mapping, DataCoordinate): 

214 if graph is None: 

215 if not kwargs: 

216 # Already standardized to exactly what we want. 

217 return mapping 

218 elif kwargs.keys().isdisjoint(graph.dimensions.names): 

219 # User provided kwargs, but told us not to use them by 

220 # passing in dimensions that are disjoint from those kwargs. 

221 # This is not necessarily user error - it's a useful pattern 

222 # to pass in all of the key-value pairs you have and let the 

223 # code here pull out only what it needs. 

224 return mapping.subset(graph) 

225 assert universe is None or universe == mapping.universe 

226 universe = mapping.universe 

227 d.update((name, mapping[name]) for name in mapping.graph.required.names) 

228 if mapping.hasFull(): 

229 d.update((name, mapping[name]) for name in mapping.graph.implied.names) 

230 elif isinstance(mapping, NamedKeyMapping): 

231 d.update(mapping.byName()) 

232 elif mapping is not None: 

233 d.update(mapping) 

234 d.update(kwargs) 

235 if graph is None: 

236 if defaults is not None: 

237 universe = defaults.universe 

238 elif universe is None: 

239 raise TypeError("universe must be provided if graph is not.") 

240 graph = DimensionGraph(universe, names=d.keys()) 

241 if not graph.dimensions: 

242 return DataCoordinate.makeEmpty(graph.universe) 

243 if defaults is not None: 

244 if defaults.hasFull(): 

245 for k, v in defaults.full.items(): 

246 d.setdefault(k.name, v) 

247 else: 

248 for k, v in defaults.items(): 

249 d.setdefault(k.name, v) 

250 if d.keys() >= graph.dimensions.names: 

251 values = tuple(d[name] for name in graph._dataCoordinateIndices) 

252 else: 

253 try: 

254 values = tuple(d[name] for name in graph.required.names) 

255 except KeyError as err: 

256 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err 

257 # Some backends cannot handle numpy.int64 type which is a subclass of 

258 # numbers.Integral; convert that to int. 

259 values = tuple( 

260 int(val) if isinstance(val, numbers.Integral) else val for val in values # type: ignore 

261 ) 

262 return _BasicTupleDataCoordinate(graph, values) 

263 

264 @staticmethod 

265 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate: 

266 """Return an empty `DataCoordinate`. 

267 

268 It identifies the null set of dimensions. 

269 

270 Parameters 

271 ---------- 

272 universe : `DimensionUniverse` 

273 Universe to which this null dimension set belongs. 

274 

275 Returns 

276 ------- 

277 dataId : `DataCoordinate` 

278 A data ID object that identifies no dimensions. `hasFull` and 

279 `hasRecords` are guaranteed to return `True`, because both `full` 

280 and `records` are just empty mappings. 

281 """ 

282 return _ExpandedTupleDataCoordinate(universe.empty, (), {}) 

283 

284 @staticmethod 

285 def fromRequiredValues(graph: DimensionGraph, values: tuple[DataIdValue, ...]) -> DataCoordinate: 

286 """Construct a `DataCoordinate` from required dimension values. 

287 

288 This is a low-level interface with at most assertion-level checking of 

289 inputs. Most callers should use `standardize` instead. 

290 

291 Parameters 

292 ---------- 

293 graph : `DimensionGraph` 

294 Dimensions this data ID will identify. 

295 values : `tuple` [ `int` or `str` ] 

296 Tuple of primary key values corresponding to ``graph.required``, 

297 in that order. 

298 

299 Returns 

300 ------- 

301 dataId : `DataCoordinate` 

302 A data ID object that identifies the given dimensions. 

303 ``dataId.hasFull()`` will return `True` if and only if 

304 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

305 return `True`. 

306 """ 

307 assert len(graph.required) == len( 

308 values 

309 ), f"Inconsistency between dimensions {graph.required} and required values {values}." 

310 return _BasicTupleDataCoordinate(graph, values) 

311 

312 @staticmethod 

313 def fromFullValues(graph: DimensionGraph, values: tuple[DataIdValue, ...]) -> DataCoordinate: 

314 """Construct a `DataCoordinate` from all dimension values. 

315 

316 This is a low-level interface with at most assertion-level checking of 

317 inputs. Most callers should use `standardize` instead. 

318 

319 Parameters 

320 ---------- 

321 graph : `DimensionGraph` 

322 Dimensions this data ID will identify. 

323 values : `tuple` [ `int` or `str` ] 

324 Tuple of primary key values corresponding to 

325 ``itertools.chain(graph.required, graph.implied)``, in that order. 

326 Note that this is _not_ the same order as ``graph.dimensions``, 

327 though these contain the same elements. 

328 

329 Returns 

330 ------- 

331 dataId : `DataCoordinate` 

332 A data ID object that identifies the given dimensions. 

333 ``dataId.hasFull()`` will return `True` if and only if 

334 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

335 return `True`. 

336 """ 

337 assert len(graph.dimensions) == len( 

338 values 

339 ), f"Inconsistency between dimensions {graph.dimensions} and full values {values}." 

340 return _BasicTupleDataCoordinate(graph, values) 

341 

342 def __hash__(self) -> int: 

343 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required)) 

344 

345 def __eq__(self, other: Any) -> bool: 

346 if not isinstance(other, DataCoordinate): 

347 other = DataCoordinate.standardize(other, universe=self.universe) 

348 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required) 

349 

350 def __repr__(self) -> str: 

351 # We can't make repr yield something that could be exec'd here without 

352 # printing out the whole DimensionUniverse the graph is derived from. 

353 # So we print something that mostly looks like a dict, but doesn't 

354 # quote its keys: that's both more compact and something that can't 

355 # be mistaken for an actual dict or something that could be exec'd. 

356 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names] 

357 if self.hasFull() and self.graph.required != self.graph.dimensions: 

358 terms.append("...") 

359 return "{{{}}}".format(", ".join(terms)) 

360 

361 def __lt__(self, other: Any) -> bool: 

362 # Allow DataCoordinate to be sorted 

363 if not isinstance(other, type(self)): 

364 return NotImplemented 

365 # Form tuple of tuples for each DataCoordinate: 

366 # Unlike repr() we only use required keys here to ensure that 

367 # __eq__ can not be true simultaneously with __lt__ being true. 

368 self_kv = tuple(self.items()) 

369 other_kv = tuple(other.items()) 

370 

371 return self_kv < other_kv 

372 

373 def __iter__(self) -> Iterator[Dimension]: 

374 return iter(self.keys()) 

375 

376 def __len__(self) -> int: 

377 return len(self.keys()) 

378 

379 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

380 return self.graph.required 

381 

382 @property 

383 def names(self) -> Set[str]: 

384 """Names of the required dimensions identified by this data ID. 

385 

386 They are returned in the same order as `keys` 

387 (`collections.abc.Set` [ `str` ]). 

388 """ 

389 return self.keys().names 

390 

391 @abstractmethod 

392 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

393 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``. 

394 

395 Parameters 

396 ---------- 

397 graph : `DimensionGraph` 

398 The dimensions identified by the returned `DataCoordinate`. 

399 

400 Returns 

401 ------- 

402 coordinate : `DataCoordinate` 

403 A `DataCoordinate` instance that identifies only the given 

404 dimensions. May be ``self`` if ``graph == self.graph``. 

405 

406 Raises 

407 ------ 

408 KeyError 

409 Raised if the primary key value for one or more required dimensions 

410 is unknown. This may happen if ``graph.issubset(self.graph)`` is 

411 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if 

412 ``self.hasFull()`` is `False` and 

413 ``graph.required.issubset(self.graph.required)`` is `False`. As 

414 an example of the latter case, consider trying to go from a data ID 

415 with dimensions {instrument, physical_filter, band} to 

416 just {instrument, band}; band is implied by 

417 physical_filter and hence would have no value in the original data 

418 ID if ``self.hasFull()`` is `False`. 

419 

420 Notes 

421 ----- 

422 If `hasFull` and `hasRecords` return `True` on ``self``, they will 

423 return `True` (respectively) on the returned `DataCoordinate` as well. 

424 The converse does not hold. 

425 """ 

426 raise NotImplementedError() 

427 

428 @abstractmethod 

429 def union(self, other: DataCoordinate) -> DataCoordinate: 

430 """Combine two data IDs. 

431 

432 Yields a new one that identifies all dimensions that either of them 

433 identify. 

434 

435 Parameters 

436 ---------- 

437 other : `DataCoordinate` 

438 Data ID to combine with ``self``. 

439 

440 Returns 

441 ------- 

442 unioned : `DataCoordinate` 

443 A `DataCoordinate` instance that satisfies 

444 ``unioned.graph == self.graph.union(other.graph)``. Will preserve 

445 ``hasFull`` and ``hasRecords`` whenever possible. 

446 

447 Notes 

448 ----- 

449 No checking for consistency is performed on values for keys that 

450 ``self`` and ``other`` have in common, and which value is included in 

451 the returned data ID is not specified. 

452 """ 

453 raise NotImplementedError() 

454 

455 @abstractmethod 

456 def expanded( 

457 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None] 

458 ) -> DataCoordinate: 

459 """Return a `DataCoordinate` that holds the given records. 

460 

461 Guarantees that `hasRecords` returns `True`. 

462 

463 This is a low-level interface with at most assertion-level checking of 

464 inputs. Most callers should use `Registry.expandDataId` instead. 

465 

466 Parameters 

467 ---------- 

468 records : `~collections.abc.Mapping` [ `str`, `DimensionRecord` or \ 

469 `None` ] 

470 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

471 `~collections.abc.Mapping` with `str` (`DimensionElement` name) 

472 keys and `DimensionRecord` values. Keys must cover all elements in 

473 ``self.graph.elements``. Values may be `None`, but only to reflect 

474 actual NULL values in the database, not just records that have not 

475 been fetched. 

476 """ 

477 raise NotImplementedError() 

478 

479 @property 

480 def universe(self) -> DimensionUniverse: 

481 """Universe that defines all known compatible dimensions. 

482 

483 The univers will be compatible with this coordinate 

484 (`DimensionUniverse`). 

485 """ 

486 return self.graph.universe 

487 

488 @property 

489 @abstractmethod 

490 def graph(self) -> DimensionGraph: 

491 """Dimensions identified by this data ID (`DimensionGraph`). 

492 

493 Note that values are only required to be present for dimensions in 

494 ``self.graph.required``; all others may be retrieved (from a 

495 `Registry`) given these. 

496 """ 

497 raise NotImplementedError() 

498 

499 @abstractmethod 

500 def hasFull(self) -> bool: 

501 """Whether this data ID contains implied and required values. 

502 

503 Returns 

504 ------- 

505 state : `bool` 

506 If `True`, `__getitem__`, `get`, and `__contains__` (but not 

507 `keys`!) will act as though the mapping includes key-value pairs 

508 for implied dimensions, and the `full` property may be used. If 

509 `False`, these operations only include key-value pairs for required 

510 dimensions, and accessing `full` is an error. Always `True` if 

511 there are no implied dimensions. 

512 """ 

513 raise NotImplementedError() 

514 

515 @property 

516 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]: 

517 """Return mapping for all dimensions in ``self.graph``. 

518 

519 The mapping includes key-value pairs for all dimensions in 

520 ``self.graph``, including implied (`NamedKeyMapping`). 

521 

522 Accessing this attribute if `hasFull` returns `False` is a logic error 

523 that may raise an exception of unspecified type either immediately or 

524 when implied keys are accessed via the returned mapping, depending on 

525 the implementation and whether assertions are enabled. 

526 """ 

527 assert self.hasFull(), "full may only be accessed if hasFull() returns True." 

528 return _DataCoordinateFullView(self) 

529 

530 @abstractmethod 

531 def hasRecords(self) -> bool: 

532 """Whether this data ID contains records. 

533 

534 These are the records for all of the dimension elements it identifies. 

535 

536 Returns 

537 ------- 

538 state : `bool` 

539 If `True`, the following attributes may be accessed: 

540 

541 - `records` 

542 - `region` 

543 - `timespan` 

544 - `pack` 

545 

546 If `False`, accessing any of these is considered a logic error. 

547 """ 

548 raise NotImplementedError() 

549 

550 @property 

551 def records(self) -> NamedKeyMapping[DimensionElement, DimensionRecord | None]: 

552 """Return the records. 

553 

554 Returns a mapping that contains `DimensionRecord` objects for all 

555 elements identified by this data ID (`NamedKeyMapping`). 

556 

557 The values of this mapping may be `None` if and only if there is no 

558 record for that element with these dimensions in the database (which 

559 means some foreign key field must have a NULL value). 

560 

561 Accessing this attribute if `hasRecords` returns `False` is a logic 

562 error that may raise an exception of unspecified type either 

563 immediately or when the returned mapping is used, depending on the 

564 implementation and whether assertions are enabled. 

565 """ 

566 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True." 

567 return _DataCoordinateRecordsView(self) 

568 

569 @abstractmethod 

570 def _record(self, name: str) -> DimensionRecord | None: 

571 """Protected implementation hook that backs the ``records`` attribute. 

572 

573 Parameters 

574 ---------- 

575 name : `str` 

576 The name of a `DimensionElement`, guaranteed to be in 

577 ``self.graph.elements.names``. 

578 

579 Returns 

580 ------- 

581 record : `DimensionRecord` or `None` 

582 The dimension record for the given element identified by this 

583 data ID, or `None` if there is no such record. 

584 """ 

585 raise NotImplementedError() 

586 

587 @property 

588 def region(self) -> Region | None: 

589 """Spatial region associated with this data ID. 

590 

591 (`lsst.sphgeom.Region` or `None`). 

592 

593 This is `None` if and only if ``self.graph.spatial`` is empty. 

594 

595 Accessing this attribute if `hasRecords` returns `False` is a logic 

596 error that may or may not raise an exception, depending on the 

597 implementation and whether assertions are enabled. 

598 """ 

599 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True." 

600 regions = [] 

601 for family in self.graph.spatial: 

602 element = family.choose(self.graph.elements) 

603 record = self._record(element.name) 

604 if record is None or record.region is None: 

605 return None 

606 else: 

607 regions.append(record.region) 

608 return _intersectRegions(*regions) 

609 

610 @property 

611 def timespan(self) -> Timespan | None: 

612 """Temporal interval associated with this data ID. 

613 

614 (`Timespan` or `None`). 

615 

616 This is `None` if and only if ``self.graph.timespan`` is empty. 

617 

618 Accessing this attribute if `hasRecords` returns `False` is a logic 

619 error that may or may not raise an exception, depending on the 

620 implementation and whether assertions are enabled. 

621 """ 

622 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True." 

623 timespans = [] 

624 for family in self.graph.temporal: 

625 element = family.choose(self.graph.elements) 

626 record = self._record(element.name) 

627 # DimensionRecord subclasses for temporal elements always have 

628 # .timespan, but they're dynamic so this can't be type-checked. 

629 if record is None or record.timespan is None: 

630 return None 

631 else: 

632 timespans.append(record.timespan) 

633 if not timespans: 

634 return None 

635 elif len(timespans) == 1: 

636 return timespans[0] 

637 else: 

638 return Timespan.intersection(*timespans) 

639 

640 @overload 

641 def pack(self, name: str, *, returnMaxBits: Literal[True]) -> tuple[int, int]: 

642 ... 

643 

644 @overload 

645 def pack(self, name: str, *, returnMaxBits: Literal[False]) -> int: 

646 ... 

647 

648 # TODO: Remove this method and its overloads above on DM-38687. 

649 @deprecated( 

650 "Deprecated in favor of configurable dimension packers. Will be removed after v26.", 

651 version="v26", 

652 category=FutureWarning, 

653 ) 

654 def pack(self, name: str, *, returnMaxBits: bool = False) -> tuple[int, int] | int: 

655 """Pack this data ID into an integer. 

656 

657 Parameters 

658 ---------- 

659 name : `str` 

660 Name of the `DimensionPacker` algorithm (as defined in the 

661 dimension configuration). 

662 returnMaxBits : `bool`, optional 

663 If `True` (`False` is default), return the maximum number of 

664 nonzero bits in the returned integer across all data IDs. 

665 

666 Returns 

667 ------- 

668 packed : `int` 

669 Integer ID. This ID is unique only across data IDs that have 

670 the same values for the packer's "fixed" dimensions. 

671 maxBits : `int`, optional 

672 Maximum number of nonzero bits in ``packed``. Not returned unless 

673 ``returnMaxBits`` is `True`. 

674 

675 Notes 

676 ----- 

677 Accessing this attribute if `hasRecords` returns `False` is a logic 

678 error that may or may not raise an exception, depending on the 

679 implementation and whether assertions are enabled. 

680 """ 

681 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." 

682 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) 

683 

684 def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate: 

685 """Convert this class to a simple python type. 

686 

687 This is suitable for serialization. 

688 

689 Parameters 

690 ---------- 

691 minimal : `bool`, optional 

692 Use minimal serialization. If set the records will not be attached. 

693 

694 Returns 

695 ------- 

696 simple : `SerializedDataCoordinate` 

697 The object converted to simple form. 

698 """ 

699 # Convert to a dict form 

700 if self.hasFull(): 

701 dataId = self.full.byName() 

702 else: 

703 dataId = self.byName() 

704 records: dict[str, SerializedDimensionRecord] | None 

705 if not minimal and self.hasRecords(): 

706 records = {k: v.to_simple() for k, v in self.records.byName().items() if v is not None} 

707 else: 

708 records = None 

709 

710 return SerializedDataCoordinate(dataId=dataId, records=records) 

711 

712 @classmethod 

713 def from_simple( 

714 cls, 

715 simple: SerializedDataCoordinate, 

716 universe: DimensionUniverse | None = None, 

717 registry: Registry | None = None, 

718 ) -> DataCoordinate: 

719 """Construct a new object from the simplified form. 

720 

721 The data is assumed to be of the form returned from the `to_simple` 

722 method. 

723 

724 Parameters 

725 ---------- 

726 simple : `dict` of [`str`, `Any`] 

727 The `dict` returned by `to_simple()`. 

728 universe : `DimensionUniverse` 

729 The special graph of all known dimensions. 

730 registry : `lsst.daf.butler.Registry`, optional 

731 Registry from which a universe can be extracted. Can be `None` 

732 if universe is provided explicitly. 

733 

734 Returns 

735 ------- 

736 dataId : `DataCoordinate` 

737 Newly-constructed object. 

738 """ 

739 key = (frozenset(simple.dataId.items()), simple.records is not None) 

740 cache = PersistenceContextVars.dataCoordinates.get() 

741 if cache is not None and (result := cache.get(key)) is not None: 

742 return result 

743 if universe is None and registry is None: 

744 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

745 if universe is None and registry is not None: 

746 universe = registry.dimensions 

747 if universe is None: 

748 # this is for mypy 

749 raise ValueError("Unable to determine a usable universe") 

750 

751 dataId = cls.standardize(simple.dataId, universe=universe) 

752 if simple.records: 

753 dataId = dataId.expanded( 

754 {k: DimensionRecord.from_simple(v, universe=universe) for k, v in simple.records.items()} 

755 ) 

756 if cache is not None: 

757 cache[key] = dataId 

758 return dataId 

759 

760 to_json = to_json_pydantic 

761 from_json: ClassVar = classmethod(from_json_pydantic) 

762 

763 

764DataId = DataCoordinate | Mapping[str, Any] 

765"""A type-annotation alias for signatures that accept both informal data ID 

766dictionaries and validated `DataCoordinate` instances. 

767""" 

768 

769 

770class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]): 

771 """View class for `DataCoordinate.full`. 

772 

773 Provides the default implementation for 

774 `DataCoordinate.full`. 

775 

776 Parameters 

777 ---------- 

778 target : `DataCoordinate` 

779 The `DataCoordinate` instance this object provides a view of. 

780 """ 

781 

782 def __init__(self, target: DataCoordinate): 

783 self._target = target 

784 

785 __slots__ = ("_target",) 

786 

787 def __repr__(self) -> str: 

788 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names] 

789 return "{{{}}}".format(", ".join(terms)) 

790 

791 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

792 return self._target[key] 

793 

794 def __iter__(self) -> Iterator[Dimension]: 

795 return iter(self.keys()) 

796 

797 def __len__(self) -> int: 

798 return len(self.keys()) 

799 

800 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

801 return self._target.graph.dimensions 

802 

803 @property 

804 def names(self) -> Set[str]: 

805 # Docstring inherited from `NamedKeyMapping`. 

806 return self.keys().names 

807 

808 

809class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, DimensionRecord | None]): 

810 """View class for `DataCoordinate.records`. 

811 

812 Provides the default implementation for 

813 `DataCoordinate.records`. 

814 

815 Parameters 

816 ---------- 

817 target : `DataCoordinate` 

818 The `DataCoordinate` instance this object provides a view of. 

819 """ 

820 

821 def __init__(self, target: DataCoordinate): 

822 self._target = target 

823 

824 __slots__ = ("_target",) 

825 

826 def __repr__(self) -> str: 

827 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names] 

828 return "{{{}}}".format(", ".join(terms)) 

829 

830 def __str__(self) -> str: 

831 return "\n".join(str(v) for v in self.values()) 

832 

833 def __getitem__(self, key: DimensionElement | str) -> DimensionRecord | None: 

834 if isinstance(key, DimensionElement): 

835 key = key.name 

836 return self._target._record(key) 

837 

838 def __iter__(self) -> Iterator[DimensionElement]: 

839 return iter(self.keys()) 

840 

841 def __len__(self) -> int: 

842 return len(self.keys()) 

843 

844 def keys(self) -> NamedValueAbstractSet[DimensionElement]: # type: ignore 

845 return self._target.graph.elements 

846 

847 @property 

848 def names(self) -> Set[str]: 

849 # Docstring inherited from `NamedKeyMapping`. 

850 return self.keys().names 

851 

852 

853class _BasicTupleDataCoordinate(DataCoordinate): 

854 """Standard implementation of `DataCoordinate`. 

855 

856 Backed by a tuple of values. 

857 

858 This class should only be accessed outside this module via the 

859 `DataCoordinate` interface, and should only be constructed via the static 

860 methods there. 

861 

862 Parameters 

863 ---------- 

864 graph : `DimensionGraph` 

865 The dimensions to be identified. 

866 values : `tuple` [ `int` or `str` ] 

867 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May 

868 include values for just required dimensions (which always come first) 

869 or all dimensions. 

870 """ 

871 

872 def __init__(self, graph: DimensionGraph, values: tuple[DataIdValue, ...]): 

873 self._graph = graph 

874 self._values = values 

875 

876 __slots__ = ("_graph", "_values") 

877 

878 @property 

879 def graph(self) -> DimensionGraph: 

880 # Docstring inherited from DataCoordinate. 

881 return self._graph 

882 

883 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

884 # Docstring inherited from DataCoordinate. 

885 if isinstance(key, Dimension): 

886 key = key.name 

887 index = self._graph._dataCoordinateIndices[key] 

888 try: 

889 return self._values[index] 

890 except IndexError: 

891 # Caller asked for an implied dimension, but this object only has 

892 # values for the required ones. 

893 raise KeyError(key) from None 

894 

895 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

896 # Docstring inherited from DataCoordinate. 

897 if self._graph == graph: 

898 return self 

899 elif self.hasFull() or self._graph.required >= graph.dimensions: 

900 return _BasicTupleDataCoordinate( 

901 graph, 

902 tuple(self[k] for k in graph._dataCoordinateIndices), 

903 ) 

904 else: 

905 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names)) 

906 

907 def union(self, other: DataCoordinate) -> DataCoordinate: 

908 # Docstring inherited from DataCoordinate. 

909 graph = self.graph.union(other.graph) 

910 # See if one or both input data IDs is already what we want to return; 

911 # if so, return the most complete one we have. 

912 if other.graph == graph: 

913 if self.graph == graph: 

914 # Input data IDs have the same graph (which is also the result 

915 # graph), but may not have the same content. 

916 # other might have records; self does not, so try other first. 

917 # If it at least has full values, it's no worse than self. 

918 if other.hasFull(): 

919 return other 

920 else: 

921 return self 

922 elif other.hasFull(): 

923 return other 

924 # There's some chance that neither self nor other has full values, 

925 # but together provide enough to the union to. Let the general 

926 # case below handle that. 

927 elif self.graph == graph and self.hasFull(): 

928 # No chance at returning records. If self has full values, it's 

929 # the best we can do. 

930 return self 

931 # General case with actual merging of dictionaries. 

932 values = self.full.byName() if self.hasFull() else self.byName() 

933 values.update(other.full.byName() if other.hasFull() else other.byName()) 

934 return DataCoordinate.standardize(values, graph=graph) 

935 

936 def expanded( 

937 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None] 

938 ) -> DataCoordinate: 

939 # Docstring inherited from DataCoordinate 

940 values = self._values 

941 if not self.hasFull(): 

942 # Extract a complete values tuple from the attributes of the given 

943 # records. It's possible for these to be inconsistent with 

944 # self._values (which is a serious problem, of course), but we've 

945 # documented this as a no-checking API. 

946 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied) 

947 return _ExpandedTupleDataCoordinate(self._graph, values, records) 

948 

949 def hasFull(self) -> bool: 

950 # Docstring inherited from DataCoordinate. 

951 return len(self._values) == len(self._graph._dataCoordinateIndices) 

952 

953 def hasRecords(self) -> bool: 

954 # Docstring inherited from DataCoordinate. 

955 return False 

956 

957 def _record(self, name: str) -> DimensionRecord | None: 

958 # Docstring inherited from DataCoordinate. 

959 raise AssertionError() 

960 

961 def __reduce__(self) -> tuple[Any, ...]: 

962 return (_BasicTupleDataCoordinate, (self._graph, self._values)) 

963 

964 def __getattr__(self, name: str) -> Any: 

965 if name in self.graph.elements.names: 

966 raise AttributeError( 

967 f"Dimension record attribute {name!r} is only available on expanded DataCoordinates." 

968 ) 

969 raise AttributeError(name) 

970 

971 

972class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate): 

973 """A `DataCoordinate` implementation that can hold `DimensionRecord`. 

974 

975 This class should only be accessed outside this module via the 

976 `DataCoordinate` interface, and should only be constructed via calls to 

977 `DataCoordinate.expanded`. 

978 

979 Parameters 

980 ---------- 

981 graph : `DimensionGraph` 

982 The dimensions to be identified. 

983 values : `tuple` [ `int` or `str` ] 

984 Data ID values, ordered to match ``graph._dataCoordinateIndices``. 

985 May include values for just required dimensions (which always come 

986 first) or all dimensions. 

987 records : `~collections.abc.Mapping` [ `str`, `DimensionRecord` or `None` ] 

988 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

989 `~collections.abc.Mapping` with `str` (`DimensionElement` name) keys 

990 and `DimensionRecord` values. Keys must cover all elements in 

991 ``self.graph.elements``. Values may be `None`, but only to reflect 

992 actual NULL values in the database, not just records that have not 

993 been fetched. 

994 """ 

995 

996 def __init__( 

997 self, 

998 graph: DimensionGraph, 

999 values: tuple[DataIdValue, ...], 

1000 records: NameLookupMapping[DimensionElement, DimensionRecord | None], 

1001 ): 

1002 super().__init__(graph, values) 

1003 assert super().hasFull(), "This implementation requires full dimension records." 

1004 self._records = records 

1005 

1006 __slots__ = ("_records",) 

1007 

1008 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

1009 # Docstring inherited from DataCoordinate. 

1010 if self._graph == graph: 

1011 return self 

1012 return _ExpandedTupleDataCoordinate( 

1013 graph, tuple(self[k] for k in graph._dataCoordinateIndices), records=self._records 

1014 ) 

1015 

1016 def expanded( 

1017 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None] 

1018 ) -> DataCoordinate: 

1019 # Docstring inherited from DataCoordinate. 

1020 return self 

1021 

1022 def union(self, other: DataCoordinate) -> DataCoordinate: 

1023 # Docstring inherited from DataCoordinate. 

1024 graph = self.graph.union(other.graph) 

1025 # See if one or both input data IDs is already what we want to return; 

1026 # if so, return the most complete one we have. 

1027 if self.graph == graph: 

1028 # self has records, so even if other is also a valid result, it's 

1029 # no better. 

1030 return self 

1031 if other.graph == graph and other.hasFull(): 

1032 # If other has full values, and self does not identify some of 

1033 # those, it's the base we can do. It may have records, too. 

1034 return other 

1035 # If other does not have full values, there's a chance self may 

1036 # provide the values needed to complete it. For example, self 

1037 # could be {band} while other could be 

1038 # {instrument, physical_filter, band}, with band unknown. 

1039 # General case with actual merging of dictionaries. 

1040 values = self.full.byName() 

1041 values.update(other.full.byName() if other.hasFull() else other.byName()) 

1042 basic = DataCoordinate.standardize(values, graph=graph) 

1043 # See if we can add records. 

1044 if self.hasRecords() and other.hasRecords(): 

1045 # Sometimes the elements of a union of graphs can contain elements 

1046 # that weren't in either input graph (because graph unions are only 

1047 # on dimensions). e.g. {visit} | {detector} brings along 

1048 # visit_detector_region. 

1049 elements = set(graph.elements.names) 

1050 elements -= self.graph.elements.names 

1051 elements -= other.graph.elements.names 

1052 if not elements: 

1053 records = NamedKeyDict[DimensionElement, DimensionRecord | None](self.records) 

1054 records.update(other.records) 

1055 return basic.expanded(records.freeze()) 

1056 return basic 

1057 

1058 def hasFull(self) -> bool: 

1059 # Docstring inherited from DataCoordinate. 

1060 return True 

1061 

1062 def hasRecords(self) -> bool: 

1063 # Docstring inherited from DataCoordinate. 

1064 return True 

1065 

1066 def _record(self, name: str) -> DimensionRecord | None: 

1067 # Docstring inherited from DataCoordinate. 

1068 return self._records[name] 

1069 

1070 def __reduce__(self) -> tuple[Any, ...]: 

1071 return (_ExpandedTupleDataCoordinate, (self._graph, self._values, self._records)) 

1072 

1073 def __getattr__(self, name: str) -> Any: 

1074 try: 

1075 return self._record(name) 

1076 except KeyError: 

1077 raise AttributeError(name) from None 

1078 

1079 def __dir__(self) -> list[str]: 

1080 result = list(super().__dir__()) 

1081 result.extend(self.graph.elements.names) 

1082 return result