Coverage for python/lsst/daf/butler/core/dimensions/_coordinate.py: 27%

345 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-07 02:10 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22# 

23# Design notes for this module are in 

24# doc/lsst.daf.butler/dev/dataCoordinate.py. 

25# 

26 

27from __future__ import annotations 

28 

29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue", "SerializedDataCoordinate") 

30 

31import numbers 

32from abc import abstractmethod 

33from typing import ( 

34 TYPE_CHECKING, 

35 AbstractSet, 

36 Any, 

37 ClassVar, 

38 Dict, 

39 Iterator, 

40 Literal, 

41 Mapping, 

42 Optional, 

43 Tuple, 

44 Union, 

45 overload, 

46) 

47 

48from deprecated.sphinx import deprecated 

49from lsst.sphgeom import IntersectionRegion, Region 

50from pydantic import BaseModel 

51 

52from ..json import from_json_pydantic, to_json_pydantic 

53from ..named import NamedKeyDict, NamedKeyMapping, NamedValueAbstractSet, NameLookupMapping 

54from ..timespan import Timespan 

55from ._elements import Dimension, DimensionElement 

56from ._graph import DimensionGraph 

57from ._records import DimensionRecord, SerializedDimensionRecord 

58 

59if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

60 from ...registry import Registry 

61 from ._universe import DimensionUniverse 

62 

63DataIdKey = Union[str, Dimension] 

64"""Type annotation alias for the keys that can be used to index a 

65DataCoordinate. 

66""" 

67 

68# Pydantic will cast int to str if str is first in the Union. 

69DataIdValue = Union[int, str, None] 

70"""Type annotation alias for the values that can be present in a 

71DataCoordinate or other data ID. 

72""" 

73 

74 

75class SerializedDataCoordinate(BaseModel): 

76 """Simplified model for serializing a `DataCoordinate`.""" 

77 

78 dataId: Dict[str, DataIdValue] 

79 records: Optional[Dict[str, SerializedDimensionRecord]] = None 

80 

81 @classmethod 

82 def direct(cls, *, dataId: Dict[str, DataIdValue], records: Dict[str, Dict]) -> SerializedDataCoordinate: 

83 """Construct a `SerializedDataCoordinate` directly without validators. 

84 

85 This differs from the pydantic "construct" method in that the arguments 

86 are explicitly what the model requires, and it will recurse through 

87 members, constructing them from their corresponding `direct` methods. 

88 

89 This method should only be called when the inputs are trusted. 

90 """ 

91 node = SerializedDataCoordinate.__new__(cls) 

92 setter = object.__setattr__ 

93 setter(node, "dataId", dataId) 

94 setter( 

95 node, 

96 "records", 

97 records 

98 if records is None 

99 else {k: SerializedDimensionRecord.direct(**v) for k, v in records.items()}, 

100 ) 

101 setter(node, "__fields_set__", {"dataId", "records"}) 

102 return node 

103 

104 

105def _intersectRegions(*args: Region) -> Optional[Region]: 

106 """Return the intersection of several regions. 

107 

108 For internal use by `ExpandedDataCoordinate` only. 

109 

110 If no regions are provided, returns `None`. 

111 """ 

112 if len(args) == 0: 

113 return None 

114 else: 

115 result = args[0] 

116 for n in range(1, len(args)): 

117 result = IntersectionRegion(result, args[n]) 

118 return result 

119 

120 

121class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]): 

122 """Data ID dictionary. 

123 

124 An immutable data ID dictionary that guarantees that its key-value pairs 

125 identify at least all required dimensions in a `DimensionGraph`. 

126 

127 `DataCoordinate` itself is an ABC, but provides `staticmethod` factory 

128 functions for private concrete implementations that should be sufficient 

129 for most purposes. `standardize` is the most flexible and safe of these; 

130 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are 

131 more specialized and perform little or no checking of inputs. 

132 

133 Notes 

134 ----- 

135 Like any data ID class, `DataCoordinate` behaves like a dictionary, but 

136 with some subtleties: 

137 

138 - Both `Dimension` instances and `str` names thereof may be used as keys 

139 in lookup operations, but iteration (and `keys`) will yield `Dimension` 

140 instances. The `names` property can be used to obtain the corresponding 

141 `str` names. 

142 

143 - Lookups for implied dimensions (those in ``self.graph.implied``) are 

144 supported if and only if `hasFull` returns `True`, and are never 

145 included in iteration or `keys`. The `full` property may be used to 

146 obtain a mapping whose keys do include implied dimensions. 

147 

148 - Equality comparison with other mappings is supported, but it always 

149 considers only required dimensions (as well as requiring both operands 

150 to identify the same dimensions). This is not quite consistent with the 

151 way mappings usually work - normally differing keys imply unequal 

152 mappings - but it makes sense in this context because data IDs with the 

153 same values for required dimensions but different values for implied 

154 dimensions represent a serious problem with the data that 

155 `DataCoordinate` cannot generally recognize on its own, and a data ID 

156 that knows implied dimension values should still be able to compare as 

157 equal to one that does not. This is of course not the way comparisons 

158 between simple `dict` data IDs work, and hence using a `DataCoordinate` 

159 instance for at least one operand in any data ID comparison is strongly 

160 recommended. 

161 

162 See also 

163 -------- 

164 :ref:`lsst.daf.butler-dimensions_data_ids` 

165 """ 

166 

167 __slots__ = () 

168 

169 _serializedType = SerializedDataCoordinate 

170 

171 @staticmethod 

172 def standardize( 

173 mapping: Optional[NameLookupMapping[Dimension, DataIdValue]] = None, 

174 *, 

175 graph: Optional[DimensionGraph] = None, 

176 universe: Optional[DimensionUniverse] = None, 

177 defaults: Optional[DataCoordinate] = None, 

178 **kwargs: Any, 

179 ) -> DataCoordinate: 

180 """Standardize the supplied dataId. 

181 

182 Adapts an arbitrary mapping and/or additional arguments into a true 

183 `DataCoordinate`, or augment an existing one. 

184 

185 Parameters 

186 ---------- 

187 mapping : `~collections.abc.Mapping`, optional 

188 An informal data ID that maps dimensions or dimension names to 

189 their primary key values (may also be a true `DataCoordinate`). 

190 graph : `DimensionGraph` 

191 The dimensions to be identified by the new `DataCoordinate`. 

192 If not provided, will be inferred from the keys of ``mapping`` and 

193 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

194 is already a `DataCoordinate`. 

195 universe : `DimensionUniverse` 

196 All known dimensions and their relationships; used to expand 

197 and validate dependencies when ``graph`` is not provided. 

198 defaults : `DataCoordinate`, optional 

199 Default dimension key-value pairs to use when needed. These are 

200 never used to infer ``graph``, and are ignored if a different value 

201 is provided for the same key in ``mapping`` or `**kwargs``. 

202 **kwargs 

203 Additional keyword arguments are treated like additional key-value 

204 pairs in ``mapping``. 

205 

206 Returns 

207 ------- 

208 coordinate : `DataCoordinate` 

209 A validated `DataCoordinate` instance. 

210 

211 Raises 

212 ------ 

213 TypeError 

214 Raised if the set of optional arguments provided is not supported. 

215 KeyError 

216 Raised if a key-value pair for a required dimension is missing. 

217 """ 

218 d: Dict[str, DataIdValue] = {} 

219 if isinstance(mapping, DataCoordinate): 

220 if graph is None: 

221 if not kwargs: 

222 # Already standardized to exactly what we want. 

223 return mapping 

224 elif kwargs.keys().isdisjoint(graph.dimensions.names): 

225 # User provided kwargs, but told us not to use them by 

226 # passing in dimensions that are disjoint from those kwargs. 

227 # This is not necessarily user error - it's a useful pattern 

228 # to pass in all of the key-value pairs you have and let the 

229 # code here pull out only what it needs. 

230 return mapping.subset(graph) 

231 assert universe is None or universe == mapping.universe 

232 universe = mapping.universe 

233 d.update((name, mapping[name]) for name in mapping.graph.required.names) 

234 if mapping.hasFull(): 

235 d.update((name, mapping[name]) for name in mapping.graph.implied.names) 

236 elif isinstance(mapping, NamedKeyMapping): 

237 d.update(mapping.byName()) 

238 elif mapping is not None: 

239 d.update(mapping) 

240 d.update(kwargs) 

241 if graph is None: 

242 if defaults is not None: 

243 universe = defaults.universe 

244 elif universe is None: 

245 raise TypeError("universe must be provided if graph is not.") 

246 graph = DimensionGraph(universe, names=d.keys()) 

247 if not graph.dimensions: 

248 return DataCoordinate.makeEmpty(graph.universe) 

249 if defaults is not None: 

250 if defaults.hasFull(): 

251 for k, v in defaults.full.items(): 

252 d.setdefault(k.name, v) 

253 else: 

254 for k, v in defaults.items(): 

255 d.setdefault(k.name, v) 

256 if d.keys() >= graph.dimensions.names: 

257 values = tuple(d[name] for name in graph._dataCoordinateIndices.keys()) 

258 else: 

259 try: 

260 values = tuple(d[name] for name in graph.required.names) 

261 except KeyError as err: 

262 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err 

263 # Some backends cannot handle numpy.int64 type which is a subclass of 

264 # numbers.Integral; convert that to int. 

265 values = tuple( 

266 int(val) if isinstance(val, numbers.Integral) else val for val in values # type: ignore 

267 ) 

268 return _BasicTupleDataCoordinate(graph, values) 

269 

270 @staticmethod 

271 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate: 

272 """Return an empty `DataCoordinate`. 

273 

274 It identifies the null set of dimensions. 

275 

276 Parameters 

277 ---------- 

278 universe : `DimensionUniverse` 

279 Universe to which this null dimension set belongs. 

280 

281 Returns 

282 ------- 

283 dataId : `DataCoordinate` 

284 A data ID object that identifies no dimensions. `hasFull` and 

285 `hasRecords` are guaranteed to return `True`, because both `full` 

286 and `records` are just empty mappings. 

287 """ 

288 return _ExpandedTupleDataCoordinate(universe.empty, (), {}) 

289 

290 @staticmethod 

291 def fromRequiredValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

292 """Construct a `DataCoordinate` from required dimension values. 

293 

294 This is a low-level interface with at most assertion-level checking of 

295 inputs. Most callers should use `standardize` instead. 

296 

297 Parameters 

298 ---------- 

299 graph : `DimensionGraph` 

300 Dimensions this data ID will identify. 

301 values : `tuple` [ `int` or `str` ] 

302 Tuple of primary key values corresponding to ``graph.required``, 

303 in that order. 

304 

305 Returns 

306 ------- 

307 dataId : `DataCoordinate` 

308 A data ID object that identifies the given dimensions. 

309 ``dataId.hasFull()`` will return `True` if and only if 

310 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

311 return `True`. 

312 """ 

313 assert len(graph.required) == len( 

314 values 

315 ), f"Inconsistency between dimensions {graph.required} and required values {values}." 

316 return _BasicTupleDataCoordinate(graph, values) 

317 

318 @staticmethod 

319 def fromFullValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

320 """Construct a `DataCoordinate` from all dimension values. 

321 

322 This is a low-level interface with at most assertion-level checking of 

323 inputs. Most callers should use `standardize` instead. 

324 

325 Parameters 

326 ---------- 

327 graph : `DimensionGraph` 

328 Dimensions this data ID will identify. 

329 values : `tuple` [ `int` or `str` ] 

330 Tuple of primary key values corresponding to 

331 ``itertools.chain(graph.required, graph.implied)``, in that order. 

332 Note that this is _not_ the same order as ``graph.dimensions``, 

333 though these contain the same elements. 

334 

335 Returns 

336 ------- 

337 dataId : `DataCoordinate` 

338 A data ID object that identifies the given dimensions. 

339 ``dataId.hasFull()`` will return `True` if and only if 

340 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

341 return `True`. 

342 """ 

343 assert len(graph.dimensions) == len( 

344 values 

345 ), f"Inconsistency between dimensions {graph.dimensions} and full values {values}." 

346 return _BasicTupleDataCoordinate(graph, values) 

347 

348 def __hash__(self) -> int: 

349 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required)) 

350 

351 def __eq__(self, other: Any) -> bool: 

352 if not isinstance(other, DataCoordinate): 

353 other = DataCoordinate.standardize(other, universe=self.universe) 

354 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required) 

355 

356 def __repr__(self) -> str: 

357 # We can't make repr yield something that could be exec'd here without 

358 # printing out the whole DimensionUniverse the graph is derived from. 

359 # So we print something that mostly looks like a dict, but doesn't 

360 # quote its keys: that's both more compact and something that can't 

361 # be mistaken for an actual dict or something that could be exec'd. 

362 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names] 

363 if self.hasFull() and self.graph.required != self.graph.dimensions: 

364 terms.append("...") 

365 return "{{{}}}".format(", ".join(terms)) 

366 

367 def __lt__(self, other: Any) -> bool: 

368 # Allow DataCoordinate to be sorted 

369 if not isinstance(other, type(self)): 

370 return NotImplemented 

371 # Form tuple of tuples for each DataCoordinate: 

372 # Unlike repr() we only use required keys here to ensure that 

373 # __eq__ can not be true simultaneously with __lt__ being true. 

374 self_kv = tuple(self.items()) 

375 other_kv = tuple(other.items()) 

376 

377 return self_kv < other_kv 

378 

379 def __iter__(self) -> Iterator[Dimension]: 

380 return iter(self.keys()) 

381 

382 def __len__(self) -> int: 

383 return len(self.keys()) 

384 

385 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

386 return self.graph.required 

387 

388 @property 

389 def names(self) -> AbstractSet[str]: 

390 """Names of the required dimensions identified by this data ID. 

391 

392 They are returned in the same order as `keys` 

393 (`collections.abc.Set` [ `str` ]). 

394 """ 

395 return self.keys().names 

396 

397 @abstractmethod 

398 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

399 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``. 

400 

401 Parameters 

402 ---------- 

403 graph : `DimensionGraph` 

404 The dimensions identified by the returned `DataCoordinate`. 

405 

406 Returns 

407 ------- 

408 coordinate : `DataCoordinate` 

409 A `DataCoordinate` instance that identifies only the given 

410 dimensions. May be ``self`` if ``graph == self.graph``. 

411 

412 Raises 

413 ------ 

414 KeyError 

415 Raised if the primary key value for one or more required dimensions 

416 is unknown. This may happen if ``graph.issubset(self.graph)`` is 

417 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if 

418 ``self.hasFull()`` is `False` and 

419 ``graph.required.issubset(self.graph.required)`` is `False`. As 

420 an example of the latter case, consider trying to go from a data ID 

421 with dimensions {instrument, physical_filter, band} to 

422 just {instrument, band}; band is implied by 

423 physical_filter and hence would have no value in the original data 

424 ID if ``self.hasFull()`` is `False`. 

425 

426 Notes 

427 ----- 

428 If `hasFull` and `hasRecords` return `True` on ``self``, they will 

429 return `True` (respectively) on the returned `DataCoordinate` as well. 

430 The converse does not hold. 

431 """ 

432 raise NotImplementedError() 

433 

434 @abstractmethod 

435 def union(self, other: DataCoordinate) -> DataCoordinate: 

436 """Combine two data IDs. 

437 

438 Yields a new one that identifies all dimensions that either of them 

439 identify. 

440 

441 Parameters 

442 ---------- 

443 other : `DataCoordinate` 

444 Data ID to combine with ``self``. 

445 

446 Returns 

447 ------- 

448 unioned : `DataCoordinate` 

449 A `DataCoordinate` instance that satisfies 

450 ``unioned.graph == self.graph.union(other.graph)``. Will preserve 

451 ``hasFull`` and ``hasRecords`` whenever possible. 

452 

453 Notes 

454 ----- 

455 No checking for consistency is performed on values for keys that 

456 ``self`` and ``other`` have in common, and which value is included in 

457 the returned data ID is not specified. 

458 """ 

459 raise NotImplementedError() 

460 

461 @abstractmethod 

462 def expanded( 

463 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

464 ) -> DataCoordinate: 

465 """Return a `DataCoordinate` that holds the given records. 

466 

467 Guarantees that `hasRecords` returns `True`. 

468 

469 This is a low-level interface with at most assertion-level checking of 

470 inputs. Most callers should use `Registry.expandDataId` instead. 

471 

472 Parameters 

473 ---------- 

474 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

475 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

476 `Mapping` with `str` (`DimensionElement` name) keys and 

477 `DimensionRecord` values. Keys must cover all elements in 

478 ``self.graph.elements``. Values may be `None`, but only to reflect 

479 actual NULL values in the database, not just records that have not 

480 been fetched. 

481 """ 

482 raise NotImplementedError() 

483 

484 @property 

485 def universe(self) -> DimensionUniverse: 

486 """Universe that defines all known compatible dimensions. 

487 

488 The univers will be compatible with this coordinate 

489 (`DimensionUniverse`). 

490 """ 

491 return self.graph.universe 

492 

493 @property 

494 @abstractmethod 

495 def graph(self) -> DimensionGraph: 

496 """Dimensions identified by this data ID (`DimensionGraph`). 

497 

498 Note that values are only required to be present for dimensions in 

499 ``self.graph.required``; all others may be retrieved (from a 

500 `Registry`) given these. 

501 """ 

502 raise NotImplementedError() 

503 

504 @abstractmethod 

505 def hasFull(self) -> bool: 

506 """Whether this data ID contains implied and required values. 

507 

508 Returns 

509 ------- 

510 state : `bool` 

511 If `True`, `__getitem__`, `get`, and `__contains__` (but not 

512 `keys`!) will act as though the mapping includes key-value pairs 

513 for implied dimensions, and the `full` property may be used. If 

514 `False`, these operations only include key-value pairs for required 

515 dimensions, and accessing `full` is an error. Always `True` if 

516 there are no implied dimensions. 

517 """ 

518 raise NotImplementedError() 

519 

520 @property 

521 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]: 

522 """Return mapping for all dimensions in ``self.graph``. 

523 

524 The mapping includes key-value pairs for all dimensions in 

525 ``self.graph``, including implied (`NamedKeyMapping`). 

526 

527 Accessing this attribute if `hasFull` returns `False` is a logic error 

528 that may raise an exception of unspecified type either immediately or 

529 when implied keys are accessed via the returned mapping, depending on 

530 the implementation and whether assertions are enabled. 

531 """ 

532 assert self.hasFull(), "full may only be accessed if hasFull() returns True." 

533 return _DataCoordinateFullView(self) 

534 

535 @abstractmethod 

536 def hasRecords(self) -> bool: 

537 """Whether this data ID contains records. 

538 

539 These are the records for all of the dimension elements it identifies. 

540 

541 Returns 

542 ------- 

543 state : `bool` 

544 If `True`, the following attributes may be accessed: 

545 

546 - `records` 

547 - `region` 

548 - `timespan` 

549 - `pack` 

550 

551 If `False`, accessing any of these is considered a logic error. 

552 """ 

553 raise NotImplementedError() 

554 

555 @property 

556 def records(self) -> NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]: 

557 """Return the records. 

558 

559 Returns a mapping that contains `DimensionRecord` objects for all 

560 elements identified by this data ID (`NamedKeyMapping`). 

561 

562 The values of this mapping may be `None` if and only if there is no 

563 record for that element with these dimensions in the database (which 

564 means some foreign key field must have a NULL value). 

565 

566 Accessing this attribute if `hasRecords` returns `False` is a logic 

567 error that may raise an exception of unspecified type either 

568 immediately or when the returned mapping is used, depending on the 

569 implementation and whether assertions are enabled. 

570 """ 

571 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True." 

572 return _DataCoordinateRecordsView(self) 

573 

574 @abstractmethod 

575 def _record(self, name: str) -> Optional[DimensionRecord]: 

576 """Protected implementation hook that backs the ``records`` attribute. 

577 

578 Parameters 

579 ---------- 

580 name : `str` 

581 The name of a `DimensionElement`, guaranteed to be in 

582 ``self.graph.elements.names``. 

583 

584 Returns 

585 ------- 

586 record : `DimensionRecord` or `None` 

587 The dimension record for the given element identified by this 

588 data ID, or `None` if there is no such record. 

589 """ 

590 raise NotImplementedError() 

591 

592 @property 

593 def region(self) -> Optional[Region]: 

594 """Spatial region associated with this data ID. 

595 

596 (`lsst.sphgeom.Region` or `None`). 

597 

598 This is `None` if and only if ``self.graph.spatial`` is empty. 

599 

600 Accessing this attribute if `hasRecords` returns `False` is a logic 

601 error that may or may not raise an exception, depending on the 

602 implementation and whether assertions are enabled. 

603 """ 

604 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True." 

605 regions = [] 

606 for family in self.graph.spatial: 

607 element = family.choose(self.graph.elements) 

608 record = self._record(element.name) 

609 if record is None or record.region is None: 

610 return None 

611 else: 

612 regions.append(record.region) 

613 return _intersectRegions(*regions) 

614 

615 @property 

616 def timespan(self) -> Optional[Timespan]: 

617 """Temporal interval associated with this data ID. 

618 

619 (`Timespan` or `None`). 

620 

621 This is `None` if and only if ``self.graph.timespan`` is empty. 

622 

623 Accessing this attribute if `hasRecords` returns `False` is a logic 

624 error that may or may not raise an exception, depending on the 

625 implementation and whether assertions are enabled. 

626 """ 

627 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True." 

628 timespans = [] 

629 for family in self.graph.temporal: 

630 element = family.choose(self.graph.elements) 

631 record = self._record(element.name) 

632 # DimensionRecord subclasses for temporal elements always have 

633 # .timespan, but they're dynamic so this can't be type-checked. 

634 if record is None or record.timespan is None: 

635 return None 

636 else: 

637 timespans.append(record.timespan) 

638 if not timespans: 

639 return None 

640 elif len(timespans) == 1: 

641 return timespans[0] 

642 else: 

643 return Timespan.intersection(*timespans) 

644 

645 @overload 

646 def pack(self, name: str, *, returnMaxBits: Literal[True]) -> Tuple[int, int]: 

647 ... 

648 

649 @overload 

650 def pack(self, name: str, *, returnMaxBits: Literal[False]) -> int: 

651 ... 

652 

653 # TODO: Remove this method and its overloads above on DM-38687. 

654 @deprecated( 

655 "Deprecated in favor of configurable dimension packers. Will be removed after v27.", 

656 version="v26", 

657 category=FutureWarning, 

658 ) 

659 def pack(self, name: str, *, returnMaxBits: bool = False) -> Union[Tuple[int, int], int]: 

660 """Pack this data ID into an integer. 

661 

662 Parameters 

663 ---------- 

664 name : `str` 

665 Name of the `DimensionPacker` algorithm (as defined in the 

666 dimension configuration). 

667 returnMaxBits : `bool`, optional 

668 If `True` (`False` is default), return the maximum number of 

669 nonzero bits in the returned integer across all data IDs. 

670 

671 Returns 

672 ------- 

673 packed : `int` 

674 Integer ID. This ID is unique only across data IDs that have 

675 the same values for the packer's "fixed" dimensions. 

676 maxBits : `int`, optional 

677 Maximum number of nonzero bits in ``packed``. Not returned unless 

678 ``returnMaxBits`` is `True`. 

679 

680 Notes 

681 ----- 

682 Accessing this attribute if `hasRecords` returns `False` is a logic 

683 error that may or may not raise an exception, depending on the 

684 implementation and whether assertions are enabled. 

685 """ 

686 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." 

687 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) 

688 

689 def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate: 

690 """Convert this class to a simple python type. 

691 

692 This is suitable for serialization. 

693 

694 Parameters 

695 ---------- 

696 minimal : `bool`, optional 

697 Use minimal serialization. If set the records will not be attached. 

698 

699 Returns 

700 ------- 

701 simple : `SerializedDataCoordinate` 

702 The object converted to simple form. 

703 """ 

704 # Convert to a dict form 

705 if self.hasFull(): 

706 dataId = self.full.byName() 

707 else: 

708 dataId = self.byName() 

709 records: Optional[Dict[str, SerializedDimensionRecord]] 

710 if not minimal and self.hasRecords(): 

711 records = {k: v.to_simple() for k, v in self.records.byName().items() if v is not None} 

712 else: 

713 records = None 

714 

715 return SerializedDataCoordinate(dataId=dataId, records=records) 

716 

717 @classmethod 

718 def from_simple( 

719 cls, 

720 simple: SerializedDataCoordinate, 

721 universe: Optional[DimensionUniverse] = None, 

722 registry: Optional[Registry] = None, 

723 ) -> DataCoordinate: 

724 """Construct a new object from the simplified form. 

725 

726 The data is assumed to be of the form returned from the `to_simple` 

727 method. 

728 

729 Parameters 

730 ---------- 

731 simple : `dict` of [`str`, `Any`] 

732 The `dict` returned by `to_simple()`. 

733 universe : `DimensionUniverse` 

734 The special graph of all known dimensions. 

735 registry : `lsst.daf.butler.Registry`, optional 

736 Registry from which a universe can be extracted. Can be `None` 

737 if universe is provided explicitly. 

738 

739 Returns 

740 ------- 

741 dataId : `DataCoordinate` 

742 Newly-constructed object. 

743 """ 

744 if universe is None and registry is None: 

745 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

746 if universe is None and registry is not None: 

747 universe = registry.dimensions 

748 if universe is None: 

749 # this is for mypy 

750 raise ValueError("Unable to determine a usable universe") 

751 

752 dataId = cls.standardize(simple.dataId, universe=universe) 

753 if simple.records: 

754 dataId = dataId.expanded( 

755 {k: DimensionRecord.from_simple(v, universe=universe) for k, v in simple.records.items()} 

756 ) 

757 return dataId 

758 

759 to_json = to_json_pydantic 

760 from_json: ClassVar = classmethod(from_json_pydantic) 

761 

762 

763DataId = Union[DataCoordinate, Mapping[str, Any]] 

764"""A type-annotation alias for signatures that accept both informal data ID 

765dictionaries and validated `DataCoordinate` instances. 

766""" 

767 

768 

769class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]): 

770 """View class for `DataCoordinate.full`. 

771 

772 Provides the default implementation for 

773 `DataCoordinate.full`. 

774 

775 Parameters 

776 ---------- 

777 target : `DataCoordinate` 

778 The `DataCoordinate` instance this object provides a view of. 

779 """ 

780 

781 def __init__(self, target: DataCoordinate): 

782 self._target = target 

783 

784 __slots__ = ("_target",) 

785 

786 def __repr__(self) -> str: 

787 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names] 

788 return "{{{}}}".format(", ".join(terms)) 

789 

790 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

791 return self._target[key] 

792 

793 def __iter__(self) -> Iterator[Dimension]: 

794 return iter(self.keys()) 

795 

796 def __len__(self) -> int: 

797 return len(self.keys()) 

798 

799 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

800 return self._target.graph.dimensions 

801 

802 @property 

803 def names(self) -> AbstractSet[str]: 

804 # Docstring inherited from `NamedKeyMapping`. 

805 return self.keys().names 

806 

807 

808class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]): 

809 """View class for `DataCoordinate.records`. 

810 

811 Provides the default implementation for 

812 `DataCoordinate.records`. 

813 

814 Parameters 

815 ---------- 

816 target : `DataCoordinate` 

817 The `DataCoordinate` instance this object provides a view of. 

818 """ 

819 

820 def __init__(self, target: DataCoordinate): 

821 self._target = target 

822 

823 __slots__ = ("_target",) 

824 

825 def __repr__(self) -> str: 

826 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names] 

827 return "{{{}}}".format(", ".join(terms)) 

828 

829 def __str__(self) -> str: 

830 return "\n".join(str(v) for v in self.values()) 

831 

832 def __getitem__(self, key: Union[DimensionElement, str]) -> Optional[DimensionRecord]: 

833 if isinstance(key, DimensionElement): 

834 key = key.name 

835 return self._target._record(key) 

836 

837 def __iter__(self) -> Iterator[DimensionElement]: 

838 return iter(self.keys()) 

839 

840 def __len__(self) -> int: 

841 return len(self.keys()) 

842 

843 def keys(self) -> NamedValueAbstractSet[DimensionElement]: # type: ignore 

844 return self._target.graph.elements 

845 

846 @property 

847 def names(self) -> AbstractSet[str]: 

848 # Docstring inherited from `NamedKeyMapping`. 

849 return self.keys().names 

850 

851 

852class _BasicTupleDataCoordinate(DataCoordinate): 

853 """Standard implementation of `DataCoordinate`. 

854 

855 Backed by a tuple of values. 

856 

857 This class should only be accessed outside this module via the 

858 `DataCoordinate` interface, and should only be constructed via the static 

859 methods there. 

860 

861 Parameters 

862 ---------- 

863 graph : `DimensionGraph` 

864 The dimensions to be identified. 

865 values : `tuple` [ `int` or `str` ] 

866 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May 

867 include values for just required dimensions (which always come first) 

868 or all dimensions. 

869 """ 

870 

871 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...]): 

872 self._graph = graph 

873 self._values = values 

874 

875 __slots__ = ("_graph", "_values") 

876 

877 @property 

878 def graph(self) -> DimensionGraph: 

879 # Docstring inherited from DataCoordinate. 

880 return self._graph 

881 

882 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

883 # Docstring inherited from DataCoordinate. 

884 if isinstance(key, Dimension): 

885 key = key.name 

886 index = self._graph._dataCoordinateIndices[key] 

887 try: 

888 return self._values[index] 

889 except IndexError: 

890 # Caller asked for an implied dimension, but this object only has 

891 # values for the required ones. 

892 raise KeyError(key) from None 

893 

894 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

895 # Docstring inherited from DataCoordinate. 

896 if self._graph == graph: 

897 return self 

898 elif self.hasFull() or self._graph.required >= graph.dimensions: 

899 return _BasicTupleDataCoordinate( 

900 graph, 

901 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

902 ) 

903 else: 

904 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names)) 

905 

906 def union(self, other: DataCoordinate) -> DataCoordinate: 

907 # Docstring inherited from DataCoordinate. 

908 graph = self.graph.union(other.graph) 

909 # See if one or both input data IDs is already what we want to return; 

910 # if so, return the most complete one we have. 

911 if other.graph == graph: 

912 if self.graph == graph: 

913 # Input data IDs have the same graph (which is also the result 

914 # graph), but may not have the same content. 

915 # other might have records; self does not, so try other first. 

916 # If it at least has full values, it's no worse than self. 

917 if other.hasFull(): 

918 return other 

919 else: 

920 return self 

921 elif other.hasFull(): 

922 return other 

923 # There's some chance that neither self nor other has full values, 

924 # but together provide enough to the union to. Let the general 

925 # case below handle that. 

926 elif self.graph == graph: 

927 # No chance at returning records. If self has full values, it's 

928 # the best we can do. 

929 if self.hasFull(): 

930 return self 

931 # General case with actual merging of dictionaries. 

932 values = self.full.byName() if self.hasFull() else self.byName() 

933 values.update(other.full.byName() if other.hasFull() else other.byName()) 

934 return DataCoordinate.standardize(values, graph=graph) 

935 

936 def expanded( 

937 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

938 ) -> DataCoordinate: 

939 # Docstring inherited from DataCoordinate 

940 values = self._values 

941 if not self.hasFull(): 

942 # Extract a complete values tuple from the attributes of the given 

943 # records. It's possible for these to be inconsistent with 

944 # self._values (which is a serious problem, of course), but we've 

945 # documented this as a no-checking API. 

946 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied) 

947 return _ExpandedTupleDataCoordinate(self._graph, values, records) 

948 

949 def hasFull(self) -> bool: 

950 # Docstring inherited from DataCoordinate. 

951 return len(self._values) == len(self._graph._dataCoordinateIndices) 

952 

953 def hasRecords(self) -> bool: 

954 # Docstring inherited from DataCoordinate. 

955 return False 

956 

957 def _record(self, name: str) -> Optional[DimensionRecord]: 

958 # Docstring inherited from DataCoordinate. 

959 assert False 

960 

961 def __reduce__(self) -> tuple[Any, ...]: 

962 return (_BasicTupleDataCoordinate, (self._graph, self._values)) 

963 

964 def __getattr__(self, name: str) -> Any: 

965 if name in self.graph.elements.names: 

966 raise AttributeError( 

967 f"Dimension record attribute {name!r} is only available on expanded DataCoordinates." 

968 ) 

969 raise AttributeError(name) 

970 

971 

972class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate): 

973 """A `DataCoordinate` implementation that can hold `DimensionRecord`. 

974 

975 This class should only be accessed outside this module via the 

976 `DataCoordinate` interface, and should only be constructed via calls to 

977 `DataCoordinate.expanded`. 

978 

979 Parameters 

980 ---------- 

981 graph : `DimensionGraph` 

982 The dimensions to be identified. 

983 values : `tuple` [ `int` or `str` ] 

984 Data ID values, ordered to match ``graph._dataCoordinateIndices``. 

985 May include values for just required dimensions (which always come 

986 first) or all dimensions. 

987 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

988 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

989 `Mapping` with `str` (`DimensionElement` name) keys and 

990 `DimensionRecord` values. Keys must cover all elements in 

991 ``self.graph.elements``. Values may be `None`, but only to reflect 

992 actual NULL values in the database, not just records that have not 

993 been fetched. 

994 """ 

995 

996 def __init__( 

997 self, 

998 graph: DimensionGraph, 

999 values: Tuple[DataIdValue, ...], 

1000 records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]], 

1001 ): 

1002 super().__init__(graph, values) 

1003 assert super().hasFull(), "This implementation requires full dimension records." 

1004 self._records = records 

1005 

1006 __slots__ = ("_records",) 

1007 

1008 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

1009 # Docstring inherited from DataCoordinate. 

1010 if self._graph == graph: 

1011 return self 

1012 return _ExpandedTupleDataCoordinate( 

1013 graph, tuple(self[k] for k in graph._dataCoordinateIndices.keys()), records=self._records 

1014 ) 

1015 

1016 def expanded( 

1017 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

1018 ) -> DataCoordinate: 

1019 # Docstring inherited from DataCoordinate. 

1020 return self 

1021 

1022 def union(self, other: DataCoordinate) -> DataCoordinate: 

1023 # Docstring inherited from DataCoordinate. 

1024 graph = self.graph.union(other.graph) 

1025 # See if one or both input data IDs is already what we want to return; 

1026 # if so, return the most complete one we have. 

1027 if self.graph == graph: 

1028 # self has records, so even if other is also a valid result, it's 

1029 # no better. 

1030 return self 

1031 if other.graph == graph: 

1032 # If other has full values, and self does not identify some of 

1033 # those, it's the base we can do. It may have records, too. 

1034 if other.hasFull(): 

1035 return other 

1036 # If other does not have full values, there's a chance self may 

1037 # provide the values needed to complete it. For example, self 

1038 # could be {band} while other could be 

1039 # {instrument, physical_filter, band}, with band unknown. 

1040 # General case with actual merging of dictionaries. 

1041 values = self.full.byName() 

1042 values.update(other.full.byName() if other.hasFull() else other.byName()) 

1043 basic = DataCoordinate.standardize(values, graph=graph) 

1044 # See if we can add records. 

1045 if self.hasRecords() and other.hasRecords(): 

1046 # Sometimes the elements of a union of graphs can contain elements 

1047 # that weren't in either input graph (because graph unions are only 

1048 # on dimensions). e.g. {visit} | {detector} brings along 

1049 # visit_detector_region. 

1050 elements = set(graph.elements.names) 

1051 elements -= self.graph.elements.names 

1052 elements -= other.graph.elements.names 

1053 if not elements: 

1054 records = NamedKeyDict[DimensionElement, Optional[DimensionRecord]](self.records) 

1055 records.update(other.records) 

1056 return basic.expanded(records.freeze()) 

1057 return basic 

1058 

1059 def hasFull(self) -> bool: 

1060 # Docstring inherited from DataCoordinate. 

1061 return True 

1062 

1063 def hasRecords(self) -> bool: 

1064 # Docstring inherited from DataCoordinate. 

1065 return True 

1066 

1067 def _record(self, name: str) -> Optional[DimensionRecord]: 

1068 # Docstring inherited from DataCoordinate. 

1069 return self._records[name] 

1070 

1071 def __reduce__(self) -> tuple[Any, ...]: 

1072 return (_ExpandedTupleDataCoordinate, (self._graph, self._values, self._records)) 

1073 

1074 def __getattr__(self, name: str) -> Any: 

1075 try: 

1076 return self._record(name) 

1077 except KeyError: 

1078 raise AttributeError(name) from None 

1079 

1080 def __dir__(self) -> list[str]: 

1081 result = list(super().__dir__()) 

1082 result.extend(self.graph.elements.names) 

1083 return result