Coverage for python/lsst/daf/butler/core/dimensions/_coordinate.py: 26%

343 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-04-13 02:34 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22# 

23# Design notes for this module are in 

24# doc/lsst.daf.butler/dev/dataCoordinate.py. 

25# 

26 

27from __future__ import annotations 

28 

29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue", "SerializedDataCoordinate") 

30 

31import numbers 

32from abc import abstractmethod 

33from typing import ( 

34 TYPE_CHECKING, 

35 AbstractSet, 

36 Any, 

37 ClassVar, 

38 Dict, 

39 Iterator, 

40 Literal, 

41 Mapping, 

42 Optional, 

43 Tuple, 

44 Union, 

45 overload, 

46) 

47 

48from lsst.sphgeom import IntersectionRegion, Region 

49from pydantic import BaseModel 

50 

51from ..json import from_json_pydantic, to_json_pydantic 

52from ..named import NamedKeyDict, NamedKeyMapping, NamedValueAbstractSet, NameLookupMapping 

53from ..timespan import Timespan 

54from ._elements import Dimension, DimensionElement 

55from ._graph import DimensionGraph 

56from ._records import DimensionRecord, SerializedDimensionRecord 

57 

58if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

59 from ...registry import Registry 

60 from ._universe import DimensionUniverse 

61 

62DataIdKey = Union[str, Dimension] 

63"""Type annotation alias for the keys that can be used to index a 

64DataCoordinate. 

65""" 

66 

67# Pydantic will cast int to str if str is first in the Union. 

68DataIdValue = Union[int, str, None] 

69"""Type annotation alias for the values that can be present in a 

70DataCoordinate or other data ID. 

71""" 

72 

73 

74class SerializedDataCoordinate(BaseModel): 

75 """Simplified model for serializing a `DataCoordinate`.""" 

76 

77 dataId: Dict[str, DataIdValue] 

78 records: Optional[Dict[str, SerializedDimensionRecord]] = None 

79 

80 @classmethod 

81 def direct(cls, *, dataId: Dict[str, DataIdValue], records: Dict[str, Dict]) -> SerializedDataCoordinate: 

82 """Construct a `SerializedDataCoordinate` directly without validators. 

83 

84 This differs from the pydantic "construct" method in that the arguments 

85 are explicitly what the model requires, and it will recurse through 

86 members, constructing them from their corresponding `direct` methods. 

87 

88 This method should only be called when the inputs are trusted. 

89 """ 

90 node = SerializedDataCoordinate.__new__(cls) 

91 setter = object.__setattr__ 

92 setter(node, "dataId", dataId) 

93 setter( 

94 node, 

95 "records", 

96 records 

97 if records is None 

98 else {k: SerializedDimensionRecord.direct(**v) for k, v in records.items()}, 

99 ) 

100 setter(node, "__fields_set__", {"dataId", "records"}) 

101 return node 

102 

103 

104def _intersectRegions(*args: Region) -> Optional[Region]: 

105 """Return the intersection of several regions. 

106 

107 For internal use by `ExpandedDataCoordinate` only. 

108 

109 If no regions are provided, returns `None`. 

110 """ 

111 if len(args) == 0: 

112 return None 

113 else: 

114 result = args[0] 

115 for n in range(1, len(args)): 

116 result = IntersectionRegion(result, args[n]) 

117 return result 

118 

119 

120class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]): 

121 """Data ID dictionary. 

122 

123 An immutable data ID dictionary that guarantees that its key-value pairs 

124 identify at least all required dimensions in a `DimensionGraph`. 

125 

126 `DataCoordinate` itself is an ABC, but provides `staticmethod` factory 

127 functions for private concrete implementations that should be sufficient 

128 for most purposes. `standardize` is the most flexible and safe of these; 

129 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are 

130 more specialized and perform little or no checking of inputs. 

131 

132 Notes 

133 ----- 

134 Like any data ID class, `DataCoordinate` behaves like a dictionary, but 

135 with some subtleties: 

136 

137 - Both `Dimension` instances and `str` names thereof may be used as keys 

138 in lookup operations, but iteration (and `keys`) will yield `Dimension` 

139 instances. The `names` property can be used to obtain the corresponding 

140 `str` names. 

141 

142 - Lookups for implied dimensions (those in ``self.graph.implied``) are 

143 supported if and only if `hasFull` returns `True`, and are never 

144 included in iteration or `keys`. The `full` property may be used to 

145 obtain a mapping whose keys do include implied dimensions. 

146 

147 - Equality comparison with other mappings is supported, but it always 

148 considers only required dimensions (as well as requiring both operands 

149 to identify the same dimensions). This is not quite consistent with the 

150 way mappings usually work - normally differing keys imply unequal 

151 mappings - but it makes sense in this context because data IDs with the 

152 same values for required dimensions but different values for implied 

153 dimensions represent a serious problem with the data that 

154 `DataCoordinate` cannot generally recognize on its own, and a data ID 

155 that knows implied dimension values should still be able to compare as 

156 equal to one that does not. This is of course not the way comparisons 

157 between simple `dict` data IDs work, and hence using a `DataCoordinate` 

158 instance for at least one operand in any data ID comparison is strongly 

159 recommended. 

160 

161 See also 

162 -------- 

163 :ref:`lsst.daf.butler-dimensions_data_ids` 

164 """ 

165 

166 __slots__ = () 

167 

168 _serializedType = SerializedDataCoordinate 

169 

170 @staticmethod 

171 def standardize( 

172 mapping: Optional[NameLookupMapping[Dimension, DataIdValue]] = None, 

173 *, 

174 graph: Optional[DimensionGraph] = None, 

175 universe: Optional[DimensionUniverse] = None, 

176 defaults: Optional[DataCoordinate] = None, 

177 **kwargs: Any, 

178 ) -> DataCoordinate: 

179 """Standardize the supplied dataId. 

180 

181 Adapts an arbitrary mapping and/or additional arguments into a true 

182 `DataCoordinate`, or augment an existing one. 

183 

184 Parameters 

185 ---------- 

186 mapping : `~collections.abc.Mapping`, optional 

187 An informal data ID that maps dimensions or dimension names to 

188 their primary key values (may also be a true `DataCoordinate`). 

189 graph : `DimensionGraph` 

190 The dimensions to be identified by the new `DataCoordinate`. 

191 If not provided, will be inferred from the keys of ``mapping`` and 

192 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

193 is already a `DataCoordinate`. 

194 universe : `DimensionUniverse` 

195 All known dimensions and their relationships; used to expand 

196 and validate dependencies when ``graph`` is not provided. 

197 defaults : `DataCoordinate`, optional 

198 Default dimension key-value pairs to use when needed. These are 

199 never used to infer ``graph``, and are ignored if a different value 

200 is provided for the same key in ``mapping`` or `**kwargs``. 

201 **kwargs 

202 Additional keyword arguments are treated like additional key-value 

203 pairs in ``mapping``. 

204 

205 Returns 

206 ------- 

207 coordinate : `DataCoordinate` 

208 A validated `DataCoordinate` instance. 

209 

210 Raises 

211 ------ 

212 TypeError 

213 Raised if the set of optional arguments provided is not supported. 

214 KeyError 

215 Raised if a key-value pair for a required dimension is missing. 

216 """ 

217 d: Dict[str, DataIdValue] = {} 

218 if isinstance(mapping, DataCoordinate): 

219 if graph is None: 

220 if not kwargs: 

221 # Already standardized to exactly what we want. 

222 return mapping 

223 elif kwargs.keys().isdisjoint(graph.dimensions.names): 

224 # User provided kwargs, but told us not to use them by 

225 # passing in dimensions that are disjoint from those kwargs. 

226 # This is not necessarily user error - it's a useful pattern 

227 # to pass in all of the key-value pairs you have and let the 

228 # code here pull out only what it needs. 

229 return mapping.subset(graph) 

230 assert universe is None or universe == mapping.universe 

231 universe = mapping.universe 

232 d.update((name, mapping[name]) for name in mapping.graph.required.names) 

233 if mapping.hasFull(): 

234 d.update((name, mapping[name]) for name in mapping.graph.implied.names) 

235 elif isinstance(mapping, NamedKeyMapping): 

236 d.update(mapping.byName()) 

237 elif mapping is not None: 

238 d.update(mapping) 

239 d.update(kwargs) 

240 if graph is None: 

241 if defaults is not None: 

242 universe = defaults.universe 

243 elif universe is None: 

244 raise TypeError("universe must be provided if graph is not.") 

245 graph = DimensionGraph(universe, names=d.keys()) 

246 if not graph.dimensions: 

247 return DataCoordinate.makeEmpty(graph.universe) 

248 if defaults is not None: 

249 if defaults.hasFull(): 

250 for k, v in defaults.full.items(): 

251 d.setdefault(k.name, v) 

252 else: 

253 for k, v in defaults.items(): 

254 d.setdefault(k.name, v) 

255 if d.keys() >= graph.dimensions.names: 

256 values = tuple(d[name] for name in graph._dataCoordinateIndices.keys()) 

257 else: 

258 try: 

259 values = tuple(d[name] for name in graph.required.names) 

260 except KeyError as err: 

261 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err 

262 # Some backends cannot handle numpy.int64 type which is a subclass of 

263 # numbers.Integral; convert that to int. 

264 values = tuple( 

265 int(val) if isinstance(val, numbers.Integral) else val for val in values # type: ignore 

266 ) 

267 return _BasicTupleDataCoordinate(graph, values) 

268 

269 @staticmethod 

270 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate: 

271 """Return an empty `DataCoordinate`. 

272 

273 It identifies the null set of dimensions. 

274 

275 Parameters 

276 ---------- 

277 universe : `DimensionUniverse` 

278 Universe to which this null dimension set belongs. 

279 

280 Returns 

281 ------- 

282 dataId : `DataCoordinate` 

283 A data ID object that identifies no dimensions. `hasFull` and 

284 `hasRecords` are guaranteed to return `True`, because both `full` 

285 and `records` are just empty mappings. 

286 """ 

287 return _ExpandedTupleDataCoordinate(universe.empty, (), {}) 

288 

289 @staticmethod 

290 def fromRequiredValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

291 """Construct a `DataCoordinate` from required dimension values. 

292 

293 This is a low-level interface with at most assertion-level checking of 

294 inputs. Most callers should use `standardize` instead. 

295 

296 Parameters 

297 ---------- 

298 graph : `DimensionGraph` 

299 Dimensions this data ID will identify. 

300 values : `tuple` [ `int` or `str` ] 

301 Tuple of primary key values corresponding to ``graph.required``, 

302 in that order. 

303 

304 Returns 

305 ------- 

306 dataId : `DataCoordinate` 

307 A data ID object that identifies the given dimensions. 

308 ``dataId.hasFull()`` will return `True` if and only if 

309 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

310 return `True`. 

311 """ 

312 assert len(graph.required) == len( 

313 values 

314 ), f"Inconsistency between dimensions {graph.required} and required values {values}." 

315 return _BasicTupleDataCoordinate(graph, values) 

316 

317 @staticmethod 

318 def fromFullValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

319 """Construct a `DataCoordinate` from all dimension values. 

320 

321 This is a low-level interface with at most assertion-level checking of 

322 inputs. Most callers should use `standardize` instead. 

323 

324 Parameters 

325 ---------- 

326 graph : `DimensionGraph` 

327 Dimensions this data ID will identify. 

328 values : `tuple` [ `int` or `str` ] 

329 Tuple of primary key values corresponding to 

330 ``itertools.chain(graph.required, graph.implied)``, in that order. 

331 Note that this is _not_ the same order as ``graph.dimensions``, 

332 though these contain the same elements. 

333 

334 Returns 

335 ------- 

336 dataId : `DataCoordinate` 

337 A data ID object that identifies the given dimensions. 

338 ``dataId.hasFull()`` will return `True` if and only if 

339 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

340 return `True`. 

341 """ 

342 assert len(graph.dimensions) == len( 

343 values 

344 ), f"Inconsistency between dimensions {graph.dimensions} and full values {values}." 

345 return _BasicTupleDataCoordinate(graph, values) 

346 

347 def __hash__(self) -> int: 

348 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required)) 

349 

350 def __eq__(self, other: Any) -> bool: 

351 if not isinstance(other, DataCoordinate): 

352 other = DataCoordinate.standardize(other, universe=self.universe) 

353 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required) 

354 

355 def __repr__(self) -> str: 

356 # We can't make repr yield something that could be exec'd here without 

357 # printing out the whole DimensionUniverse the graph is derived from. 

358 # So we print something that mostly looks like a dict, but doesn't 

359 # quote its keys: that's both more compact and something that can't 

360 # be mistaken for an actual dict or something that could be exec'd. 

361 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names] 

362 if self.hasFull() and self.graph.required != self.graph.dimensions: 

363 terms.append("...") 

364 return "{{{}}}".format(", ".join(terms)) 

365 

366 def __lt__(self, other: Any) -> bool: 

367 # Allow DataCoordinate to be sorted 

368 if not isinstance(other, type(self)): 

369 return NotImplemented 

370 # Form tuple of tuples for each DataCoordinate: 

371 # Unlike repr() we only use required keys here to ensure that 

372 # __eq__ can not be true simultaneously with __lt__ being true. 

373 self_kv = tuple(self.items()) 

374 other_kv = tuple(other.items()) 

375 

376 return self_kv < other_kv 

377 

378 def __iter__(self) -> Iterator[Dimension]: 

379 return iter(self.keys()) 

380 

381 def __len__(self) -> int: 

382 return len(self.keys()) 

383 

384 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

385 return self.graph.required 

386 

387 @property 

388 def names(self) -> AbstractSet[str]: 

389 """Names of the required dimensions identified by this data ID. 

390 

391 They are returned in the same order as `keys` 

392 (`collections.abc.Set` [ `str` ]). 

393 """ 

394 return self.keys().names 

395 

396 @abstractmethod 

397 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

398 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``. 

399 

400 Parameters 

401 ---------- 

402 graph : `DimensionGraph` 

403 The dimensions identified by the returned `DataCoordinate`. 

404 

405 Returns 

406 ------- 

407 coordinate : `DataCoordinate` 

408 A `DataCoordinate` instance that identifies only the given 

409 dimensions. May be ``self`` if ``graph == self.graph``. 

410 

411 Raises 

412 ------ 

413 KeyError 

414 Raised if the primary key value for one or more required dimensions 

415 is unknown. This may happen if ``graph.issubset(self.graph)`` is 

416 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if 

417 ``self.hasFull()`` is `False` and 

418 ``graph.required.issubset(self.graph.required)`` is `False`. As 

419 an example of the latter case, consider trying to go from a data ID 

420 with dimensions {instrument, physical_filter, band} to 

421 just {instrument, band}; band is implied by 

422 physical_filter and hence would have no value in the original data 

423 ID if ``self.hasFull()`` is `False`. 

424 

425 Notes 

426 ----- 

427 If `hasFull` and `hasRecords` return `True` on ``self``, they will 

428 return `True` (respectively) on the returned `DataCoordinate` as well. 

429 The converse does not hold. 

430 """ 

431 raise NotImplementedError() 

432 

433 @abstractmethod 

434 def union(self, other: DataCoordinate) -> DataCoordinate: 

435 """Combine two data IDs. 

436 

437 Yields a new one that identifies all dimensions that either of them 

438 identify. 

439 

440 Parameters 

441 ---------- 

442 other : `DataCoordinate` 

443 Data ID to combine with ``self``. 

444 

445 Returns 

446 ------- 

447 unioned : `DataCoordinate` 

448 A `DataCoordinate` instance that satisfies 

449 ``unioned.graph == self.graph.union(other.graph)``. Will preserve 

450 ``hasFull`` and ``hasRecords`` whenever possible. 

451 

452 Notes 

453 ----- 

454 No checking for consistency is performed on values for keys that 

455 ``self`` and ``other`` have in common, and which value is included in 

456 the returned data ID is not specified. 

457 """ 

458 raise NotImplementedError() 

459 

460 @abstractmethod 

461 def expanded( 

462 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

463 ) -> DataCoordinate: 

464 """Return a `DataCoordinate` that holds the given records. 

465 

466 Guarantees that `hasRecords` returns `True`. 

467 

468 This is a low-level interface with at most assertion-level checking of 

469 inputs. Most callers should use `Registry.expandDataId` instead. 

470 

471 Parameters 

472 ---------- 

473 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

474 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

475 `Mapping` with `str` (`DimensionElement` name) keys and 

476 `DimensionRecord` values. Keys must cover all elements in 

477 ``self.graph.elements``. Values may be `None`, but only to reflect 

478 actual NULL values in the database, not just records that have not 

479 been fetched. 

480 """ 

481 raise NotImplementedError() 

482 

483 @property 

484 def universe(self) -> DimensionUniverse: 

485 """Universe that defines all known compatible dimensions. 

486 

487 The univers will be compatible with this coordinate 

488 (`DimensionUniverse`). 

489 """ 

490 return self.graph.universe 

491 

492 @property 

493 @abstractmethod 

494 def graph(self) -> DimensionGraph: 

495 """Dimensions identified by this data ID (`DimensionGraph`). 

496 

497 Note that values are only required to be present for dimensions in 

498 ``self.graph.required``; all others may be retrieved (from a 

499 `Registry`) given these. 

500 """ 

501 raise NotImplementedError() 

502 

503 @abstractmethod 

504 def hasFull(self) -> bool: 

505 """Whether this data ID contains implied and required values. 

506 

507 Returns 

508 ------- 

509 state : `bool` 

510 If `True`, `__getitem__`, `get`, and `__contains__` (but not 

511 `keys`!) will act as though the mapping includes key-value pairs 

512 for implied dimensions, and the `full` property may be used. If 

513 `False`, these operations only include key-value pairs for required 

514 dimensions, and accessing `full` is an error. Always `True` if 

515 there are no implied dimensions. 

516 """ 

517 raise NotImplementedError() 

518 

519 @property 

520 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]: 

521 """Return mapping for all dimensions in ``self.graph``. 

522 

523 The mapping includes key-value pairs for all dimensions in 

524 ``self.graph``, including implied (`NamedKeyMapping`). 

525 

526 Accessing this attribute if `hasFull` returns `False` is a logic error 

527 that may raise an exception of unspecified type either immediately or 

528 when implied keys are accessed via the returned mapping, depending on 

529 the implementation and whether assertions are enabled. 

530 """ 

531 assert self.hasFull(), "full may only be accessed if hasFull() returns True." 

532 return _DataCoordinateFullView(self) 

533 

534 @abstractmethod 

535 def hasRecords(self) -> bool: 

536 """Whether this data ID contains records. 

537 

538 These are the records for all of the dimension elements it identifies. 

539 

540 Returns 

541 ------- 

542 state : `bool` 

543 If `True`, the following attributes may be accessed: 

544 

545 - `records` 

546 - `region` 

547 - `timespan` 

548 - `pack` 

549 

550 If `False`, accessing any of these is considered a logic error. 

551 """ 

552 raise NotImplementedError() 

553 

554 @property 

555 def records(self) -> NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]: 

556 """Return the records. 

557 

558 Returns a mapping that contains `DimensionRecord` objects for all 

559 elements identified by this data ID (`NamedKeyMapping`). 

560 

561 The values of this mapping may be `None` if and only if there is no 

562 record for that element with these dimensions in the database (which 

563 means some foreign key field must have a NULL value). 

564 

565 Accessing this attribute if `hasRecords` returns `False` is a logic 

566 error that may raise an exception of unspecified type either 

567 immediately or when the returned mapping is used, depending on the 

568 implementation and whether assertions are enabled. 

569 """ 

570 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True." 

571 return _DataCoordinateRecordsView(self) 

572 

573 @abstractmethod 

574 def _record(self, name: str) -> Optional[DimensionRecord]: 

575 """Protected implementation hook that backs the ``records`` attribute. 

576 

577 Parameters 

578 ---------- 

579 name : `str` 

580 The name of a `DimensionElement`, guaranteed to be in 

581 ``self.graph.elements.names``. 

582 

583 Returns 

584 ------- 

585 record : `DimensionRecord` or `None` 

586 The dimension record for the given element identified by this 

587 data ID, or `None` if there is no such record. 

588 """ 

589 raise NotImplementedError() 

590 

591 @property 

592 def region(self) -> Optional[Region]: 

593 """Spatial region associated with this data ID. 

594 

595 (`lsst.sphgeom.Region` or `None`). 

596 

597 This is `None` if and only if ``self.graph.spatial`` is empty. 

598 

599 Accessing this attribute if `hasRecords` returns `False` is a logic 

600 error that may or may not raise an exception, depending on the 

601 implementation and whether assertions are enabled. 

602 """ 

603 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True." 

604 regions = [] 

605 for family in self.graph.spatial: 

606 element = family.choose(self.graph.elements) 

607 record = self._record(element.name) 

608 if record is None or record.region is None: 

609 return None 

610 else: 

611 regions.append(record.region) 

612 return _intersectRegions(*regions) 

613 

614 @property 

615 def timespan(self) -> Optional[Timespan]: 

616 """Temporal interval associated with this data ID. 

617 

618 (`Timespan` or `None`). 

619 

620 This is `None` if and only if ``self.graph.timespan`` is empty. 

621 

622 Accessing this attribute if `hasRecords` returns `False` is a logic 

623 error that may or may not raise an exception, depending on the 

624 implementation and whether assertions are enabled. 

625 """ 

626 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True." 

627 timespans = [] 

628 for family in self.graph.temporal: 

629 element = family.choose(self.graph.elements) 

630 record = self._record(element.name) 

631 # DimensionRecord subclasses for temporal elements always have 

632 # .timespan, but they're dynamic so this can't be type-checked. 

633 if record is None or record.timespan is None: 

634 return None 

635 else: 

636 timespans.append(record.timespan) 

637 if not timespans: 

638 return None 

639 elif len(timespans) == 1: 

640 return timespans[0] 

641 else: 

642 return Timespan.intersection(*timespans) 

643 

644 @overload 

645 def pack(self, name: str, *, returnMaxBits: Literal[True]) -> Tuple[int, int]: 

646 ... 

647 

648 @overload 

649 def pack(self, name: str, *, returnMaxBits: Literal[False]) -> int: 

650 ... 

651 

652 def pack(self, name: str, *, returnMaxBits: bool = False) -> Union[Tuple[int, int], int]: 

653 """Pack this data ID into an integer. 

654 

655 Parameters 

656 ---------- 

657 name : `str` 

658 Name of the `DimensionPacker` algorithm (as defined in the 

659 dimension configuration). 

660 returnMaxBits : `bool`, optional 

661 If `True` (`False` is default), return the maximum number of 

662 nonzero bits in the returned integer across all data IDs. 

663 

664 Returns 

665 ------- 

666 packed : `int` 

667 Integer ID. This ID is unique only across data IDs that have 

668 the same values for the packer's "fixed" dimensions. 

669 maxBits : `int`, optional 

670 Maximum number of nonzero bits in ``packed``. Not returned unless 

671 ``returnMaxBits`` is `True`. 

672 

673 Notes 

674 ----- 

675 Accessing this attribute if `hasRecords` returns `False` is a logic 

676 error that may or may not raise an exception, depending on the 

677 implementation and whether assertions are enabled. 

678 """ 

679 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." 

680 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) 

681 

682 def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate: 

683 """Convert this class to a simple python type. 

684 

685 This is suitable for serialization. 

686 

687 Parameters 

688 ---------- 

689 minimal : `bool`, optional 

690 Use minimal serialization. If set the records will not be attached. 

691 

692 Returns 

693 ------- 

694 simple : `SerializedDataCoordinate` 

695 The object converted to simple form. 

696 """ 

697 # Convert to a dict form 

698 if self.hasFull(): 

699 dataId = self.full.byName() 

700 else: 

701 dataId = self.byName() 

702 records: Optional[Dict[str, SerializedDimensionRecord]] 

703 if not minimal and self.hasRecords(): 

704 records = {k: v.to_simple() for k, v in self.records.byName().items() if v is not None} 

705 else: 

706 records = None 

707 

708 return SerializedDataCoordinate(dataId=dataId, records=records) 

709 

710 @classmethod 

711 def from_simple( 

712 cls, 

713 simple: SerializedDataCoordinate, 

714 universe: Optional[DimensionUniverse] = None, 

715 registry: Optional[Registry] = None, 

716 ) -> DataCoordinate: 

717 """Construct a new object from the simplified form. 

718 

719 The data is assumed to be of the form returned from the `to_simple` 

720 method. 

721 

722 Parameters 

723 ---------- 

724 simple : `dict` of [`str`, `Any`] 

725 The `dict` returned by `to_simple()`. 

726 universe : `DimensionUniverse` 

727 The special graph of all known dimensions. 

728 registry : `lsst.daf.butler.Registry`, optional 

729 Registry from which a universe can be extracted. Can be `None` 

730 if universe is provided explicitly. 

731 

732 Returns 

733 ------- 

734 dataId : `DataCoordinate` 

735 Newly-constructed object. 

736 """ 

737 if universe is None and registry is None: 

738 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

739 if universe is None and registry is not None: 

740 universe = registry.dimensions 

741 if universe is None: 

742 # this is for mypy 

743 raise ValueError("Unable to determine a usable universe") 

744 

745 dataId = cls.standardize(simple.dataId, universe=universe) 

746 if simple.records: 

747 dataId = dataId.expanded( 

748 {k: DimensionRecord.from_simple(v, universe=universe) for k, v in simple.records.items()} 

749 ) 

750 return dataId 

751 

752 to_json = to_json_pydantic 

753 from_json: ClassVar = classmethod(from_json_pydantic) 

754 

755 

756DataId = Union[DataCoordinate, Mapping[str, Any]] 

757"""A type-annotation alias for signatures that accept both informal data ID 

758dictionaries and validated `DataCoordinate` instances. 

759""" 

760 

761 

762class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]): 

763 """View class for `DataCoordinate.full`. 

764 

765 Provides the default implementation for 

766 `DataCoordinate.full`. 

767 

768 Parameters 

769 ---------- 

770 target : `DataCoordinate` 

771 The `DataCoordinate` instance this object provides a view of. 

772 """ 

773 

774 def __init__(self, target: DataCoordinate): 

775 self._target = target 

776 

777 __slots__ = ("_target",) 

778 

779 def __repr__(self) -> str: 

780 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names] 

781 return "{{{}}}".format(", ".join(terms)) 

782 

783 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

784 return self._target[key] 

785 

786 def __iter__(self) -> Iterator[Dimension]: 

787 return iter(self.keys()) 

788 

789 def __len__(self) -> int: 

790 return len(self.keys()) 

791 

792 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

793 return self._target.graph.dimensions 

794 

795 @property 

796 def names(self) -> AbstractSet[str]: 

797 # Docstring inherited from `NamedKeyMapping`. 

798 return self.keys().names 

799 

800 

801class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]): 

802 """View class for `DataCoordinate.records`. 

803 

804 Provides the default implementation for 

805 `DataCoordinate.records`. 

806 

807 Parameters 

808 ---------- 

809 target : `DataCoordinate` 

810 The `DataCoordinate` instance this object provides a view of. 

811 """ 

812 

813 def __init__(self, target: DataCoordinate): 

814 self._target = target 

815 

816 __slots__ = ("_target",) 

817 

818 def __repr__(self) -> str: 

819 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names] 

820 return "{{{}}}".format(", ".join(terms)) 

821 

822 def __str__(self) -> str: 

823 return "\n".join(str(v) for v in self.values()) 

824 

825 def __getitem__(self, key: Union[DimensionElement, str]) -> Optional[DimensionRecord]: 

826 if isinstance(key, DimensionElement): 

827 key = key.name 

828 return self._target._record(key) 

829 

830 def __iter__(self) -> Iterator[DimensionElement]: 

831 return iter(self.keys()) 

832 

833 def __len__(self) -> int: 

834 return len(self.keys()) 

835 

836 def keys(self) -> NamedValueAbstractSet[DimensionElement]: # type: ignore 

837 return self._target.graph.elements 

838 

839 @property 

840 def names(self) -> AbstractSet[str]: 

841 # Docstring inherited from `NamedKeyMapping`. 

842 return self.keys().names 

843 

844 

845class _BasicTupleDataCoordinate(DataCoordinate): 

846 """Standard implementation of `DataCoordinate`. 

847 

848 Backed by a tuple of values. 

849 

850 This class should only be accessed outside this module via the 

851 `DataCoordinate` interface, and should only be constructed via the static 

852 methods there. 

853 

854 Parameters 

855 ---------- 

856 graph : `DimensionGraph` 

857 The dimensions to be identified. 

858 values : `tuple` [ `int` or `str` ] 

859 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May 

860 include values for just required dimensions (which always come first) 

861 or all dimensions. 

862 """ 

863 

864 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...]): 

865 self._graph = graph 

866 self._values = values 

867 

868 __slots__ = ("_graph", "_values") 

869 

870 @property 

871 def graph(self) -> DimensionGraph: 

872 # Docstring inherited from DataCoordinate. 

873 return self._graph 

874 

875 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

876 # Docstring inherited from DataCoordinate. 

877 if isinstance(key, Dimension): 

878 key = key.name 

879 index = self._graph._dataCoordinateIndices[key] 

880 try: 

881 return self._values[index] 

882 except IndexError: 

883 # Caller asked for an implied dimension, but this object only has 

884 # values for the required ones. 

885 raise KeyError(key) from None 

886 

887 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

888 # Docstring inherited from DataCoordinate. 

889 if self._graph == graph: 

890 return self 

891 elif self.hasFull() or self._graph.required >= graph.dimensions: 

892 return _BasicTupleDataCoordinate( 

893 graph, 

894 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

895 ) 

896 else: 

897 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names)) 

898 

899 def union(self, other: DataCoordinate) -> DataCoordinate: 

900 # Docstring inherited from DataCoordinate. 

901 graph = self.graph.union(other.graph) 

902 # See if one or both input data IDs is already what we want to return; 

903 # if so, return the most complete one we have. 

904 if other.graph == graph: 

905 if self.graph == graph: 

906 # Input data IDs have the same graph (which is also the result 

907 # graph), but may not have the same content. 

908 # other might have records; self does not, so try other first. 

909 # If it at least has full values, it's no worse than self. 

910 if other.hasFull(): 

911 return other 

912 else: 

913 return self 

914 elif other.hasFull(): 

915 return other 

916 # There's some chance that neither self nor other has full values, 

917 # but together provide enough to the union to. Let the general 

918 # case below handle that. 

919 elif self.graph == graph: 

920 # No chance at returning records. If self has full values, it's 

921 # the best we can do. 

922 if self.hasFull(): 

923 return self 

924 # General case with actual merging of dictionaries. 

925 values = self.full.byName() if self.hasFull() else self.byName() 

926 values.update(other.full.byName() if other.hasFull() else other.byName()) 

927 return DataCoordinate.standardize(values, graph=graph) 

928 

929 def expanded( 

930 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

931 ) -> DataCoordinate: 

932 # Docstring inherited from DataCoordinate 

933 values = self._values 

934 if not self.hasFull(): 

935 # Extract a complete values tuple from the attributes of the given 

936 # records. It's possible for these to be inconsistent with 

937 # self._values (which is a serious problem, of course), but we've 

938 # documented this as a no-checking API. 

939 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied) 

940 return _ExpandedTupleDataCoordinate(self._graph, values, records) 

941 

942 def hasFull(self) -> bool: 

943 # Docstring inherited from DataCoordinate. 

944 return len(self._values) == len(self._graph._dataCoordinateIndices) 

945 

946 def hasRecords(self) -> bool: 

947 # Docstring inherited from DataCoordinate. 

948 return False 

949 

950 def _record(self, name: str) -> Optional[DimensionRecord]: 

951 # Docstring inherited from DataCoordinate. 

952 assert False 

953 

954 def __reduce__(self) -> tuple[Any, ...]: 

955 return (_BasicTupleDataCoordinate, (self._graph, self._values)) 

956 

957 def __getattr__(self, name: str) -> Any: 

958 if name in self.graph.elements.names: 

959 raise AttributeError( 

960 f"Dimension record attribute {name!r} is only available on expanded DataCoordinates." 

961 ) 

962 raise AttributeError(name) 

963 

964 

965class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate): 

966 """A `DataCoordinate` implementation that can hold `DimensionRecord`. 

967 

968 This class should only be accessed outside this module via the 

969 `DataCoordinate` interface, and should only be constructed via calls to 

970 `DataCoordinate.expanded`. 

971 

972 Parameters 

973 ---------- 

974 graph : `DimensionGraph` 

975 The dimensions to be identified. 

976 values : `tuple` [ `int` or `str` ] 

977 Data ID values, ordered to match ``graph._dataCoordinateIndices``. 

978 May include values for just required dimensions (which always come 

979 first) or all dimensions. 

980 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

981 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

982 `Mapping` with `str` (`DimensionElement` name) keys and 

983 `DimensionRecord` values. Keys must cover all elements in 

984 ``self.graph.elements``. Values may be `None`, but only to reflect 

985 actual NULL values in the database, not just records that have not 

986 been fetched. 

987 """ 

988 

989 def __init__( 

990 self, 

991 graph: DimensionGraph, 

992 values: Tuple[DataIdValue, ...], 

993 records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]], 

994 ): 

995 super().__init__(graph, values) 

996 assert super().hasFull(), "This implementation requires full dimension records." 

997 self._records = records 

998 

999 __slots__ = ("_records",) 

1000 

1001 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

1002 # Docstring inherited from DataCoordinate. 

1003 if self._graph == graph: 

1004 return self 

1005 return _ExpandedTupleDataCoordinate( 

1006 graph, tuple(self[k] for k in graph._dataCoordinateIndices.keys()), records=self._records 

1007 ) 

1008 

1009 def expanded( 

1010 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

1011 ) -> DataCoordinate: 

1012 # Docstring inherited from DataCoordinate. 

1013 return self 

1014 

1015 def union(self, other: DataCoordinate) -> DataCoordinate: 

1016 # Docstring inherited from DataCoordinate. 

1017 graph = self.graph.union(other.graph) 

1018 # See if one or both input data IDs is already what we want to return; 

1019 # if so, return the most complete one we have. 

1020 if self.graph == graph: 

1021 # self has records, so even if other is also a valid result, it's 

1022 # no better. 

1023 return self 

1024 if other.graph == graph: 

1025 # If other has full values, and self does not identify some of 

1026 # those, it's the base we can do. It may have records, too. 

1027 if other.hasFull(): 

1028 return other 

1029 # If other does not have full values, there's a chance self may 

1030 # provide the values needed to complete it. For example, self 

1031 # could be {band} while other could be 

1032 # {instrument, physical_filter, band}, with band unknown. 

1033 # General case with actual merging of dictionaries. 

1034 values = self.full.byName() 

1035 values.update(other.full.byName() if other.hasFull() else other.byName()) 

1036 basic = DataCoordinate.standardize(values, graph=graph) 

1037 # See if we can add records. 

1038 if self.hasRecords() and other.hasRecords(): 

1039 # Sometimes the elements of a union of graphs can contain elements 

1040 # that weren't in either input graph (because graph unions are only 

1041 # on dimensions). e.g. {visit} | {detector} brings along 

1042 # visit_detector_region. 

1043 elements = set(graph.elements.names) 

1044 elements -= self.graph.elements.names 

1045 elements -= other.graph.elements.names 

1046 if not elements: 

1047 records = NamedKeyDict[DimensionElement, Optional[DimensionRecord]](self.records) 

1048 records.update(other.records) 

1049 return basic.expanded(records.freeze()) 

1050 return basic 

1051 

1052 def hasFull(self) -> bool: 

1053 # Docstring inherited from DataCoordinate. 

1054 return True 

1055 

1056 def hasRecords(self) -> bool: 

1057 # Docstring inherited from DataCoordinate. 

1058 return True 

1059 

1060 def _record(self, name: str) -> Optional[DimensionRecord]: 

1061 # Docstring inherited from DataCoordinate. 

1062 return self._records[name] 

1063 

1064 def __reduce__(self) -> tuple[Any, ...]: 

1065 return (_ExpandedTupleDataCoordinate, (self._graph, self._values, self._records)) 

1066 

1067 def __getattr__(self, name: str) -> Any: 

1068 try: 

1069 return self._record(name) 

1070 except KeyError: 

1071 raise AttributeError(name) from None 

1072 

1073 def __dir__(self) -> list[str]: 

1074 result = list(super().__dir__()) 

1075 result.extend(self.graph.elements.names) 

1076 return result