Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22# 

23# Design notes for this module are in 

24# doc/lsst.daf.butler/dev/dataCoordinate.py. 

25# 

26 

27from __future__ import annotations 

28 

29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue") 

30 

31from abc import abstractmethod 

32import numbers 

33from typing import ( 

34 AbstractSet, 

35 Any, 

36 Dict, 

37 Iterator, 

38 Mapping, 

39 Optional, 

40 Tuple, 

41 TYPE_CHECKING, 

42 Union, 

43) 

44 

45from lsst.sphgeom import Region 

46from ..named import NamedKeyDict, NamedKeyMapping, NameLookupMapping, NamedValueAbstractSet 

47from ..timespan import Timespan 

48from ._elements import Dimension, DimensionElement 

49from ._graph import DimensionGraph 

50from ._records import DimensionRecord 

51from ..json import from_json_generic, to_json_generic 

52 

53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true

54 from ._universe import DimensionUniverse 

55 from ...registry import Registry 

56 

57DataIdKey = Union[str, Dimension] 

58"""Type annotation alias for the keys that can be used to index a 

59DataCoordinate. 

60""" 

61 

62DataIdValue = Union[str, int, None] 

63"""Type annotation alias for the values that can be present in a 

64DataCoordinate or other data ID. 

65""" 

66 

67 

68def _intersectRegions(*args: Region) -> Optional[Region]: 

69 """Return the intersection of several regions. 

70 

71 For internal use by `ExpandedDataCoordinate` only. 

72 

73 If no regions are provided, returns `None`. 

74 

75 This is currently a placeholder; it actually returns `NotImplemented` 

76 (it does *not* raise an exception) when multiple regions are given, which 

77 propagates to `ExpandedDataCoordinate`. This reflects the fact that we 

78 don't want to fail to construct an `ExpandedDataCoordinate` entirely when 

79 we can't compute its region, and at present we don't have a high-level use 

80 case for the regions of these particular data IDs. 

81 """ 

82 if len(args) == 0: 

83 return None 

84 elif len(args) == 1: 

85 return args[0] 

86 else: 

87 return NotImplemented 

88 

89 

90class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]): 

91 """An immutable data ID dictionary that guarantees that its key-value pairs 

92 identify at least all required dimensions in a `DimensionGraph`. 

93 

94 `DataCoordinateSet` itself is an ABC, but provides `staticmethod` factory 

95 functions for private concrete implementations that should be sufficient 

96 for most purposes. `standardize` is the most flexible and safe of these; 

97 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are 

98 more specialized and perform little or no checking of inputs. 

99 

100 Notes 

101 ----- 

102 Like any data ID class, `DataCoordinate` behaves like a dictionary, but 

103 with some subtleties: 

104 

105 - Both `Dimension` instances and `str` names thereof may be used as keys 

106 in lookup operations, but iteration (and `keys`) will yield `Dimension` 

107 instances. The `names` property can be used to obtain the corresponding 

108 `str` names. 

109 

110 - Lookups for implied dimensions (those in ``self.graph.implied``) are 

111 supported if and only if `hasFull` returns `True`, and are never 

112 included in iteration or `keys`. The `full` property may be used to 

113 obtain a mapping whose keys do include implied dimensions. 

114 

115 - Equality comparison with other mappings is supported, but it always 

116 considers only required dimensions (as well as requiring both operands 

117 to identify the same dimensions). This is not quite consistent with the 

118 way mappings usually work - normally differing keys imply unequal 

119 mappings - but it makes sense in this context because data IDs with the 

120 same values for required dimensions but different values for implied 

121 dimensions represent a serious problem with the data that 

122 `DataCoordinate` cannot generally recognize on its own, and a data ID 

123 that knows implied dimension values should still be able to compare as 

124 equal to one that does not. This is of course not the way comparisons 

125 between simple `dict` data IDs work, and hence using a `DataCoordinate` 

126 instance for at least one operand in any data ID comparison is strongly 

127 recommended. 

128 """ 

129 

130 __slots__ = () 

131 

132 @staticmethod 

133 def standardize( 

134 mapping: Optional[NameLookupMapping[Dimension, DataIdValue]] = None, 

135 *, 

136 graph: Optional[DimensionGraph] = None, 

137 universe: Optional[DimensionUniverse] = None, 

138 defaults: Optional[DataCoordinate] = None, 

139 **kwargs: Any 

140 ) -> DataCoordinate: 

141 """Adapt an arbitrary mapping and/or additional arguments into a true 

142 `DataCoordinate`, or augment an existing one. 

143 

144 Parameters 

145 ---------- 

146 mapping : `~collections.abc.Mapping`, optional 

147 An informal data ID that maps dimensions or dimension names to 

148 their primary key values (may also be a true `DataCoordinate`). 

149 graph : `DimensionGraph` 

150 The dimensions to be identified by the new `DataCoordinate`. 

151 If not provided, will be inferred from the keys of ``mapping`` and 

152 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

153 is already a `DataCoordinate`. 

154 universe : `DimensionUniverse` 

155 All known dimensions and their relationships; used to expand 

156 and validate dependencies when ``graph`` is not provided. 

157 defaults : `DataCoordinate`, optional 

158 Default dimension key-value pairs to use when needed. These are 

159 never used to infer ``graph``, and are ignored if a different value 

160 is provided for the same key in ``mapping`` or `**kwargs``. 

161 **kwargs 

162 Additional keyword arguments are treated like additional key-value 

163 pairs in ``mapping``. 

164 

165 Returns 

166 ------- 

167 coordinate : `DataCoordinate` 

168 A validated `DataCoordinate` instance. 

169 

170 Raises 

171 ------ 

172 TypeError 

173 Raised if the set of optional arguments provided is not supported. 

174 KeyError 

175 Raised if a key-value pair for a required dimension is missing. 

176 """ 

177 d: Dict[str, DataIdValue] = {} 

178 if isinstance(mapping, DataCoordinate): 

179 if graph is None: 

180 if not kwargs: 

181 # Already standardized to exactly what we want. 

182 return mapping 

183 elif kwargs.keys().isdisjoint(graph.dimensions.names): 

184 # User provided kwargs, but told us not to use them by 

185 # passing in dimensions that are disjoint from those kwargs. 

186 # This is not necessarily user error - it's a useful pattern 

187 # to pass in all of the key-value pairs you have and let the 

188 # code here pull out only what it needs. 

189 return mapping.subset(graph) 

190 assert universe is None or universe == mapping.universe 

191 universe = mapping.universe 

192 d.update((name, mapping[name]) for name in mapping.graph.required.names) 

193 if mapping.hasFull(): 

194 d.update((name, mapping[name]) for name in mapping.graph.implied.names) 

195 elif isinstance(mapping, NamedKeyMapping): 

196 d.update(mapping.byName()) 

197 elif mapping is not None: 

198 d.update(mapping) 

199 d.update(kwargs) 

200 if graph is None: 

201 if defaults is not None: 

202 universe = defaults.universe 

203 elif universe is None: 

204 raise TypeError("universe must be provided if graph is not.") 

205 graph = DimensionGraph(universe, names=d.keys()) 

206 if not graph.dimensions: 

207 return DataCoordinate.makeEmpty(graph.universe) 

208 if defaults is not None: 

209 if defaults.hasFull(): 

210 for k, v in defaults.full.items(): 

211 d.setdefault(k.name, v) 

212 else: 

213 for k, v in defaults.items(): 

214 d.setdefault(k.name, v) 

215 if d.keys() >= graph.dimensions.names: 

216 values = tuple(d[name] for name in graph._dataCoordinateIndices.keys()) 

217 else: 

218 try: 

219 values = tuple(d[name] for name in graph.required.names) 

220 except KeyError as err: 

221 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err 

222 # Some backends cannot handle numpy.int64 type which is a subclass of 

223 # numbers.Integral; convert that to int. 

224 values = tuple(int(val) if isinstance(val, numbers.Integral) # type: ignore 

225 else val for val in values) 

226 return _BasicTupleDataCoordinate(graph, values) 

227 

228 @staticmethod 

229 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate: 

230 """Return an empty `DataCoordinate` that identifies the null set of 

231 dimensions. 

232 

233 Parameters 

234 ---------- 

235 universe : `DimensionUniverse` 

236 Universe to which this null dimension set belongs. 

237 

238 Returns 

239 ------- 

240 dataId : `DataCoordinate` 

241 A data ID object that identifies no dimensions. `hasFull` and 

242 `hasRecords` are guaranteed to return `True`, because both `full` 

243 and `records` are just empty mappings. 

244 """ 

245 return _ExpandedTupleDataCoordinate(universe.empty, (), {}) 

246 

247 @staticmethod 

248 def fromRequiredValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

249 """Construct a `DataCoordinate` from a tuple of dimension values that 

250 identify only required dimensions. 

251 

252 This is a low-level interface with at most assertion-level checking of 

253 inputs. Most callers should use `standardize` instead. 

254 

255 Parameters 

256 ---------- 

257 graph : `DimensionGraph` 

258 Dimensions this data ID will identify. 

259 values : `tuple` [ `int` or `str` ] 

260 Tuple of primary key values corresponding to ``graph.required``, 

261 in that order. 

262 

263 Returns 

264 ------- 

265 dataId : `DataCoordinate` 

266 A data ID object that identifies the given dimensions. 

267 ``dataId.hasFull()`` will return `True` if and only if 

268 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

269 return `True`. 

270 """ 

271 assert len(graph.required) == len(values), \ 

272 f"Inconsistency between dimensions {graph.required} and required values {values}." 

273 return _BasicTupleDataCoordinate(graph, values) 

274 

275 @staticmethod 

276 def fromFullValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

277 """Construct a `DataCoordinate` from a tuple of dimension values that 

278 identify all dimensions. 

279 

280 This is a low-level interface with at most assertion-level checking of 

281 inputs. Most callers should use `standardize` instead. 

282 

283 Parameters 

284 ---------- 

285 graph : `DimensionGraph` 

286 Dimensions this data ID will identify. 

287 values : `tuple` [ `int` or `str` ] 

288 Tuple of primary key values corresponding to 

289 ``itertools.chain(graph.required, graph.implied)``, in that order. 

290 Note that this is _not_ the same order as ``graph.dimensions``, 

291 though these contain the same elements. 

292 

293 Returns 

294 ------- 

295 dataId : `DataCoordinate` 

296 A data ID object that identifies the given dimensions. 

297 ``dataId.hasFull()`` will return `True` if and only if 

298 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

299 return `True`. 

300 """ 

301 assert len(graph.dimensions) == len(values), \ 

302 f"Inconsistency between dimensions {graph.dimensions} and full values {values}." 

303 return _BasicTupleDataCoordinate(graph, values) 

304 

305 def __hash__(self) -> int: 

306 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required)) 

307 

308 def __eq__(self, other: Any) -> bool: 

309 if not isinstance(other, DataCoordinate): 

310 other = DataCoordinate.standardize(other, universe=self.universe) 

311 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required) 

312 

313 def __repr__(self) -> str: 

314 # We can't make repr yield something that could be exec'd here without 

315 # printing out the whole DimensionUniverse the graph is derived from. 

316 # So we print something that mostly looks like a dict, but doesn't 

317 # quote its keys: that's both more compact and something that can't 

318 # be mistaken for an actual dict or something that could be exec'd. 

319 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names] 

320 if self.hasFull() and self.graph.required != self.graph.dimensions: 

321 terms.append("...") 

322 return "{{{}}}".format(', '.join(terms)) 

323 

324 def __lt__(self, other: Any) -> bool: 

325 # Allow DataCoordinate to be sorted 

326 if not isinstance(other, type(self)): 

327 return NotImplemented 

328 # Form tuple of tuples for each DataCoordinate: 

329 # Unlike repr() we only use required keys here to ensure that 

330 # __eq__ can not be true simultaneously with __lt__ being true. 

331 self_kv = tuple(self.items()) 

332 other_kv = tuple(other.items()) 

333 

334 return self_kv < other_kv 

335 

336 def __iter__(self) -> Iterator[Dimension]: 

337 return iter(self.keys()) 

338 

339 def __len__(self) -> int: 

340 return len(self.keys()) 

341 

342 def keys(self) -> NamedValueAbstractSet[Dimension]: 

343 return self.graph.required 

344 

345 @property 

346 def names(self) -> AbstractSet[str]: 

347 """The names of the required dimensions identified by this data ID, in 

348 the same order as `keys` (`collections.abc.Set` [ `str` ]). 

349 """ 

350 return self.keys().names 

351 

352 @abstractmethod 

353 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

354 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``. 

355 

356 Parameters 

357 ---------- 

358 graph : `DimensionGraph` 

359 The dimensions identified by the returned `DataCoordinate`. 

360 

361 Returns 

362 ------- 

363 coordinate : `DataCoordinate` 

364 A `DataCoordinate` instance that identifies only the given 

365 dimensions. May be ``self`` if ``graph == self.graph``. 

366 

367 Raises 

368 ------ 

369 KeyError 

370 Raised if the primary key value for one or more required dimensions 

371 is unknown. This may happen if ``graph.issubset(self.graph)`` is 

372 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if 

373 ``self.hasFull()`` is `False` and 

374 ``graph.required.issubset(self.graph.required)`` is `False`. As 

375 an example of the latter case, consider trying to go from a data ID 

376 with dimensions {instrument, physical_filter, band} to 

377 just {instrument, band}; band is implied by 

378 physical_filter and hence would have no value in the original data 

379 ID if ``self.hasFull()`` is `False`. 

380 

381 Notes 

382 ----- 

383 If `hasFull` and `hasRecords` return `True` on ``self``, they will 

384 return `True` (respectively) on the returned `DataCoordinate` as well. 

385 The converse does not hold. 

386 """ 

387 raise NotImplementedError() 

388 

389 @abstractmethod 

390 def union(self, other: DataCoordinate) -> DataCoordinate: 

391 """Combine two data IDs, yielding a new one that identifies all 

392 dimensions that either of them identify. 

393 

394 Parameters 

395 ---------- 

396 other : `DataCoordinate` 

397 Data ID to combine with ``self``. 

398 

399 Returns 

400 ------- 

401 unioned : `DataCoordinate` 

402 A `DataCoordinate` instance that satisfies 

403 ``unioned.graph == self.graph.union(other.graph)``. Will preserve 

404 ``hasFull`` and ``hasRecords`` whenever possible. 

405 

406 Notes 

407 ----- 

408 No checking for consistency is performed on values for keys that 

409 ``self`` and ``other`` have in common, and which value is included in 

410 the returned data ID is not specified. 

411 """ 

412 raise NotImplementedError() 

413 

414 @abstractmethod 

415 def expanded(self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

416 ) -> DataCoordinate: 

417 """Return a `DataCoordinate` that holds the given records and 

418 guarantees that `hasRecords` returns `True`. 

419 

420 This is a low-level interface with at most assertion-level checking of 

421 inputs. Most callers should use `Registry.expandDataId` instead. 

422 

423 Parameters 

424 ---------- 

425 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

426 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

427 `Mapping` with `str` (`DimensionElement` name) keys and 

428 `DimensionRecord` values. Keys must cover all elements in 

429 ``self.graph.elements``. Values may be `None`, but only to reflect 

430 actual NULL values in the database, not just records that have not 

431 been fetched. 

432 """ 

433 raise NotImplementedError() 

434 

435 @property 

436 def universe(self) -> DimensionUniverse: 

437 """The universe that defines all known dimensions compatible with 

438 this coordinate (`DimensionUniverse`). 

439 """ 

440 return self.graph.universe 

441 

442 @property 

443 @abstractmethod 

444 def graph(self) -> DimensionGraph: 

445 """The dimensions identified by this data ID (`DimensionGraph`). 

446 

447 Note that values are only required to be present for dimensions in 

448 ``self.graph.required``; all others may be retrieved (from a 

449 `Registry`) given these. 

450 """ 

451 raise NotImplementedError() 

452 

453 @abstractmethod 

454 def hasFull(self) -> bool: 

455 """Whether this data ID contains values for implied as well as 

456 required dimensions. 

457 

458 Returns 

459 ------- 

460 state : `bool` 

461 If `True`, `__getitem__`, `get`, and `__contains__` (but not 

462 `keys`!) will act as though the mapping includes key-value pairs 

463 for implied dimensions, and the `full` property may be used. If 

464 `False`, these operations only include key-value pairs for required 

465 dimensions, and accessing `full` is an error. Always `True` if 

466 there are no implied dimensions. 

467 """ 

468 raise NotImplementedError() 

469 

470 @property 

471 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]: 

472 """A mapping that includes key-value pairs for all dimensions in 

473 ``self.graph``, including implied (`NamedKeyMapping`). 

474 

475 Accessing this attribute if `hasFull` returns `False` is a logic error 

476 that may raise an exception of unspecified type either immediately or 

477 when implied keys are accessed via the returned mapping, depending on 

478 the implementation and whether assertions are enabled. 

479 """ 

480 assert self.hasFull(), "full may only be accessed if hasRecords() returns True." 

481 return _DataCoordinateFullView(self) 

482 

483 @abstractmethod 

484 def hasRecords(self) -> bool: 

485 """Whether this data ID contains records for all of the dimension 

486 elements it identifies. 

487 

488 Returns 

489 ------- 

490 state : `bool` 

491 If `True`, the following attributes may be accessed: 

492 

493 - `records` 

494 - `region` 

495 - `timespan` 

496 - `pack` 

497 

498 If `False`, accessing any of these is considered a logic error. 

499 """ 

500 raise NotImplementedError() 

501 

502 @property 

503 def records(self) -> NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]: 

504 """A mapping that contains `DimensionRecord` objects for all elements 

505 identified by this data ID (`NamedKeyMapping`). 

506 

507 The values of this mapping may be `None` if and only if there is no 

508 record for that element with these dimensions in the database (which 

509 means some foreign key field must have a NULL value). 

510 

511 Accessing this attribute if `hasRecords` returns `False` is a logic 

512 error that may raise an exception of unspecified type either 

513 immediately or when the returned mapping is used, depending on the 

514 implementation and whether assertions are enabled. 

515 """ 

516 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True." 

517 return _DataCoordinateRecordsView(self) 

518 

519 @abstractmethod 

520 def _record(self, name: str) -> Optional[DimensionRecord]: 

521 """Protected implementation hook that backs the ``records`` attribute. 

522 

523 Parameters 

524 ---------- 

525 name : `str` 

526 The name of a `DimensionElement`, guaranteed to be in 

527 ``self.graph.elements.names``. 

528 

529 Returns 

530 ------- 

531 record : `DimensionRecord` or `None` 

532 The dimension record for the given element identified by this 

533 data ID, or `None` if there is no such record. 

534 """ 

535 raise NotImplementedError() 

536 

537 @property 

538 def region(self) -> Optional[Region]: 

539 """The spatial region associated with this data ID 

540 (`lsst.sphgeom.Region` or `None`). 

541 

542 This is `None` if and only if ``self.graph.spatial`` is empty. 

543 

544 Accessing this attribute if `hasRecords` returns `False` is a logic 

545 error that may or may not raise an exception, depending on the 

546 implementation and whether assertions are enabled. 

547 """ 

548 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True." 

549 regions = [] 

550 for family in self.graph.spatial: 

551 element = family.choose(self.graph.elements) 

552 record = self._record(element.name) 

553 if record is None or record.region is None: 

554 return None 

555 else: 

556 regions.append(record.region) 

557 return _intersectRegions(*regions) 

558 

559 @property 

560 def timespan(self) -> Optional[Timespan]: 

561 """The temporal interval associated with this data ID 

562 (`Timespan` or `None`). 

563 

564 This is `None` if and only if ``self.graph.timespan`` is empty. 

565 

566 Accessing this attribute if `hasRecords` returns `False` is a logic 

567 error that may or may not raise an exception, depending on the 

568 implementation and whether assertions are enabled. 

569 """ 

570 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True." 

571 timespans = [] 

572 for family in self.graph.temporal: 

573 element = family.choose(self.graph.elements) 

574 record = self._record(element.name) 

575 # DimensionRecord subclasses for temporal elements always have 

576 # .timespan, but they're dynamic so this can't be type-checked. 

577 if record is None or record.timespan is None: 

578 return None 

579 else: 

580 timespans.append(record.timespan) 

581 return Timespan.intersection(*timespans) 

582 

583 def pack(self, name: str, *, returnMaxBits: bool = False) -> Union[Tuple[int, int], int]: 

584 """Pack this data ID into an integer. 

585 

586 Parameters 

587 ---------- 

588 name : `str` 

589 Name of the `DimensionPacker` algorithm (as defined in the 

590 dimension configuration). 

591 returnMaxBits : `bool`, optional 

592 If `True` (`False` is default), return the maximum number of 

593 nonzero bits in the returned integer across all data IDs. 

594 

595 Returns 

596 ------- 

597 packed : `int` 

598 Integer ID. This ID is unique only across data IDs that have 

599 the same values for the packer's "fixed" dimensions. 

600 maxBits : `int`, optional 

601 Maximum number of nonzero bits in ``packed``. Not returned unless 

602 ``returnMaxBits`` is `True`. 

603 

604 Notes 

605 ----- 

606 Accessing this attribute if `hasRecords` returns `False` is a logic 

607 error that may or may not raise an exception, depending on the 

608 implementation and whether assertions are enabled. 

609 """ 

610 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." 

611 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) 

612 

613 def to_simple(self, minimal: bool = False) -> Dict: 

614 """Convert this class to a simple python type suitable for 

615 serialization. 

616 

617 Parameters 

618 ---------- 

619 minimal : `bool`, optional 

620 Use minimal serialization. Has no effect on for this class. 

621 

622 Returns 

623 ------- 

624 as_dict : `dict` 

625 The object converted to a dictionary. 

626 """ 

627 # Convert to a dict form 

628 return self.byName() 

629 

630 @classmethod 

631 def from_simple(cls, simple: Dict[str, Any], 

632 universe: Optional[DimensionUniverse] = None, 

633 registry: Optional[Registry] = None) -> DataCoordinate: 

634 """Construct a new object from the data returned from the `to_simple` 

635 method. 

636 

637 Parameters 

638 ---------- 

639 simple : `dict` of [`str`, `Any`] 

640 The `dict` returned by `to_simple()`. 

641 universe : `DimensionUniverse` 

642 The special graph of all known dimensions. 

643 registry : `lsst.daf.butler.Registry`, optional 

644 Registry from which a universe can be extracted. Can be `None` 

645 if universe is provided explicitly. 

646 

647 Returns 

648 ------- 

649 dataId : `DataCoordinate` 

650 Newly-constructed object. 

651 """ 

652 if universe is None and registry is None: 

653 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

654 if universe is None and registry is not None: 

655 universe = registry.dimensions 

656 if universe is None: 

657 # this is for mypy 

658 raise ValueError("Unable to determine a usable universe") 

659 

660 return cls.standardize(simple, universe=universe) 

661 

662 to_json = to_json_generic 

663 from_json = classmethod(from_json_generic) 

664 

665 

666DataId = Union[DataCoordinate, Mapping[str, Any]] 

667"""A type-annotation alias for signatures that accept both informal data ID 

668dictionaries and validated `DataCoordinate` instances. 

669""" 

670 

671 

672class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]): 

673 """View class that provides the default implementation for 

674 `DataCoordinate.full`. 

675 

676 Parameters 

677 ---------- 

678 target : `DataCoordinate` 

679 The `DataCoordinate` instance this object provides a view of. 

680 """ 

681 def __init__(self, target: DataCoordinate): 

682 self._target = target 

683 

684 __slots__ = ("_target",) 

685 

686 def __repr__(self) -> str: 

687 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names] 

688 return "{{{}}}".format(', '.join(terms)) 

689 

690 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

691 return self._target[key] 

692 

693 def __iter__(self) -> Iterator[Dimension]: 

694 return iter(self.keys()) 

695 

696 def __len__(self) -> int: 

697 return len(self.keys()) 

698 

699 def keys(self) -> NamedValueAbstractSet[Dimension]: 

700 return self._target.graph.dimensions 

701 

702 @property 

703 def names(self) -> AbstractSet[str]: 

704 # Docstring inherited from `NamedKeyMapping`. 

705 return self.keys().names 

706 

707 

708class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]): 

709 """View class that provides the default implementation for 

710 `DataCoordinate.records`. 

711 

712 Parameters 

713 ---------- 

714 target : `DataCoordinate` 

715 The `DataCoordinate` instance this object provides a view of. 

716 """ 

717 def __init__(self, target: DataCoordinate): 

718 self._target = target 

719 

720 __slots__ = ("_target",) 

721 

722 def __repr__(self) -> str: 

723 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names] 

724 return "{{{}}}".format(', '.join(terms)) 

725 

726 def __str__(self) -> str: 

727 return "\n".join(str(v) for v in self.values()) 

728 

729 def __getitem__(self, key: Union[DimensionElement, str]) -> Optional[DimensionRecord]: 

730 if isinstance(key, DimensionElement): 

731 key = key.name 

732 return self._target._record(key) 

733 

734 def __iter__(self) -> Iterator[DimensionElement]: 

735 return iter(self.keys()) 

736 

737 def __len__(self) -> int: 

738 return len(self.keys()) 

739 

740 def keys(self) -> NamedValueAbstractSet[DimensionElement]: 

741 return self._target.graph.elements 

742 

743 @property 

744 def names(self) -> AbstractSet[str]: 

745 # Docstring inherited from `NamedKeyMapping`. 

746 return self.keys().names 

747 

748 

749class _BasicTupleDataCoordinate(DataCoordinate): 

750 """Standard implementation of `DataCoordinate`, backed by a tuple of 

751 values. 

752 

753 This class should only be accessed outside this module via the 

754 `DataCoordinate` interface, and should only be constructed via the static 

755 methods there. 

756 

757 Parameters 

758 ---------- 

759 graph : `DimensionGraph` 

760 The dimensions to be identified. 

761 values : `tuple` [ `int` or `str` ] 

762 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May 

763 include values for just required dimensions (which always come first) 

764 or all dimensions. 

765 """ 

766 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...]): 

767 self._graph = graph 

768 self._values = values 

769 

770 __slots__ = ("_graph", "_values") 

771 

772 @property 

773 def graph(self) -> DimensionGraph: 

774 # Docstring inherited from DataCoordinate. 

775 return self._graph 

776 

777 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

778 # Docstring inherited from DataCoordinate. 

779 if isinstance(key, Dimension): 

780 key = key.name 

781 index = self._graph._dataCoordinateIndices[key] 

782 try: 

783 return self._values[index] 

784 except IndexError: 

785 # Caller asked for an implied dimension, but this object only has 

786 # values for the required ones. 

787 raise KeyError(key) from None 

788 

789 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

790 # Docstring inherited from DataCoordinate. 

791 if self._graph == graph: 

792 return self 

793 elif self.hasFull() or self._graph.required >= graph.dimensions: 

794 return _BasicTupleDataCoordinate( 

795 graph, 

796 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

797 ) 

798 else: 

799 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names)) 

800 

801 def union(self, other: DataCoordinate) -> DataCoordinate: 

802 # Docstring inherited from DataCoordinate. 

803 graph = self.graph.union(other.graph) 

804 # See if one or both input data IDs is already what we want to return; 

805 # if so, return the most complete one we have. 

806 if other.graph == graph: 

807 if self.graph == graph: 

808 # Input data IDs have the same graph (which is also the result 

809 # graph), but may not have the same content. 

810 # other might have records; self does not, so try other first. 

811 # If it at least has full values, it's no worse than self. 

812 if other.hasFull(): 

813 return other 

814 else: 

815 return self 

816 elif other.hasFull(): 

817 return other 

818 # There's some chance that neither self nor other has full values, 

819 # but together provide enough to the union to. Let the general 

820 # case below handle that. 

821 elif self.graph == graph: 

822 # No chance at returning records. If self has full values, it's 

823 # the best we can do. 

824 if self.hasFull(): 

825 return self 

826 # General case with actual merging of dictionaries. 

827 values = self.full.byName() if self.hasFull() else self.byName() 

828 values.update(other.full.byName() if other.hasFull() else other.byName()) 

829 return DataCoordinate.standardize(values, graph=graph) 

830 

831 def expanded(self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

832 ) -> DataCoordinate: 

833 # Docstring inherited from DataCoordinate 

834 values = self._values 

835 if not self.hasFull(): 

836 # Extract a complete values tuple from the attributes of the given 

837 # records. It's possible for these to be inconsistent with 

838 # self._values (which is a serious problem, of course), but we've 

839 # documented this as a no-checking API. 

840 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied) 

841 return _ExpandedTupleDataCoordinate(self._graph, values, records) 

842 

843 def hasFull(self) -> bool: 

844 # Docstring inherited from DataCoordinate. 

845 return len(self._values) == len(self._graph._dataCoordinateIndices) 

846 

847 def hasRecords(self) -> bool: 

848 # Docstring inherited from DataCoordinate. 

849 return False 

850 

851 def _record(self, name: str) -> Optional[DimensionRecord]: 

852 # Docstring inherited from DataCoordinate. 

853 assert False 

854 

855 

856class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate): 

857 """A `DataCoordinate` implementation that can hold `DimensionRecord` 

858 objects. 

859 

860 This class should only be accessed outside this module via the 

861 `DataCoordinate` interface, and should only be constructed via calls to 

862 `DataCoordinate.expanded`. 

863 

864 Parameters 

865 ---------- 

866 graph : `DimensionGraph` 

867 The dimensions to be identified. 

868 values : `tuple` [ `int` or `str` ] 

869 Data ID values, ordered to match ``graph._dataCoordinateIndices``. 

870 May include values for just required dimensions (which always come 

871 first) or all dimensions. 

872 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

873 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

874 `Mapping` with `str` (`DimensionElement` name) keys and 

875 `DimensionRecord` values. Keys must cover all elements in 

876 ``self.graph.elements``. Values may be `None`, but only to reflect 

877 actual NULL values in the database, not just records that have not 

878 been fetched. 

879 """ 

880 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...], 

881 records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]]): 

882 super().__init__(graph, values) 

883 assert super().hasFull(), "This implementation requires full dimension records." 

884 self._records = records 

885 

886 __slots__ = ("_records",) 

887 

888 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

889 # Docstring inherited from DataCoordinate. 

890 if self._graph == graph: 

891 return self 

892 return _ExpandedTupleDataCoordinate(graph, 

893 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

894 records=self._records) 

895 

896 def expanded(self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

897 ) -> DataCoordinate: 

898 # Docstring inherited from DataCoordinate. 

899 return self 

900 

901 def union(self, other: DataCoordinate) -> DataCoordinate: 

902 # Docstring inherited from DataCoordinate. 

903 graph = self.graph.union(other.graph) 

904 # See if one or both input data IDs is already what we want to return; 

905 # if so, return the most complete one we have. 

906 if self.graph == graph: 

907 # self has records, so even if other is also a valid result, it's 

908 # no better. 

909 return self 

910 if other.graph == graph: 

911 # If other has full values, and self does not identify some of 

912 # those, it's the base we can do. It may have records, too. 

913 if other.hasFull(): 

914 return other 

915 # If other does not have full values, there's a chance self may 

916 # provide the values needed to complete it. For example, self 

917 # could be {band} while other could be 

918 # {instrument, physical_filter, band}, with band unknown. 

919 # General case with actual merging of dictionaries. 

920 values = self.full.byName() 

921 values.update(other.full.byName() if other.hasFull() else other.byName()) 

922 basic = DataCoordinate.standardize(values, graph=graph) 

923 # See if we can add records. 

924 if self.hasRecords() and other.hasRecords(): 

925 # Sometimes the elements of a union of graphs can contain elements 

926 # that weren't in either input graph (because graph unions are only 

927 # on dimensions). e.g. {visit} | {detector} brings along 

928 # visit_detector_region. 

929 elements = set(graph.elements.names) 

930 elements -= self.graph.elements.names 

931 elements -= other.graph.elements.names 

932 if not elements: 

933 records = NamedKeyDict[DimensionElement, Optional[DimensionRecord]](self.records) 

934 records.update(other.records) 

935 return basic.expanded(records.freeze()) 

936 return basic 

937 

938 def hasFull(self) -> bool: 

939 # Docstring inherited from DataCoordinate. 

940 return True 

941 

942 def hasRecords(self) -> bool: 

943 # Docstring inherited from DataCoordinate. 

944 return True 

945 

946 def _record(self, name: str) -> Optional[DimensionRecord]: 

947 # Docstring inherited from DataCoordinate. 

948 return self._records[name]