Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22# 

23# Design notes for this module are in 

24# doc/lsst.daf.butler/dev/dataCoordinate.py. 

25# 

26 

27from __future__ import annotations 

28 

29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue") 

30 

31from abc import abstractmethod 

32import numbers 

33from typing import ( 

34 AbstractSet, 

35 Any, 

36 Dict, 

37 Iterator, 

38 Mapping, 

39 Optional, 

40 Tuple, 

41 TYPE_CHECKING, 

42 Union, 

43) 

44 

45from lsst.sphgeom import Region 

46from ..named import NamedKeyDict, NamedKeyMapping, NameLookupMapping, NamedValueAbstractSet 

47from ..timespan import Timespan 

48from ._elements import Dimension, DimensionElement 

49from ._graph import DimensionGraph 

50from ._records import DimensionRecord 

51from ..json import from_json_generic, to_json_generic 

52 

53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 53 ↛ 54line 53 didn't jump to line 54, because the condition on line 53 was never true

54 from ._universe import DimensionUniverse 

55 from ...registry import Registry 

56 

57DataIdKey = Union[str, Dimension] 

58"""Type annotation alias for the keys that can be used to index a 

59DataCoordinate. 

60""" 

61 

62DataIdValue = Union[str, int, None] 

63"""Type annotation alias for the values that can be present in a 

64DataCoordinate or other data ID. 

65""" 

66 

67 

68def _intersectRegions(*args: Region) -> Optional[Region]: 

69 """Return the intersection of several regions. 

70 

71 For internal use by `ExpandedDataCoordinate` only. 

72 

73 If no regions are provided, returns `None`. 

74 

75 This is currently a placeholder; it actually returns `NotImplemented` 

76 (it does *not* raise an exception) when multiple regions are given, which 

77 propagates to `ExpandedDataCoordinate`. This reflects the fact that we 

78 don't want to fail to construct an `ExpandedDataCoordinate` entirely when 

79 we can't compute its region, and at present we don't have a high-level use 

80 case for the regions of these particular data IDs. 

81 """ 

82 if len(args) == 0: 

83 return None 

84 elif len(args) == 1: 

85 return args[0] 

86 else: 

87 return NotImplemented 

88 

89 

90class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]): 

91 """Data ID dictionary. 

92 

93 An immutable data ID dictionary that guarantees that its key-value pairs 

94 identify at least all required dimensions in a `DimensionGraph`. 

95 

96 `DataCoordinateSet` itself is an ABC, but provides `staticmethod` factory 

97 functions for private concrete implementations that should be sufficient 

98 for most purposes. `standardize` is the most flexible and safe of these; 

99 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are 

100 more specialized and perform little or no checking of inputs. 

101 

102 Notes 

103 ----- 

104 Like any data ID class, `DataCoordinate` behaves like a dictionary, but 

105 with some subtleties: 

106 

107 - Both `Dimension` instances and `str` names thereof may be used as keys 

108 in lookup operations, but iteration (and `keys`) will yield `Dimension` 

109 instances. The `names` property can be used to obtain the corresponding 

110 `str` names. 

111 

112 - Lookups for implied dimensions (those in ``self.graph.implied``) are 

113 supported if and only if `hasFull` returns `True`, and are never 

114 included in iteration or `keys`. The `full` property may be used to 

115 obtain a mapping whose keys do include implied dimensions. 

116 

117 - Equality comparison with other mappings is supported, but it always 

118 considers only required dimensions (as well as requiring both operands 

119 to identify the same dimensions). This is not quite consistent with the 

120 way mappings usually work - normally differing keys imply unequal 

121 mappings - but it makes sense in this context because data IDs with the 

122 same values for required dimensions but different values for implied 

123 dimensions represent a serious problem with the data that 

124 `DataCoordinate` cannot generally recognize on its own, and a data ID 

125 that knows implied dimension values should still be able to compare as 

126 equal to one that does not. This is of course not the way comparisons 

127 between simple `dict` data IDs work, and hence using a `DataCoordinate` 

128 instance for at least one operand in any data ID comparison is strongly 

129 recommended. 

130 """ 

131 

132 __slots__ = () 

133 

134 @staticmethod 

135 def standardize( 

136 mapping: Optional[NameLookupMapping[Dimension, DataIdValue]] = None, 

137 *, 

138 graph: Optional[DimensionGraph] = None, 

139 universe: Optional[DimensionUniverse] = None, 

140 defaults: Optional[DataCoordinate] = None, 

141 **kwargs: Any 

142 ) -> DataCoordinate: 

143 """Standardize the supplied dataId. 

144 

145 Adapts an arbitrary mapping and/or additional arguments into a true 

146 `DataCoordinate`, or augment an existing one. 

147 

148 Parameters 

149 ---------- 

150 mapping : `~collections.abc.Mapping`, optional 

151 An informal data ID that maps dimensions or dimension names to 

152 their primary key values (may also be a true `DataCoordinate`). 

153 graph : `DimensionGraph` 

154 The dimensions to be identified by the new `DataCoordinate`. 

155 If not provided, will be inferred from the keys of ``mapping`` and 

156 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

157 is already a `DataCoordinate`. 

158 universe : `DimensionUniverse` 

159 All known dimensions and their relationships; used to expand 

160 and validate dependencies when ``graph`` is not provided. 

161 defaults : `DataCoordinate`, optional 

162 Default dimension key-value pairs to use when needed. These are 

163 never used to infer ``graph``, and are ignored if a different value 

164 is provided for the same key in ``mapping`` or `**kwargs``. 

165 **kwargs 

166 Additional keyword arguments are treated like additional key-value 

167 pairs in ``mapping``. 

168 

169 Returns 

170 ------- 

171 coordinate : `DataCoordinate` 

172 A validated `DataCoordinate` instance. 

173 

174 Raises 

175 ------ 

176 TypeError 

177 Raised if the set of optional arguments provided is not supported. 

178 KeyError 

179 Raised if a key-value pair for a required dimension is missing. 

180 """ 

181 d: Dict[str, DataIdValue] = {} 

182 if isinstance(mapping, DataCoordinate): 

183 if graph is None: 

184 if not kwargs: 

185 # Already standardized to exactly what we want. 

186 return mapping 

187 elif kwargs.keys().isdisjoint(graph.dimensions.names): 

188 # User provided kwargs, but told us not to use them by 

189 # passing in dimensions that are disjoint from those kwargs. 

190 # This is not necessarily user error - it's a useful pattern 

191 # to pass in all of the key-value pairs you have and let the 

192 # code here pull out only what it needs. 

193 return mapping.subset(graph) 

194 assert universe is None or universe == mapping.universe 

195 universe = mapping.universe 

196 d.update((name, mapping[name]) for name in mapping.graph.required.names) 

197 if mapping.hasFull(): 

198 d.update((name, mapping[name]) for name in mapping.graph.implied.names) 

199 elif isinstance(mapping, NamedKeyMapping): 

200 d.update(mapping.byName()) 

201 elif mapping is not None: 

202 d.update(mapping) 

203 d.update(kwargs) 

204 if graph is None: 

205 if defaults is not None: 

206 universe = defaults.universe 

207 elif universe is None: 

208 raise TypeError("universe must be provided if graph is not.") 

209 graph = DimensionGraph(universe, names=d.keys()) 

210 if not graph.dimensions: 

211 return DataCoordinate.makeEmpty(graph.universe) 

212 if defaults is not None: 

213 if defaults.hasFull(): 

214 for k, v in defaults.full.items(): 

215 d.setdefault(k.name, v) 

216 else: 

217 for k, v in defaults.items(): 

218 d.setdefault(k.name, v) 

219 if d.keys() >= graph.dimensions.names: 

220 values = tuple(d[name] for name in graph._dataCoordinateIndices.keys()) 

221 else: 

222 try: 

223 values = tuple(d[name] for name in graph.required.names) 

224 except KeyError as err: 

225 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err 

226 # Some backends cannot handle numpy.int64 type which is a subclass of 

227 # numbers.Integral; convert that to int. 

228 values = tuple(int(val) if isinstance(val, numbers.Integral) # type: ignore 

229 else val for val in values) 

230 return _BasicTupleDataCoordinate(graph, values) 

231 

232 @staticmethod 

233 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate: 

234 """Return an empty `DataCoordinate`. 

235 

236 It identifies the null set of dimensions. 

237 

238 Parameters 

239 ---------- 

240 universe : `DimensionUniverse` 

241 Universe to which this null dimension set belongs. 

242 

243 Returns 

244 ------- 

245 dataId : `DataCoordinate` 

246 A data ID object that identifies no dimensions. `hasFull` and 

247 `hasRecords` are guaranteed to return `True`, because both `full` 

248 and `records` are just empty mappings. 

249 """ 

250 return _ExpandedTupleDataCoordinate(universe.empty, (), {}) 

251 

252 @staticmethod 

253 def fromRequiredValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

254 """Construct a `DataCoordinate` from required dimension values. 

255 

256 This is a low-level interface with at most assertion-level checking of 

257 inputs. Most callers should use `standardize` instead. 

258 

259 Parameters 

260 ---------- 

261 graph : `DimensionGraph` 

262 Dimensions this data ID will identify. 

263 values : `tuple` [ `int` or `str` ] 

264 Tuple of primary key values corresponding to ``graph.required``, 

265 in that order. 

266 

267 Returns 

268 ------- 

269 dataId : `DataCoordinate` 

270 A data ID object that identifies the given dimensions. 

271 ``dataId.hasFull()`` will return `True` if and only if 

272 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

273 return `True`. 

274 """ 

275 assert len(graph.required) == len(values), \ 

276 f"Inconsistency between dimensions {graph.required} and required values {values}." 

277 return _BasicTupleDataCoordinate(graph, values) 

278 

279 @staticmethod 

280 def fromFullValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

281 """Construct a `DataCoordinate` from all dimension values. 

282 

283 This is a low-level interface with at most assertion-level checking of 

284 inputs. Most callers should use `standardize` instead. 

285 

286 Parameters 

287 ---------- 

288 graph : `DimensionGraph` 

289 Dimensions this data ID will identify. 

290 values : `tuple` [ `int` or `str` ] 

291 Tuple of primary key values corresponding to 

292 ``itertools.chain(graph.required, graph.implied)``, in that order. 

293 Note that this is _not_ the same order as ``graph.dimensions``, 

294 though these contain the same elements. 

295 

296 Returns 

297 ------- 

298 dataId : `DataCoordinate` 

299 A data ID object that identifies the given dimensions. 

300 ``dataId.hasFull()`` will return `True` if and only if 

301 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

302 return `True`. 

303 """ 

304 assert len(graph.dimensions) == len(values), \ 

305 f"Inconsistency between dimensions {graph.dimensions} and full values {values}." 

306 return _BasicTupleDataCoordinate(graph, values) 

307 

308 def __hash__(self) -> int: 

309 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required)) 

310 

311 def __eq__(self, other: Any) -> bool: 

312 if not isinstance(other, DataCoordinate): 

313 other = DataCoordinate.standardize(other, universe=self.universe) 

314 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required) 

315 

316 def __repr__(self) -> str: 

317 # We can't make repr yield something that could be exec'd here without 

318 # printing out the whole DimensionUniverse the graph is derived from. 

319 # So we print something that mostly looks like a dict, but doesn't 

320 # quote its keys: that's both more compact and something that can't 

321 # be mistaken for an actual dict or something that could be exec'd. 

322 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names] 

323 if self.hasFull() and self.graph.required != self.graph.dimensions: 

324 terms.append("...") 

325 return "{{{}}}".format(', '.join(terms)) 

326 

327 def __lt__(self, other: Any) -> bool: 

328 # Allow DataCoordinate to be sorted 

329 if not isinstance(other, type(self)): 

330 return NotImplemented 

331 # Form tuple of tuples for each DataCoordinate: 

332 # Unlike repr() we only use required keys here to ensure that 

333 # __eq__ can not be true simultaneously with __lt__ being true. 

334 self_kv = tuple(self.items()) 

335 other_kv = tuple(other.items()) 

336 

337 return self_kv < other_kv 

338 

339 def __iter__(self) -> Iterator[Dimension]: 

340 return iter(self.keys()) 

341 

342 def __len__(self) -> int: 

343 return len(self.keys()) 

344 

345 def keys(self) -> NamedValueAbstractSet[Dimension]: 

346 return self.graph.required 

347 

348 @property 

349 def names(self) -> AbstractSet[str]: 

350 """Names of the required dimensions identified by this data ID. 

351 

352 They are returned in the same order as `keys` 

353 (`collections.abc.Set` [ `str` ]). 

354 """ 

355 return self.keys().names 

356 

357 @abstractmethod 

358 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

359 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``. 

360 

361 Parameters 

362 ---------- 

363 graph : `DimensionGraph` 

364 The dimensions identified by the returned `DataCoordinate`. 

365 

366 Returns 

367 ------- 

368 coordinate : `DataCoordinate` 

369 A `DataCoordinate` instance that identifies only the given 

370 dimensions. May be ``self`` if ``graph == self.graph``. 

371 

372 Raises 

373 ------ 

374 KeyError 

375 Raised if the primary key value for one or more required dimensions 

376 is unknown. This may happen if ``graph.issubset(self.graph)`` is 

377 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if 

378 ``self.hasFull()`` is `False` and 

379 ``graph.required.issubset(self.graph.required)`` is `False`. As 

380 an example of the latter case, consider trying to go from a data ID 

381 with dimensions {instrument, physical_filter, band} to 

382 just {instrument, band}; band is implied by 

383 physical_filter and hence would have no value in the original data 

384 ID if ``self.hasFull()`` is `False`. 

385 

386 Notes 

387 ----- 

388 If `hasFull` and `hasRecords` return `True` on ``self``, they will 

389 return `True` (respectively) on the returned `DataCoordinate` as well. 

390 The converse does not hold. 

391 """ 

392 raise NotImplementedError() 

393 

394 @abstractmethod 

395 def union(self, other: DataCoordinate) -> DataCoordinate: 

396 """Combine two data IDs. 

397 

398 Yields a new one that identifies all dimensions that either of them 

399 identify. 

400 

401 Parameters 

402 ---------- 

403 other : `DataCoordinate` 

404 Data ID to combine with ``self``. 

405 

406 Returns 

407 ------- 

408 unioned : `DataCoordinate` 

409 A `DataCoordinate` instance that satisfies 

410 ``unioned.graph == self.graph.union(other.graph)``. Will preserve 

411 ``hasFull`` and ``hasRecords`` whenever possible. 

412 

413 Notes 

414 ----- 

415 No checking for consistency is performed on values for keys that 

416 ``self`` and ``other`` have in common, and which value is included in 

417 the returned data ID is not specified. 

418 """ 

419 raise NotImplementedError() 

420 

421 @abstractmethod 

422 def expanded(self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

423 ) -> DataCoordinate: 

424 """Return a `DataCoordinate` that holds the given records. 

425 

426 Guarantees that `hasRecords` returns `True`. 

427 

428 This is a low-level interface with at most assertion-level checking of 

429 inputs. Most callers should use `Registry.expandDataId` instead. 

430 

431 Parameters 

432 ---------- 

433 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

434 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

435 `Mapping` with `str` (`DimensionElement` name) keys and 

436 `DimensionRecord` values. Keys must cover all elements in 

437 ``self.graph.elements``. Values may be `None`, but only to reflect 

438 actual NULL values in the database, not just records that have not 

439 been fetched. 

440 """ 

441 raise NotImplementedError() 

442 

443 @property 

444 def universe(self) -> DimensionUniverse: 

445 """Universe that defines all known compatible dimensions. 

446 

447 The univers will be compatible with this coordinate 

448 (`DimensionUniverse`). 

449 """ 

450 return self.graph.universe 

451 

452 @property 

453 @abstractmethod 

454 def graph(self) -> DimensionGraph: 

455 """Dimensions identified by this data ID (`DimensionGraph`). 

456 

457 Note that values are only required to be present for dimensions in 

458 ``self.graph.required``; all others may be retrieved (from a 

459 `Registry`) given these. 

460 """ 

461 raise NotImplementedError() 

462 

463 @abstractmethod 

464 def hasFull(self) -> bool: 

465 """Whether this data ID contains implied and required values. 

466 

467 Returns 

468 ------- 

469 state : `bool` 

470 If `True`, `__getitem__`, `get`, and `__contains__` (but not 

471 `keys`!) will act as though the mapping includes key-value pairs 

472 for implied dimensions, and the `full` property may be used. If 

473 `False`, these operations only include key-value pairs for required 

474 dimensions, and accessing `full` is an error. Always `True` if 

475 there are no implied dimensions. 

476 """ 

477 raise NotImplementedError() 

478 

479 @property 

480 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]: 

481 """Return mapping for all dimensions in ``self.graph``. 

482 

483 The mapping includes key-value pairs for all dimensions in 

484 ``self.graph``, including implied (`NamedKeyMapping`). 

485 

486 Accessing this attribute if `hasFull` returns `False` is a logic error 

487 that may raise an exception of unspecified type either immediately or 

488 when implied keys are accessed via the returned mapping, depending on 

489 the implementation and whether assertions are enabled. 

490 """ 

491 assert self.hasFull(), "full may only be accessed if hasRecords() returns True." 

492 return _DataCoordinateFullView(self) 

493 

494 @abstractmethod 

495 def hasRecords(self) -> bool: 

496 """Whether this data ID contains records. 

497 

498 These are the records for all of the dimension elements it identifies. 

499 

500 Returns 

501 ------- 

502 state : `bool` 

503 If `True`, the following attributes may be accessed: 

504 

505 - `records` 

506 - `region` 

507 - `timespan` 

508 - `pack` 

509 

510 If `False`, accessing any of these is considered a logic error. 

511 """ 

512 raise NotImplementedError() 

513 

514 @property 

515 def records(self) -> NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]: 

516 """Return the records. 

517 

518 Returns a mapping that contains `DimensionRecord` objects for all 

519 elements identified by this data ID (`NamedKeyMapping`). 

520 

521 The values of this mapping may be `None` if and only if there is no 

522 record for that element with these dimensions in the database (which 

523 means some foreign key field must have a NULL value). 

524 

525 Accessing this attribute if `hasRecords` returns `False` is a logic 

526 error that may raise an exception of unspecified type either 

527 immediately or when the returned mapping is used, depending on the 

528 implementation and whether assertions are enabled. 

529 """ 

530 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True." 

531 return _DataCoordinateRecordsView(self) 

532 

533 @abstractmethod 

534 def _record(self, name: str) -> Optional[DimensionRecord]: 

535 """Protected implementation hook that backs the ``records`` attribute. 

536 

537 Parameters 

538 ---------- 

539 name : `str` 

540 The name of a `DimensionElement`, guaranteed to be in 

541 ``self.graph.elements.names``. 

542 

543 Returns 

544 ------- 

545 record : `DimensionRecord` or `None` 

546 The dimension record for the given element identified by this 

547 data ID, or `None` if there is no such record. 

548 """ 

549 raise NotImplementedError() 

550 

551 @property 

552 def region(self) -> Optional[Region]: 

553 """Spatial region associated with this data ID. 

554 

555 (`lsst.sphgeom.Region` or `None`). 

556 

557 This is `None` if and only if ``self.graph.spatial`` is empty. 

558 

559 Accessing this attribute if `hasRecords` returns `False` is a logic 

560 error that may or may not raise an exception, depending on the 

561 implementation and whether assertions are enabled. 

562 """ 

563 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True." 

564 regions = [] 

565 for family in self.graph.spatial: 

566 element = family.choose(self.graph.elements) 

567 record = self._record(element.name) 

568 if record is None or record.region is None: 

569 return None 

570 else: 

571 regions.append(record.region) 

572 return _intersectRegions(*regions) 

573 

574 @property 

575 def timespan(self) -> Optional[Timespan]: 

576 """Temporal interval associated with this data ID. 

577 

578 (`Timespan` or `None`). 

579 

580 This is `None` if and only if ``self.graph.timespan`` is empty. 

581 

582 Accessing this attribute if `hasRecords` returns `False` is a logic 

583 error that may or may not raise an exception, depending on the 

584 implementation and whether assertions are enabled. 

585 """ 

586 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True." 

587 timespans = [] 

588 for family in self.graph.temporal: 

589 element = family.choose(self.graph.elements) 

590 record = self._record(element.name) 

591 # DimensionRecord subclasses for temporal elements always have 

592 # .timespan, but they're dynamic so this can't be type-checked. 

593 if record is None or record.timespan is None: 

594 return None 

595 else: 

596 timespans.append(record.timespan) 

597 return Timespan.intersection(*timespans) 

598 

599 def pack(self, name: str, *, returnMaxBits: bool = False) -> Union[Tuple[int, int], int]: 

600 """Pack this data ID into an integer. 

601 

602 Parameters 

603 ---------- 

604 name : `str` 

605 Name of the `DimensionPacker` algorithm (as defined in the 

606 dimension configuration). 

607 returnMaxBits : `bool`, optional 

608 If `True` (`False` is default), return the maximum number of 

609 nonzero bits in the returned integer across all data IDs. 

610 

611 Returns 

612 ------- 

613 packed : `int` 

614 Integer ID. This ID is unique only across data IDs that have 

615 the same values for the packer's "fixed" dimensions. 

616 maxBits : `int`, optional 

617 Maximum number of nonzero bits in ``packed``. Not returned unless 

618 ``returnMaxBits`` is `True`. 

619 

620 Notes 

621 ----- 

622 Accessing this attribute if `hasRecords` returns `False` is a logic 

623 error that may or may not raise an exception, depending on the 

624 implementation and whether assertions are enabled. 

625 """ 

626 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." 

627 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) 

628 

629 def to_simple(self, minimal: bool = False) -> Dict: 

630 """Convert this class to a simple python type. 

631 

632 This is suitable for serialization. 

633 

634 Parameters 

635 ---------- 

636 minimal : `bool`, optional 

637 Use minimal serialization. Has no effect on for this class. 

638 

639 Returns 

640 ------- 

641 as_dict : `dict` 

642 The object converted to a dictionary. 

643 """ 

644 # Convert to a dict form 

645 return self.byName() 

646 

647 @classmethod 

648 def from_simple(cls, simple: Dict[str, Any], 

649 universe: Optional[DimensionUniverse] = None, 

650 registry: Optional[Registry] = None) -> DataCoordinate: 

651 """Construct a new object from the simplified form. 

652 

653 The data is assumed to be of the form returned from the `to_simple` 

654 method. 

655 

656 Parameters 

657 ---------- 

658 simple : `dict` of [`str`, `Any`] 

659 The `dict` returned by `to_simple()`. 

660 universe : `DimensionUniverse` 

661 The special graph of all known dimensions. 

662 registry : `lsst.daf.butler.Registry`, optional 

663 Registry from which a universe can be extracted. Can be `None` 

664 if universe is provided explicitly. 

665 

666 Returns 

667 ------- 

668 dataId : `DataCoordinate` 

669 Newly-constructed object. 

670 """ 

671 if universe is None and registry is None: 

672 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

673 if universe is None and registry is not None: 

674 universe = registry.dimensions 

675 if universe is None: 

676 # this is for mypy 

677 raise ValueError("Unable to determine a usable universe") 

678 

679 return cls.standardize(simple, universe=universe) 

680 

681 to_json = to_json_generic 

682 from_json = classmethod(from_json_generic) 

683 

684 

685DataId = Union[DataCoordinate, Mapping[str, Any]] 

686"""A type-annotation alias for signatures that accept both informal data ID 

687dictionaries and validated `DataCoordinate` instances. 

688""" 

689 

690 

691class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]): 

692 """View class for `DataCoordinate.full`. 

693 

694 Provides the default implementation for 

695 `DataCoordinate.full`. 

696 

697 Parameters 

698 ---------- 

699 target : `DataCoordinate` 

700 The `DataCoordinate` instance this object provides a view of. 

701 """ 

702 

703 def __init__(self, target: DataCoordinate): 

704 self._target = target 

705 

706 __slots__ = ("_target",) 

707 

708 def __repr__(self) -> str: 

709 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names] 

710 return "{{{}}}".format(', '.join(terms)) 

711 

712 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

713 return self._target[key] 

714 

715 def __iter__(self) -> Iterator[Dimension]: 

716 return iter(self.keys()) 

717 

718 def __len__(self) -> int: 

719 return len(self.keys()) 

720 

721 def keys(self) -> NamedValueAbstractSet[Dimension]: 

722 return self._target.graph.dimensions 

723 

724 @property 

725 def names(self) -> AbstractSet[str]: 

726 # Docstring inherited from `NamedKeyMapping`. 

727 return self.keys().names 

728 

729 

730class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]): 

731 """View class for `DataCoordinate.records`. 

732 

733 Provides the default implementation for 

734 `DataCoordinate.records`. 

735 

736 Parameters 

737 ---------- 

738 target : `DataCoordinate` 

739 The `DataCoordinate` instance this object provides a view of. 

740 """ 

741 

742 def __init__(self, target: DataCoordinate): 

743 self._target = target 

744 

745 __slots__ = ("_target",) 

746 

747 def __repr__(self) -> str: 

748 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names] 

749 return "{{{}}}".format(', '.join(terms)) 

750 

751 def __str__(self) -> str: 

752 return "\n".join(str(v) for v in self.values()) 

753 

754 def __getitem__(self, key: Union[DimensionElement, str]) -> Optional[DimensionRecord]: 

755 if isinstance(key, DimensionElement): 

756 key = key.name 

757 return self._target._record(key) 

758 

759 def __iter__(self) -> Iterator[DimensionElement]: 

760 return iter(self.keys()) 

761 

762 def __len__(self) -> int: 

763 return len(self.keys()) 

764 

765 def keys(self) -> NamedValueAbstractSet[DimensionElement]: 

766 return self._target.graph.elements 

767 

768 @property 

769 def names(self) -> AbstractSet[str]: 

770 # Docstring inherited from `NamedKeyMapping`. 

771 return self.keys().names 

772 

773 

774class _BasicTupleDataCoordinate(DataCoordinate): 

775 """Standard implementation of `DataCoordinate`. 

776 

777 Backed by a tuple of values. 

778 

779 This class should only be accessed outside this module via the 

780 `DataCoordinate` interface, and should only be constructed via the static 

781 methods there. 

782 

783 Parameters 

784 ---------- 

785 graph : `DimensionGraph` 

786 The dimensions to be identified. 

787 values : `tuple` [ `int` or `str` ] 

788 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May 

789 include values for just required dimensions (which always come first) 

790 or all dimensions. 

791 """ 

792 

793 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...]): 

794 self._graph = graph 

795 self._values = values 

796 

797 __slots__ = ("_graph", "_values") 

798 

799 @property 

800 def graph(self) -> DimensionGraph: 

801 # Docstring inherited from DataCoordinate. 

802 return self._graph 

803 

804 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

805 # Docstring inherited from DataCoordinate. 

806 if isinstance(key, Dimension): 

807 key = key.name 

808 index = self._graph._dataCoordinateIndices[key] 

809 try: 

810 return self._values[index] 

811 except IndexError: 

812 # Caller asked for an implied dimension, but this object only has 

813 # values for the required ones. 

814 raise KeyError(key) from None 

815 

816 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

817 # Docstring inherited from DataCoordinate. 

818 if self._graph == graph: 

819 return self 

820 elif self.hasFull() or self._graph.required >= graph.dimensions: 

821 return _BasicTupleDataCoordinate( 

822 graph, 

823 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

824 ) 

825 else: 

826 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names)) 

827 

828 def union(self, other: DataCoordinate) -> DataCoordinate: 

829 # Docstring inherited from DataCoordinate. 

830 graph = self.graph.union(other.graph) 

831 # See if one or both input data IDs is already what we want to return; 

832 # if so, return the most complete one we have. 

833 if other.graph == graph: 

834 if self.graph == graph: 

835 # Input data IDs have the same graph (which is also the result 

836 # graph), but may not have the same content. 

837 # other might have records; self does not, so try other first. 

838 # If it at least has full values, it's no worse than self. 

839 if other.hasFull(): 

840 return other 

841 else: 

842 return self 

843 elif other.hasFull(): 

844 return other 

845 # There's some chance that neither self nor other has full values, 

846 # but together provide enough to the union to. Let the general 

847 # case below handle that. 

848 elif self.graph == graph: 

849 # No chance at returning records. If self has full values, it's 

850 # the best we can do. 

851 if self.hasFull(): 

852 return self 

853 # General case with actual merging of dictionaries. 

854 values = self.full.byName() if self.hasFull() else self.byName() 

855 values.update(other.full.byName() if other.hasFull() else other.byName()) 

856 return DataCoordinate.standardize(values, graph=graph) 

857 

858 def expanded(self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

859 ) -> DataCoordinate: 

860 # Docstring inherited from DataCoordinate 

861 values = self._values 

862 if not self.hasFull(): 

863 # Extract a complete values tuple from the attributes of the given 

864 # records. It's possible for these to be inconsistent with 

865 # self._values (which is a serious problem, of course), but we've 

866 # documented this as a no-checking API. 

867 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied) 

868 return _ExpandedTupleDataCoordinate(self._graph, values, records) 

869 

870 def hasFull(self) -> bool: 

871 # Docstring inherited from DataCoordinate. 

872 return len(self._values) == len(self._graph._dataCoordinateIndices) 

873 

874 def hasRecords(self) -> bool: 

875 # Docstring inherited from DataCoordinate. 

876 return False 

877 

878 def _record(self, name: str) -> Optional[DimensionRecord]: 

879 # Docstring inherited from DataCoordinate. 

880 assert False 

881 

882 

883class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate): 

884 """A `DataCoordinate` implementation that can hold `DimensionRecord`. 

885 

886 This class should only be accessed outside this module via the 

887 `DataCoordinate` interface, and should only be constructed via calls to 

888 `DataCoordinate.expanded`. 

889 

890 Parameters 

891 ---------- 

892 graph : `DimensionGraph` 

893 The dimensions to be identified. 

894 values : `tuple` [ `int` or `str` ] 

895 Data ID values, ordered to match ``graph._dataCoordinateIndices``. 

896 May include values for just required dimensions (which always come 

897 first) or all dimensions. 

898 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

899 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

900 `Mapping` with `str` (`DimensionElement` name) keys and 

901 `DimensionRecord` values. Keys must cover all elements in 

902 ``self.graph.elements``. Values may be `None`, but only to reflect 

903 actual NULL values in the database, not just records that have not 

904 been fetched. 

905 """ 

906 

907 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...], 

908 records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]]): 

909 super().__init__(graph, values) 

910 assert super().hasFull(), "This implementation requires full dimension records." 

911 self._records = records 

912 

913 __slots__ = ("_records",) 

914 

915 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

916 # Docstring inherited from DataCoordinate. 

917 if self._graph == graph: 

918 return self 

919 return _ExpandedTupleDataCoordinate(graph, 

920 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

921 records=self._records) 

922 

923 def expanded(self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

924 ) -> DataCoordinate: 

925 # Docstring inherited from DataCoordinate. 

926 return self 

927 

928 def union(self, other: DataCoordinate) -> DataCoordinate: 

929 # Docstring inherited from DataCoordinate. 

930 graph = self.graph.union(other.graph) 

931 # See if one or both input data IDs is already what we want to return; 

932 # if so, return the most complete one we have. 

933 if self.graph == graph: 

934 # self has records, so even if other is also a valid result, it's 

935 # no better. 

936 return self 

937 if other.graph == graph: 

938 # If other has full values, and self does not identify some of 

939 # those, it's the base we can do. It may have records, too. 

940 if other.hasFull(): 

941 return other 

942 # If other does not have full values, there's a chance self may 

943 # provide the values needed to complete it. For example, self 

944 # could be {band} while other could be 

945 # {instrument, physical_filter, band}, with band unknown. 

946 # General case with actual merging of dictionaries. 

947 values = self.full.byName() 

948 values.update(other.full.byName() if other.hasFull() else other.byName()) 

949 basic = DataCoordinate.standardize(values, graph=graph) 

950 # See if we can add records. 

951 if self.hasRecords() and other.hasRecords(): 

952 # Sometimes the elements of a union of graphs can contain elements 

953 # that weren't in either input graph (because graph unions are only 

954 # on dimensions). e.g. {visit} | {detector} brings along 

955 # visit_detector_region. 

956 elements = set(graph.elements.names) 

957 elements -= self.graph.elements.names 

958 elements -= other.graph.elements.names 

959 if not elements: 

960 records = NamedKeyDict[DimensionElement, Optional[DimensionRecord]](self.records) 

961 records.update(other.records) 

962 return basic.expanded(records.freeze()) 

963 return basic 

964 

965 def hasFull(self) -> bool: 

966 # Docstring inherited from DataCoordinate. 

967 return True 

968 

969 def hasRecords(self) -> bool: 

970 # Docstring inherited from DataCoordinate. 

971 return True 

972 

973 def _record(self, name: str) -> Optional[DimensionRecord]: 

974 # Docstring inherited from DataCoordinate. 

975 return self._records[name]