Coverage for python/lsst/daf/butler/core/dimensions/_coordinate.py: 28%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

322 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22# 

23# Design notes for this module are in 

24# doc/lsst.daf.butler/dev/dataCoordinate.py. 

25# 

26 

27from __future__ import annotations 

28 

29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue", "SerializedDataCoordinate") 

30 

31from abc import abstractmethod 

32import numbers 

33from typing import ( 

34 AbstractSet, 

35 Any, 

36 Dict, 

37 Iterator, 

38 Mapping, 

39 Optional, 

40 Tuple, 

41 TYPE_CHECKING, 

42 Union, 

43) 

44from pydantic import BaseModel 

45 

46from lsst.sphgeom import Region 

47from ..named import NamedKeyDict, NamedKeyMapping, NameLookupMapping, NamedValueAbstractSet 

48from ..timespan import Timespan 

49from ._elements import Dimension, DimensionElement 

50from ._graph import DimensionGraph 

51from ._records import DimensionRecord, SerializedDimensionRecord 

52from ..json import from_json_pydantic, to_json_pydantic 

53 

54if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 54 ↛ 55line 54 didn't jump to line 55, because the condition on line 54 was never true

55 from ._universe import DimensionUniverse 

56 from ...registry import Registry 

57 

58DataIdKey = Union[str, Dimension] 

59"""Type annotation alias for the keys that can be used to index a 

60DataCoordinate. 

61""" 

62 

63# Pydantic will cast int to str if str is first in the Union. 

64DataIdValue = Union[int, str, None] 

65"""Type annotation alias for the values that can be present in a 

66DataCoordinate or other data ID. 

67""" 

68 

69 

70class SerializedDataCoordinate(BaseModel): 

71 """Simplified model for serializing a `DataCoordinate`.""" 

72 

73 dataId: Dict[str, DataIdValue] 

74 records: Optional[Dict[str, SerializedDimensionRecord]] = None 

75 

76 @classmethod 

77 def direct(cls, *, dataId: Dict[str, DataIdValue], records: Dict[str, Dict]) -> SerializedDataCoordinate: 

78 """Construct a `SerializedDataCoordinate` directly without validators. 

79 

80 This differs from the pydantic "construct" method in that the arguments 

81 are explicitly what the model requires, and it will recurse through 

82 members, constructing them from their corresponding `direct` methods. 

83 

84 This method should only be called when the inputs are trusted. 

85 """ 

86 node = SerializedDataCoordinate.__new__(cls) 

87 setter = object.__setattr__ 

88 setter(node, 'dataId', dataId) 

89 setter(node, 'records', 

90 records if records is None else 

91 {k: SerializedDimensionRecord.direct(**v) for k, v in records.items()}) 

92 setter(node, '__fields_set__', {'dataId', 'records'}) 

93 return node 

94 

95 

96def _intersectRegions(*args: Region) -> Optional[Region]: 

97 """Return the intersection of several regions. 

98 

99 For internal use by `ExpandedDataCoordinate` only. 

100 

101 If no regions are provided, returns `None`. 

102 

103 This is currently a placeholder; it actually returns `NotImplemented` 

104 (it does *not* raise an exception) when multiple regions are given, which 

105 propagates to `ExpandedDataCoordinate`. This reflects the fact that we 

106 don't want to fail to construct an `ExpandedDataCoordinate` entirely when 

107 we can't compute its region, and at present we don't have a high-level use 

108 case for the regions of these particular data IDs. 

109 """ 

110 if len(args) == 0: 

111 return None 

112 elif len(args) == 1: 

113 return args[0] 

114 else: 

115 return NotImplemented 

116 

117 

118class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]): 

119 """Data ID dictionary. 

120 

121 An immutable data ID dictionary that guarantees that its key-value pairs 

122 identify at least all required dimensions in a `DimensionGraph`. 

123 

124 `DataCoordinate` itself is an ABC, but provides `staticmethod` factory 

125 functions for private concrete implementations that should be sufficient 

126 for most purposes. `standardize` is the most flexible and safe of these; 

127 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are 

128 more specialized and perform little or no checking of inputs. 

129 

130 Notes 

131 ----- 

132 Like any data ID class, `DataCoordinate` behaves like a dictionary, but 

133 with some subtleties: 

134 

135 - Both `Dimension` instances and `str` names thereof may be used as keys 

136 in lookup operations, but iteration (and `keys`) will yield `Dimension` 

137 instances. The `names` property can be used to obtain the corresponding 

138 `str` names. 

139 

140 - Lookups for implied dimensions (those in ``self.graph.implied``) are 

141 supported if and only if `hasFull` returns `True`, and are never 

142 included in iteration or `keys`. The `full` property may be used to 

143 obtain a mapping whose keys do include implied dimensions. 

144 

145 - Equality comparison with other mappings is supported, but it always 

146 considers only required dimensions (as well as requiring both operands 

147 to identify the same dimensions). This is not quite consistent with the 

148 way mappings usually work - normally differing keys imply unequal 

149 mappings - but it makes sense in this context because data IDs with the 

150 same values for required dimensions but different values for implied 

151 dimensions represent a serious problem with the data that 

152 `DataCoordinate` cannot generally recognize on its own, and a data ID 

153 that knows implied dimension values should still be able to compare as 

154 equal to one that does not. This is of course not the way comparisons 

155 between simple `dict` data IDs work, and hence using a `DataCoordinate` 

156 instance for at least one operand in any data ID comparison is strongly 

157 recommended. 

158 """ 

159 

160 __slots__ = () 

161 

162 _serializedType = SerializedDataCoordinate 

163 

164 @staticmethod 

165 def standardize( 

166 mapping: Optional[NameLookupMapping[Dimension, DataIdValue]] = None, 

167 *, 

168 graph: Optional[DimensionGraph] = None, 

169 universe: Optional[DimensionUniverse] = None, 

170 defaults: Optional[DataCoordinate] = None, 

171 **kwargs: Any 

172 ) -> DataCoordinate: 

173 """Standardize the supplied dataId. 

174 

175 Adapts an arbitrary mapping and/or additional arguments into a true 

176 `DataCoordinate`, or augment an existing one. 

177 

178 Parameters 

179 ---------- 

180 mapping : `~collections.abc.Mapping`, optional 

181 An informal data ID that maps dimensions or dimension names to 

182 their primary key values (may also be a true `DataCoordinate`). 

183 graph : `DimensionGraph` 

184 The dimensions to be identified by the new `DataCoordinate`. 

185 If not provided, will be inferred from the keys of ``mapping`` and 

186 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

187 is already a `DataCoordinate`. 

188 universe : `DimensionUniverse` 

189 All known dimensions and their relationships; used to expand 

190 and validate dependencies when ``graph`` is not provided. 

191 defaults : `DataCoordinate`, optional 

192 Default dimension key-value pairs to use when needed. These are 

193 never used to infer ``graph``, and are ignored if a different value 

194 is provided for the same key in ``mapping`` or `**kwargs``. 

195 **kwargs 

196 Additional keyword arguments are treated like additional key-value 

197 pairs in ``mapping``. 

198 

199 Returns 

200 ------- 

201 coordinate : `DataCoordinate` 

202 A validated `DataCoordinate` instance. 

203 

204 Raises 

205 ------ 

206 TypeError 

207 Raised if the set of optional arguments provided is not supported. 

208 KeyError 

209 Raised if a key-value pair for a required dimension is missing. 

210 """ 

211 d: Dict[str, DataIdValue] = {} 

212 if isinstance(mapping, DataCoordinate): 

213 if graph is None: 

214 if not kwargs: 

215 # Already standardized to exactly what we want. 

216 return mapping 

217 elif kwargs.keys().isdisjoint(graph.dimensions.names): 

218 # User provided kwargs, but told us not to use them by 

219 # passing in dimensions that are disjoint from those kwargs. 

220 # This is not necessarily user error - it's a useful pattern 

221 # to pass in all of the key-value pairs you have and let the 

222 # code here pull out only what it needs. 

223 return mapping.subset(graph) 

224 assert universe is None or universe == mapping.universe 

225 universe = mapping.universe 

226 d.update((name, mapping[name]) for name in mapping.graph.required.names) 

227 if mapping.hasFull(): 

228 d.update((name, mapping[name]) for name in mapping.graph.implied.names) 

229 elif isinstance(mapping, NamedKeyMapping): 

230 d.update(mapping.byName()) 

231 elif mapping is not None: 

232 d.update(mapping) 

233 d.update(kwargs) 

234 if graph is None: 

235 if defaults is not None: 

236 universe = defaults.universe 

237 elif universe is None: 

238 raise TypeError("universe must be provided if graph is not.") 

239 graph = DimensionGraph(universe, names=d.keys()) 

240 if not graph.dimensions: 

241 return DataCoordinate.makeEmpty(graph.universe) 

242 if defaults is not None: 

243 if defaults.hasFull(): 

244 for k, v in defaults.full.items(): 

245 d.setdefault(k.name, v) 

246 else: 

247 for k, v in defaults.items(): 

248 d.setdefault(k.name, v) 

249 if d.keys() >= graph.dimensions.names: 

250 values = tuple(d[name] for name in graph._dataCoordinateIndices.keys()) 

251 else: 

252 try: 

253 values = tuple(d[name] for name in graph.required.names) 

254 except KeyError as err: 

255 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err 

256 # Some backends cannot handle numpy.int64 type which is a subclass of 

257 # numbers.Integral; convert that to int. 

258 values = tuple(int(val) if isinstance(val, numbers.Integral) # type: ignore 

259 else val for val in values) 

260 return _BasicTupleDataCoordinate(graph, values) 

261 

262 @staticmethod 

263 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate: 

264 """Return an empty `DataCoordinate`. 

265 

266 It identifies the null set of dimensions. 

267 

268 Parameters 

269 ---------- 

270 universe : `DimensionUniverse` 

271 Universe to which this null dimension set belongs. 

272 

273 Returns 

274 ------- 

275 dataId : `DataCoordinate` 

276 A data ID object that identifies no dimensions. `hasFull` and 

277 `hasRecords` are guaranteed to return `True`, because both `full` 

278 and `records` are just empty mappings. 

279 """ 

280 return _ExpandedTupleDataCoordinate(universe.empty, (), {}) 

281 

282 @staticmethod 

283 def fromRequiredValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

284 """Construct a `DataCoordinate` from required dimension values. 

285 

286 This is a low-level interface with at most assertion-level checking of 

287 inputs. Most callers should use `standardize` instead. 

288 

289 Parameters 

290 ---------- 

291 graph : `DimensionGraph` 

292 Dimensions this data ID will identify. 

293 values : `tuple` [ `int` or `str` ] 

294 Tuple of primary key values corresponding to ``graph.required``, 

295 in that order. 

296 

297 Returns 

298 ------- 

299 dataId : `DataCoordinate` 

300 A data ID object that identifies the given dimensions. 

301 ``dataId.hasFull()`` will return `True` if and only if 

302 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

303 return `True`. 

304 """ 

305 assert len(graph.required) == len(values), \ 

306 f"Inconsistency between dimensions {graph.required} and required values {values}." 

307 return _BasicTupleDataCoordinate(graph, values) 

308 

309 @staticmethod 

310 def fromFullValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

311 """Construct a `DataCoordinate` from all dimension values. 

312 

313 This is a low-level interface with at most assertion-level checking of 

314 inputs. Most callers should use `standardize` instead. 

315 

316 Parameters 

317 ---------- 

318 graph : `DimensionGraph` 

319 Dimensions this data ID will identify. 

320 values : `tuple` [ `int` or `str` ] 

321 Tuple of primary key values corresponding to 

322 ``itertools.chain(graph.required, graph.implied)``, in that order. 

323 Note that this is _not_ the same order as ``graph.dimensions``, 

324 though these contain the same elements. 

325 

326 Returns 

327 ------- 

328 dataId : `DataCoordinate` 

329 A data ID object that identifies the given dimensions. 

330 ``dataId.hasFull()`` will return `True` if and only if 

331 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

332 return `True`. 

333 """ 

334 assert len(graph.dimensions) == len(values), \ 

335 f"Inconsistency between dimensions {graph.dimensions} and full values {values}." 

336 return _BasicTupleDataCoordinate(graph, values) 

337 

338 def __hash__(self) -> int: 

339 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required)) 

340 

341 def __eq__(self, other: Any) -> bool: 

342 if not isinstance(other, DataCoordinate): 

343 other = DataCoordinate.standardize(other, universe=self.universe) 

344 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required) 

345 

346 def __repr__(self) -> str: 

347 # We can't make repr yield something that could be exec'd here without 

348 # printing out the whole DimensionUniverse the graph is derived from. 

349 # So we print something that mostly looks like a dict, but doesn't 

350 # quote its keys: that's both more compact and something that can't 

351 # be mistaken for an actual dict or something that could be exec'd. 

352 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names] 

353 if self.hasFull() and self.graph.required != self.graph.dimensions: 

354 terms.append("...") 

355 return "{{{}}}".format(', '.join(terms)) 

356 

357 def __lt__(self, other: Any) -> bool: 

358 # Allow DataCoordinate to be sorted 

359 if not isinstance(other, type(self)): 

360 return NotImplemented 

361 # Form tuple of tuples for each DataCoordinate: 

362 # Unlike repr() we only use required keys here to ensure that 

363 # __eq__ can not be true simultaneously with __lt__ being true. 

364 self_kv = tuple(self.items()) 

365 other_kv = tuple(other.items()) 

366 

367 return self_kv < other_kv 

368 

369 def __iter__(self) -> Iterator[Dimension]: 

370 return iter(self.keys()) 

371 

372 def __len__(self) -> int: 

373 return len(self.keys()) 

374 

375 def keys(self) -> NamedValueAbstractSet[Dimension]: 

376 return self.graph.required 

377 

378 @property 

379 def names(self) -> AbstractSet[str]: 

380 """Names of the required dimensions identified by this data ID. 

381 

382 They are returned in the same order as `keys` 

383 (`collections.abc.Set` [ `str` ]). 

384 """ 

385 return self.keys().names 

386 

387 @abstractmethod 

388 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

389 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``. 

390 

391 Parameters 

392 ---------- 

393 graph : `DimensionGraph` 

394 The dimensions identified by the returned `DataCoordinate`. 

395 

396 Returns 

397 ------- 

398 coordinate : `DataCoordinate` 

399 A `DataCoordinate` instance that identifies only the given 

400 dimensions. May be ``self`` if ``graph == self.graph``. 

401 

402 Raises 

403 ------ 

404 KeyError 

405 Raised if the primary key value for one or more required dimensions 

406 is unknown. This may happen if ``graph.issubset(self.graph)`` is 

407 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if 

408 ``self.hasFull()`` is `False` and 

409 ``graph.required.issubset(self.graph.required)`` is `False`. As 

410 an example of the latter case, consider trying to go from a data ID 

411 with dimensions {instrument, physical_filter, band} to 

412 just {instrument, band}; band is implied by 

413 physical_filter and hence would have no value in the original data 

414 ID if ``self.hasFull()`` is `False`. 

415 

416 Notes 

417 ----- 

418 If `hasFull` and `hasRecords` return `True` on ``self``, they will 

419 return `True` (respectively) on the returned `DataCoordinate` as well. 

420 The converse does not hold. 

421 """ 

422 raise NotImplementedError() 

423 

424 @abstractmethod 

425 def union(self, other: DataCoordinate) -> DataCoordinate: 

426 """Combine two data IDs. 

427 

428 Yields a new one that identifies all dimensions that either of them 

429 identify. 

430 

431 Parameters 

432 ---------- 

433 other : `DataCoordinate` 

434 Data ID to combine with ``self``. 

435 

436 Returns 

437 ------- 

438 unioned : `DataCoordinate` 

439 A `DataCoordinate` instance that satisfies 

440 ``unioned.graph == self.graph.union(other.graph)``. Will preserve 

441 ``hasFull`` and ``hasRecords`` whenever possible. 

442 

443 Notes 

444 ----- 

445 No checking for consistency is performed on values for keys that 

446 ``self`` and ``other`` have in common, and which value is included in 

447 the returned data ID is not specified. 

448 """ 

449 raise NotImplementedError() 

450 

451 @abstractmethod 

452 def expanded(self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

453 ) -> DataCoordinate: 

454 """Return a `DataCoordinate` that holds the given records. 

455 

456 Guarantees that `hasRecords` returns `True`. 

457 

458 This is a low-level interface with at most assertion-level checking of 

459 inputs. Most callers should use `Registry.expandDataId` instead. 

460 

461 Parameters 

462 ---------- 

463 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

464 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

465 `Mapping` with `str` (`DimensionElement` name) keys and 

466 `DimensionRecord` values. Keys must cover all elements in 

467 ``self.graph.elements``. Values may be `None`, but only to reflect 

468 actual NULL values in the database, not just records that have not 

469 been fetched. 

470 """ 

471 raise NotImplementedError() 

472 

473 @property 

474 def universe(self) -> DimensionUniverse: 

475 """Universe that defines all known compatible dimensions. 

476 

477 The univers will be compatible with this coordinate 

478 (`DimensionUniverse`). 

479 """ 

480 return self.graph.universe 

481 

482 @property 

483 @abstractmethod 

484 def graph(self) -> DimensionGraph: 

485 """Dimensions identified by this data ID (`DimensionGraph`). 

486 

487 Note that values are only required to be present for dimensions in 

488 ``self.graph.required``; all others may be retrieved (from a 

489 `Registry`) given these. 

490 """ 

491 raise NotImplementedError() 

492 

493 @abstractmethod 

494 def hasFull(self) -> bool: 

495 """Whether this data ID contains implied and required values. 

496 

497 Returns 

498 ------- 

499 state : `bool` 

500 If `True`, `__getitem__`, `get`, and `__contains__` (but not 

501 `keys`!) will act as though the mapping includes key-value pairs 

502 for implied dimensions, and the `full` property may be used. If 

503 `False`, these operations only include key-value pairs for required 

504 dimensions, and accessing `full` is an error. Always `True` if 

505 there are no implied dimensions. 

506 """ 

507 raise NotImplementedError() 

508 

509 @property 

510 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]: 

511 """Return mapping for all dimensions in ``self.graph``. 

512 

513 The mapping includes key-value pairs for all dimensions in 

514 ``self.graph``, including implied (`NamedKeyMapping`). 

515 

516 Accessing this attribute if `hasFull` returns `False` is a logic error 

517 that may raise an exception of unspecified type either immediately or 

518 when implied keys are accessed via the returned mapping, depending on 

519 the implementation and whether assertions are enabled. 

520 """ 

521 assert self.hasFull(), "full may only be accessed if hasFull() returns True." 

522 return _DataCoordinateFullView(self) 

523 

524 @abstractmethod 

525 def hasRecords(self) -> bool: 

526 """Whether this data ID contains records. 

527 

528 These are the records for all of the dimension elements it identifies. 

529 

530 Returns 

531 ------- 

532 state : `bool` 

533 If `True`, the following attributes may be accessed: 

534 

535 - `records` 

536 - `region` 

537 - `timespan` 

538 - `pack` 

539 

540 If `False`, accessing any of these is considered a logic error. 

541 """ 

542 raise NotImplementedError() 

543 

544 @property 

545 def records(self) -> NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]: 

546 """Return the records. 

547 

548 Returns a mapping that contains `DimensionRecord` objects for all 

549 elements identified by this data ID (`NamedKeyMapping`). 

550 

551 The values of this mapping may be `None` if and only if there is no 

552 record for that element with these dimensions in the database (which 

553 means some foreign key field must have a NULL value). 

554 

555 Accessing this attribute if `hasRecords` returns `False` is a logic 

556 error that may raise an exception of unspecified type either 

557 immediately or when the returned mapping is used, depending on the 

558 implementation and whether assertions are enabled. 

559 """ 

560 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True." 

561 return _DataCoordinateRecordsView(self) 

562 

563 @abstractmethod 

564 def _record(self, name: str) -> Optional[DimensionRecord]: 

565 """Protected implementation hook that backs the ``records`` attribute. 

566 

567 Parameters 

568 ---------- 

569 name : `str` 

570 The name of a `DimensionElement`, guaranteed to be in 

571 ``self.graph.elements.names``. 

572 

573 Returns 

574 ------- 

575 record : `DimensionRecord` or `None` 

576 The dimension record for the given element identified by this 

577 data ID, or `None` if there is no such record. 

578 """ 

579 raise NotImplementedError() 

580 

581 @property 

582 def region(self) -> Optional[Region]: 

583 """Spatial region associated with this data ID. 

584 

585 (`lsst.sphgeom.Region` or `None`). 

586 

587 This is `None` if and only if ``self.graph.spatial`` is empty. 

588 

589 Accessing this attribute if `hasRecords` returns `False` is a logic 

590 error that may or may not raise an exception, depending on the 

591 implementation and whether assertions are enabled. 

592 """ 

593 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True." 

594 regions = [] 

595 for family in self.graph.spatial: 

596 element = family.choose(self.graph.elements) 

597 record = self._record(element.name) 

598 if record is None or record.region is None: 

599 return None 

600 else: 

601 regions.append(record.region) 

602 return _intersectRegions(*regions) 

603 

604 @property 

605 def timespan(self) -> Optional[Timespan]: 

606 """Temporal interval associated with this data ID. 

607 

608 (`Timespan` or `None`). 

609 

610 This is `None` if and only if ``self.graph.timespan`` is empty. 

611 

612 Accessing this attribute if `hasRecords` returns `False` is a logic 

613 error that may or may not raise an exception, depending on the 

614 implementation and whether assertions are enabled. 

615 """ 

616 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True." 

617 timespans = [] 

618 for family in self.graph.temporal: 

619 element = family.choose(self.graph.elements) 

620 record = self._record(element.name) 

621 # DimensionRecord subclasses for temporal elements always have 

622 # .timespan, but they're dynamic so this can't be type-checked. 

623 if record is None or record.timespan is None: 

624 return None 

625 else: 

626 timespans.append(record.timespan) 

627 return Timespan.intersection(*timespans) 

628 

629 def pack(self, name: str, *, returnMaxBits: bool = False) -> Union[Tuple[int, int], int]: 

630 """Pack this data ID into an integer. 

631 

632 Parameters 

633 ---------- 

634 name : `str` 

635 Name of the `DimensionPacker` algorithm (as defined in the 

636 dimension configuration). 

637 returnMaxBits : `bool`, optional 

638 If `True` (`False` is default), return the maximum number of 

639 nonzero bits in the returned integer across all data IDs. 

640 

641 Returns 

642 ------- 

643 packed : `int` 

644 Integer ID. This ID is unique only across data IDs that have 

645 the same values for the packer's "fixed" dimensions. 

646 maxBits : `int`, optional 

647 Maximum number of nonzero bits in ``packed``. Not returned unless 

648 ``returnMaxBits`` is `True`. 

649 

650 Notes 

651 ----- 

652 Accessing this attribute if `hasRecords` returns `False` is a logic 

653 error that may or may not raise an exception, depending on the 

654 implementation and whether assertions are enabled. 

655 """ 

656 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." 

657 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) 

658 

659 def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate: 

660 """Convert this class to a simple python type. 

661 

662 This is suitable for serialization. 

663 

664 Parameters 

665 ---------- 

666 minimal : `bool`, optional 

667 Use minimal serialization. If set the records will not be attached. 

668 

669 Returns 

670 ------- 

671 simple : `SerializedDataCoordinate` 

672 The object converted to simple form. 

673 """ 

674 # Convert to a dict form 

675 if self.hasFull(): 

676 dataId = self.full.byName() 

677 else: 

678 dataId = self.byName() 

679 records: Optional[Dict[str, SerializedDimensionRecord]] 

680 if not minimal and self.hasRecords(): 

681 records = {k: v.to_simple() for k, v in self.records.byName().items() if v is not None} 

682 else: 

683 records = None 

684 

685 return SerializedDataCoordinate(dataId=dataId, records=records) 

686 

687 @classmethod 

688 def from_simple(cls, simple: SerializedDataCoordinate, 

689 universe: Optional[DimensionUniverse] = None, 

690 registry: Optional[Registry] = None) -> DataCoordinate: 

691 """Construct a new object from the simplified form. 

692 

693 The data is assumed to be of the form returned from the `to_simple` 

694 method. 

695 

696 Parameters 

697 ---------- 

698 simple : `dict` of [`str`, `Any`] 

699 The `dict` returned by `to_simple()`. 

700 universe : `DimensionUniverse` 

701 The special graph of all known dimensions. 

702 registry : `lsst.daf.butler.Registry`, optional 

703 Registry from which a universe can be extracted. Can be `None` 

704 if universe is provided explicitly. 

705 

706 Returns 

707 ------- 

708 dataId : `DataCoordinate` 

709 Newly-constructed object. 

710 """ 

711 if universe is None and registry is None: 

712 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

713 if universe is None and registry is not None: 

714 universe = registry.dimensions 

715 if universe is None: 

716 # this is for mypy 

717 raise ValueError("Unable to determine a usable universe") 

718 

719 dataId = cls.standardize(simple.dataId, universe=universe) 

720 if simple.records: 

721 dataId = dataId.expanded({k: DimensionRecord.from_simple(v, universe=universe) 

722 for k, v in simple.records.items()}) 

723 return dataId 

724 

725 to_json = to_json_pydantic 

726 from_json = classmethod(from_json_pydantic) 

727 

728 

729DataId = Union[DataCoordinate, Mapping[str, Any]] 

730"""A type-annotation alias for signatures that accept both informal data ID 

731dictionaries and validated `DataCoordinate` instances. 

732""" 

733 

734 

735class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]): 

736 """View class for `DataCoordinate.full`. 

737 

738 Provides the default implementation for 

739 `DataCoordinate.full`. 

740 

741 Parameters 

742 ---------- 

743 target : `DataCoordinate` 

744 The `DataCoordinate` instance this object provides a view of. 

745 """ 

746 

747 def __init__(self, target: DataCoordinate): 

748 self._target = target 

749 

750 __slots__ = ("_target",) 

751 

752 def __repr__(self) -> str: 

753 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names] 

754 return "{{{}}}".format(', '.join(terms)) 

755 

756 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

757 return self._target[key] 

758 

759 def __iter__(self) -> Iterator[Dimension]: 

760 return iter(self.keys()) 

761 

762 def __len__(self) -> int: 

763 return len(self.keys()) 

764 

765 def keys(self) -> NamedValueAbstractSet[Dimension]: 

766 return self._target.graph.dimensions 

767 

768 @property 

769 def names(self) -> AbstractSet[str]: 

770 # Docstring inherited from `NamedKeyMapping`. 

771 return self.keys().names 

772 

773 

774class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]): 

775 """View class for `DataCoordinate.records`. 

776 

777 Provides the default implementation for 

778 `DataCoordinate.records`. 

779 

780 Parameters 

781 ---------- 

782 target : `DataCoordinate` 

783 The `DataCoordinate` instance this object provides a view of. 

784 """ 

785 

786 def __init__(self, target: DataCoordinate): 

787 self._target = target 

788 

789 __slots__ = ("_target",) 

790 

791 def __repr__(self) -> str: 

792 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names] 

793 return "{{{}}}".format(', '.join(terms)) 

794 

795 def __str__(self) -> str: 

796 return "\n".join(str(v) for v in self.values()) 

797 

798 def __getitem__(self, key: Union[DimensionElement, str]) -> Optional[DimensionRecord]: 

799 if isinstance(key, DimensionElement): 

800 key = key.name 

801 return self._target._record(key) 

802 

803 def __iter__(self) -> Iterator[DimensionElement]: 

804 return iter(self.keys()) 

805 

806 def __len__(self) -> int: 

807 return len(self.keys()) 

808 

809 def keys(self) -> NamedValueAbstractSet[DimensionElement]: 

810 return self._target.graph.elements 

811 

812 @property 

813 def names(self) -> AbstractSet[str]: 

814 # Docstring inherited from `NamedKeyMapping`. 

815 return self.keys().names 

816 

817 

818class _BasicTupleDataCoordinate(DataCoordinate): 

819 """Standard implementation of `DataCoordinate`. 

820 

821 Backed by a tuple of values. 

822 

823 This class should only be accessed outside this module via the 

824 `DataCoordinate` interface, and should only be constructed via the static 

825 methods there. 

826 

827 Parameters 

828 ---------- 

829 graph : `DimensionGraph` 

830 The dimensions to be identified. 

831 values : `tuple` [ `int` or `str` ] 

832 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May 

833 include values for just required dimensions (which always come first) 

834 or all dimensions. 

835 """ 

836 

837 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...]): 

838 self._graph = graph 

839 self._values = values 

840 

841 __slots__ = ("_graph", "_values") 

842 

843 @property 

844 def graph(self) -> DimensionGraph: 

845 # Docstring inherited from DataCoordinate. 

846 return self._graph 

847 

848 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

849 # Docstring inherited from DataCoordinate. 

850 if isinstance(key, Dimension): 

851 key = key.name 

852 index = self._graph._dataCoordinateIndices[key] 

853 try: 

854 return self._values[index] 

855 except IndexError: 

856 # Caller asked for an implied dimension, but this object only has 

857 # values for the required ones. 

858 raise KeyError(key) from None 

859 

860 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

861 # Docstring inherited from DataCoordinate. 

862 if self._graph == graph: 

863 return self 

864 elif self.hasFull() or self._graph.required >= graph.dimensions: 

865 return _BasicTupleDataCoordinate( 

866 graph, 

867 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

868 ) 

869 else: 

870 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names)) 

871 

872 def union(self, other: DataCoordinate) -> DataCoordinate: 

873 # Docstring inherited from DataCoordinate. 

874 graph = self.graph.union(other.graph) 

875 # See if one or both input data IDs is already what we want to return; 

876 # if so, return the most complete one we have. 

877 if other.graph == graph: 

878 if self.graph == graph: 

879 # Input data IDs have the same graph (which is also the result 

880 # graph), but may not have the same content. 

881 # other might have records; self does not, so try other first. 

882 # If it at least has full values, it's no worse than self. 

883 if other.hasFull(): 

884 return other 

885 else: 

886 return self 

887 elif other.hasFull(): 

888 return other 

889 # There's some chance that neither self nor other has full values, 

890 # but together provide enough to the union to. Let the general 

891 # case below handle that. 

892 elif self.graph == graph: 

893 # No chance at returning records. If self has full values, it's 

894 # the best we can do. 

895 if self.hasFull(): 

896 return self 

897 # General case with actual merging of dictionaries. 

898 values = self.full.byName() if self.hasFull() else self.byName() 

899 values.update(other.full.byName() if other.hasFull() else other.byName()) 

900 return DataCoordinate.standardize(values, graph=graph) 

901 

902 def expanded(self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

903 ) -> DataCoordinate: 

904 # Docstring inherited from DataCoordinate 

905 values = self._values 

906 if not self.hasFull(): 

907 # Extract a complete values tuple from the attributes of the given 

908 # records. It's possible for these to be inconsistent with 

909 # self._values (which is a serious problem, of course), but we've 

910 # documented this as a no-checking API. 

911 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied) 

912 return _ExpandedTupleDataCoordinate(self._graph, values, records) 

913 

914 def hasFull(self) -> bool: 

915 # Docstring inherited from DataCoordinate. 

916 return len(self._values) == len(self._graph._dataCoordinateIndices) 

917 

918 def hasRecords(self) -> bool: 

919 # Docstring inherited from DataCoordinate. 

920 return False 

921 

922 def _record(self, name: str) -> Optional[DimensionRecord]: 

923 # Docstring inherited from DataCoordinate. 

924 assert False 

925 

926 

927class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate): 

928 """A `DataCoordinate` implementation that can hold `DimensionRecord`. 

929 

930 This class should only be accessed outside this module via the 

931 `DataCoordinate` interface, and should only be constructed via calls to 

932 `DataCoordinate.expanded`. 

933 

934 Parameters 

935 ---------- 

936 graph : `DimensionGraph` 

937 The dimensions to be identified. 

938 values : `tuple` [ `int` or `str` ] 

939 Data ID values, ordered to match ``graph._dataCoordinateIndices``. 

940 May include values for just required dimensions (which always come 

941 first) or all dimensions. 

942 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

943 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

944 `Mapping` with `str` (`DimensionElement` name) keys and 

945 `DimensionRecord` values. Keys must cover all elements in 

946 ``self.graph.elements``. Values may be `None`, but only to reflect 

947 actual NULL values in the database, not just records that have not 

948 been fetched. 

949 """ 

950 

951 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...], 

952 records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]]): 

953 super().__init__(graph, values) 

954 assert super().hasFull(), "This implementation requires full dimension records." 

955 self._records = records 

956 

957 __slots__ = ("_records",) 

958 

959 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

960 # Docstring inherited from DataCoordinate. 

961 if self._graph == graph: 

962 return self 

963 return _ExpandedTupleDataCoordinate(graph, 

964 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

965 records=self._records) 

966 

967 def expanded(self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

968 ) -> DataCoordinate: 

969 # Docstring inherited from DataCoordinate. 

970 return self 

971 

972 def union(self, other: DataCoordinate) -> DataCoordinate: 

973 # Docstring inherited from DataCoordinate. 

974 graph = self.graph.union(other.graph) 

975 # See if one or both input data IDs is already what we want to return; 

976 # if so, return the most complete one we have. 

977 if self.graph == graph: 

978 # self has records, so even if other is also a valid result, it's 

979 # no better. 

980 return self 

981 if other.graph == graph: 

982 # If other has full values, and self does not identify some of 

983 # those, it's the base we can do. It may have records, too. 

984 if other.hasFull(): 

985 return other 

986 # If other does not have full values, there's a chance self may 

987 # provide the values needed to complete it. For example, self 

988 # could be {band} while other could be 

989 # {instrument, physical_filter, band}, with band unknown. 

990 # General case with actual merging of dictionaries. 

991 values = self.full.byName() 

992 values.update(other.full.byName() if other.hasFull() else other.byName()) 

993 basic = DataCoordinate.standardize(values, graph=graph) 

994 # See if we can add records. 

995 if self.hasRecords() and other.hasRecords(): 

996 # Sometimes the elements of a union of graphs can contain elements 

997 # that weren't in either input graph (because graph unions are only 

998 # on dimensions). e.g. {visit} | {detector} brings along 

999 # visit_detector_region. 

1000 elements = set(graph.elements.names) 

1001 elements -= self.graph.elements.names 

1002 elements -= other.graph.elements.names 

1003 if not elements: 

1004 records = NamedKeyDict[DimensionElement, Optional[DimensionRecord]](self.records) 

1005 records.update(other.records) 

1006 return basic.expanded(records.freeze()) 

1007 return basic 

1008 

1009 def hasFull(self) -> bool: 

1010 # Docstring inherited from DataCoordinate. 

1011 return True 

1012 

1013 def hasRecords(self) -> bool: 

1014 # Docstring inherited from DataCoordinate. 

1015 return True 

1016 

1017 def _record(self, name: str) -> Optional[DimensionRecord]: 

1018 # Docstring inherited from DataCoordinate. 

1019 return self._records[name]