Coverage for python/lsst/daf/butler/core/dimensions/_coordinate.py: 26%

329 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-06 12:40 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22# 

23# Design notes for this module are in 

24# doc/lsst.daf.butler/dev/dataCoordinate.py. 

25# 

26 

27from __future__ import annotations 

28 

29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue", "SerializedDataCoordinate") 

30 

31import numbers 

32from abc import abstractmethod 

33from typing import TYPE_CHECKING, AbstractSet, Any, Dict, Iterator, Mapping, Optional, Tuple, Union 

34 

35from lsst.sphgeom import Region 

36from pydantic import BaseModel 

37 

38from ..json import from_json_pydantic, to_json_pydantic 

39from ..named import NamedKeyDict, NamedKeyMapping, NamedValueAbstractSet, NameLookupMapping 

40from ..timespan import Timespan 

41from ._elements import Dimension, DimensionElement 

42from ._graph import DimensionGraph 

43from ._records import DimensionRecord, SerializedDimensionRecord 

44 

45if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 45 ↛ 46line 45 didn't jump to line 46, because the condition on line 45 was never true

46 from ...registry import Registry 

47 from ._universe import DimensionUniverse 

48 

49DataIdKey = Union[str, Dimension] 

50"""Type annotation alias for the keys that can be used to index a 

51DataCoordinate. 

52""" 

53 

54# Pydantic will cast int to str if str is first in the Union. 

55DataIdValue = Union[int, str, None] 

56"""Type annotation alias for the values that can be present in a 

57DataCoordinate or other data ID. 

58""" 

59 

60 

61class SerializedDataCoordinate(BaseModel): 

62 """Simplified model for serializing a `DataCoordinate`.""" 

63 

64 dataId: Dict[str, DataIdValue] 

65 records: Optional[Dict[str, SerializedDimensionRecord]] = None 

66 

67 @classmethod 

68 def direct(cls, *, dataId: Dict[str, DataIdValue], records: Dict[str, Dict]) -> SerializedDataCoordinate: 

69 """Construct a `SerializedDataCoordinate` directly without validators. 

70 

71 This differs from the pydantic "construct" method in that the arguments 

72 are explicitly what the model requires, and it will recurse through 

73 members, constructing them from their corresponding `direct` methods. 

74 

75 This method should only be called when the inputs are trusted. 

76 """ 

77 node = SerializedDataCoordinate.__new__(cls) 

78 setter = object.__setattr__ 

79 setter(node, "dataId", dataId) 

80 setter( 

81 node, 

82 "records", 

83 records 

84 if records is None 

85 else {k: SerializedDimensionRecord.direct(**v) for k, v in records.items()}, 

86 ) 

87 setter(node, "__fields_set__", {"dataId", "records"}) 

88 return node 

89 

90 

91def _intersectRegions(*args: Region) -> Optional[Region]: 

92 """Return the intersection of several regions. 

93 

94 For internal use by `ExpandedDataCoordinate` only. 

95 

96 If no regions are provided, returns `None`. 

97 

98 This is currently a placeholder; it actually returns `NotImplemented` 

99 (it does *not* raise an exception) when multiple regions are given, which 

100 propagates to `ExpandedDataCoordinate`. This reflects the fact that we 

101 don't want to fail to construct an `ExpandedDataCoordinate` entirely when 

102 we can't compute its region, and at present we don't have a high-level use 

103 case for the regions of these particular data IDs. 

104 """ 

105 if len(args) == 0: 

106 return None 

107 elif len(args) == 1: 

108 return args[0] 

109 else: 

110 return NotImplemented 

111 

112 

113class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]): 

114 """Data ID dictionary. 

115 

116 An immutable data ID dictionary that guarantees that its key-value pairs 

117 identify at least all required dimensions in a `DimensionGraph`. 

118 

119 `DataCoordinate` itself is an ABC, but provides `staticmethod` factory 

120 functions for private concrete implementations that should be sufficient 

121 for most purposes. `standardize` is the most flexible and safe of these; 

122 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are 

123 more specialized and perform little or no checking of inputs. 

124 

125 Notes 

126 ----- 

127 Like any data ID class, `DataCoordinate` behaves like a dictionary, but 

128 with some subtleties: 

129 

130 - Both `Dimension` instances and `str` names thereof may be used as keys 

131 in lookup operations, but iteration (and `keys`) will yield `Dimension` 

132 instances. The `names` property can be used to obtain the corresponding 

133 `str` names. 

134 

135 - Lookups for implied dimensions (those in ``self.graph.implied``) are 

136 supported if and only if `hasFull` returns `True`, and are never 

137 included in iteration or `keys`. The `full` property may be used to 

138 obtain a mapping whose keys do include implied dimensions. 

139 

140 - Equality comparison with other mappings is supported, but it always 

141 considers only required dimensions (as well as requiring both operands 

142 to identify the same dimensions). This is not quite consistent with the 

143 way mappings usually work - normally differing keys imply unequal 

144 mappings - but it makes sense in this context because data IDs with the 

145 same values for required dimensions but different values for implied 

146 dimensions represent a serious problem with the data that 

147 `DataCoordinate` cannot generally recognize on its own, and a data ID 

148 that knows implied dimension values should still be able to compare as 

149 equal to one that does not. This is of course not the way comparisons 

150 between simple `dict` data IDs work, and hence using a `DataCoordinate` 

151 instance for at least one operand in any data ID comparison is strongly 

152 recommended. 

153 """ 

154 

155 __slots__ = () 

156 

157 _serializedType = SerializedDataCoordinate 

158 

159 @staticmethod 

160 def standardize( 

161 mapping: Optional[NameLookupMapping[Dimension, DataIdValue]] = None, 

162 *, 

163 graph: Optional[DimensionGraph] = None, 

164 universe: Optional[DimensionUniverse] = None, 

165 defaults: Optional[DataCoordinate] = None, 

166 **kwargs: Any, 

167 ) -> DataCoordinate: 

168 """Standardize the supplied dataId. 

169 

170 Adapts an arbitrary mapping and/or additional arguments into a true 

171 `DataCoordinate`, or augment an existing one. 

172 

173 Parameters 

174 ---------- 

175 mapping : `~collections.abc.Mapping`, optional 

176 An informal data ID that maps dimensions or dimension names to 

177 their primary key values (may also be a true `DataCoordinate`). 

178 graph : `DimensionGraph` 

179 The dimensions to be identified by the new `DataCoordinate`. 

180 If not provided, will be inferred from the keys of ``mapping`` and 

181 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

182 is already a `DataCoordinate`. 

183 universe : `DimensionUniverse` 

184 All known dimensions and their relationships; used to expand 

185 and validate dependencies when ``graph`` is not provided. 

186 defaults : `DataCoordinate`, optional 

187 Default dimension key-value pairs to use when needed. These are 

188 never used to infer ``graph``, and are ignored if a different value 

189 is provided for the same key in ``mapping`` or `**kwargs``. 

190 **kwargs 

191 Additional keyword arguments are treated like additional key-value 

192 pairs in ``mapping``. 

193 

194 Returns 

195 ------- 

196 coordinate : `DataCoordinate` 

197 A validated `DataCoordinate` instance. 

198 

199 Raises 

200 ------ 

201 TypeError 

202 Raised if the set of optional arguments provided is not supported. 

203 KeyError 

204 Raised if a key-value pair for a required dimension is missing. 

205 """ 

206 d: Dict[str, DataIdValue] = {} 

207 if isinstance(mapping, DataCoordinate): 

208 if graph is None: 

209 if not kwargs: 

210 # Already standardized to exactly what we want. 

211 return mapping 

212 elif kwargs.keys().isdisjoint(graph.dimensions.names): 

213 # User provided kwargs, but told us not to use them by 

214 # passing in dimensions that are disjoint from those kwargs. 

215 # This is not necessarily user error - it's a useful pattern 

216 # to pass in all of the key-value pairs you have and let the 

217 # code here pull out only what it needs. 

218 return mapping.subset(graph) 

219 assert universe is None or universe == mapping.universe 

220 universe = mapping.universe 

221 d.update((name, mapping[name]) for name in mapping.graph.required.names) 

222 if mapping.hasFull(): 

223 d.update((name, mapping[name]) for name in mapping.graph.implied.names) 

224 elif isinstance(mapping, NamedKeyMapping): 

225 d.update(mapping.byName()) 

226 elif mapping is not None: 

227 d.update(mapping) 

228 d.update(kwargs) 

229 if graph is None: 

230 if defaults is not None: 

231 universe = defaults.universe 

232 elif universe is None: 

233 raise TypeError("universe must be provided if graph is not.") 

234 graph = DimensionGraph(universe, names=d.keys()) 

235 if not graph.dimensions: 

236 return DataCoordinate.makeEmpty(graph.universe) 

237 if defaults is not None: 

238 if defaults.hasFull(): 

239 for k, v in defaults.full.items(): 

240 d.setdefault(k.name, v) 

241 else: 

242 for k, v in defaults.items(): 

243 d.setdefault(k.name, v) 

244 if d.keys() >= graph.dimensions.names: 

245 values = tuple(d[name] for name in graph._dataCoordinateIndices.keys()) 

246 else: 

247 try: 

248 values = tuple(d[name] for name in graph.required.names) 

249 except KeyError as err: 

250 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err 

251 # Some backends cannot handle numpy.int64 type which is a subclass of 

252 # numbers.Integral; convert that to int. 

253 values = tuple( 

254 int(val) if isinstance(val, numbers.Integral) else val for val in values # type: ignore 

255 ) 

256 return _BasicTupleDataCoordinate(graph, values) 

257 

258 @staticmethod 

259 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate: 

260 """Return an empty `DataCoordinate`. 

261 

262 It identifies the null set of dimensions. 

263 

264 Parameters 

265 ---------- 

266 universe : `DimensionUniverse` 

267 Universe to which this null dimension set belongs. 

268 

269 Returns 

270 ------- 

271 dataId : `DataCoordinate` 

272 A data ID object that identifies no dimensions. `hasFull` and 

273 `hasRecords` are guaranteed to return `True`, because both `full` 

274 and `records` are just empty mappings. 

275 """ 

276 return _ExpandedTupleDataCoordinate(universe.empty, (), {}) 

277 

278 @staticmethod 

279 def fromRequiredValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

280 """Construct a `DataCoordinate` from required dimension values. 

281 

282 This is a low-level interface with at most assertion-level checking of 

283 inputs. Most callers should use `standardize` instead. 

284 

285 Parameters 

286 ---------- 

287 graph : `DimensionGraph` 

288 Dimensions this data ID will identify. 

289 values : `tuple` [ `int` or `str` ] 

290 Tuple of primary key values corresponding to ``graph.required``, 

291 in that order. 

292 

293 Returns 

294 ------- 

295 dataId : `DataCoordinate` 

296 A data ID object that identifies the given dimensions. 

297 ``dataId.hasFull()`` will return `True` if and only if 

298 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

299 return `True`. 

300 """ 

301 assert len(graph.required) == len( 

302 values 

303 ), f"Inconsistency between dimensions {graph.required} and required values {values}." 

304 return _BasicTupleDataCoordinate(graph, values) 

305 

306 @staticmethod 

307 def fromFullValues(graph: DimensionGraph, values: Tuple[DataIdValue, ...]) -> DataCoordinate: 

308 """Construct a `DataCoordinate` from all dimension values. 

309 

310 This is a low-level interface with at most assertion-level checking of 

311 inputs. Most callers should use `standardize` instead. 

312 

313 Parameters 

314 ---------- 

315 graph : `DimensionGraph` 

316 Dimensions this data ID will identify. 

317 values : `tuple` [ `int` or `str` ] 

318 Tuple of primary key values corresponding to 

319 ``itertools.chain(graph.required, graph.implied)``, in that order. 

320 Note that this is _not_ the same order as ``graph.dimensions``, 

321 though these contain the same elements. 

322 

323 Returns 

324 ------- 

325 dataId : `DataCoordinate` 

326 A data ID object that identifies the given dimensions. 

327 ``dataId.hasFull()`` will return `True` if and only if 

328 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

329 return `True`. 

330 """ 

331 assert len(graph.dimensions) == len( 

332 values 

333 ), f"Inconsistency between dimensions {graph.dimensions} and full values {values}." 

334 return _BasicTupleDataCoordinate(graph, values) 

335 

336 def __hash__(self) -> int: 

337 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required)) 

338 

339 def __eq__(self, other: Any) -> bool: 

340 if not isinstance(other, DataCoordinate): 

341 other = DataCoordinate.standardize(other, universe=self.universe) 

342 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required) 

343 

344 def __repr__(self) -> str: 

345 # We can't make repr yield something that could be exec'd here without 

346 # printing out the whole DimensionUniverse the graph is derived from. 

347 # So we print something that mostly looks like a dict, but doesn't 

348 # quote its keys: that's both more compact and something that can't 

349 # be mistaken for an actual dict or something that could be exec'd. 

350 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names] 

351 if self.hasFull() and self.graph.required != self.graph.dimensions: 

352 terms.append("...") 

353 return "{{{}}}".format(", ".join(terms)) 

354 

355 def __lt__(self, other: Any) -> bool: 

356 # Allow DataCoordinate to be sorted 

357 if not isinstance(other, type(self)): 

358 return NotImplemented 

359 # Form tuple of tuples for each DataCoordinate: 

360 # Unlike repr() we only use required keys here to ensure that 

361 # __eq__ can not be true simultaneously with __lt__ being true. 

362 self_kv = tuple(self.items()) 

363 other_kv = tuple(other.items()) 

364 

365 return self_kv < other_kv 

366 

367 def __iter__(self) -> Iterator[Dimension]: 

368 return iter(self.keys()) 

369 

370 def __len__(self) -> int: 

371 return len(self.keys()) 

372 

373 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

374 return self.graph.required 

375 

376 @property 

377 def names(self) -> AbstractSet[str]: 

378 """Names of the required dimensions identified by this data ID. 

379 

380 They are returned in the same order as `keys` 

381 (`collections.abc.Set` [ `str` ]). 

382 """ 

383 return self.keys().names 

384 

385 @abstractmethod 

386 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

387 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``. 

388 

389 Parameters 

390 ---------- 

391 graph : `DimensionGraph` 

392 The dimensions identified by the returned `DataCoordinate`. 

393 

394 Returns 

395 ------- 

396 coordinate : `DataCoordinate` 

397 A `DataCoordinate` instance that identifies only the given 

398 dimensions. May be ``self`` if ``graph == self.graph``. 

399 

400 Raises 

401 ------ 

402 KeyError 

403 Raised if the primary key value for one or more required dimensions 

404 is unknown. This may happen if ``graph.issubset(self.graph)`` is 

405 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if 

406 ``self.hasFull()`` is `False` and 

407 ``graph.required.issubset(self.graph.required)`` is `False`. As 

408 an example of the latter case, consider trying to go from a data ID 

409 with dimensions {instrument, physical_filter, band} to 

410 just {instrument, band}; band is implied by 

411 physical_filter and hence would have no value in the original data 

412 ID if ``self.hasFull()`` is `False`. 

413 

414 Notes 

415 ----- 

416 If `hasFull` and `hasRecords` return `True` on ``self``, they will 

417 return `True` (respectively) on the returned `DataCoordinate` as well. 

418 The converse does not hold. 

419 """ 

420 raise NotImplementedError() 

421 

422 @abstractmethod 

423 def union(self, other: DataCoordinate) -> DataCoordinate: 

424 """Combine two data IDs. 

425 

426 Yields a new one that identifies all dimensions that either of them 

427 identify. 

428 

429 Parameters 

430 ---------- 

431 other : `DataCoordinate` 

432 Data ID to combine with ``self``. 

433 

434 Returns 

435 ------- 

436 unioned : `DataCoordinate` 

437 A `DataCoordinate` instance that satisfies 

438 ``unioned.graph == self.graph.union(other.graph)``. Will preserve 

439 ``hasFull`` and ``hasRecords`` whenever possible. 

440 

441 Notes 

442 ----- 

443 No checking for consistency is performed on values for keys that 

444 ``self`` and ``other`` have in common, and which value is included in 

445 the returned data ID is not specified. 

446 """ 

447 raise NotImplementedError() 

448 

449 @abstractmethod 

450 def expanded( 

451 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

452 ) -> DataCoordinate: 

453 """Return a `DataCoordinate` that holds the given records. 

454 

455 Guarantees that `hasRecords` returns `True`. 

456 

457 This is a low-level interface with at most assertion-level checking of 

458 inputs. Most callers should use `Registry.expandDataId` instead. 

459 

460 Parameters 

461 ---------- 

462 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

463 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

464 `Mapping` with `str` (`DimensionElement` name) keys and 

465 `DimensionRecord` values. Keys must cover all elements in 

466 ``self.graph.elements``. Values may be `None`, but only to reflect 

467 actual NULL values in the database, not just records that have not 

468 been fetched. 

469 """ 

470 raise NotImplementedError() 

471 

472 @property 

473 def universe(self) -> DimensionUniverse: 

474 """Universe that defines all known compatible dimensions. 

475 

476 The univers will be compatible with this coordinate 

477 (`DimensionUniverse`). 

478 """ 

479 return self.graph.universe 

480 

481 @property 

482 @abstractmethod 

483 def graph(self) -> DimensionGraph: 

484 """Dimensions identified by this data ID (`DimensionGraph`). 

485 

486 Note that values are only required to be present for dimensions in 

487 ``self.graph.required``; all others may be retrieved (from a 

488 `Registry`) given these. 

489 """ 

490 raise NotImplementedError() 

491 

492 @abstractmethod 

493 def hasFull(self) -> bool: 

494 """Whether this data ID contains implied and required values. 

495 

496 Returns 

497 ------- 

498 state : `bool` 

499 If `True`, `__getitem__`, `get`, and `__contains__` (but not 

500 `keys`!) will act as though the mapping includes key-value pairs 

501 for implied dimensions, and the `full` property may be used. If 

502 `False`, these operations only include key-value pairs for required 

503 dimensions, and accessing `full` is an error. Always `True` if 

504 there are no implied dimensions. 

505 """ 

506 raise NotImplementedError() 

507 

508 @property 

509 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]: 

510 """Return mapping for all dimensions in ``self.graph``. 

511 

512 The mapping includes key-value pairs for all dimensions in 

513 ``self.graph``, including implied (`NamedKeyMapping`). 

514 

515 Accessing this attribute if `hasFull` returns `False` is a logic error 

516 that may raise an exception of unspecified type either immediately or 

517 when implied keys are accessed via the returned mapping, depending on 

518 the implementation and whether assertions are enabled. 

519 """ 

520 assert self.hasFull(), "full may only be accessed if hasFull() returns True." 

521 return _DataCoordinateFullView(self) 

522 

523 @abstractmethod 

524 def hasRecords(self) -> bool: 

525 """Whether this data ID contains records. 

526 

527 These are the records for all of the dimension elements it identifies. 

528 

529 Returns 

530 ------- 

531 state : `bool` 

532 If `True`, the following attributes may be accessed: 

533 

534 - `records` 

535 - `region` 

536 - `timespan` 

537 - `pack` 

538 

539 If `False`, accessing any of these is considered a logic error. 

540 """ 

541 raise NotImplementedError() 

542 

543 @property 

544 def records(self) -> NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]: 

545 """Return the records. 

546 

547 Returns a mapping that contains `DimensionRecord` objects for all 

548 elements identified by this data ID (`NamedKeyMapping`). 

549 

550 The values of this mapping may be `None` if and only if there is no 

551 record for that element with these dimensions in the database (which 

552 means some foreign key field must have a NULL value). 

553 

554 Accessing this attribute if `hasRecords` returns `False` is a logic 

555 error that may raise an exception of unspecified type either 

556 immediately or when the returned mapping is used, depending on the 

557 implementation and whether assertions are enabled. 

558 """ 

559 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True." 

560 return _DataCoordinateRecordsView(self) 

561 

562 @abstractmethod 

563 def _record(self, name: str) -> Optional[DimensionRecord]: 

564 """Protected implementation hook that backs the ``records`` attribute. 

565 

566 Parameters 

567 ---------- 

568 name : `str` 

569 The name of a `DimensionElement`, guaranteed to be in 

570 ``self.graph.elements.names``. 

571 

572 Returns 

573 ------- 

574 record : `DimensionRecord` or `None` 

575 The dimension record for the given element identified by this 

576 data ID, or `None` if there is no such record. 

577 """ 

578 raise NotImplementedError() 

579 

580 @property 

581 def region(self) -> Optional[Region]: 

582 """Spatial region associated with this data ID. 

583 

584 (`lsst.sphgeom.Region` or `None`). 

585 

586 This is `None` if and only if ``self.graph.spatial`` is empty. 

587 

588 Accessing this attribute if `hasRecords` returns `False` is a logic 

589 error that may or may not raise an exception, depending on the 

590 implementation and whether assertions are enabled. 

591 """ 

592 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True." 

593 regions = [] 

594 for family in self.graph.spatial: 

595 element = family.choose(self.graph.elements) 

596 record = self._record(element.name) 

597 if record is None or record.region is None: 

598 return None 

599 else: 

600 regions.append(record.region) 

601 return _intersectRegions(*regions) 

602 

603 @property 

604 def timespan(self) -> Optional[Timespan]: 

605 """Temporal interval associated with this data ID. 

606 

607 (`Timespan` or `None`). 

608 

609 This is `None` if and only if ``self.graph.timespan`` is empty. 

610 

611 Accessing this attribute if `hasRecords` returns `False` is a logic 

612 error that may or may not raise an exception, depending on the 

613 implementation and whether assertions are enabled. 

614 """ 

615 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True." 

616 timespans = [] 

617 for family in self.graph.temporal: 

618 element = family.choose(self.graph.elements) 

619 record = self._record(element.name) 

620 # DimensionRecord subclasses for temporal elements always have 

621 # .timespan, but they're dynamic so this can't be type-checked. 

622 if record is None or record.timespan is None: 

623 return None 

624 else: 

625 timespans.append(record.timespan) 

626 if not timespans: 

627 return None 

628 elif len(timespans) == 1: 

629 return timespans[0] 

630 else: 

631 return Timespan.intersection(*timespans) 

632 

633 def pack(self, name: str, *, returnMaxBits: bool = False) -> Union[Tuple[int, int], int]: 

634 """Pack this data ID into an integer. 

635 

636 Parameters 

637 ---------- 

638 name : `str` 

639 Name of the `DimensionPacker` algorithm (as defined in the 

640 dimension configuration). 

641 returnMaxBits : `bool`, optional 

642 If `True` (`False` is default), return the maximum number of 

643 nonzero bits in the returned integer across all data IDs. 

644 

645 Returns 

646 ------- 

647 packed : `int` 

648 Integer ID. This ID is unique only across data IDs that have 

649 the same values for the packer's "fixed" dimensions. 

650 maxBits : `int`, optional 

651 Maximum number of nonzero bits in ``packed``. Not returned unless 

652 ``returnMaxBits`` is `True`. 

653 

654 Notes 

655 ----- 

656 Accessing this attribute if `hasRecords` returns `False` is a logic 

657 error that may or may not raise an exception, depending on the 

658 implementation and whether assertions are enabled. 

659 """ 

660 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." 

661 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) 

662 

663 def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate: 

664 """Convert this class to a simple python type. 

665 

666 This is suitable for serialization. 

667 

668 Parameters 

669 ---------- 

670 minimal : `bool`, optional 

671 Use minimal serialization. If set the records will not be attached. 

672 

673 Returns 

674 ------- 

675 simple : `SerializedDataCoordinate` 

676 The object converted to simple form. 

677 """ 

678 # Convert to a dict form 

679 if self.hasFull(): 

680 dataId = self.full.byName() 

681 else: 

682 dataId = self.byName() 

683 records: Optional[Dict[str, SerializedDimensionRecord]] 

684 if not minimal and self.hasRecords(): 

685 records = {k: v.to_simple() for k, v in self.records.byName().items() if v is not None} 

686 else: 

687 records = None 

688 

689 return SerializedDataCoordinate(dataId=dataId, records=records) 

690 

691 @classmethod 

692 def from_simple( 

693 cls, 

694 simple: SerializedDataCoordinate, 

695 universe: Optional[DimensionUniverse] = None, 

696 registry: Optional[Registry] = None, 

697 ) -> DataCoordinate: 

698 """Construct a new object from the simplified form. 

699 

700 The data is assumed to be of the form returned from the `to_simple` 

701 method. 

702 

703 Parameters 

704 ---------- 

705 simple : `dict` of [`str`, `Any`] 

706 The `dict` returned by `to_simple()`. 

707 universe : `DimensionUniverse` 

708 The special graph of all known dimensions. 

709 registry : `lsst.daf.butler.Registry`, optional 

710 Registry from which a universe can be extracted. Can be `None` 

711 if universe is provided explicitly. 

712 

713 Returns 

714 ------- 

715 dataId : `DataCoordinate` 

716 Newly-constructed object. 

717 """ 

718 if universe is None and registry is None: 

719 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

720 if universe is None and registry is not None: 

721 universe = registry.dimensions 

722 if universe is None: 

723 # this is for mypy 

724 raise ValueError("Unable to determine a usable universe") 

725 

726 dataId = cls.standardize(simple.dataId, universe=universe) 

727 if simple.records: 

728 dataId = dataId.expanded( 

729 {k: DimensionRecord.from_simple(v, universe=universe) for k, v in simple.records.items()} 

730 ) 

731 return dataId 

732 

733 to_json = to_json_pydantic 

734 from_json = classmethod(from_json_pydantic) 

735 

736 

737DataId = Union[DataCoordinate, Mapping[str, Any]] 

738"""A type-annotation alias for signatures that accept both informal data ID 

739dictionaries and validated `DataCoordinate` instances. 

740""" 

741 

742 

743class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]): 

744 """View class for `DataCoordinate.full`. 

745 

746 Provides the default implementation for 

747 `DataCoordinate.full`. 

748 

749 Parameters 

750 ---------- 

751 target : `DataCoordinate` 

752 The `DataCoordinate` instance this object provides a view of. 

753 """ 

754 

755 def __init__(self, target: DataCoordinate): 

756 self._target = target 

757 

758 __slots__ = ("_target",) 

759 

760 def __repr__(self) -> str: 

761 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names] 

762 return "{{{}}}".format(", ".join(terms)) 

763 

764 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

765 return self._target[key] 

766 

767 def __iter__(self) -> Iterator[Dimension]: 

768 return iter(self.keys()) 

769 

770 def __len__(self) -> int: 

771 return len(self.keys()) 

772 

773 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

774 return self._target.graph.dimensions 

775 

776 @property 

777 def names(self) -> AbstractSet[str]: 

778 # Docstring inherited from `NamedKeyMapping`. 

779 return self.keys().names 

780 

781 

782class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, Optional[DimensionRecord]]): 

783 """View class for `DataCoordinate.records`. 

784 

785 Provides the default implementation for 

786 `DataCoordinate.records`. 

787 

788 Parameters 

789 ---------- 

790 target : `DataCoordinate` 

791 The `DataCoordinate` instance this object provides a view of. 

792 """ 

793 

794 def __init__(self, target: DataCoordinate): 

795 self._target = target 

796 

797 __slots__ = ("_target",) 

798 

799 def __repr__(self) -> str: 

800 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names] 

801 return "{{{}}}".format(", ".join(terms)) 

802 

803 def __str__(self) -> str: 

804 return "\n".join(str(v) for v in self.values()) 

805 

806 def __getitem__(self, key: Union[DimensionElement, str]) -> Optional[DimensionRecord]: 

807 if isinstance(key, DimensionElement): 

808 key = key.name 

809 return self._target._record(key) 

810 

811 def __iter__(self) -> Iterator[DimensionElement]: 

812 return iter(self.keys()) 

813 

814 def __len__(self) -> int: 

815 return len(self.keys()) 

816 

817 def keys(self) -> NamedValueAbstractSet[DimensionElement]: # type: ignore 

818 return self._target.graph.elements 

819 

820 @property 

821 def names(self) -> AbstractSet[str]: 

822 # Docstring inherited from `NamedKeyMapping`. 

823 return self.keys().names 

824 

825 

826class _BasicTupleDataCoordinate(DataCoordinate): 

827 """Standard implementation of `DataCoordinate`. 

828 

829 Backed by a tuple of values. 

830 

831 This class should only be accessed outside this module via the 

832 `DataCoordinate` interface, and should only be constructed via the static 

833 methods there. 

834 

835 Parameters 

836 ---------- 

837 graph : `DimensionGraph` 

838 The dimensions to be identified. 

839 values : `tuple` [ `int` or `str` ] 

840 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May 

841 include values for just required dimensions (which always come first) 

842 or all dimensions. 

843 """ 

844 

845 def __init__(self, graph: DimensionGraph, values: Tuple[DataIdValue, ...]): 

846 self._graph = graph 

847 self._values = values 

848 

849 __slots__ = ("_graph", "_values") 

850 

851 @property 

852 def graph(self) -> DimensionGraph: 

853 # Docstring inherited from DataCoordinate. 

854 return self._graph 

855 

856 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

857 # Docstring inherited from DataCoordinate. 

858 if isinstance(key, Dimension): 

859 key = key.name 

860 index = self._graph._dataCoordinateIndices[key] 

861 try: 

862 return self._values[index] 

863 except IndexError: 

864 # Caller asked for an implied dimension, but this object only has 

865 # values for the required ones. 

866 raise KeyError(key) from None 

867 

868 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

869 # Docstring inherited from DataCoordinate. 

870 if self._graph == graph: 

871 return self 

872 elif self.hasFull() or self._graph.required >= graph.dimensions: 

873 return _BasicTupleDataCoordinate( 

874 graph, 

875 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

876 ) 

877 else: 

878 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names)) 

879 

880 def union(self, other: DataCoordinate) -> DataCoordinate: 

881 # Docstring inherited from DataCoordinate. 

882 graph = self.graph.union(other.graph) 

883 # See if one or both input data IDs is already what we want to return; 

884 # if so, return the most complete one we have. 

885 if other.graph == graph: 

886 if self.graph == graph: 

887 # Input data IDs have the same graph (which is also the result 

888 # graph), but may not have the same content. 

889 # other might have records; self does not, so try other first. 

890 # If it at least has full values, it's no worse than self. 

891 if other.hasFull(): 

892 return other 

893 else: 

894 return self 

895 elif other.hasFull(): 

896 return other 

897 # There's some chance that neither self nor other has full values, 

898 # but together provide enough to the union to. Let the general 

899 # case below handle that. 

900 elif self.graph == graph: 

901 # No chance at returning records. If self has full values, it's 

902 # the best we can do. 

903 if self.hasFull(): 

904 return self 

905 # General case with actual merging of dictionaries. 

906 values = self.full.byName() if self.hasFull() else self.byName() 

907 values.update(other.full.byName() if other.hasFull() else other.byName()) 

908 return DataCoordinate.standardize(values, graph=graph) 

909 

910 def expanded( 

911 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

912 ) -> DataCoordinate: 

913 # Docstring inherited from DataCoordinate 

914 values = self._values 

915 if not self.hasFull(): 

916 # Extract a complete values tuple from the attributes of the given 

917 # records. It's possible for these to be inconsistent with 

918 # self._values (which is a serious problem, of course), but we've 

919 # documented this as a no-checking API. 

920 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied) 

921 return _ExpandedTupleDataCoordinate(self._graph, values, records) 

922 

923 def hasFull(self) -> bool: 

924 # Docstring inherited from DataCoordinate. 

925 return len(self._values) == len(self._graph._dataCoordinateIndices) 

926 

927 def hasRecords(self) -> bool: 

928 # Docstring inherited from DataCoordinate. 

929 return False 

930 

931 def _record(self, name: str) -> Optional[DimensionRecord]: 

932 # Docstring inherited from DataCoordinate. 

933 assert False 

934 

935 

936class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate): 

937 """A `DataCoordinate` implementation that can hold `DimensionRecord`. 

938 

939 This class should only be accessed outside this module via the 

940 `DataCoordinate` interface, and should only be constructed via calls to 

941 `DataCoordinate.expanded`. 

942 

943 Parameters 

944 ---------- 

945 graph : `DimensionGraph` 

946 The dimensions to be identified. 

947 values : `tuple` [ `int` or `str` ] 

948 Data ID values, ordered to match ``graph._dataCoordinateIndices``. 

949 May include values for just required dimensions (which always come 

950 first) or all dimensions. 

951 records : `Mapping` [ `str`, `DimensionRecord` or `None` ] 

952 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

953 `Mapping` with `str` (`DimensionElement` name) keys and 

954 `DimensionRecord` values. Keys must cover all elements in 

955 ``self.graph.elements``. Values may be `None`, but only to reflect 

956 actual NULL values in the database, not just records that have not 

957 been fetched. 

958 """ 

959 

960 def __init__( 

961 self, 

962 graph: DimensionGraph, 

963 values: Tuple[DataIdValue, ...], 

964 records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]], 

965 ): 

966 super().__init__(graph, values) 

967 assert super().hasFull(), "This implementation requires full dimension records." 

968 self._records = records 

969 

970 __slots__ = ("_records",) 

971 

972 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

973 # Docstring inherited from DataCoordinate. 

974 if self._graph == graph: 

975 return self 

976 return _ExpandedTupleDataCoordinate( 

977 graph, tuple(self[k] for k in graph._dataCoordinateIndices.keys()), records=self._records 

978 ) 

979 

980 def expanded( 

981 self, records: NameLookupMapping[DimensionElement, Optional[DimensionRecord]] 

982 ) -> DataCoordinate: 

983 # Docstring inherited from DataCoordinate. 

984 return self 

985 

986 def union(self, other: DataCoordinate) -> DataCoordinate: 

987 # Docstring inherited from DataCoordinate. 

988 graph = self.graph.union(other.graph) 

989 # See if one or both input data IDs is already what we want to return; 

990 # if so, return the most complete one we have. 

991 if self.graph == graph: 

992 # self has records, so even if other is also a valid result, it's 

993 # no better. 

994 return self 

995 if other.graph == graph: 

996 # If other has full values, and self does not identify some of 

997 # those, it's the base we can do. It may have records, too. 

998 if other.hasFull(): 

999 return other 

1000 # If other does not have full values, there's a chance self may 

1001 # provide the values needed to complete it. For example, self 

1002 # could be {band} while other could be 

1003 # {instrument, physical_filter, band}, with band unknown. 

1004 # General case with actual merging of dictionaries. 

1005 values = self.full.byName() 

1006 values.update(other.full.byName() if other.hasFull() else other.byName()) 

1007 basic = DataCoordinate.standardize(values, graph=graph) 

1008 # See if we can add records. 

1009 if self.hasRecords() and other.hasRecords(): 

1010 # Sometimes the elements of a union of graphs can contain elements 

1011 # that weren't in either input graph (because graph unions are only 

1012 # on dimensions). e.g. {visit} | {detector} brings along 

1013 # visit_detector_region. 

1014 elements = set(graph.elements.names) 

1015 elements -= self.graph.elements.names 

1016 elements -= other.graph.elements.names 

1017 if not elements: 

1018 records = NamedKeyDict[DimensionElement, Optional[DimensionRecord]](self.records) 

1019 records.update(other.records) 

1020 return basic.expanded(records.freeze()) 

1021 return basic 

1022 

1023 def hasFull(self) -> bool: 

1024 # Docstring inherited from DataCoordinate. 

1025 return True 

1026 

1027 def hasRecords(self) -> bool: 

1028 # Docstring inherited from DataCoordinate. 

1029 return True 

1030 

1031 def _record(self, name: str) -> Optional[DimensionRecord]: 

1032 # Docstring inherited from DataCoordinate. 

1033 return self._records[name]