Coverage for python/lsst/daf/butler/core/dimensions/_coordinate.py: 27%

346 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-14 09:11 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22# 

23# Design notes for this module are in 

24# doc/lsst.daf.butler/dev/dataCoordinate.py. 

25# 

26 

27from __future__ import annotations 

28 

29__all__ = ("DataCoordinate", "DataId", "DataIdKey", "DataIdValue", "SerializedDataCoordinate") 

30 

31import numbers 

32from abc import abstractmethod 

33from collections.abc import Iterator, Mapping, Set 

34from typing import TYPE_CHECKING, Any, ClassVar, Literal, overload 

35 

36from deprecated.sphinx import deprecated 

37from lsst.sphgeom import IntersectionRegion, Region 

38from pydantic import BaseModel 

39 

40from ..json import from_json_pydantic, to_json_pydantic 

41from ..named import NamedKeyDict, NamedKeyMapping, NamedValueAbstractSet, NameLookupMapping 

42from ..timespan import Timespan 

43from ._elements import Dimension, DimensionElement 

44from ._graph import DimensionGraph 

45from ._records import DimensionRecord, SerializedDimensionRecord 

46 

47if TYPE_CHECKING: # Imports needed only for type annotations; may be circular. 

48 from ...registry import Registry 

49 from ._universe import DimensionUniverse 

50 

51DataIdKey = str | Dimension 

52"""Type annotation alias for the keys that can be used to index a 

53DataCoordinate. 

54""" 

55 

56# Pydantic will cast int to str if str is first in the Union. 

57DataIdValue = int | str | None 

58"""Type annotation alias for the values that can be present in a 

59DataCoordinate or other data ID. 

60""" 

61 

62 

63class SerializedDataCoordinate(BaseModel): 

64 """Simplified model for serializing a `DataCoordinate`.""" 

65 

66 dataId: dict[str, DataIdValue] 

67 records: dict[str, SerializedDimensionRecord] | None = None 

68 

69 @classmethod 

70 def direct(cls, *, dataId: dict[str, DataIdValue], records: dict[str, dict]) -> SerializedDataCoordinate: 

71 """Construct a `SerializedDataCoordinate` directly without validators. 

72 

73 This differs from the pydantic "construct" method in that the arguments 

74 are explicitly what the model requires, and it will recurse through 

75 members, constructing them from their corresponding `direct` methods. 

76 

77 This method should only be called when the inputs are trusted. 

78 """ 

79 node = SerializedDataCoordinate.__new__(cls) 

80 setter = object.__setattr__ 

81 setter(node, "dataId", dataId) 

82 setter( 

83 node, 

84 "records", 

85 records 

86 if records is None 

87 else {k: SerializedDimensionRecord.direct(**v) for k, v in records.items()}, 

88 ) 

89 setter(node, "__fields_set__", {"dataId", "records"}) 

90 return node 

91 

92 

93def _intersectRegions(*args: Region) -> Region | None: 

94 """Return the intersection of several regions. 

95 

96 For internal use by `ExpandedDataCoordinate` only. 

97 

98 If no regions are provided, returns `None`. 

99 """ 

100 if len(args) == 0: 

101 return None 

102 else: 

103 result = args[0] 

104 for n in range(1, len(args)): 

105 result = IntersectionRegion(result, args[n]) 

106 return result 

107 

108 

109class DataCoordinate(NamedKeyMapping[Dimension, DataIdValue]): 

110 """Data ID dictionary. 

111 

112 An immutable data ID dictionary that guarantees that its key-value pairs 

113 identify at least all required dimensions in a `DimensionGraph`. 

114 

115 `DataCoordinate` itself is an ABC, but provides `staticmethod` factory 

116 functions for private concrete implementations that should be sufficient 

117 for most purposes. `standardize` is the most flexible and safe of these; 

118 the others (`makeEmpty`, `fromRequiredValues`, and `fromFullValues`) are 

119 more specialized and perform little or no checking of inputs. 

120 

121 Notes 

122 ----- 

123 Like any data ID class, `DataCoordinate` behaves like a dictionary, but 

124 with some subtleties: 

125 

126 - Both `Dimension` instances and `str` names thereof may be used as keys 

127 in lookup operations, but iteration (and `keys`) will yield `Dimension` 

128 instances. The `names` property can be used to obtain the corresponding 

129 `str` names. 

130 

131 - Lookups for implied dimensions (those in ``self.graph.implied``) are 

132 supported if and only if `hasFull` returns `True`, and are never 

133 included in iteration or `keys`. The `full` property may be used to 

134 obtain a mapping whose keys do include implied dimensions. 

135 

136 - Equality comparison with other mappings is supported, but it always 

137 considers only required dimensions (as well as requiring both operands 

138 to identify the same dimensions). This is not quite consistent with the 

139 way mappings usually work - normally differing keys imply unequal 

140 mappings - but it makes sense in this context because data IDs with the 

141 same values for required dimensions but different values for implied 

142 dimensions represent a serious problem with the data that 

143 `DataCoordinate` cannot generally recognize on its own, and a data ID 

144 that knows implied dimension values should still be able to compare as 

145 equal to one that does not. This is of course not the way comparisons 

146 between simple `dict` data IDs work, and hence using a `DataCoordinate` 

147 instance for at least one operand in any data ID comparison is strongly 

148 recommended. 

149 

150 See Also 

151 -------- 

152 :ref:`lsst.daf.butler-dimensions_data_ids` 

153 """ 

154 

155 __slots__ = () 

156 

157 _serializedType = SerializedDataCoordinate 

158 

159 @staticmethod 

160 def standardize( 

161 mapping: NameLookupMapping[Dimension, DataIdValue] | None = None, 

162 *, 

163 graph: DimensionGraph | None = None, 

164 universe: DimensionUniverse | None = None, 

165 defaults: DataCoordinate | None = None, 

166 **kwargs: Any, 

167 ) -> DataCoordinate: 

168 """Standardize the supplied dataId. 

169 

170 Adapts an arbitrary mapping and/or additional arguments into a true 

171 `DataCoordinate`, or augment an existing one. 

172 

173 Parameters 

174 ---------- 

175 mapping : `~collections.abc.Mapping`, optional 

176 An informal data ID that maps dimensions or dimension names to 

177 their primary key values (may also be a true `DataCoordinate`). 

178 graph : `DimensionGraph` 

179 The dimensions to be identified by the new `DataCoordinate`. 

180 If not provided, will be inferred from the keys of ``mapping`` and 

181 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

182 is already a `DataCoordinate`. 

183 universe : `DimensionUniverse` 

184 All known dimensions and their relationships; used to expand 

185 and validate dependencies when ``graph`` is not provided. 

186 defaults : `DataCoordinate`, optional 

187 Default dimension key-value pairs to use when needed. These are 

188 never used to infer ``graph``, and are ignored if a different value 

189 is provided for the same key in ``mapping`` or `**kwargs``. 

190 **kwargs 

191 Additional keyword arguments are treated like additional key-value 

192 pairs in ``mapping``. 

193 

194 Returns 

195 ------- 

196 coordinate : `DataCoordinate` 

197 A validated `DataCoordinate` instance. 

198 

199 Raises 

200 ------ 

201 TypeError 

202 Raised if the set of optional arguments provided is not supported. 

203 KeyError 

204 Raised if a key-value pair for a required dimension is missing. 

205 """ 

206 d: dict[str, DataIdValue] = {} 

207 if isinstance(mapping, DataCoordinate): 

208 if graph is None: 

209 if not kwargs: 

210 # Already standardized to exactly what we want. 

211 return mapping 

212 elif kwargs.keys().isdisjoint(graph.dimensions.names): 

213 # User provided kwargs, but told us not to use them by 

214 # passing in dimensions that are disjoint from those kwargs. 

215 # This is not necessarily user error - it's a useful pattern 

216 # to pass in all of the key-value pairs you have and let the 

217 # code here pull out only what it needs. 

218 return mapping.subset(graph) 

219 assert universe is None or universe == mapping.universe 

220 universe = mapping.universe 

221 d.update((name, mapping[name]) for name in mapping.graph.required.names) 

222 if mapping.hasFull(): 

223 d.update((name, mapping[name]) for name in mapping.graph.implied.names) 

224 elif isinstance(mapping, NamedKeyMapping): 

225 d.update(mapping.byName()) 

226 elif mapping is not None: 

227 d.update(mapping) 

228 d.update(kwargs) 

229 if graph is None: 

230 if defaults is not None: 

231 universe = defaults.universe 

232 elif universe is None: 

233 raise TypeError("universe must be provided if graph is not.") 

234 graph = DimensionGraph(universe, names=d.keys()) 

235 if not graph.dimensions: 

236 return DataCoordinate.makeEmpty(graph.universe) 

237 if defaults is not None: 

238 if defaults.hasFull(): 

239 for k, v in defaults.full.items(): 

240 d.setdefault(k.name, v) 

241 else: 

242 for k, v in defaults.items(): 

243 d.setdefault(k.name, v) 

244 if d.keys() >= graph.dimensions.names: 

245 values = tuple(d[name] for name in graph._dataCoordinateIndices.keys()) 

246 else: 

247 try: 

248 values = tuple(d[name] for name in graph.required.names) 

249 except KeyError as err: 

250 raise KeyError(f"No value in data ID ({mapping}) for required dimension {err}.") from err 

251 # Some backends cannot handle numpy.int64 type which is a subclass of 

252 # numbers.Integral; convert that to int. 

253 values = tuple( 

254 int(val) if isinstance(val, numbers.Integral) else val for val in values # type: ignore 

255 ) 

256 return _BasicTupleDataCoordinate(graph, values) 

257 

258 @staticmethod 

259 def makeEmpty(universe: DimensionUniverse) -> DataCoordinate: 

260 """Return an empty `DataCoordinate`. 

261 

262 It identifies the null set of dimensions. 

263 

264 Parameters 

265 ---------- 

266 universe : `DimensionUniverse` 

267 Universe to which this null dimension set belongs. 

268 

269 Returns 

270 ------- 

271 dataId : `DataCoordinate` 

272 A data ID object that identifies no dimensions. `hasFull` and 

273 `hasRecords` are guaranteed to return `True`, because both `full` 

274 and `records` are just empty mappings. 

275 """ 

276 return _ExpandedTupleDataCoordinate(universe.empty, (), {}) 

277 

278 @staticmethod 

279 def fromRequiredValues(graph: DimensionGraph, values: tuple[DataIdValue, ...]) -> DataCoordinate: 

280 """Construct a `DataCoordinate` from required dimension values. 

281 

282 This is a low-level interface with at most assertion-level checking of 

283 inputs. Most callers should use `standardize` instead. 

284 

285 Parameters 

286 ---------- 

287 graph : `DimensionGraph` 

288 Dimensions this data ID will identify. 

289 values : `tuple` [ `int` or `str` ] 

290 Tuple of primary key values corresponding to ``graph.required``, 

291 in that order. 

292 

293 Returns 

294 ------- 

295 dataId : `DataCoordinate` 

296 A data ID object that identifies the given dimensions. 

297 ``dataId.hasFull()`` will return `True` if and only if 

298 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

299 return `True`. 

300 """ 

301 assert len(graph.required) == len( 

302 values 

303 ), f"Inconsistency between dimensions {graph.required} and required values {values}." 

304 return _BasicTupleDataCoordinate(graph, values) 

305 

306 @staticmethod 

307 def fromFullValues(graph: DimensionGraph, values: tuple[DataIdValue, ...]) -> DataCoordinate: 

308 """Construct a `DataCoordinate` from all dimension values. 

309 

310 This is a low-level interface with at most assertion-level checking of 

311 inputs. Most callers should use `standardize` instead. 

312 

313 Parameters 

314 ---------- 

315 graph : `DimensionGraph` 

316 Dimensions this data ID will identify. 

317 values : `tuple` [ `int` or `str` ] 

318 Tuple of primary key values corresponding to 

319 ``itertools.chain(graph.required, graph.implied)``, in that order. 

320 Note that this is _not_ the same order as ``graph.dimensions``, 

321 though these contain the same elements. 

322 

323 Returns 

324 ------- 

325 dataId : `DataCoordinate` 

326 A data ID object that identifies the given dimensions. 

327 ``dataId.hasFull()`` will return `True` if and only if 

328 ``graph.implied`` is empty, and ``dataId.hasRecords()`` will never 

329 return `True`. 

330 """ 

331 assert len(graph.dimensions) == len( 

332 values 

333 ), f"Inconsistency between dimensions {graph.dimensions} and full values {values}." 

334 return _BasicTupleDataCoordinate(graph, values) 

335 

336 def __hash__(self) -> int: 

337 return hash((self.graph,) + tuple(self[d.name] for d in self.graph.required)) 

338 

339 def __eq__(self, other: Any) -> bool: 

340 if not isinstance(other, DataCoordinate): 

341 other = DataCoordinate.standardize(other, universe=self.universe) 

342 return self.graph == other.graph and all(self[d.name] == other[d.name] for d in self.graph.required) 

343 

344 def __repr__(self) -> str: 

345 # We can't make repr yield something that could be exec'd here without 

346 # printing out the whole DimensionUniverse the graph is derived from. 

347 # So we print something that mostly looks like a dict, but doesn't 

348 # quote its keys: that's both more compact and something that can't 

349 # be mistaken for an actual dict or something that could be exec'd. 

350 terms = [f"{d}: {self[d]!r}" for d in self.graph.required.names] 

351 if self.hasFull() and self.graph.required != self.graph.dimensions: 

352 terms.append("...") 

353 return "{{{}}}".format(", ".join(terms)) 

354 

355 def __lt__(self, other: Any) -> bool: 

356 # Allow DataCoordinate to be sorted 

357 if not isinstance(other, type(self)): 

358 return NotImplemented 

359 # Form tuple of tuples for each DataCoordinate: 

360 # Unlike repr() we only use required keys here to ensure that 

361 # __eq__ can not be true simultaneously with __lt__ being true. 

362 self_kv = tuple(self.items()) 

363 other_kv = tuple(other.items()) 

364 

365 return self_kv < other_kv 

366 

367 def __iter__(self) -> Iterator[Dimension]: 

368 return iter(self.keys()) 

369 

370 def __len__(self) -> int: 

371 return len(self.keys()) 

372 

373 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

374 return self.graph.required 

375 

376 @property 

377 def names(self) -> Set[str]: 

378 """Names of the required dimensions identified by this data ID. 

379 

380 They are returned in the same order as `keys` 

381 (`collections.abc.Set` [ `str` ]). 

382 """ 

383 return self.keys().names 

384 

385 @abstractmethod 

386 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

387 """Return a `DataCoordinate` whose graph is a subset of ``self.graph``. 

388 

389 Parameters 

390 ---------- 

391 graph : `DimensionGraph` 

392 The dimensions identified by the returned `DataCoordinate`. 

393 

394 Returns 

395 ------- 

396 coordinate : `DataCoordinate` 

397 A `DataCoordinate` instance that identifies only the given 

398 dimensions. May be ``self`` if ``graph == self.graph``. 

399 

400 Raises 

401 ------ 

402 KeyError 

403 Raised if the primary key value for one or more required dimensions 

404 is unknown. This may happen if ``graph.issubset(self.graph)`` is 

405 `False`, or even if ``graph.issubset(self.graph)`` is `True`, if 

406 ``self.hasFull()`` is `False` and 

407 ``graph.required.issubset(self.graph.required)`` is `False`. As 

408 an example of the latter case, consider trying to go from a data ID 

409 with dimensions {instrument, physical_filter, band} to 

410 just {instrument, band}; band is implied by 

411 physical_filter and hence would have no value in the original data 

412 ID if ``self.hasFull()`` is `False`. 

413 

414 Notes 

415 ----- 

416 If `hasFull` and `hasRecords` return `True` on ``self``, they will 

417 return `True` (respectively) on the returned `DataCoordinate` as well. 

418 The converse does not hold. 

419 """ 

420 raise NotImplementedError() 

421 

422 @abstractmethod 

423 def union(self, other: DataCoordinate) -> DataCoordinate: 

424 """Combine two data IDs. 

425 

426 Yields a new one that identifies all dimensions that either of them 

427 identify. 

428 

429 Parameters 

430 ---------- 

431 other : `DataCoordinate` 

432 Data ID to combine with ``self``. 

433 

434 Returns 

435 ------- 

436 unioned : `DataCoordinate` 

437 A `DataCoordinate` instance that satisfies 

438 ``unioned.graph == self.graph.union(other.graph)``. Will preserve 

439 ``hasFull`` and ``hasRecords`` whenever possible. 

440 

441 Notes 

442 ----- 

443 No checking for consistency is performed on values for keys that 

444 ``self`` and ``other`` have in common, and which value is included in 

445 the returned data ID is not specified. 

446 """ 

447 raise NotImplementedError() 

448 

449 @abstractmethod 

450 def expanded( 

451 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None] 

452 ) -> DataCoordinate: 

453 """Return a `DataCoordinate` that holds the given records. 

454 

455 Guarantees that `hasRecords` returns `True`. 

456 

457 This is a low-level interface with at most assertion-level checking of 

458 inputs. Most callers should use `Registry.expandDataId` instead. 

459 

460 Parameters 

461 ---------- 

462 records : `~collections.abc.Mapping` [ `str`, `DimensionRecord` or \ 

463 `None` ] 

464 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

465 `~collections.abc.Mapping` with `str` (`DimensionElement` name) 

466 keys and `DimensionRecord` values. Keys must cover all elements in 

467 ``self.graph.elements``. Values may be `None`, but only to reflect 

468 actual NULL values in the database, not just records that have not 

469 been fetched. 

470 """ 

471 raise NotImplementedError() 

472 

473 @property 

474 def universe(self) -> DimensionUniverse: 

475 """Universe that defines all known compatible dimensions. 

476 

477 The univers will be compatible with this coordinate 

478 (`DimensionUniverse`). 

479 """ 

480 return self.graph.universe 

481 

482 @property 

483 @abstractmethod 

484 def graph(self) -> DimensionGraph: 

485 """Dimensions identified by this data ID (`DimensionGraph`). 

486 

487 Note that values are only required to be present for dimensions in 

488 ``self.graph.required``; all others may be retrieved (from a 

489 `Registry`) given these. 

490 """ 

491 raise NotImplementedError() 

492 

493 @abstractmethod 

494 def hasFull(self) -> bool: 

495 """Whether this data ID contains implied and required values. 

496 

497 Returns 

498 ------- 

499 state : `bool` 

500 If `True`, `__getitem__`, `get`, and `__contains__` (but not 

501 `keys`!) will act as though the mapping includes key-value pairs 

502 for implied dimensions, and the `full` property may be used. If 

503 `False`, these operations only include key-value pairs for required 

504 dimensions, and accessing `full` is an error. Always `True` if 

505 there are no implied dimensions. 

506 """ 

507 raise NotImplementedError() 

508 

509 @property 

510 def full(self) -> NamedKeyMapping[Dimension, DataIdValue]: 

511 """Return mapping for all dimensions in ``self.graph``. 

512 

513 The mapping includes key-value pairs for all dimensions in 

514 ``self.graph``, including implied (`NamedKeyMapping`). 

515 

516 Accessing this attribute if `hasFull` returns `False` is a logic error 

517 that may raise an exception of unspecified type either immediately or 

518 when implied keys are accessed via the returned mapping, depending on 

519 the implementation and whether assertions are enabled. 

520 """ 

521 assert self.hasFull(), "full may only be accessed if hasFull() returns True." 

522 return _DataCoordinateFullView(self) 

523 

524 @abstractmethod 

525 def hasRecords(self) -> bool: 

526 """Whether this data ID contains records. 

527 

528 These are the records for all of the dimension elements it identifies. 

529 

530 Returns 

531 ------- 

532 state : `bool` 

533 If `True`, the following attributes may be accessed: 

534 

535 - `records` 

536 - `region` 

537 - `timespan` 

538 - `pack` 

539 

540 If `False`, accessing any of these is considered a logic error. 

541 """ 

542 raise NotImplementedError() 

543 

544 @property 

545 def records(self) -> NamedKeyMapping[DimensionElement, DimensionRecord | None]: 

546 """Return the records. 

547 

548 Returns a mapping that contains `DimensionRecord` objects for all 

549 elements identified by this data ID (`NamedKeyMapping`). 

550 

551 The values of this mapping may be `None` if and only if there is no 

552 record for that element with these dimensions in the database (which 

553 means some foreign key field must have a NULL value). 

554 

555 Accessing this attribute if `hasRecords` returns `False` is a logic 

556 error that may raise an exception of unspecified type either 

557 immediately or when the returned mapping is used, depending on the 

558 implementation and whether assertions are enabled. 

559 """ 

560 assert self.hasRecords(), "records may only be accessed if hasRecords() returns True." 

561 return _DataCoordinateRecordsView(self) 

562 

563 @abstractmethod 

564 def _record(self, name: str) -> DimensionRecord | None: 

565 """Protected implementation hook that backs the ``records`` attribute. 

566 

567 Parameters 

568 ---------- 

569 name : `str` 

570 The name of a `DimensionElement`, guaranteed to be in 

571 ``self.graph.elements.names``. 

572 

573 Returns 

574 ------- 

575 record : `DimensionRecord` or `None` 

576 The dimension record for the given element identified by this 

577 data ID, or `None` if there is no such record. 

578 """ 

579 raise NotImplementedError() 

580 

581 @property 

582 def region(self) -> Region | None: 

583 """Spatial region associated with this data ID. 

584 

585 (`lsst.sphgeom.Region` or `None`). 

586 

587 This is `None` if and only if ``self.graph.spatial`` is empty. 

588 

589 Accessing this attribute if `hasRecords` returns `False` is a logic 

590 error that may or may not raise an exception, depending on the 

591 implementation and whether assertions are enabled. 

592 """ 

593 assert self.hasRecords(), "region may only be accessed if hasRecords() returns True." 

594 regions = [] 

595 for family in self.graph.spatial: 

596 element = family.choose(self.graph.elements) 

597 record = self._record(element.name) 

598 if record is None or record.region is None: 

599 return None 

600 else: 

601 regions.append(record.region) 

602 return _intersectRegions(*regions) 

603 

604 @property 

605 def timespan(self) -> Timespan | None: 

606 """Temporal interval associated with this data ID. 

607 

608 (`Timespan` or `None`). 

609 

610 This is `None` if and only if ``self.graph.timespan`` is empty. 

611 

612 Accessing this attribute if `hasRecords` returns `False` is a logic 

613 error that may or may not raise an exception, depending on the 

614 implementation and whether assertions are enabled. 

615 """ 

616 assert self.hasRecords(), "timespan may only be accessed if hasRecords() returns True." 

617 timespans = [] 

618 for family in self.graph.temporal: 

619 element = family.choose(self.graph.elements) 

620 record = self._record(element.name) 

621 # DimensionRecord subclasses for temporal elements always have 

622 # .timespan, but they're dynamic so this can't be type-checked. 

623 if record is None or record.timespan is None: 

624 return None 

625 else: 

626 timespans.append(record.timespan) 

627 if not timespans: 

628 return None 

629 elif len(timespans) == 1: 

630 return timespans[0] 

631 else: 

632 return Timespan.intersection(*timespans) 

633 

634 @overload 

635 def pack(self, name: str, *, returnMaxBits: Literal[True]) -> tuple[int, int]: 

636 ... 

637 

638 @overload 

639 def pack(self, name: str, *, returnMaxBits: Literal[False]) -> int: 

640 ... 

641 

642 # TODO: Remove this method and its overloads above on DM-38687. 

643 @deprecated( 

644 "Deprecated in favor of configurable dimension packers. Will be removed after v27.", 

645 version="v26", 

646 category=FutureWarning, 

647 ) 

648 def pack(self, name: str, *, returnMaxBits: bool = False) -> tuple[int, int] | int: 

649 """Pack this data ID into an integer. 

650 

651 Parameters 

652 ---------- 

653 name : `str` 

654 Name of the `DimensionPacker` algorithm (as defined in the 

655 dimension configuration). 

656 returnMaxBits : `bool`, optional 

657 If `True` (`False` is default), return the maximum number of 

658 nonzero bits in the returned integer across all data IDs. 

659 

660 Returns 

661 ------- 

662 packed : `int` 

663 Integer ID. This ID is unique only across data IDs that have 

664 the same values for the packer's "fixed" dimensions. 

665 maxBits : `int`, optional 

666 Maximum number of nonzero bits in ``packed``. Not returned unless 

667 ``returnMaxBits`` is `True`. 

668 

669 Notes 

670 ----- 

671 Accessing this attribute if `hasRecords` returns `False` is a logic 

672 error that may or may not raise an exception, depending on the 

673 implementation and whether assertions are enabled. 

674 """ 

675 assert self.hasRecords(), "pack() may only be called if hasRecords() returns True." 

676 return self.universe.makePacker(name, self).pack(self, returnMaxBits=returnMaxBits) 

677 

678 def to_simple(self, minimal: bool = False) -> SerializedDataCoordinate: 

679 """Convert this class to a simple python type. 

680 

681 This is suitable for serialization. 

682 

683 Parameters 

684 ---------- 

685 minimal : `bool`, optional 

686 Use minimal serialization. If set the records will not be attached. 

687 

688 Returns 

689 ------- 

690 simple : `SerializedDataCoordinate` 

691 The object converted to simple form. 

692 """ 

693 # Convert to a dict form 

694 if self.hasFull(): 

695 dataId = self.full.byName() 

696 else: 

697 dataId = self.byName() 

698 records: dict[str, SerializedDimensionRecord] | None 

699 if not minimal and self.hasRecords(): 

700 records = {k: v.to_simple() for k, v in self.records.byName().items() if v is not None} 

701 else: 

702 records = None 

703 

704 return SerializedDataCoordinate(dataId=dataId, records=records) 

705 

706 @classmethod 

707 def from_simple( 

708 cls, 

709 simple: SerializedDataCoordinate, 

710 universe: DimensionUniverse | None = None, 

711 registry: Registry | None = None, 

712 ) -> DataCoordinate: 

713 """Construct a new object from the simplified form. 

714 

715 The data is assumed to be of the form returned from the `to_simple` 

716 method. 

717 

718 Parameters 

719 ---------- 

720 simple : `dict` of [`str`, `Any`] 

721 The `dict` returned by `to_simple()`. 

722 universe : `DimensionUniverse` 

723 The special graph of all known dimensions. 

724 registry : `lsst.daf.butler.Registry`, optional 

725 Registry from which a universe can be extracted. Can be `None` 

726 if universe is provided explicitly. 

727 

728 Returns 

729 ------- 

730 dataId : `DataCoordinate` 

731 Newly-constructed object. 

732 """ 

733 if universe is None and registry is None: 

734 raise ValueError("One of universe or registry is required to convert a dict to a DataCoordinate") 

735 if universe is None and registry is not None: 

736 universe = registry.dimensions 

737 if universe is None: 

738 # this is for mypy 

739 raise ValueError("Unable to determine a usable universe") 

740 

741 dataId = cls.standardize(simple.dataId, universe=universe) 

742 if simple.records: 

743 dataId = dataId.expanded( 

744 {k: DimensionRecord.from_simple(v, universe=universe) for k, v in simple.records.items()} 

745 ) 

746 return dataId 

747 

748 to_json = to_json_pydantic 

749 from_json: ClassVar = classmethod(from_json_pydantic) 

750 

751 

752DataId = DataCoordinate | Mapping[str, Any] 

753"""A type-annotation alias for signatures that accept both informal data ID 

754dictionaries and validated `DataCoordinate` instances. 

755""" 

756 

757 

758class _DataCoordinateFullView(NamedKeyMapping[Dimension, DataIdValue]): 

759 """View class for `DataCoordinate.full`. 

760 

761 Provides the default implementation for 

762 `DataCoordinate.full`. 

763 

764 Parameters 

765 ---------- 

766 target : `DataCoordinate` 

767 The `DataCoordinate` instance this object provides a view of. 

768 """ 

769 

770 def __init__(self, target: DataCoordinate): 

771 self._target = target 

772 

773 __slots__ = ("_target",) 

774 

775 def __repr__(self) -> str: 

776 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.dimensions.names] 

777 return "{{{}}}".format(", ".join(terms)) 

778 

779 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

780 return self._target[key] 

781 

782 def __iter__(self) -> Iterator[Dimension]: 

783 return iter(self.keys()) 

784 

785 def __len__(self) -> int: 

786 return len(self.keys()) 

787 

788 def keys(self) -> NamedValueAbstractSet[Dimension]: # type: ignore 

789 return self._target.graph.dimensions 

790 

791 @property 

792 def names(self) -> Set[str]: 

793 # Docstring inherited from `NamedKeyMapping`. 

794 return self.keys().names 

795 

796 

797class _DataCoordinateRecordsView(NamedKeyMapping[DimensionElement, DimensionRecord | None]): 

798 """View class for `DataCoordinate.records`. 

799 

800 Provides the default implementation for 

801 `DataCoordinate.records`. 

802 

803 Parameters 

804 ---------- 

805 target : `DataCoordinate` 

806 The `DataCoordinate` instance this object provides a view of. 

807 """ 

808 

809 def __init__(self, target: DataCoordinate): 

810 self._target = target 

811 

812 __slots__ = ("_target",) 

813 

814 def __repr__(self) -> str: 

815 terms = [f"{d}: {self[d]!r}" for d in self._target.graph.elements.names] 

816 return "{{{}}}".format(", ".join(terms)) 

817 

818 def __str__(self) -> str: 

819 return "\n".join(str(v) for v in self.values()) 

820 

821 def __getitem__(self, key: DimensionElement | str) -> DimensionRecord | None: 

822 if isinstance(key, DimensionElement): 

823 key = key.name 

824 return self._target._record(key) 

825 

826 def __iter__(self) -> Iterator[DimensionElement]: 

827 return iter(self.keys()) 

828 

829 def __len__(self) -> int: 

830 return len(self.keys()) 

831 

832 def keys(self) -> NamedValueAbstractSet[DimensionElement]: # type: ignore 

833 return self._target.graph.elements 

834 

835 @property 

836 def names(self) -> Set[str]: 

837 # Docstring inherited from `NamedKeyMapping`. 

838 return self.keys().names 

839 

840 

841class _BasicTupleDataCoordinate(DataCoordinate): 

842 """Standard implementation of `DataCoordinate`. 

843 

844 Backed by a tuple of values. 

845 

846 This class should only be accessed outside this module via the 

847 `DataCoordinate` interface, and should only be constructed via the static 

848 methods there. 

849 

850 Parameters 

851 ---------- 

852 graph : `DimensionGraph` 

853 The dimensions to be identified. 

854 values : `tuple` [ `int` or `str` ] 

855 Data ID values, ordered to match ``graph._dataCoordinateIndices``. May 

856 include values for just required dimensions (which always come first) 

857 or all dimensions. 

858 """ 

859 

860 def __init__(self, graph: DimensionGraph, values: tuple[DataIdValue, ...]): 

861 self._graph = graph 

862 self._values = values 

863 

864 __slots__ = ("_graph", "_values") 

865 

866 @property 

867 def graph(self) -> DimensionGraph: 

868 # Docstring inherited from DataCoordinate. 

869 return self._graph 

870 

871 def __getitem__(self, key: DataIdKey) -> DataIdValue: 

872 # Docstring inherited from DataCoordinate. 

873 if isinstance(key, Dimension): 

874 key = key.name 

875 index = self._graph._dataCoordinateIndices[key] 

876 try: 

877 return self._values[index] 

878 except IndexError: 

879 # Caller asked for an implied dimension, but this object only has 

880 # values for the required ones. 

881 raise KeyError(key) from None 

882 

883 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

884 # Docstring inherited from DataCoordinate. 

885 if self._graph == graph: 

886 return self 

887 elif self.hasFull() or self._graph.required >= graph.dimensions: 

888 return _BasicTupleDataCoordinate( 

889 graph, 

890 tuple(self[k] for k in graph._dataCoordinateIndices.keys()), 

891 ) 

892 else: 

893 return _BasicTupleDataCoordinate(graph, tuple(self[k] for k in graph.required.names)) 

894 

895 def union(self, other: DataCoordinate) -> DataCoordinate: 

896 # Docstring inherited from DataCoordinate. 

897 graph = self.graph.union(other.graph) 

898 # See if one or both input data IDs is already what we want to return; 

899 # if so, return the most complete one we have. 

900 if other.graph == graph: 

901 if self.graph == graph: 

902 # Input data IDs have the same graph (which is also the result 

903 # graph), but may not have the same content. 

904 # other might have records; self does not, so try other first. 

905 # If it at least has full values, it's no worse than self. 

906 if other.hasFull(): 

907 return other 

908 else: 

909 return self 

910 elif other.hasFull(): 

911 return other 

912 # There's some chance that neither self nor other has full values, 

913 # but together provide enough to the union to. Let the general 

914 # case below handle that. 

915 elif self.graph == graph: 

916 # No chance at returning records. If self has full values, it's 

917 # the best we can do. 

918 if self.hasFull(): 

919 return self 

920 # General case with actual merging of dictionaries. 

921 values = self.full.byName() if self.hasFull() else self.byName() 

922 values.update(other.full.byName() if other.hasFull() else other.byName()) 

923 return DataCoordinate.standardize(values, graph=graph) 

924 

925 def expanded( 

926 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None] 

927 ) -> DataCoordinate: 

928 # Docstring inherited from DataCoordinate 

929 values = self._values 

930 if not self.hasFull(): 

931 # Extract a complete values tuple from the attributes of the given 

932 # records. It's possible for these to be inconsistent with 

933 # self._values (which is a serious problem, of course), but we've 

934 # documented this as a no-checking API. 

935 values += tuple(getattr(records[d.name], d.primaryKey.name) for d in self._graph.implied) 

936 return _ExpandedTupleDataCoordinate(self._graph, values, records) 

937 

938 def hasFull(self) -> bool: 

939 # Docstring inherited from DataCoordinate. 

940 return len(self._values) == len(self._graph._dataCoordinateIndices) 

941 

942 def hasRecords(self) -> bool: 

943 # Docstring inherited from DataCoordinate. 

944 return False 

945 

946 def _record(self, name: str) -> DimensionRecord | None: 

947 # Docstring inherited from DataCoordinate. 

948 assert False 

949 

950 def __reduce__(self) -> tuple[Any, ...]: 

951 return (_BasicTupleDataCoordinate, (self._graph, self._values)) 

952 

953 def __getattr__(self, name: str) -> Any: 

954 if name in self.graph.elements.names: 

955 raise AttributeError( 

956 f"Dimension record attribute {name!r} is only available on expanded DataCoordinates." 

957 ) 

958 raise AttributeError(name) 

959 

960 

961class _ExpandedTupleDataCoordinate(_BasicTupleDataCoordinate): 

962 """A `DataCoordinate` implementation that can hold `DimensionRecord`. 

963 

964 This class should only be accessed outside this module via the 

965 `DataCoordinate` interface, and should only be constructed via calls to 

966 `DataCoordinate.expanded`. 

967 

968 Parameters 

969 ---------- 

970 graph : `DimensionGraph` 

971 The dimensions to be identified. 

972 values : `tuple` [ `int` or `str` ] 

973 Data ID values, ordered to match ``graph._dataCoordinateIndices``. 

974 May include values for just required dimensions (which always come 

975 first) or all dimensions. 

976 records : `~collections.abc.Mapping` [ `str`, `DimensionRecord` or `None` ] 

977 A `NamedKeyMapping` with `DimensionElement` keys or a regular 

978 `~collections.abc.Mapping` with `str` (`DimensionElement` name) keys 

979 and `DimensionRecord` values. Keys must cover all elements in 

980 ``self.graph.elements``. Values may be `None`, but only to reflect 

981 actual NULL values in the database, not just records that have not 

982 been fetched. 

983 """ 

984 

985 def __init__( 

986 self, 

987 graph: DimensionGraph, 

988 values: tuple[DataIdValue, ...], 

989 records: NameLookupMapping[DimensionElement, DimensionRecord | None], 

990 ): 

991 super().__init__(graph, values) 

992 assert super().hasFull(), "This implementation requires full dimension records." 

993 self._records = records 

994 

995 __slots__ = ("_records",) 

996 

997 def subset(self, graph: DimensionGraph) -> DataCoordinate: 

998 # Docstring inherited from DataCoordinate. 

999 if self._graph == graph: 

1000 return self 

1001 return _ExpandedTupleDataCoordinate( 

1002 graph, tuple(self[k] for k in graph._dataCoordinateIndices.keys()), records=self._records 

1003 ) 

1004 

1005 def expanded( 

1006 self, records: NameLookupMapping[DimensionElement, DimensionRecord | None] 

1007 ) -> DataCoordinate: 

1008 # Docstring inherited from DataCoordinate. 

1009 return self 

1010 

1011 def union(self, other: DataCoordinate) -> DataCoordinate: 

1012 # Docstring inherited from DataCoordinate. 

1013 graph = self.graph.union(other.graph) 

1014 # See if one or both input data IDs is already what we want to return; 

1015 # if so, return the most complete one we have. 

1016 if self.graph == graph: 

1017 # self has records, so even if other is also a valid result, it's 

1018 # no better. 

1019 return self 

1020 if other.graph == graph: 

1021 # If other has full values, and self does not identify some of 

1022 # those, it's the base we can do. It may have records, too. 

1023 if other.hasFull(): 

1024 return other 

1025 # If other does not have full values, there's a chance self may 

1026 # provide the values needed to complete it. For example, self 

1027 # could be {band} while other could be 

1028 # {instrument, physical_filter, band}, with band unknown. 

1029 # General case with actual merging of dictionaries. 

1030 values = self.full.byName() 

1031 values.update(other.full.byName() if other.hasFull() else other.byName()) 

1032 basic = DataCoordinate.standardize(values, graph=graph) 

1033 # See if we can add records. 

1034 if self.hasRecords() and other.hasRecords(): 

1035 # Sometimes the elements of a union of graphs can contain elements 

1036 # that weren't in either input graph (because graph unions are only 

1037 # on dimensions). e.g. {visit} | {detector} brings along 

1038 # visit_detector_region. 

1039 elements = set(graph.elements.names) 

1040 elements -= self.graph.elements.names 

1041 elements -= other.graph.elements.names 

1042 if not elements: 

1043 records = NamedKeyDict[DimensionElement, DimensionRecord | None](self.records) 

1044 records.update(other.records) 

1045 return basic.expanded(records.freeze()) 

1046 return basic 

1047 

1048 def hasFull(self) -> bool: 

1049 # Docstring inherited from DataCoordinate. 

1050 return True 

1051 

1052 def hasRecords(self) -> bool: 

1053 # Docstring inherited from DataCoordinate. 

1054 return True 

1055 

1056 def _record(self, name: str) -> DimensionRecord | None: 

1057 # Docstring inherited from DataCoordinate. 

1058 return self._records[name] 

1059 

1060 def __reduce__(self) -> tuple[Any, ...]: 

1061 return (_ExpandedTupleDataCoordinate, (self._graph, self._values, self._records)) 

1062 

1063 def __getattr__(self, name: str) -> Any: 

1064 try: 

1065 return self._record(name) 

1066 except KeyError: 

1067 raise AttributeError(name) from None 

1068 

1069 def __dir__(self) -> list[str]: 

1070 result = list(super().__dir__()) 

1071 result.extend(self.graph.elements.names) 

1072 return result