Coverage for python/lsst/daf/butler/_dataset

1# This file is part of daf_butler.

3# Developed for the LSST Data Management System.

4# This product includes software developed by the LSST Project

5# (http://www.lsst.org).

6# See the COPYRIGHT file at the top-level directory of this distribution

7# for details of code ownership.

9# This software is dual licensed under the GNU General Public License and also

10# under a 3-clause BSD license. Recipients may choose which of these licenses

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,

12# respectively. If you choose the GPL option then the following text applies

13# (but note that there is still no warranty even if you opt for BSD instead):

14#

15# This program is free software: you can redistribute it and/or modify

16# it under the terms of the GNU General Public License as published by

17# the Free Software Foundation, either version 3 of the License, or

18# (at your option) any later version.

19#

20# This program is distributed in the hope that it will be useful,

21# but WITHOUT ANY WARRANTY; without even the implied warranty of

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

23# GNU General Public License for more details.

24#

25# You should have received a copy of the GNU General Public License

26# along with this program. If not, see <http://www.gnu.org/licenses/>.

28from __future__ import annotations

30__all__ = ["DatasetType", "SerializedDatasetType"]

32import re

33from collections.abc import Callable, Iterable, Mapping

34from copy import deepcopy

35from types import MappingProxyType

36from typing import TYPE_CHECKING, Any, ClassVar, Self, cast

38from pydantic import BaseModel, StrictBool, StrictStr

40from ._config_support import LookupKey

41from ._exceptions import InconsistentUniverseError, UnknownComponentError

42from ._storage_class import StorageClass, StorageClassFactory

43from .dimensions import DimensionGroup

44from .json import from_json_pydantic, to_json_pydantic

45from .persistence_context import PersistenceContextVars

47if TYPE_CHECKING:

48 from .dimensions import DimensionUniverse

49 from .registry import Registry

52def _safeMakeMappingProxyType(data: Mapping | None) -> Mapping:

53 if data is None:

54 data = {}

55 return MappingProxyType(data)

58class SerializedDatasetType(BaseModel):

59 """Simplified model of a `DatasetType` suitable for serialization."""

61 name: StrictStr

62 storageClass: StrictStr | None = None

63 dimensions: list[StrictStr] | None = None

64 parentStorageClass: StrictStr | None = None

65 isCalibration: StrictBool = False

67 @classmethod

68 def direct(

69 cls,

70 *,

71 name: str,

72 storageClass: str | None = None,

73 dimensions: list | None = None,

74 parentStorageClass: str | None = None,

75 isCalibration: bool = False,

76 ) -> SerializedDatasetType:

77 """Construct a `SerializedDatasetType` directly without validators.

79 This differs from Pydantic's model_construct method in that the

80 arguments are explicitly what the model requires, and it will recurse

81 through members, constructing them from their corresponding `direct`

82 methods.

84 This method should only be called when the inputs are trusted.

86 Parameters

87 ----------

88 name : `str`

89 The name of the dataset type.

90 storageClass : `str` or `None`

91 The name of the storage class.

92 dimensions : `list` or `None`

93 The dimensions associated with this dataset type.

94 parentStorageClass : `str` or `None`

95 The parent storage class name if this is a component.

96 isCalibration : `bool`

97 Whether this dataset type represents calibrations.

99 Returns

100 -------

101 `SerializedDatasetType`

102 A Pydantic model representing a dataset type.

103 """

104 cache = PersistenceContextVars.serializedDatasetTypeMapping.get()

105 key = (name, storageClass or "")

106 if cache is not None and (type_ := cache.get(key, None)) is not None: 106 ↛ 107line 106 didn't jump to line 107 because the condition on line 106 was never true

107 return type_

108 serialized_dimensions = dimensions if dimensions is not None else None

109 node = cls.model_construct(

110 name=name,

111 storageClass=storageClass,

112 dimensions=serialized_dimensions,

113 parentStorageClass=parentStorageClass,

114 isCalibration=isCalibration,

115 )

116

117 if cache is not None: 117 ↛ 118line 117 didn't jump to line 118 because the condition on line 117 was never true

118 cache[key] = node

119 return node

120

121

122class DatasetType:

123 """A named category of Datasets.

124

125 Defines how they are organized, related, and stored.

126

127 A concrete, final class whose instances represent a `DatasetType`.

128 `DatasetType` instances may be constructed without a `Registry`,

129 but they must be registered

130 via `Registry.registerDatasetType()` before corresponding Datasets

131 may be added.

132 `DatasetType` instances are immutable.

133

134 Parameters

135 ----------

136 name : `str`

137 A string name for the Dataset; must correspond to the same

138 `DatasetType` across all Registries. Names must start with an

139 upper or lowercase letter, and may contain only letters, numbers,

140 and underscores. Component dataset types should contain a single

141 period separating the base dataset type name from the component name

142 (and may be recursive).

143 dimensions : `DimensionGroup` or `~collections.abc.Iterable` [ `str` ]

144 Dimensions used to label and relate instances of this `DatasetType`.

145 If not a `DimensionGroup`, ``universe`` must be provided as well.

146 storageClass : `StorageClass` or `str`

147 Instance of a `StorageClass` or name of `StorageClass` that defines

148 how this `DatasetType` is persisted.

149 parentStorageClass : `StorageClass` or `str`, optional

150 Instance of a `StorageClass` or name of `StorageClass` that defines

151 how the composite parent is persisted. Must be `None` if this

152 is not a component.

153 universe : `DimensionUniverse`, optional

154 Set of all known dimensions, used to normalize ``dimensions`` if it

155 is not already a `DimensionGroup`.

156 isCalibration : `bool`, optional

157 If `True`, this dataset type may be included in

158 `~CollectionType.CALIBRATION` collections.

159

160 Notes

161 -----

162 See also :ref:`daf_butler_organizing_datasets`.

163 """

164

165 __slots__ = (

166 "_name",

167 "_dimensions",

168 "_storageClass",

169 "_storageClassName",

170 "_parentStorageClass",

171 "_parentStorageClassName",

172 "_isCalibration",

173 )

174

175 _serializedType: ClassVar[type[BaseModel]] = SerializedDatasetType

176

177 VALID_NAME_REGEX = re.compile("^[a-zA-Z_][a-zA-Z0-9_]*(\\.[a-zA-Z_][a-zA-Z0-9_]*)*$")

178

179 @staticmethod

180 def nameWithComponent(datasetTypeName: str, componentName: str) -> str:

181 """Form a valid DatasetTypeName from a parent and component.

182

183 No validation is performed.

184

185 Parameters

186 ----------

187 datasetTypeName : `str`

188 Base type name.

189 componentName : `str`

190 Name of component.

191

192 Returns

193 -------

194 compTypeName : `str`

195 Name to use for component DatasetType.

196 """

197 return f"{datasetTypeName}.{componentName}"

198

199 def __init__(

200 self,

201 name: str,

202 dimensions: DimensionGroup | Iterable[str],

203 storageClass: StorageClass | str,

204 parentStorageClass: StorageClass | str | None = None,

205 *,

206 universe: DimensionUniverse | None = None,

207 isCalibration: bool = False,

208 ):

209 if self.VALID_NAME_REGEX.match(name) is None:

210 raise ValueError(f"DatasetType name '{name}' is invalid.")

211 self._name = name

212 universe = universe or getattr(dimensions, "universe", None)

213 if universe is None: 213 ↛ 214line 213 didn't jump to line 214 because the condition on line 213 was never true

214 raise ValueError("If dimensions is not a DimensionGroup, a universe must be provided.")

215 self._dimensions = universe.conform(dimensions)

216 if name in self._dimensions.universe.governor_dimensions: 216 ↛ 217line 216 didn't jump to line 217 because the condition on line 216 was never true

217 raise ValueError(f"Governor dimension name {name} cannot be used as a dataset type name.")

218 if not isinstance(storageClass, StorageClass | str): 218 ↛ 219line 218 didn't jump to line 219 because the condition on line 218 was never true

219 raise ValueError(f"StorageClass argument must be StorageClass or str. Got {storageClass}")

220 self._storageClass: StorageClass | None

221 if isinstance(storageClass, StorageClass):

222 self._storageClass = storageClass

223 self._storageClassName = storageClass.name

224 else:

225 self._storageClass = None

226 self._storageClassName = storageClass

227

228 self._parentStorageClass: StorageClass | None = None

229 self._parentStorageClassName: str | None = None

230 if parentStorageClass is not None:

231 if not isinstance(storageClass, StorageClass | str): 231 ↛ 232line 231 didn't jump to line 232 because the condition on line 231 was never true

232 raise ValueError(

233 f"Parent StorageClass argument must be StorageClass or str. Got {parentStorageClass}"

234 )

235

236 # Only allowed for a component dataset type

237 _, componentName = self.splitDatasetTypeName(self._name)

238 if componentName is None:

239 raise ValueError(

240 f"Can not specify a parent storage class if this is not a component ({self._name})"

241 )

242 if isinstance(parentStorageClass, StorageClass):

243 self._parentStorageClass = parentStorageClass

244 self._parentStorageClassName = parentStorageClass.name

245 else:

246 self._parentStorageClassName = parentStorageClass

247

248 # Ensure that parent storage class is specified when we have

249 # a component and is not specified when we don't

250 _, componentName = self.splitDatasetTypeName(self._name)

251 if parentStorageClass is None and componentName is not None:

252 raise ValueError(

253 f"Component dataset type '{self._name}' constructed without parent storage class"

254 )

255 if parentStorageClass is not None and componentName is None: 255 ↛ 256line 255 didn't jump to line 256 because the condition on line 255 was never true

256 raise ValueError(f"Parent storage class specified by {self._name} is not a composite")

257 self._isCalibration = isCalibration

258

259 def __repr__(self) -> str:

260 extra = ""

261 if self._parentStorageClassName:

262 extra = f", parentStorageClass={self._parentStorageClassName}"

263 if self._isCalibration:

264 extra += ", isCalibration=True"

265 return f"DatasetType({self.name!r}, {self._dimensions}, {self._storageClassName}{extra})"

266

267 def _equal_ignoring_storage_class(self, other: Any) -> bool:

268 """Check everything is equal except the storage class.

269

270 Parameters

271 ----------

272 other : Any

273 Object to check against this one.

274

275 Returns

276 -------

277 mostly : `bool`

278 Returns `True` if everything except the storage class is equal.

279 """

280 if not isinstance(other, type(self)): 280 ↛ 281line 280 didn't jump to line 281 because the condition on line 280 was never true

281 return False

282 if self._name != other._name:

283 return False

284 if self._dimensions != other._dimensions:

285 return False

286 if self._isCalibration != other._isCalibration:

287 return False

288 return True

289

290 def __eq__(self, other: Any) -> bool:

291 mostly_equal = self._equal_ignoring_storage_class(other)

292 if not mostly_equal:

293 return False

294

295 # Be careful not to force a storage class to import the corresponding

296 # python code.

297 if self._parentStorageClass is not None and other._parentStorageClass is not None:

298 if self._parentStorageClass != other._parentStorageClass:

299 return False

300 else:

301 if self._parentStorageClassName != other._parentStorageClassName:

302 return False

303

304 if self._storageClass is not None and other._storageClass is not None:

305 if self._storageClass != other._storageClass:

306 return False

307 else:

308 if self._storageClassName != other._storageClassName:

309 return False

310 return True

311

312 def is_compatible_with(self, other: DatasetType) -> bool:

313 """Determine if the given `DatasetType` is compatible with this one.

314

315 Compatibility requires a matching name and dimensions and a storage

316 class for this dataset type that can convert the python type associated

317 with the other storage class to this python type. Parent storage class

318 compatibility is not checked at all for components.

319

320 Parameters

321 ----------

322 other : `DatasetType`

323 Dataset type to check.

324

325 Returns

326 -------

327 is_compatible : `bool`

328 Returns `True` if the other dataset type is either the same as this

329 or the storage class associated with the other can be converted to

330 this.

331 """

332 mostly_equal = self._equal_ignoring_storage_class(other)

333 if not mostly_equal: 333 ↛ 334line 333 didn't jump to line 334 because the condition on line 333 was never true

334 return False

335

336 # If the storage class names match then they are compatible.

337 if self._storageClassName == other._storageClassName:

338 return True

339

340 # Now required to check the full storage class.

341 self_sc = self.storageClass

342 other_sc = other.storageClass

343

344 return self_sc.can_convert(other_sc)

345

346 def conform_to(self, universe: DimensionUniverse) -> DatasetType:

347 """Rebuild this dataset type in a different dimension universe.

348

349 Parameters

350 ----------

351 universe : `DimensionUniverse`

352 Target dimension universe.

353

354 Returns

355 -------

356 dataset_type : `DatasetType`

357 This dataset type if its dimensions already belong to

358 ``universe``, else an otherwise-identical dataset type whose

359 dimensions belong to ``universe``.

360

361 Raises

362 ------

363 InconsistentUniverseError

364 Raised if the target universe has a different namespace, does not

365 contain all of this dataset type's dimensions, or conforms those

366 dimensions to a group with different names or a different split

367 between required and implied dimensions.

368 """

369 source_universe = self._dimensions.universe

370 if source_universe is universe:

371 return self

372 if source_universe.namespace != universe.namespace:

373 raise InconsistentUniverseError(

374 f"Dataset type {self._name!r} has universe {source_universe} with a different "

375 f"namespace than target universe {universe}."

376 )

377 try:

378 dimensions = universe.conform(self._dimensions.names)

379 except KeyError as exc:

380 raise InconsistentUniverseError(

381 f"Dimensions {self._dimensions} of dataset type {self._name!r} from universe "

382 f"{source_universe} do not all exist in target universe {universe}."

383 ) from exc

384 if dimensions.names != self._dimensions.names or set(dimensions.required) != set(

385 self._dimensions.required

386 ):

387 raise InconsistentUniverseError(

388 f"Dimensions {self._dimensions} of dataset type {self._name!r} from universe "

389 f"{source_universe} are different from the conforming set of target universe "

390 f"{universe} dimensions {dimensions}."

391 )

392 return DatasetType(

393 self._name,

394 dimensions,

395 self._storageClass or self._storageClassName,

396 parentStorageClass=self._parentStorageClass or self._parentStorageClassName,

397 isCalibration=self._isCalibration,

398 )

399

400 def __hash__(self) -> int:

401 """Hash DatasetType instance.

402

403 This only uses StorageClass name which is it consistent with the

404 implementation of StorageClass hash method.

405 """

406 return hash((self._name, self._dimensions, self._storageClassName, self._parentStorageClassName))

407

408 def __lt__(self, other: Any) -> bool:

409 """Sort using the dataset type name."""

410 if not isinstance(other, type(self)):

411 return NotImplemented

412 return self.name < other.name

413

414 @property

415 def name(self) -> str:

416 """Return a string name for the Dataset.

417

418 Must correspond to the same `DatasetType` across all Registries.

419 """

420 return self._name

421

422 @property

423 def dimensions(self) -> DimensionGroup:

424 """Return the dimensions of this dataset type (`DimensionGroup`).

425

426 The dimensions of a define the keys of its datasets' data IDs..

427 """

428 return self._dimensions

429

430 @property

431 def storageClass(self) -> StorageClass:

432 """Return `StorageClass` instance associated with this dataset type.

433

434 The `StorageClass` defines how this `DatasetType`

435 is persisted. Note that if DatasetType was constructed with a name

436 of a StorageClass then Butler has to be initialized before using

437 this property.

438 """

439 if self._storageClass is None:

440 self._storageClass = StorageClassFactory().getStorageClass(self._storageClassName)

441 return self._storageClass

442

443 @property

444 def storageClass_name(self) -> str:

445 """Return the storage class name.

446

447 This will never force the storage class to be imported.

448 """

449 return self._storageClassName

450

451 @property

452 def parentStorageClass(self) -> StorageClass | None:

453 """Return the storage class of the composite containing this component.

454

455 Note that if DatasetType was constructed with a name of a

456 StorageClass then Butler has to be initialized before using this

457 property. Can be `None` if this is not a component of a composite.

458 Must be defined if this is a component.

459 """

460 if self._parentStorageClass is None and self._parentStorageClassName is None:

461 return None

462 if self._parentStorageClass is None and self._parentStorageClassName is not None:

463 self._parentStorageClass = StorageClassFactory().getStorageClass(self._parentStorageClassName)

464 return self._parentStorageClass

465

466 def isCalibration(self) -> bool:

467 """Return if datasets of this type can be in calibration collections.

468

469 Returns

470 -------

471 flag : `bool`

472 `True` if datasets of this type may be included in calibration

473 collections.

474 """

475 return self._isCalibration

476

477 @staticmethod

478 def splitDatasetTypeName(datasetTypeName: str) -> tuple[str, str | None]:

479 """Return the root name and the component from a composite name.

480

481 Parameters

482 ----------

483 datasetTypeName : `str`

484 The name of the dataset type, can include a component using

485 a "."-separator.

486

487 Returns

488 -------

489 rootName : `str`

490 Root name without any components.

491 componentName : `str`

492 The component if it has been specified, else `None`.

493

494 Notes

495 -----

496 If the dataset type name is ``a.b.c`` this method will return a

497 root name of ``a`` and a component name of ``b.c``.

498 """

499 comp = None

500 root = datasetTypeName

501 if "." in root:

502 # If there is doubt, the component is after the first "."

503 root, comp = root.split(".", maxsplit=1)

504 return root, comp

505

506 def nameAndComponent(self) -> tuple[str, str | None]:

507 """Return the root name of this dataset type and any component.

508

509 Returns

510 -------

511 rootName : `str`

512 Root name for this `DatasetType` without any components.

513 componentName : `str`

514 The component if it has been specified, else `None`.

515 """

516 return self.splitDatasetTypeName(self.name)

517

518 def component(self) -> str | None:

519 """Return the component name (if defined).

520

521 Returns

522 -------

523 comp : `str`

524 Name of component part of DatasetType name. `None` if this

525 `DatasetType` is not associated with a component.

526 """

527 _, comp = self.nameAndComponent()

528 return comp

529

530 def componentTypeName(self, component: str) -> str:

531 """Derive a component dataset type from a composite.

532

533 Parameters

534 ----------

535 component : `str`

536 Name of component.

537

538 Returns

539 -------

540 derived : `str`

541 Compound name of this `DatasetType` and the component.

542

543 Raises

544 ------

545 KeyError

546 Requested component is not supported by this `DatasetType`.

547 """

548 if component in self.storageClass.allComponents():

549 return self.nameWithComponent(self.name, component)

550 raise UnknownComponentError(

551 f"Requested component ({component}) not understood by this DatasetType ({self})"

552 )

553

554 def makeCompositeDatasetType(self) -> DatasetType:

555 """Return a composite dataset type from the component.

556

557 Returns

558 -------

559 composite : `DatasetType`

560 The composite dataset type.

561

562 Raises

563 ------

564 RuntimeError

565 Raised if this dataset type is not a component dataset type.

566 """

567 if not self.isComponent(): 567 ↛ 568line 567 didn't jump to line 568 because the condition on line 567 was never true

568 raise RuntimeError(f"DatasetType {self.name} must be a component to form the composite")

569 composite_name, _ = self.nameAndComponent()

570 if self.parentStorageClass is None: 570 ↛ 571line 570 didn't jump to line 571 because the condition on line 570 was never true

571 raise ValueError(

572 f"Parent storage class is not set. Unable to create composite type from {self.name}"

573 )

574 return DatasetType(

575 composite_name,

576 dimensions=self._dimensions,

577 storageClass=self.parentStorageClass,

578 isCalibration=self.isCalibration(),

579 )

580

581 def makeComponentDatasetType(self, component: str) -> DatasetType:

582 """Return a component dataset type from a composite.

583

584 Assumes the same dimensions as the parent.

585

586 Parameters

587 ----------

588 component : `str`

589 Name of component.

590

591 Returns

592 -------

593 datasetType : `DatasetType`

594 A new DatasetType instance.

595 """

596 # The component could be a read/write or read component

597 return DatasetType(

598 self.componentTypeName(component),

599 dimensions=self._dimensions,

600 storageClass=self.storageClass.allComponents()[component],

601 parentStorageClass=self.storageClass,

602 isCalibration=self.isCalibration(),

603 )

604

605 def makeAllComponentDatasetTypes(self) -> list[DatasetType]:

606 """Return all component dataset types for this composite.

607

608 Returns

609 -------

610 all : `list` of `DatasetType`

611 All the component dataset types. If this is not a composite

612 then returns an empty list.

613 """

614 return [

615 self.makeComponentDatasetType(componentName)

616 for componentName in self.storageClass.allComponents()

617 ]

618

619 def overrideStorageClass(self, storageClass: str | StorageClass) -> DatasetType:

620 """Create a new `DatasetType` from this one but with an updated

621 `StorageClass`.

622

623 Parameters

624 ----------

625 storageClass : `str` or `StorageClass`

626 The new storage class.

627

628 Returns

629 -------

630 modified : `DatasetType`

631 A dataset type that is the same as the current one but with a

632 different storage class. Will be ``self`` if the given storage

633 class is the current one.

634

635 Notes

636 -----

637 If this is a component dataset type, the parent storage class will be

638 retained.

639 """

640 if storageClass == self._storageClassName or storageClass == self._storageClass:

641 return self

642 parent = self._parentStorageClass if self._parentStorageClass else self._parentStorageClassName

643 new = DatasetType(

644 self.name,

645 dimensions=self._dimensions,

646 storageClass=storageClass,

647 parentStorageClass=parent,

648 isCalibration=self.isCalibration(),

649 )

650 # Check validity.

651 if new.is_compatible_with(self) or self.is_compatible_with(new):

652 return new

653 raise ValueError(

654 f"The new storage class ({new.storageClass}) is not compatible with the "

655 f"existing storage class ({self.storageClass})."

656 )

657

658 def isComponent(self) -> bool:

659 """Return whether this `DatasetType` refers to a component.

660

661 Returns

662 -------

663 isComponent : `bool`

664 `True` if this `DatasetType` is a component, `False` otherwise.

665 """

666 if self.component():

667 return True

668 return False

669

670 def isComposite(self) -> bool:

671 """Return whether this `DatasetType` is a composite.

672

673 Returns

674 -------

675 isComposite : `bool`

676 `True` if this `DatasetType` is a composite type, `False`

677 otherwise.

678 """

679 return self.storageClass.isComposite()

680

681 def _lookupNames(self) -> tuple[LookupKey, ...]:

682 """Return name keys to use for lookups in configurations.

683

684 The names are returned in order of priority.

685

686 Returns

687 -------

688 names : `tuple` of `LookupKey`

689 Tuple of the `DatasetType` name and the `StorageClass` name.

690 If the name includes a component the name with the component

691 is first, then the name without the component and finally

692 the storage class name and the storage class name of the

693 composite.

694 """

695 rootName, componentName = self.nameAndComponent()

696 lookups: tuple[LookupKey, ...] = (LookupKey(name=self.name),)

697 if componentName is not None:

698 lookups = lookups + (LookupKey(name=rootName),)

699

700 if self._dimensions:

701 # Dimensions are a lower priority than dataset type name

702 lookups = lookups + (LookupKey(dimensions=self._dimensions),)

703

704 storageClasses = self.storageClass._lookupNames()

705 if componentName is not None and self.parentStorageClass is not None:

706 storageClasses += self.parentStorageClass._lookupNames()

707

708 return lookups + storageClasses

709

710 def to_simple(self, minimal: bool = False) -> SerializedDatasetType:

711 """Convert this class to a simple python type.

712

713 This makes it suitable for serialization.

714

715 Parameters

716 ----------

717 minimal : `bool`, optional

718 Use minimal serialization. Requires Registry to convert

719 back to a full type.

720

721 Returns

722 -------

723 simple : `SerializedDatasetType`

724 The object converted to a class suitable for serialization.

725 """

726 as_dict: dict[str, Any]

727 if minimal:

728 # Only needs the name.

729 as_dict = {"name": self.name}

730 else:

731 # Convert to a dict form

732 as_dict = {

733 "name": self.name,

734 "storageClass": self._storageClassName,

735 "isCalibration": self._isCalibration,

736 "dimensions": list(self._dimensions.required),

737 }

738

739 if self._parentStorageClassName is not None:

740 as_dict["parentStorageClass"] = self._parentStorageClassName

741 return SerializedDatasetType(**as_dict)

742

743 @classmethod

744 def from_simple(

745 cls,

746 simple: SerializedDatasetType,

747 universe: DimensionUniverse | None = None,

748 registry: Registry | None = None,

749 ) -> DatasetType:

750 """Construct a new object from the simplified form.

751

752 This is usually data returned from the `to_simple` method.

753

754 Parameters

755 ----------

756 simple : `SerializedDatasetType`

757 The value returned by `to_simple()`.

758 universe : `DimensionUniverse`

759 The special graph of all known dimensions of which this graph will

760 be a subset. Can be `None` if a registry is provided.

761 registry : `lsst.daf.butler.Registry`, optional

762 Registry to use to convert simple name of a DatasetType to

763 a full `DatasetType`. Can be `None` if a full description of

764 the type is provided along with a universe.

765

766 Returns

767 -------

768 datasetType : `DatasetType`

769 Newly-constructed object.

770 """

771 # check to see if there is a cache, and if there is, if there is a

772 # cached dataset type

773 cache = PersistenceContextVars.loadedTypes.get()

774 key = (simple.name, simple.storageClass or "")

775 if cache is not None and (type_ := cache.get(key, None)) is not None: 775 ↛ 776line 775 didn't jump to line 776 because the condition on line 775 was never true

776 return type_

777

778 if simple.storageClass is None:

779 # Treat this as minimalist representation

780 if registry is None: 780 ↛ 781line 780 didn't jump to line 781 because the condition on line 780 was never true

781 raise ValueError(

782 f"Unable to convert a DatasetType name '{simple}' to DatasetType without a Registry"

783 )

784 return registry.getDatasetType(simple.name)

785

786 if universe is None and registry is None: 786 ↛ 787line 786 didn't jump to line 787 because the condition on line 786 was never true

787 raise ValueError("One of universe or registry must be provided.")

788

789 if universe is None and registry is not None:

790 # registry should not be none by now but test helps mypy

791 universe = registry.dimensions

792

793 if universe is None: 793 ↛ 795line 793 didn't jump to line 795 because the condition on line 793 was never true

794 # this is for mypy

795 raise ValueError("Unable to determine a usable universe")

796 if simple.dimensions is None: 796 ↛ 797line 796 didn't jump to line 797 because the condition on line 796 was never true

797 raise ValueError(f"Dimensions must be specified in {simple}")

798 dimensions = universe.conform(simple.dimensions)

799

800 newType = cls(

801 name=simple.name,

802 dimensions=dimensions,

803 storageClass=simple.storageClass,

804 isCalibration=simple.isCalibration,

805 parentStorageClass=simple.parentStorageClass,

806 universe=universe,

807 )

808 if cache is not None: 808 ↛ 809line 808 didn't jump to line 809 because the condition on line 808 was never true

809 cache[key] = newType

810 return newType

811

812 to_json = to_json_pydantic

813 from_json: ClassVar[Callable[..., Self]] = cast(Callable[..., Self], classmethod(from_json_pydantic))

814

815 def __reduce__(

816 self,

817 ) -> tuple[

818 Callable, tuple[type[DatasetType], tuple[str, DimensionGroup, str, str | None], dict[str, bool]]

819 ]:

820 """Support pickling.

821

822 StorageClass instances can not normally be pickled, so we pickle

823 StorageClass name instead of instance.

824 """

825 return _unpickle_via_factory, (

826 self.__class__,

827 (self.name, self._dimensions, self._storageClassName, self._parentStorageClassName),

828 {"isCalibration": self._isCalibration},

829 )

830

831 def __deepcopy__(self, memo: Any) -> DatasetType:

832 """Support for deep copy method.

833

834 Normally ``deepcopy`` will use pickle mechanism to make copies.

835 We want to avoid that to support (possibly degenerate) use case when

836 DatasetType is constructed with StorageClass instance which is not

837 registered with StorageClassFactory (this happens in unit tests).

838 Instead we re-implement ``__deepcopy__`` method.

839 """

840 return DatasetType(

841 name=deepcopy(self.name, memo),

842 dimensions=deepcopy(self._dimensions, memo),

843 storageClass=deepcopy(self._storageClass or self._storageClassName, memo),

844 parentStorageClass=deepcopy(self._parentStorageClass or self._parentStorageClassName, memo),

845 isCalibration=deepcopy(self._isCalibration, memo),

846 )

847

848

849def _unpickle_via_factory(factory: Callable, args: Any, kwargs: Any) -> DatasetType:

850 """Unpickle something by calling a factory.

851

852 Allows subclasses to unpickle using `__reduce__` with keyword

853 arguments as well as positional arguments.

854 """

855 return factory(*args, **kwargs)

856

857

858def get_dataset_type_name(datasetTypeOrName: DatasetType | str) -> str:

859 """Given a `DatasetType` object or a dataset type name, return a dataset

860 type name.

861

862 Parameters

863 ----------

864 datasetTypeOrName : `DatasetType` | `str`

865 A DatasetType, or the name of a DatasetType.

866

867 Returns

868 -------

869 name

870 The name associated with the given DatasetType, or the given string.

871 """

872 if isinstance(datasetTypeOrName, DatasetType):

873 return datasetTypeOrName.name

874 elif isinstance(datasetTypeOrName, str): 874 ↛ 877line 874 didn't jump to line 877 because the condition on line 874 was always true

875 return datasetTypeOrName

876 else:

877 raise TypeError(f"Expected DatasetType or str, got unexpected object: {datasetTypeOrName}")

Coverage for python/lsst/daf/butler/_dataset_type.py: 89%

257 statements