Coverage for python/lsst/daf/butler/registry/_registry.py: 60%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = (
25 "Registry",
26)
28from abc import ABC, abstractmethod
29import contextlib
30import logging
31from typing import (
32 Any,
33 Dict,
34 Iterable,
35 Iterator,
36 List,
37 Mapping,
38 Optional,
39 Tuple,
40 Type,
41 TYPE_CHECKING,
42 Union,
43)
45from lsst.utils import doImportType
47from ..core import (
48 ButlerURI,
49 Config,
50 DataCoordinate,
51 DataCoordinateIterable,
52 DataId,
53 DatasetAssociation,
54 DatasetId,
55 DatasetRef,
56 DatasetType,
57 Dimension,
58 DimensionConfig,
59 DimensionElement,
60 DimensionGraph,
61 DimensionRecord,
62 DimensionUniverse,
63 NameLookupMapping,
64 StorageClassFactory,
65 Timespan,
66)
68from ._config import RegistryConfig
69from ._collectionType import CollectionType
70from ._defaults import RegistryDefaults
71from .interfaces import DatasetIdGenEnum
72from .wildcards import CollectionSearch
73from .summaries import CollectionSummary
75if TYPE_CHECKING: 75 ↛ 76line 75 didn't jump to line 76, because the condition on line 75 was never true
76 from .._butlerConfig import ButlerConfig
77 from .interfaces import (
78 CollectionRecord,
79 DatastoreRegistryBridgeManager,
80 )
82_LOG = logging.getLogger(__name__)
85class Registry(ABC):
86 """Abstract Registry interface.
88 Each registry implementation can have its own constructor parameters.
89 The assumption is that an instance of a specific subclass will be
90 constructed from configuration using `Registry.fromConfig()`.
91 The base class will look for a ``cls`` entry and call that specific
92 `fromConfig()` method.
94 All subclasses should store `RegistryDefaults` in a ``_defaults``
95 property. No other properties are assumed shared between implementations.
96 """
98 defaultConfigFile: Optional[str] = None
99 """Path to configuration defaults. Accessed within the ``configs`` resource
100 or relative to a search path. Can be None if no defaults specified.
101 """
103 @classmethod
104 def forceRegistryConfig(cls, config: Optional[Union[ButlerConfig,
105 RegistryConfig, Config, str]]) -> RegistryConfig:
106 """Force the supplied config to a `RegistryConfig`.
108 Parameters
109 ----------
110 config : `RegistryConfig`, `Config` or `str` or `None`
111 Registry configuration, if missing then default configuration will
112 be loaded from registry.yaml.
114 Returns
115 -------
116 registry_config : `RegistryConfig`
117 A registry config.
118 """
119 if not isinstance(config, RegistryConfig):
120 if isinstance(config, (str, Config)) or config is None:
121 config = RegistryConfig(config)
122 else:
123 raise ValueError(f"Incompatible Registry configuration: {config}")
124 return config
126 @classmethod
127 def determineTrampoline(cls,
128 config: Optional[Union[ButlerConfig,
129 RegistryConfig,
130 Config,
131 str]]) -> Tuple[Type[Registry], RegistryConfig]:
132 """Return class to use to instantiate real registry.
134 Parameters
135 ----------
136 config : `RegistryConfig` or `str`, optional
137 Registry configuration, if missing then default configuration will
138 be loaded from registry.yaml.
140 Returns
141 -------
142 requested_cls : `type` of `Registry`
143 The real registry class to use.
144 registry_config : `RegistryConfig`
145 The `RegistryConfig` to use.
146 """
147 config = cls.forceRegistryConfig(config)
149 # Default to the standard registry
150 registry_cls_name = config.get("cls", "lsst.daf.butler.registries.sql.SqlRegistry")
151 registry_cls = doImportType(registry_cls_name)
152 if registry_cls is cls:
153 raise ValueError("Can not instantiate the abstract base Registry from config")
154 if not issubclass(registry_cls, Registry):
155 raise TypeError(f"Registry class obtained from config {registry_cls_name} is "
156 "not a Registry class.")
157 return registry_cls, config
159 @classmethod
160 def createFromConfig(cls, config: Optional[Union[RegistryConfig, str]] = None,
161 dimensionConfig: Optional[Union[DimensionConfig, str]] = None,
162 butlerRoot: Optional[str] = None) -> Registry:
163 """Create registry database and return `Registry` instance.
165 This method initializes database contents, database must be empty
166 prior to calling this method.
168 Parameters
169 ----------
170 config : `RegistryConfig` or `str`, optional
171 Registry configuration, if missing then default configuration will
172 be loaded from registry.yaml.
173 dimensionConfig : `DimensionConfig` or `str`, optional
174 Dimensions configuration, if missing then default configuration
175 will be loaded from dimensions.yaml.
176 butlerRoot : `str`, optional
177 Path to the repository root this `Registry` will manage.
179 Returns
180 -------
181 registry : `Registry`
182 A new `Registry` instance.
184 Notes
185 -----
186 This class will determine the concrete `Registry` subclass to
187 use from configuration. Each subclass should implement this method
188 even if it can not create a registry.
189 """
190 registry_cls, registry_config = cls.determineTrampoline(config)
191 return registry_cls.createFromConfig(registry_config, dimensionConfig, butlerRoot)
193 @classmethod
194 def fromConfig(cls, config: Union[ButlerConfig, RegistryConfig, Config, str],
195 butlerRoot: Optional[Union[str, ButlerURI]] = None, writeable: bool = True,
196 defaults: Optional[RegistryDefaults] = None) -> Registry:
197 """Create `Registry` subclass instance from `config`.
199 Registry database must be initialized prior to calling this method.
201 Parameters
202 ----------
203 config : `ButlerConfig`, `RegistryConfig`, `Config` or `str`
204 Registry configuration
205 butlerRoot : `str` or `ButlerURI`, optional
206 Path to the repository root this `Registry` will manage.
207 writeable : `bool`, optional
208 If `True` (default) create a read-write connection to the database.
209 defaults : `RegistryDefaults`, optional
210 Default collection search path and/or output `~CollectionType.RUN`
211 collection.
213 Returns
214 -------
215 registry : `Registry` (subclass)
216 A new `Registry` subclass instance.
218 Notes
219 -----
220 This class will determine the concrete `Registry` subclass to
221 use from configuration. Each subclass should implement this method.
222 """
223 # The base class implementation should trampoline to the correct
224 # subclass. No implementation should ever use this implementation
225 # directly. If no class is specified, default to the standard
226 # registry.
227 registry_cls, registry_config = cls.determineTrampoline(config)
228 return registry_cls.fromConfig(config, butlerRoot, writeable, defaults)
230 @abstractmethod
231 def isWriteable(self) -> bool:
232 """Return `True` if this registry allows write operations, and `False`
233 otherwise.
234 """
235 raise NotImplementedError()
237 @abstractmethod
238 def copy(self, defaults: Optional[RegistryDefaults] = None) -> Registry:
239 """Create a new `Registry` backed by the same data repository and
240 connection as this one, but independent defaults.
242 Parameters
243 ----------
244 defaults : `RegistryDefaults`, optional
245 Default collections and data ID values for the new registry. If
246 not provided, ``self.defaults`` will be used (but future changes
247 to either registry's defaults will not affect the other).
249 Returns
250 -------
251 copy : `Registry`
252 A new `Registry` instance with its own defaults.
254 Notes
255 -----
256 Because the new registry shares a connection with the original, they
257 also share transaction state (despite the fact that their `transaction`
258 context manager methods do not reflect this), and must be used with
259 care.
260 """
261 raise NotImplementedError()
263 @property
264 @abstractmethod
265 def dimensions(self) -> DimensionUniverse:
266 """All dimensions recognized by this `Registry` (`DimensionUniverse`).
267 """
268 raise NotImplementedError()
270 @property
271 def defaults(self) -> RegistryDefaults:
272 """Default collection search path and/or output `~CollectionType.RUN`
273 collection (`RegistryDefaults`).
275 This is an immutable struct whose components may not be set
276 individually, but the entire struct can be set by assigning to this
277 property.
278 """
279 return self._defaults
281 @defaults.setter
282 def defaults(self, value: RegistryDefaults) -> None:
283 if value.run is not None:
284 self.registerRun(value.run)
285 value.finish(self)
286 self._defaults = value
288 @abstractmethod
289 def refresh(self) -> None:
290 """Refresh all in-memory state by querying the database.
292 This may be necessary to enable querying for entities added by other
293 registry instances after this one was constructed.
294 """
295 raise NotImplementedError()
297 @contextlib.contextmanager
298 @abstractmethod
299 def transaction(self, *, savepoint: bool = False) -> Iterator[None]:
300 """Return a context manager that represents a transaction.
301 """
302 raise NotImplementedError()
304 def resetConnectionPool(self) -> None:
305 """Reset connection pool for registry if relevant.
307 This operation can be used reset connections to servers when
308 using registry with fork-based multiprocessing. This method should
309 usually be called by the child process immediately
310 after the fork.
312 The base class implementation is a no-op.
313 """
314 pass
316 @abstractmethod
317 def registerCollection(self, name: str, type: CollectionType = CollectionType.TAGGED,
318 doc: Optional[str] = None) -> bool:
319 """Add a new collection if one with the given name does not exist.
321 Parameters
322 ----------
323 name : `str`
324 The name of the collection to create.
325 type : `CollectionType`
326 Enum value indicating the type of collection to create.
327 doc : `str`, optional
328 Documentation string for the collection.
330 Returns
331 -------
332 registered : `bool`
333 Boolean indicating whether the collection was already registered
334 or was created by this call.
336 Notes
337 -----
338 This method cannot be called within transactions, as it needs to be
339 able to perform its own transaction to be concurrent.
340 """
341 raise NotImplementedError()
343 @abstractmethod
344 def getCollectionType(self, name: str) -> CollectionType:
345 """Return an enumeration value indicating the type of the given
346 collection.
348 Parameters
349 ----------
350 name : `str`
351 The name of the collection.
353 Returns
354 -------
355 type : `CollectionType`
356 Enum value indicating the type of this collection.
358 Raises
359 ------
360 MissingCollectionError
361 Raised if no collection with the given name exists.
362 """
363 raise NotImplementedError()
365 @abstractmethod
366 def _get_collection_record(self, name: str) -> CollectionRecord:
367 """Return the record for this collection.
369 Parameters
370 ----------
371 name : `str`
372 Name of the collection for which the record is to be retrieved.
374 Returns
375 -------
376 record : `CollectionRecord`
377 The record for this collection.
378 """
379 raise NotImplementedError()
381 @abstractmethod
382 def registerRun(self, name: str, doc: Optional[str] = None) -> bool:
383 """Add a new run if one with the given name does not exist.
385 Parameters
386 ----------
387 name : `str`
388 The name of the run to create.
389 doc : `str`, optional
390 Documentation string for the collection.
392 Returns
393 -------
394 registered : `bool`
395 Boolean indicating whether a new run was registered. `False`
396 if it already existed.
398 Notes
399 -----
400 This method cannot be called within transactions, as it needs to be
401 able to perform its own transaction to be concurrent.
402 """
403 raise NotImplementedError()
405 @abstractmethod
406 def removeCollection(self, name: str) -> None:
407 """Completely remove the given collection.
409 Parameters
410 ----------
411 name : `str`
412 The name of the collection to remove.
414 Raises
415 ------
416 MissingCollectionError
417 Raised if no collection with the given name exists.
419 Notes
420 -----
421 If this is a `~CollectionType.RUN` collection, all datasets and quanta
422 in it are also fully removed. This requires that those datasets be
423 removed (or at least trashed) from any datastores that hold them first.
425 A collection may not be deleted as long as it is referenced by a
426 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must
427 be deleted or redefined first.
428 """
429 raise NotImplementedError()
431 @abstractmethod
432 def getCollectionChain(self, parent: str) -> CollectionSearch:
433 """Return the child collections in a `~CollectionType.CHAINED`
434 collection.
436 Parameters
437 ----------
438 parent : `str`
439 Name of the chained collection. Must have already been added via
440 a call to `Registry.registerCollection`.
442 Returns
443 -------
444 children : `CollectionSearch`
445 An object that defines the search path of the collection.
446 See :ref:`daf_butler_collection_expressions` for more information.
448 Raises
449 ------
450 MissingCollectionError
451 Raised if ``parent`` does not exist in the `Registry`.
452 TypeError
453 Raised if ``parent`` does not correspond to a
454 `~CollectionType.CHAINED` collection.
455 """
456 raise NotImplementedError()
458 @abstractmethod
459 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None:
460 """Define or redefine a `~CollectionType.CHAINED` collection.
462 Parameters
463 ----------
464 parent : `str`
465 Name of the chained collection. Must have already been added via
466 a call to `Registry.registerCollection`.
467 children : `Any`
468 An expression defining an ordered search of child collections,
469 generally an iterable of `str`; see
470 :ref:`daf_butler_collection_expressions` for more information.
471 flatten : `bool`, optional
472 If `True` (`False` is default), recursively flatten out any nested
473 `~CollectionType.CHAINED` collections in ``children`` first.
475 Raises
476 ------
477 MissingCollectionError
478 Raised when any of the given collections do not exist in the
479 `Registry`.
480 TypeError
481 Raised if ``parent`` does not correspond to a
482 `~CollectionType.CHAINED` collection.
483 ValueError
484 Raised if the given collections contains a cycle.
485 """
486 raise NotImplementedError()
488 @abstractmethod
489 def getCollectionDocumentation(self, collection: str) -> Optional[str]:
490 """Retrieve the documentation string for a collection.
492 Parameters
493 ----------
494 name : `str`
495 Name of the collection.
497 Returns
498 -------
499 docs : `str` or `None`
500 Docstring for the collection with the given name.
501 """
502 raise NotImplementedError()
504 @abstractmethod
505 def setCollectionDocumentation(self, collection: str, doc: Optional[str]) -> None:
506 """Set the documentation string for a collection.
508 Parameters
509 ----------
510 name : `str`
511 Name of the collection.
512 docs : `str` or `None`
513 Docstring for the collection with the given name; will replace any
514 existing docstring. Passing `None` will remove any existing
515 docstring.
516 """
517 raise NotImplementedError()
519 @abstractmethod
520 def getCollectionSummary(self, collection: str) -> CollectionSummary:
521 """Return a summary for the given collection.
523 Parameters
524 ----------
525 collection : `str`
526 Name of the collection for which a summary is to be retrieved.
528 Returns
529 -------
530 summary : `CollectionSummary`
531 Summary of the dataset types and governor dimension values in
532 this collection.
533 """
534 raise NotImplementedError()
536 @abstractmethod
537 def registerDatasetType(self, datasetType: DatasetType) -> bool:
538 """
539 Add a new `DatasetType` to the Registry.
541 It is not an error to register the same `DatasetType` twice.
543 Parameters
544 ----------
545 datasetType : `DatasetType`
546 The `DatasetType` to be added.
548 Returns
549 -------
550 inserted : `bool`
551 `True` if ``datasetType`` was inserted, `False` if an identical
552 existing `DatsetType` was found. Note that in either case the
553 DatasetType is guaranteed to be defined in the Registry
554 consistently with the given definition.
556 Raises
557 ------
558 ValueError
559 Raised if the dimensions or storage class are invalid.
560 ConflictingDefinitionError
561 Raised if this DatasetType is already registered with a different
562 definition.
564 Notes
565 -----
566 This method cannot be called within transactions, as it needs to be
567 able to perform its own transaction to be concurrent.
568 """
569 raise NotImplementedError()
571 @abstractmethod
572 def removeDatasetType(self, name: str) -> None:
573 """Remove the named `DatasetType` from the registry.
575 .. warning::
577 Registry implementations can cache the dataset type definitions.
578 This means that deleting the dataset type definition may result in
579 unexpected behavior from other butler processes that are active
580 that have not seen the deletion.
582 Parameters
583 ----------
584 name : `str`
585 Name of the type to be removed.
587 Raises
588 ------
589 lsst.daf.butler.registry.OrphanedRecordError
590 Raised if an attempt is made to remove the dataset type definition
591 when there are already datasets associated with it.
593 Notes
594 -----
595 If the dataset type is not registered the method will return without
596 action.
597 """
598 raise NotImplementedError()
600 @abstractmethod
601 def getDatasetType(self, name: str) -> DatasetType:
602 """Get the `DatasetType`.
604 Parameters
605 ----------
606 name : `str`
607 Name of the type.
609 Returns
610 -------
611 type : `DatasetType`
612 The `DatasetType` associated with the given name.
614 Raises
615 ------
616 KeyError
617 Requested named DatasetType could not be found in registry.
618 """
619 raise NotImplementedError()
621 @abstractmethod
622 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool:
623 """Test whether the given dataset ID generation mode is supported by
624 `insertDatasets`.
626 Parameters
627 ----------
628 mode : `DatasetIdGenEnum`
629 Enum value for the mode to test.
631 Returns
632 -------
633 supported : `bool`
634 Whether the given mode is supported.
635 """
636 raise NotImplementedError()
638 @abstractmethod
639 def findDataset(self, datasetType: Union[DatasetType, str], dataId: Optional[DataId] = None, *,
640 collections: Any = None, timespan: Optional[Timespan] = None,
641 **kwargs: Any) -> Optional[DatasetRef]:
642 """Find a dataset given its `DatasetType` and data ID.
644 This can be used to obtain a `DatasetRef` that permits the dataset to
645 be read from a `Datastore`. If the dataset is a component and can not
646 be found using the provided dataset type, a dataset ref for the parent
647 will be returned instead but with the correct dataset type.
649 Parameters
650 ----------
651 datasetType : `DatasetType` or `str`
652 A `DatasetType` or the name of one.
653 dataId : `dict` or `DataCoordinate`, optional
654 A `dict`-like object containing the `Dimension` links that identify
655 the dataset within a collection.
656 collections, optional.
657 An expression that fully or partially identifies the collections to
658 search for the dataset; see
659 :ref:`daf_butler_collection_expressions` for more information.
660 Defaults to ``self.defaults.collections``.
661 timespan : `Timespan`, optional
662 A timespan that the validity range of the dataset must overlap.
663 If not provided, any `~CollectionType.CALIBRATION` collections
664 matched by the ``collections`` argument will not be searched.
665 **kwargs
666 Additional keyword arguments passed to
667 `DataCoordinate.standardize` to convert ``dataId`` to a true
668 `DataCoordinate` or augment an existing one.
670 Returns
671 -------
672 ref : `DatasetRef`
673 A reference to the dataset, or `None` if no matching Dataset
674 was found.
676 Raises
677 ------
678 TypeError
679 Raised if ``collections`` is `None` and
680 ``self.defaults.collections`` is `None`.
681 LookupError
682 Raised if one or more data ID keys are missing.
683 KeyError
684 Raised if the dataset type does not exist.
685 MissingCollectionError
686 Raised if any of ``collections`` does not exist in the registry.
688 Notes
689 -----
690 This method simply returns `None` and does not raise an exception even
691 when the set of collections searched is intrinsically incompatible with
692 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
693 only `~CollectionType.CALIBRATION` collections are being searched.
694 This may make it harder to debug some lookup failures, but the behavior
695 is intentional; we consider it more important that failed searches are
696 reported consistently, regardless of the reason, and that adding
697 additional collections that do not contain a match to the search path
698 never changes the behavior.
699 """
700 raise NotImplementedError()
702 @abstractmethod
703 def insertDatasets(self, datasetType: Union[DatasetType, str], dataIds: Iterable[DataId],
704 run: Optional[str] = None, expand: bool = True,
705 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE) -> List[DatasetRef]:
706 """Insert one or more datasets into the `Registry`
708 This always adds new datasets; to associate existing datasets with
709 a new collection, use ``associate``.
711 Parameters
712 ----------
713 datasetType : `DatasetType` or `str`
714 A `DatasetType` or the name of one.
715 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate`
716 Dimension-based identifiers for the new datasets.
717 run : `str`, optional
718 The name of the run that produced the datasets. Defaults to
719 ``self.defaults.run``.
720 expand : `bool`, optional
721 If `True` (default), expand data IDs as they are inserted. This is
722 necessary in general to allow datastore to generate file templates,
723 but it may be disabled if the caller can guarantee this is
724 unnecessary.
725 idGenerationMode : `DatasetIdGenEnum`, optional
726 Specifies option for generating dataset IDs. By default unique IDs
727 are generated for each inserted dataset.
729 Returns
730 -------
731 refs : `list` of `DatasetRef`
732 Resolved `DatasetRef` instances for all given data IDs (in the same
733 order).
735 Raises
736 ------
737 TypeError
738 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
739 ConflictingDefinitionError
740 If a dataset with the same dataset type and data ID as one of those
741 given already exists in ``run``.
742 MissingCollectionError
743 Raised if ``run`` does not exist in the registry.
744 """
745 raise NotImplementedError()
747 @abstractmethod
748 def _importDatasets(self, datasets: Iterable[DatasetRef], expand: bool = True,
749 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE,
750 reuseIds: bool = False) -> List[DatasetRef]:
751 """Import one or more datasets into the `Registry`.
753 Difference from `insertDatasets` method is that this method accepts
754 `DatasetRef` instances which should already be resolved and have a
755 dataset ID. If registry supports globally-unique dataset IDs (e.g.
756 `uuid.UUID`) then datasets which already exist in the registry will be
757 ignored if imported again.
759 Parameters
760 ----------
761 datasets : `~collections.abc.Iterable` of `DatasetRef`
762 Datasets to be inserted. All `DatasetRef` instances must have
763 identical ``datasetType`` and ``run`` attributes. ``run``
764 attribute can be `None` and defaults to ``self.defaults.run``.
765 Datasets can specify ``id`` attribute which will be used for
766 inserted datasets. All dataset IDs must have the same type
767 (`int` or `uuid.UUID`), if type of dataset IDs does not match
768 configured backend then IDs will be ignored and new IDs will be
769 generated by backend.
770 expand : `bool`, optional
771 If `True` (default), expand data IDs as they are inserted. This is
772 necessary in general to allow datastore to generate file templates,
773 but it may be disabled if the caller can guarantee this is
774 unnecessary.
775 idGenerationMode : `DatasetIdGenEnum`, optional
776 Specifies option for generating dataset IDs when IDs are not
777 provided or their type does not match backend type. By default
778 unique IDs are generated for each inserted dataset.
779 reuseIds : `bool`, optional
780 If `True` then forces re-use of imported dataset IDs for integer
781 IDs which are normally generated as auto-incremented; exception
782 will be raised if imported IDs clash with existing ones. This
783 option has no effect on the use of globally-unique IDs which are
784 always re-used (or generated if integer IDs are being imported).
786 Returns
787 -------
788 refs : `list` of `DatasetRef`
789 Resolved `DatasetRef` instances for all given data IDs (in the same
790 order). If any of ``datasets`` has an ID which already exists in
791 the database then it will not be inserted or updated, but a
792 resolved `DatasetRef` will be returned for it in any case.
794 Raises
795 ------
796 TypeError
797 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
798 ConflictingDefinitionError
799 If a dataset with the same dataset type and data ID as one of those
800 given already exists in ``run``.
801 MissingCollectionError
802 Raised if ``run`` does not exist in the registry.
804 Notes
805 -----
806 This method is considered package-private and internal to Butler
807 implementation. Clients outside daf_butler package should not use this
808 method.
809 """
810 raise NotImplementedError()
812 @abstractmethod
813 def getDataset(self, id: DatasetId) -> Optional[DatasetRef]:
814 """Retrieve a Dataset entry.
816 Parameters
817 ----------
818 id : `DatasetId`
819 The unique identifier for the dataset.
821 Returns
822 -------
823 ref : `DatasetRef` or `None`
824 A ref to the Dataset, or `None` if no matching Dataset
825 was found.
826 """
827 raise NotImplementedError()
829 @abstractmethod
830 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None:
831 """Remove datasets from the Registry.
833 The datasets will be removed unconditionally from all collections, and
834 any `Quantum` that consumed this dataset will instead be marked with
835 having a NULL input. `Datastore` records will *not* be deleted; the
836 caller is responsible for ensuring that the dataset has already been
837 removed from all Datastores.
839 Parameters
840 ----------
841 refs : `Iterable` of `DatasetRef`
842 References to the datasets to be removed. Must include a valid
843 ``id`` attribute, and should be considered invalidated upon return.
845 Raises
846 ------
847 AmbiguousDatasetError
848 Raised if any ``ref.id`` is `None`.
849 OrphanedRecordError
850 Raised if any dataset is still present in any `Datastore`.
851 """
852 raise NotImplementedError()
854 @abstractmethod
855 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
856 """Add existing datasets to a `~CollectionType.TAGGED` collection.
858 If a DatasetRef with the same exact ID is already in a collection
859 nothing is changed. If a `DatasetRef` with the same `DatasetType` and
860 data ID but with different ID exists in the collection,
861 `ConflictingDefinitionError` is raised.
863 Parameters
864 ----------
865 collection : `str`
866 Indicates the collection the datasets should be associated with.
867 refs : `Iterable` [ `DatasetRef` ]
868 An iterable of resolved `DatasetRef` instances that already exist
869 in this `Registry`.
871 Raises
872 ------
873 ConflictingDefinitionError
874 If a Dataset with the given `DatasetRef` already exists in the
875 given collection.
876 AmbiguousDatasetError
877 Raised if ``any(ref.id is None for ref in refs)``.
878 MissingCollectionError
879 Raised if ``collection`` does not exist in the registry.
880 TypeError
881 Raise adding new datasets to the given ``collection`` is not
882 allowed.
883 """
884 raise NotImplementedError()
886 @abstractmethod
887 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
888 """Remove existing datasets from a `~CollectionType.TAGGED` collection.
890 ``collection`` and ``ref`` combinations that are not currently
891 associated are silently ignored.
893 Parameters
894 ----------
895 collection : `str`
896 The collection the datasets should no longer be associated with.
897 refs : `Iterable` [ `DatasetRef` ]
898 An iterable of resolved `DatasetRef` instances that already exist
899 in this `Registry`.
901 Raises
902 ------
903 AmbiguousDatasetError
904 Raised if any of the given dataset references is unresolved.
905 MissingCollectionError
906 Raised if ``collection`` does not exist in the registry.
907 TypeError
908 Raise adding new datasets to the given ``collection`` is not
909 allowed.
910 """
911 raise NotImplementedError()
913 @abstractmethod
914 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None:
915 """Associate one or more datasets with a calibration collection and a
916 validity range within it.
918 Parameters
919 ----------
920 collection : `str`
921 The name of an already-registered `~CollectionType.CALIBRATION`
922 collection.
923 refs : `Iterable` [ `DatasetRef` ]
924 Datasets to be associated.
925 timespan : `Timespan`
926 The validity range for these datasets within the collection.
928 Raises
929 ------
930 AmbiguousDatasetError
931 Raised if any of the given `DatasetRef` instances is unresolved.
932 ConflictingDefinitionError
933 Raised if the collection already contains a different dataset with
934 the same `DatasetType` and data ID and an overlapping validity
935 range.
936 TypeError
937 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
938 collection or if one or more datasets are of a dataset type for
939 which `DatasetType.isCalibration` returns `False`.
940 """
941 raise NotImplementedError()
943 @abstractmethod
944 def decertify(self, collection: str, datasetType: Union[str, DatasetType], timespan: Timespan, *,
945 dataIds: Optional[Iterable[DataId]] = None) -> None:
946 """Remove or adjust datasets to clear a validity range within a
947 calibration collection.
949 Parameters
950 ----------
951 collection : `str`
952 The name of an already-registered `~CollectionType.CALIBRATION`
953 collection.
954 datasetType : `str` or `DatasetType`
955 Name or `DatasetType` instance for the datasets to be decertified.
956 timespan : `Timespan`, optional
957 The validity range to remove datasets from within the collection.
958 Datasets that overlap this range but are not contained by it will
959 have their validity ranges adjusted to not overlap it, which may
960 split a single dataset validity range into two.
961 dataIds : `Iterable` [ `DataId` ], optional
962 Data IDs that should be decertified within the given validity range
963 If `None`, all data IDs for ``self.datasetType`` will be
964 decertified.
966 Raises
967 ------
968 TypeError
969 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
970 collection or if ``datasetType.isCalibration() is False``.
971 """
972 raise NotImplementedError()
974 @abstractmethod
975 def getDatastoreBridgeManager(self) -> DatastoreRegistryBridgeManager:
976 """Return an object that allows a new `Datastore` instance to
977 communicate with this `Registry`.
979 Returns
980 -------
981 manager : `DatastoreRegistryBridgeManager`
982 Object that mediates communication between this `Registry` and its
983 associated datastores.
984 """
985 raise NotImplementedError()
987 @abstractmethod
988 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]:
989 """Retrieve datastore locations for a given dataset.
991 Parameters
992 ----------
993 ref : `DatasetRef`
994 A reference to the dataset for which to retrieve storage
995 information.
997 Returns
998 -------
999 datastores : `Iterable` [ `str` ]
1000 All the matching datastores holding this dataset.
1002 Raises
1003 ------
1004 AmbiguousDatasetError
1005 Raised if ``ref.id`` is `None`.
1006 """
1007 raise NotImplementedError()
1009 @abstractmethod
1010 def expandDataId(self, dataId: Optional[DataId] = None, *, graph: Optional[DimensionGraph] = None,
1011 records: Optional[NameLookupMapping[DimensionElement, Optional[DimensionRecord]]] = None,
1012 withDefaults: bool = True,
1013 **kwargs: Any) -> DataCoordinate:
1014 """Expand a dimension-based data ID to include additional information.
1016 Parameters
1017 ----------
1018 dataId : `DataCoordinate` or `dict`, optional
1019 Data ID to be expanded; augmented and overridden by ``kwargs``.
1020 graph : `DimensionGraph`, optional
1021 Set of dimensions for the expanded ID. If `None`, the dimensions
1022 will be inferred from the keys of ``dataId`` and ``kwargs``.
1023 Dimensions that are in ``dataId`` or ``kwargs`` but not in
1024 ``graph`` are silently ignored, providing a way to extract and
1025 ``graph`` expand a subset of a data ID.
1026 records : `Mapping` [`str`, `DimensionRecord`], optional
1027 Dimension record data to use before querying the database for that
1028 data, keyed by element name.
1029 withDefaults : `bool`, optional
1030 Utilize ``self.defaults.dataId`` to fill in missing governor
1031 dimension key-value pairs. Defaults to `True` (i.e. defaults are
1032 used).
1033 **kwargs
1034 Additional keywords are treated like additional key-value pairs for
1035 ``dataId``, extending and overriding
1037 Returns
1038 -------
1039 expanded : `DataCoordinate`
1040 A data ID that includes full metadata for all of the dimensions it
1041 identifieds, i.e. guarantees that ``expanded.hasRecords()`` and
1042 ``expanded.hasFull()`` both return `True`.
1043 """
1044 raise NotImplementedError()
1046 @abstractmethod
1047 def insertDimensionData(self, element: Union[DimensionElement, str],
1048 *data: Union[Mapping[str, Any], DimensionRecord],
1049 conform: bool = True,
1050 replace: bool = False) -> None:
1051 """Insert one or more dimension records into the database.
1053 Parameters
1054 ----------
1055 element : `DimensionElement` or `str`
1056 The `DimensionElement` or name thereof that identifies the table
1057 records will be inserted into.
1058 data : `dict` or `DimensionRecord` (variadic)
1059 One or more records to insert.
1060 conform : `bool`, optional
1061 If `False` (`True` is default) perform no checking or conversions,
1062 and assume that ``element`` is a `DimensionElement` instance and
1063 ``data`` is a one or more `DimensionRecord` instances of the
1064 appropriate subclass.
1065 replace: `bool`, optional
1066 If `True` (`False` is default), replace existing records in the
1067 database if there is a conflict.
1068 """
1069 raise NotImplementedError()
1071 @abstractmethod
1072 def syncDimensionData(self, element: Union[DimensionElement, str],
1073 row: Union[Mapping[str, Any], DimensionRecord],
1074 conform: bool = True,
1075 update: bool = False) -> Union[bool, Dict[str, Any]]:
1076 """Synchronize the given dimension record with the database, inserting
1077 if it does not already exist and comparing values if it does.
1079 Parameters
1080 ----------
1081 element : `DimensionElement` or `str`
1082 The `DimensionElement` or name thereof that identifies the table
1083 records will be inserted into.
1084 row : `dict` or `DimensionRecord`
1085 The record to insert.
1086 conform : `bool`, optional
1087 If `False` (`True` is default) perform no checking or conversions,
1088 and assume that ``element`` is a `DimensionElement` instance and
1089 ``data`` is a one or more `DimensionRecord` instances of the
1090 appropriate subclass.
1091 update: `bool`, optional
1092 If `True` (`False` is default), update the existing record in the
1093 database if there is a conflict.
1095 Returns
1096 -------
1097 inserted_or_updated : `bool` or `dict`
1098 `True` if a new row was inserted, `False` if no changes were
1099 needed, or a `dict` mapping updated column names to their old
1100 values if an update was performed (only possible if
1101 ``update=True``).
1103 Raises
1104 ------
1105 ConflictingDefinitionError
1106 Raised if the record exists in the database (according to primary
1107 key lookup) but is inconsistent with the given one.
1108 """
1109 raise NotImplementedError()
1111 @abstractmethod
1112 def queryDatasetTypes(self, expression: Any = ..., *, components: Optional[bool] = None,
1113 missing: Optional[List[str]] = None,
1114 ) -> Iterator[DatasetType]:
1115 """Iterate over the dataset types whose names match an expression.
1117 Parameters
1118 ----------
1119 expression : `Any`, optional
1120 An expression that fully or partially identifies the dataset types
1121 to return, such as a `str`, `re.Pattern`, or iterable thereof.
1122 `...` can be used to return all dataset types, and is the default.
1123 See :ref:`daf_butler_dataset_type_expressions` for more
1124 information.
1125 components : `bool`, optional
1126 If `True`, apply all expression patterns to component dataset type
1127 names as well. If `False`, never apply patterns to components.
1128 If `None` (default), apply patterns to components only if their
1129 parent datasets were not matched by the expression.
1130 Fully-specified component datasets (`str` or `DatasetType`
1131 instances) are always included.
1132 missing : `list` of `str`, optional
1133 String dataset type names that were explicitly given (i.e. not
1134 regular expression patterns) but not found will be appended to this
1135 list, if it is provided.
1137 Yields
1138 ------
1139 datasetType : `DatasetType`
1140 A `DatasetType` instance whose name matches ``expression``.
1141 """
1142 raise NotImplementedError()
1144 @abstractmethod
1145 def queryCollections(self, expression: Any = ...,
1146 datasetType: Optional[DatasetType] = None,
1147 collectionTypes: Iterable[CollectionType] = CollectionType.all(),
1148 flattenChains: bool = False,
1149 includeChains: Optional[bool] = None) -> Iterator[str]:
1150 """Iterate over the collections whose names match an expression.
1152 Parameters
1153 ----------
1154 expression : `Any`, optional
1155 An expression that identifies the collections to return, such as
1156 a `str` (for full matches or partial matches via globs),
1157 `re.Pattern` (for partial matches), or iterable thereof. `...`
1158 can be used to return all collections, and is the default.
1159 See :ref:`daf_butler_collection_expressions` for more information.
1160 datasetType : `DatasetType`, optional
1161 If provided, only yield collections that may contain datasets of
1162 this type. This is a conservative approximation in general; it may
1163 yield collections that do not have any such datasets.
1164 collectionTypes : `AbstractSet` [ `CollectionType` ], optional
1165 If provided, only yield collections of these types.
1166 flattenChains : `bool`, optional
1167 If `True` (`False` is default), recursively yield the child
1168 collections of matching `~CollectionType.CHAINED` collections.
1169 includeChains : `bool`, optional
1170 If `True`, yield records for matching `~CollectionType.CHAINED`
1171 collections. Default is the opposite of ``flattenChains``: include
1172 either CHAINED collections or their children, but not both.
1174 Yields
1175 ------
1176 collection : `str`
1177 The name of a collection that matches ``expression``.
1178 """
1179 raise NotImplementedError()
1181 @abstractmethod
1182 def queryDatasets(self, datasetType: Any, *,
1183 collections: Any = None,
1184 dimensions: Optional[Iterable[Union[Dimension, str]]] = None,
1185 dataId: Optional[DataId] = None,
1186 where: Optional[str] = None,
1187 findFirst: bool = False,
1188 components: Optional[bool] = None,
1189 bind: Optional[Mapping[str, Any]] = None,
1190 check: bool = True,
1191 **kwargs: Any) -> Iterable[DatasetRef]:
1192 """Query for and iterate over dataset references matching user-provided
1193 criteria.
1195 Parameters
1196 ----------
1197 datasetType
1198 An expression that fully or partially identifies the dataset types
1199 to be queried. Allowed types include `DatasetType`, `str`,
1200 `re.Pattern`, and iterables thereof. The special value `...` can
1201 be used to query all dataset types. See
1202 :ref:`daf_butler_dataset_type_expressions` for more information.
1203 collections: optional
1204 An expression that identifies the collections to search, such as a
1205 `str` (for full matches or partial matches via globs), `re.Pattern`
1206 (for partial matches), or iterable thereof. `...` can be used to
1207 search all collections (actually just all `~CollectionType.RUN`
1208 collections, because this will still find all datasets).
1209 If not provided, ``self.default.collections`` is used. See
1210 :ref:`daf_butler_collection_expressions` for more information.
1211 dimensions : `~collections.abc.Iterable` of `Dimension` or `str`
1212 Dimensions to include in the query (in addition to those used
1213 to identify the queried dataset type(s)), either to constrain
1214 the resulting datasets to those for which a matching dimension
1215 exists, or to relate the dataset type's dimensions to dimensions
1216 referenced by the ``dataId`` or ``where`` arguments.
1217 dataId : `dict` or `DataCoordinate`, optional
1218 A data ID whose key-value pairs are used as equality constraints
1219 in the query.
1220 where : `str`, optional
1221 A string expression similar to a SQL WHERE clause. May involve
1222 any column of a dimension table or (as a shortcut for the primary
1223 key column of a dimension table) dimension name. See
1224 :ref:`daf_butler_dimension_expressions` for more information.
1225 findFirst : `bool`, optional
1226 If `True` (`False` is default), for each result data ID, only
1227 yield one `DatasetRef` of each `DatasetType`, from the first
1228 collection in which a dataset of that dataset type appears
1229 (according to the order of ``collections`` passed in). If `True`,
1230 ``collections`` must not contain regular expressions and may not
1231 be `...`.
1232 components : `bool`, optional
1233 If `True`, apply all dataset expression patterns to component
1234 dataset type names as well. If `False`, never apply patterns to
1235 components. If `None` (default), apply patterns to components only
1236 if their parent datasets were not matched by the expression.
1237 Fully-specified component datasets (`str` or `DatasetType`
1238 instances) are always included.
1239 bind : `Mapping`, optional
1240 Mapping containing literal values that should be injected into the
1241 ``where`` expression, keyed by the identifiers they replace.
1242 check : `bool`, optional
1243 If `True` (default) check the query for consistency before
1244 executing it. This may reject some valid queries that resemble
1245 common mistakes (e.g. queries for visits without specifying an
1246 instrument).
1247 **kwargs
1248 Additional keyword arguments are forwarded to
1249 `DataCoordinate.standardize` when processing the ``dataId``
1250 argument (and may be used to provide a constraining data ID even
1251 when the ``dataId`` argument is `None`).
1253 Returns
1254 -------
1255 refs : `queries.DatasetQueryResults`
1256 Dataset references matching the given query criteria. Nested data
1257 IDs are guaranteed to include values for all implied dimensions
1258 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
1259 include dimension records (`DataCoordinate.hasRecords` will be
1260 `False`) unless `~queries.DatasetQueryResults.expanded` is called
1261 on the result object (which returns a new one).
1263 Raises
1264 ------
1265 TypeError
1266 Raised when the arguments are incompatible, such as when a
1267 collection wildcard is passed when ``findFirst`` is `True`, or
1268 when ``collections`` is `None` and``self.defaults.collections`` is
1269 also `None`.
1271 Notes
1272 -----
1273 When multiple dataset types are queried in a single call, the
1274 results of this operation are equivalent to querying for each dataset
1275 type separately in turn, and no information about the relationships
1276 between datasets of different types is included. In contexts where
1277 that kind of information is important, the recommended pattern is to
1278 use `queryDataIds` to first obtain data IDs (possibly with the
1279 desired dataset types and collections passed as constraints to the
1280 query), and then use multiple (generally much simpler) calls to
1281 `queryDatasets` with the returned data IDs passed as constraints.
1282 """
1283 raise NotImplementedError()
1285 @abstractmethod
1286 def queryDataIds(self, dimensions: Union[Iterable[Union[Dimension, str]], Dimension, str], *,
1287 dataId: Optional[DataId] = None,
1288 datasets: Any = None,
1289 collections: Any = None,
1290 where: Optional[str] = None,
1291 components: Optional[bool] = None,
1292 bind: Optional[Mapping[str, Any]] = None,
1293 check: bool = True,
1294 **kwargs: Any) -> DataCoordinateIterable:
1295 """Query for data IDs matching user-provided criteria.
1297 Parameters
1298 ----------
1299 dimensions : `Dimension` or `str`, or iterable thereof
1300 The dimensions of the data IDs to yield, as either `Dimension`
1301 instances or `str`. Will be automatically expanded to a complete
1302 `DimensionGraph`.
1303 dataId : `dict` or `DataCoordinate`, optional
1304 A data ID whose key-value pairs are used as equality constraints
1305 in the query.
1306 datasets : `Any`, optional
1307 An expression that fully or partially identifies dataset types
1308 that should constrain the yielded data IDs. For example, including
1309 "raw" here would constrain the yielded ``instrument``,
1310 ``exposure``, ``detector``, and ``physical_filter`` values to only
1311 those for which at least one "raw" dataset exists in
1312 ``collections``. Allowed types include `DatasetType`, `str`,
1313 `re.Pattern`, and iterables thereof. Unlike other dataset type
1314 expressions, ``...`` is not permitted - it doesn't make sense to
1315 constrain data IDs on the existence of *all* datasets.
1316 See :ref:`daf_butler_dataset_type_expressions` for more
1317 information.
1318 collections: `Any`, optional
1319 An expression that identifies the collections to search for
1320 datasets, such as a `str` (for full matches or partial matches
1321 via globs), `re.Pattern` (for partial matches), or iterable
1322 thereof. `...` can be used to search all collections (actually
1323 just all `~CollectionType.RUN` collections, because this will
1324 still find all datasets). If not provided,
1325 ``self.default.collections`` is used. Ignored unless ``datasets``
1326 is also passed. See :ref:`daf_butler_collection_expressions` for
1327 more information.
1328 where : `str`, optional
1329 A string expression similar to a SQL WHERE clause. May involve
1330 any column of a dimension table or (as a shortcut for the primary
1331 key column of a dimension table) dimension name. See
1332 :ref:`daf_butler_dimension_expressions` for more information.
1333 components : `bool`, optional
1334 If `True`, apply all dataset expression patterns to component
1335 dataset type names as well. If `False`, never apply patterns to
1336 components. If `None` (default), apply patterns to components only
1337 if their parent datasets were not matched by the expression.
1338 Fully-specified component datasets (`str` or `DatasetType`
1339 instances) are always included.
1340 bind : `Mapping`, optional
1341 Mapping containing literal values that should be injected into the
1342 ``where`` expression, keyed by the identifiers they replace.
1343 check : `bool`, optional
1344 If `True` (default) check the query for consistency before
1345 executing it. This may reject some valid queries that resemble
1346 common mistakes (e.g. queries for visits without specifying an
1347 instrument).
1348 **kwargs
1349 Additional keyword arguments are forwarded to
1350 `DataCoordinate.standardize` when processing the ``dataId``
1351 argument (and may be used to provide a constraining data ID even
1352 when the ``dataId`` argument is `None`).
1354 Returns
1355 -------
1356 dataIds : `DataCoordinateQueryResults`
1357 Data IDs matching the given query parameters. These are guaranteed
1358 to identify all dimensions (`DataCoordinate.hasFull` returns
1359 `True`), but will not contain `DimensionRecord` objects
1360 (`DataCoordinate.hasRecords` returns `False`). Call
1361 `DataCoordinateQueryResults.expanded` on the returned object to
1362 fetch those (and consider using
1363 `DataCoordinateQueryResults.materialize` on the returned object
1364 first if the expected number of rows is very large). See
1365 documentation for those methods for additional information.
1367 Raises
1368 ------
1369 TypeError
1370 Raised if ``collections`` is `None`, ``self.defaults.collections``
1371 is `None`, and ``datasets`` is not `None`.
1372 """
1373 raise NotImplementedError()
1375 @abstractmethod
1376 def queryDimensionRecords(self, element: Union[DimensionElement, str], *,
1377 dataId: Optional[DataId] = None,
1378 datasets: Any = None,
1379 collections: Any = None,
1380 where: Optional[str] = None,
1381 components: Optional[bool] = None,
1382 bind: Optional[Mapping[str, Any]] = None,
1383 check: bool = True,
1384 **kwargs: Any) -> Iterator[DimensionRecord]:
1385 """Query for dimension information matching user-provided criteria.
1387 Parameters
1388 ----------
1389 element : `DimensionElement` or `str`
1390 The dimension element to obtain records for.
1391 dataId : `dict` or `DataCoordinate`, optional
1392 A data ID whose key-value pairs are used as equality constraints
1393 in the query.
1394 datasets : `Any`, optional
1395 An expression that fully or partially identifies dataset types
1396 that should constrain the yielded records. See `queryDataIds` and
1397 :ref:`daf_butler_dataset_type_expressions` for more information.
1398 collections: `Any`, optional
1399 An expression that identifies the collections to search for
1400 datasets, such as a `str` (for full matches or partial matches
1401 via globs), `re.Pattern` (for partial matches), or iterable
1402 thereof. `...` can be used to search all collections (actually
1403 just all `~CollectionType.RUN` collections, because this will
1404 still find all datasets). If not provided,
1405 ``self.default.collections`` is used. Ignored unless ``datasets``
1406 is also passed. See :ref:`daf_butler_collection_expressions` for
1407 more information.
1408 where : `str`, optional
1409 A string expression similar to a SQL WHERE clause. See
1410 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
1411 information.
1412 components : `bool`, optional
1413 Whether to apply dataset expressions to components as well.
1414 See `queryDataIds` for more information.
1415 bind : `Mapping`, optional
1416 Mapping containing literal values that should be injected into the
1417 ``where`` expression, keyed by the identifiers they replace.
1418 check : `bool`, optional
1419 If `True` (default) check the query for consistency before
1420 executing it. This may reject some valid queries that resemble
1421 common mistakes (e.g. queries for visits without specifying an
1422 instrument).
1423 **kwargs
1424 Additional keyword arguments are forwarded to
1425 `DataCoordinate.standardize` when processing the ``dataId``
1426 argument (and may be used to provide a constraining data ID even
1427 when the ``dataId`` argument is `None`).
1429 Returns
1430 -------
1431 dataIds : `DataCoordinateQueryResults`
1432 Data IDs matching the given query parameters.
1433 """
1434 raise NotImplementedError()
1436 @abstractmethod
1437 def queryDatasetAssociations(
1438 self,
1439 datasetType: Union[str, DatasetType],
1440 collections: Any = ...,
1441 *,
1442 collectionTypes: Iterable[CollectionType] = CollectionType.all(),
1443 flattenChains: bool = False,
1444 ) -> Iterator[DatasetAssociation]:
1445 """Iterate over dataset-collection combinations where the dataset is in
1446 the collection.
1448 This method is a temporary placeholder for better support for
1449 assocation results in `queryDatasets`. It will probably be
1450 removed in the future, and should be avoided in production code
1451 whenever possible.
1453 Parameters
1454 ----------
1455 datasetType : `DatasetType` or `str`
1456 A dataset type object or the name of one.
1457 collections: `Any`, optional
1458 An expression that identifies the collections to search for
1459 datasets, such as a `str` (for full matches or partial matches
1460 via globs), `re.Pattern` (for partial matches), or iterable
1461 thereof. `...` can be used to search all collections (actually
1462 just all `~CollectionType.RUN` collections, because this will still
1463 find all datasets). If not provided, ``self.default.collections``
1464 is used. See :ref:`daf_butler_collection_expressions` for more
1465 information.
1466 collectionTypes : `AbstractSet` [ `CollectionType` ], optional
1467 If provided, only yield associations from collections of these
1468 types.
1469 flattenChains : `bool`, optional
1470 If `True` (default) search in the children of
1471 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED``
1472 collections are ignored.
1474 Yields
1475 ------
1476 association : `DatasetAssociation`
1477 Object representing the relationship beween a single dataset and
1478 a single collection.
1480 Raises
1481 ------
1482 TypeError
1483 Raised if ``collections`` is `None` and
1484 ``self.defaults.collections`` is `None`.
1485 """
1486 raise NotImplementedError()
1488 storageClasses: StorageClassFactory
1489 """All storage classes known to the registry (`StorageClassFactory`).
1490 """