Coverage for python/lsst/daf/butler/registry/_registry.py : 62%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = (
25 "Registry",
26)
28from abc import ABC, abstractmethod
29import contextlib
30import logging
31from typing import (
32 Any,
33 Dict,
34 Iterable,
35 Iterator,
36 List,
37 Mapping,
38 Optional,
39 Tuple,
40 Type,
41 TYPE_CHECKING,
42 Union,
43)
45from lsst.utils import doImport
47from ..core import (
48 ButlerURI,
49 Config,
50 DataCoordinate,
51 DataCoordinateIterable,
52 DataId,
53 DatasetAssociation,
54 DatasetId,
55 DatasetRef,
56 DatasetType,
57 Dimension,
58 DimensionConfig,
59 DimensionElement,
60 DimensionGraph,
61 DimensionRecord,
62 DimensionUniverse,
63 NameLookupMapping,
64 StorageClassFactory,
65 Timespan,
66)
68from ._config import RegistryConfig
69from ._collectionType import CollectionType
70from ._defaults import RegistryDefaults
71from .interfaces import DatasetIdGenEnum
72from .wildcards import CollectionSearch
73from .summaries import CollectionSummary
75if TYPE_CHECKING: 75 ↛ 76line 75 didn't jump to line 76, because the condition on line 75 was never true
76 from .._butlerConfig import ButlerConfig
77 from .interfaces import (
78 CollectionRecord,
79 DatastoreRegistryBridgeManager,
80 )
82_LOG = logging.getLogger(__name__)
85class Registry(ABC):
86 """Abstract Registry interface.
88 Each registry implementation can have its own constructor parameters.
89 The assumption is that an instance of a specific subclass will be
90 constructed from configuration using `Registry.fromConfig()`.
91 The base class will look for a ``cls`` entry and call that specific
92 `fromConfig()` method.
94 All subclasses should store `RegistryDefaults` in a ``_defaults``
95 property. No other properties are assumed shared between implementations.
96 """
98 defaultConfigFile: Optional[str] = None
99 """Path to configuration defaults. Accessed within the ``configs`` resource
100 or relative to a search path. Can be None if no defaults specified.
101 """
103 @classmethod
104 def forceRegistryConfig(cls, config: Optional[Union[ButlerConfig,
105 RegistryConfig, Config, str]]) -> RegistryConfig:
106 """Force the supplied config to a `RegistryConfig`.
108 Parameters
109 ----------
110 config : `RegistryConfig`, `Config` or `str` or `None`
111 Registry configuration, if missing then default configuration will
112 be loaded from registry.yaml.
114 Returns
115 -------
116 registry_config : `RegistryConfig`
117 A registry config.
118 """
119 if not isinstance(config, RegistryConfig):
120 if isinstance(config, (str, Config)) or config is None:
121 config = RegistryConfig(config)
122 else:
123 raise ValueError(f"Incompatible Registry configuration: {config}")
124 return config
126 @classmethod
127 def determineTrampoline(cls,
128 config: Optional[Union[ButlerConfig,
129 RegistryConfig,
130 Config,
131 str]]) -> Tuple[Type[Registry], RegistryConfig]:
132 """Return class to use to instantiate real registry.
134 Parameters
135 ----------
136 config : `RegistryConfig` or `str`, optional
137 Registry configuration, if missing then default configuration will
138 be loaded from registry.yaml.
140 Returns
141 -------
142 requested_cls : `type` of `Registry`
143 The real registry class to use.
144 registry_config : `RegistryConfig`
145 The `RegistryConfig` to use.
146 """
147 config = cls.forceRegistryConfig(config)
149 # Default to the standard registry
150 registry_cls = doImport(config.get("cls", "lsst.daf.butler.registries.sql.SqlRegistry"))
151 if registry_cls is cls:
152 raise ValueError("Can not instantiate the abstract base Registry from config")
153 return registry_cls, config
155 @classmethod
156 def createFromConfig(cls, config: Optional[Union[RegistryConfig, str]] = None,
157 dimensionConfig: Optional[Union[DimensionConfig, str]] = None,
158 butlerRoot: Optional[str] = None) -> Registry:
159 """Create registry database and return `Registry` instance.
161 This method initializes database contents, database must be empty
162 prior to calling this method.
164 Parameters
165 ----------
166 config : `RegistryConfig` or `str`, optional
167 Registry configuration, if missing then default configuration will
168 be loaded from registry.yaml.
169 dimensionConfig : `DimensionConfig` or `str`, optional
170 Dimensions configuration, if missing then default configuration
171 will be loaded from dimensions.yaml.
172 butlerRoot : `str`, optional
173 Path to the repository root this `Registry` will manage.
175 Returns
176 -------
177 registry : `Registry`
178 A new `Registry` instance.
180 Notes
181 -----
182 This class will determine the concrete `Registry` subclass to
183 use from configuration. Each subclass should implement this method
184 even if it can not create a registry.
185 """
186 registry_cls, registry_config = cls.determineTrampoline(config)
187 return registry_cls.createFromConfig(registry_config, dimensionConfig, butlerRoot)
189 @classmethod
190 def fromConfig(cls, config: Union[ButlerConfig, RegistryConfig, Config, str],
191 butlerRoot: Optional[Union[str, ButlerURI]] = None, writeable: bool = True,
192 defaults: Optional[RegistryDefaults] = None) -> Registry:
193 """Create `Registry` subclass instance from `config`.
195 Registry database must be initialized prior to calling this method.
197 Parameters
198 ----------
199 config : `ButlerConfig`, `RegistryConfig`, `Config` or `str`
200 Registry configuration
201 butlerRoot : `str` or `ButlerURI`, optional
202 Path to the repository root this `Registry` will manage.
203 writeable : `bool`, optional
204 If `True` (default) create a read-write connection to the database.
205 defaults : `RegistryDefaults`, optional
206 Default collection search path and/or output `~CollectionType.RUN`
207 collection.
209 Returns
210 -------
211 registry : `Registry` (subclass)
212 A new `Registry` subclass instance.
214 Notes
215 -----
216 This class will determine the concrete `Registry` subclass to
217 use from configuration. Each subclass should implement this method.
218 """
219 # The base class implementation should trampoline to the correct
220 # subclass. No implementation should ever use this implementation
221 # directly. If no class is specified, default to the standard
222 # registry.
223 registry_cls, registry_config = cls.determineTrampoline(config)
224 return registry_cls.fromConfig(config, butlerRoot, writeable, defaults)
226 @abstractmethod
227 def isWriteable(self) -> bool:
228 """Return `True` if this registry allows write operations, and `False`
229 otherwise.
230 """
231 raise NotImplementedError()
233 @abstractmethod
234 def copy(self, defaults: Optional[RegistryDefaults] = None) -> Registry:
235 """Create a new `Registry` backed by the same data repository and
236 connection as this one, but independent defaults.
238 Parameters
239 ----------
240 defaults : `RegistryDefaults`, optional
241 Default collections and data ID values for the new registry. If
242 not provided, ``self.defaults`` will be used (but future changes
243 to either registry's defaults will not affect the other).
245 Returns
246 -------
247 copy : `Registry`
248 A new `Registry` instance with its own defaults.
250 Notes
251 -----
252 Because the new registry shares a connection with the original, they
253 also share transaction state (despite the fact that their `transaction`
254 context manager methods do not reflect this), and must be used with
255 care.
256 """
257 raise NotImplementedError()
259 @property
260 @abstractmethod
261 def dimensions(self) -> DimensionUniverse:
262 """All dimensions recognized by this `Registry` (`DimensionUniverse`).
263 """
264 raise NotImplementedError()
266 @property
267 def defaults(self) -> RegistryDefaults:
268 """Default collection search path and/or output `~CollectionType.RUN`
269 collection (`RegistryDefaults`).
271 This is an immutable struct whose components may not be set
272 individually, but the entire struct can be set by assigning to this
273 property.
274 """
275 return self._defaults
277 @defaults.setter
278 def defaults(self, value: RegistryDefaults) -> None:
279 if value.run is not None:
280 self.registerRun(value.run)
281 value.finish(self)
282 self._defaults = value
284 @abstractmethod
285 def refresh(self) -> None:
286 """Refresh all in-memory state by querying the database.
288 This may be necessary to enable querying for entities added by other
289 registry instances after this one was constructed.
290 """
291 raise NotImplementedError()
293 @contextlib.contextmanager
294 @abstractmethod
295 def transaction(self, *, savepoint: bool = False) -> Iterator[None]:
296 """Return a context manager that represents a transaction.
297 """
298 raise NotImplementedError()
300 def resetConnectionPool(self) -> None:
301 """Reset connection pool for registry if relevant.
303 This operation can be used reset connections to servers when
304 using registry with fork-based multiprocessing. This method should
305 usually be called by the child process immediately
306 after the fork.
308 The base class implementation is a no-op.
309 """
310 pass
312 @abstractmethod
313 def registerCollection(self, name: str, type: CollectionType = CollectionType.TAGGED,
314 doc: Optional[str] = None) -> None:
315 """Add a new collection if one with the given name does not exist.
317 Parameters
318 ----------
319 name : `str`
320 The name of the collection to create.
321 type : `CollectionType`
322 Enum value indicating the type of collection to create.
323 doc : `str`, optional
324 Documentation string for the collection.
326 Notes
327 -----
328 This method cannot be called within transactions, as it needs to be
329 able to perform its own transaction to be concurrent.
330 """
331 raise NotImplementedError()
333 @abstractmethod
334 def getCollectionType(self, name: str) -> CollectionType:
335 """Return an enumeration value indicating the type of the given
336 collection.
338 Parameters
339 ----------
340 name : `str`
341 The name of the collection.
343 Returns
344 -------
345 type : `CollectionType`
346 Enum value indicating the type of this collection.
348 Raises
349 ------
350 MissingCollectionError
351 Raised if no collection with the given name exists.
352 """
353 raise NotImplementedError()
355 @abstractmethod
356 def _get_collection_record(self, name: str) -> CollectionRecord:
357 """Return the record for this collection.
359 Parameters
360 ----------
361 name : `str`
362 Name of the collection for which the record is to be retrieved.
364 Returns
365 -------
366 record : `CollectionRecord`
367 The record for this collection.
368 """
369 raise NotImplementedError()
371 @abstractmethod
372 def registerRun(self, name: str, doc: Optional[str] = None) -> None:
373 """Add a new run if one with the given name does not exist.
375 Parameters
376 ----------
377 name : `str`
378 The name of the run to create.
379 doc : `str`, optional
380 Documentation string for the collection.
382 Notes
383 -----
384 This method cannot be called within transactions, as it needs to be
385 able to perform its own transaction to be concurrent.
386 """
387 raise NotImplementedError()
389 @abstractmethod
390 def removeCollection(self, name: str) -> None:
391 """Completely remove the given collection.
393 Parameters
394 ----------
395 name : `str`
396 The name of the collection to remove.
398 Raises
399 ------
400 MissingCollectionError
401 Raised if no collection with the given name exists.
403 Notes
404 -----
405 If this is a `~CollectionType.RUN` collection, all datasets and quanta
406 in it are also fully removed. This requires that those datasets be
407 removed (or at least trashed) from any datastores that hold them first.
409 A collection may not be deleted as long as it is referenced by a
410 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must
411 be deleted or redefined first.
412 """
413 raise NotImplementedError()
415 @abstractmethod
416 def getCollectionChain(self, parent: str) -> CollectionSearch:
417 """Return the child collections in a `~CollectionType.CHAINED`
418 collection.
420 Parameters
421 ----------
422 parent : `str`
423 Name of the chained collection. Must have already been added via
424 a call to `Registry.registerCollection`.
426 Returns
427 -------
428 children : `CollectionSearch`
429 An object that defines the search path of the collection.
430 See :ref:`daf_butler_collection_expressions` for more information.
432 Raises
433 ------
434 MissingCollectionError
435 Raised if ``parent`` does not exist in the `Registry`.
436 TypeError
437 Raised if ``parent`` does not correspond to a
438 `~CollectionType.CHAINED` collection.
439 """
440 raise NotImplementedError()
442 @abstractmethod
443 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None:
444 """Define or redefine a `~CollectionType.CHAINED` collection.
446 Parameters
447 ----------
448 parent : `str`
449 Name of the chained collection. Must have already been added via
450 a call to `Registry.registerCollection`.
451 children : `Any`
452 An expression defining an ordered search of child collections,
453 generally an iterable of `str`; see
454 :ref:`daf_butler_collection_expressions` for more information.
455 flatten : `bool`, optional
456 If `True` (`False` is default), recursively flatten out any nested
457 `~CollectionType.CHAINED` collections in ``children`` first.
459 Raises
460 ------
461 MissingCollectionError
462 Raised when any of the given collections do not exist in the
463 `Registry`.
464 TypeError
465 Raised if ``parent`` does not correspond to a
466 `~CollectionType.CHAINED` collection.
467 ValueError
468 Raised if the given collections contains a cycle.
469 """
470 raise NotImplementedError()
472 @abstractmethod
473 def getCollectionDocumentation(self, collection: str) -> Optional[str]:
474 """Retrieve the documentation string for a collection.
476 Parameters
477 ----------
478 name : `str`
479 Name of the collection.
481 Returns
482 -------
483 docs : `str` or `None`
484 Docstring for the collection with the given name.
485 """
486 raise NotImplementedError()
488 @abstractmethod
489 def setCollectionDocumentation(self, collection: str, doc: Optional[str]) -> None:
490 """Set the documentation string for a collection.
492 Parameters
493 ----------
494 name : `str`
495 Name of the collection.
496 docs : `str` or `None`
497 Docstring for the collection with the given name; will replace any
498 existing docstring. Passing `None` will remove any existing
499 docstring.
500 """
501 raise NotImplementedError()
503 @abstractmethod
504 def getCollectionSummary(self, collection: str) -> CollectionSummary:
505 """Return a summary for the given collection.
507 Parameters
508 ----------
509 collection : `str`
510 Name of the collection for which a summary is to be retrieved.
512 Returns
513 -------
514 summary : `CollectionSummary`
515 Summary of the dataset types and governor dimension values in
516 this collection.
517 """
518 raise NotImplementedError()
520 @abstractmethod
521 def registerDatasetType(self, datasetType: DatasetType) -> bool:
522 """
523 Add a new `DatasetType` to the Registry.
525 It is not an error to register the same `DatasetType` twice.
527 Parameters
528 ----------
529 datasetType : `DatasetType`
530 The `DatasetType` to be added.
532 Returns
533 -------
534 inserted : `bool`
535 `True` if ``datasetType`` was inserted, `False` if an identical
536 existing `DatsetType` was found. Note that in either case the
537 DatasetType is guaranteed to be defined in the Registry
538 consistently with the given definition.
540 Raises
541 ------
542 ValueError
543 Raised if the dimensions or storage class are invalid.
544 ConflictingDefinitionError
545 Raised if this DatasetType is already registered with a different
546 definition.
548 Notes
549 -----
550 This method cannot be called within transactions, as it needs to be
551 able to perform its own transaction to be concurrent.
552 """
553 raise NotImplementedError()
555 @abstractmethod
556 def removeDatasetType(self, name: str) -> None:
557 """Remove the named `DatasetType` from the registry.
559 .. warning::
561 Registry implementations can cache the dataset type definitions.
562 This means that deleting the dataset type definition may result in
563 unexpected behavior from other butler processes that are active
564 that have not seen the deletion.
566 Parameters
567 ----------
568 name : `str`
569 Name of the type to be removed.
571 Raises
572 ------
573 lsst.daf.butler.registry.OrphanedRecordError
574 Raised if an attempt is made to remove the dataset type definition
575 when there are already datasets associated with it.
577 Notes
578 -----
579 If the dataset type is not registered the method will return without
580 action.
581 """
582 raise NotImplementedError()
584 @abstractmethod
585 def getDatasetType(self, name: str) -> DatasetType:
586 """Get the `DatasetType`.
588 Parameters
589 ----------
590 name : `str`
591 Name of the type.
593 Returns
594 -------
595 type : `DatasetType`
596 The `DatasetType` associated with the given name.
598 Raises
599 ------
600 KeyError
601 Requested named DatasetType could not be found in registry.
602 """
603 raise NotImplementedError()
605 @abstractmethod
606 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool:
607 """Test whether the given dataset ID generation mode is supported by
608 `insertDatasets`.
610 Parameters
611 ----------
612 mode : `DatasetIdGenEnum`
613 Enum value for the mode to test.
615 Returns
616 -------
617 supported : `bool`
618 Whether the given mode is supported.
619 """
620 raise NotImplementedError()
622 @abstractmethod
623 def findDataset(self, datasetType: Union[DatasetType, str], dataId: Optional[DataId] = None, *,
624 collections: Any = None, timespan: Optional[Timespan] = None,
625 **kwargs: Any) -> Optional[DatasetRef]:
626 """Find a dataset given its `DatasetType` and data ID.
628 This can be used to obtain a `DatasetRef` that permits the dataset to
629 be read from a `Datastore`. If the dataset is a component and can not
630 be found using the provided dataset type, a dataset ref for the parent
631 will be returned instead but with the correct dataset type.
633 Parameters
634 ----------
635 datasetType : `DatasetType` or `str`
636 A `DatasetType` or the name of one.
637 dataId : `dict` or `DataCoordinate`, optional
638 A `dict`-like object containing the `Dimension` links that identify
639 the dataset within a collection.
640 collections, optional.
641 An expression that fully or partially identifies the collections to
642 search for the dataset; see
643 :ref:`daf_butler_collection_expressions` for more information.
644 Defaults to ``self.defaults.collections``.
645 timespan : `Timespan`, optional
646 A timespan that the validity range of the dataset must overlap.
647 If not provided, any `~CollectionType.CALIBRATION` collections
648 matched by the ``collections`` argument will not be searched.
649 **kwargs
650 Additional keyword arguments passed to
651 `DataCoordinate.standardize` to convert ``dataId`` to a true
652 `DataCoordinate` or augment an existing one.
654 Returns
655 -------
656 ref : `DatasetRef`
657 A reference to the dataset, or `None` if no matching Dataset
658 was found.
660 Raises
661 ------
662 TypeError
663 Raised if ``collections`` is `None` and
664 ``self.defaults.collections`` is `None`.
665 LookupError
666 Raised if one or more data ID keys are missing.
667 KeyError
668 Raised if the dataset type does not exist.
669 MissingCollectionError
670 Raised if any of ``collections`` does not exist in the registry.
672 Notes
673 -----
674 This method simply returns `None` and does not raise an exception even
675 when the set of collections searched is intrinsically incompatible with
676 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
677 only `~CollectionType.CALIBRATION` collections are being searched.
678 This may make it harder to debug some lookup failures, but the behavior
679 is intentional; we consider it more important that failed searches are
680 reported consistently, regardless of the reason, and that adding
681 additional collections that do not contain a match to the search path
682 never changes the behavior.
683 """
684 raise NotImplementedError()
686 @abstractmethod
687 def insertDatasets(self, datasetType: Union[DatasetType, str], dataIds: Iterable[DataId],
688 run: Optional[str] = None, expand: bool = True,
689 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE) -> List[DatasetRef]:
690 """Insert one or more datasets into the `Registry`
692 This always adds new datasets; to associate existing datasets with
693 a new collection, use ``associate``.
695 Parameters
696 ----------
697 datasetType : `DatasetType` or `str`
698 A `DatasetType` or the name of one.
699 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate`
700 Dimension-based identifiers for the new datasets.
701 run : `str`, optional
702 The name of the run that produced the datasets. Defaults to
703 ``self.defaults.run``.
704 expand : `bool`, optional
705 If `True` (default), expand data IDs as they are inserted. This is
706 necessary in general to allow datastore to generate file templates,
707 but it may be disabled if the caller can guarantee this is
708 unnecessary.
709 idGenerationMode : `DatasetIdGenEnum`, optional
710 Specifies option for generating dataset IDs. By default unique IDs
711 are generated for each inserted dataset.
713 Returns
714 -------
715 refs : `list` of `DatasetRef`
716 Resolved `DatasetRef` instances for all given data IDs (in the same
717 order).
719 Raises
720 ------
721 TypeError
722 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
723 ConflictingDefinitionError
724 If a dataset with the same dataset type and data ID as one of those
725 given already exists in ``run``.
726 MissingCollectionError
727 Raised if ``run`` does not exist in the registry.
728 """
729 raise NotImplementedError()
731 @abstractmethod
732 def _importDatasets(self, datasets: Iterable[DatasetRef], expand: bool = True,
733 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE,
734 reuseIds: bool = False) -> List[DatasetRef]:
735 """Import one or more datasets into the `Registry`.
737 Difference from `insertDatasets` method is that this method accepts
738 `DatasetRef` instances which should already be resolved and have a
739 dataset ID. If registry supports globally-unique dataset IDs (e.g.
740 `uuid.UUID`) then datasets which already exist in the registry will be
741 ignored if imported again.
743 Parameters
744 ----------
745 datasets : `~collections.abc.Iterable` of `DatasetRef`
746 Datasets to be inserted. All `DatasetRef` instances must have
747 identical ``datasetType`` and ``run`` attributes. ``run``
748 attribute can be `None` and defaults to ``self.defaults.run``.
749 Datasets can specify ``id`` attribute which will be used for
750 inserted datasets. All dataset IDs must have the same type
751 (`int` or `uuid.UUID`), if type of dataset IDs does not match
752 configured backend then IDs will be ignored and new IDs will be
753 generated by backend.
754 expand : `bool`, optional
755 If `True` (default), expand data IDs as they are inserted. This is
756 necessary in general to allow datastore to generate file templates,
757 but it may be disabled if the caller can guarantee this is
758 unnecessary.
759 idGenerationMode : `DatasetIdGenEnum`, optional
760 Specifies option for generating dataset IDs when IDs are not
761 provided or their type does not match backend type. By default
762 unique IDs are generated for each inserted dataset.
763 reuseIds : `bool`, optional
764 If `True` then forces re-use of imported dataset IDs for integer
765 IDs which are normally generated as auto-incremented; exception
766 will be raised if imported IDs clash with existing ones. This
767 option has no effect on the use of globally-unique IDs which are
768 always re-used (or generated if integer IDs are being imported).
770 Returns
771 -------
772 refs : `list` of `DatasetRef`
773 Resolved `DatasetRef` instances for all given data IDs (in the same
774 order). If any of ``datasets`` has an ID which already exists in
775 the database then it will not be inserted or updated, but a
776 resolved `DatasetRef` will be returned for it in any case.
778 Raises
779 ------
780 TypeError
781 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
782 ConflictingDefinitionError
783 If a dataset with the same dataset type and data ID as one of those
784 given already exists in ``run``.
785 MissingCollectionError
786 Raised if ``run`` does not exist in the registry.
788 Notes
789 -----
790 This method is considered package-private and internal to Butler
791 implementation. Clients outside daf_butler package should not use this
792 method.
793 """
794 raise NotImplementedError()
796 @abstractmethod
797 def getDataset(self, id: DatasetId) -> Optional[DatasetRef]:
798 """Retrieve a Dataset entry.
800 Parameters
801 ----------
802 id : `DatasetId`
803 The unique identifier for the dataset.
805 Returns
806 -------
807 ref : `DatasetRef` or `None`
808 A ref to the Dataset, or `None` if no matching Dataset
809 was found.
810 """
811 raise NotImplementedError()
813 @abstractmethod
814 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None:
815 """Remove datasets from the Registry.
817 The datasets will be removed unconditionally from all collections, and
818 any `Quantum` that consumed this dataset will instead be marked with
819 having a NULL input. `Datastore` records will *not* be deleted; the
820 caller is responsible for ensuring that the dataset has already been
821 removed from all Datastores.
823 Parameters
824 ----------
825 refs : `Iterable` of `DatasetRef`
826 References to the datasets to be removed. Must include a valid
827 ``id`` attribute, and should be considered invalidated upon return.
829 Raises
830 ------
831 AmbiguousDatasetError
832 Raised if any ``ref.id`` is `None`.
833 OrphanedRecordError
834 Raised if any dataset is still present in any `Datastore`.
835 """
836 raise NotImplementedError()
838 @abstractmethod
839 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
840 """Add existing datasets to a `~CollectionType.TAGGED` collection.
842 If a DatasetRef with the same exact ID is already in a collection
843 nothing is changed. If a `DatasetRef` with the same `DatasetType` and
844 data ID but with different ID exists in the collection,
845 `ConflictingDefinitionError` is raised.
847 Parameters
848 ----------
849 collection : `str`
850 Indicates the collection the datasets should be associated with.
851 refs : `Iterable` [ `DatasetRef` ]
852 An iterable of resolved `DatasetRef` instances that already exist
853 in this `Registry`.
855 Raises
856 ------
857 ConflictingDefinitionError
858 If a Dataset with the given `DatasetRef` already exists in the
859 given collection.
860 AmbiguousDatasetError
861 Raised if ``any(ref.id is None for ref in refs)``.
862 MissingCollectionError
863 Raised if ``collection`` does not exist in the registry.
864 TypeError
865 Raise adding new datasets to the given ``collection`` is not
866 allowed.
867 """
868 raise NotImplementedError()
870 @abstractmethod
871 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
872 """Remove existing datasets from a `~CollectionType.TAGGED` collection.
874 ``collection`` and ``ref`` combinations that are not currently
875 associated are silently ignored.
877 Parameters
878 ----------
879 collection : `str`
880 The collection the datasets should no longer be associated with.
881 refs : `Iterable` [ `DatasetRef` ]
882 An iterable of resolved `DatasetRef` instances that already exist
883 in this `Registry`.
885 Raises
886 ------
887 AmbiguousDatasetError
888 Raised if any of the given dataset references is unresolved.
889 MissingCollectionError
890 Raised if ``collection`` does not exist in the registry.
891 TypeError
892 Raise adding new datasets to the given ``collection`` is not
893 allowed.
894 """
895 raise NotImplementedError()
897 @abstractmethod
898 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None:
899 """Associate one or more datasets with a calibration collection and a
900 validity range within it.
902 Parameters
903 ----------
904 collection : `str`
905 The name of an already-registered `~CollectionType.CALIBRATION`
906 collection.
907 refs : `Iterable` [ `DatasetRef` ]
908 Datasets to be associated.
909 timespan : `Timespan`
910 The validity range for these datasets within the collection.
912 Raises
913 ------
914 AmbiguousDatasetError
915 Raised if any of the given `DatasetRef` instances is unresolved.
916 ConflictingDefinitionError
917 Raised if the collection already contains a different dataset with
918 the same `DatasetType` and data ID and an overlapping validity
919 range.
920 TypeError
921 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
922 collection or if one or more datasets are of a dataset type for
923 which `DatasetType.isCalibration` returns `False`.
924 """
925 raise NotImplementedError()
927 @abstractmethod
928 def decertify(self, collection: str, datasetType: Union[str, DatasetType], timespan: Timespan, *,
929 dataIds: Optional[Iterable[DataId]] = None) -> None:
930 """Remove or adjust datasets to clear a validity range within a
931 calibration collection.
933 Parameters
934 ----------
935 collection : `str`
936 The name of an already-registered `~CollectionType.CALIBRATION`
937 collection.
938 datasetType : `str` or `DatasetType`
939 Name or `DatasetType` instance for the datasets to be decertified.
940 timespan : `Timespan`, optional
941 The validity range to remove datasets from within the collection.
942 Datasets that overlap this range but are not contained by it will
943 have their validity ranges adjusted to not overlap it, which may
944 split a single dataset validity range into two.
945 dataIds : `Iterable` [ `DataId` ], optional
946 Data IDs that should be decertified within the given validity range
947 If `None`, all data IDs for ``self.datasetType`` will be
948 decertified.
950 Raises
951 ------
952 TypeError
953 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
954 collection or if ``datasetType.isCalibration() is False``.
955 """
956 raise NotImplementedError()
958 @abstractmethod
959 def getDatastoreBridgeManager(self) -> DatastoreRegistryBridgeManager:
960 """Return an object that allows a new `Datastore` instance to
961 communicate with this `Registry`.
963 Returns
964 -------
965 manager : `DatastoreRegistryBridgeManager`
966 Object that mediates communication between this `Registry` and its
967 associated datastores.
968 """
969 raise NotImplementedError()
971 @abstractmethod
972 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]:
973 """Retrieve datastore locations for a given dataset.
975 Parameters
976 ----------
977 ref : `DatasetRef`
978 A reference to the dataset for which to retrieve storage
979 information.
981 Returns
982 -------
983 datastores : `Iterable` [ `str` ]
984 All the matching datastores holding this dataset.
986 Raises
987 ------
988 AmbiguousDatasetError
989 Raised if ``ref.id`` is `None`.
990 """
991 raise NotImplementedError()
993 @abstractmethod
994 def expandDataId(self, dataId: Optional[DataId] = None, *, graph: Optional[DimensionGraph] = None,
995 records: Optional[NameLookupMapping[DimensionElement, Optional[DimensionRecord]]] = None,
996 withDefaults: bool = True,
997 **kwargs: Any) -> DataCoordinate:
998 """Expand a dimension-based data ID to include additional information.
1000 Parameters
1001 ----------
1002 dataId : `DataCoordinate` or `dict`, optional
1003 Data ID to be expanded; augmented and overridden by ``kwargs``.
1004 graph : `DimensionGraph`, optional
1005 Set of dimensions for the expanded ID. If `None`, the dimensions
1006 will be inferred from the keys of ``dataId`` and ``kwargs``.
1007 Dimensions that are in ``dataId`` or ``kwargs`` but not in
1008 ``graph`` are silently ignored, providing a way to extract and
1009 ``graph`` expand a subset of a data ID.
1010 records : `Mapping` [`str`, `DimensionRecord`], optional
1011 Dimension record data to use before querying the database for that
1012 data, keyed by element name.
1013 withDefaults : `bool`, optional
1014 Utilize ``self.defaults.dataId`` to fill in missing governor
1015 dimension key-value pairs. Defaults to `True` (i.e. defaults are
1016 used).
1017 **kwargs
1018 Additional keywords are treated like additional key-value pairs for
1019 ``dataId``, extending and overriding
1021 Returns
1022 -------
1023 expanded : `DataCoordinate`
1024 A data ID that includes full metadata for all of the dimensions it
1025 identifieds, i.e. guarantees that ``expanded.hasRecords()`` and
1026 ``expanded.hasFull()`` both return `True`.
1027 """
1028 raise NotImplementedError()
1030 @abstractmethod
1031 def insertDimensionData(self, element: Union[DimensionElement, str],
1032 *data: Union[Mapping[str, Any], DimensionRecord],
1033 conform: bool = True,
1034 replace: bool = False) -> None:
1035 """Insert one or more dimension records into the database.
1037 Parameters
1038 ----------
1039 element : `DimensionElement` or `str`
1040 The `DimensionElement` or name thereof that identifies the table
1041 records will be inserted into.
1042 data : `dict` or `DimensionRecord` (variadic)
1043 One or more records to insert.
1044 conform : `bool`, optional
1045 If `False` (`True` is default) perform no checking or conversions,
1046 and assume that ``element`` is a `DimensionElement` instance and
1047 ``data`` is a one or more `DimensionRecord` instances of the
1048 appropriate subclass.
1049 replace: `bool`, optional
1050 If `True` (`False` is default), replace existing records in the
1051 database if there is a conflict.
1052 """
1053 raise NotImplementedError()
1055 @abstractmethod
1056 def syncDimensionData(self, element: Union[DimensionElement, str],
1057 row: Union[Mapping[str, Any], DimensionRecord],
1058 conform: bool = True,
1059 update: bool = False) -> Union[bool, Dict[str, Any]]:
1060 """Synchronize the given dimension record with the database, inserting
1061 if it does not already exist and comparing values if it does.
1063 Parameters
1064 ----------
1065 element : `DimensionElement` or `str`
1066 The `DimensionElement` or name thereof that identifies the table
1067 records will be inserted into.
1068 row : `dict` or `DimensionRecord`
1069 The record to insert.
1070 conform : `bool`, optional
1071 If `False` (`True` is default) perform no checking or conversions,
1072 and assume that ``element`` is a `DimensionElement` instance and
1073 ``data`` is a one or more `DimensionRecord` instances of the
1074 appropriate subclass.
1075 update: `bool`, optional
1076 If `True` (`False` is default), update the existing record in the
1077 database if there is a conflict.
1079 Returns
1080 -------
1081 inserted_or_updated : `bool` or `dict`
1082 `True` if a new row was inserted, `False` if no changes were
1083 needed, or a `dict` mapping updated column names to their old
1084 values if an update was performed (only possible if
1085 ``update=True``).
1087 Raises
1088 ------
1089 ConflictingDefinitionError
1090 Raised if the record exists in the database (according to primary
1091 key lookup) but is inconsistent with the given one.
1092 """
1093 raise NotImplementedError()
1095 @abstractmethod
1096 def queryDatasetTypes(self, expression: Any = ..., *, components: Optional[bool] = None
1097 ) -> Iterator[DatasetType]:
1098 """Iterate over the dataset types whose names match an expression.
1100 Parameters
1101 ----------
1102 expression : `Any`, optional
1103 An expression that fully or partially identifies the dataset types
1104 to return, such as a `str`, `re.Pattern`, or iterable thereof.
1105 `...` can be used to return all dataset types, and is the default.
1106 See :ref:`daf_butler_dataset_type_expressions` for more
1107 information.
1108 components : `bool`, optional
1109 If `True`, apply all expression patterns to component dataset type
1110 names as well. If `False`, never apply patterns to components.
1111 If `None` (default), apply patterns to components only if their
1112 parent datasets were not matched by the expression.
1113 Fully-specified component datasets (`str` or `DatasetType`
1114 instances) are always included.
1116 Yields
1117 ------
1118 datasetType : `DatasetType`
1119 A `DatasetType` instance whose name matches ``expression``.
1120 """
1121 raise NotImplementedError()
1123 @abstractmethod
1124 def queryCollections(self, expression: Any = ...,
1125 datasetType: Optional[DatasetType] = None,
1126 collectionTypes: Iterable[CollectionType] = CollectionType.all(),
1127 flattenChains: bool = False,
1128 includeChains: Optional[bool] = None) -> Iterator[str]:
1129 """Iterate over the collections whose names match an expression.
1131 Parameters
1132 ----------
1133 expression : `Any`, optional
1134 An expression that identifies the collections to return, such as
1135 a `str` (for full matches or partial matches via globs),
1136 `re.Pattern` (for partial matches), or iterable thereof. `...`
1137 can be used to return all collections, and is the default.
1138 See :ref:`daf_butler_collection_expressions` for more information.
1139 datasetType : `DatasetType`, optional
1140 If provided, only yield collections that may contain datasets of
1141 this type. This is a conservative approximation in general; it may
1142 yield collections that do not have any such datasets.
1143 collectionTypes : `AbstractSet` [ `CollectionType` ], optional
1144 If provided, only yield collections of these types.
1145 flattenChains : `bool`, optional
1146 If `True` (`False` is default), recursively yield the child
1147 collections of matching `~CollectionType.CHAINED` collections.
1148 includeChains : `bool`, optional
1149 If `True`, yield records for matching `~CollectionType.CHAINED`
1150 collections. Default is the opposite of ``flattenChains``: include
1151 either CHAINED collections or their children, but not both.
1153 Yields
1154 ------
1155 collection : `str`
1156 The name of a collection that matches ``expression``.
1157 """
1158 raise NotImplementedError()
1160 @abstractmethod
1161 def queryDatasets(self, datasetType: Any, *,
1162 collections: Any = None,
1163 dimensions: Optional[Iterable[Union[Dimension, str]]] = None,
1164 dataId: Optional[DataId] = None,
1165 where: Optional[str] = None,
1166 findFirst: bool = False,
1167 components: Optional[bool] = None,
1168 bind: Optional[Mapping[str, Any]] = None,
1169 check: bool = True,
1170 **kwargs: Any) -> Iterable[DatasetRef]:
1171 """Query for and iterate over dataset references matching user-provided
1172 criteria.
1174 Parameters
1175 ----------
1176 datasetType
1177 An expression that fully or partially identifies the dataset types
1178 to be queried. Allowed types include `DatasetType`, `str`,
1179 `re.Pattern`, and iterables thereof. The special value `...` can
1180 be used to query all dataset types. See
1181 :ref:`daf_butler_dataset_type_expressions` for more information.
1182 collections: optional
1183 An expression that identifies the collections to search, such as a
1184 `str` (for full matches or partial matches via globs), `re.Pattern`
1185 (for partial matches), or iterable thereof. `...` can be used to
1186 search all collections (actually just all `~CollectionType.RUN`
1187 collections, because this will still find all datasets).
1188 If not provided, ``self.default.collections`` is used. See
1189 :ref:`daf_butler_collection_expressions` for more information.
1190 dimensions : `~collections.abc.Iterable` of `Dimension` or `str`
1191 Dimensions to include in the query (in addition to those used
1192 to identify the queried dataset type(s)), either to constrain
1193 the resulting datasets to those for which a matching dimension
1194 exists, or to relate the dataset type's dimensions to dimensions
1195 referenced by the ``dataId`` or ``where`` arguments.
1196 dataId : `dict` or `DataCoordinate`, optional
1197 A data ID whose key-value pairs are used as equality constraints
1198 in the query.
1199 where : `str`, optional
1200 A string expression similar to a SQL WHERE clause. May involve
1201 any column of a dimension table or (as a shortcut for the primary
1202 key column of a dimension table) dimension name. See
1203 :ref:`daf_butler_dimension_expressions` for more information.
1204 findFirst : `bool`, optional
1205 If `True` (`False` is default), for each result data ID, only
1206 yield one `DatasetRef` of each `DatasetType`, from the first
1207 collection in which a dataset of that dataset type appears
1208 (according to the order of ``collections`` passed in). If `True`,
1209 ``collections`` must not contain regular expressions and may not
1210 be `...`.
1211 components : `bool`, optional
1212 If `True`, apply all dataset expression patterns to component
1213 dataset type names as well. If `False`, never apply patterns to
1214 components. If `None` (default), apply patterns to components only
1215 if their parent datasets were not matched by the expression.
1216 Fully-specified component datasets (`str` or `DatasetType`
1217 instances) are always included.
1218 bind : `Mapping`, optional
1219 Mapping containing literal values that should be injected into the
1220 ``where`` expression, keyed by the identifiers they replace.
1221 check : `bool`, optional
1222 If `True` (default) check the query for consistency before
1223 executing it. This may reject some valid queries that resemble
1224 common mistakes (e.g. queries for visits without specifying an
1225 instrument).
1226 **kwargs
1227 Additional keyword arguments are forwarded to
1228 `DataCoordinate.standardize` when processing the ``dataId``
1229 argument (and may be used to provide a constraining data ID even
1230 when the ``dataId`` argument is `None`).
1232 Returns
1233 -------
1234 refs : `queries.DatasetQueryResults`
1235 Dataset references matching the given query criteria. Nested data
1236 IDs are guaranteed to include values for all implied dimensions
1237 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
1238 include dimension records (`DataCoordinate.hasRecords` will be
1239 `False`) unless `~queries.DatasetQueryResults.expanded` is called
1240 on the result object (which returns a new one).
1242 Raises
1243 ------
1244 TypeError
1245 Raised when the arguments are incompatible, such as when a
1246 collection wildcard is passed when ``findFirst`` is `True`, or
1247 when ``collections`` is `None` and``self.defaults.collections`` is
1248 also `None`.
1250 Notes
1251 -----
1252 When multiple dataset types are queried in a single call, the
1253 results of this operation are equivalent to querying for each dataset
1254 type separately in turn, and no information about the relationships
1255 between datasets of different types is included. In contexts where
1256 that kind of information is important, the recommended pattern is to
1257 use `queryDataIds` to first obtain data IDs (possibly with the
1258 desired dataset types and collections passed as constraints to the
1259 query), and then use multiple (generally much simpler) calls to
1260 `queryDatasets` with the returned data IDs passed as constraints.
1261 """
1262 raise NotImplementedError()
1264 @abstractmethod
1265 def queryDataIds(self, dimensions: Union[Iterable[Union[Dimension, str]], Dimension, str], *,
1266 dataId: Optional[DataId] = None,
1267 datasets: Any = None,
1268 collections: Any = None,
1269 where: Optional[str] = None,
1270 components: Optional[bool] = None,
1271 bind: Optional[Mapping[str, Any]] = None,
1272 check: bool = True,
1273 **kwargs: Any) -> DataCoordinateIterable:
1274 """Query for data IDs matching user-provided criteria.
1276 Parameters
1277 ----------
1278 dimensions : `Dimension` or `str`, or iterable thereof
1279 The dimensions of the data IDs to yield, as either `Dimension`
1280 instances or `str`. Will be automatically expanded to a complete
1281 `DimensionGraph`.
1282 dataId : `dict` or `DataCoordinate`, optional
1283 A data ID whose key-value pairs are used as equality constraints
1284 in the query.
1285 datasets : `Any`, optional
1286 An expression that fully or partially identifies dataset types
1287 that should constrain the yielded data IDs. For example, including
1288 "raw" here would constrain the yielded ``instrument``,
1289 ``exposure``, ``detector``, and ``physical_filter`` values to only
1290 those for which at least one "raw" dataset exists in
1291 ``collections``. Allowed types include `DatasetType`, `str`,
1292 `re.Pattern`, and iterables thereof. Unlike other dataset type
1293 expressions, ``...`` is not permitted - it doesn't make sense to
1294 constrain data IDs on the existence of *all* datasets.
1295 See :ref:`daf_butler_dataset_type_expressions` for more
1296 information.
1297 collections: `Any`, optional
1298 An expression that identifies the collections to search for
1299 datasets, such as a `str` (for full matches or partial matches
1300 via globs), `re.Pattern` (for partial matches), or iterable
1301 thereof. `...` can be used to search all collections (actually
1302 just all `~CollectionType.RUN` collections, because this will
1303 still find all datasets). If not provided,
1304 ``self.default.collections`` is used. Ignored unless ``datasets``
1305 is also passed. See :ref:`daf_butler_collection_expressions` for
1306 more information.
1307 where : `str`, optional
1308 A string expression similar to a SQL WHERE clause. May involve
1309 any column of a dimension table or (as a shortcut for the primary
1310 key column of a dimension table) dimension name. See
1311 :ref:`daf_butler_dimension_expressions` for more information.
1312 components : `bool`, optional
1313 If `True`, apply all dataset expression patterns to component
1314 dataset type names as well. If `False`, never apply patterns to
1315 components. If `None` (default), apply patterns to components only
1316 if their parent datasets were not matched by the expression.
1317 Fully-specified component datasets (`str` or `DatasetType`
1318 instances) are always included.
1319 bind : `Mapping`, optional
1320 Mapping containing literal values that should be injected into the
1321 ``where`` expression, keyed by the identifiers they replace.
1322 check : `bool`, optional
1323 If `True` (default) check the query for consistency before
1324 executing it. This may reject some valid queries that resemble
1325 common mistakes (e.g. queries for visits without specifying an
1326 instrument).
1327 **kwargs
1328 Additional keyword arguments are forwarded to
1329 `DataCoordinate.standardize` when processing the ``dataId``
1330 argument (and may be used to provide a constraining data ID even
1331 when the ``dataId`` argument is `None`).
1333 Returns
1334 -------
1335 dataIds : `DataCoordinateQueryResults`
1336 Data IDs matching the given query parameters. These are guaranteed
1337 to identify all dimensions (`DataCoordinate.hasFull` returns
1338 `True`), but will not contain `DimensionRecord` objects
1339 (`DataCoordinate.hasRecords` returns `False`). Call
1340 `DataCoordinateQueryResults.expanded` on the returned object to
1341 fetch those (and consider using
1342 `DataCoordinateQueryResults.materialize` on the returned object
1343 first if the expected number of rows is very large). See
1344 documentation for those methods for additional information.
1346 Raises
1347 ------
1348 TypeError
1349 Raised if ``collections`` is `None`, ``self.defaults.collections``
1350 is `None`, and ``datasets`` is not `None`.
1351 """
1352 raise NotImplementedError()
1354 @abstractmethod
1355 def queryDimensionRecords(self, element: Union[DimensionElement, str], *,
1356 dataId: Optional[DataId] = None,
1357 datasets: Any = None,
1358 collections: Any = None,
1359 where: Optional[str] = None,
1360 components: Optional[bool] = None,
1361 bind: Optional[Mapping[str, Any]] = None,
1362 check: bool = True,
1363 **kwargs: Any) -> Iterator[DimensionRecord]:
1364 """Query for dimension information matching user-provided criteria.
1366 Parameters
1367 ----------
1368 element : `DimensionElement` or `str`
1369 The dimension element to obtain records for.
1370 dataId : `dict` or `DataCoordinate`, optional
1371 A data ID whose key-value pairs are used as equality constraints
1372 in the query.
1373 datasets : `Any`, optional
1374 An expression that fully or partially identifies dataset types
1375 that should constrain the yielded records. See `queryDataIds` and
1376 :ref:`daf_butler_dataset_type_expressions` for more information.
1377 collections: `Any`, optional
1378 An expression that identifies the collections to search for
1379 datasets, such as a `str` (for full matches or partial matches
1380 via globs), `re.Pattern` (for partial matches), or iterable
1381 thereof. `...` can be used to search all collections (actually
1382 just all `~CollectionType.RUN` collections, because this will
1383 still find all datasets). If not provided,
1384 ``self.default.collections`` is used. Ignored unless ``datasets``
1385 is also passed. See :ref:`daf_butler_collection_expressions` for
1386 more information.
1387 where : `str`, optional
1388 A string expression similar to a SQL WHERE clause. See
1389 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
1390 information.
1391 components : `bool`, optional
1392 Whether to apply dataset expressions to components as well.
1393 See `queryDataIds` for more information.
1394 bind : `Mapping`, optional
1395 Mapping containing literal values that should be injected into the
1396 ``where`` expression, keyed by the identifiers they replace.
1397 check : `bool`, optional
1398 If `True` (default) check the query for consistency before
1399 executing it. This may reject some valid queries that resemble
1400 common mistakes (e.g. queries for visits without specifying an
1401 instrument).
1402 **kwargs
1403 Additional keyword arguments are forwarded to
1404 `DataCoordinate.standardize` when processing the ``dataId``
1405 argument (and may be used to provide a constraining data ID even
1406 when the ``dataId`` argument is `None`).
1408 Returns
1409 -------
1410 dataIds : `DataCoordinateQueryResults`
1411 Data IDs matching the given query parameters.
1412 """
1413 raise NotImplementedError()
1415 @abstractmethod
1416 def queryDatasetAssociations(
1417 self,
1418 datasetType: Union[str, DatasetType],
1419 collections: Any = ...,
1420 *,
1421 collectionTypes: Iterable[CollectionType] = CollectionType.all(),
1422 flattenChains: bool = False,
1423 ) -> Iterator[DatasetAssociation]:
1424 """Iterate over dataset-collection combinations where the dataset is in
1425 the collection.
1427 This method is a temporary placeholder for better support for
1428 assocation results in `queryDatasets`. It will probably be
1429 removed in the future, and should be avoided in production code
1430 whenever possible.
1432 Parameters
1433 ----------
1434 datasetType : `DatasetType` or `str`
1435 A dataset type object or the name of one.
1436 collections: `Any`, optional
1437 An expression that identifies the collections to search for
1438 datasets, such as a `str` (for full matches or partial matches
1439 via globs), `re.Pattern` (for partial matches), or iterable
1440 thereof. `...` can be used to search all collections (actually
1441 just all `~CollectionType.RUN` collections, because this will still
1442 find all datasets). If not provided, ``self.default.collections``
1443 is used. See :ref:`daf_butler_collection_expressions` for more
1444 information.
1445 collectionTypes : `AbstractSet` [ `CollectionType` ], optional
1446 If provided, only yield associations from collections of these
1447 types.
1448 flattenChains : `bool`, optional
1449 If `True` (default) search in the children of
1450 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED``
1451 collections are ignored.
1453 Yields
1454 ------
1455 association : `DatasetAssociation`
1456 Object representing the relationship beween a single dataset and
1457 a single collection.
1459 Raises
1460 ------
1461 TypeError
1462 Raised if ``collections`` is `None` and
1463 ``self.defaults.collections`` is `None`.
1464 """
1465 raise NotImplementedError()
1467 storageClasses: StorageClassFactory
1468 """All storage classes known to the registry (`StorageClassFactory`).
1469 """