Coverage for python/lsst/daf/butler/registry/_registry.py : 62%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = (
25 "Registry",
26)
28from abc import ABC, abstractmethod
29import contextlib
30import logging
31from typing import (
32 Any,
33 Dict,
34 Iterable,
35 Iterator,
36 List,
37 Mapping,
38 Optional,
39 Tuple,
40 Type,
41 TYPE_CHECKING,
42 Union,
43)
45from lsst.utils import doImport
47from ..core import (
48 ButlerURI,
49 Config,
50 DataCoordinate,
51 DataCoordinateIterable,
52 DataId,
53 DatasetAssociation,
54 DatasetId,
55 DatasetRef,
56 DatasetType,
57 Dimension,
58 DimensionConfig,
59 DimensionElement,
60 DimensionGraph,
61 DimensionRecord,
62 DimensionUniverse,
63 NameLookupMapping,
64 StorageClassFactory,
65 Timespan,
66)
68from ._config import RegistryConfig
69from ._collectionType import CollectionType
70from ._defaults import RegistryDefaults
71from .interfaces import DatasetIdGenEnum
72from .wildcards import CollectionSearch
73from .summaries import CollectionSummary
75if TYPE_CHECKING: 75 ↛ 76line 75 didn't jump to line 76, because the condition on line 75 was never true
76 from .._butlerConfig import ButlerConfig
77 from .interfaces import (
78 CollectionRecord,
79 DatastoreRegistryBridgeManager,
80 )
82_LOG = logging.getLogger(__name__)
85class Registry(ABC):
86 """Abstract Registry interface.
88 Each registry implementation can have its own constructor parameters.
89 The assumption is that an instance of a specific subclass will be
90 constructed from configuration using `Registry.fromConfig()`.
91 The base class will look for a ``cls`` entry and call that specific
92 `fromConfig()` method.
94 All subclasses should store `RegistryDefaults` in a ``_defaults``
95 property. No other properties are assumed shared between implementations.
96 """
98 defaultConfigFile: Optional[str] = None
99 """Path to configuration defaults. Accessed within the ``configs`` resource
100 or relative to a search path. Can be None if no defaults specified.
101 """
103 @classmethod
104 def forceRegistryConfig(cls, config: Optional[Union[ButlerConfig,
105 RegistryConfig, Config, str]]) -> RegistryConfig:
106 """Force the supplied config to a `RegistryConfig`.
108 Parameters
109 ----------
110 config : `RegistryConfig`, `Config` or `str` or `None`
111 Registry configuration, if missing then default configuration will
112 be loaded from registry.yaml.
114 Returns
115 -------
116 registry_config : `RegistryConfig`
117 A registry config.
118 """
119 if not isinstance(config, RegistryConfig):
120 if isinstance(config, (str, Config)) or config is None:
121 config = RegistryConfig(config)
122 else:
123 raise ValueError(f"Incompatible Registry configuration: {config}")
124 return config
126 @classmethod
127 def determineTrampoline(cls,
128 config: Optional[Union[ButlerConfig,
129 RegistryConfig,
130 Config,
131 str]]) -> Tuple[Type[Registry], RegistryConfig]:
132 """Return class to use to instantiate real registry.
134 Parameters
135 ----------
136 config : `RegistryConfig` or `str`, optional
137 Registry configuration, if missing then default configuration will
138 be loaded from registry.yaml.
140 Returns
141 -------
142 requested_cls : `type` of `Registry`
143 The real registry class to use.
144 registry_config : `RegistryConfig`
145 The `RegistryConfig` to use.
146 """
147 config = cls.forceRegistryConfig(config)
149 # Default to the standard registry
150 registry_cls = doImport(config.get("cls", "lsst.daf.butler.registries.sql.SqlRegistry"))
151 if registry_cls is cls:
152 raise ValueError("Can not instantiate the abstract base Registry from config")
153 return registry_cls, config
155 @classmethod
156 def createFromConfig(cls, config: Optional[Union[RegistryConfig, str]] = None,
157 dimensionConfig: Optional[Union[DimensionConfig, str]] = None,
158 butlerRoot: Optional[str] = None) -> Registry:
159 """Create registry database and return `Registry` instance.
161 This method initializes database contents, database must be empty
162 prior to calling this method.
164 Parameters
165 ----------
166 config : `RegistryConfig` or `str`, optional
167 Registry configuration, if missing then default configuration will
168 be loaded from registry.yaml.
169 dimensionConfig : `DimensionConfig` or `str`, optional
170 Dimensions configuration, if missing then default configuration
171 will be loaded from dimensions.yaml.
172 butlerRoot : `str`, optional
173 Path to the repository root this `Registry` will manage.
175 Returns
176 -------
177 registry : `Registry`
178 A new `Registry` instance.
180 Notes
181 -----
182 This class will determine the concrete `Registry` subclass to
183 use from configuration. Each subclass should implement this method
184 even if it can not create a registry.
185 """
186 registry_cls, registry_config = cls.determineTrampoline(config)
187 return registry_cls.createFromConfig(registry_config, dimensionConfig, butlerRoot)
189 @classmethod
190 def fromConfig(cls, config: Union[ButlerConfig, RegistryConfig, Config, str],
191 butlerRoot: Optional[Union[str, ButlerURI]] = None, writeable: bool = True,
192 defaults: Optional[RegistryDefaults] = None) -> Registry:
193 """Create `Registry` subclass instance from `config`.
195 Registry database must be initialized prior to calling this method.
197 Parameters
198 ----------
199 config : `ButlerConfig`, `RegistryConfig`, `Config` or `str`
200 Registry configuration
201 butlerRoot : `str` or `ButlerURI`, optional
202 Path to the repository root this `Registry` will manage.
203 writeable : `bool`, optional
204 If `True` (default) create a read-write connection to the database.
205 defaults : `RegistryDefaults`, optional
206 Default collection search path and/or output `~CollectionType.RUN`
207 collection.
209 Returns
210 -------
211 registry : `Registry` (subclass)
212 A new `Registry` subclass instance.
214 Notes
215 -----
216 This class will determine the concrete `Registry` subclass to
217 use from configuration. Each subclass should implement this method.
218 """
219 # The base class implementation should trampoline to the correct
220 # subclass. No implementation should ever use this implementation
221 # directly. If no class is specified, default to the standard
222 # registry.
223 registry_cls, registry_config = cls.determineTrampoline(config)
224 return registry_cls.fromConfig(config, butlerRoot, writeable, defaults)
226 @abstractmethod
227 def isWriteable(self) -> bool:
228 """Return `True` if this registry allows write operations, and `False`
229 otherwise.
230 """
231 raise NotImplementedError()
233 @abstractmethod
234 def copy(self, defaults: Optional[RegistryDefaults] = None) -> Registry:
235 """Create a new `Registry` backed by the same data repository and
236 connection as this one, but independent defaults.
238 Parameters
239 ----------
240 defaults : `RegistryDefaults`, optional
241 Default collections and data ID values for the new registry. If
242 not provided, ``self.defaults`` will be used (but future changes
243 to either registry's defaults will not affect the other).
245 Returns
246 -------
247 copy : `Registry`
248 A new `Registry` instance with its own defaults.
250 Notes
251 -----
252 Because the new registry shares a connection with the original, they
253 also share transaction state (despite the fact that their `transaction`
254 context manager methods do not reflect this), and must be used with
255 care.
256 """
257 raise NotImplementedError()
259 @property
260 @abstractmethod
261 def dimensions(self) -> DimensionUniverse:
262 """All dimensions recognized by this `Registry` (`DimensionUniverse`).
263 """
264 raise NotImplementedError()
266 @property
267 def defaults(self) -> RegistryDefaults:
268 """Default collection search path and/or output `~CollectionType.RUN`
269 collection (`RegistryDefaults`).
271 This is an immutable struct whose components may not be set
272 individually, but the entire struct can be set by assigning to this
273 property.
274 """
275 return self._defaults
277 @defaults.setter
278 def defaults(self, value: RegistryDefaults) -> None:
279 if value.run is not None:
280 self.registerRun(value.run)
281 value.finish(self)
282 self._defaults = value
284 @abstractmethod
285 def refresh(self) -> None:
286 """Refresh all in-memory state by querying the database.
288 This may be necessary to enable querying for entities added by other
289 registry instances after this one was constructed.
290 """
291 raise NotImplementedError()
293 @contextlib.contextmanager
294 @abstractmethod
295 def transaction(self, *, savepoint: bool = False) -> Iterator[None]:
296 """Return a context manager that represents a transaction.
297 """
298 raise NotImplementedError()
300 def resetConnectionPool(self) -> None:
301 """Reset connection pool for registry if relevant.
303 This operation can be used reset connections to servers when
304 using registry with fork-based multiprocessing. This method should
305 usually be called by the child process immediately
306 after the fork.
308 The base class implementation is a no-op.
309 """
310 pass
312 @abstractmethod
313 def registerCollection(self, name: str, type: CollectionType = CollectionType.TAGGED,
314 doc: Optional[str] = None) -> None:
315 """Add a new collection if one with the given name does not exist.
317 Parameters
318 ----------
319 name : `str`
320 The name of the collection to create.
321 type : `CollectionType`
322 Enum value indicating the type of collection to create.
323 doc : `str`, optional
324 Documentation string for the collection.
326 Notes
327 -----
328 This method cannot be called within transactions, as it needs to be
329 able to perform its own transaction to be concurrent.
330 """
331 raise NotImplementedError()
333 @abstractmethod
334 def getCollectionType(self, name: str) -> CollectionType:
335 """Return an enumeration value indicating the type of the given
336 collection.
338 Parameters
339 ----------
340 name : `str`
341 The name of the collection.
343 Returns
344 -------
345 type : `CollectionType`
346 Enum value indicating the type of this collection.
348 Raises
349 ------
350 MissingCollectionError
351 Raised if no collection with the given name exists.
352 """
353 raise NotImplementedError()
355 @abstractmethod
356 def _get_collection_record(self, name: str) -> CollectionRecord:
357 """Return the record for this collection.
359 Parameters
360 ----------
361 name : `str`
362 Name of the collection for which the record is to be retrieved.
364 Returns
365 -------
366 record : `CollectionRecord`
367 The record for this collection.
368 """
369 raise NotImplementedError()
371 @abstractmethod
372 def registerRun(self, name: str, doc: Optional[str] = None) -> None:
373 """Add a new run if one with the given name does not exist.
375 Parameters
376 ----------
377 name : `str`
378 The name of the run to create.
379 doc : `str`, optional
380 Documentation string for the collection.
382 Notes
383 -----
384 This method cannot be called within transactions, as it needs to be
385 able to perform its own transaction to be concurrent.
386 """
387 raise NotImplementedError()
389 @abstractmethod
390 def removeCollection(self, name: str) -> None:
391 """Completely remove the given collection.
393 Parameters
394 ----------
395 name : `str`
396 The name of the collection to remove.
398 Raises
399 ------
400 MissingCollectionError
401 Raised if no collection with the given name exists.
403 Notes
404 -----
405 If this is a `~CollectionType.RUN` collection, all datasets and quanta
406 in it are also fully removed. This requires that those datasets be
407 removed (or at least trashed) from any datastores that hold them first.
409 A collection may not be deleted as long as it is referenced by a
410 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must
411 be deleted or redefined first.
412 """
413 raise NotImplementedError()
415 @abstractmethod
416 def getCollectionChain(self, parent: str) -> CollectionSearch:
417 """Return the child collections in a `~CollectionType.CHAINED`
418 collection.
420 Parameters
421 ----------
422 parent : `str`
423 Name of the chained collection. Must have already been added via
424 a call to `Registry.registerCollection`.
426 Returns
427 -------
428 children : `CollectionSearch`
429 An object that defines the search path of the collection.
430 See :ref:`daf_butler_collection_expressions` for more information.
432 Raises
433 ------
434 MissingCollectionError
435 Raised if ``parent`` does not exist in the `Registry`.
436 TypeError
437 Raised if ``parent`` does not correspond to a
438 `~CollectionType.CHAINED` collection.
439 """
440 raise NotImplementedError()
442 @abstractmethod
443 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None:
444 """Define or redefine a `~CollectionType.CHAINED` collection.
446 Parameters
447 ----------
448 parent : `str`
449 Name of the chained collection. Must have already been added via
450 a call to `Registry.registerCollection`.
451 children : `Any`
452 An expression defining an ordered search of child collections,
453 generally an iterable of `str`; see
454 :ref:`daf_butler_collection_expressions` for more information.
455 flatten : `bool`, optional
456 If `True` (`False` is default), recursively flatten out any nested
457 `~CollectionType.CHAINED` collections in ``children`` first.
459 Raises
460 ------
461 MissingCollectionError
462 Raised when any of the given collections do not exist in the
463 `Registry`.
464 TypeError
465 Raised if ``parent`` does not correspond to a
466 `~CollectionType.CHAINED` collection.
467 ValueError
468 Raised if the given collections contains a cycle.
469 """
470 raise NotImplementedError()
472 @abstractmethod
473 def getCollectionDocumentation(self, collection: str) -> Optional[str]:
474 """Retrieve the documentation string for a collection.
476 Parameters
477 ----------
478 name : `str`
479 Name of the collection.
481 Returns
482 -------
483 docs : `str` or `None`
484 Docstring for the collection with the given name.
485 """
486 raise NotImplementedError()
488 @abstractmethod
489 def setCollectionDocumentation(self, collection: str, doc: Optional[str]) -> None:
490 """Set the documentation string for a collection.
492 Parameters
493 ----------
494 name : `str`
495 Name of the collection.
496 docs : `str` or `None`
497 Docstring for the collection with the given name; will replace any
498 existing docstring. Passing `None` will remove any existing
499 docstring.
500 """
501 raise NotImplementedError()
503 @abstractmethod
504 def getCollectionSummary(self, collection: str) -> CollectionSummary:
505 """Return a summary for the given collection.
507 Parameters
508 ----------
509 collection : `str`
510 Name of the collection for which a summary is to be retrieved.
512 Returns
513 -------
514 summary : `CollectionSummary`
515 Summary of the dataset types and governor dimension values in
516 this collection.
517 """
518 raise NotImplementedError()
520 @abstractmethod
521 def registerDatasetType(self, datasetType: DatasetType) -> bool:
522 """
523 Add a new `DatasetType` to the Registry.
525 It is not an error to register the same `DatasetType` twice.
527 Parameters
528 ----------
529 datasetType : `DatasetType`
530 The `DatasetType` to be added.
532 Returns
533 -------
534 inserted : `bool`
535 `True` if ``datasetType`` was inserted, `False` if an identical
536 existing `DatsetType` was found. Note that in either case the
537 DatasetType is guaranteed to be defined in the Registry
538 consistently with the given definition.
540 Raises
541 ------
542 ValueError
543 Raised if the dimensions or storage class are invalid.
544 ConflictingDefinitionError
545 Raised if this DatasetType is already registered with a different
546 definition.
548 Notes
549 -----
550 This method cannot be called within transactions, as it needs to be
551 able to perform its own transaction to be concurrent.
552 """
553 raise NotImplementedError()
555 @abstractmethod
556 def removeDatasetType(self, name: str) -> None:
557 """Remove the named `DatasetType` from the registry.
559 .. warning::
561 Registry implementations can cache the dataset type definitions.
562 This means that deleting the dataset type definition may result in
563 unexpected behavior from other butler processes that are active
564 that have not seen the deletion.
566 Parameters
567 ----------
568 name : `str`
569 Name of the type to be removed.
571 Raises
572 ------
573 lsst.daf.butler.registry.OrphanedRecordError
574 Raised if an attempt is made to remove the dataset type definition
575 when there are already datasets associated with it.
577 Notes
578 -----
579 If the dataset type is not registered the method will return without
580 action.
581 """
582 raise NotImplementedError()
584 @abstractmethod
585 def getDatasetType(self, name: str) -> DatasetType:
586 """Get the `DatasetType`.
588 Parameters
589 ----------
590 name : `str`
591 Name of the type.
593 Returns
594 -------
595 type : `DatasetType`
596 The `DatasetType` associated with the given name.
598 Raises
599 ------
600 KeyError
601 Requested named DatasetType could not be found in registry.
602 """
603 raise NotImplementedError()
605 @abstractmethod
606 def findDataset(self, datasetType: Union[DatasetType, str], dataId: Optional[DataId] = None, *,
607 collections: Any = None, timespan: Optional[Timespan] = None,
608 **kwargs: Any) -> Optional[DatasetRef]:
609 """Find a dataset given its `DatasetType` and data ID.
611 This can be used to obtain a `DatasetRef` that permits the dataset to
612 be read from a `Datastore`. If the dataset is a component and can not
613 be found using the provided dataset type, a dataset ref for the parent
614 will be returned instead but with the correct dataset type.
616 Parameters
617 ----------
618 datasetType : `DatasetType` or `str`
619 A `DatasetType` or the name of one.
620 dataId : `dict` or `DataCoordinate`, optional
621 A `dict`-like object containing the `Dimension` links that identify
622 the dataset within a collection.
623 collections, optional.
624 An expression that fully or partially identifies the collections to
625 search for the dataset; see
626 :ref:`daf_butler_collection_expressions` for more information.
627 Defaults to ``self.defaults.collections``.
628 timespan : `Timespan`, optional
629 A timespan that the validity range of the dataset must overlap.
630 If not provided, any `~CollectionType.CALIBRATION` collections
631 matched by the ``collections`` argument will not be searched.
632 **kwargs
633 Additional keyword arguments passed to
634 `DataCoordinate.standardize` to convert ``dataId`` to a true
635 `DataCoordinate` or augment an existing one.
637 Returns
638 -------
639 ref : `DatasetRef`
640 A reference to the dataset, or `None` if no matching Dataset
641 was found.
643 Raises
644 ------
645 TypeError
646 Raised if ``collections`` is `None` and
647 ``self.defaults.collections`` is `None`.
648 LookupError
649 Raised if one or more data ID keys are missing.
650 KeyError
651 Raised if the dataset type does not exist.
652 MissingCollectionError
653 Raised if any of ``collections`` does not exist in the registry.
655 Notes
656 -----
657 This method simply returns `None` and does not raise an exception even
658 when the set of collections searched is intrinsically incompatible with
659 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
660 only `~CollectionType.CALIBRATION` collections are being searched.
661 This may make it harder to debug some lookup failures, but the behavior
662 is intentional; we consider it more important that failed searches are
663 reported consistently, regardless of the reason, and that adding
664 additional collections that do not contain a match to the search path
665 never changes the behavior.
666 """
667 raise NotImplementedError()
669 @abstractmethod
670 def insertDatasets(self, datasetType: Union[DatasetType, str], dataIds: Iterable[DataId],
671 run: Optional[str] = None, expand: bool = True,
672 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE) -> List[DatasetRef]:
673 """Insert one or more datasets into the `Registry`
675 This always adds new datasets; to associate existing datasets with
676 a new collection, use ``associate``.
678 Parameters
679 ----------
680 datasetType : `DatasetType` or `str`
681 A `DatasetType` or the name of one.
682 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate`
683 Dimension-based identifiers for the new datasets.
684 run : `str`, optional
685 The name of the run that produced the datasets. Defaults to
686 ``self.defaults.run``.
687 expand : `bool`, optional
688 If `True` (default), expand data IDs as they are inserted. This is
689 necessary in general to allow datastore to generate file templates,
690 but it may be disabled if the caller can guarantee this is
691 unnecessary.
692 idGenerationMode : `DatasetIdGenEnum`, optional
693 Specifies option for generating dataset IDs. By default unique IDs
694 are generated for each inserted dataset.
696 Returns
697 -------
698 refs : `list` of `DatasetRef`
699 Resolved `DatasetRef` instances for all given data IDs (in the same
700 order).
702 Raises
703 ------
704 TypeError
705 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
706 ConflictingDefinitionError
707 If a dataset with the same dataset type and data ID as one of those
708 given already exists in ``run``.
709 MissingCollectionError
710 Raised if ``run`` does not exist in the registry.
711 """
712 raise NotImplementedError()
714 @abstractmethod
715 def _importDatasets(self, datasets: Iterable[DatasetRef], expand: bool = True,
716 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE,
717 reuseIds: bool = False) -> List[DatasetRef]:
718 """Import one or more datasets into the `Registry`.
720 Difference from `insertDatasets` method is that this method accepts
721 `DatasetRef` instances which should already be resolved and have a
722 dataset ID. If registry supports globally-unique dataset IDs (e.g.
723 `uuid.UUID`) then datasets which already exist in the registry will be
724 ignored if imported again.
726 Parameters
727 ----------
728 datasets : `~collections.abc.Iterable` of `DatasetRef`
729 Datasets to be inserted. All `DatasetRef` instances must have
730 identical ``datasetType`` and ``run`` attributes. ``run``
731 attribute can be `None` and defaults to ``self.defaults.run``.
732 Datasets can specify ``id`` attribute which will be used for
733 inserted datasets. All dataset IDs must have the same type
734 (`int` or `uuid.UUID`), if type of dataset IDs does not match
735 configured backend then IDs will be ignored and new IDs will be
736 generated by backend.
737 expand : `bool`, optional
738 If `True` (default), expand data IDs as they are inserted. This is
739 necessary in general to allow datastore to generate file templates,
740 but it may be disabled if the caller can guarantee this is
741 unnecessary.
742 idGenerationMode : `DatasetIdGenEnum`, optional
743 Specifies option for generating dataset IDs when IDs are not
744 provided or their type does not match backend type. By default
745 unique IDs are generated for each inserted dataset.
746 reuseIds : `bool`, optional
747 If `True` then forces re-use of imported dataset IDs for integer
748 IDs which are normally generated as auto-incremented; exception
749 will be raised if imported IDs clash with existing ones. This
750 option has no effect on the use of globally-unique IDs which are
751 always re-used (or generated if integer IDs are being imported).
753 Returns
754 -------
755 refs : `list` of `DatasetRef`
756 Resolved `DatasetRef` instances for all given data IDs (in the same
757 order). If any of ``datasets`` has an ID which already exists in
758 the database then it will not be inserted or updated, but a
759 resolved `DatasetRef` will be returned for it in any case.
761 Raises
762 ------
763 TypeError
764 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
765 ConflictingDefinitionError
766 If a dataset with the same dataset type and data ID as one of those
767 given already exists in ``run``.
768 MissingCollectionError
769 Raised if ``run`` does not exist in the registry.
771 Notes
772 -----
773 This method is considered package-private and internal to Butler
774 implementation. Clients outside daf_butler package should not use this
775 method.
776 """
777 raise NotImplementedError()
779 @abstractmethod
780 def getDataset(self, id: DatasetId) -> Optional[DatasetRef]:
781 """Retrieve a Dataset entry.
783 Parameters
784 ----------
785 id : `DatasetId`
786 The unique identifier for the dataset.
788 Returns
789 -------
790 ref : `DatasetRef` or `None`
791 A ref to the Dataset, or `None` if no matching Dataset
792 was found.
793 """
794 raise NotImplementedError()
796 @abstractmethod
797 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None:
798 """Remove datasets from the Registry.
800 The datasets will be removed unconditionally from all collections, and
801 any `Quantum` that consumed this dataset will instead be marked with
802 having a NULL input. `Datastore` records will *not* be deleted; the
803 caller is responsible for ensuring that the dataset has already been
804 removed from all Datastores.
806 Parameters
807 ----------
808 refs : `Iterable` of `DatasetRef`
809 References to the datasets to be removed. Must include a valid
810 ``id`` attribute, and should be considered invalidated upon return.
812 Raises
813 ------
814 AmbiguousDatasetError
815 Raised if any ``ref.id`` is `None`.
816 OrphanedRecordError
817 Raised if any dataset is still present in any `Datastore`.
818 """
819 raise NotImplementedError()
821 @abstractmethod
822 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
823 """Add existing datasets to a `~CollectionType.TAGGED` collection.
825 If a DatasetRef with the same exact ID is already in a collection
826 nothing is changed. If a `DatasetRef` with the same `DatasetType` and
827 data ID but with different ID exists in the collection,
828 `ConflictingDefinitionError` is raised.
830 Parameters
831 ----------
832 collection : `str`
833 Indicates the collection the datasets should be associated with.
834 refs : `Iterable` [ `DatasetRef` ]
835 An iterable of resolved `DatasetRef` instances that already exist
836 in this `Registry`.
838 Raises
839 ------
840 ConflictingDefinitionError
841 If a Dataset with the given `DatasetRef` already exists in the
842 given collection.
843 AmbiguousDatasetError
844 Raised if ``any(ref.id is None for ref in refs)``.
845 MissingCollectionError
846 Raised if ``collection`` does not exist in the registry.
847 TypeError
848 Raise adding new datasets to the given ``collection`` is not
849 allowed.
850 """
851 raise NotImplementedError()
853 @abstractmethod
854 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
855 """Remove existing datasets from a `~CollectionType.TAGGED` collection.
857 ``collection`` and ``ref`` combinations that are not currently
858 associated are silently ignored.
860 Parameters
861 ----------
862 collection : `str`
863 The collection the datasets should no longer be associated with.
864 refs : `Iterable` [ `DatasetRef` ]
865 An iterable of resolved `DatasetRef` instances that already exist
866 in this `Registry`.
868 Raises
869 ------
870 AmbiguousDatasetError
871 Raised if any of the given dataset references is unresolved.
872 MissingCollectionError
873 Raised if ``collection`` does not exist in the registry.
874 TypeError
875 Raise adding new datasets to the given ``collection`` is not
876 allowed.
877 """
878 raise NotImplementedError()
880 @abstractmethod
881 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None:
882 """Associate one or more datasets with a calibration collection and a
883 validity range within it.
885 Parameters
886 ----------
887 collection : `str`
888 The name of an already-registered `~CollectionType.CALIBRATION`
889 collection.
890 refs : `Iterable` [ `DatasetRef` ]
891 Datasets to be associated.
892 timespan : `Timespan`
893 The validity range for these datasets within the collection.
895 Raises
896 ------
897 AmbiguousDatasetError
898 Raised if any of the given `DatasetRef` instances is unresolved.
899 ConflictingDefinitionError
900 Raised if the collection already contains a different dataset with
901 the same `DatasetType` and data ID and an overlapping validity
902 range.
903 TypeError
904 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
905 collection or if one or more datasets are of a dataset type for
906 which `DatasetType.isCalibration` returns `False`.
907 """
908 raise NotImplementedError()
910 @abstractmethod
911 def decertify(self, collection: str, datasetType: Union[str, DatasetType], timespan: Timespan, *,
912 dataIds: Optional[Iterable[DataId]] = None) -> None:
913 """Remove or adjust datasets to clear a validity range within a
914 calibration collection.
916 Parameters
917 ----------
918 collection : `str`
919 The name of an already-registered `~CollectionType.CALIBRATION`
920 collection.
921 datasetType : `str` or `DatasetType`
922 Name or `DatasetType` instance for the datasets to be decertified.
923 timespan : `Timespan`, optional
924 The validity range to remove datasets from within the collection.
925 Datasets that overlap this range but are not contained by it will
926 have their validity ranges adjusted to not overlap it, which may
927 split a single dataset validity range into two.
928 dataIds : `Iterable` [ `DataId` ], optional
929 Data IDs that should be decertified within the given validity range
930 If `None`, all data IDs for ``self.datasetType`` will be
931 decertified.
933 Raises
934 ------
935 TypeError
936 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
937 collection or if ``datasetType.isCalibration() is False``.
938 """
939 raise NotImplementedError()
941 @abstractmethod
942 def getDatastoreBridgeManager(self) -> DatastoreRegistryBridgeManager:
943 """Return an object that allows a new `Datastore` instance to
944 communicate with this `Registry`.
946 Returns
947 -------
948 manager : `DatastoreRegistryBridgeManager`
949 Object that mediates communication between this `Registry` and its
950 associated datastores.
951 """
952 raise NotImplementedError()
954 @abstractmethod
955 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]:
956 """Retrieve datastore locations for a given dataset.
958 Parameters
959 ----------
960 ref : `DatasetRef`
961 A reference to the dataset for which to retrieve storage
962 information.
964 Returns
965 -------
966 datastores : `Iterable` [ `str` ]
967 All the matching datastores holding this dataset.
969 Raises
970 ------
971 AmbiguousDatasetError
972 Raised if ``ref.id`` is `None`.
973 """
974 raise NotImplementedError()
976 @abstractmethod
977 def expandDataId(self, dataId: Optional[DataId] = None, *, graph: Optional[DimensionGraph] = None,
978 records: Optional[NameLookupMapping[DimensionElement, Optional[DimensionRecord]]] = None,
979 withDefaults: bool = True,
980 **kwargs: Any) -> DataCoordinate:
981 """Expand a dimension-based data ID to include additional information.
983 Parameters
984 ----------
985 dataId : `DataCoordinate` or `dict`, optional
986 Data ID to be expanded; augmented and overridden by ``kwargs``.
987 graph : `DimensionGraph`, optional
988 Set of dimensions for the expanded ID. If `None`, the dimensions
989 will be inferred from the keys of ``dataId`` and ``kwargs``.
990 Dimensions that are in ``dataId`` or ``kwargs`` but not in
991 ``graph`` are silently ignored, providing a way to extract and
992 ``graph`` expand a subset of a data ID.
993 records : `Mapping` [`str`, `DimensionRecord`], optional
994 Dimension record data to use before querying the database for that
995 data, keyed by element name.
996 withDefaults : `bool`, optional
997 Utilize ``self.defaults.dataId`` to fill in missing governor
998 dimension key-value pairs. Defaults to `True` (i.e. defaults are
999 used).
1000 **kwargs
1001 Additional keywords are treated like additional key-value pairs for
1002 ``dataId``, extending and overriding
1004 Returns
1005 -------
1006 expanded : `DataCoordinate`
1007 A data ID that includes full metadata for all of the dimensions it
1008 identifieds, i.e. guarantees that ``expanded.hasRecords()`` and
1009 ``expanded.hasFull()`` both return `True`.
1010 """
1011 raise NotImplementedError()
1013 @abstractmethod
1014 def insertDimensionData(self, element: Union[DimensionElement, str],
1015 *data: Union[Mapping[str, Any], DimensionRecord],
1016 conform: bool = True,
1017 replace: bool = False) -> None:
1018 """Insert one or more dimension records into the database.
1020 Parameters
1021 ----------
1022 element : `DimensionElement` or `str`
1023 The `DimensionElement` or name thereof that identifies the table
1024 records will be inserted into.
1025 data : `dict` or `DimensionRecord` (variadic)
1026 One or more records to insert.
1027 conform : `bool`, optional
1028 If `False` (`True` is default) perform no checking or conversions,
1029 and assume that ``element`` is a `DimensionElement` instance and
1030 ``data`` is a one or more `DimensionRecord` instances of the
1031 appropriate subclass.
1032 replace: `bool`, optional
1033 If `True` (`False` is default), replace existing records in the
1034 database if there is a conflict.
1035 """
1036 raise NotImplementedError()
1038 @abstractmethod
1039 def syncDimensionData(self, element: Union[DimensionElement, str],
1040 row: Union[Mapping[str, Any], DimensionRecord],
1041 conform: bool = True,
1042 update: bool = False) -> Union[bool, Dict[str, Any]]:
1043 """Synchronize the given dimension record with the database, inserting
1044 if it does not already exist and comparing values if it does.
1046 Parameters
1047 ----------
1048 element : `DimensionElement` or `str`
1049 The `DimensionElement` or name thereof that identifies the table
1050 records will be inserted into.
1051 row : `dict` or `DimensionRecord`
1052 The record to insert.
1053 conform : `bool`, optional
1054 If `False` (`True` is default) perform no checking or conversions,
1055 and assume that ``element`` is a `DimensionElement` instance and
1056 ``data`` is a one or more `DimensionRecord` instances of the
1057 appropriate subclass.
1058 update: `bool`, optional
1059 If `True` (`False` is default), update the existing record in the
1060 database if there is a conflict.
1062 Returns
1063 -------
1064 inserted_or_updated : `bool` or `dict`
1065 `True` if a new row was inserted, `False` if no changes were
1066 needed, or a `dict` mapping updated column names to their old
1067 values if an update was performed (only possible if
1068 ``update=True``).
1070 Raises
1071 ------
1072 ConflictingDefinitionError
1073 Raised if the record exists in the database (according to primary
1074 key lookup) but is inconsistent with the given one.
1075 """
1076 raise NotImplementedError()
1078 @abstractmethod
1079 def queryDatasetTypes(self, expression: Any = ..., *, components: Optional[bool] = None
1080 ) -> Iterator[DatasetType]:
1081 """Iterate over the dataset types whose names match an expression.
1083 Parameters
1084 ----------
1085 expression : `Any`, optional
1086 An expression that fully or partially identifies the dataset types
1087 to return, such as a `str`, `re.Pattern`, or iterable thereof.
1088 `...` can be used to return all dataset types, and is the default.
1089 See :ref:`daf_butler_dataset_type_expressions` for more
1090 information.
1091 components : `bool`, optional
1092 If `True`, apply all expression patterns to component dataset type
1093 names as well. If `False`, never apply patterns to components.
1094 If `None` (default), apply patterns to components only if their
1095 parent datasets were not matched by the expression.
1096 Fully-specified component datasets (`str` or `DatasetType`
1097 instances) are always included.
1099 Yields
1100 ------
1101 datasetType : `DatasetType`
1102 A `DatasetType` instance whose name matches ``expression``.
1103 """
1104 raise NotImplementedError()
1106 @abstractmethod
1107 def queryCollections(self, expression: Any = ...,
1108 datasetType: Optional[DatasetType] = None,
1109 collectionTypes: Iterable[CollectionType] = CollectionType.all(),
1110 flattenChains: bool = False,
1111 includeChains: Optional[bool] = None) -> Iterator[str]:
1112 """Iterate over the collections whose names match an expression.
1114 Parameters
1115 ----------
1116 expression : `Any`, optional
1117 An expression that identifies the collections to return, such as
1118 a `str` (for full matches or partial matches via globs),
1119 `re.Pattern` (for partial matches), or iterable thereof. `...`
1120 can be used to return all collections, and is the default.
1121 See :ref:`daf_butler_collection_expressions` for more information.
1122 datasetType : `DatasetType`, optional
1123 If provided, only yield collections that may contain datasets of
1124 this type. This is a conservative approximation in general; it may
1125 yield collections that do not have any such datasets.
1126 collectionTypes : `AbstractSet` [ `CollectionType` ], optional
1127 If provided, only yield collections of these types.
1128 flattenChains : `bool`, optional
1129 If `True` (`False` is default), recursively yield the child
1130 collections of matching `~CollectionType.CHAINED` collections.
1131 includeChains : `bool`, optional
1132 If `True`, yield records for matching `~CollectionType.CHAINED`
1133 collections. Default is the opposite of ``flattenChains``: include
1134 either CHAINED collections or their children, but not both.
1136 Yields
1137 ------
1138 collection : `str`
1139 The name of a collection that matches ``expression``.
1140 """
1141 raise NotImplementedError()
1143 @abstractmethod
1144 def queryDatasets(self, datasetType: Any, *,
1145 collections: Any = None,
1146 dimensions: Optional[Iterable[Union[Dimension, str]]] = None,
1147 dataId: Optional[DataId] = None,
1148 where: Optional[str] = None,
1149 findFirst: bool = False,
1150 components: Optional[bool] = None,
1151 bind: Optional[Mapping[str, Any]] = None,
1152 check: bool = True,
1153 **kwargs: Any) -> Iterable[DatasetRef]:
1154 """Query for and iterate over dataset references matching user-provided
1155 criteria.
1157 Parameters
1158 ----------
1159 datasetType
1160 An expression that fully or partially identifies the dataset types
1161 to be queried. Allowed types include `DatasetType`, `str`,
1162 `re.Pattern`, and iterables thereof. The special value `...` can
1163 be used to query all dataset types. See
1164 :ref:`daf_butler_dataset_type_expressions` for more information.
1165 collections: optional
1166 An expression that identifies the collections to search, such as a
1167 `str` (for full matches or partial matches via globs), `re.Pattern`
1168 (for partial matches), or iterable thereof. `...` can be used to
1169 search all collections (actually just all `~CollectionType.RUN`
1170 collections, because this will still find all datasets).
1171 If not provided, ``self.default.collections`` is used. See
1172 :ref:`daf_butler_collection_expressions` for more information.
1173 dimensions : `~collections.abc.Iterable` of `Dimension` or `str`
1174 Dimensions to include in the query (in addition to those used
1175 to identify the queried dataset type(s)), either to constrain
1176 the resulting datasets to those for which a matching dimension
1177 exists, or to relate the dataset type's dimensions to dimensions
1178 referenced by the ``dataId`` or ``where`` arguments.
1179 dataId : `dict` or `DataCoordinate`, optional
1180 A data ID whose key-value pairs are used as equality constraints
1181 in the query.
1182 where : `str`, optional
1183 A string expression similar to a SQL WHERE clause. May involve
1184 any column of a dimension table or (as a shortcut for the primary
1185 key column of a dimension table) dimension name. See
1186 :ref:`daf_butler_dimension_expressions` for more information.
1187 findFirst : `bool`, optional
1188 If `True` (`False` is default), for each result data ID, only
1189 yield one `DatasetRef` of each `DatasetType`, from the first
1190 collection in which a dataset of that dataset type appears
1191 (according to the order of ``collections`` passed in). If `True`,
1192 ``collections`` must not contain regular expressions and may not
1193 be `...`.
1194 components : `bool`, optional
1195 If `True`, apply all dataset expression patterns to component
1196 dataset type names as well. If `False`, never apply patterns to
1197 components. If `None` (default), apply patterns to components only
1198 if their parent datasets were not matched by the expression.
1199 Fully-specified component datasets (`str` or `DatasetType`
1200 instances) are always included.
1201 bind : `Mapping`, optional
1202 Mapping containing literal values that should be injected into the
1203 ``where`` expression, keyed by the identifiers they replace.
1204 check : `bool`, optional
1205 If `True` (default) check the query for consistency before
1206 executing it. This may reject some valid queries that resemble
1207 common mistakes (e.g. queries for visits without specifying an
1208 instrument).
1209 **kwargs
1210 Additional keyword arguments are forwarded to
1211 `DataCoordinate.standardize` when processing the ``dataId``
1212 argument (and may be used to provide a constraining data ID even
1213 when the ``dataId`` argument is `None`).
1215 Returns
1216 -------
1217 refs : `queries.DatasetQueryResults`
1218 Dataset references matching the given query criteria. Nested data
1219 IDs are guaranteed to include values for all implied dimensions
1220 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
1221 include dimension records (`DataCoordinate.hasRecords` will be
1222 `False`) unless `~queries.DatasetQueryResults.expanded` is called
1223 on the result object (which returns a new one).
1225 Raises
1226 ------
1227 TypeError
1228 Raised when the arguments are incompatible, such as when a
1229 collection wildcard is passed when ``findFirst`` is `True`, or
1230 when ``collections`` is `None` and``self.defaults.collections`` is
1231 also `None`.
1233 Notes
1234 -----
1235 When multiple dataset types are queried in a single call, the
1236 results of this operation are equivalent to querying for each dataset
1237 type separately in turn, and no information about the relationships
1238 between datasets of different types is included. In contexts where
1239 that kind of information is important, the recommended pattern is to
1240 use `queryDataIds` to first obtain data IDs (possibly with the
1241 desired dataset types and collections passed as constraints to the
1242 query), and then use multiple (generally much simpler) calls to
1243 `queryDatasets` with the returned data IDs passed as constraints.
1244 """
1245 raise NotImplementedError()
1247 @abstractmethod
1248 def queryDataIds(self, dimensions: Union[Iterable[Union[Dimension, str]], Dimension, str], *,
1249 dataId: Optional[DataId] = None,
1250 datasets: Any = None,
1251 collections: Any = None,
1252 where: Optional[str] = None,
1253 components: Optional[bool] = None,
1254 bind: Optional[Mapping[str, Any]] = None,
1255 check: bool = True,
1256 **kwargs: Any) -> DataCoordinateIterable:
1257 """Query for data IDs matching user-provided criteria.
1259 Parameters
1260 ----------
1261 dimensions : `Dimension` or `str`, or iterable thereof
1262 The dimensions of the data IDs to yield, as either `Dimension`
1263 instances or `str`. Will be automatically expanded to a complete
1264 `DimensionGraph`.
1265 dataId : `dict` or `DataCoordinate`, optional
1266 A data ID whose key-value pairs are used as equality constraints
1267 in the query.
1268 datasets : `Any`, optional
1269 An expression that fully or partially identifies dataset types
1270 that should constrain the yielded data IDs. For example, including
1271 "raw" here would constrain the yielded ``instrument``,
1272 ``exposure``, ``detector``, and ``physical_filter`` values to only
1273 those for which at least one "raw" dataset exists in
1274 ``collections``. Allowed types include `DatasetType`, `str`,
1275 `re.Pattern`, and iterables thereof. Unlike other dataset type
1276 expressions, ``...`` is not permitted - it doesn't make sense to
1277 constrain data IDs on the existence of *all* datasets.
1278 See :ref:`daf_butler_dataset_type_expressions` for more
1279 information.
1280 collections: `Any`, optional
1281 An expression that identifies the collections to search for
1282 datasets, such as a `str` (for full matches or partial matches
1283 via globs), `re.Pattern` (for partial matches), or iterable
1284 thereof. `...` can be used to search all collections (actually
1285 just all `~CollectionType.RUN` collections, because this will
1286 still find all datasets). If not provided,
1287 ``self.default.collections`` is used. Ignored unless ``datasets``
1288 is also passed. See :ref:`daf_butler_collection_expressions` for
1289 more information.
1290 where : `str`, optional
1291 A string expression similar to a SQL WHERE clause. May involve
1292 any column of a dimension table or (as a shortcut for the primary
1293 key column of a dimension table) dimension name. See
1294 :ref:`daf_butler_dimension_expressions` for more information.
1295 components : `bool`, optional
1296 If `True`, apply all dataset expression patterns to component
1297 dataset type names as well. If `False`, never apply patterns to
1298 components. If `None` (default), apply patterns to components only
1299 if their parent datasets were not matched by the expression.
1300 Fully-specified component datasets (`str` or `DatasetType`
1301 instances) are always included.
1302 bind : `Mapping`, optional
1303 Mapping containing literal values that should be injected into the
1304 ``where`` expression, keyed by the identifiers they replace.
1305 check : `bool`, optional
1306 If `True` (default) check the query for consistency before
1307 executing it. This may reject some valid queries that resemble
1308 common mistakes (e.g. queries for visits without specifying an
1309 instrument).
1310 **kwargs
1311 Additional keyword arguments are forwarded to
1312 `DataCoordinate.standardize` when processing the ``dataId``
1313 argument (and may be used to provide a constraining data ID even
1314 when the ``dataId`` argument is `None`).
1316 Returns
1317 -------
1318 dataIds : `DataCoordinateQueryResults`
1319 Data IDs matching the given query parameters. These are guaranteed
1320 to identify all dimensions (`DataCoordinate.hasFull` returns
1321 `True`), but will not contain `DimensionRecord` objects
1322 (`DataCoordinate.hasRecords` returns `False`). Call
1323 `DataCoordinateQueryResults.expanded` on the returned object to
1324 fetch those (and consider using
1325 `DataCoordinateQueryResults.materialize` on the returned object
1326 first if the expected number of rows is very large). See
1327 documentation for those methods for additional information.
1329 Raises
1330 ------
1331 TypeError
1332 Raised if ``collections`` is `None`, ``self.defaults.collections``
1333 is `None`, and ``datasets`` is not `None`.
1334 """
1335 raise NotImplementedError()
1337 @abstractmethod
1338 def queryDimensionRecords(self, element: Union[DimensionElement, str], *,
1339 dataId: Optional[DataId] = None,
1340 datasets: Any = None,
1341 collections: Any = None,
1342 where: Optional[str] = None,
1343 components: Optional[bool] = None,
1344 bind: Optional[Mapping[str, Any]] = None,
1345 check: bool = True,
1346 **kwargs: Any) -> Iterator[DimensionRecord]:
1347 """Query for dimension information matching user-provided criteria.
1349 Parameters
1350 ----------
1351 element : `DimensionElement` or `str`
1352 The dimension element to obtain records for.
1353 dataId : `dict` or `DataCoordinate`, optional
1354 A data ID whose key-value pairs are used as equality constraints
1355 in the query.
1356 datasets : `Any`, optional
1357 An expression that fully or partially identifies dataset types
1358 that should constrain the yielded records. See `queryDataIds` and
1359 :ref:`daf_butler_dataset_type_expressions` for more information.
1360 collections: `Any`, optional
1361 An expression that identifies the collections to search for
1362 datasets, such as a `str` (for full matches or partial matches
1363 via globs), `re.Pattern` (for partial matches), or iterable
1364 thereof. `...` can be used to search all collections (actually
1365 just all `~CollectionType.RUN` collections, because this will
1366 still find all datasets). If not provided,
1367 ``self.default.collections`` is used. Ignored unless ``datasets``
1368 is also passed. See :ref:`daf_butler_collection_expressions` for
1369 more information.
1370 where : `str`, optional
1371 A string expression similar to a SQL WHERE clause. See
1372 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
1373 information.
1374 components : `bool`, optional
1375 Whether to apply dataset expressions to components as well.
1376 See `queryDataIds` for more information.
1377 bind : `Mapping`, optional
1378 Mapping containing literal values that should be injected into the
1379 ``where`` expression, keyed by the identifiers they replace.
1380 check : `bool`, optional
1381 If `True` (default) check the query for consistency before
1382 executing it. This may reject some valid queries that resemble
1383 common mistakes (e.g. queries for visits without specifying an
1384 instrument).
1385 **kwargs
1386 Additional keyword arguments are forwarded to
1387 `DataCoordinate.standardize` when processing the ``dataId``
1388 argument (and may be used to provide a constraining data ID even
1389 when the ``dataId`` argument is `None`).
1391 Returns
1392 -------
1393 dataIds : `DataCoordinateQueryResults`
1394 Data IDs matching the given query parameters.
1395 """
1396 raise NotImplementedError()
1398 @abstractmethod
1399 def queryDatasetAssociations(
1400 self,
1401 datasetType: Union[str, DatasetType],
1402 collections: Any = ...,
1403 *,
1404 collectionTypes: Iterable[CollectionType] = CollectionType.all(),
1405 flattenChains: bool = False,
1406 ) -> Iterator[DatasetAssociation]:
1407 """Iterate over dataset-collection combinations where the dataset is in
1408 the collection.
1410 This method is a temporary placeholder for better support for
1411 assocation results in `queryDatasets`. It will probably be
1412 removed in the future, and should be avoided in production code
1413 whenever possible.
1415 Parameters
1416 ----------
1417 datasetType : `DatasetType` or `str`
1418 A dataset type object or the name of one.
1419 collections: `Any`, optional
1420 An expression that identifies the collections to search for
1421 datasets, such as a `str` (for full matches or partial matches
1422 via globs), `re.Pattern` (for partial matches), or iterable
1423 thereof. `...` can be used to search all collections (actually
1424 just all `~CollectionType.RUN` collections, because this will still
1425 find all datasets). If not provided, ``self.default.collections``
1426 is used. See :ref:`daf_butler_collection_expressions` for more
1427 information.
1428 collectionTypes : `AbstractSet` [ `CollectionType` ], optional
1429 If provided, only yield associations from collections of these
1430 types.
1431 flattenChains : `bool`, optional
1432 If `True` (default) search in the children of
1433 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED``
1434 collections are ignored.
1436 Yields
1437 ------
1438 association : `DatasetAssociation`
1439 Object representing the relationship beween a single dataset and
1440 a single collection.
1442 Raises
1443 ------
1444 TypeError
1445 Raised if ``collections`` is `None` and
1446 ``self.defaults.collections`` is `None`.
1447 """
1448 raise NotImplementedError()
1450 storageClasses: StorageClassFactory
1451 """All storage classes known to the registry (`StorageClassFactory`).
1452 """