Coverage for python/lsst/daf/butler/registry/_registry.py: 95%
116 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-25 15:14 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-25 15:14 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("Registry",)
26import contextlib
27import logging
28import re
29from abc import ABC, abstractmethod
30from collections.abc import Iterable, Iterator, Mapping, Sequence
31from types import EllipsisType
32from typing import TYPE_CHECKING, Any
34from lsst.resources import ResourcePathExpression
36from ..core import (
37 DataCoordinate,
38 DataId,
39 DatasetAssociation,
40 DatasetId,
41 DatasetIdGenEnum,
42 DatasetRef,
43 DatasetType,
44 Dimension,
45 DimensionConfig,
46 DimensionElement,
47 DimensionGraph,
48 DimensionRecord,
49 DimensionUniverse,
50 NameLookupMapping,
51 StorageClassFactory,
52 Timespan,
53)
54from ._collection_summary import CollectionSummary
55from ._collectionType import CollectionType
56from ._config import RegistryConfig
57from ._defaults import RegistryDefaults
58from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults
59from .wildcards import CollectionWildcard
61if TYPE_CHECKING:
62 from .interfaces import ObsCoreTableManager
64_LOG = logging.getLogger(__name__)
66# TYpe alias for `collections` arguments.
67CollectionArgType = str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard
70class Registry(ABC):
71 """Abstract Registry interface.
73 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in
74 a ``_defaults`` property. No other properties are assumed shared between
75 implementations.
76 """
78 @classmethod
79 def createFromConfig(
80 cls,
81 config: RegistryConfig | str | None = None,
82 dimensionConfig: DimensionConfig | str | None = None,
83 butlerRoot: ResourcePathExpression | None = None,
84 ) -> Registry:
85 """Create registry database and return `Registry` instance.
87 This method initializes database contents, database must be empty
88 prior to calling this method.
90 Parameters
91 ----------
92 config : `RegistryConfig` or `str`, optional
93 Registry configuration, if missing then default configuration will
94 be loaded from registry.yaml.
95 dimensionConfig : `DimensionConfig` or `str`, optional
96 Dimensions configuration, if missing then default configuration
97 will be loaded from dimensions.yaml.
98 butlerRoot : convertible to `lsst.resources.ResourcePath`, optional
99 Path to the repository root this `Registry` will manage.
101 Returns
102 -------
103 registry : `Registry`
104 A new `Registry` instance.
106 Notes
107 -----
108 This method is for backward compatibility only, until all clients
109 migrate to use new `~lsst.daf.butler.registry._RegistryFactory` factory
110 class. Regular clients of registry class do not use this method, it is
111 only used by tests in multiple packages.
112 """
113 from ._registry_factory import _RegistryFactory
115 return _RegistryFactory(config).create_from_config(dimensionConfig, butlerRoot)
117 @abstractmethod
118 def isWriteable(self) -> bool:
119 """Return `True` if this registry allows write operations, and `False`
120 otherwise.
121 """
122 raise NotImplementedError()
124 @property
125 @abstractmethod
126 def dimensions(self) -> DimensionUniverse:
127 """Definitions of all dimensions recognized by this `Registry`
128 (`DimensionUniverse`).
129 """
130 raise NotImplementedError()
132 @property
133 def defaults(self) -> RegistryDefaults:
134 """Default collection search path and/or output `~CollectionType.RUN`
135 collection (`~lsst.daf.butler.registry.RegistryDefaults`).
137 This is an immutable struct whose components may not be set
138 individually, but the entire struct can be set by assigning to this
139 property.
140 """
141 return self._defaults
143 @defaults.setter
144 def defaults(self, value: RegistryDefaults) -> None:
145 if value.run is not None:
146 self.registerRun(value.run)
147 value.finish(self)
148 self._defaults = value
150 @abstractmethod
151 def refresh(self) -> None:
152 """Refresh all in-memory state by querying the database.
154 This may be necessary to enable querying for entities added by other
155 registry instances after this one was constructed.
156 """
157 raise NotImplementedError()
159 @contextlib.contextmanager
160 @abstractmethod
161 def transaction(self, *, savepoint: bool = False) -> Iterator[None]:
162 """Return a context manager that represents a transaction."""
163 raise NotImplementedError()
165 def resetConnectionPool(self) -> None:
166 """Reset connection pool for registry if relevant.
168 This operation can be used reset connections to servers when
169 using registry with fork-based multiprocessing. This method should
170 usually be called by the child process immediately
171 after the fork.
173 The base class implementation is a no-op.
174 """
175 pass
177 @abstractmethod
178 def registerCollection(
179 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None
180 ) -> bool:
181 """Add a new collection if one with the given name does not exist.
183 Parameters
184 ----------
185 name : `str`
186 The name of the collection to create.
187 type : `CollectionType`
188 Enum value indicating the type of collection to create.
189 doc : `str`, optional
190 Documentation string for the collection.
192 Returns
193 -------
194 registered : `bool`
195 Boolean indicating whether the collection was already registered
196 or was created by this call.
198 Notes
199 -----
200 This method cannot be called within transactions, as it needs to be
201 able to perform its own transaction to be concurrent.
202 """
203 raise NotImplementedError()
205 @abstractmethod
206 def getCollectionType(self, name: str) -> CollectionType:
207 """Return an enumeration value indicating the type of the given
208 collection.
210 Parameters
211 ----------
212 name : `str`
213 The name of the collection.
215 Returns
216 -------
217 type : `CollectionType`
218 Enum value indicating the type of this collection.
220 Raises
221 ------
222 lsst.daf.butler.registry.MissingCollectionError
223 Raised if no collection with the given name exists.
224 """
225 raise NotImplementedError()
227 @abstractmethod
228 def registerRun(self, name: str, doc: str | None = None) -> bool:
229 """Add a new run if one with the given name does not exist.
231 Parameters
232 ----------
233 name : `str`
234 The name of the run to create.
235 doc : `str`, optional
236 Documentation string for the collection.
238 Returns
239 -------
240 registered : `bool`
241 Boolean indicating whether a new run was registered. `False`
242 if it already existed.
244 Notes
245 -----
246 This method cannot be called within transactions, as it needs to be
247 able to perform its own transaction to be concurrent.
248 """
249 raise NotImplementedError()
251 @abstractmethod
252 def removeCollection(self, name: str) -> None:
253 """Remove the given collection from the registry.
255 Parameters
256 ----------
257 name : `str`
258 The name of the collection to remove.
260 Raises
261 ------
262 lsst.daf.butler.registry.MissingCollectionError
263 Raised if no collection with the given name exists.
264 sqlalchemy.exc.IntegrityError
265 Raised if the database rows associated with the collection are
266 still referenced by some other table, such as a dataset in a
267 datastore (for `~CollectionType.RUN` collections only) or a
268 `~CollectionType.CHAINED` collection of which this collection is
269 a child.
271 Notes
272 -----
273 If this is a `~CollectionType.RUN` collection, all datasets and quanta
274 in it will removed from the `Registry` database. This requires that
275 those datasets be removed (or at least trashed) from any datastores
276 that hold them first.
278 A collection may not be deleted as long as it is referenced by a
279 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must
280 be deleted or redefined first.
281 """
282 raise NotImplementedError()
284 @abstractmethod
285 def getCollectionChain(self, parent: str) -> Sequence[str]:
286 """Return the child collections in a `~CollectionType.CHAINED`
287 collection.
289 Parameters
290 ----------
291 parent : `str`
292 Name of the chained collection. Must have already been added via
293 a call to `Registry.registerCollection`.
295 Returns
296 -------
297 children : `~collections.abc.Sequence` [ `str` ]
298 An ordered sequence of collection names that are searched when the
299 given chained collection is searched.
301 Raises
302 ------
303 lsst.daf.butler.registry.MissingCollectionError
304 Raised if ``parent`` does not exist in the `Registry`.
305 lsst.daf.butler.registry.CollectionTypeError
306 Raised if ``parent`` does not correspond to a
307 `~CollectionType.CHAINED` collection.
308 """
309 raise NotImplementedError()
311 @abstractmethod
312 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None:
313 """Define or redefine a `~CollectionType.CHAINED` collection.
315 Parameters
316 ----------
317 parent : `str`
318 Name of the chained collection. Must have already been added via
319 a call to `Registry.registerCollection`.
320 children : collection expression
321 An expression defining an ordered search of child collections,
322 generally an iterable of `str`; see
323 :ref:`daf_butler_collection_expressions` for more information.
324 flatten : `bool`, optional
325 If `True` (`False` is default), recursively flatten out any nested
326 `~CollectionType.CHAINED` collections in ``children`` first.
328 Raises
329 ------
330 lsst.daf.butler.registry.MissingCollectionError
331 Raised when any of the given collections do not exist in the
332 `Registry`.
333 lsst.daf.butler.registry.CollectionTypeError
334 Raised if ``parent`` does not correspond to a
335 `~CollectionType.CHAINED` collection.
336 ValueError
337 Raised if the given collections contains a cycle.
338 """
339 raise NotImplementedError()
341 @abstractmethod
342 def getCollectionParentChains(self, collection: str) -> set[str]:
343 """Return the CHAINED collections that directly contain the given one.
345 Parameters
346 ----------
347 name : `str`
348 Name of the collection.
350 Returns
351 -------
352 chains : `set` of `str`
353 Set of `~CollectionType.CHAINED` collection names.
354 """
355 raise NotImplementedError()
357 @abstractmethod
358 def getCollectionDocumentation(self, collection: str) -> str | None:
359 """Retrieve the documentation string for a collection.
361 Parameters
362 ----------
363 name : `str`
364 Name of the collection.
366 Returns
367 -------
368 docs : `str` or `None`
369 Docstring for the collection with the given name.
370 """
371 raise NotImplementedError()
373 @abstractmethod
374 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None:
375 """Set the documentation string for a collection.
377 Parameters
378 ----------
379 name : `str`
380 Name of the collection.
381 docs : `str` or `None`
382 Docstring for the collection with the given name; will replace any
383 existing docstring. Passing `None` will remove any existing
384 docstring.
385 """
386 raise NotImplementedError()
388 @abstractmethod
389 def getCollectionSummary(self, collection: str) -> CollectionSummary:
390 """Return a summary for the given collection.
392 Parameters
393 ----------
394 collection : `str`
395 Name of the collection for which a summary is to be retrieved.
397 Returns
398 -------
399 summary : `~lsst.daf.butler.registry.CollectionSummary`
400 Summary of the dataset types and governor dimension values in
401 this collection.
402 """
403 raise NotImplementedError()
405 @abstractmethod
406 def registerDatasetType(self, datasetType: DatasetType) -> bool:
407 """Add a new `DatasetType` to the Registry.
409 It is not an error to register the same `DatasetType` twice.
411 Parameters
412 ----------
413 datasetType : `DatasetType`
414 The `DatasetType` to be added.
416 Returns
417 -------
418 inserted : `bool`
419 `True` if ``datasetType`` was inserted, `False` if an identical
420 existing `DatasetType` was found. Note that in either case the
421 DatasetType is guaranteed to be defined in the Registry
422 consistently with the given definition.
424 Raises
425 ------
426 ValueError
427 Raised if the dimensions or storage class are invalid.
428 lsst.daf.butler.registry.ConflictingDefinitionError
429 Raised if this `DatasetType` is already registered with a different
430 definition.
432 Notes
433 -----
434 This method cannot be called within transactions, as it needs to be
435 able to perform its own transaction to be concurrent.
436 """
437 raise NotImplementedError()
439 @abstractmethod
440 def removeDatasetType(self, name: str | tuple[str, ...]) -> None:
441 """Remove the named `DatasetType` from the registry.
443 .. warning::
445 Registry implementations can cache the dataset type definitions.
446 This means that deleting the dataset type definition may result in
447 unexpected behavior from other butler processes that are active
448 that have not seen the deletion.
450 Parameters
451 ----------
452 name : `str` or `tuple` [`str`]
453 Name of the type to be removed or tuple containing a list of type
454 names to be removed. Wildcards are allowed.
456 Raises
457 ------
458 lsst.daf.butler.registry.OrphanedRecordError
459 Raised if an attempt is made to remove the dataset type definition
460 when there are already datasets associated with it.
462 Notes
463 -----
464 If the dataset type is not registered the method will return without
465 action.
466 """
467 raise NotImplementedError()
469 @abstractmethod
470 def getDatasetType(self, name: str) -> DatasetType:
471 """Get the `DatasetType`.
473 Parameters
474 ----------
475 name : `str`
476 Name of the type.
478 Returns
479 -------
480 type : `DatasetType`
481 The `DatasetType` associated with the given name.
483 Raises
484 ------
485 lsst.daf.butler.registry.MissingDatasetTypeError
486 Raised if the requested dataset type has not been registered.
488 Notes
489 -----
490 This method handles component dataset types automatically, though most
491 other registry operations do not.
492 """
493 raise NotImplementedError()
495 @abstractmethod
496 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool:
497 """Test whether the given dataset ID generation mode is supported by
498 `insertDatasets`.
500 Parameters
501 ----------
502 mode : `DatasetIdGenEnum`
503 Enum value for the mode to test.
505 Returns
506 -------
507 supported : `bool`
508 Whether the given mode is supported.
509 """
510 raise NotImplementedError()
512 @abstractmethod
513 def findDataset(
514 self,
515 datasetType: DatasetType | str,
516 dataId: DataId | None = None,
517 *,
518 collections: CollectionArgType | None = None,
519 timespan: Timespan | None = None,
520 **kwargs: Any,
521 ) -> DatasetRef | None:
522 """Find a dataset given its `DatasetType` and data ID.
524 This can be used to obtain a `DatasetRef` that permits the dataset to
525 be read from a `Datastore`. If the dataset is a component and can not
526 be found using the provided dataset type, a dataset ref for the parent
527 will be returned instead but with the correct dataset type.
529 Parameters
530 ----------
531 datasetType : `DatasetType` or `str`
532 A `DatasetType` or the name of one. If this is a `DatasetType`
533 instance, its storage class will be respected and propagated to
534 the output, even if it differs from the dataset type definition
535 in the registry, as long as the storage classes are convertible.
536 dataId : `dict` or `DataCoordinate`, optional
537 A `dict`-like object containing the `Dimension` links that identify
538 the dataset within a collection.
539 collections : collection expression, optional
540 An expression that fully or partially identifies the collections to
541 search for the dataset; see
542 :ref:`daf_butler_collection_expressions` for more information.
543 Defaults to ``self.defaults.collections``.
544 timespan : `Timespan`, optional
545 A timespan that the validity range of the dataset must overlap.
546 If not provided, any `~CollectionType.CALIBRATION` collections
547 matched by the ``collections`` argument will not be searched.
548 **kwargs
549 Additional keyword arguments passed to
550 `DataCoordinate.standardize` to convert ``dataId`` to a true
551 `DataCoordinate` or augment an existing one.
553 Returns
554 -------
555 ref : `DatasetRef`
556 A reference to the dataset, or `None` if no matching Dataset
557 was found.
559 Raises
560 ------
561 lsst.daf.butler.registry.NoDefaultCollectionError
562 Raised if ``collections`` is `None` and
563 ``self.defaults.collections`` is `None`.
564 LookupError
565 Raised if one or more data ID keys are missing.
566 lsst.daf.butler.registry.MissingDatasetTypeError
567 Raised if the dataset type does not exist.
568 lsst.daf.butler.registry.MissingCollectionError
569 Raised if any of ``collections`` does not exist in the registry.
571 Notes
572 -----
573 This method simply returns `None` and does not raise an exception even
574 when the set of collections searched is intrinsically incompatible with
575 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
576 only `~CollectionType.CALIBRATION` collections are being searched.
577 This may make it harder to debug some lookup failures, but the behavior
578 is intentional; we consider it more important that failed searches are
579 reported consistently, regardless of the reason, and that adding
580 additional collections that do not contain a match to the search path
581 never changes the behavior.
583 This method handles component dataset types automatically, though most
584 other registry operations do not.
585 """
586 raise NotImplementedError()
588 @abstractmethod
589 def insertDatasets(
590 self,
591 datasetType: DatasetType | str,
592 dataIds: Iterable[DataId],
593 run: str | None = None,
594 expand: bool = True,
595 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE,
596 ) -> list[DatasetRef]:
597 """Insert one or more datasets into the `Registry`.
599 This always adds new datasets; to associate existing datasets with
600 a new collection, use ``associate``.
602 Parameters
603 ----------
604 datasetType : `DatasetType` or `str`
605 A `DatasetType` or the name of one.
606 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate`
607 Dimension-based identifiers for the new datasets.
608 run : `str`, optional
609 The name of the run that produced the datasets. Defaults to
610 ``self.defaults.run``.
611 expand : `bool`, optional
612 If `True` (default), expand data IDs as they are inserted. This is
613 necessary in general to allow datastore to generate file templates,
614 but it may be disabled if the caller can guarantee this is
615 unnecessary.
616 idGenerationMode : `DatasetIdGenEnum`, optional
617 Specifies option for generating dataset IDs. By default unique IDs
618 are generated for each inserted dataset.
620 Returns
621 -------
622 refs : `list` of `DatasetRef`
623 Resolved `DatasetRef` instances for all given data IDs (in the same
624 order).
626 Raises
627 ------
628 lsst.daf.butler.registry.DatasetTypeError
629 Raised if ``datasetType`` is not known to registry.
630 lsst.daf.butler.registry.CollectionTypeError
631 Raised if ``run`` collection type is not `~CollectionType.RUN`.
632 lsst.daf.butler.registry.NoDefaultCollectionError
633 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
634 lsst.daf.butler.registry.ConflictingDefinitionError
635 If a dataset with the same dataset type and data ID as one of those
636 given already exists in ``run``.
637 lsst.daf.butler.registry.MissingCollectionError
638 Raised if ``run`` does not exist in the registry.
639 """
640 raise NotImplementedError()
642 @abstractmethod
643 def _importDatasets(
644 self,
645 datasets: Iterable[DatasetRef],
646 expand: bool = True,
647 ) -> list[DatasetRef]:
648 """Import one or more datasets into the `Registry`.
650 Difference from `insertDatasets` method is that this method accepts
651 `DatasetRef` instances which should already be resolved and have a
652 dataset ID. If registry supports globally-unique dataset IDs (e.g.
653 `uuid.UUID`) then datasets which already exist in the registry will be
654 ignored if imported again.
656 Parameters
657 ----------
658 datasets : `~collections.abc.Iterable` of `DatasetRef`
659 Datasets to be inserted. All `DatasetRef` instances must have
660 identical ``datasetType`` and ``run`` attributes. ``run``
661 attribute can be `None` and defaults to ``self.defaults.run``.
662 Datasets can specify ``id`` attribute which will be used for
663 inserted datasets. All dataset IDs must have the same type
664 (`int` or `uuid.UUID`), if type of dataset IDs does not match
665 configured backend then IDs will be ignored and new IDs will be
666 generated by backend.
667 expand : `bool`, optional
668 If `True` (default), expand data IDs as they are inserted. This is
669 necessary in general to allow datastore to generate file templates,
670 but it may be disabled if the caller can guarantee this is
671 unnecessary.
673 Returns
674 -------
675 refs : `list` of `DatasetRef`
676 Resolved `DatasetRef` instances for all given data IDs (in the same
677 order). If any of ``datasets`` has an ID which already exists in
678 the database then it will not be inserted or updated, but a
679 resolved `DatasetRef` will be returned for it in any case.
681 Raises
682 ------
683 lsst.daf.butler.registry.NoDefaultCollectionError
684 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
685 lsst.daf.butler.registry.DatasetTypeError
686 Raised if datasets correspond to more than one dataset type or
687 dataset type is not known to registry.
688 lsst.daf.butler.registry.ConflictingDefinitionError
689 If a dataset with the same dataset type and data ID as one of those
690 given already exists in ``run``.
691 lsst.daf.butler.registry.MissingCollectionError
692 Raised if ``run`` does not exist in the registry.
694 Notes
695 -----
696 This method is considered package-private and internal to Butler
697 implementation. Clients outside daf_butler package should not use this
698 method.
699 """
700 raise NotImplementedError()
702 @abstractmethod
703 def getDataset(self, id: DatasetId) -> DatasetRef | None:
704 """Retrieve a Dataset entry.
706 Parameters
707 ----------
708 id : `DatasetId`
709 The unique identifier for the dataset.
711 Returns
712 -------
713 ref : `DatasetRef` or `None`
714 A ref to the Dataset, or `None` if no matching Dataset
715 was found.
716 """
717 raise NotImplementedError()
719 @abstractmethod
720 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None:
721 """Remove datasets from the Registry.
723 The datasets will be removed unconditionally from all collections, and
724 any `Quantum` that consumed this dataset will instead be marked with
725 having a NULL input. `Datastore` records will *not* be deleted; the
726 caller is responsible for ensuring that the dataset has already been
727 removed from all Datastores.
729 Parameters
730 ----------
731 refs : `~collections.abc.Iterable` [`DatasetRef`]
732 References to the datasets to be removed. Must include a valid
733 ``id`` attribute, and should be considered invalidated upon return.
735 Raises
736 ------
737 lsst.daf.butler.AmbiguousDatasetError
738 Raised if any ``ref.id`` is `None`.
739 lsst.daf.butler.registry.OrphanedRecordError
740 Raised if any dataset is still present in any `Datastore`.
741 """
742 raise NotImplementedError()
744 @abstractmethod
745 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
746 """Add existing datasets to a `~CollectionType.TAGGED` collection.
748 If a DatasetRef with the same exact ID is already in a collection
749 nothing is changed. If a `DatasetRef` with the same `DatasetType` and
750 data ID but with different ID exists in the collection,
751 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised.
753 Parameters
754 ----------
755 collection : `str`
756 Indicates the collection the datasets should be associated with.
757 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
758 An iterable of resolved `DatasetRef` instances that already exist
759 in this `Registry`.
761 Raises
762 ------
763 lsst.daf.butler.registry.ConflictingDefinitionError
764 If a Dataset with the given `DatasetRef` already exists in the
765 given collection.
766 lsst.daf.butler.registry.MissingCollectionError
767 Raised if ``collection`` does not exist in the registry.
768 lsst.daf.butler.registry.CollectionTypeError
769 Raise adding new datasets to the given ``collection`` is not
770 allowed.
771 """
772 raise NotImplementedError()
774 @abstractmethod
775 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
776 """Remove existing datasets from a `~CollectionType.TAGGED` collection.
778 ``collection`` and ``ref`` combinations that are not currently
779 associated are silently ignored.
781 Parameters
782 ----------
783 collection : `str`
784 The collection the datasets should no longer be associated with.
785 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
786 An iterable of resolved `DatasetRef` instances that already exist
787 in this `Registry`.
789 Raises
790 ------
791 lsst.daf.butler.AmbiguousDatasetError
792 Raised if any of the given dataset references is unresolved.
793 lsst.daf.butler.registry.MissingCollectionError
794 Raised if ``collection`` does not exist in the registry.
795 lsst.daf.butler.registry.CollectionTypeError
796 Raise adding new datasets to the given ``collection`` is not
797 allowed.
798 """
799 raise NotImplementedError()
801 @abstractmethod
802 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None:
803 """Associate one or more datasets with a calibration collection and a
804 validity range within it.
806 Parameters
807 ----------
808 collection : `str`
809 The name of an already-registered `~CollectionType.CALIBRATION`
810 collection.
811 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
812 Datasets to be associated.
813 timespan : `Timespan`
814 The validity range for these datasets within the collection.
816 Raises
817 ------
818 lsst.daf.butler.AmbiguousDatasetError
819 Raised if any of the given `DatasetRef` instances is unresolved.
820 lsst.daf.butler.registry.ConflictingDefinitionError
821 Raised if the collection already contains a different dataset with
822 the same `DatasetType` and data ID and an overlapping validity
823 range.
824 lsst.daf.butler.registry.CollectionTypeError
825 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
826 collection or if one or more datasets are of a dataset type for
827 which `DatasetType.isCalibration` returns `False`.
828 """
829 raise NotImplementedError()
831 @abstractmethod
832 def decertify(
833 self,
834 collection: str,
835 datasetType: str | DatasetType,
836 timespan: Timespan,
837 *,
838 dataIds: Iterable[DataId] | None = None,
839 ) -> None:
840 """Remove or adjust datasets to clear a validity range within a
841 calibration collection.
843 Parameters
844 ----------
845 collection : `str`
846 The name of an already-registered `~CollectionType.CALIBRATION`
847 collection.
848 datasetType : `str` or `DatasetType`
849 Name or `DatasetType` instance for the datasets to be decertified.
850 timespan : `Timespan`, optional
851 The validity range to remove datasets from within the collection.
852 Datasets that overlap this range but are not contained by it will
853 have their validity ranges adjusted to not overlap it, which may
854 split a single dataset validity range into two.
855 dataIds : iterable [`dict` or `DataCoordinate`], optional
856 Data IDs that should be decertified within the given validity range
857 If `None`, all data IDs for ``self.datasetType`` will be
858 decertified.
860 Raises
861 ------
862 lsst.daf.butler.registry.CollectionTypeError
863 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
864 collection or if ``datasetType.isCalibration() is False``.
865 """
866 raise NotImplementedError()
868 @abstractmethod
869 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]:
870 """Retrieve datastore locations for a given dataset.
872 Parameters
873 ----------
874 ref : `DatasetRef`
875 A reference to the dataset for which to retrieve storage
876 information.
878 Returns
879 -------
880 datastores : `~collections.abc.Iterable` [ `str` ]
881 All the matching datastores holding this dataset.
883 Raises
884 ------
885 lsst.daf.butler.AmbiguousDatasetError
886 Raised if ``ref.id`` is `None`.
887 """
888 raise NotImplementedError()
890 @abstractmethod
891 def expandDataId(
892 self,
893 dataId: DataId | None = None,
894 *,
895 graph: DimensionGraph | None = None,
896 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None,
897 withDefaults: bool = True,
898 **kwargs: Any,
899 ) -> DataCoordinate:
900 """Expand a dimension-based data ID to include additional information.
902 Parameters
903 ----------
904 dataId : `DataCoordinate` or `dict`, optional
905 Data ID to be expanded; augmented and overridden by ``kwargs``.
906 graph : `DimensionGraph`, optional
907 Set of dimensions for the expanded ID. If `None`, the dimensions
908 will be inferred from the keys of ``dataId`` and ``kwargs``.
909 Dimensions that are in ``dataId`` or ``kwargs`` but not in
910 ``graph`` are silently ignored, providing a way to extract and
911 ``graph`` expand a subset of a data ID.
912 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \
913 optional
914 Dimension record data to use before querying the database for that
915 data, keyed by element name.
916 withDefaults : `bool`, optional
917 Utilize ``self.defaults.dataId`` to fill in missing governor
918 dimension key-value pairs. Defaults to `True` (i.e. defaults are
919 used).
920 **kwargs
921 Additional keywords are treated like additional key-value pairs for
922 ``dataId``, extending and overriding
924 Returns
925 -------
926 expanded : `DataCoordinate`
927 A data ID that includes full metadata for all of the dimensions it
928 identifies, i.e. guarantees that ``expanded.hasRecords()`` and
929 ``expanded.hasFull()`` both return `True`.
931 Raises
932 ------
933 lsst.daf.butler.registry.DataIdError
934 Raised when ``dataId`` or keyword arguments specify unknown
935 dimensions or values, or when a resulting data ID contains
936 contradictory key-value pairs, according to dimension
937 relationships.
939 Notes
940 -----
941 This method cannot be relied upon to reject invalid data ID values
942 for dimensions that do actually not have any record columns. For
943 efficiency reasons the records for these dimensions (which have only
944 dimension key values that are given by the caller) may be constructed
945 directly rather than obtained from the registry database.
946 """
947 raise NotImplementedError()
949 @abstractmethod
950 def insertDimensionData(
951 self,
952 element: DimensionElement | str,
953 *data: Mapping[str, Any] | DimensionRecord,
954 conform: bool = True,
955 replace: bool = False,
956 skip_existing: bool = False,
957 ) -> None:
958 """Insert one or more dimension records into the database.
960 Parameters
961 ----------
962 element : `DimensionElement` or `str`
963 The `DimensionElement` or name thereof that identifies the table
964 records will be inserted into.
965 *data : `dict` or `DimensionRecord`
966 One or more records to insert.
967 conform : `bool`, optional
968 If `False` (`True` is default) perform no checking or conversions,
969 and assume that ``element`` is a `DimensionElement` instance and
970 ``data`` is a one or more `DimensionRecord` instances of the
971 appropriate subclass.
972 replace : `bool`, optional
973 If `True` (`False` is default), replace existing records in the
974 database if there is a conflict.
975 skip_existing : `bool`, optional
976 If `True` (`False` is default), skip insertion if a record with
977 the same primary key values already exists. Unlike
978 `syncDimensionData`, this will not detect when the given record
979 differs from what is in the database, and should not be used when
980 this is a concern.
981 """
982 raise NotImplementedError()
984 @abstractmethod
985 def syncDimensionData(
986 self,
987 element: DimensionElement | str,
988 row: Mapping[str, Any] | DimensionRecord,
989 conform: bool = True,
990 update: bool = False,
991 ) -> bool | dict[str, Any]:
992 """Synchronize the given dimension record with the database, inserting
993 if it does not already exist and comparing values if it does.
995 Parameters
996 ----------
997 element : `DimensionElement` or `str`
998 The `DimensionElement` or name thereof that identifies the table
999 records will be inserted into.
1000 row : `dict` or `DimensionRecord`
1001 The record to insert.
1002 conform : `bool`, optional
1003 If `False` (`True` is default) perform no checking or conversions,
1004 and assume that ``element`` is a `DimensionElement` instance and
1005 ``data`` is a one or more `DimensionRecord` instances of the
1006 appropriate subclass.
1007 update : `bool`, optional
1008 If `True` (`False` is default), update the existing record in the
1009 database if there is a conflict.
1011 Returns
1012 -------
1013 inserted_or_updated : `bool` or `dict`
1014 `True` if a new row was inserted, `False` if no changes were
1015 needed, or a `dict` mapping updated column names to their old
1016 values if an update was performed (only possible if
1017 ``update=True``).
1019 Raises
1020 ------
1021 lsst.daf.butler.registry.ConflictingDefinitionError
1022 Raised if the record exists in the database (according to primary
1023 key lookup) but is inconsistent with the given one.
1024 """
1025 raise NotImplementedError()
1027 @abstractmethod
1028 def queryDatasetTypes(
1029 self,
1030 expression: Any = ...,
1031 *,
1032 components: bool | None = False,
1033 missing: list[str] | None = None,
1034 ) -> Iterable[DatasetType]:
1035 """Iterate over the dataset types whose names match an expression.
1037 Parameters
1038 ----------
1039 expression : dataset type expression, optional
1040 An expression that fully or partially identifies the dataset types
1041 to return, such as a `str`, `re.Pattern`, or iterable thereof.
1042 ``...`` can be used to return all dataset types, and is the
1043 default. See :ref:`daf_butler_dataset_type_expressions` for more
1044 information.
1045 components : `bool`, optional
1046 If `True`, apply all expression patterns to component dataset type
1047 names as well. If `False`, never apply patterns to components.
1048 If `None`, apply patterns to components only if their
1049 parent datasets were not matched by the expression.
1050 Fully-specified component datasets (`str` or `DatasetType`
1051 instances) are always included.
1053 Values other than `False` are deprecated, and only `False` will be
1054 supported after v26. After v27 this argument will be removed
1055 entirely.
1056 missing : `list` of `str`, optional
1057 String dataset type names that were explicitly given (i.e. not
1058 regular expression patterns) but not found will be appended to this
1059 list, if it is provided.
1061 Returns
1062 -------
1063 dataset_types : `~collections.abc.Iterable` [ `DatasetType`]
1064 An `~collections.abc.Iterable` of `DatasetType` instances whose
1065 names match ``expression``.
1067 Raises
1068 ------
1069 lsst.daf.butler.registry.DatasetTypeExpressionError
1070 Raised when ``expression`` is invalid.
1071 """
1072 raise NotImplementedError()
1074 @abstractmethod
1075 def queryCollections(
1076 self,
1077 expression: Any = ...,
1078 datasetType: DatasetType | None = None,
1079 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(),
1080 flattenChains: bool = False,
1081 includeChains: bool | None = None,
1082 ) -> Sequence[str]:
1083 """Iterate over the collections whose names match an expression.
1085 Parameters
1086 ----------
1087 expression : collection expression, optional
1088 An expression that identifies the collections to return, such as
1089 a `str` (for full matches or partial matches via globs),
1090 `re.Pattern` (for partial matches), or iterable thereof. ``...``
1091 can be used to return all collections, and is the default.
1092 See :ref:`daf_butler_collection_expressions` for more information.
1093 datasetType : `DatasetType`, optional
1094 If provided, only yield collections that may contain datasets of
1095 this type. This is a conservative approximation in general; it may
1096 yield collections that do not have any such datasets.
1097 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \
1098 `CollectionType`, optional
1099 If provided, only yield collections of these types.
1100 flattenChains : `bool`, optional
1101 If `True` (`False` is default), recursively yield the child
1102 collections of matching `~CollectionType.CHAINED` collections.
1103 includeChains : `bool`, optional
1104 If `True`, yield records for matching `~CollectionType.CHAINED`
1105 collections. Default is the opposite of ``flattenChains``: include
1106 either CHAINED collections or their children, but not both.
1108 Returns
1109 -------
1110 collections : `~collections.abc.Sequence` [ `str` ]
1111 The names of collections that match ``expression``.
1113 Raises
1114 ------
1115 lsst.daf.butler.registry.CollectionExpressionError
1116 Raised when ``expression`` is invalid.
1118 Notes
1119 -----
1120 The order in which collections are returned is unspecified, except that
1121 the children of a `~CollectionType.CHAINED` collection are guaranteed
1122 to be in the order in which they are searched. When multiple parent
1123 `~CollectionType.CHAINED` collections match the same criteria, the
1124 order in which the two lists appear is unspecified, and the lists of
1125 children may be incomplete if a child has multiple parents.
1126 """
1127 raise NotImplementedError()
1129 @abstractmethod
1130 def queryDatasets(
1131 self,
1132 datasetType: Any,
1133 *,
1134 collections: CollectionArgType | None = None,
1135 dimensions: Iterable[Dimension | str] | None = None,
1136 dataId: DataId | None = None,
1137 where: str = "",
1138 findFirst: bool = False,
1139 components: bool | None = False,
1140 bind: Mapping[str, Any] | None = None,
1141 check: bool = True,
1142 **kwargs: Any,
1143 ) -> DatasetQueryResults:
1144 """Query for and iterate over dataset references matching user-provided
1145 criteria.
1147 Parameters
1148 ----------
1149 datasetType : dataset type expression
1150 An expression that fully or partially identifies the dataset types
1151 to be queried. Allowed types include `DatasetType`, `str`,
1152 `re.Pattern`, and iterables thereof. The special value ``...`` can
1153 be used to query all dataset types. See
1154 :ref:`daf_butler_dataset_type_expressions` for more information.
1155 collections : collection expression, optional
1156 An expression that identifies the collections to search, such as a
1157 `str` (for full matches or partial matches via globs), `re.Pattern`
1158 (for partial matches), or iterable thereof. ``...`` can be used to
1159 search all collections (actually just all `~CollectionType.RUN`
1160 collections, because this will still find all datasets).
1161 If not provided, ``self.default.collections`` is used. See
1162 :ref:`daf_butler_collection_expressions` for more information.
1163 dimensions : `~collections.abc.Iterable` of `Dimension` or `str`
1164 Dimensions to include in the query (in addition to those used
1165 to identify the queried dataset type(s)), either to constrain
1166 the resulting datasets to those for which a matching dimension
1167 exists, or to relate the dataset type's dimensions to dimensions
1168 referenced by the ``dataId`` or ``where`` arguments.
1169 dataId : `dict` or `DataCoordinate`, optional
1170 A data ID whose key-value pairs are used as equality constraints
1171 in the query.
1172 where : `str`, optional
1173 A string expression similar to a SQL WHERE clause. May involve
1174 any column of a dimension table or (as a shortcut for the primary
1175 key column of a dimension table) dimension name. See
1176 :ref:`daf_butler_dimension_expressions` for more information.
1177 findFirst : `bool`, optional
1178 If `True` (`False` is default), for each result data ID, only
1179 yield one `DatasetRef` of each `DatasetType`, from the first
1180 collection in which a dataset of that dataset type appears
1181 (according to the order of ``collections`` passed in). If `True`,
1182 ``collections`` must not contain regular expressions and may not
1183 be ``...``.
1184 components : `bool`, optional
1185 If `True`, apply all dataset expression patterns to component
1186 dataset type names as well. If `False`, never apply patterns to
1187 components. If `None`, apply patterns to components only
1188 if their parent datasets were not matched by the expression.
1189 Fully-specified component datasets (`str` or `DatasetType`
1190 instances) are always included.
1192 Values other than `False` are deprecated, and only `False` will be
1193 supported after v26. After v27 this argument will be removed
1194 entirely.
1195 bind : `~collections.abc.Mapping`, optional
1196 Mapping containing literal values that should be injected into the
1197 ``where`` expression, keyed by the identifiers they replace.
1198 Values of collection type can be expanded in some cases; see
1199 :ref:`daf_butler_dimension_expressions_identifiers` for more
1200 information.
1201 check : `bool`, optional
1202 If `True` (default) check the query for consistency before
1203 executing it. This may reject some valid queries that resemble
1204 common mistakes (e.g. queries for visits without specifying an
1205 instrument).
1206 **kwargs
1207 Additional keyword arguments are forwarded to
1208 `DataCoordinate.standardize` when processing the ``dataId``
1209 argument (and may be used to provide a constraining data ID even
1210 when the ``dataId`` argument is `None`).
1212 Returns
1213 -------
1214 refs : `.queries.DatasetQueryResults`
1215 Dataset references matching the given query criteria. Nested data
1216 IDs are guaranteed to include values for all implied dimensions
1217 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
1218 include dimension records (`DataCoordinate.hasRecords` will be
1219 `False`) unless `~.queries.DatasetQueryResults.expanded` is
1220 called on the result object (which returns a new one).
1222 Raises
1223 ------
1224 lsst.daf.butler.registry.DatasetTypeExpressionError
1225 Raised when ``datasetType`` expression is invalid.
1226 TypeError
1227 Raised when the arguments are incompatible, such as when a
1228 collection wildcard is passed when ``findFirst`` is `True`, or
1229 when ``collections`` is `None` and ``self.defaults.collections`` is
1230 also `None`.
1231 lsst.daf.butler.registry.DataIdError
1232 Raised when ``dataId`` or keyword arguments specify unknown
1233 dimensions or values, or when they contain inconsistent values.
1234 lsst.daf.butler.registry.UserExpressionError
1235 Raised when ``where`` expression is invalid.
1237 Notes
1238 -----
1239 When multiple dataset types are queried in a single call, the
1240 results of this operation are equivalent to querying for each dataset
1241 type separately in turn, and no information about the relationships
1242 between datasets of different types is included. In contexts where
1243 that kind of information is important, the recommended pattern is to
1244 use `queryDataIds` to first obtain data IDs (possibly with the
1245 desired dataset types and collections passed as constraints to the
1246 query), and then use multiple (generally much simpler) calls to
1247 `queryDatasets` with the returned data IDs passed as constraints.
1248 """
1249 raise NotImplementedError()
1251 @abstractmethod
1252 def queryDataIds(
1253 self,
1254 dimensions: Iterable[Dimension | str] | Dimension | str,
1255 *,
1256 dataId: DataId | None = None,
1257 datasets: Any = None,
1258 collections: CollectionArgType | None = None,
1259 where: str = "",
1260 components: bool | None = False,
1261 bind: Mapping[str, Any] | None = None,
1262 check: bool = True,
1263 **kwargs: Any,
1264 ) -> DataCoordinateQueryResults:
1265 """Query for data IDs matching user-provided criteria.
1267 Parameters
1268 ----------
1269 dimensions : `Dimension` or `str`, or iterable thereof
1270 The dimensions of the data IDs to yield, as either `Dimension`
1271 instances or `str`. Will be automatically expanded to a complete
1272 `DimensionGraph`.
1273 dataId : `dict` or `DataCoordinate`, optional
1274 A data ID whose key-value pairs are used as equality constraints
1275 in the query.
1276 datasets : dataset type expression, optional
1277 An expression that fully or partially identifies dataset types
1278 that should constrain the yielded data IDs. For example, including
1279 "raw" here would constrain the yielded ``instrument``,
1280 ``exposure``, ``detector``, and ``physical_filter`` values to only
1281 those for which at least one "raw" dataset exists in
1282 ``collections``. Allowed types include `DatasetType`, `str`,
1283 and iterables thereof. Regular expression objects (i.e.
1284 `re.Pattern`) are deprecated and will be removed after the v26
1285 release. See :ref:`daf_butler_dataset_type_expressions` for more
1286 information.
1287 collections : collection expression, optional
1288 An expression that identifies the collections to search for
1289 datasets, such as a `str` (for full matches or partial matches
1290 via globs), `re.Pattern` (for partial matches), or iterable
1291 thereof. ``...`` can be used to search all collections (actually
1292 just all `~CollectionType.RUN` collections, because this will
1293 still find all datasets). If not provided,
1294 ``self.default.collections`` is used. Ignored unless ``datasets``
1295 is also passed. See :ref:`daf_butler_collection_expressions` for
1296 more information.
1297 where : `str`, optional
1298 A string expression similar to a SQL WHERE clause. May involve
1299 any column of a dimension table or (as a shortcut for the primary
1300 key column of a dimension table) dimension name. See
1301 :ref:`daf_butler_dimension_expressions` for more information.
1302 components : `bool`, optional
1303 If `True`, apply all dataset expression patterns to component
1304 dataset type names as well. If `False`, never apply patterns to
1305 components. If `None`, apply patterns to components only
1306 if their parent datasets were not matched by the expression.
1307 Fully-specified component datasets (`str` or `DatasetType`
1308 instances) are always included.
1310 Values other than `False` are deprecated, and only `False` will be
1311 supported after v26. After v27 this argument will be removed
1312 entirely.
1313 bind : `~collections.abc.Mapping`, optional
1314 Mapping containing literal values that should be injected into the
1315 ``where`` expression, keyed by the identifiers they replace.
1316 Values of collection type can be expanded in some cases; see
1317 :ref:`daf_butler_dimension_expressions_identifiers` for more
1318 information.
1319 check : `bool`, optional
1320 If `True` (default) check the query for consistency before
1321 executing it. This may reject some valid queries that resemble
1322 common mistakes (e.g. queries for visits without specifying an
1323 instrument).
1324 **kwargs
1325 Additional keyword arguments are forwarded to
1326 `DataCoordinate.standardize` when processing the ``dataId``
1327 argument (and may be used to provide a constraining data ID even
1328 when the ``dataId`` argument is `None`).
1330 Returns
1331 -------
1332 dataIds : `.queries.DataCoordinateQueryResults`
1333 Data IDs matching the given query parameters. These are guaranteed
1334 to identify all dimensions (`DataCoordinate.hasFull` returns
1335 `True`), but will not contain `DimensionRecord` objects
1336 (`DataCoordinate.hasRecords` returns `False`). Call
1337 `~.queries.DataCoordinateQueryResults.expanded` on the
1338 returned object to fetch those (and consider using
1339 `~.queries.DataCoordinateQueryResults.materialize` on the
1340 returned object first if the expected number of rows is very
1341 large). See documentation for those methods for additional
1342 information.
1344 Raises
1345 ------
1346 lsst.daf.butler.registry.NoDefaultCollectionError
1347 Raised if ``collections`` is `None` and
1348 ``self.defaults.collections`` is `None`.
1349 lsst.daf.butler.registry.CollectionExpressionError
1350 Raised when ``collections`` expression is invalid.
1351 lsst.daf.butler.registry.DataIdError
1352 Raised when ``dataId`` or keyword arguments specify unknown
1353 dimensions or values, or when they contain inconsistent values.
1354 lsst.daf.butler.registry.DatasetTypeExpressionError
1355 Raised when ``datasetType`` expression is invalid.
1356 lsst.daf.butler.registry.UserExpressionError
1357 Raised when ``where`` expression is invalid.
1358 """
1359 raise NotImplementedError()
1361 @abstractmethod
1362 def queryDimensionRecords(
1363 self,
1364 element: DimensionElement | str,
1365 *,
1366 dataId: DataId | None = None,
1367 datasets: Any = None,
1368 collections: CollectionArgType | None = None,
1369 where: str = "",
1370 components: bool | None = False,
1371 bind: Mapping[str, Any] | None = None,
1372 check: bool = True,
1373 **kwargs: Any,
1374 ) -> DimensionRecordQueryResults:
1375 """Query for dimension information matching user-provided criteria.
1377 Parameters
1378 ----------
1379 element : `DimensionElement` or `str`
1380 The dimension element to obtain records for.
1381 dataId : `dict` or `DataCoordinate`, optional
1382 A data ID whose key-value pairs are used as equality constraints
1383 in the query.
1384 datasets : dataset type expression, optional
1385 An expression that fully or partially identifies dataset types
1386 that should constrain the yielded records. See `queryDataIds` and
1387 :ref:`daf_butler_dataset_type_expressions` for more information.
1388 collections : collection expression, optional
1389 An expression that identifies the collections to search for
1390 datasets, such as a `str` (for full matches or partial matches
1391 via globs), `re.Pattern` (for partial matches), or iterable
1392 thereof. ``...`` can be used to search all collections (actually
1393 just all `~CollectionType.RUN` collections, because this will
1394 still find all datasets). If not provided,
1395 ``self.default.collections`` is used. Ignored unless ``datasets``
1396 is also passed. See :ref:`daf_butler_collection_expressions` for
1397 more information.
1398 where : `str`, optional
1399 A string expression similar to a SQL WHERE clause. See
1400 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
1401 information.
1402 components : `bool`, optional
1403 Whether to apply dataset expressions to components as well.
1404 See `queryDataIds` for more information.
1406 Values other than `False` are deprecated, and only `False` will be
1407 supported after v26. After v27 this argument will be removed
1408 entirely.
1409 bind : `~collections.abc.Mapping`, optional
1410 Mapping containing literal values that should be injected into the
1411 ``where`` expression, keyed by the identifiers they replace.
1412 Values of collection type can be expanded in some cases; see
1413 :ref:`daf_butler_dimension_expressions_identifiers` for more
1414 information.
1415 check : `bool`, optional
1416 If `True` (default) check the query for consistency before
1417 executing it. This may reject some valid queries that resemble
1418 common mistakes (e.g. queries for visits without specifying an
1419 instrument).
1420 **kwargs
1421 Additional keyword arguments are forwarded to
1422 `DataCoordinate.standardize` when processing the ``dataId``
1423 argument (and may be used to provide a constraining data ID even
1424 when the ``dataId`` argument is `None`).
1426 Returns
1427 -------
1428 dataIds : `.queries.DimensionRecordQueryResults`
1429 Data IDs matching the given query parameters.
1431 Raises
1432 ------
1433 lsst.daf.butler.registry.NoDefaultCollectionError
1434 Raised if ``collections`` is `None` and
1435 ``self.defaults.collections`` is `None`.
1436 lsst.daf.butler.registry.CollectionExpressionError
1437 Raised when ``collections`` expression is invalid.
1438 lsst.daf.butler.registry.DataIdError
1439 Raised when ``dataId`` or keyword arguments specify unknown
1440 dimensions or values, or when they contain inconsistent values.
1441 lsst.daf.butler.registry.DatasetTypeExpressionError
1442 Raised when ``datasetType`` expression is invalid.
1443 lsst.daf.butler.registry.UserExpressionError
1444 Raised when ``where`` expression is invalid.
1445 """
1446 raise NotImplementedError()
1448 @abstractmethod
1449 def queryDatasetAssociations(
1450 self,
1451 datasetType: str | DatasetType,
1452 collections: CollectionArgType | None = ...,
1453 *,
1454 collectionTypes: Iterable[CollectionType] = CollectionType.all(),
1455 flattenChains: bool = False,
1456 ) -> Iterator[DatasetAssociation]:
1457 """Iterate over dataset-collection combinations where the dataset is in
1458 the collection.
1460 This method is a temporary placeholder for better support for
1461 association results in `queryDatasets`. It will probably be
1462 removed in the future, and should be avoided in production code
1463 whenever possible.
1465 Parameters
1466 ----------
1467 datasetType : `DatasetType` or `str`
1468 A dataset type object or the name of one.
1469 collections : collection expression, optional
1470 An expression that identifies the collections to search for
1471 datasets, such as a `str` (for full matches or partial matches
1472 via globs), `re.Pattern` (for partial matches), or iterable
1473 thereof. ``...`` can be used to search all collections (actually
1474 just all `~CollectionType.RUN` collections, because this will still
1475 find all datasets). If not provided, ``self.default.collections``
1476 is used. See :ref:`daf_butler_collection_expressions` for more
1477 information.
1478 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional
1479 If provided, only yield associations from collections of these
1480 types.
1481 flattenChains : `bool`, optional
1482 If `True`, search in the children of `~CollectionType.CHAINED`
1483 collections. If `False`, ``CHAINED`` collections are ignored.
1485 Yields
1486 ------
1487 association : `.DatasetAssociation`
1488 Object representing the relationship between a single dataset and
1489 a single collection.
1491 Raises
1492 ------
1493 lsst.daf.butler.registry.NoDefaultCollectionError
1494 Raised if ``collections`` is `None` and
1495 ``self.defaults.collections`` is `None`.
1496 lsst.daf.butler.registry.CollectionExpressionError
1497 Raised when ``collections`` expression is invalid.
1498 """
1499 raise NotImplementedError()
1501 @property
1502 def obsCoreTableManager(self) -> ObsCoreTableManager | None:
1503 """The ObsCore manager instance for this registry
1504 (`~.interfaces.ObsCoreTableManager`
1505 or `None`).
1507 ObsCore manager may not be implemented for all registry backend, or
1508 may not be enabled for many repositories.
1509 """
1510 return None
1512 storageClasses: StorageClassFactory
1513 """All storage classes known to the registry (`StorageClassFactory`).
1514 """