Coverage for python/lsst/daf/butler/registry/_registry.py: 99%
115 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-01 10:59 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-01 10:59 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ("Registry", "CollectionArgType")
32import contextlib
33import logging
34import re
35from abc import ABC, abstractmethod
36from collections.abc import Iterable, Iterator, Mapping, Sequence
37from types import EllipsisType
38from typing import TYPE_CHECKING, Any, TypeAlias
40from .._dataset_association import DatasetAssociation
41from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef
42from .._dataset_type import DatasetType
43from .._named import NameLookupMapping
44from .._storage_class import StorageClassFactory
45from .._timespan import Timespan
46from ..dimensions import (
47 DataCoordinate,
48 DataId,
49 Dimension,
50 DimensionElement,
51 DimensionGraph,
52 DimensionGroup,
53 DimensionRecord,
54 DimensionUniverse,
55)
56from ._collection_summary import CollectionSummary
57from ._collection_type import CollectionType
58from ._defaults import RegistryDefaults
59from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults
60from .wildcards import CollectionWildcard
62if TYPE_CHECKING:
63 from .interfaces import ObsCoreTableManager
65_LOG = logging.getLogger(__name__)
67# TYpe alias for `collections` arguments.
68CollectionArgType: TypeAlias = (
69 str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard
70)
73class Registry(ABC):
74 """Abstract Registry interface.
76 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in
77 a ``_defaults`` property. No other properties are assumed shared between
78 implementations.
79 """
81 @abstractmethod
82 def isWriteable(self) -> bool:
83 """Return `True` if this registry allows write operations, and `False`
84 otherwise.
85 """
86 raise NotImplementedError()
88 @property
89 @abstractmethod
90 def dimensions(self) -> DimensionUniverse:
91 """Definitions of all dimensions recognized by this `Registry`
92 (`DimensionUniverse`).
93 """
94 raise NotImplementedError()
96 @property
97 @abstractmethod
98 def defaults(self) -> RegistryDefaults:
99 """Default collection search path and/or output `~CollectionType.RUN`
100 collection (`~lsst.daf.butler.registry.RegistryDefaults`).
102 This is an immutable struct whose components may not be set
103 individually, but the entire struct can be set by assigning to this
104 property.
105 """
106 raise NotImplementedError()
108 @defaults.setter
109 @abstractmethod
110 def defaults(self, value: RegistryDefaults) -> None:
111 raise NotImplementedError()
113 @abstractmethod
114 def refresh(self) -> None:
115 """Refresh all in-memory state by querying the database.
117 This may be necessary to enable querying for entities added by other
118 registry instances after this one was constructed.
119 """
120 raise NotImplementedError()
122 @abstractmethod
123 def caching_context(self) -> contextlib.AbstractContextManager[None]:
124 """Context manager that enables caching."""
125 raise NotImplementedError()
127 @contextlib.contextmanager
128 @abstractmethod
129 def transaction(self, *, savepoint: bool = False) -> Iterator[None]:
130 """Return a context manager that represents a transaction."""
131 raise NotImplementedError()
133 def resetConnectionPool(self) -> None:
134 """Reset connection pool for registry if relevant.
136 This operation can be used reset connections to servers when
137 using registry with fork-based multiprocessing. This method should
138 usually be called by the child process immediately
139 after the fork.
141 The base class implementation is a no-op.
142 """
143 pass
145 @abstractmethod
146 def registerCollection(
147 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None
148 ) -> bool:
149 """Add a new collection if one with the given name does not exist.
151 Parameters
152 ----------
153 name : `str`
154 The name of the collection to create.
155 type : `CollectionType`
156 Enum value indicating the type of collection to create.
157 doc : `str`, optional
158 Documentation string for the collection.
160 Returns
161 -------
162 registered : `bool`
163 Boolean indicating whether the collection was already registered
164 or was created by this call.
166 Notes
167 -----
168 This method cannot be called within transactions, as it needs to be
169 able to perform its own transaction to be concurrent.
170 """
171 raise NotImplementedError()
173 @abstractmethod
174 def getCollectionType(self, name: str) -> CollectionType:
175 """Return an enumeration value indicating the type of the given
176 collection.
178 Parameters
179 ----------
180 name : `str`
181 The name of the collection.
183 Returns
184 -------
185 type : `CollectionType`
186 Enum value indicating the type of this collection.
188 Raises
189 ------
190 lsst.daf.butler.registry.MissingCollectionError
191 Raised if no collection with the given name exists.
192 """
193 raise NotImplementedError()
195 @abstractmethod
196 def registerRun(self, name: str, doc: str | None = None) -> bool:
197 """Add a new run if one with the given name does not exist.
199 Parameters
200 ----------
201 name : `str`
202 The name of the run to create.
203 doc : `str`, optional
204 Documentation string for the collection.
206 Returns
207 -------
208 registered : `bool`
209 Boolean indicating whether a new run was registered. `False`
210 if it already existed.
212 Notes
213 -----
214 This method cannot be called within transactions, as it needs to be
215 able to perform its own transaction to be concurrent.
216 """
217 raise NotImplementedError()
219 @abstractmethod
220 def removeCollection(self, name: str) -> None:
221 """Remove the given collection from the registry.
223 Parameters
224 ----------
225 name : `str`
226 The name of the collection to remove.
228 Raises
229 ------
230 lsst.daf.butler.registry.MissingCollectionError
231 Raised if no collection with the given name exists.
232 sqlalchemy.exc.IntegrityError
233 Raised if the database rows associated with the collection are
234 still referenced by some other table, such as a dataset in a
235 datastore (for `~CollectionType.RUN` collections only) or a
236 `~CollectionType.CHAINED` collection of which this collection is
237 a child.
239 Notes
240 -----
241 If this is a `~CollectionType.RUN` collection, all datasets and quanta
242 in it will removed from the `Registry` database. This requires that
243 those datasets be removed (or at least trashed) from any datastores
244 that hold them first.
246 A collection may not be deleted as long as it is referenced by a
247 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must
248 be deleted or redefined first.
249 """
250 raise NotImplementedError()
252 @abstractmethod
253 def getCollectionChain(self, parent: str) -> Sequence[str]:
254 """Return the child collections in a `~CollectionType.CHAINED`
255 collection.
257 Parameters
258 ----------
259 parent : `str`
260 Name of the chained collection. Must have already been added via
261 a call to `Registry.registerCollection`.
263 Returns
264 -------
265 children : `~collections.abc.Sequence` [ `str` ]
266 An ordered sequence of collection names that are searched when the
267 given chained collection is searched.
269 Raises
270 ------
271 lsst.daf.butler.registry.MissingCollectionError
272 Raised if ``parent`` does not exist in the `Registry`.
273 lsst.daf.butler.registry.CollectionTypeError
274 Raised if ``parent`` does not correspond to a
275 `~CollectionType.CHAINED` collection.
276 """
277 raise NotImplementedError()
279 @abstractmethod
280 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None:
281 """Define or redefine a `~CollectionType.CHAINED` collection.
283 Parameters
284 ----------
285 parent : `str`
286 Name of the chained collection. Must have already been added via
287 a call to `Registry.registerCollection`.
288 children : collection expression
289 An expression defining an ordered search of child collections,
290 generally an iterable of `str`; see
291 :ref:`daf_butler_collection_expressions` for more information.
292 flatten : `bool`, optional
293 If `True` (`False` is default), recursively flatten out any nested
294 `~CollectionType.CHAINED` collections in ``children`` first.
296 Raises
297 ------
298 lsst.daf.butler.registry.MissingCollectionError
299 Raised when any of the given collections do not exist in the
300 `Registry`.
301 lsst.daf.butler.registry.CollectionTypeError
302 Raised if ``parent`` does not correspond to a
303 `~CollectionType.CHAINED` collection.
304 ValueError
305 Raised if the given collections contains a cycle.
306 """
307 raise NotImplementedError()
309 @abstractmethod
310 def getCollectionParentChains(self, collection: str) -> set[str]:
311 """Return the CHAINED collections that directly contain the given one.
313 Parameters
314 ----------
315 name : `str`
316 Name of the collection.
318 Returns
319 -------
320 chains : `set` of `str`
321 Set of `~CollectionType.CHAINED` collection names.
322 """
323 raise NotImplementedError()
325 @abstractmethod
326 def getCollectionDocumentation(self, collection: str) -> str | None:
327 """Retrieve the documentation string for a collection.
329 Parameters
330 ----------
331 name : `str`
332 Name of the collection.
334 Returns
335 -------
336 docs : `str` or `None`
337 Docstring for the collection with the given name.
338 """
339 raise NotImplementedError()
341 @abstractmethod
342 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None:
343 """Set the documentation string for a collection.
345 Parameters
346 ----------
347 name : `str`
348 Name of the collection.
349 docs : `str` or `None`
350 Docstring for the collection with the given name; will replace any
351 existing docstring. Passing `None` will remove any existing
352 docstring.
353 """
354 raise NotImplementedError()
356 @abstractmethod
357 def getCollectionSummary(self, collection: str) -> CollectionSummary:
358 """Return a summary for the given collection.
360 Parameters
361 ----------
362 collection : `str`
363 Name of the collection for which a summary is to be retrieved.
365 Returns
366 -------
367 summary : `~lsst.daf.butler.registry.CollectionSummary`
368 Summary of the dataset types and governor dimension values in
369 this collection.
370 """
371 raise NotImplementedError()
373 @abstractmethod
374 def registerDatasetType(self, datasetType: DatasetType) -> bool:
375 """Add a new `DatasetType` to the Registry.
377 It is not an error to register the same `DatasetType` twice.
379 Parameters
380 ----------
381 datasetType : `DatasetType`
382 The `DatasetType` to be added.
384 Returns
385 -------
386 inserted : `bool`
387 `True` if ``datasetType`` was inserted, `False` if an identical
388 existing `DatasetType` was found. Note that in either case the
389 DatasetType is guaranteed to be defined in the Registry
390 consistently with the given definition.
392 Raises
393 ------
394 ValueError
395 Raised if the dimensions or storage class are invalid.
396 lsst.daf.butler.registry.ConflictingDefinitionError
397 Raised if this `DatasetType` is already registered with a different
398 definition.
400 Notes
401 -----
402 This method cannot be called within transactions, as it needs to be
403 able to perform its own transaction to be concurrent.
404 """
405 raise NotImplementedError()
407 @abstractmethod
408 def removeDatasetType(self, name: str | tuple[str, ...]) -> None:
409 """Remove the named `DatasetType` from the registry.
411 .. warning::
413 Registry implementations can cache the dataset type definitions.
414 This means that deleting the dataset type definition may result in
415 unexpected behavior from other butler processes that are active
416 that have not seen the deletion.
418 Parameters
419 ----------
420 name : `str` or `tuple` [`str`]
421 Name of the type to be removed or tuple containing a list of type
422 names to be removed. Wildcards are allowed.
424 Raises
425 ------
426 lsst.daf.butler.registry.OrphanedRecordError
427 Raised if an attempt is made to remove the dataset type definition
428 when there are already datasets associated with it.
430 Notes
431 -----
432 If the dataset type is not registered the method will return without
433 action.
434 """
435 raise NotImplementedError()
437 @abstractmethod
438 def getDatasetType(self, name: str) -> DatasetType:
439 """Get the `DatasetType`.
441 Parameters
442 ----------
443 name : `str`
444 Name of the type.
446 Returns
447 -------
448 type : `DatasetType`
449 The `DatasetType` associated with the given name.
451 Raises
452 ------
453 lsst.daf.butler.registry.MissingDatasetTypeError
454 Raised if the requested dataset type has not been registered.
456 Notes
457 -----
458 This method handles component dataset types automatically, though most
459 other registry operations do not.
460 """
461 raise NotImplementedError()
463 @abstractmethod
464 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool:
465 """Test whether the given dataset ID generation mode is supported by
466 `insertDatasets`.
468 Parameters
469 ----------
470 mode : `DatasetIdGenEnum`
471 Enum value for the mode to test.
473 Returns
474 -------
475 supported : `bool`
476 Whether the given mode is supported.
477 """
478 raise NotImplementedError()
480 @abstractmethod
481 def findDataset(
482 self,
483 datasetType: DatasetType | str,
484 dataId: DataId | None = None,
485 *,
486 collections: CollectionArgType | None = None,
487 timespan: Timespan | None = None,
488 datastore_records: bool = False,
489 **kwargs: Any,
490 ) -> DatasetRef | None:
491 """Find a dataset given its `DatasetType` and data ID.
493 This can be used to obtain a `DatasetRef` that permits the dataset to
494 be read from a `Datastore`. If the dataset is a component and can not
495 be found using the provided dataset type, a dataset ref for the parent
496 will be returned instead but with the correct dataset type.
498 Parameters
499 ----------
500 datasetType : `DatasetType` or `str`
501 A `DatasetType` or the name of one. If this is a `DatasetType`
502 instance, its storage class will be respected and propagated to
503 the output, even if it differs from the dataset type definition
504 in the registry, as long as the storage classes are convertible.
505 dataId : `dict` or `DataCoordinate`, optional
506 A `dict`-like object containing the `Dimension` links that identify
507 the dataset within a collection.
508 collections : collection expression, optional
509 An expression that fully or partially identifies the collections to
510 search for the dataset; see
511 :ref:`daf_butler_collection_expressions` for more information.
512 Defaults to ``self.defaults.collections``.
513 timespan : `Timespan`, optional
514 A timespan that the validity range of the dataset must overlap.
515 If not provided, any `~CollectionType.CALIBRATION` collections
516 matched by the ``collections`` argument will not be searched.
517 **kwargs
518 Additional keyword arguments passed to
519 `DataCoordinate.standardize` to convert ``dataId`` to a true
520 `DataCoordinate` or augment an existing one.
522 Returns
523 -------
524 ref : `DatasetRef`
525 A reference to the dataset, or `None` if no matching Dataset
526 was found.
528 Raises
529 ------
530 lsst.daf.butler.registry.NoDefaultCollectionError
531 Raised if ``collections`` is `None` and
532 ``self.defaults.collections`` is `None`.
533 LookupError
534 Raised if one or more data ID keys are missing.
535 lsst.daf.butler.registry.MissingDatasetTypeError
536 Raised if the dataset type does not exist.
537 lsst.daf.butler.registry.MissingCollectionError
538 Raised if any of ``collections`` does not exist in the registry.
540 Notes
541 -----
542 This method simply returns `None` and does not raise an exception even
543 when the set of collections searched is intrinsically incompatible with
544 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but
545 only `~CollectionType.CALIBRATION` collections are being searched.
546 This may make it harder to debug some lookup failures, but the behavior
547 is intentional; we consider it more important that failed searches are
548 reported consistently, regardless of the reason, and that adding
549 additional collections that do not contain a match to the search path
550 never changes the behavior.
552 This method handles component dataset types automatically, though most
553 other registry operations do not.
554 """
555 raise NotImplementedError()
557 @abstractmethod
558 def insertDatasets(
559 self,
560 datasetType: DatasetType | str,
561 dataIds: Iterable[DataId],
562 run: str | None = None,
563 expand: bool = True,
564 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE,
565 ) -> list[DatasetRef]:
566 """Insert one or more datasets into the `Registry`.
568 This always adds new datasets; to associate existing datasets with
569 a new collection, use ``associate``.
571 Parameters
572 ----------
573 datasetType : `DatasetType` or `str`
574 A `DatasetType` or the name of one.
575 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate`
576 Dimension-based identifiers for the new datasets.
577 run : `str`, optional
578 The name of the run that produced the datasets. Defaults to
579 ``self.defaults.run``.
580 expand : `bool`, optional
581 If `True` (default), expand data IDs as they are inserted. This is
582 necessary in general to allow datastore to generate file templates,
583 but it may be disabled if the caller can guarantee this is
584 unnecessary.
585 idGenerationMode : `DatasetIdGenEnum`, optional
586 Specifies option for generating dataset IDs. By default unique IDs
587 are generated for each inserted dataset.
589 Returns
590 -------
591 refs : `list` of `DatasetRef`
592 Resolved `DatasetRef` instances for all given data IDs (in the same
593 order).
595 Raises
596 ------
597 lsst.daf.butler.registry.DatasetTypeError
598 Raised if ``datasetType`` is not known to registry.
599 lsst.daf.butler.registry.CollectionTypeError
600 Raised if ``run`` collection type is not `~CollectionType.RUN`.
601 lsst.daf.butler.registry.NoDefaultCollectionError
602 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
603 lsst.daf.butler.registry.ConflictingDefinitionError
604 If a dataset with the same dataset type and data ID as one of those
605 given already exists in ``run``.
606 lsst.daf.butler.registry.MissingCollectionError
607 Raised if ``run`` does not exist in the registry.
608 """
609 raise NotImplementedError()
611 @abstractmethod
612 def _importDatasets(
613 self,
614 datasets: Iterable[DatasetRef],
615 expand: bool = True,
616 ) -> list[DatasetRef]:
617 """Import one or more datasets into the `Registry`.
619 Difference from `insertDatasets` method is that this method accepts
620 `DatasetRef` instances which should already be resolved and have a
621 dataset ID. If registry supports globally-unique dataset IDs (e.g.
622 `uuid.UUID`) then datasets which already exist in the registry will be
623 ignored if imported again.
625 Parameters
626 ----------
627 datasets : `~collections.abc.Iterable` of `DatasetRef`
628 Datasets to be inserted. All `DatasetRef` instances must have
629 identical ``datasetType`` and ``run`` attributes. ``run``
630 attribute can be `None` and defaults to ``self.defaults.run``.
631 Datasets can specify ``id`` attribute which will be used for
632 inserted datasets. All dataset IDs must have the same type
633 (`int` or `uuid.UUID`), if type of dataset IDs does not match
634 configured backend then IDs will be ignored and new IDs will be
635 generated by backend.
636 expand : `bool`, optional
637 If `True` (default), expand data IDs as they are inserted. This is
638 necessary in general, but it may be disabled if the caller can
639 guarantee this is unnecessary.
641 Returns
642 -------
643 refs : `list` of `DatasetRef`
644 Resolved `DatasetRef` instances for all given data IDs (in the same
645 order). If any of ``datasets`` has an ID which already exists in
646 the database then it will not be inserted or updated, but a
647 resolved `DatasetRef` will be returned for it in any case.
649 Raises
650 ------
651 lsst.daf.butler.registry.NoDefaultCollectionError
652 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`.
653 lsst.daf.butler.registry.DatasetTypeError
654 Raised if datasets correspond to more than one dataset type or
655 dataset type is not known to registry.
656 lsst.daf.butler.registry.ConflictingDefinitionError
657 If a dataset with the same dataset type and data ID as one of those
658 given already exists in ``run``.
659 lsst.daf.butler.registry.MissingCollectionError
660 Raised if ``run`` does not exist in the registry.
662 Notes
663 -----
664 This method is considered package-private and internal to Butler
665 implementation. Clients outside daf_butler package should not use this
666 method.
667 """
668 raise NotImplementedError()
670 @abstractmethod
671 def getDataset(self, id: DatasetId) -> DatasetRef | None:
672 """Retrieve a Dataset entry.
674 Parameters
675 ----------
676 id : `DatasetId`
677 The unique identifier for the dataset.
679 Returns
680 -------
681 ref : `DatasetRef` or `None`
682 A ref to the Dataset, or `None` if no matching Dataset
683 was found.
684 """
685 raise NotImplementedError()
687 @abstractmethod
688 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None:
689 """Remove datasets from the Registry.
691 The datasets will be removed unconditionally from all collections, and
692 any `Quantum` that consumed this dataset will instead be marked with
693 having a NULL input. `Datastore` records will *not* be deleted; the
694 caller is responsible for ensuring that the dataset has already been
695 removed from all Datastores.
697 Parameters
698 ----------
699 refs : `~collections.abc.Iterable` [`DatasetRef`]
700 References to the datasets to be removed. Must include a valid
701 ``id`` attribute, and should be considered invalidated upon return.
703 Raises
704 ------
705 lsst.daf.butler.AmbiguousDatasetError
706 Raised if any ``ref.id`` is `None`.
707 lsst.daf.butler.registry.OrphanedRecordError
708 Raised if any dataset is still present in any `Datastore`.
709 """
710 raise NotImplementedError()
712 @abstractmethod
713 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
714 """Add existing datasets to a `~CollectionType.TAGGED` collection.
716 If a DatasetRef with the same exact ID is already in a collection
717 nothing is changed. If a `DatasetRef` with the same `DatasetType` and
718 data ID but with different ID exists in the collection,
719 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised.
721 Parameters
722 ----------
723 collection : `str`
724 Indicates the collection the datasets should be associated with.
725 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
726 An iterable of resolved `DatasetRef` instances that already exist
727 in this `Registry`.
729 Raises
730 ------
731 lsst.daf.butler.registry.ConflictingDefinitionError
732 If a Dataset with the given `DatasetRef` already exists in the
733 given collection.
734 lsst.daf.butler.registry.MissingCollectionError
735 Raised if ``collection`` does not exist in the registry.
736 lsst.daf.butler.registry.CollectionTypeError
737 Raise adding new datasets to the given ``collection`` is not
738 allowed.
739 """
740 raise NotImplementedError()
742 @abstractmethod
743 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None:
744 """Remove existing datasets from a `~CollectionType.TAGGED` collection.
746 ``collection`` and ``ref`` combinations that are not currently
747 associated are silently ignored.
749 Parameters
750 ----------
751 collection : `str`
752 The collection the datasets should no longer be associated with.
753 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
754 An iterable of resolved `DatasetRef` instances that already exist
755 in this `Registry`.
757 Raises
758 ------
759 lsst.daf.butler.AmbiguousDatasetError
760 Raised if any of the given dataset references is unresolved.
761 lsst.daf.butler.registry.MissingCollectionError
762 Raised if ``collection`` does not exist in the registry.
763 lsst.daf.butler.registry.CollectionTypeError
764 Raise adding new datasets to the given ``collection`` is not
765 allowed.
766 """
767 raise NotImplementedError()
769 @abstractmethod
770 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None:
771 """Associate one or more datasets with a calibration collection and a
772 validity range within it.
774 Parameters
775 ----------
776 collection : `str`
777 The name of an already-registered `~CollectionType.CALIBRATION`
778 collection.
779 refs : `~collections.abc.Iterable` [ `DatasetRef` ]
780 Datasets to be associated.
781 timespan : `Timespan`
782 The validity range for these datasets within the collection.
784 Raises
785 ------
786 lsst.daf.butler.AmbiguousDatasetError
787 Raised if any of the given `DatasetRef` instances is unresolved.
788 lsst.daf.butler.registry.ConflictingDefinitionError
789 Raised if the collection already contains a different dataset with
790 the same `DatasetType` and data ID and an overlapping validity
791 range.
792 lsst.daf.butler.registry.CollectionTypeError
793 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
794 collection or if one or more datasets are of a dataset type for
795 which `DatasetType.isCalibration` returns `False`.
796 """
797 raise NotImplementedError()
799 @abstractmethod
800 def decertify(
801 self,
802 collection: str,
803 datasetType: str | DatasetType,
804 timespan: Timespan,
805 *,
806 dataIds: Iterable[DataId] | None = None,
807 ) -> None:
808 """Remove or adjust datasets to clear a validity range within a
809 calibration collection.
811 Parameters
812 ----------
813 collection : `str`
814 The name of an already-registered `~CollectionType.CALIBRATION`
815 collection.
816 datasetType : `str` or `DatasetType`
817 Name or `DatasetType` instance for the datasets to be decertified.
818 timespan : `Timespan`, optional
819 The validity range to remove datasets from within the collection.
820 Datasets that overlap this range but are not contained by it will
821 have their validity ranges adjusted to not overlap it, which may
822 split a single dataset validity range into two.
823 dataIds : iterable [`dict` or `DataCoordinate`], optional
824 Data IDs that should be decertified within the given validity range
825 If `None`, all data IDs for ``self.datasetType`` will be
826 decertified.
828 Raises
829 ------
830 lsst.daf.butler.registry.CollectionTypeError
831 Raised if ``collection`` is not a `~CollectionType.CALIBRATION`
832 collection or if ``datasetType.isCalibration() is False``.
833 """
834 raise NotImplementedError()
836 @abstractmethod
837 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]:
838 """Retrieve datastore locations for a given dataset.
840 Parameters
841 ----------
842 ref : `DatasetRef`
843 A reference to the dataset for which to retrieve storage
844 information.
846 Returns
847 -------
848 datastores : `~collections.abc.Iterable` [ `str` ]
849 All the matching datastores holding this dataset.
851 Raises
852 ------
853 lsst.daf.butler.AmbiguousDatasetError
854 Raised if ``ref.id`` is `None`.
855 """
856 raise NotImplementedError()
858 @abstractmethod
859 def expandDataId(
860 self,
861 dataId: DataId | None = None,
862 *,
863 dimensions: Iterable[str] | DimensionGroup | DimensionGraph | None = None,
864 graph: DimensionGraph | None = None,
865 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None,
866 withDefaults: bool = True,
867 **kwargs: Any,
868 ) -> DataCoordinate:
869 """Expand a dimension-based data ID to include additional information.
871 Parameters
872 ----------
873 dataId : `DataCoordinate` or `dict`, optional
874 Data ID to be expanded; augmented and overridden by ``kwargs``.
875 dimensions : `~collections.abc.Iterable` [ `str` ], \
876 `DimensionGroup`, or `DimensionGraph`, optional
877 The dimensions to be identified by the new `DataCoordinate`.
878 If not provided, will be inferred from the keys of ``mapping`` and
879 ``**kwargs``, and ``universe`` must be provided unless ``mapping``
880 is already a `DataCoordinate`.
881 graph : `DimensionGraph`, optional
882 Like ``dimensions``, but as a ``DimensionGraph`` instance. Ignored
883 if ``dimensions`` is provided. Deprecated and will be removed
884 after v27.
885 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \
886 optional
887 Dimension record data to use before querying the database for that
888 data, keyed by element name.
889 withDefaults : `bool`, optional
890 Utilize ``self.defaults.dataId`` to fill in missing governor
891 dimension key-value pairs. Defaults to `True` (i.e. defaults are
892 used).
893 **kwargs
894 Additional keywords are treated like additional key-value pairs for
895 ``dataId``, extending and overriding
897 Returns
898 -------
899 expanded : `DataCoordinate`
900 A data ID that includes full metadata for all of the dimensions it
901 identifies, i.e. guarantees that ``expanded.hasRecords()`` and
902 ``expanded.hasFull()`` both return `True`.
904 Raises
905 ------
906 lsst.daf.butler.registry.DataIdError
907 Raised when ``dataId`` or keyword arguments specify unknown
908 dimensions or values, or when a resulting data ID contains
909 contradictory key-value pairs, according to dimension
910 relationships.
912 Notes
913 -----
914 This method cannot be relied upon to reject invalid data ID values
915 for dimensions that do actually not have any record columns. For
916 efficiency reasons the records for these dimensions (which have only
917 dimension key values that are given by the caller) may be constructed
918 directly rather than obtained from the registry database.
919 """
920 raise NotImplementedError()
922 @abstractmethod
923 def insertDimensionData(
924 self,
925 element: DimensionElement | str,
926 *data: Mapping[str, Any] | DimensionRecord,
927 conform: bool = True,
928 replace: bool = False,
929 skip_existing: bool = False,
930 ) -> None:
931 """Insert one or more dimension records into the database.
933 Parameters
934 ----------
935 element : `DimensionElement` or `str`
936 The `DimensionElement` or name thereof that identifies the table
937 records will be inserted into.
938 *data : `dict` or `DimensionRecord`
939 One or more records to insert.
940 conform : `bool`, optional
941 If `False` (`True` is default) perform no checking or conversions,
942 and assume that ``element`` is a `DimensionElement` instance and
943 ``data`` is a one or more `DimensionRecord` instances of the
944 appropriate subclass.
945 replace : `bool`, optional
946 If `True` (`False` is default), replace existing records in the
947 database if there is a conflict.
948 skip_existing : `bool`, optional
949 If `True` (`False` is default), skip insertion if a record with
950 the same primary key values already exists. Unlike
951 `syncDimensionData`, this will not detect when the given record
952 differs from what is in the database, and should not be used when
953 this is a concern.
954 """
955 raise NotImplementedError()
957 @abstractmethod
958 def syncDimensionData(
959 self,
960 element: DimensionElement | str,
961 row: Mapping[str, Any] | DimensionRecord,
962 conform: bool = True,
963 update: bool = False,
964 ) -> bool | dict[str, Any]:
965 """Synchronize the given dimension record with the database, inserting
966 if it does not already exist and comparing values if it does.
968 Parameters
969 ----------
970 element : `DimensionElement` or `str`
971 The `DimensionElement` or name thereof that identifies the table
972 records will be inserted into.
973 row : `dict` or `DimensionRecord`
974 The record to insert.
975 conform : `bool`, optional
976 If `False` (`True` is default) perform no checking or conversions,
977 and assume that ``element`` is a `DimensionElement` instance and
978 ``data`` is a one or more `DimensionRecord` instances of the
979 appropriate subclass.
980 update : `bool`, optional
981 If `True` (`False` is default), update the existing record in the
982 database if there is a conflict.
984 Returns
985 -------
986 inserted_or_updated : `bool` or `dict`
987 `True` if a new row was inserted, `False` if no changes were
988 needed, or a `dict` mapping updated column names to their old
989 values if an update was performed (only possible if
990 ``update=True``).
992 Raises
993 ------
994 lsst.daf.butler.registry.ConflictingDefinitionError
995 Raised if the record exists in the database (according to primary
996 key lookup) but is inconsistent with the given one.
997 """
998 raise NotImplementedError()
1000 @abstractmethod
1001 def queryDatasetTypes(
1002 self,
1003 expression: Any = ...,
1004 *,
1005 components: bool | None = False,
1006 missing: list[str] | None = None,
1007 ) -> Iterable[DatasetType]:
1008 """Iterate over the dataset types whose names match an expression.
1010 Parameters
1011 ----------
1012 expression : dataset type expression, optional
1013 An expression that fully or partially identifies the dataset types
1014 to return, such as a `str`, `re.Pattern`, or iterable thereof.
1015 ``...`` can be used to return all dataset types, and is the
1016 default. See :ref:`daf_butler_dataset_type_expressions` for more
1017 information.
1018 components : `bool`, optional
1019 If `True`, apply all expression patterns to component dataset type
1020 names as well. If `False`, never apply patterns to components.
1021 If `None`, apply patterns to components only if their
1022 parent datasets were not matched by the expression.
1023 Fully-specified component datasets (`str` or `DatasetType`
1024 instances) are always included.
1026 Values other than `False` are deprecated, and only `False` will be
1027 supported after v26. After v27 this argument will be removed
1028 entirely.
1029 missing : `list` of `str`, optional
1030 String dataset type names that were explicitly given (i.e. not
1031 regular expression patterns) but not found will be appended to this
1032 list, if it is provided.
1034 Returns
1035 -------
1036 dataset_types : `~collections.abc.Iterable` [ `DatasetType`]
1037 An `~collections.abc.Iterable` of `DatasetType` instances whose
1038 names match ``expression``.
1040 Raises
1041 ------
1042 lsst.daf.butler.registry.DatasetTypeExpressionError
1043 Raised when ``expression`` is invalid.
1044 """
1045 raise NotImplementedError()
1047 @abstractmethod
1048 def queryCollections(
1049 self,
1050 expression: Any = ...,
1051 datasetType: DatasetType | None = None,
1052 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(),
1053 flattenChains: bool = False,
1054 includeChains: bool | None = None,
1055 ) -> Sequence[str]:
1056 """Iterate over the collections whose names match an expression.
1058 Parameters
1059 ----------
1060 expression : collection expression, optional
1061 An expression that identifies the collections to return, such as
1062 a `str` (for full matches or partial matches via globs),
1063 `re.Pattern` (for partial matches), or iterable thereof. ``...``
1064 can be used to return all collections, and is the default.
1065 See :ref:`daf_butler_collection_expressions` for more information.
1066 datasetType : `DatasetType`, optional
1067 If provided, only yield collections that may contain datasets of
1068 this type. This is a conservative approximation in general; it may
1069 yield collections that do not have any such datasets.
1070 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \
1071 `CollectionType`, optional
1072 If provided, only yield collections of these types.
1073 flattenChains : `bool`, optional
1074 If `True` (`False` is default), recursively yield the child
1075 collections of matching `~CollectionType.CHAINED` collections.
1076 includeChains : `bool`, optional
1077 If `True`, yield records for matching `~CollectionType.CHAINED`
1078 collections. Default is the opposite of ``flattenChains``: include
1079 either CHAINED collections or their children, but not both.
1081 Returns
1082 -------
1083 collections : `~collections.abc.Sequence` [ `str` ]
1084 The names of collections that match ``expression``.
1086 Raises
1087 ------
1088 lsst.daf.butler.registry.CollectionExpressionError
1089 Raised when ``expression`` is invalid.
1091 Notes
1092 -----
1093 The order in which collections are returned is unspecified, except that
1094 the children of a `~CollectionType.CHAINED` collection are guaranteed
1095 to be in the order in which they are searched. When multiple parent
1096 `~CollectionType.CHAINED` collections match the same criteria, the
1097 order in which the two lists appear is unspecified, and the lists of
1098 children may be incomplete if a child has multiple parents.
1099 """
1100 raise NotImplementedError()
1102 @abstractmethod
1103 def queryDatasets(
1104 self,
1105 datasetType: Any,
1106 *,
1107 collections: CollectionArgType | None = None,
1108 dimensions: Iterable[Dimension | str] | None = None,
1109 dataId: DataId | None = None,
1110 where: str = "",
1111 findFirst: bool = False,
1112 components: bool | None = False,
1113 bind: Mapping[str, Any] | None = None,
1114 check: bool = True,
1115 **kwargs: Any,
1116 ) -> DatasetQueryResults:
1117 """Query for and iterate over dataset references matching user-provided
1118 criteria.
1120 Parameters
1121 ----------
1122 datasetType : dataset type expression
1123 An expression that fully or partially identifies the dataset types
1124 to be queried. Allowed types include `DatasetType`, `str`,
1125 `re.Pattern`, and iterables thereof. The special value ``...`` can
1126 be used to query all dataset types. See
1127 :ref:`daf_butler_dataset_type_expressions` for more information.
1128 collections : collection expression, optional
1129 An expression that identifies the collections to search, such as a
1130 `str` (for full matches or partial matches via globs), `re.Pattern`
1131 (for partial matches), or iterable thereof. ``...`` can be used to
1132 search all collections (actually just all `~CollectionType.RUN`
1133 collections, because this will still find all datasets).
1134 If not provided, ``self.default.collections`` is used. See
1135 :ref:`daf_butler_collection_expressions` for more information.
1136 dimensions : `~collections.abc.Iterable` of `Dimension` or `str`
1137 Dimensions to include in the query (in addition to those used
1138 to identify the queried dataset type(s)), either to constrain
1139 the resulting datasets to those for which a matching dimension
1140 exists, or to relate the dataset type's dimensions to dimensions
1141 referenced by the ``dataId`` or ``where`` arguments.
1142 dataId : `dict` or `DataCoordinate`, optional
1143 A data ID whose key-value pairs are used as equality constraints
1144 in the query.
1145 where : `str`, optional
1146 A string expression similar to a SQL WHERE clause. May involve
1147 any column of a dimension table or (as a shortcut for the primary
1148 key column of a dimension table) dimension name. See
1149 :ref:`daf_butler_dimension_expressions` for more information.
1150 findFirst : `bool`, optional
1151 If `True` (`False` is default), for each result data ID, only
1152 yield one `DatasetRef` of each `DatasetType`, from the first
1153 collection in which a dataset of that dataset type appears
1154 (according to the order of ``collections`` passed in). If `True`,
1155 ``collections`` must not contain regular expressions and may not
1156 be ``...``.
1157 components : `bool`, optional
1158 If `True`, apply all dataset expression patterns to component
1159 dataset type names as well. If `False`, never apply patterns to
1160 components. If `None`, apply patterns to components only
1161 if their parent datasets were not matched by the expression.
1162 Fully-specified component datasets (`str` or `DatasetType`
1163 instances) are always included.
1165 Values other than `False` are deprecated, and only `False` will be
1166 supported after v26. After v27 this argument will be removed
1167 entirely.
1168 bind : `~collections.abc.Mapping`, optional
1169 Mapping containing literal values that should be injected into the
1170 ``where`` expression, keyed by the identifiers they replace.
1171 Values of collection type can be expanded in some cases; see
1172 :ref:`daf_butler_dimension_expressions_identifiers` for more
1173 information.
1174 check : `bool`, optional
1175 If `True` (default) check the query for consistency before
1176 executing it. This may reject some valid queries that resemble
1177 common mistakes (e.g. queries for visits without specifying an
1178 instrument).
1179 **kwargs
1180 Additional keyword arguments are forwarded to
1181 `DataCoordinate.standardize` when processing the ``dataId``
1182 argument (and may be used to provide a constraining data ID even
1183 when the ``dataId`` argument is `None`).
1185 Returns
1186 -------
1187 refs : `.queries.DatasetQueryResults`
1188 Dataset references matching the given query criteria. Nested data
1189 IDs are guaranteed to include values for all implied dimensions
1190 (i.e. `DataCoordinate.hasFull` will return `True`), but will not
1191 include dimension records (`DataCoordinate.hasRecords` will be
1192 `False`) unless `~.queries.DatasetQueryResults.expanded` is
1193 called on the result object (which returns a new one).
1195 Raises
1196 ------
1197 lsst.daf.butler.registry.DatasetTypeExpressionError
1198 Raised when ``datasetType`` expression is invalid.
1199 TypeError
1200 Raised when the arguments are incompatible, such as when a
1201 collection wildcard is passed when ``findFirst`` is `True`, or
1202 when ``collections`` is `None` and ``self.defaults.collections`` is
1203 also `None`.
1204 lsst.daf.butler.registry.DataIdError
1205 Raised when ``dataId`` or keyword arguments specify unknown
1206 dimensions or values, or when they contain inconsistent values.
1207 lsst.daf.butler.registry.UserExpressionError
1208 Raised when ``where`` expression is invalid.
1210 Notes
1211 -----
1212 When multiple dataset types are queried in a single call, the
1213 results of this operation are equivalent to querying for each dataset
1214 type separately in turn, and no information about the relationships
1215 between datasets of different types is included. In contexts where
1216 that kind of information is important, the recommended pattern is to
1217 use `queryDataIds` to first obtain data IDs (possibly with the
1218 desired dataset types and collections passed as constraints to the
1219 query), and then use multiple (generally much simpler) calls to
1220 `queryDatasets` with the returned data IDs passed as constraints.
1221 """
1222 raise NotImplementedError()
1224 @abstractmethod
1225 def queryDataIds(
1226 self,
1227 # TODO: Drop `Dimension` objects on DM-41326.
1228 dimensions: DimensionGroup | Iterable[Dimension | str] | Dimension | str,
1229 *,
1230 dataId: DataId | None = None,
1231 datasets: Any = None,
1232 collections: CollectionArgType | None = None,
1233 where: str = "",
1234 components: bool | None = False,
1235 bind: Mapping[str, Any] | None = None,
1236 check: bool = True,
1237 **kwargs: Any,
1238 ) -> DataCoordinateQueryResults:
1239 """Query for data IDs matching user-provided criteria.
1241 Parameters
1242 ----------
1243 dimensions : `DimensionGroup`, `Dimension`, or `str`, or \
1244 `~collections.abc.Iterable` [ `Dimension` or `str` ]
1245 The dimensions of the data IDs to yield, as either `Dimension`
1246 instances or `str`. Will be automatically expanded to a complete
1247 `DimensionGroup`. Support for `Dimension` instances is deprecated
1248 and will not be supported after v27.
1249 dataId : `dict` or `DataCoordinate`, optional
1250 A data ID whose key-value pairs are used as equality constraints
1251 in the query.
1252 datasets : dataset type expression, optional
1253 An expression that fully or partially identifies dataset types
1254 that should constrain the yielded data IDs. For example, including
1255 "raw" here would constrain the yielded ``instrument``,
1256 ``exposure``, ``detector``, and ``physical_filter`` values to only
1257 those for which at least one "raw" dataset exists in
1258 ``collections``. Allowed types include `DatasetType`, `str`,
1259 and iterables thereof. Regular expression objects (i.e.
1260 `re.Pattern`) are deprecated and will be removed after the v26
1261 release. See :ref:`daf_butler_dataset_type_expressions` for more
1262 information.
1263 collections : collection expression, optional
1264 An expression that identifies the collections to search for
1265 datasets, such as a `str` (for full matches or partial matches
1266 via globs), `re.Pattern` (for partial matches), or iterable
1267 thereof. ``...`` can be used to search all collections (actually
1268 just all `~CollectionType.RUN` collections, because this will
1269 still find all datasets). If not provided,
1270 ``self.default.collections`` is used. Ignored unless ``datasets``
1271 is also passed. See :ref:`daf_butler_collection_expressions` for
1272 more information.
1273 where : `str`, optional
1274 A string expression similar to a SQL WHERE clause. May involve
1275 any column of a dimension table or (as a shortcut for the primary
1276 key column of a dimension table) dimension name. See
1277 :ref:`daf_butler_dimension_expressions` for more information.
1278 components : `bool`, optional
1279 If `True`, apply all dataset expression patterns to component
1280 dataset type names as well. If `False`, never apply patterns to
1281 components. If `None`, apply patterns to components only
1282 if their parent datasets were not matched by the expression.
1283 Fully-specified component datasets (`str` or `DatasetType`
1284 instances) are always included.
1286 Values other than `False` are deprecated, and only `False` will be
1287 supported after v26. After v27 this argument will be removed
1288 entirely.
1289 bind : `~collections.abc.Mapping`, optional
1290 Mapping containing literal values that should be injected into the
1291 ``where`` expression, keyed by the identifiers they replace.
1292 Values of collection type can be expanded in some cases; see
1293 :ref:`daf_butler_dimension_expressions_identifiers` for more
1294 information.
1295 check : `bool`, optional
1296 If `True` (default) check the query for consistency before
1297 executing it. This may reject some valid queries that resemble
1298 common mistakes (e.g. queries for visits without specifying an
1299 instrument).
1300 **kwargs
1301 Additional keyword arguments are forwarded to
1302 `DataCoordinate.standardize` when processing the ``dataId``
1303 argument (and may be used to provide a constraining data ID even
1304 when the ``dataId`` argument is `None`).
1306 Returns
1307 -------
1308 dataIds : `.queries.DataCoordinateQueryResults`
1309 Data IDs matching the given query parameters. These are guaranteed
1310 to identify all dimensions (`DataCoordinate.hasFull` returns
1311 `True`), but will not contain `DimensionRecord` objects
1312 (`DataCoordinate.hasRecords` returns `False`). Call
1313 `~.queries.DataCoordinateQueryResults.expanded` on the
1314 returned object to fetch those (and consider using
1315 `~.queries.DataCoordinateQueryResults.materialize` on the
1316 returned object first if the expected number of rows is very
1317 large). See documentation for those methods for additional
1318 information.
1320 Raises
1321 ------
1322 lsst.daf.butler.registry.NoDefaultCollectionError
1323 Raised if ``collections`` is `None` and
1324 ``self.defaults.collections`` is `None`.
1325 lsst.daf.butler.registry.CollectionExpressionError
1326 Raised when ``collections`` expression is invalid.
1327 lsst.daf.butler.registry.DataIdError
1328 Raised when ``dataId`` or keyword arguments specify unknown
1329 dimensions or values, or when they contain inconsistent values.
1330 lsst.daf.butler.registry.DatasetTypeExpressionError
1331 Raised when ``datasetType`` expression is invalid.
1332 lsst.daf.butler.registry.UserExpressionError
1333 Raised when ``where`` expression is invalid.
1334 """
1335 raise NotImplementedError()
1337 @abstractmethod
1338 def queryDimensionRecords(
1339 self,
1340 element: DimensionElement | str,
1341 *,
1342 dataId: DataId | None = None,
1343 datasets: Any = None,
1344 collections: CollectionArgType | None = None,
1345 where: str = "",
1346 components: bool | None = False,
1347 bind: Mapping[str, Any] | None = None,
1348 check: bool = True,
1349 **kwargs: Any,
1350 ) -> DimensionRecordQueryResults:
1351 """Query for dimension information matching user-provided criteria.
1353 Parameters
1354 ----------
1355 element : `DimensionElement` or `str`
1356 The dimension element to obtain records for.
1357 dataId : `dict` or `DataCoordinate`, optional
1358 A data ID whose key-value pairs are used as equality constraints
1359 in the query.
1360 datasets : dataset type expression, optional
1361 An expression that fully or partially identifies dataset types
1362 that should constrain the yielded records. See `queryDataIds` and
1363 :ref:`daf_butler_dataset_type_expressions` for more information.
1364 collections : collection expression, optional
1365 An expression that identifies the collections to search for
1366 datasets, such as a `str` (for full matches or partial matches
1367 via globs), `re.Pattern` (for partial matches), or iterable
1368 thereof. ``...`` can be used to search all collections (actually
1369 just all `~CollectionType.RUN` collections, because this will
1370 still find all datasets). If not provided,
1371 ``self.default.collections`` is used. Ignored unless ``datasets``
1372 is also passed. See :ref:`daf_butler_collection_expressions` for
1373 more information.
1374 where : `str`, optional
1375 A string expression similar to a SQL WHERE clause. See
1376 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more
1377 information.
1378 components : `bool`, optional
1379 Whether to apply dataset expressions to components as well.
1380 See `queryDataIds` for more information.
1382 Values other than `False` are deprecated, and only `False` will be
1383 supported after v26. After v27 this argument will be removed
1384 entirely.
1385 bind : `~collections.abc.Mapping`, optional
1386 Mapping containing literal values that should be injected into the
1387 ``where`` expression, keyed by the identifiers they replace.
1388 Values of collection type can be expanded in some cases; see
1389 :ref:`daf_butler_dimension_expressions_identifiers` for more
1390 information.
1391 check : `bool`, optional
1392 If `True` (default) check the query for consistency before
1393 executing it. This may reject some valid queries that resemble
1394 common mistakes (e.g. queries for visits without specifying an
1395 instrument).
1396 **kwargs
1397 Additional keyword arguments are forwarded to
1398 `DataCoordinate.standardize` when processing the ``dataId``
1399 argument (and may be used to provide a constraining data ID even
1400 when the ``dataId`` argument is `None`).
1402 Returns
1403 -------
1404 dataIds : `.queries.DimensionRecordQueryResults`
1405 Data IDs matching the given query parameters.
1407 Raises
1408 ------
1409 lsst.daf.butler.registry.NoDefaultCollectionError
1410 Raised if ``collections`` is `None` and
1411 ``self.defaults.collections`` is `None`.
1412 lsst.daf.butler.registry.CollectionExpressionError
1413 Raised when ``collections`` expression is invalid.
1414 lsst.daf.butler.registry.DataIdError
1415 Raised when ``dataId`` or keyword arguments specify unknown
1416 dimensions or values, or when they contain inconsistent values.
1417 lsst.daf.butler.registry.DatasetTypeExpressionError
1418 Raised when ``datasetType`` expression is invalid.
1419 lsst.daf.butler.registry.UserExpressionError
1420 Raised when ``where`` expression is invalid.
1421 """
1422 raise NotImplementedError()
1424 @abstractmethod
1425 def queryDatasetAssociations(
1426 self,
1427 datasetType: str | DatasetType,
1428 collections: CollectionArgType | None = ...,
1429 *,
1430 collectionTypes: Iterable[CollectionType] = CollectionType.all(),
1431 flattenChains: bool = False,
1432 ) -> Iterator[DatasetAssociation]:
1433 """Iterate over dataset-collection combinations where the dataset is in
1434 the collection.
1436 This method is a temporary placeholder for better support for
1437 association results in `queryDatasets`. It will probably be
1438 removed in the future, and should be avoided in production code
1439 whenever possible.
1441 Parameters
1442 ----------
1443 datasetType : `DatasetType` or `str`
1444 A dataset type object or the name of one.
1445 collections : collection expression, optional
1446 An expression that identifies the collections to search for
1447 datasets, such as a `str` (for full matches or partial matches
1448 via globs), `re.Pattern` (for partial matches), or iterable
1449 thereof. ``...`` can be used to search all collections (actually
1450 just all `~CollectionType.RUN` collections, because this will still
1451 find all datasets). If not provided, ``self.default.collections``
1452 is used. See :ref:`daf_butler_collection_expressions` for more
1453 information.
1454 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional
1455 If provided, only yield associations from collections of these
1456 types.
1457 flattenChains : `bool`, optional
1458 If `True`, search in the children of `~CollectionType.CHAINED`
1459 collections. If `False`, ``CHAINED`` collections are ignored.
1461 Yields
1462 ------
1463 association : `.DatasetAssociation`
1464 Object representing the relationship between a single dataset and
1465 a single collection.
1467 Raises
1468 ------
1469 lsst.daf.butler.registry.NoDefaultCollectionError
1470 Raised if ``collections`` is `None` and
1471 ``self.defaults.collections`` is `None`.
1472 lsst.daf.butler.registry.CollectionExpressionError
1473 Raised when ``collections`` expression is invalid.
1474 """
1475 raise NotImplementedError()
1477 @property
1478 def obsCoreTableManager(self) -> ObsCoreTableManager | None:
1479 """The ObsCore manager instance for this registry
1480 (`~.interfaces.ObsCoreTableManager`
1481 or `None`).
1483 ObsCore manager may not be implemented for all registry backend, or
1484 may not be enabled for many repositories.
1485 """
1486 return None
1488 storageClasses: StorageClassFactory
1489 """All storage classes known to the registry (`StorageClassFactory`).
1490 """