Coverage for python/lsst/daf/butler/registry/interfaces/_datasets.py: 77%
77 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-10-09 03:23 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30from ... import ddl
32__all__ = ("DatasetRecordStorageManager",)
34from abc import abstractmethod
35from collections.abc import Iterable, Mapping, Sequence, Set
36from typing import TYPE_CHECKING, Any
38from lsst.daf.relation import Relation
40from ..._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef
41from ..._dataset_type import DatasetType
42from ..._exceptions import DatasetTypeError, DatasetTypeNotSupportedError
43from ..._timespan import Timespan
44from ...dimensions import DataCoordinate
45from ._versioning import VersionedExtension, VersionTuple
47if TYPE_CHECKING:
48 from ...direct_query_driver import SqlJoinsBuilder # new query system, server+direct only
49 from .._caching_context import CachingContext
50 from .._collection_summary import CollectionSummary
51 from ..queries import SqlQueryContext # old registry query system
52 from ._collections import CollectionManager, CollectionRecord, RunRecord
53 from ._database import Database, StaticTablesContext
54 from ._dimensions import DimensionRecordStorageManager
57class DatasetRecordStorageManager(VersionedExtension):
58 """An interface that manages the tables that describe datasets.
60 `DatasetRecordStorageManager` primarily serves as a container and factory
61 for `DatasetRecordStorage` instances, which each provide access to the
62 records for a different `DatasetType`.
64 Parameters
65 ----------
66 registry_schema_version : `VersionTuple` or `None`, optional
67 Version of registry schema.
68 """
70 def __init__(self, *, registry_schema_version: VersionTuple | None = None) -> None:
71 super().__init__(registry_schema_version=registry_schema_version)
73 @abstractmethod
74 def clone(
75 self,
76 *,
77 db: Database,
78 collections: CollectionManager,
79 dimensions: DimensionRecordStorageManager,
80 caching_context: CachingContext,
81 ) -> DatasetRecordStorageManager:
82 """Make an independent copy of this manager instance bound to new
83 instances of `Database` and other managers.
85 Parameters
86 ----------
87 db : `Database`
88 New `Database` object to use when instantiating the manager.
89 collections : `CollectionManager`
90 New `CollectionManager` object to use when instantiating the
91 manager.
92 dimensions : `DimensionRecordStorageManager`
93 New `DimensionRecordStorageManager` object to use when
94 instantiating the manager.
95 caching_context : `CachingContext`
96 New `CachingContext` object to use when instantiating the manager.
98 Returns
99 -------
100 instance : `DatasetRecordStorageManager`
101 New manager instance with the same configuration as this instance,
102 but bound to a new Database object.
103 """
104 raise NotImplementedError()
106 @abstractmethod
107 def preload_cache(self) -> None:
108 """Fetch data from the database and use it to pre-populate caches to
109 speed up later operations.
110 """
111 raise NotImplementedError()
113 @classmethod
114 @abstractmethod
115 def initialize(
116 cls,
117 db: Database,
118 context: StaticTablesContext,
119 *,
120 collections: CollectionManager,
121 dimensions: DimensionRecordStorageManager,
122 caching_context: CachingContext,
123 registry_schema_version: VersionTuple | None = None,
124 ) -> DatasetRecordStorageManager:
125 """Construct an instance of the manager.
127 Parameters
128 ----------
129 db : `Database`
130 Interface to the underlying database engine and namespace.
131 context : `StaticTablesContext`
132 Context object obtained from `Database.declareStaticTables`; used
133 to declare any tables that should always be present.
134 collections : `CollectionManager`
135 Manager object for the collections in this `Registry`.
136 dimensions : `DimensionRecordStorageManager`
137 Manager object for the dimensions in this `Registry`.
138 caching_context : `CachingContext`
139 Object controlling caching of information returned by managers.
140 registry_schema_version : `VersionTuple` or `None`
141 Schema version of this extension as defined in registry.
143 Returns
144 -------
145 manager : `DatasetRecordStorageManager`
146 An instance of a concrete `DatasetRecordStorageManager` subclass.
147 """
148 raise NotImplementedError()
150 @classmethod
151 @abstractmethod
152 def addDatasetForeignKey(
153 cls,
154 tableSpec: ddl.TableSpec,
155 *,
156 name: str = "dataset",
157 constraint: bool = True,
158 onDelete: str | None = None,
159 **kwargs: Any,
160 ) -> ddl.FieldSpec:
161 """Add a foreign key (field and constraint) referencing the dataset
162 table.
164 Parameters
165 ----------
166 tableSpec : `ddl.TableSpec`
167 Specification for the table that should reference the dataset
168 table. Will be modified in place.
169 name : `str`, optional
170 A name to use for the prefix of the new field; the full name is
171 ``{name}_id``.
172 constraint : `bool`, optional
173 If `False` (`True` is default), add a field that can be joined to
174 the dataset primary key, but do not add a foreign key constraint.
175 onDelete : `str`, optional
176 One of "CASCADE" or "SET NULL", indicating what should happen to
177 the referencing row if the collection row is deleted. `None`
178 indicates that this should be an integrity error.
179 **kwargs
180 Additional keyword arguments are forwarded to the `ddl.FieldSpec`
181 constructor (only the ``name`` and ``dtype`` arguments are
182 otherwise provided).
184 Returns
185 -------
186 idSpec : `ddl.FieldSpec`
187 Specification for the ID field.
188 """
189 raise NotImplementedError()
191 @abstractmethod
192 def refresh(self) -> None:
193 """Ensure all other operations on this manager are aware of any
194 dataset types that may have been registered by other clients since
195 it was initialized or last refreshed.
196 """
197 raise NotImplementedError()
199 @abstractmethod
200 def get_dataset_type(self, name: str) -> DatasetType:
201 """Look up a dataset type by name.
203 Parameters
204 ----------
205 name : `str`
206 Name of a parent dataset type.
208 Returns
209 -------
210 dataset_type : `DatasetType`
211 The object representing the records for the given dataset type.
213 Raises
214 ------
215 MissingDatasetTypeError
216 Raised if there is no dataset type with the given name.
217 """
218 raise NotImplementedError()
220 def conform_exact_dataset_type(self, dataset_type: DatasetType | str) -> DatasetType:
221 """Conform a value that may be a dataset type or dataset type name to
222 just the dataset type name, while checking that the dataset type is not
223 a component and (if a `DatasetType` instance is given) has the exact
224 same definition in the registry.
226 Parameters
227 ----------
228 dataset_type : `str` or `DatasetType`
229 Dataset type object or name.
231 Returns
232 -------
233 dataset_type : `DatasetType`
234 The corresponding registered dataset type.
236 Raises
237 ------
238 DatasetTypeError
239 Raised if ``dataset_type`` is a component, or if its definition
240 does not exactly match the registered dataset type.
241 MissingDatasetTypeError
242 Raised if this dataset type is not registered at all.
243 """
244 if isinstance(dataset_type, DatasetType):
245 dataset_type_name = dataset_type.name
246 given_dataset_type = dataset_type
247 else:
248 dataset_type_name = dataset_type
249 given_dataset_type = None
250 parent_name, component = DatasetType.splitDatasetTypeName(dataset_type_name)
251 if component is not None:
252 raise DatasetTypeNotSupportedError(
253 f"Component dataset {dataset_type_name!r} is not supported in this context."
254 )
255 registered_dataset_type = self.get_dataset_type(dataset_type_name)
256 if given_dataset_type is not None and registered_dataset_type != given_dataset_type:
257 raise DatasetTypeError(
258 f"Given dataset type {given_dataset_type} is not identical to the "
259 f"registered one {registered_dataset_type}."
260 )
261 return registered_dataset_type
263 @abstractmethod
264 def register_dataset_type(self, dataset_type: DatasetType) -> bool:
265 """Ensure that this `Registry` can hold records for the given
266 `DatasetType`, creating new tables as necessary.
268 Parameters
269 ----------
270 dataset_type : `DatasetType`
271 Dataset type for which a table should created (as necessary) and
272 an associated `DatasetRecordStorage` returned.
274 Returns
275 -------
276 inserted : `bool`
277 `True` if the dataset type did not exist in the registry before.
279 Notes
280 -----
281 This operation may not be invoked within a `Database.transaction`
282 context.
283 """
284 raise NotImplementedError()
286 @abstractmethod
287 def remove_dataset_type(self, name: str) -> None:
288 """Remove the dataset type.
290 Parameters
291 ----------
292 name : `str`
293 Name of the dataset type.
294 """
295 raise NotImplementedError()
297 @abstractmethod
298 def resolve_wildcard(
299 self,
300 expression: Any,
301 missing: list[str] | None = None,
302 explicit_only: bool = False,
303 ) -> list[DatasetType]:
304 """Resolve a dataset type wildcard expression.
306 Parameters
307 ----------
308 expression : `~typing.Any`
309 Expression to resolve. Will be passed to
310 `DatasetTypeWildcard.from_expression`.
311 missing : `list` of `str`, optional
312 String dataset type names that were explicitly given (i.e. not
313 regular expression patterns) but not found will be appended to this
314 list, if it is provided.
315 explicit_only : `bool`, optional
316 If `True`, require explicit `DatasetType` instances or `str` names,
317 with `re.Pattern` instances deprecated and ``...`` prohibited.
319 Returns
320 -------
321 dataset_types : `list` [ `DatasetType` ]
322 A list of resolved dataset types.
323 """
324 raise NotImplementedError()
326 @abstractmethod
327 def getDatasetRef(self, id: DatasetId) -> DatasetRef | None:
328 """Return a `DatasetRef` for the given dataset primary key
329 value.
331 Parameters
332 ----------
333 id : `DatasetId`
334 Primary key value for the dataset.
336 Returns
337 -------
338 ref : `DatasetRef` or `None`
339 Object representing the dataset, or `None` if no dataset with the
340 given primary key values exists in this layer.
341 """
342 raise NotImplementedError()
344 @abstractmethod
345 def getCollectionSummary(self, collection: CollectionRecord) -> CollectionSummary:
346 """Return a summary for the given collection.
348 Parameters
349 ----------
350 collection : `CollectionRecord`
351 Record describing the collection for which a summary is to be
352 retrieved.
354 Returns
355 -------
356 summary : `CollectionSummary`
357 Summary of the dataset types and governor dimension values in
358 this collection.
359 """
360 raise NotImplementedError()
362 @abstractmethod
363 def fetch_summaries(
364 self,
365 collections: Iterable[CollectionRecord],
366 dataset_types: Iterable[DatasetType] | Iterable[str] | None = None,
367 ) -> Mapping[Any, CollectionSummary]:
368 """Fetch collection summaries given their names and dataset types.
370 Parameters
371 ----------
372 collections : `~collections.abc.Iterable` [`CollectionRecord`]
373 Collection records to query.
374 dataset_types : `~collections.abc.Iterable` [`DatasetType`] or `None`
375 Dataset types to include into returned summaries. If `None` then
376 all dataset types will be included.
378 Returns
379 -------
380 summaries : `~collections.abc.Mapping` [`Any`, `CollectionSummary`]
381 Collection summaries indexed by collection record key. This mapping
382 will also contain all nested non-chained collections of the chained
383 collections.
384 """
385 raise NotImplementedError()
387 @abstractmethod
388 def ingest_date_dtype(self) -> type:
389 """Return type of the ``ingest_date`` column."""
390 raise NotImplementedError()
392 @abstractmethod
393 def insert(
394 self,
395 dataset_type_name: str,
396 run: RunRecord,
397 data_ids: Iterable[DataCoordinate],
398 id_generation_mode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE,
399 ) -> list[DatasetRef]:
400 """Insert one or more dataset entries into the database.
402 Parameters
403 ----------
404 dataset_type_name : `str`
405 Name of the dataset type.
406 run : `RunRecord`
407 The record object describing the `~CollectionType.RUN` collection
408 these datasets will be associated with.
409 data_ids : `~collections.abc.Iterable` [ `DataCoordinate` ]
410 Expanded data IDs (`DataCoordinate` instances) for the
411 datasets to be added. The dimensions of all data IDs must be the
412 same as ``dataset_type.dimensions``.
413 id_generation_mode : `DatasetIdGenEnum`
414 With `UNIQUE` each new dataset is inserted with its new unique ID.
415 With non-`UNIQUE` mode ID is computed from some combination of
416 dataset type, dataId, and run collection name; if the same ID is
417 already in the database then new record is not inserted.
419 Returns
420 -------
421 datasets : `list` [ `DatasetRef` ]
422 References to the inserted datasets.
423 """
424 raise NotImplementedError()
426 @abstractmethod
427 def import_(self, run: RunRecord, refs: list[DatasetRef]) -> None:
428 """Insert one or more dataset entries into the database.
430 Parameters
431 ----------
432 run : `RunRecord`
433 The record object describing the `~CollectionType.RUN` collection
434 these datasets will be associated with.
435 refs : `list` [ `DatasetRef` ]
436 List of datasets to be be inserted. All of the ``DatasetRef``
437 ``run`` attributes must match the ``run`` parameter.
438 """
439 raise NotImplementedError()
441 @abstractmethod
442 def delete(self, datasets: Iterable[DatasetId | DatasetRef]) -> None:
443 """Fully delete the given datasets from the registry.
445 Parameters
446 ----------
447 datasets : `~collections.abc.Iterable` [ `DatasetId` or `DatasetRef` ]
448 Datasets to be deleted. If `DatasetRef` instances are passed,
449 only the `DatasetRef.id` attribute is used.
450 """
451 raise NotImplementedError()
453 @abstractmethod
454 def associate(
455 self, dataset_type: DatasetType, collection: CollectionRecord, datasets: Iterable[DatasetRef]
456 ) -> None:
457 """Associate one or more datasets with a collection.
459 Parameters
460 ----------
461 dataset_type : `DatasetType`
462 Type of all datasets.
463 collection : `CollectionRecord`
464 The record object describing the collection. ``collection.type``
465 must be `~CollectionType.TAGGED`.
466 datasets : `~collections.abc.Iterable` [ `DatasetRef` ]
467 Datasets to be associated. All datasets must have the same
468 `DatasetType` as ``dataset_type``, but this is not checked.
470 Notes
471 -----
472 Associating a dataset into collection that already contains a
473 different dataset with the same `DatasetType` and data ID will remove
474 the existing dataset from that collection.
476 Associating the same dataset into a collection multiple times is a
477 no-op, but is still not permitted on read-only databases.
478 """
479 raise NotImplementedError()
481 @abstractmethod
482 def disassociate(
483 self, dataset_type: DatasetType, collection: CollectionRecord, datasets: Iterable[DatasetRef]
484 ) -> None:
485 """Remove one or more datasets from a collection.
487 Parameters
488 ----------
489 dataset_type : `DatasetType`
490 Type of all datasets.
491 collection : `CollectionRecord`
492 The record object describing the collection. ``collection.type``
493 must be `~CollectionType.TAGGED`.
494 datasets : `~collections.abc.Iterable` [ `DatasetRef` ]
495 Datasets to be disassociated. All datasets must have the same
496 `DatasetType` as ``dataset_type``, but this is not checked.
497 """
498 raise NotImplementedError()
500 @abstractmethod
501 def certify(
502 self,
503 dataset_type: DatasetType,
504 collection: CollectionRecord,
505 datasets: Iterable[DatasetRef],
506 timespan: Timespan,
507 context: SqlQueryContext,
508 ) -> None:
509 """Associate one or more datasets with a calibration collection and a
510 validity range within it.
512 Parameters
513 ----------
514 dataset_type : `DatasetType`
515 Type of all datasets.
516 collection : `CollectionRecord`
517 The record object describing the collection. ``collection.type``
518 must be `~CollectionType.CALIBRATION`.
519 datasets : `~collections.abc.Iterable` [ `DatasetRef` ]
520 Datasets to be associated. All datasets must have the same
521 `DatasetType` as ``dataset_type``, but this is not checked.
522 timespan : `Timespan`
523 The validity range for these datasets within the collection.
524 context : `SqlQueryContext`
525 The object that manages database connections, temporary tables and
526 relation engines for this query.
528 Raises
529 ------
530 ConflictingDefinitionError
531 Raised if the collection already contains a different dataset with
532 the same `DatasetType` and data ID and an overlapping validity
533 range.
534 DatasetTypeError
535 Raised if ``dataset_type.isCalibration() is False``.
536 CollectionTypeError
537 Raised if
538 ``collection.type is not CollectionType.CALIBRATION``.
539 """
540 raise NotImplementedError()
542 @abstractmethod
543 def decertify(
544 self,
545 dataset_type: DatasetType,
546 collection: CollectionRecord,
547 timespan: Timespan,
548 *,
549 data_ids: Iterable[DataCoordinate] | None = None,
550 context: SqlQueryContext,
551 ) -> None:
552 """Remove or adjust datasets to clear a validity range within a
553 calibration collection.
555 Parameters
556 ----------
557 dataset_type : `DatasetType`
558 Type of all datasets.
559 collection : `CollectionRecord`
560 The record object describing the collection. ``collection.type``
561 must be `~CollectionType.CALIBRATION`.
562 timespan : `Timespan`
563 The validity range to remove datasets from within the collection.
564 Datasets that overlap this range but are not contained by it will
565 have their validity ranges adjusted to not overlap it, which may
566 split a single dataset validity range into two.
567 data_ids : `~collections.abc.Iterable` [ `DataCoordinate` ], optional
568 Data IDs that should be decertified within the given validity range
569 If `None`, all data IDs for ``dataset_type`` in ``collection`` will
570 be decertified.
571 context : `SqlQueryContext`
572 The object that manages database connections, temporary tables and
573 relation engines for this query.
575 Raises
576 ------
577 DatasetTypeError
578 Raised if ``dataset_type.isCalibration() is False``.
579 CollectionTypeError
580 Raised if
581 ``collection.type is not CollectionType.CALIBRATION``.
582 """
583 raise NotImplementedError()
585 @abstractmethod
586 def make_relation(
587 self,
588 dataset_type: DatasetType,
589 *collections: CollectionRecord,
590 columns: Set[str],
591 context: SqlQueryContext,
592 ) -> Relation:
593 """Return a `sql.Relation` that represents a query for this
594 `DatasetType` in one or more collections.
596 Parameters
597 ----------
598 dataset_type : `DatasetType`
599 Type of dataset to query for.
600 *collections : `CollectionRecord`
601 The record object(s) describing the collection(s) to query. May
602 not be of type `CollectionType.CHAINED`. If multiple collections
603 are passed, the query will search all of them in an unspecified
604 order, and all collections must have the same type. Must include
605 at least one collection.
606 columns : `~collections.abc.Set` [ `str` ]
607 Columns to include in the relation. See `Query.find_datasets` for
608 most options, but this method supports one more:
610 - ``rank``: a calculated integer column holding the index of the
611 collection the dataset was found in, within the ``collections``
612 sequence given.
613 context : `SqlQueryContext`
614 The object that manages database connections, temporary tables and
615 relation engines for this query.
617 Returns
618 -------
619 relation : `~lsst.daf.relation.Relation`
620 Representation of the query.
621 """
622 raise NotImplementedError()
624 @abstractmethod
625 def make_joins_builder(
626 self,
627 dataset_type: DatasetType,
628 collections: Sequence[CollectionRecord],
629 fields: Set[str],
630 is_union: bool = False,
631 ) -> SqlJoinsBuilder:
632 """Make a `..direct_query_driver.SqlJoinsBuilder` that represents a
633 search for datasets of this type.
635 Parameters
636 ----------
637 dataset_type : `DatasetType`
638 Type of dataset to query for.
639 collections : `~collections.abc.Sequence` [ `CollectionRecord` ]
640 Collections to search, in order, after filtering out collections
641 with no datasets of this type via collection summaries.
642 fields : `~collections.abc.Set` [ `str` ]
643 Names of fields to make available in the builder. Options include:
645 - ``dataset_id`` (UUID)
646 - ``run`` (collection name, `str`)
647 - ``collection`` (collection name, `str`)
648 - ``collection_key`` (collection primary key, manager-dependent)
649 - ``timespan`` (validity range, or unbounded for non-calibrations)
650 - ``ingest_date`` (time dataset was ingested into repository)
652 Dimension keys for the dataset type's required dimensions are
653 always included.
654 is_union : `bool`, optional
655 If `True`, this search is being joined in as part of one term in
656 a union over all dataset types. This causes fields to be added to
657 the builder via the special ``...`` instad of the dataset type
658 name.
660 Returns
661 -------
662 builder : `..direct_query_driver.SqlJoinsBuilder`
663 A query-construction object representing a table or subquery.
664 """
665 raise NotImplementedError()
667 @abstractmethod
668 def refresh_collection_summaries(self, dataset_type: DatasetType) -> None:
669 """Make sure that collection summaries for this dataset type are
670 consistent with the contents of the dataset tables.
672 Parameters
673 ----------
674 dataset_type : `DatasetType`
675 Dataset type whose summary entries should be refreshed.
676 """
677 raise NotImplementedError()