Coverage for python/lsst/daf/butler/transfers/_interfaces.py: 81%
32 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-24 23:49 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-24 23:49 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["RepoExportBackend", "RepoImportBackend", "RepoTransferFormatConfig"]
26from abc import ABC, abstractmethod
27from typing import Iterable, Optional, Set
29from ..core import (
30 ConfigSubset,
31 DatasetAssociation,
32 DatasetType,
33 Datastore,
34 DimensionElement,
35 DimensionRecord,
36 FileDataset,
37)
38from ..registry import CollectionType
39from ..registry.interfaces import CollectionRecord, DatasetIdGenEnum
42class RepoTransferFormatConfig(ConfigSubset):
43 """The section of butler configuration that associates repo import/export
44 backends with file formats.
45 """
47 component = "repo_transfer_formats"
48 defaultConfigFile = "repo_transfer_formats.yaml"
51class RepoExportBackend(ABC):
52 """An abstract interface for data repository export implementations.
54 Methods are guaranteed to be called in ways that reflect foreign key
55 dependencies.
56 """
58 @abstractmethod
59 def saveDimensionData(self, element: DimensionElement, *data: DimensionRecord) -> None:
60 """Export one or more dimension element records.
62 Parameters
63 ----------
64 element : `DimensionElement`
65 The `DimensionElement` whose elements are being exported.
66 data : `DimensionRecord` (variadic)
67 One or more records to export.
68 """
69 raise NotImplementedError()
71 @abstractmethod
72 def saveCollection(self, record: CollectionRecord, doc: Optional[str]) -> None:
73 """Export a collection.
75 This only exports the collection's own state, not its associations with
76 datasets.
78 Parameters
79 ----------
80 record: `CollectionRecord`
81 Object representing the collection to export.
82 doc : `str` or `None`
83 Documentation string for the collection.
84 """
85 raise NotImplementedError()
87 @abstractmethod
88 def saveDatasets(self, datasetType: DatasetType, run: str, *datasets: FileDataset) -> None:
89 """Export one or more datasets, including their associated DatasetType
90 and run information (but not including associated dimension
91 information).
93 Parameters
94 ----------
95 datasetType : `DatasetType`
96 Type of all datasets being exported with this call.
97 run : `str`
98 Run associated with all datasets being exported with this call.
99 datasets : `FileDataset`, variadic
100 Per-dataset information to be exported. `FileDataset.formatter`
101 attributes should be strings, not `Formatter` instances or classes.
102 """
103 raise NotImplementedError()
105 @abstractmethod
106 def saveDatasetAssociations(
107 self, collection: str, collectionType: CollectionType, associations: Iterable[DatasetAssociation]
108 ) -> None:
109 """Export the dataset-collection associations for a single collection.
111 Parameters
112 ----------
113 collection : `str`
114 The name of the collection.
115 collectionType : `CollectionType`
116 The type of the collection; either `CollectionType.TAGGED` or
117 `CollectionType.CALIBRATION` (as other collection types are
118 exported in other ways).
119 associations : `Iterable` [ `DatasetAssociation` ]
120 Structs representing an association between this collection and
121 this dataset.
122 """
123 raise NotImplementedError()
125 @abstractmethod
126 def finish(self) -> None:
127 """Complete the export process."""
128 raise NotImplementedError()
131class RepoImportBackend(ABC):
132 """An abstract interface for data repository import implementations.
134 Import backends are expected to be constructed with a description of
135 the objects that need to be imported (from, e.g., a file written by the
136 corresponding export backend), along with a `Registry`.
137 """
139 @abstractmethod
140 def register(self) -> None:
141 """Register all runs and dataset types associated with the backend with
142 the `Registry` the backend was constructed with.
144 These operations cannot be performed inside transactions, unlike those
145 performed by `load`, and must in general be performed before `load`.
146 """
148 @abstractmethod
149 def load(
150 self,
151 datastore: Optional[Datastore],
152 *,
153 directory: Optional[str] = None,
154 transfer: Optional[str] = None,
155 skip_dimensions: Optional[Set] = None,
156 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE,
157 reuseIds: bool = False,
158 ) -> None:
159 """Import information associated with the backend into the given
160 registry and datastore.
162 This must be run after `register`, and may be performed inside a
163 transaction.
165 Parameters
166 ----------
167 datastore : `Datastore`
168 Datastore to import into. If `None`, datasets will only be
169 inserted into the `Registry` (primarily intended for tests).
170 directory : `str`, optional
171 File all dataset paths are relative to.
172 transfer : `str`, optional
173 Transfer mode forwarded to `Datastore.ingest`.
174 skip_dimensions : `set`, optional
175 Dimensions that should be skipped and not imported. This can
176 be useful when importing into a registry that already knows
177 about a specific instrument.
178 idGenerationMode : `DatasetIdGenEnum`, optional
179 Specifies option for generating dataset IDs when IDs are not
180 provided or their type does not match backend type. By default
181 unique IDs are generated for each inserted dataset.
182 reuseIds : `bool`, optional
183 If `True` then forces re-use of imported dataset IDs for integer
184 IDs which are normally generated as auto-incremented. This option
185 has no effect on the use of globally-unique IDs which are always
186 re-used (or generated if integer IDs are being imported).
187 """
188 raise NotImplementedError()