Coverage for python/lsst/daf/butler/transfers/_interfaces.py: 81%
32 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:54 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:54 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["RepoExportBackend", "RepoImportBackend", "RepoTransferFormatConfig"]
26from abc import ABC, abstractmethod
27from typing import (
28 Iterable,
29 Optional,
30 Set,
31)
33from ..core import (
34 ConfigSubset,
35 DatasetAssociation,
36 DatasetType,
37 Datastore,
38 DimensionElement,
39 DimensionRecord,
40 FileDataset,
41)
43from ..registry import CollectionType
44from ..registry.interfaces import CollectionRecord, DatasetIdGenEnum
47class RepoTransferFormatConfig(ConfigSubset):
48 """The section of butler configuration that associates repo import/export
49 backends with file formats.
50 """
51 component = "repo_transfer_formats"
52 defaultConfigFile = "repo_transfer_formats.yaml"
55class RepoExportBackend(ABC):
56 """An abstract interface for data repository export implementations.
58 Methods are guaranteed to be called in ways that reflect foreign key
59 dependencies.
60 """
62 @abstractmethod
63 def saveDimensionData(self, element: DimensionElement, *data: DimensionRecord) -> None:
64 """Export one or more dimension element records.
66 Parameters
67 ----------
68 element : `DimensionElement`
69 The `DimensionElement` whose elements are being exported.
70 data : `DimensionRecord` (variadic)
71 One or more records to export.
72 """
73 raise NotImplementedError()
75 @abstractmethod
76 def saveCollection(self, record: CollectionRecord, doc: Optional[str]) -> None:
77 """Export a collection.
79 This only exports the collection's own state, not its associations with
80 datasets.
82 Parameters
83 ----------
84 record: `CollectionRecord`
85 Object representing the collection to export.
86 doc : `str` or `None`
87 Documentation string for the collection.
88 """
89 raise NotImplementedError()
91 @abstractmethod
92 def saveDatasets(self, datasetType: DatasetType, run: str, *datasets: FileDataset) -> None:
93 """Export one or more datasets, including their associated DatasetType
94 and run information (but not including associated dimension
95 information).
97 Parameters
98 ----------
99 datasetType : `DatasetType`
100 Type of all datasets being exported with this call.
101 run : `str`
102 Run associated with all datasets being exported with this call.
103 datasets : `FileDataset`, variadic
104 Per-dataset information to be exported. `FileDataset.formatter`
105 attributes should be strings, not `Formatter` instances or classes.
106 """
107 raise NotImplementedError()
109 @abstractmethod
110 def saveDatasetAssociations(self, collection: str, collectionType: CollectionType,
111 associations: Iterable[DatasetAssociation]) -> None:
112 """Export the dataset-collection associations for a single collection.
114 Parameters
115 ----------
116 collection : `str`
117 The name of the collection.
118 collectionType : `CollectionType`
119 The type of the collection; either `CollectionType.TAGGED` or
120 `CollectionType.CALIBRATION` (as other collection types are
121 exported in other ways).
122 associations : `Iterable` [ `DatasetAssociation` ]
123 Structs representing an association between this collection and
124 this dataset.
125 """
126 raise NotImplementedError()
128 @abstractmethod
129 def finish(self) -> None:
130 """Complete the export process.
131 """
132 raise NotImplementedError()
135class RepoImportBackend(ABC):
136 """An abstract interface for data repository import implementations.
138 Import backends are expected to be constructed with a description of
139 the objects that need to be imported (from, e.g., a file written by the
140 corresponding export backend), along with a `Registry`.
141 """
143 @abstractmethod
144 def register(self) -> None:
145 """Register all runs and dataset types associated with the backend with
146 the `Registry` the backend was constructed with.
148 These operations cannot be performed inside transactions, unlike those
149 performed by `load`, and must in general be performed before `load`.
150 """
152 @abstractmethod
153 def load(self, datastore: Optional[Datastore], *,
154 directory: Optional[str] = None, transfer: Optional[str] = None,
155 skip_dimensions: Optional[Set] = None,
156 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE,
157 reuseIds: bool = False) -> None:
158 """Import information associated with the backend into the given
159 registry and datastore.
161 This must be run after `register`, and may be performed inside a
162 transaction.
164 Parameters
165 ----------
166 datastore : `Datastore`
167 Datastore to import into. If `None`, datasets will only be
168 inserted into the `Registry` (primarily intended for tests).
169 directory : `str`, optional
170 File all dataset paths are relative to.
171 transfer : `str`, optional
172 Transfer mode forwarded to `Datastore.ingest`.
173 skip_dimensions : `set`, optional
174 Dimensions that should be skipped and not imported. This can
175 be useful when importing into a registry that already knows
176 about a specific instrument.
177 idGenerationMode : `DatasetIdGenEnum`, optional
178 Specifies option for generating dataset IDs when IDs are not
179 provided or their type does not match backend type. By default
180 unique IDs are generated for each inserted dataset.
181 reuseIds : `bool`, optional
182 If `True` then forces re-use of imported dataset IDs for integer
183 IDs which are normally generated as auto-incremented. This option
184 has no effect on the use of globally-unique IDs which are always
185 re-used (or generated if integer IDs are being imported).
186 """
187 raise NotImplementedError()