Coverage for python/lsst/daf/butler/transfers/_interfaces.py : 77%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ["RepoExportBackend", "RepoImportBackend", "RepoTransferFormatConfig"]
26from abc import ABC, abstractmethod
27from typing import (
28 Iterable,
29 Optional,
30 Set,
31)
33from ..core import (
34 ConfigSubset,
35 DatasetAssociation,
36 DatasetType,
37 Datastore,
38 DimensionElement,
39 DimensionRecord,
40 FileDataset,
41)
43from ..registry import CollectionType
44from ..registry.interfaces import CollectionRecord
47class RepoTransferFormatConfig(ConfigSubset):
48 """The section of butler configuration that associates repo import/export
49 backends with file formats.
50 """
51 component = "repo_transfer_formats"
52 defaultConfigFile = "repo_transfer_formats.yaml"
55class RepoExportBackend(ABC):
56 """An abstract interface for data repository export implementations.
58 Methods are guaranteed to be called in ways that reflect foreign key
59 dependencies.
60 """
62 @abstractmethod
63 def saveDimensionData(self, element: DimensionElement, *data: DimensionRecord) -> None:
64 """Export one or more dimension element records.
66 Parameters
67 ----------
68 element : `DimensionElement`
69 The `DimensionElement` whose elements are being exported.
70 data : `DimensionRecord` (variadic)
71 One or more records to export.
72 """
73 raise NotImplementedError()
75 @abstractmethod
76 def saveCollection(self, record: CollectionRecord, doc: Optional[str]) -> None:
77 """Export a collection.
79 This only exports the collection's own state, not its associations with
80 datasets.
82 Parameters
83 ----------
84 record: `CollectionRecord`
85 Object representing the collection to export.
86 doc : `str` or `None`
87 Documentation string for the collection.
88 """
89 raise NotImplementedError()
91 @abstractmethod
92 def saveDatasets(self, datasetType: DatasetType, run: str, *datasets: FileDataset) -> None:
93 """Export one or more datasets, including their associated DatasetType
94 and run information (but not including associated dimension
95 information).
97 Parameters
98 ----------
99 datasetType : `DatasetType`
100 Type of all datasets being exported with this call.
101 run : `str`
102 Run associated with all datasets being exported with this call.
103 datasets : `FileDataset`, variadic
104 Per-dataset information to be exported. `FileDataset.formatter`
105 attributes should be strings, not `Formatter` instances or classes.
106 """
107 raise NotImplementedError()
109 @abstractmethod
110 def saveDatasetAssociations(self, collection: str, collectionType: CollectionType,
111 associations: Iterable[DatasetAssociation]) -> None:
112 """Export the dataset-collection associations for a single collection.
114 Parameters
115 ----------
116 collection : `str`
117 The name of the collection.
118 collectionType : `CollectionType`
119 The type of the collection; either `CollectionType.TAGGED` or
120 `CollectionType.CALIBRATION` (as other collection types are
121 exported in other ways).
122 associations : `Iterable` [ `DatasetAssociation` ]
123 Structs representing an association between this collection and
124 this dataset.
125 """
126 raise NotImplementedError()
128 @abstractmethod
129 def finish(self) -> None:
130 """Complete the export process.
131 """
132 raise NotImplementedError()
135class RepoImportBackend(ABC):
136 """An abstract interface for data repository import implementations.
138 Import backends are expected to be constructed with a description of
139 the objects that need to be imported (from, e.g., a file written by the
140 corresponding export backend), along with a `Registry`.
141 """
143 @abstractmethod
144 def register(self) -> None:
145 """Register all runs and dataset types associated with the backend with
146 the `Registry` the backend was constructed with.
148 These operations cannot be performed inside transactions, unlike those
149 performed by `load`, and must in general be performed before `load`.
150 """
152 @abstractmethod
153 def load(self, datastore: Optional[Datastore], *,
154 directory: Optional[str] = None, transfer: Optional[str] = None,
155 skip_dimensions: Optional[Set] = None) -> None:
156 """Import information associated with the backend into the given
157 registry and datastore.
159 This must be run after `register`, and may be performed inside a
160 transaction.
162 Parameters
163 ----------
164 datastore : `Datastore`
165 Datastore to import into. If `None`, datasets will only be
166 inserted into the `Registry` (primarily intended for tests).
167 directory : `str`, optional
168 File all dataset paths are relative to.
169 transfer : `str`, optional
170 Transfer mode forwarded to `Datastore.ingest`.
171 skip_dimensions : `set`, optional
172 Dimensions that should be skipped and not imported. This can
173 be useful when importing into a registry that already knows
174 about a specific instrument.
175 """
176 raise NotImplementedError()