Coverage for python/lsst/ap/verify/dataset.py : 27%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import os
26import lsst.daf.persistence as dafPersistence
27import lsst.daf.butler as dafButler
28import lsst.obs.base as obsBase
29import lsst.pex.exceptions as pexExcept
30from lsst.utils import getPackageDir
32from .config import Config
35class Dataset:
36 """A dataset supported by ``ap_verify``.
38 Any object of this class is guaranteed to represent a ready-for-use
39 ap_verify dataset, barring concurrent changes to the file system or EUPS
40 operations. Constructing a Dataset does not create a compatible output
41 repository(ies), which can be done by calling `makeCompatibleRepo`.
43 Parameters
44 ----------
45 datasetId : `str`
46 A tag identifying the dataset.
48 Raises
49 ------
50 RuntimeError
51 Raised if `datasetId` exists, but is not correctly organized or incomplete
52 ValueError
53 Raised if `datasetId` is not a recognized ap_verify dataset. No side
54 effects if this exception is raised.
55 """
57 def __init__(self, datasetId):
58 # daf.persistence.Policy's behavior on missing keys is apparently undefined
59 # test for __getattr__ *either* raising KeyError or returning None
60 try:
61 datasetPackage = self._getDatasetInfo()[datasetId]
62 if datasetPackage is None:
63 raise KeyError
64 except KeyError:
65 raise ValueError('Unsupported dataset: ' + datasetId)
67 try:
68 self._dataRootDir = getPackageDir(datasetPackage)
69 except pexExcept.NotFoundError as e:
70 error = 'Dataset %s requires the %s package, which has not been set up.' \
71 % (datasetId, datasetPackage)
72 raise RuntimeError(error) from e
73 else:
74 self._validatePackage()
76 self._initPackage(datasetPackage)
78 def _initPackage(self, name):
79 """Prepare the package backing this ap_verify dataset.
81 Parameters
82 ----------
83 name : `str`
84 The EUPS package identifier for the desired package.
85 """
86 # No initialization required at present
87 pass
89 @staticmethod
90 def getSupportedDatasets():
91 """The ap_verify dataset IDs that can be passed to this class's constructor.
93 Returns
94 -------
95 datasets : `set` of `str`
96 the set of IDs that will be accepted
98 Raises
99 ------
100 IoError
101 Raised if the config file does not exist or is not readable
102 RuntimeError
103 Raised if the config file exists, but does not contain the expected data
104 """
105 return Dataset._getDatasetInfo().keys()
107 @staticmethod
108 def _getDatasetInfo():
109 """Return external data on supported ap_verify datasets.
111 If an exception is raised, the program state shall be unchanged.
113 Returns
114 -------
115 datasetToPackage : `dict`-like
116 a map from dataset IDs to package names.
118 Raises
119 ------
120 RuntimeError
121 Raised if the config file exists, but does not contain the expected data
122 """
123 return Config.instance['datasets']
125 @property
126 def datasetRoot(self):
127 """The parent directory containing everything related to the
128 ap_verify dataset (`str`, read-only).
129 """
130 return self._dataRootDir
132 @property
133 def rawLocation(self):
134 """The directory containing the "raw" input data (`str`, read-only).
135 """
136 return os.path.join(self.datasetRoot, 'raw')
138 @property
139 def calibLocation(self):
140 """The directory containing the calibration data (`str`, read-only).
141 """
142 return os.path.join(self.datasetRoot, 'calib')
144 @property
145 def refcatsLocation(self):
146 """The directory containing external astrometric and photometric
147 reference catalogs (`str`, read-only).
148 """
149 return os.path.join(self.datasetRoot, 'refcats')
151 @property
152 def templateLocation(self):
153 """The directory containing the image subtraction templates (`str`, read-only).
154 """
155 return os.path.join(self.datasetRoot, 'templates')
157 @property
158 def configLocation(self):
159 """The directory containing configs that can be used to process the data (`str`, read-only).
160 """
161 return os.path.join(self.datasetRoot, 'config')
163 @property
164 def obsPackage(self):
165 """The name of the obs package associated with this data (`str`, read-only).
166 """
167 return dafPersistence.Butler.getMapperClass(self._stubInputRepo).getPackageName()
169 @property
170 def camera(self):
171 """The name of the Gen 2 camera associated with this data (`str`, read-only).
172 """
173 return dafPersistence.Butler.getMapperClass(self._stubInputRepo).getCameraName()
175 @property
176 def instrument(self):
177 """The Gen 3 instrument associated with this data (`lsst.obs.base.Instrument`, read-only).
178 """
179 butler = dafButler.Butler(self._preloadedRepo, writeable=False)
180 instruments = list(butler.registry.queryDataIds('instrument'))
181 if len(instruments) != 1:
182 raise RuntimeError(f"Dataset does not have exactly one instrument; got {instruments}.")
183 else:
184 return obsBase.Instrument.fromName(instruments[0]["instrument"], butler.registry)
186 @property
187 def _stubInputRepo(self):
188 """The directory containing the data set's input stub (`str`, read-only).
189 """
190 return os.path.join(self.datasetRoot, 'repo')
192 @property
193 def _preloadedRepo(self):
194 """The directory containing the pre-ingested Gen 3 repo (`str`, read-only).
195 """
196 return os.path.join(self.datasetRoot, 'preloaded')
198 @property
199 def _preloadedExport(self):
200 """The file containing an exported registry of `_preloadedRepo` (`str`, read-only).
201 """
202 return os.path.join(self.configLocation, 'export.yaml')
204 def _validatePackage(self):
205 """Confirm that the dataset directory satisfies all assumptions.
207 Raises
208 ------
209 RuntimeError
210 Raised if the package represented by this object does not conform to the
211 dataset framework
213 Notes
214 -----
215 Requires that `self._dataRootDir` has been initialized.
216 """
217 if not os.path.exists(self.datasetRoot):
218 raise RuntimeError('Could not find dataset at ' + self.datasetRoot)
219 if not os.path.exists(self.rawLocation):
220 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing data directory')
221 if not os.path.exists(self.calibLocation):
222 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing calibration directory')
223 # Template and refcat directories might not be subdirectories of self.datasetRoot
224 if not os.path.exists(self.templateLocation):
225 raise RuntimeError('Dataset is missing template directory at ' + self.templateLocation)
226 if not os.path.exists(self.refcatsLocation):
227 raise RuntimeError('Dataset is missing reference catalog directory at ' + self.refcatsLocation)
228 if not os.path.exists(self._stubInputRepo):
229 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing stub repo')
230 if not _isRepo(self._stubInputRepo):
231 raise RuntimeError('Stub repo at ' + self._stubInputRepo + 'is missing mapper file')
233 def makeCompatibleRepo(self, repoDir, calibRepoDir):
234 """Set up a directory as a Gen 2 repository compatible with this ap_verify dataset.
236 If the directory already exists, any files required by the dataset will
237 be added if absent; otherwise the directory will remain unchanged.
239 Parameters
240 ----------
241 repoDir : `str`
242 The directory where the output repository will be created.
243 calibRepoDir : `str`
244 The directory where the output calibration repository will be created.
245 """
246 mapperArgs = {'mapperArgs': {'calibRoot': calibRepoDir}}
247 if _isRepo(self.templateLocation):
248 # Stub repo is not a parent because can't mix v1 and v2 repositories in parents list
249 dafPersistence.Butler(inputs=[{"root": self.templateLocation, "mode": "r"}],
250 outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}])
251 else:
252 dafPersistence.Butler(inputs=[{"root": self._stubInputRepo, "mode": "r"}],
253 outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}])
255 def makeCompatibleRepoGen3(self, repoDir):
256 """Set up a directory as a Gen 3 repository compatible with this ap_verify dataset.
258 If the repository already exists, this call has no effect.
260 Parameters
261 ----------
262 repoDir : `str`
263 The directory where the output repository will be created.
264 """
265 # No way to tell makeRepo "create only what's missing"
266 try:
267 repoConfig = dafButler.Butler.makeRepo(repoDir)
268 butler = dafButler.Butler(repoConfig, writeable=True)
269 butler.import_(directory=self._preloadedRepo, filename=self._preloadedExport,
270 transfer="auto")
271 except FileExistsError:
272 pass
275def _isRepo(repoDir):
276 """Test whether a directory has been set up as a repository.
277 """
278 return os.path.exists(os.path.join(repoDir, '_mapper')) \
279 or os.path.exists(os.path.join(repoDir, 'repositoryCfg.yaml'))