Coverage for python/lsst/ap/verify/dataset.py : 26%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import os
26from lsst.daf.persistence import Butler
27import lsst.pex.exceptions as pexExcept
28from lsst.utils import getPackageDir
30from .config import Config
33class Dataset:
34 """A dataset supported by ``ap_verify``.
36 Any object of this class is guaranteed to represent a ready-for-use
37 dataset, barring concurrent changes to the file system or EUPS operations.
38 Constructing a Dataset does not create a compatible output repository(ies),
39 which can be done by calling `makeCompatibleRepo`.
41 Parameters
42 ----------
43 datasetId : `str`
44 A tag identifying the dataset.
46 Raises
47 ------
48 RuntimeError
49 Raised if `datasetId` exists, but is not correctly organized or incomplete
50 ValueError
51 Raised if `datasetId` is not a recognized dataset. No side effects if this
52 exception is raised.
53 """
55 def __init__(self, datasetId):
56 # daf.persistence.Policy's behavior on missing keys is apparently undefined
57 # test for __getattr__ *either* raising KeyError or returning None
58 try:
59 datasetPackage = self._getDatasetInfo()[datasetId]
60 if datasetPackage is None:
61 raise KeyError
62 except KeyError:
63 raise ValueError('Unsupported dataset: ' + datasetId)
65 try:
66 self._dataRootDir = getPackageDir(datasetPackage)
67 except pexExcept.NotFoundError as e:
68 error = 'Dataset %s requires the %s package, which has not been set up.' \
69 % (datasetId, datasetPackage)
70 raise RuntimeError(error) from e
71 else:
72 self._validatePackage()
74 self._initPackage(datasetPackage)
76 def _initPackage(self, name):
77 """Prepare the package backing this dataset.
79 Parameters
80 ----------
81 name : `str`
82 The EUPS package identifier for the desired package.
83 """
84 # No initialization required at present
85 pass
87 @staticmethod
88 def getSupportedDatasets():
89 """The dataset IDs that can be passed to this class's constructor.
91 Returns
92 -------
93 datasets : `set` of `str`
94 the set of IDs that will be accepted
96 Raises
97 ------
98 IoError
99 Raised if the config file does not exist or is not readable
100 RuntimeError
101 Raised if the config file exists, but does not contain the expected data
102 """
103 return Dataset._getDatasetInfo().keys()
105 @staticmethod
106 def _getDatasetInfo():
107 """Return external data on supported datasets.
109 If an exception is raised, the program state shall be unchanged.
111 Returns
112 -------
113 datasetToPackage : `dict`-like
114 a map from dataset IDs to package names.
116 Raises
117 ------
118 RuntimeError
119 Raised if the config file exists, but does not contain the expected data
120 """
121 return Config.instance['datasets']
123 @property
124 def datasetRoot(self):
125 """The parent directory containing everything related to the dataset (`str`, read-only).
126 """
127 return self._dataRootDir
129 @property
130 def rawLocation(self):
131 """The directory containing the "raw" input data (`str`, read-only).
132 """
133 return os.path.join(self.datasetRoot, 'raw')
135 @property
136 def calibLocation(self):
137 """The directory containing the calibration data (`str`, read-only).
138 """
139 return os.path.join(self.datasetRoot, 'calib')
141 @property
142 def refcatsLocation(self):
143 """The directory containing external astrometric and photometric
144 reference catalogs (`str`, read-only).
145 """
146 return os.path.join(self.datasetRoot, 'refcats')
148 @property
149 def templateLocation(self):
150 """The directory containing the image subtraction templates (`str`, read-only).
151 """
152 return os.path.join(self.datasetRoot, 'templates')
154 @property
155 def configLocation(self):
156 """The directory containing configs that can be used to process the dataset (`str`, read-only).
157 """
158 return os.path.join(self.datasetRoot, 'config')
160 @property
161 def obsPackage(self):
162 """The name of the obs package associated with this dataset (`str`, read-only).
163 """
164 return Butler.getMapperClass(self._stubInputRepo).getPackageName()
166 @property
167 def camera(self):
168 """The name of the camera associated with this dataset (`str`, read-only).
169 """
170 return Butler.getMapperClass(self._stubInputRepo).getCameraName()
172 @property
173 def _stubInputRepo(self):
174 """The directory containing the data set's input stub (`str`, read-only).
175 """
176 return os.path.join(self.datasetRoot, 'repo')
178 def _validatePackage(self):
179 """Confirm that the dataset directory satisfies all assumptions.
181 Raises
182 ------
183 RuntimeError
184 Raised if the package represented by this object does not conform to the
185 dataset framework
187 Notes
188 -----
189 Requires that `self._dataRootDir` has been initialized.
190 """
191 if not os.path.exists(self.datasetRoot):
192 raise RuntimeError('Could not find dataset at ' + self.datasetRoot)
193 if not os.path.exists(self.rawLocation):
194 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing data directory')
195 if not os.path.exists(self.calibLocation):
196 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing calibration directory')
197 # Template and refcat directories might not be subdirectories of self.datasetRoot
198 if not os.path.exists(self.templateLocation):
199 raise RuntimeError('Dataset is missing template directory at ' + self.templateLocation)
200 if not os.path.exists(self.refcatsLocation):
201 raise RuntimeError('Dataset is missing reference catalog directory at ' + self.refcatsLocation)
202 if not os.path.exists(self._stubInputRepo):
203 raise RuntimeError('Dataset at ' + self.datasetRoot + 'is missing stub repo')
204 if not _isRepo(self._stubInputRepo):
205 raise RuntimeError('Stub repo at ' + self._stubInputRepo + 'is missing mapper file')
207 def makeCompatibleRepo(self, repoDir, calibRepoDir):
208 """Set up a directory as a repository compatible with this dataset.
210 If the directory already exists, any files required by the dataset will
211 be added if absent; otherwise the directory will remain unchanged.
213 Parameters
214 ----------
215 repoDir : `str`
216 The directory where the output repository will be created.
217 calibRepoDir : `str`
218 The directory where the output calibration repository will be created.
219 """
220 mapperArgs = {'mapperArgs': {'calibRoot': calibRepoDir}}
221 if _isRepo(self.templateLocation):
222 # Stub repo is not a parent because can't mix v1 and v2 repositories in parents list
223 Butler(inputs=[{"root": self.templateLocation, "mode": "r"}],
224 outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}])
225 else:
226 Butler(inputs=[{"root": self._stubInputRepo, "mode": "r"}],
227 outputs=[{"root": repoDir, "mode": "rw", **mapperArgs}])
230def _isRepo(repoDir):
231 """Test whether a directory has been set up as a repository.
232 """
233 return os.path.exists(os.path.join(repoDir, '_mapper')) \
234 or os.path.exists(os.path.join(repoDir, 'repositoryCfg.yaml'))