Coverage for tests/test_datastoreFits.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import unittest
24import tempfile
25import shutil
27import lsst.utils.tests
28from lsst.utils import doImport
30from lsst.daf.butler import StorageClassFactory
31from lsst.daf.butler import DatastoreConfig
33from lsst.daf.butler.tests import (FitsCatalogDatasetsHelper, DatasetTestHelper, DatastoreTestHelper,
34 DummyRegistry)
36try:
37 import lsst.afw.table
38 import lsst.afw.image
39 import lsst.geom
40except ImportError:
41 lsst.afw = None
43TESTDIR = os.path.dirname(__file__)
46class DatastoreFitsTests(FitsCatalogDatasetsHelper, DatasetTestHelper, DatastoreTestHelper):
47 root = None
49 @classmethod
50 def setUpClass(cls):
51 if lsst.afw is None:
52 raise unittest.SkipTest("afw not available.")
54 # Base classes need to know where the test directory is
55 cls.testDir = TESTDIR
57 # Storage Classes are fixed for all datastores in these tests
58 scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml")
59 cls.storageClassFactory = StorageClassFactory()
60 cls.storageClassFactory.addFromConfig(scConfigFile)
62 # Read the Datastore config so we can get the class
63 # information (since we should not assume the constructor
64 # name here, but rely on the configuration file itself)
65 datastoreConfig = DatastoreConfig(cls.configFile)
66 cls.datastoreType = doImport(datastoreConfig["cls"])
68 def setUp(self):
69 self.setUpDatastoreTests(DummyRegistry, DatastoreConfig)
71 def tearDown(self):
72 if self.root is not None and os.path.exists(self.root):
73 shutil.rmtree(self.root, ignore_errors=True)
75 def testConstructor(self):
76 datastore = self.makeDatastore()
77 self.assertIsNotNone(datastore)
79 def testBasicPutGet(self):
80 catalog = self.makeExampleCatalog()
81 datastore = self.makeDatastore()
83 # Put
84 dimensions = self.registry.dimensions.extract(("visit", "physical_filter"))
85 dataId = {"visit": 123456, "physical_filter": "blue", "instrument": "dummy"}
86 storageClass = self.storageClassFactory.getStorageClass("SourceCatalog")
88 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId)
90 datastore.put(catalog, ref)
92 # Does it exist?
93 self.assertTrue(datastore.exists(ref))
95 uri = datastore.getUri(ref)
96 if self.fileExt is not None:
97 self.assertTrue(uri.endswith(self.fileExt))
98 self.assertTrue(uri.startswith(self.uriScheme))
100 # Get
101 catalogOut = datastore.get(ref, parameters=None)
102 self.assertCatalogEqual(catalog, catalogOut)
104 # These should raise
105 ref = self.makeDatasetRef("calexp2", dimensions, storageClass, dataId)
106 with self.assertRaises(FileNotFoundError):
107 # non-existing file
108 datastore.get(ref, parameters=None)
110 def testRemove(self):
111 catalog = self.makeExampleCatalog()
112 datastore = self.makeDatastore()
114 # Put
115 storageClass = self.storageClassFactory.getStorageClass("SourceCatalog")
116 dimensions = self.registry.dimensions.extract(("visit", "physical_filter"))
117 dataId = {"visit": 1234567, "physical_filter": "blue", "instrument": "dummy"}
119 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId)
120 datastore.put(catalog, ref)
122 # Does it exist?
123 self.assertTrue(datastore.exists(ref))
125 # Get
126 catalogOut = datastore.get(ref)
127 self.assertCatalogEqual(catalog, catalogOut)
129 # Remove
130 datastore.remove(ref)
132 # Does it exist?
133 self.assertFalse(datastore.exists(ref))
135 # Get should now fail
136 with self.assertRaises(FileNotFoundError):
137 datastore.get(ref)
138 # Can only delete once
139 with self.assertRaises(FileNotFoundError):
140 datastore.remove(ref)
142 def testTransfer(self):
143 catalog = self.makeExampleCatalog()
144 dimensions = self.registry.dimensions.extract(("visit", "physical_filter"))
145 dataId = {"visit": 12345, "physical_filter": "red", "instrument": "dummy"}
147 storageClass = self.storageClassFactory.getStorageClass("SourceCatalog")
148 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId)
150 inputDatastore = self.makeDatastore("test_input_datastore")
151 outputDatastore = self.makeDatastore("test_output_datastore")
153 inputDatastore.put(catalog, ref)
154 outputDatastore.transfer(inputDatastore, ref)
156 catalogOut = outputDatastore.get(ref)
157 self.assertCatalogEqual(catalog, catalogOut)
159 def testExposurePutGet(self):
160 example = os.path.join(self.testDir, "data", "basic", "small.fits")
161 exposure = lsst.afw.image.ExposureF(example)
162 datastore = self.makeDatastore()
163 # Put
164 dimensions = self.registry.dimensions.extract(("visit", "physical_filter"))
165 dataId = {"visit": 231, "physical_filter": "Fc", "instrument": "dummy"}
166 storageClass = datastore.storageClassFactory.getStorageClass("ExposureF")
167 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId)
169 datastore.put(exposure, ref)
171 # Does it exist?
172 self.assertTrue(datastore.exists(ref))
174 # Get
175 exposureOut = datastore.get(ref)
176 self.assertEqual(type(exposure), type(exposureOut))
178 # Get some components
179 # Could not test the following components as they were not known:
180 # bbox, xy0, filter, polygon, detector, extras, and exposureInfo
181 for compName, isNone in (("wcs", False),
182 ("image", False),
183 ("mask", False),
184 ("coaddInputs", False),
185 ("psf", False),
186 ("variance", False),
187 ("photoCalib", False),
188 ("metadata", False),
189 ("visitInfo", False),
190 ("apCorrMap", True),
191 ("transmissionCurve", True),
192 ("metadata", False)):
193 with self.subTest(component=compName):
194 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions,
195 storageClass.components[compName], dataId, id=ref.id)
196 component = datastore.get(compRef)
197 if isNone:
198 self.assertIsNone(component)
199 else:
200 self.assertIsInstance(component, compRef.datasetType.storageClass.pytype)
202 # Get the WCS component to check it
203 wcsRef = self.makeDatasetRef(ref.datasetType.componentTypeName("wcs"), dimensions,
204 storageClass.components["wcs"], dataId, id=ref.id)
205 wcs = datastore.get(wcsRef)
207 # Simple check of WCS
208 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
209 lsst.geom.Extent2I(9, 9))
210 self.assertWcsAlmostEqualOverBBox(wcs, exposure.getWcs(), bbox)
212 # Check basic metadata
213 metadataRef = self.makeDatasetRef(ref.datasetType.componentTypeName("metadata"), dimensions,
214 storageClass.components["metadata"], dataId, id=ref.id)
215 metadata = datastore.get(metadataRef)
216 self.assertEqual(metadata["WCS_ID"], 3)
218 def testExposureCompositePutGet(self):
219 example = os.path.join(self.testDir, "data", "basic", "small.fits")
220 exposure = lsst.afw.image.ExposureF(example)
221 datastore = self.makeDatastore()
222 # Put
223 dimensions = self.registry.dimensions.extract(("visit", "physical_filter"))
224 dataId = {"visit": 23, "physical_filter": "F", "instrument": "dummy"}
225 storageClass = datastore.storageClassFactory.getStorageClass("ExposureCompositeF")
226 ref = self.makeDatasetRef("calexp", dimensions, storageClass, dataId)
228 # Get the predicted URI
229 self.assertFalse(datastore.exists(ref))
230 uri = datastore.getUri(ref, predict=True)
231 self.assertTrue(uri.endswith("#predicted"))
233 components = storageClass.assembler().disassemble(exposure)
234 self.assertTrue(components)
236 # Get a component
237 compsRead = {}
238 for compName in ("wcs", "image", "mask", "coaddInputs", "psf"):
239 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions,
240 components[compName].storageClass, dataId)
242 datastore.put(components[compName].component, compRef)
244 # Does it exist?
245 self.assertTrue(datastore.exists(compRef))
247 component = datastore.get(compRef)
248 self.assertIsInstance(component, compRef.datasetType.storageClass.pytype)
249 compsRead[compName] = component
251 # Simple check of WCS
252 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
253 lsst.geom.Extent2I(9, 9))
254 self.assertWcsAlmostEqualOverBBox(compsRead["wcs"], exposure.getWcs(), bbox)
256 # Try to reassemble the exposure
257 retrievedExposure = storageClass.assembler().assemble(compsRead)
258 self.assertIsInstance(retrievedExposure, type(exposure))
261class PosixDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase):
262 """PosixDatastore specialization"""
263 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
264 uriScheme = "file:"
265 fileExt = ".fits"
267 def setUp(self):
268 # Override the working directory before calling the base class
269 self.root = tempfile.mkdtemp(dir=TESTDIR)
270 super().setUp()
273class InMemoryDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase):
274 """PosixDatastore specialization"""
275 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml")
276 uriScheme = "mem:"
277 fileExt = None
280class ChainedDatastoreTestCase(PosixDatastoreTestCase):
281 """PosixDatastore specialization"""
282 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml")
285if __name__ == "__main__": 285 ↛ 286line 285 didn't jump to line 286, because the condition on line 285 was never true
286 unittest.main()