# This file is part of daf_butler.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (http://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import unittest
import lsst.utils.tests
from lsst.daf.butler import StorageClassFactory
from lsst.daf.butler.datastores.posixDatastore import DatastoreConfig
from lsst.daf.butler.core.utils import doImport
from datasetsHelper import FitsCatalogDatasetsHelper, DatasetTestHelper
from dummyRegistry import DummyRegistry
try:
import lsst.afw.table
import lsst.afw.image
import lsst.afw.geom
except ImportError:
lsst.afw.table = None
lsst.afw.image = None
TESTDIR = os.path.dirname(__file__)
class DatastoreFitsTests(FitsCatalogDatasetsHelper, DatasetTestHelper):
@classmethod
def setUpClass(cls):
51 ↛ 52line 51 didn't jump to line 52, because the condition on line 51 was never true if lsst.afw.table is None:
raise unittest.SkipTest("afw not available.")
# Base classes need to know where the test directory is
cls.testDir = TESTDIR
# Storage Classes are fixed for all datastores in these tests
scConfigFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
cls.storageClassFactory = StorageClassFactory()
cls.storageClassFactory.addFromConfig(scConfigFile)
# Read the Datastore config so we can get the class
# information (since we should not assume the constructor
# name here, but rely on the configuration file itself)
datastoreConfig = DatastoreConfig(cls.configFile)
cls.datastoreType = doImport(datastoreConfig["cls"])
def setUp(self):
self.registry = DummyRegistry()
# Need to keep ID for each datasetRef since we have no butler
# for these tests
self.id = 1
def testConstructor(self):
datastore = self.datastoreType(config=self.configFile, registry=self.registry)
self.assertIsNotNone(datastore)
def testBasicPutGet(self):
catalog = self.makeExampleCatalog()
datastore = self.datastoreType(config=self.configFile, registry=self.registry)
# Put
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 123456, "filter": "blue"}
storageClass = self.storageClassFactory.getStorageClass("SourceCatalog")
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
datastore.put(catalog, ref)
# Does it exist?
self.assertTrue(datastore.exists(ref))
uri = datastore.getUri(ref)
if self.fileExt is not None:
self.assertTrue(uri.endswith(self.fileExt))
self.assertTrue(uri.startswith(self.uriScheme))
# Get
catalogOut = datastore.get(ref, parameters=None)
self.assertCatalogEqual(catalog, catalogOut)
# These should raise
ref = self.makeDatasetRef("calexp2", dataUnits, storageClass, dataId)
with self.assertRaises(FileNotFoundError):
# non-existing file
datastore.get(ref, parameters=None)
def testRemove(self):
catalog = self.makeExampleCatalog()
datastore = self.datastoreType(config=self.configFile, registry=self.registry)
# Put
storageClass = self.storageClassFactory.getStorageClass("SourceCatalog")
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 1234567, "filter": "blue"}
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
datastore.put(catalog, ref)
# Does it exist?
self.assertTrue(datastore.exists(ref))
# Get
catalogOut = datastore.get(ref)
self.assertCatalogEqual(catalog, catalogOut)
# Remove
datastore.remove(ref)
# Does it exist?
self.assertFalse(datastore.exists(ref))
# Get should now fail
with self.assertRaises(FileNotFoundError):
datastore.get(ref)
# Can only delete once
with self.assertRaises(FileNotFoundError):
datastore.remove(ref)
def testTransfer(self):
catalog = self.makeExampleCatalog()
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 12345, "filter": "red"}
storageClass = self.storageClassFactory.getStorageClass("SourceCatalog")
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
inputConfig = DatastoreConfig(self.configFile)
inputConfig["root"] = os.path.join(self.testDir, "./test_input_datastore")
inputPosixDatastore = self.datastoreType(config=inputConfig, registry=self.registry)
outputConfig = inputConfig.copy()
outputConfig["root"] = os.path.join(self.testDir, "./test_output_datastore")
outputPosixDatastore = self.datastoreType(config=outputConfig,
registry=DummyRegistry())
inputPosixDatastore.put(catalog, ref)
outputPosixDatastore.transfer(inputPosixDatastore, ref)
catalogOut = outputPosixDatastore.get(ref)
self.assertCatalogEqual(catalog, catalogOut)
def testExposurePutGet(self):
example = os.path.join(self.testDir, "data", "basic", "small.fits")
exposure = lsst.afw.image.ExposureF(example)
datastore = self.datastoreType(config=self.configFile, registry=self.registry)
# Put
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 231, "filter": "Fc"}
storageClass = datastore.storageClassFactory.getStorageClass("ExposureF")
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
datastore.put(exposure, ref)
# Does it exist?
self.assertTrue(datastore.exists(ref))
# Get
exposureOut = datastore.get(ref)
self.assertEqual(type(exposure), type(exposureOut))
# Get some components
for compName in ("wcs", "image", "mask", "coaddInputs", "psf"):
compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dataUnits,
storageClass.components[compName], dataId, id=ref.id)
component = datastore.get(compRef)
self.assertIsInstance(component, compRef.datasetType.storageClass.pytype)
# Get the WCS component to check it
wcsRef = self.makeDatasetRef(ref.datasetType.componentTypeName("wcs"), dataUnits,
storageClass.components["wcs"], dataId, id=ref.id)
wcs = datastore.get(wcsRef)
# Simple check of WCS
bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0, 0),
lsst.afw.geom.Extent2I(9, 9))
self.assertWcsAlmostEqualOverBBox(wcs, exposure.getWcs(), bbox)
def testExposureCompositePutGet(self):
example = os.path.join(self.testDir, "data", "basic", "small.fits")
exposure = lsst.afw.image.ExposureF(example)
datastore = self.datastoreType(config=self.configFile, registry=self.registry)
# Put
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 23, "filter": "F"}
storageClass = datastore.storageClassFactory.getStorageClass("ExposureCompositeF")
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
# Get the predicted URI
self.assertFalse(datastore.exists(ref))
uri = datastore.getUri(ref, predict=True)
self.assertTrue(uri.endswith("#predicted"))
components = storageClass.assembler().disassemble(exposure)
self.assertTrue(components)
# Get a component
compsRead = {}
for compName in ("wcs", "image", "mask", "coaddInputs", "psf"):
compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dataUnits,
components[compName].storageClass, dataId)
datastore.put(components[compName].component, compRef)
# Does it exist?
self.assertTrue(datastore.exists(compRef))
component = datastore.get(compRef)
self.assertIsInstance(component, compRef.datasetType.storageClass.pytype)
compsRead[compName] = component
# Simple check of WCS
bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0, 0),
lsst.afw.geom.Extent2I(9, 9))
self.assertWcsAlmostEqualOverBBox(compsRead["wcs"], exposure.getWcs(), bbox)
# Try to reassemble the exposure
retrievedExposure = storageClass.assembler().assemble(compsRead)
self.assertIsInstance(retrievedExposure, type(exposure))
class PosixDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase):
"""PosixDatastore specialization"""
configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
uriScheme = "file:"
fileExt = ".fits"
class InMemoryDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase):
"""PosixDatastore specialization"""
configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml")
uriScheme = "mem:"
fileExt = None
class ChainedDatastoreTestCase(DatastoreFitsTests, lsst.utils.tests.TestCase):
"""PosixDatastore specialization"""
configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml")
uriScheme = "file:"
fileExt = ".fits"
class MemoryTester(lsst.utils.tests.MemoryTestCase):
pass
def setup_module(module):
lsst.utils.tests.init()
271 ↛ 272line 271 didn't jump to line 272, because the condition on line 271 was never trueif __name__ == "__main__":
lsst.utils.tests.init()
unittest.main()
|