Coverage for tests/test_cameraMapper.py: 22%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import gc
23import os
24import sqlite3
25import unittest
26import tempfile
28import numpy as np
30import lsst.utils.tests
31import lsst.geom as geom
32import lsst.afw.image as afwImage
33import lsst.afw.table as afwTable
34import lsst.daf.persistence as dafPersist
35import lsst.obs.base
36import shutil
38from lsst.obs.base.test import BaseMapper
40ROOT = os.path.abspath(os.path.dirname(__file__))
43def setup_module(module):
44 lsst.utils.tests.init()
47class MinCam(lsst.obs.base.Instrument):
48 @property
49 def filterDefinitions(self):
50 return lsst.obs.base.FilterDefinitionCollection(
51 lsst.obs.base.FilterDefinition(physical_filter="u.MP9301", band="u", lambdaEff=374),
52 lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g", lambdaEff=487),
53 lsst.obs.base.FilterDefinition(physical_filter="r.MP9601", band="r", alias={"old-r"},
54 lambdaEff=628),
55 lsst.obs.base.FilterDefinition(physical_filter="i.MP9701", band="i", alias={"old-i"},
56 lambdaEff=778),
57 lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z", lambdaEff=1170),
58 # afw_name is so special-cased that only a real example will work
59 lsst.obs.base.FilterDefinition(physical_filter="HSC-I2", band="i", afw_name="i2", lambdaEff=623),
60 )
62 @classmethod
63 def getName(cls):
64 return "min"
66 def getCamera(self):
67 raise NotImplementedError()
69 def register(self, registry):
70 raise NotImplementedError()
72 def getRawFormatter(self, dataId):
73 raise NotImplementedError()
75 def makeDataIdTranslatorFactory(self):
76 raise NotImplementedError()
79class MinMapper1(lsst.obs.base.CameraMapper):
80 packageName = 'larry'
82 def __init__(self, **kwargs):
83 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml"))
84 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs)
85 return
87 def std_x(self, item, dataId):
88 return float(item)
90 @classmethod
91 def getCameraName(cls):
92 """Return the name of the camera that this CameraMapper is for."""
93 return "min"
95 @classmethod
96 def getPackageDir(cls):
97 return "/path/to/nowhere"
100class MinMapper2(lsst.obs.base.CameraMapper):
101 packageName = 'moe'
102 _gen3instrument = MinCam
104 # CalibRoot in policy
105 # needCalibRegistry
106 def __init__(self, **kwargs):
107 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper2.yaml"))
108 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT,
109 registry="cfhtls.sqlite3", **kwargs)
110 return
112 def _transformId(self, dataId):
113 return dataId
115 def _extractDetectorName(self, dataId):
116 return "ccd00"
118 def std_x(self, item, dataId):
119 return float(item)
121 @classmethod
122 def getCameraName(cls):
123 """Return the name of the camera that this CameraMapper is for."""
124 return "min"
126 @classmethod
127 def getPackageDir(cls):
128 return "/path/to/nowhere"
131# does not assign packageName
132class MinMapper3(lsst.obs.base.CameraMapper):
134 def __init__(self, **kwargs):
135 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml"))
136 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, root=ROOT)
137 return
139 @classmethod
140 def getPackageDir(cls):
141 return "/path/to/nowhere"
144def checkCompression(testCase, additionalData):
145 """Check that compression settings are present
147 We check that we can access the required settings, and that
148 the seed is non-zero (zero causes lsst.afw.math.Random to fail).
149 """
150 for plane in ("image", "mask", "variance"):
151 for entry in ("compression.algorithm",
152 "compression.columns",
153 "compression.rows",
154 "compression.quantizeLevel",
155 "scaling.algorithm",
156 "scaling.bitpix",
157 "scaling.maskPlanes",
158 "scaling.seed",
159 "scaling.quantizeLevel",
160 "scaling.quantizePad",
161 "scaling.fuzz",
162 "scaling.bscale",
163 "scaling.bzero",
164 ):
165 additionalData.getScalar(plane + "." + entry)
166 testCase.assertNotEqual(additionalData.getScalar(plane + ".scaling.seed"), 0)
169class Mapper1TestCase(unittest.TestCase):
170 """A test case for the mapper used by the data butler."""
172 def setUp(self):
173 self.mapper = MinMapper1(root=ROOT)
175 def tearDown(self):
176 del self.mapper
178 def testGetDatasetTypes(self):
179 expectedTypes = BaseMapper(ROOT).getDatasetTypes()
180 # Add the expected additional types to what the base class provides
181 expectedTypes.extend(["x", "x_filename",
182 "badSourceHist", "badSourceHist_filename", ])
183 self.assertEqual(set(self.mapper.getDatasetTypes()), set(expectedTypes))
185 def testMap(self):
186 loc = self.mapper.map("x", {"sensor": "1,1"}, write=True)
187 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI")
188 self.assertEqual(loc.getCppType(), "BoxI")
189 self.assertEqual(loc.getStorageName(), "PickleStorage")
190 expectedRoot = ROOT
191 expectedLocations = ["foo-1,1.pickle"]
192 self.assertEqual(loc.getStorage().root, expectedRoot)
193 self.assertEqual(loc.getLocations(), expectedLocations)
194 self.assertEqual(loc.getAdditionalData().toString(),
195 "sensor = \"1,1\"\n")
197 def testQueryMetadata(self):
198 self.assertEqual(self.mapper.queryMetadata("x", ["sensor"], None), [("1,1",)])
200 def testStandardize(self):
201 self.assertTrue(self.mapper.canStandardize("x"))
202 self.assertFalse(self.mapper.canStandardize("badSourceHist"))
203 self.assertFalse(self.mapper.canStandardize("notPresent"))
204 result = self.mapper.standardize("x", 3, None)
205 self.assertIsInstance(result, float)
206 self.assertEqual(result, 3.0)
207 result = self.mapper.standardize("x", 3.14, None)
208 self.assertIsInstance(result, float)
209 self.assertEqual(result, 3.14)
210 result = self.mapper.standardize("x", "3.14", None)
211 self.assertIsInstance(result, float)
212 self.assertEqual(result, 3.14)
214 def testNames(self):
215 self.assertEqual(MinMapper1.getCameraName(), "min")
216 self.assertEqual(MinMapper1.getPackageName(), "larry")
219class Mapper2TestCase(unittest.TestCase):
220 """A test case for the mapper used by the data butler."""
222 def setUp(self):
223 super().setUp()
224 # Force a standard set of filters even for tests that don't use
225 # MinCam directly.
226 MinCam()
228 def testGetDatasetTypes(self):
229 mapper = MinMapper2(root=ROOT)
230 expectedTypes = BaseMapper(ROOT).getDatasetTypes()
231 # Add the expected additional types to what the base class provides
232 expectedTypes.extend(["flat", "flat_md", "flat_filename", "flat_sub",
233 "raw", "raw_md", "raw_filename", "raw_sub",
234 "some", "some_filename", "some_md", "some_sub",
235 "someCatalog", "someCatalog_md", "someCatalog_filename",
236 "someCatalog_len", "someCatalog_schema",
237 "forced_src", "forced_src_md", "forced_src_filename",
238 "forced_src_len", "forced_src_schema",
239 "other_sub", "other_filename", "other_md", "other",
240 "someGz", "someGz_filename", "someFz", "someFz_filename", "someGz_md",
241 "someFz_sub", "someFz_md", "someGz_sub",
242 "someGz_bbox", "someFz_bbox", "some_bbox", "other_bbox",
243 "someExp", "someExp_filename", "someExp_md", "someExp_sub",
244 "someExp_bbox", "someExp_filterLabel", "someExp_photoCalib",
245 "someExp_visitInfo", "someExp_detector", "someExp_filter",
246 "someExp_header_wcs", "someExp_wcs",
247 ])
248 self.assertEqual(set(mapper.getDatasetTypes()),
249 set(expectedTypes))
251 def testMap(self):
252 mapper = MinMapper2(root=ROOT)
253 loc = mapper.map("raw", {"ccd": 13}, write=True)
254 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
255 self.assertEqual(loc.getCppType(), "ImageU")
256 self.assertEqual(loc.getStorageName(), "FitsStorage")
257 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
258 self.assertEqual(loc.getStorage().root, ROOT)
259 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
260 checkCompression(self, loc.getAdditionalData())
262 def testSubMap(self):
263 bbox = geom.BoxI(geom.Point2I(200, 100),
264 geom.Extent2I(300, 400))
265 mapper = MinMapper2(root=ROOT)
266 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True)
267 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
268 self.assertEqual(loc.getCppType(), "ImageU")
269 self.assertEqual(loc.getStorageName(), "FitsStorage")
270 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
271 self.assertEqual(loc.getStorage().root, ROOT)
272 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
273 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
274 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
275 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
276 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
277 checkCompression(self, loc.getAdditionalData())
279 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox, "imageOrigin": "PARENT"}, write=True)
280 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
281 self.assertEqual(loc.getCppType(), "ImageU")
282 self.assertEqual(loc.getStorageName(), "FitsStorage")
283 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
284 self.assertEqual(loc.getStorage().root, ROOT)
285 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
286 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
287 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
288 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
289 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
290 self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"), "PARENT")
291 checkCompression(self, loc.getAdditionalData())
293 def testCatalogExtras(self):
294 butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2)
295 schema = afwTable.Schema()
296 aa = schema.addField("a", type=np.int32, doc="a")
297 bb = schema.addField("b", type=np.float64, doc="b")
298 catalog = lsst.afw.table.BaseCatalog(schema)
299 row = catalog.addNew()
300 row.set(aa, 12345)
301 row.set(bb, 1.2345)
302 size = len(catalog)
303 dataId = dict(visit=123, ccd=45)
304 butler.put(catalog, "someCatalog", dataId)
305 filename = butler.get("someCatalog_filename", dataId)[0]
306 try:
307 self.assertTrue(os.path.exists(filename))
308 self.assertEqual(butler.get("someCatalog_schema", dataId), schema)
309 self.assertEqual(butler.get("someCatalog_len", dataId), size)
310 header = butler.get("someCatalog_md", dataId)
311 self.assertEqual(header.getScalar("NAXIS2"), size)
312 finally:
313 try:
314 os.remove(filename)
315 except OSError as exc:
316 print("Warning: could not remove file %r: %s" % (filename, exc))
318 def testImage(self):
319 mapper = MinMapper2(root=ROOT)
320 loc = mapper.map("some", dict(ccd=35))
321 expectedLocations = ["bar-35.fits"]
322 self.assertEqual(loc.getStorage().root, ROOT)
323 self.assertEqual(loc.getLocations(), expectedLocations)
325 butler = dafPersist.ButlerFactory(mapper=mapper).create()
326 image = butler.get("some", ccd=35)
327 self.assertEqual(image.getFilter().getName(), "r")
328 self.assertEqual(image.getFilterLabel().bandLabel, "r")
330 self.assertEqual(butler.get("some_bbox", ccd=35), image.getBBox())
332 bbox = geom.BoxI(geom.Point2I(200, 100),
333 geom.Extent2I(300, 400))
334 image = butler.get("some_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
335 self.assertEqual(image.getHeight(), 400)
336 self.assertEqual(image.getWidth(), 300)
338 def testFilter(self):
339 """Test that the same (patched) filter is returned through all Butler
340 retrieval paths.
341 """
342 mapper = MinMapper2(root=ROOT)
344 butler = dafPersist.ButlerFactory(mapper=mapper).create()
345 image = butler.get("someExp", ccd=35)
346 filter = butler.get("someExp_filterLabel", ccd=35)
347 # Test only valid with a complete filter
348 self.assertEqual(image.getFilterLabel(), afwImage.FilterLabel(band="r", physical="r.MP9601"))
349 # Datasets should give consistent answers
350 self.assertEqual(filter, image.getFilterLabel())
352 def testDetector(self):
353 mapper = MinMapper2(root=ROOT)
354 butler = dafPersist.ButlerFactory(mapper=mapper).create()
355 detector = butler.get("raw_detector", ccd=0)
356 self.assertEqual(detector.getName(), "ccd00")
358 def testGzImage(self):
359 mapper = MinMapper2(root=ROOT)
360 loc = mapper.map("someGz", dict(ccd=35))
361 expectedLocations = [os.path.join("gz", "bar-35.fits.gz")]
362 self.assertEqual(loc.getStorage().root, ROOT)
363 self.assertEqual(loc.getLocations(), expectedLocations)
365 butler = dafPersist.ButlerFactory(mapper=mapper).create()
366 image = butler.get("someGz", ccd=35)
367 self.assertEqual(image.getFilter().getName(), "r")
368 self.assertEqual(image.getFilterLabel().bandLabel, "r")
370 bbox = geom.BoxI(geom.Point2I(200, 100),
371 geom.Extent2I(300, 400))
372 image = butler.get("someGz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
373 self.assertEqual(image.getHeight(), 400)
374 self.assertEqual(image.getWidth(), 300)
376 def testFzImage(self):
377 mapper = MinMapper2(root=ROOT)
378 loc = mapper.map("someFz", dict(ccd=35))
379 expectedRoot = ROOT
380 expectedLocations = [os.path.join("fz", "bar-35.fits.fz")]
381 self.assertEqual(loc.getStorage().root, expectedRoot)
382 self.assertEqual(loc.getLocations(), expectedLocations)
384 butler = dafPersist.ButlerFactory(mapper=mapper).create()
385 image = butler.get("someFz", ccd=35)
386 self.assertEqual(image.getFilter().getName(), "r")
387 self.assertEqual(image.getFilterLabel().bandLabel, "r")
389 bbox = geom.BoxI(geom.Point2I(200, 100),
390 geom.Extent2I(300, 400))
391 image = butler.get("someFz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
392 self.assertEqual(image.getHeight(), 400)
393 self.assertEqual(image.getWidth(), 300)
395 def testButlerQueryMetadata(self):
396 mapper = MinMapper2(root=ROOT)
397 butler = dafPersist.ButlerFactory(mapper=mapper).create()
398 kwargs = {"ccd": 35, "filter": "r", "visit": 787731,
399 "taiObs": "2005-04-02T09:24:49.933440000"}
400 self.assertEqual(butler.queryMetadata("other", "visit", **kwargs), [787731])
401 self.assertEqual(butler.queryMetadata("other", "visit",
402 visit=kwargs["visit"], ccd=kwargs["ccd"],
403 taiObs=kwargs["taiObs"], filter=kwargs["filter"]),
404 [787731])
405 # now test we get no matches if ccd is out of range
406 self.assertEqual(butler.queryMetadata("raw", "ccd", ccd=36, filter="r", visit=787731), [])
408 def testQueryMetadata(self):
409 mapper = MinMapper2(root=ROOT)
410 self.assertEqual(mapper.queryMetadata("raw", ["ccd"], None),
411 [(x,) for x in range(36) if x != 3])
413 def testStandardize(self):
414 mapper = MinMapper2(root=ROOT)
415 self.assertEqual(mapper.canStandardize("raw"), True)
416 self.assertEqual(mapper.canStandardize("notPresent"), False)
418 def testStandardizeFiltersFilterDefs(self):
419 # tuples are (input, desired output)
420 testLabels = [
421 (None, None),
422 (afwImage.FilterLabel(band="i", physical="i.MP9701"),
423 afwImage.FilterLabel(band="i", physical="i.MP9701")),
424 (afwImage.FilterLabel(band="i"), afwImage.FilterLabel(band="i", physical="i.MP9701")),
425 (afwImage.FilterLabel(physical="i.MP9701"),
426 afwImage.FilterLabel(band="i", physical="i.MP9701")),
427 (afwImage.FilterLabel(band="i", physical="old-i"),
428 afwImage.FilterLabel(band="i", physical="i.MP9701")),
429 (afwImage.FilterLabel(physical="old-i"),
430 afwImage.FilterLabel(band="i", physical="i.MP9701")),
431 (afwImage.FilterLabel(physical="i2"), afwImage.FilterLabel(band="i", physical="HSC-I2")),
432 ]
433 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in {
434 "i", "i.MP9701", "old-i", "i2",
435 }]
436 testData = []
437 # Resolve special combinations where the expected output is different
438 for input, corrected in testLabels:
439 for dataId in testIds:
440 if input is None:
441 if dataId["filter"] == "i2":
442 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
443 else:
444 data = (input, dataId, afwImage.FilterLabel(band="i", physical="i.MP9701"))
445 elif input == afwImage.FilterLabel(band="i") and dataId["filter"] == "i2":
446 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
447 elif corrected.physicalLabel == "HSC-I2" and dataId["filter"] in ("i.MP9701", "old-i"):
448 # Contradictory inputs, leave as-is
449 data = (input, dataId, input)
450 elif corrected.physicalLabel == "i.MP9701" and dataId["filter"] == "i2":
451 # Contradictory inputs, leave as-is
452 data = (input, dataId, input)
453 else:
454 data = (input, dataId, corrected)
455 testData.append(data)
457 mapper = MinMapper2(root=ROOT)
458 for label, dataId, corrected in testData:
459 exposure = afwImage.ExposureF()
460 exposure.setFilterLabel(label)
461 mapper._setFilter(mapper.exposures['raw'], exposure, dataId)
462 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}")
464 def testStandardizeFiltersFilterNoDefs(self):
465 testLabels = [
466 None,
467 afwImage.FilterLabel(band="i", physical="i.MP9701"),
468 afwImage.FilterLabel(band="i"),
469 afwImage.FilterLabel(physical="i.MP9701"),
470 afwImage.FilterLabel(band="i", physical="old-i"),
471 afwImage.FilterLabel(physical="old-i"),
472 afwImage.FilterLabel(physical="i2"),
473 ]
474 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in {
475 "i", "i.MP9701", "old-i", "i2",
476 }]
477 testData = []
478 # Resolve special combinations where the expected output is different
479 for input in testLabels:
480 for dataId in testIds:
481 if input is None:
482 # Can still get some filter info out of the Filter registry
483 if dataId["filter"] == "i2":
484 data = (input, dataId,
485 afwImage.FilterLabel(band="i", physical="HSC-I2"))
486 else:
487 # Data ID maps to filter(s) with aliases; can't
488 # unambiguously determine physical filter.
489 data = (input, dataId, afwImage.FilterLabel(band="i"))
490 else:
491 data = (input, dataId, input)
492 testData.append(data)
494 mapper = MinMapper1(root=ROOT)
495 for label, dataId, corrected in testData:
496 exposure = afwImage.ExposureF()
497 exposure.setFilterLabel(label)
498 mapper._setFilter(mapper.exposures['raw'], exposure, dataId)
499 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}")
501 def testCalib(self):
502 mapper = MinMapper2(root=ROOT)
503 loc = mapper.map("flat", {"visit": 787650, "ccd": 13}, write=True)
504 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureF")
505 self.assertEqual(loc.getCppType(), "ExposureF")
506 self.assertEqual(loc.getStorageName(), "FitsStorage")
507 expectedRoot = ROOT
508 expectedLocations = ["flat-05Am03-fi.fits"]
509 self.assertEqual(loc.getStorage().root, expectedRoot)
510 self.assertEqual(loc.getLocations(), expectedLocations)
511 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
512 self.assertEqual(loc.getAdditionalData().getScalar("visit"), 787650)
513 self.assertEqual(loc.getAdditionalData().getScalar("derivedRunId"), "05Am03")
514 self.assertEqual(loc.getAdditionalData().getScalar("filter"), "i")
515 checkCompression(self, loc.getAdditionalData())
517 def testNames(self):
518 self.assertEqual(MinMapper2.getCameraName(), "min")
519 self.assertEqual(MinMapper2.getPackageName(), "moe")
521 @unittest.expectedFailure
522 def testParentSearch(self):
523 mapper = MinMapper2(root=ROOT)
524 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'),
525 os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits')))
526 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits'))])
527 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'),
528 os.path.join(ROOT,
529 os.path.join('testParentSearch', 'bar.fits[1]')))
530 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits[1]'))])
532 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'),
533 os.path.join(ROOT, os.path.join('testParentSearch', 'baz.fits')))
534 self.assertEqual(paths, [os.path.join(ROOT,
535 os.path.join('testParentSearch', '_parent', 'baz.fits'))])
536 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'),
537 os.path.join(ROOT,
538 os.path.join('testParentSearch', 'baz.fits[1]')))
539 self.assertEqual(paths, [os.path.join(ROOT,
540 os.path.join('testParentSearch', '_parent', 'baz.fits[1]'))])
542 def testSkymapLookups(self):
543 """Test that metadata lookups don't try to get skymap data ID values
544 from the registry.
545 """
546 mapper = MinMapper2(root=ROOT)
547 butler = dafPersist.Butler(mapper=mapper)
548 with self.assertRaises(RuntimeError) as manager:
549 butler.dataRef("forced_src", visit=787650, ccd=13)
550 self.assertIn("Cannot lookup skymap key 'tract'", str(manager.exception))
551 # We're mostly concerned that the statements below will raise an
552 # exception; if they don't, it's not likely the following tests will
553 # fail.
554 subset = butler.subset("forced_src", visit=787650, ccd=13, tract=0)
555 self.assertEqual(len(subset), 1)
556 dataRef = butler.dataRef("forced_src", visit=787650, ccd=13, tract=0)
557 self.assertFalse(dataRef.datasetExists("forced_src"))
560class Mapper3TestCase(unittest.TestCase):
561 """A test case for a mapper subclass which does not assign packageName."""
563 def testPackageName(self):
564 with self.assertRaises(ValueError):
565 MinMapper3()
566 with self.assertRaises(ValueError):
567 MinMapper3.getPackageName()
570class ParentRegistryTestCase(unittest.TestCase):
572 @staticmethod
573 def _createRegistry(path):
574 cmd = """CREATE TABLE x(
575 id INT,
576 visit INT,
577 filter TEXT,
578 snap INT,
579 raft TEXT,
580 sensor TEXT,
581 channel TEXT,
582 taiObs TEXT,
583 expTime REAL
584 );
585 """
586 conn = sqlite3.connect(path)
587 conn.cursor().execute(cmd)
588 conn.commit()
589 conn.close()
591 def setUp(self):
592 self.ROOT = tempfile.mkdtemp(dir=ROOT, prefix="ParentRegistryTestCase-")
593 self.repoARoot = os.path.join(self.ROOT, 'a')
594 args = dafPersist.RepositoryArgs(root=self.repoARoot, mapper=MinMapper1)
595 butler = dafPersist.Butler(outputs=args)
596 self._createRegistry(os.path.join(self.repoARoot, 'registry.sqlite3'))
597 del butler
599 def tearDown(self):
600 # the butler sql registry closes its database connection in __del__.
601 # To trigger __del__ we explicitly collect the garbage here. If we
602 # find having or closing the open database connection is a problem in
603 # production code, we may need to add api to butler to explicity
604 # release database connections (and maybe other things like in-memory
605 # cached objects).
606 gc.collect()
607 if os.path.exists(self.ROOT):
608 shutil.rmtree(self.ROOT)
610 def test(self):
611 """Verify that when the child repo does not have a registry it is
612 assigned the registry from the parent.
613 """
614 repoBRoot = os.path.join(self.ROOT, 'b')
615 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot)
616 # This way of getting the registry from the mapping is obviously going
617 # way into private members and the python lambda implementation code.
618 # It is very brittle and should not be duplicated in user code
619 # or any location that is not trivial to fix along with changes to the
620 # CameraMapper or Mapping.
621 registryA = butler._repos.inputs()[0].repo._mapper.registry
622 registryB = butler._repos.outputs()[0].repo._mapper.registry
623 self.assertEqual(id(registryA), id(registryB))
625 self._createRegistry(os.path.join(repoBRoot, 'registry.sqlite3'))
626 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot)
627 # see above; don't copy this way of getting the registry.
628 registryA = butler._repos.inputs()[0].repo._mapper.registry
629 registryB = butler._repos.outputs()[0].repo._mapper.registry
630 self.assertNotEqual(id(registryA), id(registryB))
633class MissingPolicyKeyTestCase(unittest.TestCase):
635 def testGetRaises(self):
636 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1})
637 # MinMapper1 does not specify a template for the raw dataset type so
638 # trying to use it for get should raise
639 with self.assertRaises(RuntimeError) as contextManager:
640 butler.get('raw')
641 # This test demonstrates and verifies that simple use of the incomplete
642 # dataset type returns a helpful (I hope) error message.
643 self.assertEqual(
644 str(contextManager.exception),
645 'Template is not defined for the raw dataset type, '
646 'it must be set before it can be used.')
647 with self.assertRaises(RuntimeError) as contextManager:
648 butler.queryMetadata('raw', 'unused', {})
650 def testQueryMetadataRaises(self):
651 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1})
652 # MinMapper1 does not specify a template for the raw dataset type so
653 # trying to use it for queryMetadata should raise
654 with self.assertRaises(RuntimeError) as contextManager:
655 butler.queryMetadata('raw', 'unused', {})
656 # This test demonstrates and verifies that simple use of the incomplete
657 # dataset type returns a helpful (I hope) error message.
658 self.assertEqual(
659 str(contextManager.exception),
660 'Template is not defined for the raw dataset type, '
661 'it must be set before it can be used.')
663 def testFilenameRaises(self):
664 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1})
665 # MinMapper1 does not specify a template for the raw dataset type so
666 # trying to use it for <datasetType>_filename should raise
667 with self.assertRaises(RuntimeError) as contextManager:
668 butler.get('raw_filename')
669 # This test demonstrates and verifies that simple use of the
670 # incomplete dataset type returns a helpful (I hope) error message.
671 self.assertEqual(
672 str(contextManager.exception),
673 'Template is not defined for the raw dataset type, '
674 'it must be set before it can be used.')
676 def testWcsRaises(self):
677 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1})
678 # MinMapper1 does not specify a template for the raw dataset type so
679 # trying to use it for <datasetType>_wcs should raise
680 with self.assertRaises(RuntimeError) as contextManager:
681 butler.get('raw_wcs')
682 # This test demonstrates and verifies that simple use of the
683 # incomplete dataset type returns a helpful (I hope) error message.
684 self.assertEqual(
685 str(contextManager.exception),
686 'Template is not defined for the raw dataset type, '
687 'it must be set before it can be used.')
689 def testConflictRaises(self):
690 policy = dafPersist.Policy(os.path.join(ROOT, "ConflictMapper.yaml"))
691 with self.assertRaisesRegex(
692 ValueError,
693 r"Duplicate mapping policy for dataset type packages"):
694 mapper = lsst.obs.base.CameraMapper(policy=policy, repositoryDir=ROOT, root=ROOT) # noqa F841
697class MemoryTester(lsst.utils.tests.MemoryTestCase):
698 pass
701if __name__ == '__main__': 701 ↛ 702line 701 didn't jump to line 702, because the condition on line 701 was never true
702 lsst.utils.tests.init()
703 unittest.main()