Coverage for tests/test_cameraMapper.py : 20%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import gc
23import os
24import sqlite3
25import unittest
26import tempfile
28import numpy as np
30import lsst.utils.tests
31import lsst.geom as geom
32import lsst.afw.image as afwImage
33import lsst.afw.table as afwTable
34import lsst.daf.persistence as dafPersist
35import lsst.obs.base
36import shutil
38from lsst.obs.base.test import BaseMapper
40ROOT = os.path.abspath(os.path.dirname(__file__))
43def setup_module(module):
44 lsst.utils.tests.init()
47class MinCam(lsst.obs.base.Instrument):
48 @property
49 def filterDefinitions(self):
50 return lsst.obs.base.FilterDefinitionCollection(
51 lsst.obs.base.FilterDefinition(physical_filter="u.MP9301", band="u", lambdaEff=374),
52 lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g", lambdaEff=487),
53 lsst.obs.base.FilterDefinition(physical_filter="r.MP9601", band="r", alias={"old-r"},
54 lambdaEff=628),
55 lsst.obs.base.FilterDefinition(physical_filter="i.MP9701", band="i", alias={"old-i"},
56 lambdaEff=778),
57 lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z", lambdaEff=1170),
58 # afw_name is so special-cased that only a real example will work
59 lsst.obs.base.FilterDefinition(physical_filter="HSC-I2", band="i", afw_name="i2", lambdaEff=623),
60 )
62 @classmethod
63 def getName(cls):
64 return "min"
66 def getCamera(self):
67 raise NotImplementedError()
69 def register(self, registry):
70 raise NotImplementedError()
72 def getRawFormatter(self, dataId):
73 raise NotImplementedError()
75 def makeDataIdTranslatorFactory(self):
76 raise NotImplementedError()
79class MinMapper1(lsst.obs.base.CameraMapper):
80 packageName = 'larry'
82 def __init__(self, **kwargs):
83 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml"))
84 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs)
85 return
87 def std_x(self, item, dataId):
88 return float(item)
90 @classmethod
91 def getCameraName(cls):
92 """Return the name of the camera that this CameraMapper is for."""
93 return "min"
95 @classmethod
96 def getPackageDir(cls):
97 return "/path/to/nowhere"
100class MinMapper2(lsst.obs.base.CameraMapper):
101 packageName = 'moe'
102 _gen3instrument = MinCam
104 # CalibRoot in policy
105 # needCalibRegistry
106 def __init__(self, **kwargs):
107 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper2.yaml"))
108 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT,
109 registry="cfhtls.sqlite3", **kwargs)
110 return
112 def _transformId(self, dataId):
113 return dataId
115 def _extractDetectorName(self, dataId):
116 return "ccd00"
118 def std_x(self, item, dataId):
119 return float(item)
121 @classmethod
122 def getCameraName(cls):
123 """Return the name of the camera that this CameraMapper is for."""
124 return "min"
126 @classmethod
127 def getPackageDir(cls):
128 return "/path/to/nowhere"
131# does not assign packageName
132class MinMapper3(lsst.obs.base.CameraMapper):
134 def __init__(self, **kwargs):
135 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml"))
136 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, root=ROOT)
137 return
139 @classmethod
140 def getPackageDir(cls):
141 return "/path/to/nowhere"
144def checkCompression(testCase, additionalData):
145 """Check that compression settings are present
147 We check that we can access the required settings, and that
148 the seed is non-zero (zero causes lsst.afw.math.Random to fail).
149 """
150 for plane in ("image", "mask", "variance"):
151 for entry in ("compression.algorithm",
152 "compression.columns",
153 "compression.rows",
154 "compression.quantizeLevel",
155 "scaling.algorithm",
156 "scaling.bitpix",
157 "scaling.maskPlanes",
158 "scaling.seed",
159 "scaling.quantizeLevel",
160 "scaling.quantizePad",
161 "scaling.fuzz",
162 "scaling.bscale",
163 "scaling.bzero",
164 ):
165 additionalData.getScalar(plane + "." + entry)
166 testCase.assertNotEqual(additionalData.getScalar(plane + ".scaling.seed"), 0)
169class Mapper1TestCase(unittest.TestCase):
170 """A test case for the mapper used by the data butler."""
172 def setUp(self):
173 self.mapper = MinMapper1(root=ROOT)
175 def tearDown(self):
176 del self.mapper
178 def testGetDatasetTypes(self):
179 expectedTypes = BaseMapper(ROOT).getDatasetTypes()
180 # Add the expected additional types to what the base class provides
181 expectedTypes.extend(["x", "x_filename",
182 "badSourceHist", "badSourceHist_filename", ])
183 self.assertEqual(set(self.mapper.getDatasetTypes()), set(expectedTypes))
185 def testMap(self):
186 loc = self.mapper.map("x", {"sensor": "1,1"}, write=True)
187 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI")
188 self.assertEqual(loc.getCppType(), "BoxI")
189 self.assertEqual(loc.getStorageName(), "PickleStorage")
190 expectedRoot = ROOT
191 expectedLocations = ["foo-1,1.pickle"]
192 self.assertEqual(loc.getStorage().root, expectedRoot)
193 self.assertEqual(loc.getLocations(), expectedLocations)
194 self.assertEqual(loc.getAdditionalData().toString(),
195 "sensor = \"1,1\"\n")
197 def testQueryMetadata(self):
198 self.assertEqual(self.mapper.queryMetadata("x", ["sensor"], None), [("1,1",)])
200 def testStandardize(self):
201 self.assertTrue(self.mapper.canStandardize("x"))
202 self.assertFalse(self.mapper.canStandardize("badSourceHist"))
203 self.assertFalse(self.mapper.canStandardize("notPresent"))
204 result = self.mapper.standardize("x", 3, None)
205 self.assertIsInstance(result, float)
206 self.assertEqual(result, 3.0)
207 result = self.mapper.standardize("x", 3.14, None)
208 self.assertIsInstance(result, float)
209 self.assertEqual(result, 3.14)
210 result = self.mapper.standardize("x", "3.14", None)
211 self.assertIsInstance(result, float)
212 self.assertEqual(result, 3.14)
214 def testNames(self):
215 self.assertEqual(MinMapper1.getCameraName(), "min")
216 self.assertEqual(MinMapper1.getPackageName(), "larry")
219class Mapper2TestCase(unittest.TestCase):
220 """A test case for the mapper used by the data butler."""
222 def setUp(self):
223 super().setUp()
224 # Force a standard set of filters even for tests that don't use
225 # MinCam directly.
226 MinCam()
228 def testGetDatasetTypes(self):
229 mapper = MinMapper2(root=ROOT)
230 expectedTypes = BaseMapper(ROOT).getDatasetTypes()
231 # Add the expected additional types to what the base class provides
232 expectedTypes.extend(["flat", "flat_md", "flat_filename", "flat_sub",
233 "raw", "raw_md", "raw_filename", "raw_sub",
234 "some", "some_filename", "some_md", "some_sub",
235 "someCatalog", "someCatalog_md", "someCatalog_filename",
236 "someCatalog_len", "someCatalog_schema",
237 "forced_src", "forced_src_md", "forced_src_filename",
238 "forced_src_len", "forced_src_schema",
239 "other_sub", "other_filename", "other_md", "other",
240 "someGz", "someGz_filename", "someFz", "someFz_filename", "someGz_md",
241 "someFz_sub", "someFz_md", "someGz_sub",
242 "someGz_bbox", "someFz_bbox", "some_bbox", "other_bbox",
243 "someExp", "someExp_filename", "someExp_md", "someExp_sub",
244 "someExp_bbox", "someExp_filterLabel", "someExp_photoCalib",
245 "someExp_visitInfo", "someExp_detector", "someExp_filter",
246 "someExp_header_wcs", "someExp_wcs",
247 ])
248 self.assertEqual(set(mapper.getDatasetTypes()),
249 set(expectedTypes))
251 def testMap(self):
252 mapper = MinMapper2(root=ROOT)
253 loc = mapper.map("raw", {"ccd": 13}, write=True)
254 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
255 self.assertEqual(loc.getCppType(), "ImageU")
256 self.assertEqual(loc.getStorageName(), "FitsStorage")
257 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
258 self.assertEqual(loc.getStorage().root, ROOT)
259 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
260 checkCompression(self, loc.getAdditionalData())
262 def testSubMap(self):
263 bbox = geom.BoxI(geom.Point2I(200, 100),
264 geom.Extent2I(300, 400))
265 mapper = MinMapper2(root=ROOT)
266 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True)
267 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
268 self.assertEqual(loc.getCppType(), "ImageU")
269 self.assertEqual(loc.getStorageName(), "FitsStorage")
270 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
271 self.assertEqual(loc.getStorage().root, ROOT)
272 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
273 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
274 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
275 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
276 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
277 checkCompression(self, loc.getAdditionalData())
279 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox, "imageOrigin": "PARENT"}, write=True)
280 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
281 self.assertEqual(loc.getCppType(), "ImageU")
282 self.assertEqual(loc.getStorageName(), "FitsStorage")
283 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
284 self.assertEqual(loc.getStorage().root, ROOT)
285 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
286 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
287 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
288 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
289 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
290 self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"), "PARENT")
291 checkCompression(self, loc.getAdditionalData())
293 def testCatalogExtras(self):
294 butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2)
295 schema = afwTable.Schema()
296 aa = schema.addField("a", type=np.int32, doc="a")
297 bb = schema.addField("b", type=np.float64, doc="b")
298 catalog = lsst.afw.table.BaseCatalog(schema)
299 row = catalog.addNew()
300 row.set(aa, 12345)
301 row.set(bb, 1.2345)
302 size = len(catalog)
303 dataId = dict(visit=123, ccd=45)
304 butler.put(catalog, "someCatalog", dataId)
305 filename = butler.get("someCatalog_filename", dataId)[0]
306 try:
307 self.assertTrue(os.path.exists(filename))
308 self.assertEqual(butler.get("someCatalog_schema", dataId), schema)
309 self.assertEqual(butler.get("someCatalog_len", dataId), size)
310 header = butler.get("someCatalog_md", dataId)
311 self.assertEqual(header.getScalar("NAXIS2"), size)
312 finally:
313 try:
314 os.remove(filename)
315 except OSError as exc:
316 print("Warning: could not remove file %r: %s" % (filename, exc))
318 def testImage(self):
319 mapper = MinMapper2(root=ROOT)
320 loc = mapper.map("some", dict(ccd=35))
321 expectedLocations = ["bar-35.fits"]
322 self.assertEqual(loc.getStorage().root, ROOT)
323 self.assertEqual(loc.getLocations(), expectedLocations)
325 butler = dafPersist.ButlerFactory(mapper=mapper).create()
326 image = butler.get("some", ccd=35)
327 self.assertEqual(image.getFilter().getName(), "r")
328 self.assertEqual(image.getFilterLabel().bandLabel, "r")
330 self.assertEqual(butler.get("some_bbox", ccd=35), image.getBBox())
332 bbox = geom.BoxI(geom.Point2I(200, 100),
333 geom.Extent2I(300, 400))
334 image = butler.get("some_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
335 self.assertEqual(image.getHeight(), 400)
336 self.assertEqual(image.getWidth(), 300)
338 def testFilter(self):
339 """Test that the same (patched) filter is returned through all Butler
340 retrieval paths.
341 """
342 mapper = MinMapper2(root=ROOT)
344 butler = dafPersist.ButlerFactory(mapper=mapper).create()
345 image = butler.get("someExp", ccd=35)
346 filter = butler.get("someExp_filterLabel", ccd=35)
347 # Test only valid with a complete filter
348 self.assertEqual(image.getFilterLabel(), afwImage.FilterLabel(band="r", physical="r.MP9601"))
349 # Datasets should give consistent answers
350 self.assertEqual(filter, image.getFilterLabel())
352 def testDetector(self):
353 mapper = MinMapper2(root=ROOT)
354 butler = dafPersist.ButlerFactory(mapper=mapper).create()
355 detector = butler.get("raw_detector", ccd=0)
356 self.assertEqual(detector.getName(), "ccd00")
358 def testGzImage(self):
359 mapper = MinMapper2(root=ROOT)
360 loc = mapper.map("someGz", dict(ccd=35))
361 expectedLocations = [os.path.join("gz", "bar-35.fits.gz")]
362 self.assertEqual(loc.getStorage().root, ROOT)
363 self.assertEqual(loc.getLocations(), expectedLocations)
365 butler = dafPersist.ButlerFactory(mapper=mapper).create()
366 image = butler.get("someGz", ccd=35)
367 self.assertEqual(image.getFilter().getName(), "r")
368 self.assertEqual(image.getFilterLabel().bandLabel, "r")
370 bbox = geom.BoxI(geom.Point2I(200, 100),
371 geom.Extent2I(300, 400))
372 image = butler.get("someGz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
373 self.assertEqual(image.getHeight(), 400)
374 self.assertEqual(image.getWidth(), 300)
376 def testFzImage(self):
377 mapper = MinMapper2(root=ROOT)
378 loc = mapper.map("someFz", dict(ccd=35))
379 expectedRoot = ROOT
380 expectedLocations = [os.path.join("fz", "bar-35.fits.fz")]
381 self.assertEqual(loc.getStorage().root, expectedRoot)
382 self.assertEqual(loc.getLocations(), expectedLocations)
384 butler = dafPersist.ButlerFactory(mapper=mapper).create()
385 image = butler.get("someFz", ccd=35)
386 self.assertEqual(image.getFilter().getName(), "r")
387 self.assertEqual(image.getFilterLabel().bandLabel, "r")
389 bbox = geom.BoxI(geom.Point2I(200, 100),
390 geom.Extent2I(300, 400))
391 image = butler.get("someFz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
392 self.assertEqual(image.getHeight(), 400)
393 self.assertEqual(image.getWidth(), 300)
395 def testButlerQueryMetadata(self):
396 mapper = MinMapper2(root=ROOT)
397 butler = dafPersist.ButlerFactory(mapper=mapper).create()
398 kwargs = {"ccd": 35, "filter": "r", "visit": 787731,
399 "taiObs": "2005-04-02T09:24:49.933440000"}
400 self.assertEqual(butler.queryMetadata("other", "visit", **kwargs), [787731])
401 self.assertEqual(butler.queryMetadata("other", "visit",
402 visit=kwargs["visit"], ccd=kwargs["ccd"],
403 taiObs=kwargs["taiObs"], filter=kwargs["filter"]),
404 [787731])
405 # now test we get no matches if ccd is out of range
406 self.assertEqual(butler.queryMetadata("raw", "ccd", ccd=36, filter="r", visit=787731), [])
408 def testQueryMetadata(self):
409 mapper = MinMapper2(root=ROOT)
410 self.assertEqual(mapper.queryMetadata("raw", ["ccd"], None),
411 [(x,) for x in range(36) if x != 3])
413 def testStandardize(self):
414 mapper = MinMapper2(root=ROOT)
415 self.assertEqual(mapper.canStandardize("raw"), True)
416 self.assertEqual(mapper.canStandardize("notPresent"), False)
418 def testStandardizeFiltersFilterDefs(self):
419 testLabels = [
420 (None, None),
421 (afwImage.FilterLabel(band="i", physical="i.MP9701"),
422 afwImage.FilterLabel(band="i", physical="i.MP9701")),
423 (afwImage.FilterLabel(band="i"), afwImage.FilterLabel(band="r", physical="i.MP9701")),
424 (afwImage.FilterLabel(physical="i.MP9701"),
425 afwImage.FilterLabel(band="i", physical="i.MP9701")),
426 (afwImage.FilterLabel(band="i", physical="old-i"),
427 afwImage.FilterLabel(band="i", physical="i.MP9701")),
428 (afwImage.FilterLabel(physical="old-i"),
429 afwImage.FilterLabel(band="i", physical="i.MP9701")),
430 (afwImage.FilterLabel(physical="i2"), afwImage.FilterLabel(band="i", physical="HSC-I2")),
431 ]
432 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in {
433 "i", "i.MP9701", "old-i", "i2",
434 }]
435 testData = []
436 # Resolve special combinations where the expected output is different
437 for input, corrected in testLabels:
438 for dataId in testIds:
439 if input is None:
440 if dataId["filter"] == "i":
441 data = (input, dataId, afwImage.FilterLabel(band="i"))
442 elif dataId["filter"] == "i2":
443 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
444 else:
445 data = (input, dataId, afwImage.FilterLabel(band="i", physical="i.MP9701"))
446 elif input == afwImage.FilterLabel(band="i"):
447 if dataId["filter"] == "i":
448 # There are two "i" filters, can't tell which
449 data = (input, dataId, input)
450 elif dataId["filter"] == "i2":
451 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
452 elif corrected.physicalLabel == "HSC-I2" and dataId["filter"] in ("i.MP9701", "old-i"):
453 # Contradictory inputs, leave as-is
454 data = (input, dataId, input)
455 elif corrected.physicalLabel == "i.MP9701" and dataId["filter"] == "i2":
456 # Contradictory inputs, leave as-is
457 data = (input, dataId, input)
458 else:
459 data = (input, dataId, corrected)
460 testData.append(data)
462 mapper = MinMapper2(root=ROOT)
463 for label, dataId, corrected in testData:
464 exposure = afwImage.ExposureF()
465 exposure.setFilterLabel(label)
466 mapper._setFilter(mapper.exposures['raw'], exposure, dataId)
467 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}")
469 def testStandardizeFiltersFilterNoDefs(self):
470 testLabels = [
471 None,
472 afwImage.FilterLabel(band="i", physical="i.MP9701"),
473 afwImage.FilterLabel(band="i"),
474 afwImage.FilterLabel(physical="i.MP9701"),
475 afwImage.FilterLabel(band="i", physical="old-i"),
476 afwImage.FilterLabel(physical="old-i"),
477 afwImage.FilterLabel(physical="i2"),
478 ]
479 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in {
480 "i", "i.MP9701", "old-i", "i2",
481 }]
482 testData = []
483 # Resolve special combinations where the expected output is different
484 for input in testLabels:
485 for dataId in testIds:
486 if input is None:
487 # Can still get some filter info out of the Filter registry
488 if dataId["filter"] == "i2":
489 data = (input, dataId,
490 afwImage.FilterLabel(band="i", physical="HSC-I2"))
491 else:
492 # Data ID maps to filter(s) with aliases; can't
493 # unambiguously determine physical filter.
494 data = (input, dataId, afwImage.FilterLabel(band="i"))
495 else:
496 data = (input, dataId, input)
497 testData.append(data)
499 mapper = MinMapper1(root=ROOT)
500 for label, dataId, corrected in testData:
501 exposure = afwImage.ExposureF()
502 exposure.setFilterLabel(label)
503 mapper._setFilter(mapper.exposures['raw'], exposure, dataId)
504 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}")
506 def testCalib(self):
507 mapper = MinMapper2(root=ROOT)
508 loc = mapper.map("flat", {"visit": 787650, "ccd": 13}, write=True)
509 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureF")
510 self.assertEqual(loc.getCppType(), "ExposureF")
511 self.assertEqual(loc.getStorageName(), "FitsStorage")
512 expectedRoot = ROOT
513 expectedLocations = ["flat-05Am03-fi.fits"]
514 self.assertEqual(loc.getStorage().root, expectedRoot)
515 self.assertEqual(loc.getLocations(), expectedLocations)
516 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
517 self.assertEqual(loc.getAdditionalData().getScalar("visit"), 787650)
518 self.assertEqual(loc.getAdditionalData().getScalar("derivedRunId"), "05Am03")
519 self.assertEqual(loc.getAdditionalData().getScalar("filter"), "i")
520 checkCompression(self, loc.getAdditionalData())
522 def testNames(self):
523 self.assertEqual(MinMapper2.getCameraName(), "min")
524 self.assertEqual(MinMapper2.getPackageName(), "moe")
526 @unittest.expectedFailure
527 def testParentSearch(self):
528 mapper = MinMapper2(root=ROOT)
529 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'),
530 os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits')))
531 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits'))])
532 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'),
533 os.path.join(ROOT,
534 os.path.join('testParentSearch', 'bar.fits[1]')))
535 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits[1]'))])
537 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'),
538 os.path.join(ROOT, os.path.join('testParentSearch', 'baz.fits')))
539 self.assertEqual(paths, [os.path.join(ROOT,
540 os.path.join('testParentSearch', '_parent', 'baz.fits'))])
541 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'),
542 os.path.join(ROOT,
543 os.path.join('testParentSearch', 'baz.fits[1]')))
544 self.assertEqual(paths, [os.path.join(ROOT,
545 os.path.join('testParentSearch', '_parent', 'baz.fits[1]'))])
547 def testSkymapLookups(self):
548 """Test that metadata lookups don't try to get skymap data ID values
549 from the registry.
550 """
551 mapper = MinMapper2(root=ROOT)
552 butler = dafPersist.Butler(mapper=mapper)
553 with self.assertRaises(RuntimeError) as manager:
554 butler.dataRef("forced_src", visit=787650, ccd=13)
555 self.assertIn("Cannot lookup skymap key 'tract'", str(manager.exception))
556 # We're mostly concerned that the statements below will raise an
557 # exception; if they don't, it's not likely the following tests will
558 # fail.
559 subset = butler.subset("forced_src", visit=787650, ccd=13, tract=0)
560 self.assertEqual(len(subset), 1)
561 dataRef = butler.dataRef("forced_src", visit=787650, ccd=13, tract=0)
562 self.assertFalse(dataRef.datasetExists("forced_src"))
565class Mapper3TestCase(unittest.TestCase):
566 """A test case for a mapper subclass which does not assign packageName."""
568 def testPackageName(self):
569 with self.assertRaises(ValueError):
570 MinMapper3()
571 with self.assertRaises(ValueError):
572 MinMapper3.getPackageName()
575class ParentRegistryTestCase(unittest.TestCase):
577 @staticmethod
578 def _createRegistry(path):
579 cmd = """CREATE TABLE x(
580 id INT,
581 visit INT,
582 filter TEXT,
583 snap INT,
584 raft TEXT,
585 sensor TEXT,
586 channel TEXT,
587 taiObs TEXT,
588 expTime REAL
589 );
590 """
591 conn = sqlite3.connect(path)
592 conn.cursor().execute(cmd)
593 conn.commit()
594 conn.close()
596 def setUp(self):
597 self.ROOT = tempfile.mkdtemp(dir=ROOT, prefix="ParentRegistryTestCase-")
598 self.repoARoot = os.path.join(self.ROOT, 'a')
599 args = dafPersist.RepositoryArgs(root=self.repoARoot, mapper=MinMapper1)
600 butler = dafPersist.Butler(outputs=args)
601 self._createRegistry(os.path.join(self.repoARoot, 'registry.sqlite3'))
602 del butler
604 def tearDown(self):
605 # the butler sql registry closes its database connection in __del__.
606 # To trigger __del__ we explicitly collect the garbage here. If we
607 # find having or closing the open database connection is a problem in
608 # production code, we may need to add api to butler to explicity
609 # release database connections (and maybe other things like in-memory
610 # cached objects).
611 gc.collect()
612 if os.path.exists(self.ROOT):
613 shutil.rmtree(self.ROOT)
615 def test(self):
616 """Verify that when the child repo does not have a registry it is
617 assigned the registry from the parent.
618 """
619 repoBRoot = os.path.join(self.ROOT, 'b')
620 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot)
621 # This way of getting the registry from the mapping is obviously going
622 # way into private members and the python lambda implementation code.
623 # It is very brittle and should not be duplicated in user code
624 # or any location that is not trivial to fix along with changes to the
625 # CameraMapper or Mapping.
626 registryA = butler._repos.inputs()[0].repo._mapper.registry
627 registryB = butler._repos.outputs()[0].repo._mapper.registry
628 self.assertEqual(id(registryA), id(registryB))
630 self._createRegistry(os.path.join(repoBRoot, 'registry.sqlite3'))
631 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot)
632 # see above; don't copy this way of getting the registry.
633 registryA = butler._repos.inputs()[0].repo._mapper.registry
634 registryB = butler._repos.outputs()[0].repo._mapper.registry
635 self.assertNotEqual(id(registryA), id(registryB))
638class MissingPolicyKeyTestCase(unittest.TestCase):
640 def testGetRaises(self):
641 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1})
642 # MinMapper1 does not specify a template for the raw dataset type so
643 # trying to use it for get should raise
644 with self.assertRaises(RuntimeError) as contextManager:
645 butler.get('raw')
646 # This test demonstrates and verifies that simple use of the incomplete
647 # dataset type returns a helpful (I hope) error message.
648 self.assertEqual(
649 str(contextManager.exception),
650 'Template is not defined for the raw dataset type, '
651 'it must be set before it can be used.')
652 with self.assertRaises(RuntimeError) as contextManager:
653 butler.queryMetadata('raw', 'unused', {})
655 def testQueryMetadataRaises(self):
656 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1})
657 # MinMapper1 does not specify a template for the raw dataset type so
658 # trying to use it for queryMetadata should raise
659 with self.assertRaises(RuntimeError) as contextManager:
660 butler.queryMetadata('raw', 'unused', {})
661 # This test demonstrates and verifies that simple use of the incomplete
662 # dataset type returns a helpful (I hope) error message.
663 self.assertEqual(
664 str(contextManager.exception),
665 'Template is not defined for the raw dataset type, '
666 'it must be set before it can be used.')
668 def testFilenameRaises(self):
669 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1})
670 # MinMapper1 does not specify a template for the raw dataset type so
671 # trying to use it for <datasetType>_filename should raise
672 with self.assertRaises(RuntimeError) as contextManager:
673 butler.get('raw_filename')
674 # This test demonstrates and verifies that simple use of the
675 # incomplete dataset type returns a helpful (I hope) error message.
676 self.assertEqual(
677 str(contextManager.exception),
678 'Template is not defined for the raw dataset type, '
679 'it must be set before it can be used.')
681 def testWcsRaises(self):
682 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1})
683 # MinMapper1 does not specify a template for the raw dataset type so
684 # trying to use it for <datasetType>_wcs should raise
685 with self.assertRaises(RuntimeError) as contextManager:
686 butler.get('raw_wcs')
687 # This test demonstrates and verifies that simple use of the
688 # incomplete dataset type returns a helpful (I hope) error message.
689 self.assertEqual(
690 str(contextManager.exception),
691 'Template is not defined for the raw dataset type, '
692 'it must be set before it can be used.')
694 def testConflictRaises(self):
695 policy = dafPersist.Policy(os.path.join(ROOT, "ConflictMapper.yaml"))
696 with self.assertRaisesRegex(
697 ValueError,
698 r"Duplicate mapping policy for dataset type packages"):
699 mapper = lsst.obs.base.CameraMapper(policy=policy, repositoryDir=ROOT, root=ROOT) # noqa F841
702class MemoryTester(lsst.utils.tests.MemoryTestCase):
703 pass
706if __name__ == '__main__': 706 ↛ 707line 706 didn't jump to line 707, because the condition on line 706 was never true
707 lsst.utils.tests.init()
708 unittest.main()