Coverage for tests/test_cameraMapper.py: 25%
397 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-23 02:50 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-23 02:50 -0700
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import gc
23import os
24import shutil
25import sqlite3
26import tempfile
27import unittest
29import lsst.afw.image as afwImage
30import lsst.afw.table as afwTable
31import lsst.daf.persistence as dafPersist
32import lsst.geom as geom
33import lsst.obs.base
34import lsst.utils.tests
35import numpy as np
36from lsst.obs.base.test import BaseMapper
38ROOT = os.path.abspath(os.path.dirname(__file__))
41def setup_module(module):
42 lsst.utils.tests.init()
45class MinCam(lsst.obs.base.Instrument):
46 @property
47 def filterDefinitions(self):
48 return lsst.obs.base.FilterDefinitionCollection(
49 lsst.obs.base.FilterDefinition(physical_filter="u.MP9301", band="u"),
50 lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g"),
51 lsst.obs.base.FilterDefinition(physical_filter="r.MP9601", band="r", alias={"old-r"}),
52 lsst.obs.base.FilterDefinition(physical_filter="i.MP9701", band="i", alias={"old-i"}),
53 lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z"),
54 # afw_name is so special-cased that only a real example will work
55 lsst.obs.base.FilterDefinition(physical_filter="HSC-I2", band="i", afw_name="i2"),
56 )
58 @classmethod
59 def getName(cls):
60 return "min"
62 def getCamera(self):
63 raise NotImplementedError()
65 def register(self, registry):
66 raise NotImplementedError()
68 def getRawFormatter(self, dataId):
69 raise NotImplementedError()
71 def makeDataIdTranslatorFactory(self):
72 raise NotImplementedError()
75class MinMapper1(lsst.obs.base.CameraMapper):
76 packageName = "larry"
78 def __init__(self, **kwargs):
79 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml"))
80 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs)
81 return
83 def std_x(self, item, dataId):
84 return float(item)
86 @classmethod
87 def getCameraName(cls):
88 """Return the name of the camera that this CameraMapper is for."""
89 return "min"
91 @classmethod
92 def getPackageDir(cls):
93 return "/path/to/nowhere"
96class MinMapper2(lsst.obs.base.CameraMapper):
97 packageName = "moe"
98 _gen3instrument = MinCam
100 # CalibRoot in policy
101 # needCalibRegistry
102 def __init__(self, **kwargs):
103 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper2.yaml"))
104 lsst.obs.base.CameraMapper.__init__(
105 self, policy=policy, repositoryDir=ROOT, registry="cfhtls.sqlite3", **kwargs
106 )
107 return
109 def _transformId(self, dataId):
110 return dataId
112 def _extractDetectorName(self, dataId):
113 return "ccd00"
115 def std_x(self, item, dataId):
116 return float(item)
118 @classmethod
119 def getCameraName(cls):
120 """Return the name of the camera that this CameraMapper is for."""
121 return "min"
123 @classmethod
124 def getPackageDir(cls):
125 return "/path/to/nowhere"
128# does not assign packageName
129class MinMapper3(lsst.obs.base.CameraMapper):
130 def __init__(self, **kwargs):
131 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml"))
132 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, root=ROOT)
133 return
135 @classmethod
136 def getPackageDir(cls):
137 return "/path/to/nowhere"
140def checkCompression(testCase, additionalData):
141 """Check that compression settings are present
143 We check that we can access the required settings, and that
144 the seed is non-zero (zero causes lsst.afw.math.Random to fail).
145 """
146 for plane in ("image", "mask", "variance"):
147 for entry in (
148 "compression.algorithm",
149 "compression.columns",
150 "compression.rows",
151 "compression.quantizeLevel",
152 "scaling.algorithm",
153 "scaling.bitpix",
154 "scaling.maskPlanes",
155 "scaling.seed",
156 "scaling.quantizeLevel",
157 "scaling.quantizePad",
158 "scaling.fuzz",
159 "scaling.bscale",
160 "scaling.bzero",
161 ):
162 additionalData.getScalar(plane + "." + entry)
163 testCase.assertNotEqual(additionalData.getScalar(plane + ".scaling.seed"), 0)
166class Mapper1TestCase(unittest.TestCase):
167 """A test case for the mapper used by the data butler."""
169 def setUp(self):
170 self.mapper = MinMapper1(root=ROOT)
172 def tearDown(self):
173 del self.mapper
175 def testGetDatasetTypes(self):
176 expectedTypes = BaseMapper(ROOT).getDatasetTypes()
177 # Add the expected additional types to what the base class provides
178 expectedTypes.extend(
179 [
180 "x",
181 "x_filename",
182 "badSourceHist",
183 "badSourceHist_filename",
184 ]
185 )
186 self.assertEqual(set(self.mapper.getDatasetTypes()), set(expectedTypes))
188 def testMap(self):
189 loc = self.mapper.map("x", {"sensor": "1,1"}, write=True)
190 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI")
191 self.assertEqual(loc.getCppType(), "BoxI")
192 self.assertEqual(loc.getStorageName(), "PickleStorage")
193 expectedRoot = ROOT
194 expectedLocations = ["foo-1,1.pickle"]
195 self.assertEqual(loc.getStorage().root, expectedRoot)
196 self.assertEqual(loc.getLocations(), expectedLocations)
197 self.assertEqual(loc.getAdditionalData().toString(), 'sensor = "1,1"\n')
199 def testQueryMetadata(self):
200 self.assertEqual(self.mapper.queryMetadata("x", ["sensor"], None), [("1,1",)])
202 def testStandardize(self):
203 self.assertTrue(self.mapper.canStandardize("x"))
204 self.assertFalse(self.mapper.canStandardize("badSourceHist"))
205 self.assertFalse(self.mapper.canStandardize("notPresent"))
206 result = self.mapper.standardize("x", 3, None)
207 self.assertIsInstance(result, float)
208 self.assertEqual(result, 3.0)
209 result = self.mapper.standardize("x", 3.14, None)
210 self.assertIsInstance(result, float)
211 self.assertEqual(result, 3.14)
212 result = self.mapper.standardize("x", "3.14", None)
213 self.assertIsInstance(result, float)
214 self.assertEqual(result, 3.14)
216 def testNames(self):
217 self.assertEqual(MinMapper1.getCameraName(), "min")
218 self.assertEqual(MinMapper1.getPackageName(), "larry")
221class Mapper2TestCase(unittest.TestCase):
222 """A test case for the mapper used by the data butler."""
224 def setUp(self):
225 super().setUp()
226 # Force a standard set of filters even for tests that don't use
227 # MinCam directly.
228 MinCam()
230 def testGetDatasetTypes(self):
231 mapper = MinMapper2(root=ROOT)
232 expectedTypes = BaseMapper(ROOT).getDatasetTypes()
233 # Add the expected additional types to what the base class provides
234 expectedTypes.extend(
235 [
236 "flat",
237 "flat_md",
238 "flat_filename",
239 "flat_sub",
240 "raw",
241 "raw_md",
242 "raw_filename",
243 "raw_sub",
244 "some",
245 "some_filename",
246 "some_md",
247 "some_sub",
248 "someCatalog",
249 "someCatalog_md",
250 "someCatalog_filename",
251 "someCatalog_len",
252 "someCatalog_schema",
253 "forced_src",
254 "forced_src_md",
255 "forced_src_filename",
256 "forced_src_len",
257 "forced_src_schema",
258 "other_sub",
259 "other_filename",
260 "other_md",
261 "other",
262 "someGz",
263 "someGz_filename",
264 "someFz",
265 "someFz_filename",
266 "someGz_md",
267 "someFz_sub",
268 "someFz_md",
269 "someGz_sub",
270 "someGz_bbox",
271 "someFz_bbox",
272 "some_bbox",
273 "other_bbox",
274 "someExp",
275 "someExp_filename",
276 "someExp_md",
277 "someExp_sub",
278 "someExp_bbox",
279 "someExp_filterLabel",
280 "someExp_photoCalib",
281 "someExp_visitInfo",
282 "someExp_detector",
283 "someExp_filter",
284 "someExp_header_wcs",
285 "someExp_wcs",
286 ]
287 )
288 self.assertEqual(set(mapper.getDatasetTypes()), set(expectedTypes))
290 def testMap(self):
291 mapper = MinMapper2(root=ROOT)
292 loc = mapper.map("raw", {"ccd": 13}, write=True)
293 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
294 self.assertEqual(loc.getCppType(), "ImageU")
295 self.assertEqual(loc.getStorageName(), "FitsStorage")
296 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
297 self.assertEqual(loc.getStorage().root, ROOT)
298 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
299 checkCompression(self, loc.getAdditionalData())
301 def testSubMap(self):
302 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
303 mapper = MinMapper2(root=ROOT)
304 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True)
305 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
306 self.assertEqual(loc.getCppType(), "ImageU")
307 self.assertEqual(loc.getStorageName(), "FitsStorage")
308 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
309 self.assertEqual(loc.getStorage().root, ROOT)
310 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
311 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
312 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
313 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
314 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
315 checkCompression(self, loc.getAdditionalData())
317 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox, "imageOrigin": "PARENT"}, write=True)
318 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
319 self.assertEqual(loc.getCppType(), "ImageU")
320 self.assertEqual(loc.getStorageName(), "FitsStorage")
321 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
322 self.assertEqual(loc.getStorage().root, ROOT)
323 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
324 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
325 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
326 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
327 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
328 self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"), "PARENT")
329 checkCompression(self, loc.getAdditionalData())
331 def testCatalogExtras(self):
332 butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2)
333 schema = afwTable.Schema()
334 aa = schema.addField("a", type=np.int32, doc="a")
335 bb = schema.addField("b", type=np.float64, doc="b")
336 catalog = lsst.afw.table.BaseCatalog(schema)
337 row = catalog.addNew()
338 row.set(aa, 12345)
339 row.set(bb, 1.2345)
340 size = len(catalog)
341 dataId = dict(visit=123, ccd=45)
342 butler.put(catalog, "someCatalog", dataId)
343 filename = butler.get("someCatalog_filename", dataId)[0]
344 try:
345 self.assertTrue(os.path.exists(filename))
346 self.assertEqual(butler.get("someCatalog_schema", dataId), schema)
347 self.assertEqual(butler.get("someCatalog_len", dataId), size)
348 header = butler.get("someCatalog_md", dataId)
349 self.assertEqual(header.getScalar("NAXIS2"), size)
350 finally:
351 try:
352 os.remove(filename)
353 except OSError as exc:
354 print("Warning: could not remove file %r: %s" % (filename, exc))
356 def testImage(self):
357 mapper = MinMapper2(root=ROOT)
358 loc = mapper.map("some", dict(ccd=35))
359 expectedLocations = ["bar-35.fits"]
360 self.assertEqual(loc.getStorage().root, ROOT)
361 self.assertEqual(loc.getLocations(), expectedLocations)
363 butler = dafPersist.ButlerFactory(mapper=mapper).create()
364 image = butler.get("some", ccd=35)
365 self.assertEqual(image.getFilter().bandLabel, "r")
367 self.assertEqual(butler.get("some_bbox", ccd=35), image.getBBox())
369 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
370 image = butler.get("some_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
371 self.assertEqual(image.getHeight(), 400)
372 self.assertEqual(image.getWidth(), 300)
374 def testFilter(self):
375 """Test that the same (patched) filter is returned through all Butler
376 retrieval paths.
377 """
378 mapper = MinMapper2(root=ROOT)
380 butler = dafPersist.ButlerFactory(mapper=mapper).create()
381 image = butler.get("someExp", ccd=35)
382 filter = butler.get("someExp_filter", ccd=35)
383 # Test only valid with a complete filter
384 self.assertEqual(image.getFilter(), afwImage.FilterLabel(band="r", physical="r.MP9601"))
385 # Datasets should give consistent answers
386 self.assertEqual(filter, image.getFilter())
388 def testDetector(self):
389 mapper = MinMapper2(root=ROOT)
390 butler = dafPersist.ButlerFactory(mapper=mapper).create()
391 detector = butler.get("raw_detector", ccd=0)
392 self.assertEqual(detector.getName(), "ccd00")
394 def testGzImage(self):
395 mapper = MinMapper2(root=ROOT)
396 loc = mapper.map("someGz", dict(ccd=35))
397 expectedLocations = [os.path.join("gz", "bar-35.fits.gz")]
398 self.assertEqual(loc.getStorage().root, ROOT)
399 self.assertEqual(loc.getLocations(), expectedLocations)
401 butler = dafPersist.ButlerFactory(mapper=mapper).create()
402 image = butler.get("someGz", ccd=35)
403 self.assertEqual(image.getFilter().bandLabel, "r")
405 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
406 image = butler.get("someGz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
407 self.assertEqual(image.getHeight(), 400)
408 self.assertEqual(image.getWidth(), 300)
410 def testFzImage(self):
411 mapper = MinMapper2(root=ROOT)
412 loc = mapper.map("someFz", dict(ccd=35))
413 expectedRoot = ROOT
414 expectedLocations = [os.path.join("fz", "bar-35.fits.fz")]
415 self.assertEqual(loc.getStorage().root, expectedRoot)
416 self.assertEqual(loc.getLocations(), expectedLocations)
418 butler = dafPersist.ButlerFactory(mapper=mapper).create()
419 image = butler.get("someFz", ccd=35)
420 self.assertEqual(image.getFilter().bandLabel, "r")
422 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
423 image = butler.get("someFz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
424 self.assertEqual(image.getHeight(), 400)
425 self.assertEqual(image.getWidth(), 300)
427 def testButlerQueryMetadata(self):
428 mapper = MinMapper2(root=ROOT)
429 butler = dafPersist.ButlerFactory(mapper=mapper).create()
430 kwargs = {"ccd": 35, "filter": "r", "visit": 787731, "taiObs": "2005-04-02T09:24:49.933440000"}
431 self.assertEqual(butler.queryMetadata("other", "visit", **kwargs), [787731])
432 self.assertEqual(
433 butler.queryMetadata(
434 "other",
435 "visit",
436 visit=kwargs["visit"],
437 ccd=kwargs["ccd"],
438 taiObs=kwargs["taiObs"],
439 filter=kwargs["filter"],
440 ),
441 [787731],
442 )
443 # now test we get no matches if ccd is out of range
444 self.assertEqual(butler.queryMetadata("raw", "ccd", ccd=36, filter="r", visit=787731), [])
446 def testQueryMetadata(self):
447 mapper = MinMapper2(root=ROOT)
448 self.assertEqual(mapper.queryMetadata("raw", ["ccd"], None), [(x,) for x in range(36) if x != 3])
450 def testStandardize(self):
451 mapper = MinMapper2(root=ROOT)
452 self.assertEqual(mapper.canStandardize("raw"), True)
453 self.assertEqual(mapper.canStandardize("notPresent"), False)
455 def testStandardizeFiltersFilterDefs(self):
456 # tuples are (input, desired output)
457 testLabels = [
458 (None, None),
459 (
460 afwImage.FilterLabel(band="i", physical="i.MP9701"),
461 afwImage.FilterLabel(band="i", physical="i.MP9701"),
462 ),
463 (afwImage.FilterLabel(band="i"), afwImage.FilterLabel(band="i", physical="i.MP9701")),
464 (afwImage.FilterLabel(physical="i.MP9701"), afwImage.FilterLabel(band="i", physical="i.MP9701")),
465 (
466 afwImage.FilterLabel(band="i", physical="old-i"),
467 afwImage.FilterLabel(band="i", physical="i.MP9701"),
468 ),
469 (afwImage.FilterLabel(physical="old-i"), afwImage.FilterLabel(band="i", physical="i.MP9701")),
470 (afwImage.FilterLabel(physical="i2"), afwImage.FilterLabel(band="i", physical="HSC-I2")),
471 ]
472 testIds = [
473 {"visit": 12345, "ccd": 42, "filter": f}
474 for f in {
475 "i",
476 "i.MP9701",
477 "old-i",
478 "i2",
479 }
480 ]
481 testData = []
482 # Resolve special combinations where the expected output is different
483 for input, corrected in testLabels:
484 for dataId in testIds:
485 if input is None:
486 if dataId["filter"] == "i2":
487 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
488 else:
489 data = (input, dataId, afwImage.FilterLabel(band="i", physical="i.MP9701"))
490 elif input == afwImage.FilterLabel(band="i") and dataId["filter"] == "i2":
491 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
492 elif corrected.physicalLabel == "HSC-I2" and dataId["filter"] in ("i.MP9701", "old-i"):
493 # Contradictory inputs, leave as-is
494 data = (input, dataId, input)
495 elif corrected.physicalLabel == "i.MP9701" and dataId["filter"] == "i2":
496 # Contradictory inputs, leave as-is
497 data = (input, dataId, input)
498 else:
499 data = (input, dataId, corrected)
500 testData.append(data)
502 mapper = MinMapper2(root=ROOT)
503 for label, dataId, corrected in testData:
504 exposure = afwImage.ExposureF()
505 exposure.setFilter(label)
506 mapper._setFilter(mapper.exposures["raw"], exposure, dataId)
507 self.assertEqual(exposure.getFilter(), corrected, msg=f"Started from {label} and {dataId}")
509 def testStandardizeFiltersFilterNoDefs(self):
510 testLabels = [
511 None,
512 afwImage.FilterLabel(band="i", physical="i.MP9701"),
513 afwImage.FilterLabel(band="i"),
514 afwImage.FilterLabel(physical="i.MP9701"),
515 afwImage.FilterLabel(band="i", physical="old-i"),
516 afwImage.FilterLabel(physical="old-i"),
517 afwImage.FilterLabel(physical="i2"),
518 ]
519 testIds = [
520 {"visit": 12345, "ccd": 42, "filter": f}
521 for f in {
522 "i",
523 "i.MP9701",
524 "old-i",
525 "i2",
526 }
527 ]
528 testData = []
529 # Resolve special combinations where the expected output is different
530 for input in testLabels:
531 for dataId in testIds:
532 # FilterLabel is only source of truth if no FilterDefinitions.
533 data = (input, dataId, input)
534 testData.append(data)
536 mapper = MinMapper1(root=ROOT)
537 for label, dataId, corrected in testData:
538 exposure = afwImage.ExposureF()
539 exposure.setFilter(label)
540 mapper._setFilter(mapper.exposures["raw"], exposure, dataId)
541 self.assertEqual(exposure.getFilter(), corrected, msg=f"Started from {label} and {dataId}")
543 def testCalib(self):
544 mapper = MinMapper2(root=ROOT)
545 loc = mapper.map("flat", {"visit": 787650, "ccd": 13}, write=True)
546 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureF")
547 self.assertEqual(loc.getCppType(), "ExposureF")
548 self.assertEqual(loc.getStorageName(), "FitsStorage")
549 expectedRoot = ROOT
550 expectedLocations = ["flat-05Am03-fi.fits"]
551 self.assertEqual(loc.getStorage().root, expectedRoot)
552 self.assertEqual(loc.getLocations(), expectedLocations)
553 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
554 self.assertEqual(loc.getAdditionalData().getScalar("visit"), 787650)
555 self.assertEqual(loc.getAdditionalData().getScalar("derivedRunId"), "05Am03")
556 self.assertEqual(loc.getAdditionalData().getScalar("filter"), "i")
557 checkCompression(self, loc.getAdditionalData())
559 def testNames(self):
560 self.assertEqual(MinMapper2.getCameraName(), "min")
561 self.assertEqual(MinMapper2.getPackageName(), "moe")
563 @unittest.expectedFailure
564 def testParentSearch(self):
565 mapper = MinMapper2(root=ROOT)
566 paths = mapper.parentSearch(
567 os.path.join(ROOT, "testParentSearch"),
568 os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits")),
569 )
570 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits"))])
571 paths = mapper.parentSearch(
572 os.path.join(ROOT, "testParentSearch"),
573 os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits[1]")),
574 )
575 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits[1]"))])
577 paths = mapper.parentSearch(
578 os.path.join(ROOT, "testParentSearch"),
579 os.path.join(ROOT, os.path.join("testParentSearch", "baz.fits")),
580 )
581 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "_parent", "baz.fits"))])
582 paths = mapper.parentSearch(
583 os.path.join(ROOT, "testParentSearch"),
584 os.path.join(ROOT, os.path.join("testParentSearch", "baz.fits[1]")),
585 )
586 self.assertEqual(
587 paths, [os.path.join(ROOT, os.path.join("testParentSearch", "_parent", "baz.fits[1]"))]
588 )
590 def testSkymapLookups(self):
591 """Test that metadata lookups don't try to get skymap data ID values
592 from the registry.
593 """
594 mapper = MinMapper2(root=ROOT)
595 butler = dafPersist.Butler(mapper=mapper)
596 with self.assertRaises(RuntimeError) as manager:
597 butler.dataRef("forced_src", visit=787650, ccd=13)
598 self.assertIn("Cannot lookup skymap key 'tract'", str(manager.exception))
599 # We're mostly concerned that the statements below will raise an
600 # exception; if they don't, it's not likely the following tests will
601 # fail.
602 subset = butler.subset("forced_src", visit=787650, ccd=13, tract=0)
603 self.assertEqual(len(subset), 1)
604 dataRef = butler.dataRef("forced_src", visit=787650, ccd=13, tract=0)
605 self.assertFalse(dataRef.datasetExists("forced_src"))
608class Mapper3TestCase(unittest.TestCase):
609 """A test case for a mapper subclass which does not assign packageName."""
611 def testPackageName(self):
612 with self.assertRaises(ValueError):
613 MinMapper3()
614 with self.assertRaises(ValueError):
615 MinMapper3.getPackageName()
618class ParentRegistryTestCase(unittest.TestCase):
619 @staticmethod
620 def _createRegistry(path):
621 cmd = """CREATE TABLE x(
622 id INT,
623 visit INT,
624 filter TEXT,
625 snap INT,
626 raft TEXT,
627 sensor TEXT,
628 channel TEXT,
629 taiObs TEXT,
630 expTime REAL
631 );
632 """
633 conn = sqlite3.connect(path)
634 conn.cursor().execute(cmd)
635 conn.commit()
636 conn.close()
638 def setUp(self):
639 self.ROOT = tempfile.mkdtemp(dir=ROOT, prefix="ParentRegistryTestCase-")
640 self.repoARoot = os.path.join(self.ROOT, "a")
641 args = dafPersist.RepositoryArgs(root=self.repoARoot, mapper=MinMapper1)
642 butler = dafPersist.Butler(outputs=args)
643 self._createRegistry(os.path.join(self.repoARoot, "registry.sqlite3"))
644 del butler
646 def tearDown(self):
647 # the butler sql registry closes its database connection in __del__.
648 # To trigger __del__ we explicitly collect the garbage here. If we
649 # find having or closing the open database connection is a problem in
650 # production code, we may need to add api to butler to explicity
651 # release database connections (and maybe other things like in-memory
652 # cached objects).
653 gc.collect()
654 if os.path.exists(self.ROOT):
655 shutil.rmtree(self.ROOT)
657 def test(self):
658 """Verify that when the child repo does not have a registry it is
659 assigned the registry from the parent.
660 """
661 repoBRoot = os.path.join(self.ROOT, "b")
662 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot)
663 # This way of getting the registry from the mapping is obviously going
664 # way into private members and the python lambda implementation code.
665 # It is very brittle and should not be duplicated in user code
666 # or any location that is not trivial to fix along with changes to the
667 # CameraMapper or Mapping.
668 registryA = butler._repos.inputs()[0].repo._mapper.registry
669 registryB = butler._repos.outputs()[0].repo._mapper.registry
670 self.assertEqual(id(registryA), id(registryB))
672 self._createRegistry(os.path.join(repoBRoot, "registry.sqlite3"))
673 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot)
674 # see above; don't copy this way of getting the registry.
675 registryA = butler._repos.inputs()[0].repo._mapper.registry
676 registryB = butler._repos.outputs()[0].repo._mapper.registry
677 self.assertNotEqual(id(registryA), id(registryB))
680class MissingPolicyKeyTestCase(unittest.TestCase):
681 def testGetRaises(self):
682 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1})
683 # MinMapper1 does not specify a template for the raw dataset type so
684 # trying to use it for get should raise
685 with self.assertRaises(RuntimeError) as contextManager:
686 butler.get("raw")
687 # This test demonstrates and verifies that simple use of the incomplete
688 # dataset type returns a helpful (I hope) error message.
689 self.assertEqual(
690 str(contextManager.exception),
691 "Template is not defined for the raw dataset type, it must be set before it can be used.",
692 )
693 with self.assertRaises(RuntimeError) as contextManager:
694 butler.queryMetadata("raw", "unused", {})
696 def testQueryMetadataRaises(self):
697 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1})
698 # MinMapper1 does not specify a template for the raw dataset type so
699 # trying to use it for queryMetadata should raise
700 with self.assertRaises(RuntimeError) as contextManager:
701 butler.queryMetadata("raw", "unused", {})
702 # This test demonstrates and verifies that simple use of the incomplete
703 # dataset type returns a helpful (I hope) error message.
704 self.assertEqual(
705 str(contextManager.exception),
706 "Template is not defined for the raw dataset type, it must be set before it can be used.",
707 )
709 def testFilenameRaises(self):
710 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1})
711 # MinMapper1 does not specify a template for the raw dataset type so
712 # trying to use it for <datasetType>_filename should raise
713 with self.assertRaises(RuntimeError) as contextManager:
714 butler.get("raw_filename")
715 # This test demonstrates and verifies that simple use of the
716 # incomplete dataset type returns a helpful (I hope) error message.
717 self.assertEqual(
718 str(contextManager.exception),
719 "Template is not defined for the raw dataset type, it must be set before it can be used.",
720 )
722 def testWcsRaises(self):
723 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1})
724 # MinMapper1 does not specify a template for the raw dataset type so
725 # trying to use it for <datasetType>_wcs should raise
726 with self.assertRaises(RuntimeError) as contextManager:
727 butler.get("raw_wcs")
728 # This test demonstrates and verifies that simple use of the
729 # incomplete dataset type returns a helpful (I hope) error message.
730 self.assertEqual(
731 str(contextManager.exception),
732 "Template is not defined for the raw dataset type, it must be set before it can be used.",
733 )
735 def testConflictRaises(self):
736 policy = dafPersist.Policy(os.path.join(ROOT, "ConflictMapper.yaml"))
737 with self.assertRaisesRegex(ValueError, r"Duplicate mapping policy for dataset type packages"):
738 mapper = lsst.obs.base.CameraMapper(policy=policy, repositoryDir=ROOT, root=ROOT) # noqa F841
741class MemoryTester(lsst.utils.tests.MemoryTestCase):
742 pass
745if __name__ == "__main__": 745 ↛ 746line 745 didn't jump to line 746, because the condition on line 745 was never true
746 lsst.utils.tests.init()
747 unittest.main()