Coverage for tests/test_cameraMapper.py: 24%
404 statements
« prev ^ index » next coverage.py v6.4, created at 2022-05-27 11:22 +0000
« prev ^ index » next coverage.py v6.4, created at 2022-05-27 11:22 +0000
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import gc
23import os
24import shutil
25import sqlite3
26import tempfile
27import unittest
29import lsst.afw.image as afwImage
30import lsst.afw.table as afwTable
31import lsst.daf.persistence as dafPersist
32import lsst.geom as geom
33import lsst.obs.base
34import lsst.utils.tests
35import numpy as np
36from lsst.obs.base.test import BaseMapper
38ROOT = os.path.abspath(os.path.dirname(__file__))
41def setup_module(module):
42 lsst.utils.tests.init()
45class MinCam(lsst.obs.base.Instrument):
46 @property
47 def filterDefinitions(self):
48 return lsst.obs.base.FilterDefinitionCollection(
49 lsst.obs.base.FilterDefinition(physical_filter="u.MP9301", band="u", lambdaEff=374),
50 lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g", lambdaEff=487),
51 lsst.obs.base.FilterDefinition(
52 physical_filter="r.MP9601", band="r", alias={"old-r"}, lambdaEff=628
53 ),
54 lsst.obs.base.FilterDefinition(
55 physical_filter="i.MP9701", band="i", alias={"old-i"}, lambdaEff=778
56 ),
57 lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z", lambdaEff=1170),
58 # afw_name is so special-cased that only a real example will work
59 lsst.obs.base.FilterDefinition(physical_filter="HSC-I2", band="i", afw_name="i2", lambdaEff=623),
60 )
62 @classmethod
63 def getName(cls):
64 return "min"
66 def getCamera(self):
67 raise NotImplementedError()
69 def register(self, registry):
70 raise NotImplementedError()
72 def getRawFormatter(self, dataId):
73 raise NotImplementedError()
75 def makeDataIdTranslatorFactory(self):
76 raise NotImplementedError()
79class MinMapper1(lsst.obs.base.CameraMapper):
80 packageName = "larry"
82 def __init__(self, **kwargs):
83 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml"))
84 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs)
85 return
87 def std_x(self, item, dataId):
88 return float(item)
90 @classmethod
91 def getCameraName(cls):
92 """Return the name of the camera that this CameraMapper is for."""
93 return "min"
95 @classmethod
96 def getPackageDir(cls):
97 return "/path/to/nowhere"
100class MinMapper2(lsst.obs.base.CameraMapper):
101 packageName = "moe"
102 _gen3instrument = MinCam
104 # CalibRoot in policy
105 # needCalibRegistry
106 def __init__(self, **kwargs):
107 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper2.yaml"))
108 lsst.obs.base.CameraMapper.__init__(
109 self, policy=policy, repositoryDir=ROOT, registry="cfhtls.sqlite3", **kwargs
110 )
111 return
113 def _transformId(self, dataId):
114 return dataId
116 def _extractDetectorName(self, dataId):
117 return "ccd00"
119 def std_x(self, item, dataId):
120 return float(item)
122 @classmethod
123 def getCameraName(cls):
124 """Return the name of the camera that this CameraMapper is for."""
125 return "min"
127 @classmethod
128 def getPackageDir(cls):
129 return "/path/to/nowhere"
132# does not assign packageName
133class MinMapper3(lsst.obs.base.CameraMapper):
134 def __init__(self, **kwargs):
135 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml"))
136 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, root=ROOT)
137 return
139 @classmethod
140 def getPackageDir(cls):
141 return "/path/to/nowhere"
144def checkCompression(testCase, additionalData):
145 """Check that compression settings are present
147 We check that we can access the required settings, and that
148 the seed is non-zero (zero causes lsst.afw.math.Random to fail).
149 """
150 for plane in ("image", "mask", "variance"):
151 for entry in (
152 "compression.algorithm",
153 "compression.columns",
154 "compression.rows",
155 "compression.quantizeLevel",
156 "scaling.algorithm",
157 "scaling.bitpix",
158 "scaling.maskPlanes",
159 "scaling.seed",
160 "scaling.quantizeLevel",
161 "scaling.quantizePad",
162 "scaling.fuzz",
163 "scaling.bscale",
164 "scaling.bzero",
165 ):
166 additionalData.getScalar(plane + "." + entry)
167 testCase.assertNotEqual(additionalData.getScalar(plane + ".scaling.seed"), 0)
170class Mapper1TestCase(unittest.TestCase):
171 """A test case for the mapper used by the data butler."""
173 def setUp(self):
174 self.mapper = MinMapper1(root=ROOT)
176 def tearDown(self):
177 del self.mapper
179 def testGetDatasetTypes(self):
180 expectedTypes = BaseMapper(ROOT).getDatasetTypes()
181 # Add the expected additional types to what the base class provides
182 expectedTypes.extend(
183 [
184 "x",
185 "x_filename",
186 "badSourceHist",
187 "badSourceHist_filename",
188 ]
189 )
190 self.assertEqual(set(self.mapper.getDatasetTypes()), set(expectedTypes))
192 def testMap(self):
193 loc = self.mapper.map("x", {"sensor": "1,1"}, write=True)
194 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI")
195 self.assertEqual(loc.getCppType(), "BoxI")
196 self.assertEqual(loc.getStorageName(), "PickleStorage")
197 expectedRoot = ROOT
198 expectedLocations = ["foo-1,1.pickle"]
199 self.assertEqual(loc.getStorage().root, expectedRoot)
200 self.assertEqual(loc.getLocations(), expectedLocations)
201 self.assertEqual(loc.getAdditionalData().toString(), 'sensor = "1,1"\n')
203 def testQueryMetadata(self):
204 self.assertEqual(self.mapper.queryMetadata("x", ["sensor"], None), [("1,1",)])
206 def testStandardize(self):
207 self.assertTrue(self.mapper.canStandardize("x"))
208 self.assertFalse(self.mapper.canStandardize("badSourceHist"))
209 self.assertFalse(self.mapper.canStandardize("notPresent"))
210 result = self.mapper.standardize("x", 3, None)
211 self.assertIsInstance(result, float)
212 self.assertEqual(result, 3.0)
213 result = self.mapper.standardize("x", 3.14, None)
214 self.assertIsInstance(result, float)
215 self.assertEqual(result, 3.14)
216 result = self.mapper.standardize("x", "3.14", None)
217 self.assertIsInstance(result, float)
218 self.assertEqual(result, 3.14)
220 def testNames(self):
221 self.assertEqual(MinMapper1.getCameraName(), "min")
222 self.assertEqual(MinMapper1.getPackageName(), "larry")
225class Mapper2TestCase(unittest.TestCase):
226 """A test case for the mapper used by the data butler."""
228 def setUp(self):
229 super().setUp()
230 # Force a standard set of filters even for tests that don't use
231 # MinCam directly.
232 MinCam()
234 def testGetDatasetTypes(self):
235 mapper = MinMapper2(root=ROOT)
236 expectedTypes = BaseMapper(ROOT).getDatasetTypes()
237 # Add the expected additional types to what the base class provides
238 expectedTypes.extend(
239 [
240 "flat",
241 "flat_md",
242 "flat_filename",
243 "flat_sub",
244 "raw",
245 "raw_md",
246 "raw_filename",
247 "raw_sub",
248 "some",
249 "some_filename",
250 "some_md",
251 "some_sub",
252 "someCatalog",
253 "someCatalog_md",
254 "someCatalog_filename",
255 "someCatalog_len",
256 "someCatalog_schema",
257 "forced_src",
258 "forced_src_md",
259 "forced_src_filename",
260 "forced_src_len",
261 "forced_src_schema",
262 "other_sub",
263 "other_filename",
264 "other_md",
265 "other",
266 "someGz",
267 "someGz_filename",
268 "someFz",
269 "someFz_filename",
270 "someGz_md",
271 "someFz_sub",
272 "someFz_md",
273 "someGz_sub",
274 "someGz_bbox",
275 "someFz_bbox",
276 "some_bbox",
277 "other_bbox",
278 "someExp",
279 "someExp_filename",
280 "someExp_md",
281 "someExp_sub",
282 "someExp_bbox",
283 "someExp_filterLabel",
284 "someExp_photoCalib",
285 "someExp_visitInfo",
286 "someExp_detector",
287 "someExp_filter",
288 "someExp_header_wcs",
289 "someExp_wcs",
290 ]
291 )
292 self.assertEqual(set(mapper.getDatasetTypes()), set(expectedTypes))
294 def testMap(self):
295 mapper = MinMapper2(root=ROOT)
296 loc = mapper.map("raw", {"ccd": 13}, write=True)
297 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
298 self.assertEqual(loc.getCppType(), "ImageU")
299 self.assertEqual(loc.getStorageName(), "FitsStorage")
300 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
301 self.assertEqual(loc.getStorage().root, ROOT)
302 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
303 checkCompression(self, loc.getAdditionalData())
305 def testSubMap(self):
306 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
307 mapper = MinMapper2(root=ROOT)
308 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True)
309 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
310 self.assertEqual(loc.getCppType(), "ImageU")
311 self.assertEqual(loc.getStorageName(), "FitsStorage")
312 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
313 self.assertEqual(loc.getStorage().root, ROOT)
314 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
315 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
316 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
317 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
318 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
319 checkCompression(self, loc.getAdditionalData())
321 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox, "imageOrigin": "PARENT"}, write=True)
322 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
323 self.assertEqual(loc.getCppType(), "ImageU")
324 self.assertEqual(loc.getStorageName(), "FitsStorage")
325 self.assertEqual(loc.getLocations(), ["foo-13.fits"])
326 self.assertEqual(loc.getStorage().root, ROOT)
327 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
328 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
329 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
330 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
331 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
332 self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"), "PARENT")
333 checkCompression(self, loc.getAdditionalData())
335 def testCatalogExtras(self):
336 butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2)
337 schema = afwTable.Schema()
338 aa = schema.addField("a", type=np.int32, doc="a")
339 bb = schema.addField("b", type=np.float64, doc="b")
340 catalog = lsst.afw.table.BaseCatalog(schema)
341 row = catalog.addNew()
342 row.set(aa, 12345)
343 row.set(bb, 1.2345)
344 size = len(catalog)
345 dataId = dict(visit=123, ccd=45)
346 butler.put(catalog, "someCatalog", dataId)
347 filename = butler.get("someCatalog_filename", dataId)[0]
348 try:
349 self.assertTrue(os.path.exists(filename))
350 self.assertEqual(butler.get("someCatalog_schema", dataId), schema)
351 self.assertEqual(butler.get("someCatalog_len", dataId), size)
352 header = butler.get("someCatalog_md", dataId)
353 self.assertEqual(header.getScalar("NAXIS2"), size)
354 finally:
355 try:
356 os.remove(filename)
357 except OSError as exc:
358 print("Warning: could not remove file %r: %s" % (filename, exc))
360 def testImage(self):
361 mapper = MinMapper2(root=ROOT)
362 loc = mapper.map("some", dict(ccd=35))
363 expectedLocations = ["bar-35.fits"]
364 self.assertEqual(loc.getStorage().root, ROOT)
365 self.assertEqual(loc.getLocations(), expectedLocations)
367 butler = dafPersist.ButlerFactory(mapper=mapper).create()
368 image = butler.get("some", ccd=35)
369 self.assertEqual(image.getFilter().getName(), "r")
370 self.assertEqual(image.getFilterLabel().bandLabel, "r")
372 self.assertEqual(butler.get("some_bbox", ccd=35), image.getBBox())
374 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
375 image = butler.get("some_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
376 self.assertEqual(image.getHeight(), 400)
377 self.assertEqual(image.getWidth(), 300)
379 def testFilter(self):
380 """Test that the same (patched) filter is returned through all Butler
381 retrieval paths.
382 """
383 mapper = MinMapper2(root=ROOT)
385 butler = dafPersist.ButlerFactory(mapper=mapper).create()
386 image = butler.get("someExp", ccd=35)
387 filter = butler.get("someExp_filterLabel", ccd=35)
388 # Test only valid with a complete filter
389 self.assertEqual(image.getFilterLabel(), afwImage.FilterLabel(band="r", physical="r.MP9601"))
390 # Datasets should give consistent answers
391 self.assertEqual(filter, image.getFilterLabel())
393 def testDetector(self):
394 mapper = MinMapper2(root=ROOT)
395 butler = dafPersist.ButlerFactory(mapper=mapper).create()
396 detector = butler.get("raw_detector", ccd=0)
397 self.assertEqual(detector.getName(), "ccd00")
399 def testGzImage(self):
400 mapper = MinMapper2(root=ROOT)
401 loc = mapper.map("someGz", dict(ccd=35))
402 expectedLocations = [os.path.join("gz", "bar-35.fits.gz")]
403 self.assertEqual(loc.getStorage().root, ROOT)
404 self.assertEqual(loc.getLocations(), expectedLocations)
406 butler = dafPersist.ButlerFactory(mapper=mapper).create()
407 image = butler.get("someGz", ccd=35)
408 self.assertEqual(image.getFilter().getName(), "r")
409 self.assertEqual(image.getFilterLabel().bandLabel, "r")
411 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
412 image = butler.get("someGz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
413 self.assertEqual(image.getHeight(), 400)
414 self.assertEqual(image.getWidth(), 300)
416 def testFzImage(self):
417 mapper = MinMapper2(root=ROOT)
418 loc = mapper.map("someFz", dict(ccd=35))
419 expectedRoot = ROOT
420 expectedLocations = [os.path.join("fz", "bar-35.fits.fz")]
421 self.assertEqual(loc.getStorage().root, expectedRoot)
422 self.assertEqual(loc.getLocations(), expectedLocations)
424 butler = dafPersist.ButlerFactory(mapper=mapper).create()
425 image = butler.get("someFz", ccd=35)
426 self.assertEqual(image.getFilter().getName(), "r")
427 self.assertEqual(image.getFilterLabel().bandLabel, "r")
429 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
430 image = butler.get("someFz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True)
431 self.assertEqual(image.getHeight(), 400)
432 self.assertEqual(image.getWidth(), 300)
434 def testButlerQueryMetadata(self):
435 mapper = MinMapper2(root=ROOT)
436 butler = dafPersist.ButlerFactory(mapper=mapper).create()
437 kwargs = {"ccd": 35, "filter": "r", "visit": 787731, "taiObs": "2005-04-02T09:24:49.933440000"}
438 self.assertEqual(butler.queryMetadata("other", "visit", **kwargs), [787731])
439 self.assertEqual(
440 butler.queryMetadata(
441 "other",
442 "visit",
443 visit=kwargs["visit"],
444 ccd=kwargs["ccd"],
445 taiObs=kwargs["taiObs"],
446 filter=kwargs["filter"],
447 ),
448 [787731],
449 )
450 # now test we get no matches if ccd is out of range
451 self.assertEqual(butler.queryMetadata("raw", "ccd", ccd=36, filter="r", visit=787731), [])
453 def testQueryMetadata(self):
454 mapper = MinMapper2(root=ROOT)
455 self.assertEqual(mapper.queryMetadata("raw", ["ccd"], None), [(x,) for x in range(36) if x != 3])
457 def testStandardize(self):
458 mapper = MinMapper2(root=ROOT)
459 self.assertEqual(mapper.canStandardize("raw"), True)
460 self.assertEqual(mapper.canStandardize("notPresent"), False)
462 def testStandardizeFiltersFilterDefs(self):
463 # tuples are (input, desired output)
464 testLabels = [
465 (None, None),
466 (
467 afwImage.FilterLabel(band="i", physical="i.MP9701"),
468 afwImage.FilterLabel(band="i", physical="i.MP9701"),
469 ),
470 (afwImage.FilterLabel(band="i"), afwImage.FilterLabel(band="i", physical="i.MP9701")),
471 (afwImage.FilterLabel(physical="i.MP9701"), afwImage.FilterLabel(band="i", physical="i.MP9701")),
472 (
473 afwImage.FilterLabel(band="i", physical="old-i"),
474 afwImage.FilterLabel(band="i", physical="i.MP9701"),
475 ),
476 (afwImage.FilterLabel(physical="old-i"), afwImage.FilterLabel(band="i", physical="i.MP9701")),
477 (afwImage.FilterLabel(physical="i2"), afwImage.FilterLabel(band="i", physical="HSC-I2")),
478 ]
479 testIds = [
480 {"visit": 12345, "ccd": 42, "filter": f}
481 for f in {
482 "i",
483 "i.MP9701",
484 "old-i",
485 "i2",
486 }
487 ]
488 testData = []
489 # Resolve special combinations where the expected output is different
490 for input, corrected in testLabels:
491 for dataId in testIds:
492 if input is None:
493 if dataId["filter"] == "i2":
494 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
495 else:
496 data = (input, dataId, afwImage.FilterLabel(band="i", physical="i.MP9701"))
497 elif input == afwImage.FilterLabel(band="i") and dataId["filter"] == "i2":
498 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
499 elif corrected.physicalLabel == "HSC-I2" and dataId["filter"] in ("i.MP9701", "old-i"):
500 # Contradictory inputs, leave as-is
501 data = (input, dataId, input)
502 elif corrected.physicalLabel == "i.MP9701" and dataId["filter"] == "i2":
503 # Contradictory inputs, leave as-is
504 data = (input, dataId, input)
505 else:
506 data = (input, dataId, corrected)
507 testData.append(data)
509 mapper = MinMapper2(root=ROOT)
510 for label, dataId, corrected in testData:
511 exposure = afwImage.ExposureF()
512 exposure.setFilterLabel(label)
513 mapper._setFilter(mapper.exposures["raw"], exposure, dataId)
514 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}")
516 def testStandardizeFiltersFilterNoDefs(self):
517 testLabels = [
518 None,
519 afwImage.FilterLabel(band="i", physical="i.MP9701"),
520 afwImage.FilterLabel(band="i"),
521 afwImage.FilterLabel(physical="i.MP9701"),
522 afwImage.FilterLabel(band="i", physical="old-i"),
523 afwImage.FilterLabel(physical="old-i"),
524 afwImage.FilterLabel(physical="i2"),
525 ]
526 testIds = [
527 {"visit": 12345, "ccd": 42, "filter": f}
528 for f in {
529 "i",
530 "i.MP9701",
531 "old-i",
532 "i2",
533 }
534 ]
535 testData = []
536 # Resolve special combinations where the expected output is different
537 for input in testLabels:
538 for dataId in testIds:
539 if input is None:
540 # Can still get some filter info out of the Filter registry
541 if dataId["filter"] == "i2":
542 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
543 else:
544 # Data ID maps to filter(s) with aliases; can't
545 # unambiguously determine physical filter.
546 data = (input, dataId, afwImage.FilterLabel(band="i"))
547 else:
548 data = (input, dataId, input)
549 testData.append(data)
551 mapper = MinMapper1(root=ROOT)
552 for label, dataId, corrected in testData:
553 exposure = afwImage.ExposureF()
554 exposure.setFilterLabel(label)
555 mapper._setFilter(mapper.exposures["raw"], exposure, dataId)
556 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}")
558 def testCalib(self):
559 mapper = MinMapper2(root=ROOT)
560 loc = mapper.map("flat", {"visit": 787650, "ccd": 13}, write=True)
561 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureF")
562 self.assertEqual(loc.getCppType(), "ExposureF")
563 self.assertEqual(loc.getStorageName(), "FitsStorage")
564 expectedRoot = ROOT
565 expectedLocations = ["flat-05Am03-fi.fits"]
566 self.assertEqual(loc.getStorage().root, expectedRoot)
567 self.assertEqual(loc.getLocations(), expectedLocations)
568 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
569 self.assertEqual(loc.getAdditionalData().getScalar("visit"), 787650)
570 self.assertEqual(loc.getAdditionalData().getScalar("derivedRunId"), "05Am03")
571 self.assertEqual(loc.getAdditionalData().getScalar("filter"), "i")
572 checkCompression(self, loc.getAdditionalData())
574 def testNames(self):
575 self.assertEqual(MinMapper2.getCameraName(), "min")
576 self.assertEqual(MinMapper2.getPackageName(), "moe")
578 @unittest.expectedFailure
579 def testParentSearch(self):
580 mapper = MinMapper2(root=ROOT)
581 paths = mapper.parentSearch(
582 os.path.join(ROOT, "testParentSearch"),
583 os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits")),
584 )
585 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits"))])
586 paths = mapper.parentSearch(
587 os.path.join(ROOT, "testParentSearch"),
588 os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits[1]")),
589 )
590 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits[1]"))])
592 paths = mapper.parentSearch(
593 os.path.join(ROOT, "testParentSearch"),
594 os.path.join(ROOT, os.path.join("testParentSearch", "baz.fits")),
595 )
596 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "_parent", "baz.fits"))])
597 paths = mapper.parentSearch(
598 os.path.join(ROOT, "testParentSearch"),
599 os.path.join(ROOT, os.path.join("testParentSearch", "baz.fits[1]")),
600 )
601 self.assertEqual(
602 paths, [os.path.join(ROOT, os.path.join("testParentSearch", "_parent", "baz.fits[1]"))]
603 )
605 def testSkymapLookups(self):
606 """Test that metadata lookups don't try to get skymap data ID values
607 from the registry.
608 """
609 mapper = MinMapper2(root=ROOT)
610 butler = dafPersist.Butler(mapper=mapper)
611 with self.assertRaises(RuntimeError) as manager:
612 butler.dataRef("forced_src", visit=787650, ccd=13)
613 self.assertIn("Cannot lookup skymap key 'tract'", str(manager.exception))
614 # We're mostly concerned that the statements below will raise an
615 # exception; if they don't, it's not likely the following tests will
616 # fail.
617 subset = butler.subset("forced_src", visit=787650, ccd=13, tract=0)
618 self.assertEqual(len(subset), 1)
619 dataRef = butler.dataRef("forced_src", visit=787650, ccd=13, tract=0)
620 self.assertFalse(dataRef.datasetExists("forced_src"))
623class Mapper3TestCase(unittest.TestCase):
624 """A test case for a mapper subclass which does not assign packageName."""
626 def testPackageName(self):
627 with self.assertRaises(ValueError):
628 MinMapper3()
629 with self.assertRaises(ValueError):
630 MinMapper3.getPackageName()
633class ParentRegistryTestCase(unittest.TestCase):
634 @staticmethod
635 def _createRegistry(path):
636 cmd = """CREATE TABLE x(
637 id INT,
638 visit INT,
639 filter TEXT,
640 snap INT,
641 raft TEXT,
642 sensor TEXT,
643 channel TEXT,
644 taiObs TEXT,
645 expTime REAL
646 );
647 """
648 conn = sqlite3.connect(path)
649 conn.cursor().execute(cmd)
650 conn.commit()
651 conn.close()
653 def setUp(self):
654 self.ROOT = tempfile.mkdtemp(dir=ROOT, prefix="ParentRegistryTestCase-")
655 self.repoARoot = os.path.join(self.ROOT, "a")
656 args = dafPersist.RepositoryArgs(root=self.repoARoot, mapper=MinMapper1)
657 butler = dafPersist.Butler(outputs=args)
658 self._createRegistry(os.path.join(self.repoARoot, "registry.sqlite3"))
659 del butler
661 def tearDown(self):
662 # the butler sql registry closes its database connection in __del__.
663 # To trigger __del__ we explicitly collect the garbage here. If we
664 # find having or closing the open database connection is a problem in
665 # production code, we may need to add api to butler to explicity
666 # release database connections (and maybe other things like in-memory
667 # cached objects).
668 gc.collect()
669 if os.path.exists(self.ROOT):
670 shutil.rmtree(self.ROOT)
672 def test(self):
673 """Verify that when the child repo does not have a registry it is
674 assigned the registry from the parent.
675 """
676 repoBRoot = os.path.join(self.ROOT, "b")
677 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot)
678 # This way of getting the registry from the mapping is obviously going
679 # way into private members and the python lambda implementation code.
680 # It is very brittle and should not be duplicated in user code
681 # or any location that is not trivial to fix along with changes to the
682 # CameraMapper or Mapping.
683 registryA = butler._repos.inputs()[0].repo._mapper.registry
684 registryB = butler._repos.outputs()[0].repo._mapper.registry
685 self.assertEqual(id(registryA), id(registryB))
687 self._createRegistry(os.path.join(repoBRoot, "registry.sqlite3"))
688 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot)
689 # see above; don't copy this way of getting the registry.
690 registryA = butler._repos.inputs()[0].repo._mapper.registry
691 registryB = butler._repos.outputs()[0].repo._mapper.registry
692 self.assertNotEqual(id(registryA), id(registryB))
695class MissingPolicyKeyTestCase(unittest.TestCase):
696 def testGetRaises(self):
697 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1})
698 # MinMapper1 does not specify a template for the raw dataset type so
699 # trying to use it for get should raise
700 with self.assertRaises(RuntimeError) as contextManager:
701 butler.get("raw")
702 # This test demonstrates and verifies that simple use of the incomplete
703 # dataset type returns a helpful (I hope) error message.
704 self.assertEqual(
705 str(contextManager.exception),
706 "Template is not defined for the raw dataset type, it must be set before it can be used.",
707 )
708 with self.assertRaises(RuntimeError) as contextManager:
709 butler.queryMetadata("raw", "unused", {})
711 def testQueryMetadataRaises(self):
712 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1})
713 # MinMapper1 does not specify a template for the raw dataset type so
714 # trying to use it for queryMetadata should raise
715 with self.assertRaises(RuntimeError) as contextManager:
716 butler.queryMetadata("raw", "unused", {})
717 # This test demonstrates and verifies that simple use of the incomplete
718 # dataset type returns a helpful (I hope) error message.
719 self.assertEqual(
720 str(contextManager.exception),
721 "Template is not defined for the raw dataset type, it must be set before it can be used.",
722 )
724 def testFilenameRaises(self):
725 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1})
726 # MinMapper1 does not specify a template for the raw dataset type so
727 # trying to use it for <datasetType>_filename should raise
728 with self.assertRaises(RuntimeError) as contextManager:
729 butler.get("raw_filename")
730 # This test demonstrates and verifies that simple use of the
731 # incomplete dataset type returns a helpful (I hope) error message.
732 self.assertEqual(
733 str(contextManager.exception),
734 "Template is not defined for the raw dataset type, it must be set before it can be used.",
735 )
737 def testWcsRaises(self):
738 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1})
739 # MinMapper1 does not specify a template for the raw dataset type so
740 # trying to use it for <datasetType>_wcs should raise
741 with self.assertRaises(RuntimeError) as contextManager:
742 butler.get("raw_wcs")
743 # This test demonstrates and verifies that simple use of the
744 # incomplete dataset type returns a helpful (I hope) error message.
745 self.assertEqual(
746 str(contextManager.exception),
747 "Template is not defined for the raw dataset type, it must be set before it can be used.",
748 )
750 def testConflictRaises(self):
751 policy = dafPersist.Policy(os.path.join(ROOT, "ConflictMapper.yaml"))
752 with self.assertRaisesRegex(ValueError, r"Duplicate mapping policy for dataset type packages"):
753 mapper = lsst.obs.base.CameraMapper(policy=policy, repositoryDir=ROOT, root=ROOT) # noqa F841
756class MemoryTester(lsst.utils.tests.MemoryTestCase):
757 pass
760if __name__ == "__main__": 760 ↛ 761line 760 didn't jump to line 761, because the condition on line 760 was never true
761 lsst.utils.tests.init()
762 unittest.main()