22 """Unit test base class for the gen2 to gen3 converter.
33 import lsst.daf.persistence
34 import lsst.daf.butler
35 import lsst.meas.algorithms
37 from ..script.convertGen2RepoToGen3
import convert
as convertGen2RepoToGen3
41 """Test the `convert_gen2_repo_to_gen3.py` script.
43 Subclass this, and then `lsst.utils.tests.TestCase` and set the below
44 attributes. Uses the `lsst.obs.base.script.convertGen2RepoToGen3.convert`
45 function to do the conversion.
48 """Root path to the gen2 repo to be converted."""
51 """Path to the gen2 calib repo to be converted."""
54 """Name of the instrument for the gen3 registry, e.g. "DECam"."""
56 instrumentClass =
None
57 """Full path to the `Instrument` class of the data to be converted, e.g.
58 ``lsst.obs.decam.DarkEnergyCamera``."""
61 """Full path to a config override for ConvertRepoTask, to be applied after
62 the Instrument overrides when running the conversion function."""
65 """List dataIds to use to load gen3 biases to test that they exist."""
68 """Name of the dataset that the biases are loaded into."""
71 """List dataIds to use to load gen3 flats to test that they exist."""
74 """Name of the dataset that the flats are loaded into."""
77 """List dataIds to use to load gen3 darks to test that they exist."""
80 """Name of the dataset that the darks are loaded into."""
83 """Other keyword arguments to pass directly to the converter function,
87 """Names of the reference catalogs to query for the existence of in the
88 converted gen3 repo."""
91 """Additional collections that should appear in the gen3 repo.
93 This will automatically be populated by the base `setUp` to include
94 ``"raw/{instrumentName}"``, ``"refcats"`` (if the ``refcats``
95 class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
100 """Key to use in a gen2 dataId to refer to a detector."""
102 exposureKey =
"visit"
103 """Key to use in a gen2 dataId to refer to a visit or exposure."""
105 calibFilterType =
"physical_filter"
106 """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
107 physical_filter or abstract_filter."""
110 """Name of the Gen3 skymap."""
113 """Path to skymap config file defining the new gen3 skymap."""
126 shutil.rmtree(self.
gen3root, ignore_errors=
True)
128 def _run_convert(self):
129 """Convert a gen2 repo to gen3 for testing.
133 log = lsst.log.Log.getLogger(
"convertRepo")
134 log.setLevel(log.INFO)
143 """Check that a raw was converted correctly.
147 gen3Butler : `lsst.daf.butler.Butler`
148 The Butler to be tested.
150 The exposure/vist identifier ``get`` from both butlers.
152 The detector identifier to ``get`` from both butlers.
156 gen2Exposure = self.
gen2Butler.get(
"raw", dataId=dataIdGen2)
157 except lsst.daf.persistence.butlerExceptions.NoResults:
160 dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.
instrumentName)
161 gen3Exposure = gen3Butler.get(
"raw", dataId=dataIdGen3)
164 self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
165 self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
166 self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
169 """Test that we can get converted bias/dark/flat from the gen3 repo.
171 Note: because there is no clear way to get calibrations from a gen2
172 repo, we just test that the thing we got is an ExposureF here, and
173 assume that formatter testing is handled properly elsewhere.
178 The name of the calibration to attempt to get ("bias", "flat").
179 calibIds : `list` of `dict`
180 The list of calibration dataIds to get.
181 gen3Butler : `lsst.daf.butler.Butler`
182 The Butler to use to get the data.
184 for dataId
in calibIds:
185 with self.subTest(dtype=calibName, dataId=dataId):
186 datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
187 gen3Exposure = gen3Butler.getDirect(datasets[0])
188 self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
191 """Test that we can get converted defects from the gen3 repo.
195 gen3Butler : `lsst.daf.butler.Butler`
196 The Butler to be tested.
197 detectors : `list` of `int`
198 The detector identifiers to ``get`` from the gen3 butler.
200 for detector
in detectors:
205 with self.subTest(dtype=
"defects", dataId=dataId):
206 datasets = list(gen3Butler.registry.queryDatasets(
"defects", collections=..., dataId=dataId))
208 gen3Defects = gen3Butler.getDirect(datasets[0])
209 self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
212 """Test that each expected refcat is in the gen3 repo.
216 gen3Butler : `lsst.daf.butler.Butler`
217 The Butler to be tested.
221 query = gen3Butler.registry.queryDatasets(refcat, collections=[
"refcats"])
222 self.assertGreater(len(list(query)), 0,
223 msg=f
"refcat={refcat} has no entries in collection 'refcats'.")
226 """Test that the correct set of collections is in the gen3 repo.
230 gen3Butler : `lsst.daf.butler.Butler`
231 The Butler to be tested.
233 self.assertEqual(set(gen3Butler.registry.queryCollections()), self.
collections,
234 f
"Compare with expected collections ({self.collections})")
237 """Test that all data are converted correctly.
240 gen3Butler = lsst.daf.butler.Butler(self.
gen3root, collections=f
"raw/{self.instrumentName}")
246 for exposure, detector
in itertools.product(exposures, detectors):
247 with self.subTest(mode=
"raw", exposure=exposure, detector=detector):
248 self.
check_raw(gen3Butler, exposure, detector)
261 if __name__ ==
"__main__":