22 """Unit test base class for the gen2 to gen3 converter.
33 import lsst.daf.persistence
34 import lsst.daf.butler
35 import lsst.meas.algorithms
37 from ..script.convertGen2RepoToGen3
import convert
as convertGen2RepoToGen3
41 """Test the `convert_gen2_repo_to_gen3.py` script.
43 Subclass this, and then `lsst.utils.tests.TestCase` and set the below
44 attributes. Uses the `lsst.obs.base.script.convertGen2RepoToGen3.convert`
45 function to do the conversion.
48 """Root path to the gen2 repo to be converted."""
51 """Path to the gen2 calib repo to be converted."""
54 """Name of the instrument for the gen3 registry, e.g. "DECam"."""
56 instrumentClass =
None
57 """Full path to the `Instrument` class of the data to be converted, e.g.
58 ``lsst.obs.decam.DarkEnergyCamera``."""
61 """Full path to a config override for ConvertRepoTask, to be applied after
62 the Instrument overrides when running the conversion function."""
65 """List dataIds to use to load gen3 biases to test that they exist."""
68 """Name of the dataset that the biases are loaded into."""
71 """List dataIds to use to load gen3 flats to test that they exist."""
74 """Name of the dataset that the flats are loaded into."""
77 """List dataIds to use to load gen3 darks to test that they exist."""
80 """Name of the dataset that the darks are loaded into."""
83 """Other keyword arguments to pass directly to the converter function,
87 """Names of the reference catalogs to query for the existence of in the
88 converted gen3 repo."""
91 """Additional collections that should appear in the gen3 repo.
93 This will automatically be populated by the base `setUp` to include
94 ``"raw/{instrumentName}"``, ``"refcats"`` (if the ``refcats``
95 class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
100 """Key to use in a gen2 dataId to refer to a detector."""
102 exposureKey =
"visit"
103 """Key to use in a gen2 dataId to refer to a visit or exposure."""
105 calibFilterType =
"physical_filter"
106 """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
107 physical_filter or abstract_filter."""
110 """Name of the Gen3 skymap."""
113 """Path to skymap config file defining the new gen3 skymap."""
126 shutil.rmtree(self.
gen3root, ignore_errors=
True)
128 def _run_convert(self):
129 """Convert a gen2 repo to gen3 for testing.
133 log = lsst.log.Log.getLogger(
"convertRepo")
134 log.setLevel(log.INFO)
144 """Check that a raw was converted correctly.
148 gen3Butler : `lsst.daf.butler.Butler`
149 The Butler to be tested.
151 The exposure/vist identifier ``get`` from both butlers.
153 The detector identifier to ``get`` from both butlers.
157 gen2Exposure = self.
gen2Butler.get(
"raw", dataId=dataIdGen2)
158 except lsst.daf.persistence.butlerExceptions.NoResults:
161 dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.
instrumentName)
162 gen3Exposure = gen3Butler.get(
"raw", dataId=dataIdGen3)
165 self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
166 self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
167 self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
170 """Test that we can get converted bias/dark/flat from the gen3 repo.
172 Note: because there is no clear way to get calibrations from a gen2
173 repo, we just test that the thing we got is an ExposureF here, and
174 assume that formatter testing is handled properly elsewhere.
179 The name of the calibration to attempt to get ("bias", "flat").
180 calibIds : `list` of `dict`
181 The list of calibration dataIds to get.
182 gen3Butler : `lsst.daf.butler.Butler`
183 The Butler to use to get the data.
185 for dataId
in calibIds:
186 with self.subTest(dtype=calibName, dataId=dataId):
187 datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
188 gen3Exposure = gen3Butler.getDirect(datasets[0])
189 self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
192 """Test that we can get converted defects from the gen3 repo.
196 gen3Butler : `lsst.daf.butler.Butler`
197 The Butler to be tested.
198 detectors : `list` of `int`
199 The detector identifiers to ``get`` from the gen3 butler.
201 for detector
in detectors:
206 with self.subTest(dtype=
"defects", dataId=dataId):
207 datasets = list(gen3Butler.registry.queryDatasets(
"defects", collections=..., dataId=dataId))
209 gen3Defects = gen3Butler.getDirect(datasets[0])
210 self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
213 """Test that each expected refcat is in the gen3 repo.
217 gen3Butler : `lsst.daf.butler.Butler`
218 The Butler to be tested.
222 query = gen3Butler.registry.queryDatasets(refcat, collections=[
"refcats"])
223 self.assertGreater(len(list(query)), 0,
224 msg=f
"refcat={refcat} has no entries in collection 'refcats'.")
227 """Test that the correct set of collections is in the gen3 repo.
231 gen3Butler : `lsst.daf.butler.Butler`
232 The Butler to be tested.
234 self.assertEqual(set(gen3Butler.registry.queryCollections()), self.
collections,
235 f
"Compare with expected collections ({self.collections})")
238 """Test that all data are converted correctly.
241 gen3Butler = lsst.daf.butler.Butler(self.
gen3root, collections=f
"raw/{self.instrumentName}")
247 for exposure, detector
in itertools.product(exposures, detectors):
248 with self.subTest(mode=
"raw", exposure=exposure, detector=detector):
249 self.
check_raw(gen3Butler, exposure, detector)
262 if __name__ ==
"__main__":