22 """Unit test base class for the gen2 to gen3 converter.
34 import lsst.daf.persistence
35 import lsst.daf.butler
36 import lsst.meas.algorithms
43 """Test the `butler convert` command.
45 Subclass this, and then `lsst.utils.tests.TestCase` and set the below
46 attributes. Uses the `butler convert` command line command to do the
51 """Root path to the gen2 repo to be converted."""
54 """Path to the gen2 calib repo to be converted."""
59 """Full path to the `Instrument` class of the data to be converted,
60 e.g. ``lsst.obs.decam.DarkEnergyCamera``.
65 The fully qualified instrument class name.
71 """The instrument class."""
76 """Name of the instrument for the gen3 registry, e.g. "DECam".
81 The name of the instrument.
86 """Full path to a config override for ConvertRepoTask, to be applied after
87 the Instrument overrides when running the conversion function."""
90 """List dataIds to use to load gen3 biases to test that they exist."""
93 """Name of the dataset that the biases are loaded into."""
96 """List dataIds to use to load gen3 flats to test that they exist."""
99 """Name of the dataset that the flats are loaded into."""
102 """List dataIds to use to load gen3 darks to test that they exist."""
105 """Name of the dataset that the darks are loaded into."""
108 """Other keyword arguments to pass directly to the converter function,
112 """Names of the reference catalogs to query for the existence of in the
113 converted gen3 repo."""
116 """Additional collections that should appear in the gen3 repo.
118 This will automatically be populated by the base `setUp` to include
119 ``"{instrumentName}/raw"``, ``"refcats"`` (if the ``refcats``
120 class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
125 """Key to use in a gen2 dataId to refer to a detector."""
127 exposureKey =
"visit"
128 """Key to use in a gen2 dataId to refer to a visit or exposure."""
130 calibFilterType =
"physical_filter"
131 """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
132 physical_filter or abstract_filter."""
135 """Name of the Gen3 skymap."""
138 """Path to skymap config file defining the new gen3 skymap."""
153 shutil.rmtree(self.
gen3root, ignore_errors=
True)
155 def _run_convert(self):
156 """Convert a gen2 repo to gen3 for testing.
160 log = lsst.log.Log.getLogger(
"convertRepo")
161 log.setLevel(log.INFO)
174 """Check that a raw was converted correctly.
178 gen3Butler : `lsst.daf.butler.Butler`
179 The Butler to be tested.
181 The exposure/vist identifier ``get`` from both butlers.
183 The detector identifier to ``get`` from both butlers.
187 gen2Exposure = self.
gen2Butler.get(
"raw", dataId=dataIdGen2)
188 except lsst.daf.persistence.butlerExceptions.NoResults:
191 dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.
instrumentName)
192 gen3Exposure = gen3Butler.get(
"raw", dataId=dataIdGen3)
195 self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
196 self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
197 self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
200 """Test that we can get converted bias/dark/flat from the gen3 repo.
202 Note: because there is no clear way to get calibrations from a gen2
203 repo, we just test that the thing we got is an ExposureF here, and
204 assume that formatter testing is handled properly elsewhere.
209 The name of the calibration to attempt to get ("bias", "flat").
210 calibIds : `list` of `dict`
211 The list of calibration dataIds to get.
212 gen3Butler : `lsst.daf.butler.Butler`
213 The Butler to use to get the data.
215 for dataId
in calibIds:
216 with self.subTest(dtype=calibName, dataId=dataId):
217 datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
218 gen3Exposure = gen3Butler.getDirect(datasets[0])
219 self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
222 """Test that we can get converted defects from the gen3 repo.
226 gen3Butler : `lsst.daf.butler.Butler`
227 The Butler to be tested.
228 detectors : `list` of `int`
229 The detector identifiers to ``get`` from the gen3 butler.
231 for detector
in detectors:
236 with self.subTest(dtype=
"defects", dataId=dataId):
237 datasets = list(gen3Butler.registry.queryDatasets(
"defects", collections=..., dataId=dataId))
239 gen3Defects = gen3Butler.getDirect(datasets[0])
240 self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
243 """Test that each expected refcat is in the gen3 repo.
247 gen3Butler : `lsst.daf.butler.Butler`
248 The Butler to be tested.
252 query = gen3Butler.registry.queryDatasets(refcat, collections=[
"refcats"])
253 self.assertGreater(len(list(query)), 0,
254 msg=f
"refcat={refcat} has no entries in collection 'refcats'.")
257 """Test that the correct set of collections is in the gen3 repo.
261 gen3Butler : `lsst.daf.butler.Butler`
262 The Butler to be tested.
264 self.assertEqual(set(gen3Butler.registry.queryCollections()), self.
collections,
265 f
"Compare with expected collections ({self.collections})")
268 """Test that all data are converted correctly.
271 gen3Butler = lsst.daf.butler.Butler(self.
gen3root,
278 for exposure, detector
in itertools.product(exposures, detectors):
279 with self.subTest(mode=
"raw", exposure=exposure, detector=detector):
280 self.
check_raw(gen3Butler, exposure, detector)
293 if __name__ ==
"__main__":