22 """Unit test base class for the gen2 to gen3 converter.
34 import lsst.daf.persistence
35 import lsst.daf.butler
36 import lsst.meas.algorithms
43 """Test the `butler convert` command.
45 Subclass this, and then `lsst.utils.tests.TestCase` and set the below
46 attributes. Uses the `butler convert` command line command to do the
51 """Root path to the gen2 repo to be converted."""
54 """Path to the gen2 calib repo to be converted."""
59 """Full path to the `Instrument` class of the data to be converted,
60 e.g. ``lsst.obs.decam.DarkEnergyCamera``.
65 The fully qualified instrument class name.
71 """The instrument class."""
76 """Name of the instrument for the gen3 registry, e.g. "DECam".
81 The name of the instrument.
86 """Full path to a config override for ConvertRepoTask, to be applied after
87 the Instrument overrides when running the conversion function."""
90 """List dataIds to use to load gen3 biases to test that they exist."""
93 """Name of the dataset that the biases are loaded into."""
96 """List dataIds to use to load gen3 flats to test that they exist."""
99 """Name of the dataset that the flats are loaded into."""
102 """List dataIds to use to load gen3 darks to test that they exist."""
105 """Name of the dataset that the darks are loaded into."""
108 """Other keyword arguments to pass directly to the converter function,
112 """Names of the reference catalogs to query for the existence of in the
113 converted gen3 repo."""
116 """Additional collections that should appear in the gen3 repo.
118 This will automatically be populated by the base `setUp` to include
119 ``"{instrumentName}/raw"``, ``"refcats"`` (if the ``refcats``
120 class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
125 """Key to use in a gen2 dataId to refer to a detector."""
127 exposureKey =
"visit"
128 """Key to use in a gen2 dataId to refer to a visit or exposure."""
130 calibFilterType =
"physical_filter"
131 """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
132 physical_filter or band."""
135 """Name of the Gen3 skymap."""
138 """Path to skymap config file defining the new gen3 skymap."""
155 shutil.rmtree(self.
gen3root, ignore_errors=
True)
157 def _run_convert(self):
158 """Convert a gen2 repo to gen3 for testing.
162 log = lsst.log.Log.getLogger(
"convertRepo")
163 log.setLevel(log.INFO)
176 """Check that a raw was converted correctly.
180 gen3Butler : `lsst.daf.butler.Butler`
181 The Butler to be tested.
183 The exposure/vist identifier ``get`` from both butlers.
185 The detector identifier to ``get`` from both butlers.
189 gen2Exposure = self.
gen2Butler.get(
"raw", dataId=dataIdGen2)
190 except lsst.daf.persistence.butlerExceptions.NoResults:
193 dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.
instrumentName)
194 gen3Exposure = gen3Butler.get(
"raw", dataId=dataIdGen3)
197 self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
198 self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
199 self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
202 """Test that we can get converted bias/dark/flat from the gen3 repo.
204 Note: because there is no clear way to get calibrations from a gen2
205 repo, we just test that the thing we got is an ExposureF here, and
206 assume that formatter testing is handled properly elsewhere.
211 The name of the calibration to attempt to get ("bias", "flat").
212 calibIds : `list` of `dict`
213 The list of calibration dataIds to get.
214 gen3Butler : `lsst.daf.butler.Butler`
215 The Butler to use to get the data.
217 for dataId
in calibIds:
218 with self.subTest(dtype=calibName, dataId=dataId):
219 datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
220 gen3Exposure = gen3Butler.getDirect(datasets[0])
221 self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
224 """Test that we can get converted defects from the gen3 repo.
228 gen3Butler : `lsst.daf.butler.Butler`
229 The Butler to be tested.
230 detectors : `list` of `int`
231 The detector identifiers to ``get`` from the gen3 butler.
233 for detector
in detectors:
238 with self.subTest(dtype=
"defects", dataId=dataId):
239 datasets = list(gen3Butler.registry.queryDatasets(
"defects", collections=..., dataId=dataId))
241 gen3Defects = gen3Butler.getDirect(datasets[0])
242 self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
245 """Test that each expected refcat is in the gen3 repo.
249 gen3Butler : `lsst.daf.butler.Butler`
250 The Butler to be tested.
254 query = gen3Butler.registry.queryDatasets(refcat, collections=[
"refcats"])
255 self.assertGreater(len(list(query)), 0,
256 msg=f
"refcat={refcat} has no entries in collection 'refcats'.")
259 """Test that the correct set of collections is in the gen3 repo.
263 gen3Butler : `lsst.daf.butler.Butler`
264 The Butler to be tested.
266 self.assertEqual(set(gen3Butler.registry.queryCollections()), self.
collections,
267 f
"Compare with expected collections ({self.collections})")
270 """Test that all data are converted correctly.
273 gen3Butler = lsst.daf.butler.Butler(self.
gen3root,
280 for exposure, detector
in itertools.product(exposures, detectors):
281 with self.subTest(mode=
"raw", exposure=exposure, detector=detector):
282 self.
check_raw(gen3Butler, exposure, detector)
295 if __name__ ==
"__main__":