22 """Unit test base class for the gen2 to gen3 converter.
34 import lsst.daf.persistence
35 import lsst.daf.butler
36 import lsst.meas.algorithms
43 """Test the `butler convert` command.
45 Subclass this, and then `lsst.utils.tests.TestCase` and set the below
46 attributes. Uses the `butler convert` command line command to do the
51 """Root path to the gen2 repo to be converted."""
54 """Path to the gen2 calib repo to be converted."""
59 """Full path to the `Instrument` class of the data to be converted,
60 e.g. ``lsst.obs.decam.DarkEnergyCamera``.
65 The fully qualified instrument class name.
71 """The instrument class."""
76 """Name of the instrument for the gen3 registry, e.g. "DECam".
81 The name of the instrument.
86 """Full path to a config override for ConvertRepoTask, to be applied after
87 the Instrument overrides when running the conversion function."""
90 """List dataIds to use to load gen3 biases to test that they exist."""
93 """Name of the dataset that the biases are loaded into."""
96 """List dataIds to use to load gen3 flats to test that they exist."""
99 """Name of the dataset that the flats are loaded into."""
102 """List dataIds to use to load gen3 darks to test that they exist."""
105 """Name of the dataset that the darks are loaded into."""
108 """Other keyword arguments to pass directly to the converter function,
112 """Names of the reference catalogs to query for the existence of in the
113 converted gen3 repo."""
116 """Additional collections that should appear in the gen3 repo.
118 This will automatically be populated by the base `setUp` to include
119 ``"{instrumentName}/raw"``, ``"refcats"`` (if the ``refcats``
120 class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
125 """Key to use in a gen2 dataId to refer to a detector."""
127 exposureKey =
"visit"
128 """Key to use in a gen2 dataId to refer to a visit or exposure."""
130 calibFilterType =
"physical_filter"
131 """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
132 physical_filter or abstract_filter."""
135 """Name of the Gen3 skymap."""
138 """Path to skymap config file defining the new gen3 skymap."""
153 shutil.rmtree(self.
gen3root, ignore_errors=
True)
155 def _run_convert(self):
156 """Convert a gen2 repo to gen3 for testing.
160 log = lsst.log.Log.getLogger(
"convertRepo")
161 log.setLevel(log.INFO)
175 """Check that a raw was converted correctly.
179 gen3Butler : `lsst.daf.butler.Butler`
180 The Butler to be tested.
182 The exposure/vist identifier ``get`` from both butlers.
184 The detector identifier to ``get`` from both butlers.
188 gen2Exposure = self.
gen2Butler.get(
"raw", dataId=dataIdGen2)
189 except lsst.daf.persistence.butlerExceptions.NoResults:
192 dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.
instrumentName)
193 gen3Exposure = gen3Butler.get(
"raw", dataId=dataIdGen3)
196 self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
197 self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
198 self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
201 """Test that we can get converted bias/dark/flat from the gen3 repo.
203 Note: because there is no clear way to get calibrations from a gen2
204 repo, we just test that the thing we got is an ExposureF here, and
205 assume that formatter testing is handled properly elsewhere.
210 The name of the calibration to attempt to get ("bias", "flat").
211 calibIds : `list` of `dict`
212 The list of calibration dataIds to get.
213 gen3Butler : `lsst.daf.butler.Butler`
214 The Butler to use to get the data.
216 for dataId
in calibIds:
217 with self.subTest(dtype=calibName, dataId=dataId):
218 datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
219 gen3Exposure = gen3Butler.getDirect(datasets[0])
220 self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
223 """Test that we can get converted defects from the gen3 repo.
227 gen3Butler : `lsst.daf.butler.Butler`
228 The Butler to be tested.
229 detectors : `list` of `int`
230 The detector identifiers to ``get`` from the gen3 butler.
232 for detector
in detectors:
237 with self.subTest(dtype=
"defects", dataId=dataId):
238 datasets = list(gen3Butler.registry.queryDatasets(
"defects", collections=..., dataId=dataId))
240 gen3Defects = gen3Butler.getDirect(datasets[0])
241 self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
244 """Test that each expected refcat is in the gen3 repo.
248 gen3Butler : `lsst.daf.butler.Butler`
249 The Butler to be tested.
253 query = gen3Butler.registry.queryDatasets(refcat, collections=[
"refcats"])
254 self.assertGreater(len(list(query)), 0,
255 msg=f
"refcat={refcat} has no entries in collection 'refcats'.")
258 """Test that the correct set of collections is in the gen3 repo.
262 gen3Butler : `lsst.daf.butler.Butler`
263 The Butler to be tested.
265 self.assertEqual(set(gen3Butler.registry.queryCollections()), self.
collections,
266 f
"Compare with expected collections ({self.collections})")
269 """Test that all data are converted correctly.
272 gen3Butler = lsst.daf.butler.Butler(self.
gen3root,
279 for exposure, detector
in itertools.product(exposures, detectors):
280 with self.subTest(mode=
"raw", exposure=exposure, detector=detector):
281 self.
check_raw(gen3Butler, exposure, detector)
294 if __name__ ==
"__main__":