22 """Unit test base class for the gen2 to gen3 converter. 34 import lsst.daf.persistence
35 import lsst.daf.butler
36 import lsst.meas.algorithms
41 """Test the `convert_gen2_repo_to_gen3.py` script. 43 Subclass this, and then `lsst.utils.tests.TestCase` and set the below 47 """Root path to the gen2 repo to be converted.""" 50 """Path to the gen2 calib repo to be converted.""" 53 """Name of the instrument for the gen3 registry, e.g. "DECam".""" 55 instrumentClass =
None 56 """Full path to the `Instrument` class of the data to be converted, e.g. 57 ``lsst.obs.decam.DarkEnergyCamera``.""" 60 """Full path to a config override for ConvertRepoTask, to be applied after 61 the Instrument overrides when running `convert_gen2_repo_to_gen3.py`.""" 64 """List dataIds to use to load gen3 biases to test that they exist.""" 67 """Name of the dataset that the biases are loaded into.""" 70 """List dataIds to use to load gen3 flats to test that they exist.""" 73 """Name of the dataset that the flats are loaded into.""" 76 """List dataIds to use to load gen3 darks to test that they exist.""" 79 """Name of the dataset that the darks are loaded into.""" 82 """Other arguments to pass directly to the converter script, as a tuple.""" 85 """Names of the reference catalogs to query for the existence of in the 86 converted gen3 repo.""" 89 """Additional collections that should appear in the gen3 repo, beyond the 90 instrument name and any refcats, if ``refcats`` is non-empty above. 91 Typically the only additional one necessary would be "skymaps".""" 97 self.
cmd =
"convert_gen2_repo_to_gen3.py" 103 shutil.rmtree(self.
gen3root, ignore_errors=
True)
105 def _run_convert(self):
106 """Convert a gen2 repo to gen3 for testing. 113 if self.
config is not None:
114 cmd.extend((
"--config", self.
config))
115 if self.
args is not None:
116 cmd.extend(self.
args)
117 print(f
"Running command: {' '.join(cmd)}")
118 subprocess.run(cmd, check=
True)
121 """Check that a raw was converted correctly. 125 gen3Butler : `lsst.daf.butler.Butler` 126 The Butler to be tested. 128 The exposure/vist identifier ``get`` from both butlers. 130 The detector identifier to ``get`` from both butlers. 132 dataIdGen2 = dict(ccd=detector, visit=exposure)
134 gen2Exposure = self.
gen2Butler.get(
"raw", dataId=dataIdGen2)
135 except lsst.daf.persistence.butlerExceptions.NoResults:
138 dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.
instrumentName)
139 gen3Exposure = gen3Butler.get(
"raw", dataId=dataIdGen3)
142 self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
143 self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
144 self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
147 """Test that we can get converted bias/dark/flat from the gen3 repo. 149 Note: because there is no clear way to get calibrations from a gen2 150 repo, we just test that the thing we got is an ExposureF here, and 151 assume that formatter testing is handled properly elsewhere. 156 The name of the calibration to attempt to get ("bias", "flat"). 157 calibIds : `list` of `dict` 158 The list of calibration dataIds to get. 159 gen3Butler : `lsst.daf.butler.Butler` 160 The Butler to use to get the data. 162 for dataId
in calibIds:
163 gen3Exposure = gen3Butler.get(calibName, dataId=dataId)
164 self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
167 """Test that we can get converted defects from the gen3 repo. 171 gen3Butler : `lsst.daf.butler.Butler` 172 The Butler to be tested. 174 The detector identifiers to ``get`` from the gen3 butler. 176 for detector
in detectors:
181 datasets = list(gen3Butler.registry.queryDatasets(
"defects", collections=..., dataId=dataId))
182 gen3Defects = gen3Butler.get(
"defects", dataId=datasets[0].dataId)
183 self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
186 """Test that each expected refcat is in the gen3 repo. 190 gen3Butler : `lsst.daf.butler.Butler` 191 The Butler to be tested. 195 query = gen3Butler.registry.queryDatasets(refcat, collections=[
"refcats"])
196 self.assertGreater(len(list(query)), 0,
197 msg=f
"refcat={refcat} has no entries in collection 'refcats'.")
200 """Test that the correct set of collections is in the gen3 repo. 204 gen3Butler : `lsst.daf.butler.Butler` 205 The Butler to be tested. 207 self.assertEqual(self.
collections, gen3Butler.registry.getAllCollections())
210 """Test that raws are converted correctly. 217 detectors = self.
gen2Butler.queryMetadata(
"raw",
"ccd")
218 exposures = self.
gen2Butler.queryMetadata(
"raw",
"visit")
219 for exposure, detector
in itertools.product(exposures, detectors):
220 self.
check_raw(gen3Butler, exposure, detector)
233 if __name__ ==
"__main__":
def check_calibs(self, calibName, calibIds, gen3Butler)
def check_defects(self, gen3Butler, detectors)
def check_raw(self, gen3Butler, exposure, detector)
def check_collections(self, gen3Butler)
def check_refcat(self, gen3Butler)