lsst.obs.base  19.0.0-26-g830ab5e+1
convertTests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Unit test base class for the gen2 to gen3 converter.
23 """
24 
25 import itertools
26 import shutil
27 import subprocess
28 import tempfile
29 import unittest
30 
31 
32 import lsst.afw.image
33 import lsst.afw.table
34 import lsst.daf.persistence
35 import lsst.daf.butler
36 import lsst.meas.algorithms
37 import lsst.utils.tests
38 
39 
41  """Test the `convert_gen2_repo_to_gen3.py` script.
42 
43  Subclass this, and then `lsst.utils.tests.TestCase` and set the below
44  attributes.
45  """
46  gen2root = ""
47  """Root path to the gen2 repo to be converted."""
48 
49  gen2calib = ""
50  """Path to the gen2 calib repo to be converted."""
51 
52  instrumentName = None
53  """Name of the instrument for the gen3 registry, e.g. "DECam"."""
54 
55  instrumentClass = None
56  """Full path to the `Instrument` class of the data to be converted, e.g.
57  ``lsst.obs.decam.DarkEnergyCamera``."""
58 
59  config = None
60  """Full path to a config override for ConvertRepoTask, to be applied after
61  the Instrument overrides when running `convert_gen2_repo_to_gen3.py`."""
62 
63  biases = []
64  """List dataIds to use to load gen3 biases to test that they exist."""
65 
66  biasName = "bias"
67  """Name of the dataset that the biases are loaded into."""
68 
69  flats = []
70  """List dataIds to use to load gen3 flats to test that they exist."""
71 
72  flatName = "flat"
73  """Name of the dataset that the flats are loaded into."""
74 
75  darks = []
76  """List dataIds to use to load gen3 darks to test that they exist."""
77 
78  darkName = "dark"
79  """Name of the dataset that the darks are loaded into."""
80 
81  args = None
82  """Other arguments to pass directly to the converter script, as a tuple."""
83 
84  refcats = []
85  """Names of the reference catalogs to query for the existence of in the
86  converted gen3 repo."""
87 
88  collections = set()
89  """Additional collections that should appear in the gen3 repo, beyond the
90  instrument name and any refcats, if ``refcats`` is non-empty above.
91  Typically the only additional one necessary would be "skymaps"."""
92 
93  def setUp(self):
94  self.gen3root = tempfile.mkdtemp()
95  self.gen2Butler = lsst.daf.persistence.Butler(root=self.gen2root, calibRoot=self.gen2calib)
96  # This command is in obs_base, and we use the one that has been setup and scons'ed.
97  self.cmd = "convert_gen2_repo_to_gen3.py"
98  self.collections.add(self.instrumentName)
99  if len(self.refcats) > 0:
100  self.collections.add("refcats")
101 
102  def tearDown(self):
103  shutil.rmtree(self.gen3root, ignore_errors=True)
104 
105  def _run_convert(self):
106  """Convert a gen2 repo to gen3 for testing.
107  """
108  cmd = [self.cmd, self.instrumentClass,
109  "--gen2root", self.gen2root,
110  "--gen3root", self.gen3root,
111  "--calibs", self.gen2calib
112  ]
113  if self.config is not None:
114  cmd.extend(("--config", self.config))
115  if self.args is not None:
116  cmd.extend(self.args)
117  print(f"Running command: {' '.join(cmd)}")
118  subprocess.run(cmd, check=True)
119 
120  def check_raw(self, gen3Butler, exposure, detector):
121  """Check that a raw was converted correctly.
122 
123  Parameters
124  ----------
125  gen3Butler : `lsst.daf.butler.Butler`
126  The Butler to be tested.
127  exposure : `int`
128  The exposure/vist identifier ``get`` from both butlers.
129  detector : `int`
130  The detector identifier to ``get`` from both butlers.
131  """
132  dataIdGen2 = dict(ccd=detector, visit=exposure)
133  try:
134  gen2Exposure = self.gen2Butler.get("raw", dataId=dataIdGen2)
135  except lsst.daf.persistence.butlerExceptions.NoResults:
136  # ignore datasets that don't actually exist in the gen2 butler.
137  return
138  dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.instrumentName)
139  gen3Exposure = gen3Butler.get("raw", dataId=dataIdGen3)
140  # Check that we got an Exposure, but not what type; there is
141  # inconsistency between different obs packages.
142  self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
143  self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
144  self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
145 
146  def check_calibs(self, calibName, calibIds, gen3Butler):
147  """Test that we can get converted bias/dark/flat from the gen3 repo.
148 
149  Note: because there is no clear way to get calibrations from a gen2
150  repo, we just test that the thing we got is an ExposureF here, and
151  assume that formatter testing is handled properly elsewhere.
152 
153  Parameters
154  ----------
155  calibName : `str`
156  The name of the calibration to attempt to get ("bias", "flat").
157  calibIds : `list` of `dict`
158  The list of calibration dataIds to get.
159  gen3Butler : `lsst.daf.butler.Butler`
160  The Butler to use to get the data.
161  """
162  for dataId in calibIds:
163  gen3Exposure = gen3Butler.get(calibName, dataId=dataId)
164  self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
165 
166  def check_defects(self, gen3Butler, detectors):
167  """Test that we can get converted defects from the gen3 repo.
168 
169  Parameters
170  ----------
171  gen3Butler : `lsst.daf.butler.Butler`
172  The Butler to be tested.
173  detector : `int`
174  The detector identifiers to ``get`` from the gen3 butler.
175  """
176  for detector in detectors:
177  dataId = dict(detector=detector, instrument=self.instrumentName)
178  # Fill out the missing parts of the dataId, as we don't a-priori
179  # know e.g. the "calibration_label". Use the first element of the
180  # result because we only need to check one.
181  datasets = list(gen3Butler.registry.queryDatasets("defects", collections=..., dataId=dataId))
182  gen3Defects = gen3Butler.get("defects", dataId=datasets[0].dataId)
183  self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
184 
185  def check_refcat(self, gen3Butler):
186  """Test that each expected refcat is in the gen3 repo.
187 
188  Parameters
189  ----------
190  gen3Butler : `lsst.daf.butler.Butler`
191  The Butler to be tested.
192  """
193  if len(self.refcats) > 0:
194  for refcat in self.refcats:
195  query = gen3Butler.registry.queryDatasets(refcat, collections=["refcats"])
196  self.assertGreater(len(list(query)), 0,
197  msg=f"refcat={refcat} has no entries in collection 'refcats'.")
198 
199  def check_collections(self, gen3Butler):
200  """Test that the correct set of collections is in the gen3 repo.
201 
202  Parameters
203  ----------
204  gen3Butler : `lsst.daf.butler.Butler`
205  The Butler to be tested.
206  """
207  self.assertEqual(self.collections, gen3Butler.registry.getAllCollections())
208 
209  def test_convert(self):
210  """Test that raws are converted correctly.
211  """
212  self._run_convert()
213  gen3Butler = lsst.daf.butler.Butler(self.gen3root, run=self.instrumentName)
214  self.check_collections(gen3Butler)
215 
216  # check every raw detector that the gen2 butler knows about
217  detectors = self.gen2Butler.queryMetadata("raw", "ccd")
218  exposures = self.gen2Butler.queryMetadata("raw", "visit")
219  for exposure, detector in itertools.product(exposures, detectors):
220  self.check_raw(gen3Butler, exposure, detector)
221 
222  self.check_refcat(gen3Butler)
223  self.check_defects(gen3Butler, detectors)
224  self.check_calibs(self.biasName, self.biases, gen3Butler)
225  self.check_calibs(self.flatName, self.flats, gen3Butler)
226  self.check_calibs(self.darkName, self.darks, gen3Butler)
227 
228 
229 def setup_module(module):
231 
232 
233 if __name__ == "__main__":
235  unittest.main()
def check_calibs(self, calibName, calibIds, gen3Butler)
def check_raw(self, gen3Butler, exposure, detector)