lsst.obs.base  19.0.0-28-g99824a6
convertTests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Unit test base class for the gen2 to gen3 converter.
23 """
24 
25 import itertools
26 import shutil
27 import subprocess
28 import tempfile
29 import unittest
30 
31 
32 import lsst.afw.image
33 import lsst.afw.table
34 import lsst.daf.persistence
35 import lsst.daf.butler
36 import lsst.meas.algorithms
37 import lsst.utils.tests
38 
39 
41  """Test the `convert_gen2_repo_to_gen3.py` script.
42 
43  Subclass this, and then `lsst.utils.tests.TestCase` and set the below
44  attributes.
45  """
46  gen2root = ""
47  """Root path to the gen2 repo to be converted."""
48 
49  gen2calib = ""
50  """Path to the gen2 calib repo to be converted."""
51 
52  instrumentName = None
53  """Name of the instrument for the gen3 registry, e.g. "DECam"."""
54 
55  instrumentClass = None
56  """Full path to the `Instrument` class of the data to be converted, e.g.
57  ``lsst.obs.decam.DarkEnergyCamera``."""
58 
59  config = None
60  """Full path to a config override for ConvertRepoTask, to be applied after
61  the Instrument overrides when running `convert_gen2_repo_to_gen3.py`."""
62 
63  biases = []
64  """List dataIds to use to load gen3 biases to test that they exist."""
65 
66  biasName = "bias"
67  """Name of the dataset that the biases are loaded into."""
68 
69  flats = []
70  """List dataIds to use to load gen3 flats to test that they exist."""
71 
72  flatName = "flat"
73  """Name of the dataset that the flats are loaded into."""
74 
75  darks = []
76  """List dataIds to use to load gen3 darks to test that they exist."""
77 
78  darkName = "dark"
79  """Name of the dataset that the darks are loaded into."""
80 
81  args = None
82  """Other arguments to pass directly to the converter script, as a tuple."""
83 
84  refcats = []
85  """Names of the reference catalogs to query for the existence of in the
86  converted gen3 repo."""
87 
88  collections = set()
89  """Additional collections that should appear in the gen3 repo, beyond the
90  instrument name and any refcats, if ``refcats`` is non-empty above.
91  Typically the only additional one necessary would be "skymaps"."""
92 
93  detectorKey = "ccd"
94  """Key to use in a gen2 dataId to refer to a detector."""
95 
96  exposureKey = "visit"
97  """Key to use in a gen2 dataId to refer to a visit or exposure."""
98 
99  def setUp(self):
100  self.gen3root = tempfile.mkdtemp()
101  self.gen2Butler = lsst.daf.persistence.Butler(root=self.gen2root, calibRoot=self.gen2calib)
102  # This command is in obs_base, and we use the one that has been setup and scons'ed.
103  self.cmd = "convert_gen2_repo_to_gen3.py"
104 
105  # if the collections set is empty we do not add to it, we create
106  # a new instance version. Without this each subclass would add
107  # to the same set.
108  if not self.collections:
109  self.collections = set()
110  self.collections.add(self.instrumentName)
111  if len(self.refcats) > 0:
112  self.collections.add("refcats")
113 
114  def tearDown(self):
115  shutil.rmtree(self.gen3root, ignore_errors=True)
116 
117  def _run_convert(self):
118  """Convert a gen2 repo to gen3 for testing.
119  """
120  cmd = [self.cmd, self.instrumentClass,
121  "--gen2root", self.gen2root,
122  "--gen3root", self.gen3root,
123  "--calibs", self.gen2calib
124  ]
125  if self.config is not None:
126  cmd.extend(("--config", self.config))
127  if self.args is not None:
128  cmd.extend(self.args)
129  print(f"Running command: {' '.join(cmd)}")
130  subprocess.run(cmd, check=True)
131 
132  def check_raw(self, gen3Butler, exposure, detector):
133  """Check that a raw was converted correctly.
134 
135  Parameters
136  ----------
137  gen3Butler : `lsst.daf.butler.Butler`
138  The Butler to be tested.
139  exposure : `int`
140  The exposure/vist identifier ``get`` from both butlers.
141  detector : `int`
142  The detector identifier to ``get`` from both butlers.
143  """
144  dataIdGen2 = {self.detectorKey: detector, self.exposureKey: exposure}
145  try:
146  gen2Exposure = self.gen2Butler.get("raw", dataId=dataIdGen2)
147  except lsst.daf.persistence.butlerExceptions.NoResults:
148  # ignore datasets that don't actually exist in the gen2 butler.
149  return
150  dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.instrumentName)
151  gen3Exposure = gen3Butler.get("raw", dataId=dataIdGen3)
152  # Check that we got an Exposure, but not what type; there is
153  # inconsistency between different obs packages.
154  self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
155  self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
156  self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
157 
158  def check_calibs(self, calibName, calibIds, gen3Butler):
159  """Test that we can get converted bias/dark/flat from the gen3 repo.
160 
161  Note: because there is no clear way to get calibrations from a gen2
162  repo, we just test that the thing we got is an ExposureF here, and
163  assume that formatter testing is handled properly elsewhere.
164 
165  Parameters
166  ----------
167  calibName : `str`
168  The name of the calibration to attempt to get ("bias", "flat").
169  calibIds : `list` of `dict`
170  The list of calibration dataIds to get.
171  gen3Butler : `lsst.daf.butler.Butler`
172  The Butler to use to get the data.
173  """
174  for dataId in calibIds:
175  gen3Exposure = gen3Butler.get(calibName, dataId=dataId)
176  self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
177 
178  def check_defects(self, gen3Butler, detectors):
179  """Test that we can get converted defects from the gen3 repo.
180 
181  Parameters
182  ----------
183  gen3Butler : `lsst.daf.butler.Butler`
184  The Butler to be tested.
185  detector : `int`
186  The detector identifiers to ``get`` from the gen3 butler.
187  """
188  for detector in detectors:
189  dataId = dict(detector=detector, instrument=self.instrumentName)
190  # Fill out the missing parts of the dataId, as we don't a-priori
191  # know e.g. the "calibration_label". Use the first element of the
192  # result because we only need to check one.
193  datasets = list(gen3Butler.registry.queryDatasets("defects", collections=..., dataId=dataId))
194  if datasets:
195  gen3Defects = gen3Butler.get("defects", dataId=datasets[0].dataId)
196  self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
197 
198  def check_refcat(self, gen3Butler):
199  """Test that each expected refcat is in the gen3 repo.
200 
201  Parameters
202  ----------
203  gen3Butler : `lsst.daf.butler.Butler`
204  The Butler to be tested.
205  """
206  if len(self.refcats) > 0:
207  for refcat in self.refcats:
208  query = gen3Butler.registry.queryDatasets(refcat, collections=["refcats"])
209  self.assertGreater(len(list(query)), 0,
210  msg=f"refcat={refcat} has no entries in collection 'refcats'.")
211 
212  def check_collections(self, gen3Butler):
213  """Test that the correct set of collections is in the gen3 repo.
214 
215  Parameters
216  ----------
217  gen3Butler : `lsst.daf.butler.Butler`
218  The Butler to be tested.
219  """
220  self.assertEqual(self.collections, gen3Butler.registry.getAllCollections())
221 
222  def test_convert(self):
223  """Test that raws are converted correctly.
224  """
225  self._run_convert()
226  gen3Butler = lsst.daf.butler.Butler(self.gen3root, run=self.instrumentName)
227  self.check_collections(gen3Butler)
228 
229  # check every raw detector that the gen2 butler knows about
230  detectors = self.gen2Butler.queryMetadata("raw", self.detectorKey)
231  exposures = self.gen2Butler.queryMetadata("raw", self.exposureKey)
232  for exposure, detector in itertools.product(exposures, detectors):
233  self.check_raw(gen3Butler, exposure, detector)
234 
235  self.check_refcat(gen3Butler)
236  self.check_defects(gen3Butler, detectors)
237  self.check_calibs(self.biasName, self.biases, gen3Butler)
238  self.check_calibs(self.flatName, self.flats, gen3Butler)
239  self.check_calibs(self.darkName, self.darks, gen3Butler)
240 
241 
242 def setup_module(module):
244 
245 
246 if __name__ == "__main__":
248  unittest.main()
def check_calibs(self, calibName, calibIds, gen3Butler)
def check_raw(self, gen3Butler, exposure, detector)