lsst.obs.base  20.0.0-45-g887e66a+a9aa579b69
convertTests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Unit test base class for the gen2 to gen3 converter.
23 """
24 
25 import abc
26 import itertools
27 import shutil
28 import tempfile
29 import unittest
30 
31 
32 import lsst.afw.image
33 import lsst.afw.table
34 import lsst.daf.persistence
35 import lsst.daf.butler
36 import lsst.meas.algorithms
37 from lsst.obs.base.script import convert
38 import lsst.utils.tests
39 from lsst.utils import doImport
40 
41 
42 class ConvertGen2To3TestCase(metaclass=abc.ABCMeta):
43  """Test the `butler convert` command.
44 
45  Subclass this, and then `lsst.utils.tests.TestCase` and set the below
46  attributes. Uses the `butler convert` command line command to do the
47  conversion.
48  """
49 
50  gen2root = ""
51  """Root path to the gen2 repo to be converted."""
52 
53  gen2calib = None
54  """Path to the gen2 calib repo to be converted."""
55 
56  @property
57  @abc.abstractmethod
59  """Full path to the `Instrument` class of the data to be converted,
60  e.g. ``lsst.obs.decam.DarkEnergyCamera``.
61 
62  Returns
63  -------
64  className : `str`
65  The fully qualified instrument class name.
66  """
67  pass
68 
69  @property
70  def instrumentClass(self):
71  """The instrument class."""
72  return doImport(self.instrumentClassName)
73 
74  @property
75  def instrumentName(self):
76  """Name of the instrument for the gen3 registry, e.g. "DECam".
77 
78  Returns
79  -------
80  name : `str`
81  The name of the instrument.
82  """
83  return self.instrumentClass.getName()
84 
85  config = None
86  """Full path to a config override for ConvertRepoTask, to be applied after
87  the Instrument overrides when running the conversion function."""
88 
89  biases = []
90  """List dataIds to use to load gen3 biases to test that they exist."""
91 
92  biasName = "bias"
93  """Name of the dataset that the biases are loaded into."""
94 
95  flats = []
96  """List dataIds to use to load gen3 flats to test that they exist."""
97 
98  flatName = "flat"
99  """Name of the dataset that the flats are loaded into."""
100 
101  darks = []
102  """List dataIds to use to load gen3 darks to test that they exist."""
103 
104  darkName = "dark"
105  """Name of the dataset that the darks are loaded into."""
106 
107  kwargs = {}
108  """Other keyword arguments to pass directly to the converter function,
109  as a dict."""
110 
111  refcats = []
112  """Names of the reference catalogs to query for the existence of in the
113  converted gen3 repo."""
114 
115  collections = set()
116  """Additional collections that should appear in the gen3 repo.
117 
118  This will automatically be populated by the base `setUp` to include
119  ``"{instrumentName}/raw"``, ``"refcats"`` (if the ``refcats``
120  class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
121  not `None`).
122  """
123 
124  detectorKey = "ccd"
125  """Key to use in a gen2 dataId to refer to a detector."""
126 
127  exposureKey = "visit"
128  """Key to use in a gen2 dataId to refer to a visit or exposure."""
129 
130  calibFilterType = "physical_filter"
131  """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
132  physical_filter or abstract_filter."""
133 
134  skymapName = None
135  """Name of the Gen3 skymap."""
136 
137  skymapConfig = None
138  """Path to skymap config file defining the new gen3 skymap."""
139 
140  def setUp(self):
141  self.gen3root = tempfile.mkdtemp()
142  self.gen2Butler = lsst.daf.persistence.Butler(root=self.gen2root, calibRoot=self.gen2calib)
143  self.collections = set(type(self).collections)
144  self.collections.add(self.instrumentClass.makeDefaultRawIngestRunName())
145  if len(self.refcats) > 0:
146  self.collections.add("refcats")
147  if self.skymapName is not None:
148  self.collections.add("skymaps")
149 
150  # We always write a default calibration collection
151  # containing at least the camera
152  self.collections.add(self.instrumentClass.makeCollectionName("calib"))
153 
154  def tearDown(self):
155  shutil.rmtree(self.gen3root, ignore_errors=True)
156 
157  def _run_convert(self):
158  """Convert a gen2 repo to gen3 for testing.
159  """
160 
161  # Turn on logging
162  log = lsst.log.Log.getLogger("convertRepo")
163  log.setLevel(log.INFO)
164  log.info("Converting %s to %s", self.gen2root, self.gen3root)
165 
166  convert(repo=self.gen3root,
167  gen2root=self.gen2root,
168  skymap_name=self.skymapName,
169  skymap_config=self.skymapConfig,
170  config_file=self.config,
171  calibs=self.gen2calib,
172  reruns=None,
173  transfer="auto")
174 
175  def check_raw(self, gen3Butler, exposure, detector):
176  """Check that a raw was converted correctly.
177 
178  Parameters
179  ----------
180  gen3Butler : `lsst.daf.butler.Butler`
181  The Butler to be tested.
182  exposure : `int`
183  The exposure/vist identifier ``get`` from both butlers.
184  detector : `int`
185  The detector identifier to ``get`` from both butlers.
186  """
187  dataIdGen2 = {self.detectorKey: detector, self.exposureKey: exposure}
188  try:
189  gen2Exposure = self.gen2Butler.get("raw", dataId=dataIdGen2)
190  except lsst.daf.persistence.butlerExceptions.NoResults:
191  # ignore datasets that don't actually exist in the gen2 butler.
192  return
193  dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.instrumentName)
194  gen3Exposure = gen3Butler.get("raw", dataId=dataIdGen3)
195  # Check that we got an Exposure, but not what type; there is
196  # inconsistency between different obs packages.
197  self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
198  self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
199  self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
200 
201  def check_calibs(self, calibName, calibIds, gen3Butler):
202  """Test that we can get converted bias/dark/flat from the gen3 repo.
203 
204  Note: because there is no clear way to get calibrations from a gen2
205  repo, we just test that the thing we got is an ExposureF here, and
206  assume that formatter testing is handled properly elsewhere.
207 
208  Parameters
209  ----------
210  calibName : `str`
211  The name of the calibration to attempt to get ("bias", "flat").
212  calibIds : `list` of `dict`
213  The list of calibration dataIds to get.
214  gen3Butler : `lsst.daf.butler.Butler`
215  The Butler to use to get the data.
216  """
217  for dataId in calibIds:
218  with self.subTest(dtype=calibName, dataId=dataId):
219  datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
220  gen3Exposure = gen3Butler.getDirect(datasets[0])
221  self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
222 
223  def check_defects(self, gen3Butler, detectors):
224  """Test that we can get converted defects from the gen3 repo.
225 
226  Parameters
227  ----------
228  gen3Butler : `lsst.daf.butler.Butler`
229  The Butler to be tested.
230  detectors : `list` of `int`
231  The detector identifiers to ``get`` from the gen3 butler.
232  """
233  for detector in detectors:
234  dataId = dict(detector=detector, instrument=self.instrumentName)
235  # Fill out the missing parts of the dataId, as we don't a-priori
236  # know e.g. the "calibration_label". Use the first element of the
237  # result because we only need to check one.
238  with self.subTest(dtype="defects", dataId=dataId):
239  datasets = list(gen3Butler.registry.queryDatasets("defects", collections=..., dataId=dataId))
240  if datasets:
241  gen3Defects = gen3Butler.getDirect(datasets[0])
242  self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
243 
244  def check_refcat(self, gen3Butler):
245  """Test that each expected refcat is in the gen3 repo.
246 
247  Parameters
248  ----------
249  gen3Butler : `lsst.daf.butler.Butler`
250  The Butler to be tested.
251  """
252  if len(self.refcats) > 0:
253  for refcat in self.refcats:
254  query = gen3Butler.registry.queryDatasets(refcat, collections=["refcats"])
255  self.assertGreater(len(list(query)), 0,
256  msg=f"refcat={refcat} has no entries in collection 'refcats'.")
257 
258  def check_collections(self, gen3Butler):
259  """Test that the correct set of collections is in the gen3 repo.
260 
261  Parameters
262  ----------
263  gen3Butler : `lsst.daf.butler.Butler`
264  The Butler to be tested.
265  """
266  self.assertEqual(set(gen3Butler.registry.queryCollections()), self.collections,
267  f"Compare with expected collections ({self.collections})")
268 
269  def test_convert(self):
270  """Test that all data are converted correctly.
271  """
272  self._run_convert()
273  gen3Butler = lsst.daf.butler.Butler(self.gen3root,
274  collections=self.instrumentClass.makeDefaultRawIngestRunName())
275  self.check_collections(gen3Butler)
276 
277  # check every raw detector that the gen2 butler knows about
278  detectors = self.gen2Butler.queryMetadata("raw", self.detectorKey)
279  exposures = self.gen2Butler.queryMetadata("raw", self.exposureKey)
280  for exposure, detector in itertools.product(exposures, detectors):
281  with self.subTest(mode="raw", exposure=exposure, detector=detector):
282  self.check_raw(gen3Butler, exposure, detector)
283 
284  self.check_refcat(gen3Butler)
285  self.check_defects(gen3Butler, detectors)
286  self.check_calibs(self.biasName, self.biases, gen3Butler)
287  self.check_calibs(self.flatName, self.flats, gen3Butler)
288  self.check_calibs(self.darkName, self.darks, gen3Butler)
289 
290 
291 def setup_module(module):
293 
294 
295 if __name__ == "__main__":
297  unittest.main()
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.test_convert
def test_convert(self)
Definition: convertTests.py:269
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_collections
def check_collections(self, gen3Butler)
Definition: convertTests.py:258
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.detectorKey
string detectorKey
Definition: convertTests.py:124
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biases
list biases
Definition: convertTests.py:89
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase._run_convert
def _run_convert(self)
Definition: convertTests.py:157
lsst.obs.base.gen2to3.convertTests.setup_module
def setup_module(module)
Definition: convertTests.py:291
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_raw
def check_raw(self, gen3Butler, exposure, detector)
Definition: convertTests.py:175
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.setUp
def setUp(self)
Definition: convertTests.py:140
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flats
list flats
Definition: convertTests.py:95
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darks
list darks
Definition: convertTests.py:101
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen3root
gen3root
Definition: convertTests.py:141
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2calib
gen2calib
Definition: convertTests.py:53
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.tearDown
def tearDown(self)
Definition: convertTests.py:154
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darkName
string darkName
Definition: convertTests.py:104
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.refcats
list refcats
Definition: convertTests.py:111
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2root
string gen2root
Definition: convertTests.py:50
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.config
config
Definition: convertTests.py:85
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentClassName
def instrumentClassName(self)
Definition: convertTests.py:58
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2Butler
gen2Butler
Definition: convertTests.py:142
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_defects
def check_defects(self, gen3Butler, detectors)
Definition: convertTests.py:223
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapName
skymapName
Definition: convertTests.py:134
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flatName
string flatName
Definition: convertTests.py:98
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biasName
string biasName
Definition: convertTests.py:92
lsst::utils
cmd.commands.convert
def convert(*args, **kwargs)
Definition: commands.py:52
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_calibs
def check_calibs(self, calibName, calibIds, gen3Butler)
Definition: convertTests.py:201
lsst::utils::tests
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentClass
def instrumentClass(self)
Definition: convertTests.py:70
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.exposureKey
string exposureKey
Definition: convertTests.py:127
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapConfig
skymapConfig
Definition: convertTests.py:137
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentName
def instrumentName(self)
Definition: convertTests.py:75
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase
Definition: convertTests.py:42
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.collections
collections
Definition: convertTests.py:115
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_refcat
def check_refcat(self, gen3Butler)
Definition: convertTests.py:244
lsst.obs.base.script
Definition: __init__.py:1
lsst::utils::tests::init
def init()