lsst.obs.base  20.0.0-37-g38a3e24+ce95e46117
convertTests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Unit test base class for the gen2 to gen3 converter.
23 """
24 
25 import abc
26 import itertools
27 import shutil
28 import tempfile
29 import unittest
30 
31 
32 import lsst.afw.image
33 import lsst.afw.table
34 import lsst.daf.persistence
35 import lsst.daf.butler
36 import lsst.meas.algorithms
37 from lsst.obs.base.script import convert
38 import lsst.utils.tests
39 from lsst.utils import doImport
40 
41 
42 class ConvertGen2To3TestCase(metaclass=abc.ABCMeta):
43  """Test the `butler convert` command.
44 
45  Subclass this, and then `lsst.utils.tests.TestCase` and set the below
46  attributes. Uses the `butler convert` command line command to do the
47  conversion.
48  """
49 
50  gen2root = ""
51  """Root path to the gen2 repo to be converted."""
52 
53  gen2calib = None
54  """Path to the gen2 calib repo to be converted."""
55 
56  @property
57  @abc.abstractmethod
59  """Full path to the `Instrument` class of the data to be converted,
60  e.g. ``lsst.obs.decam.DarkEnergyCamera``.
61 
62  Returns
63  -------
64  className : `str`
65  The fully qualified instrument class name.
66  """
67  pass
68 
69  @property
70  def instrumentClass(self):
71  """The instrument class."""
72  return doImport(self.instrumentClassName)
73 
74  @property
75  def instrumentName(self):
76  """Name of the instrument for the gen3 registry, e.g. "DECam".
77 
78  Returns
79  -------
80  name : `str`
81  The name of the instrument.
82  """
83  return self.instrumentClass.getName()
84 
85  config = None
86  """Full path to a config override for ConvertRepoTask, to be applied after
87  the Instrument overrides when running the conversion function."""
88 
89  biases = []
90  """List dataIds to use to load gen3 biases to test that they exist."""
91 
92  biasName = "bias"
93  """Name of the dataset that the biases are loaded into."""
94 
95  flats = []
96  """List dataIds to use to load gen3 flats to test that they exist."""
97 
98  flatName = "flat"
99  """Name of the dataset that the flats are loaded into."""
100 
101  darks = []
102  """List dataIds to use to load gen3 darks to test that they exist."""
103 
104  darkName = "dark"
105  """Name of the dataset that the darks are loaded into."""
106 
107  kwargs = {}
108  """Other keyword arguments to pass directly to the converter function,
109  as a dict."""
110 
111  refcats = []
112  """Names of the reference catalogs to query for the existence of in the
113  converted gen3 repo."""
114 
115  collections = set()
116  """Additional collections that should appear in the gen3 repo.
117 
118  This will automatically be populated by the base `setUp` to include
119  ``"{instrumentName}/raw"``, ``"refcats"`` (if the ``refcats``
120  class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
121  not `None`).
122  """
123 
124  detectorKey = "ccd"
125  """Key to use in a gen2 dataId to refer to a detector."""
126 
127  exposureKey = "visit"
128  """Key to use in a gen2 dataId to refer to a visit or exposure."""
129 
130  calibFilterType = "physical_filter"
131  """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
132  physical_filter or abstract_filter."""
133 
134  skymapName = None
135  """Name of the Gen3 skymap."""
136 
137  skymapConfig = None
138  """Path to skymap config file defining the new gen3 skymap."""
139 
140  def setUp(self):
141  self.gen3root = tempfile.mkdtemp()
142  self.gen2Butler = lsst.daf.persistence.Butler(root=self.gen2root, calibRoot=self.gen2calib)
143  self.collections = set(type(self).collections)
144  self.collections.add(self.instrumentClass.makeDefaultRawIngestRunName())
145  if len(self.refcats) > 0:
146  self.collections.add("refcats")
147  if self.skymapName is not None:
148  self.collections.add("skymaps")
149  if self.gen2calib:
150  self.collections.add(self.instrumentClass.makeCollectionName("calib"))
151 
152  def tearDown(self):
153  shutil.rmtree(self.gen3root, ignore_errors=True)
154 
155  def _run_convert(self):
156  """Convert a gen2 repo to gen3 for testing.
157  """
158 
159  # Turn on logging
160  log = lsst.log.Log.getLogger("convertRepo")
161  log.setLevel(log.INFO)
162  log.info("Converting %s to %s", self.gen2root, self.gen3root)
163 
164  convert(repo=self.gen3root,
165  gen2root=self.gen2root,
166  skymap_name=self.skymapName,
167  skymap_config=self.skymapConfig,
168  config_file=self.config,
169  calibs=self.gen2calib,
170  reruns=None,
171  transfer="auto")
172 
173  def check_raw(self, gen3Butler, exposure, detector):
174  """Check that a raw was converted correctly.
175 
176  Parameters
177  ----------
178  gen3Butler : `lsst.daf.butler.Butler`
179  The Butler to be tested.
180  exposure : `int`
181  The exposure/vist identifier ``get`` from both butlers.
182  detector : `int`
183  The detector identifier to ``get`` from both butlers.
184  """
185  dataIdGen2 = {self.detectorKey: detector, self.exposureKey: exposure}
186  try:
187  gen2Exposure = self.gen2Butler.get("raw", dataId=dataIdGen2)
188  except lsst.daf.persistence.butlerExceptions.NoResults:
189  # ignore datasets that don't actually exist in the gen2 butler.
190  return
191  dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.instrumentName)
192  gen3Exposure = gen3Butler.get("raw", dataId=dataIdGen3)
193  # Check that we got an Exposure, but not what type; there is
194  # inconsistency between different obs packages.
195  self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
196  self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
197  self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
198 
199  def check_calibs(self, calibName, calibIds, gen3Butler):
200  """Test that we can get converted bias/dark/flat from the gen3 repo.
201 
202  Note: because there is no clear way to get calibrations from a gen2
203  repo, we just test that the thing we got is an ExposureF here, and
204  assume that formatter testing is handled properly elsewhere.
205 
206  Parameters
207  ----------
208  calibName : `str`
209  The name of the calibration to attempt to get ("bias", "flat").
210  calibIds : `list` of `dict`
211  The list of calibration dataIds to get.
212  gen3Butler : `lsst.daf.butler.Butler`
213  The Butler to use to get the data.
214  """
215  for dataId in calibIds:
216  with self.subTest(dtype=calibName, dataId=dataId):
217  datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
218  gen3Exposure = gen3Butler.getDirect(datasets[0])
219  self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
220 
221  def check_defects(self, gen3Butler, detectors):
222  """Test that we can get converted defects from the gen3 repo.
223 
224  Parameters
225  ----------
226  gen3Butler : `lsst.daf.butler.Butler`
227  The Butler to be tested.
228  detectors : `list` of `int`
229  The detector identifiers to ``get`` from the gen3 butler.
230  """
231  for detector in detectors:
232  dataId = dict(detector=detector, instrument=self.instrumentName)
233  # Fill out the missing parts of the dataId, as we don't a-priori
234  # know e.g. the "calibration_label". Use the first element of the
235  # result because we only need to check one.
236  with self.subTest(dtype="defects", dataId=dataId):
237  datasets = list(gen3Butler.registry.queryDatasets("defects", collections=..., dataId=dataId))
238  if datasets:
239  gen3Defects = gen3Butler.getDirect(datasets[0])
240  self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
241 
242  def check_refcat(self, gen3Butler):
243  """Test that each expected refcat is in the gen3 repo.
244 
245  Parameters
246  ----------
247  gen3Butler : `lsst.daf.butler.Butler`
248  The Butler to be tested.
249  """
250  if len(self.refcats) > 0:
251  for refcat in self.refcats:
252  query = gen3Butler.registry.queryDatasets(refcat, collections=["refcats"])
253  self.assertGreater(len(list(query)), 0,
254  msg=f"refcat={refcat} has no entries in collection 'refcats'.")
255 
256  def check_collections(self, gen3Butler):
257  """Test that the correct set of collections is in the gen3 repo.
258 
259  Parameters
260  ----------
261  gen3Butler : `lsst.daf.butler.Butler`
262  The Butler to be tested.
263  """
264  self.assertEqual(set(gen3Butler.registry.queryCollections()), self.collections,
265  f"Compare with expected collections ({self.collections})")
266 
267  def test_convert(self):
268  """Test that all data are converted correctly.
269  """
270  self._run_convert()
271  gen3Butler = lsst.daf.butler.Butler(self.gen3root,
272  collections=self.instrumentClass.makeDefaultRawIngestRunName())
273  self.check_collections(gen3Butler)
274 
275  # check every raw detector that the gen2 butler knows about
276  detectors = self.gen2Butler.queryMetadata("raw", self.detectorKey)
277  exposures = self.gen2Butler.queryMetadata("raw", self.exposureKey)
278  for exposure, detector in itertools.product(exposures, detectors):
279  with self.subTest(mode="raw", exposure=exposure, detector=detector):
280  self.check_raw(gen3Butler, exposure, detector)
281 
282  self.check_refcat(gen3Butler)
283  self.check_defects(gen3Butler, detectors)
284  self.check_calibs(self.biasName, self.biases, gen3Butler)
285  self.check_calibs(self.flatName, self.flats, gen3Butler)
286  self.check_calibs(self.darkName, self.darks, gen3Butler)
287 
288 
289 def setup_module(module):
291 
292 
293 if __name__ == "__main__":
295  unittest.main()
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.test_convert
def test_convert(self)
Definition: convertTests.py:267
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_collections
def check_collections(self, gen3Butler)
Definition: convertTests.py:256
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.detectorKey
string detectorKey
Definition: convertTests.py:124
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biases
list biases
Definition: convertTests.py:89
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase._run_convert
def _run_convert(self)
Definition: convertTests.py:155
lsst.obs.base.gen2to3.convertTests.setup_module
def setup_module(module)
Definition: convertTests.py:289
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_raw
def check_raw(self, gen3Butler, exposure, detector)
Definition: convertTests.py:173
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.setUp
def setUp(self)
Definition: convertTests.py:140
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flats
list flats
Definition: convertTests.py:95
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darks
list darks
Definition: convertTests.py:101
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen3root
gen3root
Definition: convertTests.py:141
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2calib
gen2calib
Definition: convertTests.py:53
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.tearDown
def tearDown(self)
Definition: convertTests.py:152
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darkName
string darkName
Definition: convertTests.py:104
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.refcats
list refcats
Definition: convertTests.py:111
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2root
string gen2root
Definition: convertTests.py:50
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.config
config
Definition: convertTests.py:85
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentClassName
def instrumentClassName(self)
Definition: convertTests.py:58
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2Butler
gen2Butler
Definition: convertTests.py:142
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_defects
def check_defects(self, gen3Butler, detectors)
Definition: convertTests.py:221
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapName
skymapName
Definition: convertTests.py:134
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flatName
string flatName
Definition: convertTests.py:98
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biasName
string biasName
Definition: convertTests.py:92
lsst::utils
cmd.commands.convert
def convert(*args, **kwargs)
Definition: commands.py:48
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_calibs
def check_calibs(self, calibName, calibIds, gen3Butler)
Definition: convertTests.py:199
lsst::utils::tests
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentClass
def instrumentClass(self)
Definition: convertTests.py:70
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.exposureKey
string exposureKey
Definition: convertTests.py:127
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapConfig
skymapConfig
Definition: convertTests.py:137
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentName
def instrumentName(self)
Definition: convertTests.py:75
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase
Definition: convertTests.py:42
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.collections
collections
Definition: convertTests.py:115
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_refcat
def check_refcat(self, gen3Butler)
Definition: convertTests.py:242
lsst.obs.base.script
Definition: __init__.py:1
lsst::utils::tests::init
def init()