lsst.obs.base  20.0.0-15-g34741e2+1
convertTests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Unit test base class for the gen2 to gen3 converter.
23 """
24 
25 import abc
26 import itertools
27 import shutil
28 import tempfile
29 import unittest
30 
31 
32 import lsst.afw.image
33 import lsst.afw.table
34 import lsst.daf.persistence
35 import lsst.daf.butler
36 import lsst.meas.algorithms
37 from lsst.obs.base.script import convert
38 import lsst.utils.tests
39 from lsst.utils import doImport
40 
41 
42 class ConvertGen2To3TestCase(metaclass=abc.ABCMeta):
43  """Test the `butler convert` command.
44 
45  Subclass this, and then `lsst.utils.tests.TestCase` and set the below
46  attributes. Uses the `butler convert` command line command to do the
47  conversion.
48  """
49 
50  gen2root = ""
51  """Root path to the gen2 repo to be converted."""
52 
53  gen2calib = None
54  """Path to the gen2 calib repo to be converted."""
55 
56  @property
57  @abc.abstractmethod
59  """Full path to the `Instrument` class of the data to be converted,
60  e.g. ``lsst.obs.decam.DarkEnergyCamera``.
61 
62  Returns
63  -------
64  className : `str`
65  The fully qualified instrument class name.
66  """
67  pass
68 
69  @property
70  def instrumentClass(self):
71  """The instrument class."""
72  return doImport(self.instrumentClassName)
73 
74  @property
75  def instrumentName(self):
76  """Name of the instrument for the gen3 registry, e.g. "DECam".
77 
78  Returns
79  -------
80  name : `str`
81  The name of the instrument.
82  """
83  return self.instrumentClass.getName()
84 
85  config = None
86  """Full path to a config override for ConvertRepoTask, to be applied after
87  the Instrument overrides when running the conversion function."""
88 
89  biases = []
90  """List dataIds to use to load gen3 biases to test that they exist."""
91 
92  biasName = "bias"
93  """Name of the dataset that the biases are loaded into."""
94 
95  flats = []
96  """List dataIds to use to load gen3 flats to test that they exist."""
97 
98  flatName = "flat"
99  """Name of the dataset that the flats are loaded into."""
100 
101  darks = []
102  """List dataIds to use to load gen3 darks to test that they exist."""
103 
104  darkName = "dark"
105  """Name of the dataset that the darks are loaded into."""
106 
107  kwargs = {}
108  """Other keyword arguments to pass directly to the converter function,
109  as a dict."""
110 
111  refcats = []
112  """Names of the reference catalogs to query for the existence of in the
113  converted gen3 repo."""
114 
115  collections = set()
116  """Additional collections that should appear in the gen3 repo.
117 
118  This will automatically be populated by the base `setUp` to include
119  ``"{instrumentName}/raw"``, ``"refcats"`` (if the ``refcats``
120  class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
121  not `None`).
122  """
123 
124  detectorKey = "ccd"
125  """Key to use in a gen2 dataId to refer to a detector."""
126 
127  exposureKey = "visit"
128  """Key to use in a gen2 dataId to refer to a visit or exposure."""
129 
130  calibFilterType = "physical_filter"
131  """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
132  physical_filter or abstract_filter."""
133 
134  skymapName = None
135  """Name of the Gen3 skymap."""
136 
137  skymapConfig = None
138  """Path to skymap config file defining the new gen3 skymap."""
139 
140  def setUp(self):
141  self.gen3root = tempfile.mkdtemp()
142  self.gen2Butler = lsst.daf.persistence.Butler(root=self.gen2root, calibRoot=self.gen2calib)
143  self.collections = set(type(self).collections)
144  self.collections.add(self.instrumentClass.makeDefaultRawIngestRunName())
145  if len(self.refcats) > 0:
146  self.collections.add("refcats")
147  if self.skymapName is not None:
148  self.collections.add("skymaps")
149  if self.gen2calib:
150  self.collections.add(self.instrumentClass.makeCollectionName("calib"))
151 
152  def tearDown(self):
153  shutil.rmtree(self.gen3root, ignore_errors=True)
154 
155  def _run_convert(self):
156  """Convert a gen2 repo to gen3 for testing.
157  """
158 
159  # Turn on logging
160  log = lsst.log.Log.getLogger("convertRepo")
161  log.setLevel(log.INFO)
162  log.info("Converting %s to %s", self.gen2root, self.gen3root)
163 
164  convert(repo=self.gen3root,
165  gen2root=self.gen2root,
166  instrument=self.instrumentClassName,
167  skymap_name=self.skymapName,
168  skymap_config=self.skymapConfig,
169  config_file=self.config,
170  calibs=self.gen2calib,
171  reruns=None,
172  transfer="auto")
173 
174  def check_raw(self, gen3Butler, exposure, detector):
175  """Check that a raw was converted correctly.
176 
177  Parameters
178  ----------
179  gen3Butler : `lsst.daf.butler.Butler`
180  The Butler to be tested.
181  exposure : `int`
182  The exposure/vist identifier ``get`` from both butlers.
183  detector : `int`
184  The detector identifier to ``get`` from both butlers.
185  """
186  dataIdGen2 = {self.detectorKey: detector, self.exposureKey: exposure}
187  try:
188  gen2Exposure = self.gen2Butler.get("raw", dataId=dataIdGen2)
189  except lsst.daf.persistence.butlerExceptions.NoResults:
190  # ignore datasets that don't actually exist in the gen2 butler.
191  return
192  dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.instrumentName)
193  gen3Exposure = gen3Butler.get("raw", dataId=dataIdGen3)
194  # Check that we got an Exposure, but not what type; there is
195  # inconsistency between different obs packages.
196  self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
197  self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
198  self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
199 
200  def check_calibs(self, calibName, calibIds, gen3Butler):
201  """Test that we can get converted bias/dark/flat from the gen3 repo.
202 
203  Note: because there is no clear way to get calibrations from a gen2
204  repo, we just test that the thing we got is an ExposureF here, and
205  assume that formatter testing is handled properly elsewhere.
206 
207  Parameters
208  ----------
209  calibName : `str`
210  The name of the calibration to attempt to get ("bias", "flat").
211  calibIds : `list` of `dict`
212  The list of calibration dataIds to get.
213  gen3Butler : `lsst.daf.butler.Butler`
214  The Butler to use to get the data.
215  """
216  for dataId in calibIds:
217  with self.subTest(dtype=calibName, dataId=dataId):
218  datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
219  gen3Exposure = gen3Butler.getDirect(datasets[0])
220  self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
221 
222  def check_defects(self, gen3Butler, detectors):
223  """Test that we can get converted defects from the gen3 repo.
224 
225  Parameters
226  ----------
227  gen3Butler : `lsst.daf.butler.Butler`
228  The Butler to be tested.
229  detectors : `list` of `int`
230  The detector identifiers to ``get`` from the gen3 butler.
231  """
232  for detector in detectors:
233  dataId = dict(detector=detector, instrument=self.instrumentName)
234  # Fill out the missing parts of the dataId, as we don't a-priori
235  # know e.g. the "calibration_label". Use the first element of the
236  # result because we only need to check one.
237  with self.subTest(dtype="defects", dataId=dataId):
238  datasets = list(gen3Butler.registry.queryDatasets("defects", collections=..., dataId=dataId))
239  if datasets:
240  gen3Defects = gen3Butler.getDirect(datasets[0])
241  self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
242 
243  def check_refcat(self, gen3Butler):
244  """Test that each expected refcat is in the gen3 repo.
245 
246  Parameters
247  ----------
248  gen3Butler : `lsst.daf.butler.Butler`
249  The Butler to be tested.
250  """
251  if len(self.refcats) > 0:
252  for refcat in self.refcats:
253  query = gen3Butler.registry.queryDatasets(refcat, collections=["refcats"])
254  self.assertGreater(len(list(query)), 0,
255  msg=f"refcat={refcat} has no entries in collection 'refcats'.")
256 
257  def check_collections(self, gen3Butler):
258  """Test that the correct set of collections is in the gen3 repo.
259 
260  Parameters
261  ----------
262  gen3Butler : `lsst.daf.butler.Butler`
263  The Butler to be tested.
264  """
265  self.assertEqual(set(gen3Butler.registry.queryCollections()), self.collections,
266  f"Compare with expected collections ({self.collections})")
267 
268  def test_convert(self):
269  """Test that all data are converted correctly.
270  """
271  self._run_convert()
272  gen3Butler = lsst.daf.butler.Butler(self.gen3root,
273  collections=self.instrumentClass.makeDefaultRawIngestRunName())
274  self.check_collections(gen3Butler)
275 
276  # check every raw detector that the gen2 butler knows about
277  detectors = self.gen2Butler.queryMetadata("raw", self.detectorKey)
278  exposures = self.gen2Butler.queryMetadata("raw", self.exposureKey)
279  for exposure, detector in itertools.product(exposures, detectors):
280  with self.subTest(mode="raw", exposure=exposure, detector=detector):
281  self.check_raw(gen3Butler, exposure, detector)
282 
283  self.check_refcat(gen3Butler)
284  self.check_defects(gen3Butler, detectors)
285  self.check_calibs(self.biasName, self.biases, gen3Butler)
286  self.check_calibs(self.flatName, self.flats, gen3Butler)
287  self.check_calibs(self.darkName, self.darks, gen3Butler)
288 
289 
290 def setup_module(module):
292 
293 
294 if __name__ == "__main__":
296  unittest.main()
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.test_convert
def test_convert(self)
Definition: convertTests.py:268
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_collections
def check_collections(self, gen3Butler)
Definition: convertTests.py:257
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.detectorKey
string detectorKey
Definition: convertTests.py:124
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biases
list biases
Definition: convertTests.py:89
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase._run_convert
def _run_convert(self)
Definition: convertTests.py:155
lsst.obs.base.gen2to3.convertTests.setup_module
def setup_module(module)
Definition: convertTests.py:290
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_raw
def check_raw(self, gen3Butler, exposure, detector)
Definition: convertTests.py:174
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.setUp
def setUp(self)
Definition: convertTests.py:140
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flats
list flats
Definition: convertTests.py:95
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darks
list darks
Definition: convertTests.py:101
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen3root
gen3root
Definition: convertTests.py:141
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2calib
gen2calib
Definition: convertTests.py:53
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.tearDown
def tearDown(self)
Definition: convertTests.py:152
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darkName
string darkName
Definition: convertTests.py:104
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.refcats
list refcats
Definition: convertTests.py:111
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2root
string gen2root
Definition: convertTests.py:50
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.config
config
Definition: convertTests.py:85
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentClassName
def instrumentClassName(self)
Definition: convertTests.py:58
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2Butler
gen2Butler
Definition: convertTests.py:142
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_defects
def check_defects(self, gen3Butler, detectors)
Definition: convertTests.py:222
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapName
skymapName
Definition: convertTests.py:134
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flatName
string flatName
Definition: convertTests.py:98
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biasName
string biasName
Definition: convertTests.py:92
lsst::utils
cmd.commands.convert
def convert(*args, **kwargs)
Definition: commands.py:51
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_calibs
def check_calibs(self, calibName, calibIds, gen3Butler)
Definition: convertTests.py:200
lsst::utils::tests
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentClass
def instrumentClass(self)
Definition: convertTests.py:70
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.exposureKey
string exposureKey
Definition: convertTests.py:127
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapConfig
skymapConfig
Definition: convertTests.py:137
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentName
def instrumentName(self)
Definition: convertTests.py:75
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase
Definition: convertTests.py:42
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.collections
collections
Definition: convertTests.py:115
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_refcat
def check_refcat(self, gen3Butler)
Definition: convertTests.py:243
lsst.obs.base.script
Definition: __init__.py:1
lsst::utils::tests::init
def init()