lsst.obs.base  19.0.0-51-gb87bce2
convertTests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Unit test base class for the gen2 to gen3 converter.
23 """
24 
25 import itertools
26 import shutil
27 import tempfile
28 import unittest
29 
30 
31 import lsst.afw.image
32 import lsst.afw.table
33 import lsst.daf.persistence
34 import lsst.daf.butler
35 import lsst.meas.algorithms
36 import lsst.utils.tests
37 from ..script.convertGen2RepoToGen3 import convert as convertGen2RepoToGen3
38 
39 
41  """Test the `convert_gen2_repo_to_gen3.py` script.
42 
43  Subclass this, and then `lsst.utils.tests.TestCase` and set the below
44  attributes. Uses the `lsst.obs.base.script.convertGen2RepoToGen3.convert`
45  function to do the conversion.
46  """
47  gen2root = ""
48  """Root path to the gen2 repo to be converted."""
49 
50  gen2calib = ""
51  """Path to the gen2 calib repo to be converted."""
52 
53  instrumentName = None
54  """Name of the instrument for the gen3 registry, e.g. "DECam"."""
55 
56  instrumentClass = None
57  """Full path to the `Instrument` class of the data to be converted, e.g.
58  ``lsst.obs.decam.DarkEnergyCamera``."""
59 
60  config = None
61  """Full path to a config override for ConvertRepoTask, to be applied after
62  the Instrument overrides when running the conversion function."""
63 
64  biases = []
65  """List dataIds to use to load gen3 biases to test that they exist."""
66 
67  biasName = "bias"
68  """Name of the dataset that the biases are loaded into."""
69 
70  flats = []
71  """List dataIds to use to load gen3 flats to test that they exist."""
72 
73  flatName = "flat"
74  """Name of the dataset that the flats are loaded into."""
75 
76  darks = []
77  """List dataIds to use to load gen3 darks to test that they exist."""
78 
79  darkName = "dark"
80  """Name of the dataset that the darks are loaded into."""
81 
82  kwargs = {}
83  """Other keyword arguments to pass directly to the converter function,
84  as a dict."""
85 
86  refcats = []
87  """Names of the reference catalogs to query for the existence of in the
88  converted gen3 repo."""
89 
90  collections = set()
91  """Additional collections that should appear in the gen3 repo.
92 
93  This will automatically be populated by the base `setUp` to include
94  ``"raw/{instrumentName}"``, ``"refcats"`` (if the ``refcats``
95  class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
96  not `None`).
97  """
98 
99  detectorKey = "ccd"
100  """Key to use in a gen2 dataId to refer to a detector."""
101 
102  exposureKey = "visit"
103  """Key to use in a gen2 dataId to refer to a visit or exposure."""
104 
105  calibFilterType = "physical_filter"
106  """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
107  physical_filter or abstract_filter."""
108 
109  skymapName = None
110  """Name of the Gen3 skymap."""
111 
112  skymapConfig = None
113  """Path to skymap config file defining the new gen3 skymap."""
114 
115  def setUp(self):
116  self.gen3root = tempfile.mkdtemp()
117  self.gen2Butler = lsst.daf.persistence.Butler(root=self.gen2root, calibRoot=self.gen2calib)
118  self.collections = set(type(self).collections)
119  self.collections.add(f"raw/{self.instrumentName}")
120  if len(self.refcats) > 0:
121  self.collections.add("refcats")
122  if self.skymapName is not None:
123  self.collections.add("skymaps")
124 
125  def tearDown(self):
126  shutil.rmtree(self.gen3root, ignore_errors=True)
127 
128  def _run_convert(self):
129  """Convert a gen2 repo to gen3 for testing.
130  """
131 
132  # Turn on logging
133  log = lsst.log.Log.getLogger("convertRepo")
134  log.setLevel(log.INFO)
135  log.info("Converting %s to %s", self.gen2root, self.gen3root)
136 
137  # Run the conversion
139  self.calibFilterType,
140  skymapName=self.skymapName, skymapConfig=self.skymapConfig,
141  config=self.config, calibs=self.gen2calib)
142 
143  def check_raw(self, gen3Butler, exposure, detector):
144  """Check that a raw was converted correctly.
145 
146  Parameters
147  ----------
148  gen3Butler : `lsst.daf.butler.Butler`
149  The Butler to be tested.
150  exposure : `int`
151  The exposure/vist identifier ``get`` from both butlers.
152  detector : `int`
153  The detector identifier to ``get`` from both butlers.
154  """
155  dataIdGen2 = {self.detectorKey: detector, self.exposureKey: exposure}
156  try:
157  gen2Exposure = self.gen2Butler.get("raw", dataId=dataIdGen2)
158  except lsst.daf.persistence.butlerExceptions.NoResults:
159  # ignore datasets that don't actually exist in the gen2 butler.
160  return
161  dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.instrumentName)
162  gen3Exposure = gen3Butler.get("raw", dataId=dataIdGen3)
163  # Check that we got an Exposure, but not what type; there is
164  # inconsistency between different obs packages.
165  self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
166  self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
167  self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
168 
169  def check_calibs(self, calibName, calibIds, gen3Butler):
170  """Test that we can get converted bias/dark/flat from the gen3 repo.
171 
172  Note: because there is no clear way to get calibrations from a gen2
173  repo, we just test that the thing we got is an ExposureF here, and
174  assume that formatter testing is handled properly elsewhere.
175 
176  Parameters
177  ----------
178  calibName : `str`
179  The name of the calibration to attempt to get ("bias", "flat").
180  calibIds : `list` of `dict`
181  The list of calibration dataIds to get.
182  gen3Butler : `lsst.daf.butler.Butler`
183  The Butler to use to get the data.
184  """
185  for dataId in calibIds:
186  with self.subTest(dtype=calibName, dataId=dataId):
187  datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
188  gen3Exposure = gen3Butler.getDirect(datasets[0])
189  self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
190 
191  def check_defects(self, gen3Butler, detectors):
192  """Test that we can get converted defects from the gen3 repo.
193 
194  Parameters
195  ----------
196  gen3Butler : `lsst.daf.butler.Butler`
197  The Butler to be tested.
198  detectors : `list` of `int`
199  The detector identifiers to ``get`` from the gen3 butler.
200  """
201  for detector in detectors:
202  dataId = dict(detector=detector, instrument=self.instrumentName)
203  # Fill out the missing parts of the dataId, as we don't a-priori
204  # know e.g. the "calibration_label". Use the first element of the
205  # result because we only need to check one.
206  with self.subTest(dtype="defects", dataId=dataId):
207  datasets = list(gen3Butler.registry.queryDatasets("defects", collections=..., dataId=dataId))
208  if datasets:
209  gen3Defects = gen3Butler.getDirect(datasets[0])
210  self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
211 
212  def check_refcat(self, gen3Butler):
213  """Test that each expected refcat is in the gen3 repo.
214 
215  Parameters
216  ----------
217  gen3Butler : `lsst.daf.butler.Butler`
218  The Butler to be tested.
219  """
220  if len(self.refcats) > 0:
221  for refcat in self.refcats:
222  query = gen3Butler.registry.queryDatasets(refcat, collections=["refcats"])
223  self.assertGreater(len(list(query)), 0,
224  msg=f"refcat={refcat} has no entries in collection 'refcats'.")
225 
226  def check_collections(self, gen3Butler):
227  """Test that the correct set of collections is in the gen3 repo.
228 
229  Parameters
230  ----------
231  gen3Butler : `lsst.daf.butler.Butler`
232  The Butler to be tested.
233  """
234  self.assertEqual(set(gen3Butler.registry.queryCollections()), self.collections,
235  f"Compare with expected collections ({self.collections})")
236 
237  def test_convert(self):
238  """Test that all data are converted correctly.
239  """
240  self._run_convert()
241  gen3Butler = lsst.daf.butler.Butler(self.gen3root, collections=f"raw/{self.instrumentName}")
242  self.check_collections(gen3Butler)
243 
244  # check every raw detector that the gen2 butler knows about
245  detectors = self.gen2Butler.queryMetadata("raw", self.detectorKey)
246  exposures = self.gen2Butler.queryMetadata("raw", self.exposureKey)
247  for exposure, detector in itertools.product(exposures, detectors):
248  with self.subTest(mode="raw", exposure=exposure, detector=detector):
249  self.check_raw(gen3Butler, exposure, detector)
250 
251  self.check_refcat(gen3Butler)
252  self.check_defects(gen3Butler, detectors)
253  self.check_calibs(self.biasName, self.biases, gen3Butler)
254  self.check_calibs(self.flatName, self.flats, gen3Butler)
255  self.check_calibs(self.darkName, self.darks, gen3Butler)
256 
257 
258 def setup_module(module):
260 
261 
262 if __name__ == "__main__":
264  unittest.main()
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.test_convert
def test_convert(self)
Definition: convertTests.py:237
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_collections
def check_collections(self, gen3Butler)
Definition: convertTests.py:226
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.detectorKey
string detectorKey
Definition: convertTests.py:99
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biases
list biases
Definition: convertTests.py:64
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase._run_convert
def _run_convert(self)
Definition: convertTests.py:128
lsst.obs.base.gen2to3.convertTests.setup_module
def setup_module(module)
Definition: convertTests.py:258
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_raw
def check_raw(self, gen3Butler, exposure, detector)
Definition: convertTests.py:143
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.setUp
def setUp(self)
Definition: convertTests.py:115
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flats
list flats
Definition: convertTests.py:70
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darks
list darks
Definition: convertTests.py:76
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentName
instrumentName
Definition: convertTests.py:53
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen3root
gen3root
Definition: convertTests.py:116
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.tearDown
def tearDown(self)
Definition: convertTests.py:125
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darkName
string darkName
Definition: convertTests.py:79
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.refcats
list refcats
Definition: convertTests.py:86
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2root
string gen2root
Definition: convertTests.py:47
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.config
config
Definition: convertTests.py:60
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.calibFilterType
string calibFilterType
Definition: convertTests.py:105
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2Butler
gen2Butler
Definition: convertTests.py:117
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentClass
instrumentClass
Definition: convertTests.py:56
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2calib
string gen2calib
Definition: convertTests.py:50
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_defects
def check_defects(self, gen3Butler, detectors)
Definition: convertTests.py:191
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapName
skymapName
Definition: convertTests.py:109
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flatName
string flatName
Definition: convertTests.py:73
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biasName
string biasName
Definition: convertTests.py:67
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_calibs
def check_calibs(self, calibName, calibIds, gen3Butler)
Definition: convertTests.py:169
lsst::utils::tests
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.exposureKey
string exposureKey
Definition: convertTests.py:102
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapConfig
skymapConfig
Definition: convertTests.py:112
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase
Definition: convertTests.py:40
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.collections
collections
Definition: convertTests.py:90
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_refcat
def check_refcat(self, gen3Butler)
Definition: convertTests.py:212
lsst::utils::tests::init
def init()
convertGen2RepoToGen3
Definition: convertGen2RepoToGen3.py:1