lsst.obs.base  19.0.0-64-gf672fef+5
convertTests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Unit test base class for the gen2 to gen3 converter.
23 """
24 
25 import itertools
26 import shutil
27 import tempfile
28 import unittest
29 
30 
31 import lsst.afw.image
32 import lsst.afw.table
33 import lsst.daf.persistence
34 import lsst.daf.butler
35 import lsst.meas.algorithms
36 import lsst.utils.tests
37 from ..script.convertGen2RepoToGen3 import convert as convertGen2RepoToGen3
38 
39 
41  """Test the `convert_gen2_repo_to_gen3.py` script.
42 
43  Subclass this, and then `lsst.utils.tests.TestCase` and set the below
44  attributes. Uses the `lsst.obs.base.script.convertGen2RepoToGen3.convert`
45  function to do the conversion.
46  """
47  gen2root = ""
48  """Root path to the gen2 repo to be converted."""
49 
50  gen2calib = None
51  """Path to the gen2 calib repo to be converted."""
52 
53  instrumentName = None
54  """Name of the instrument for the gen3 registry, e.g. "DECam"."""
55 
56  instrumentClass = None
57  """Full path to the `Instrument` class of the data to be converted, e.g.
58  ``lsst.obs.decam.DarkEnergyCamera``."""
59 
60  config = None
61  """Full path to a config override for ConvertRepoTask, to be applied after
62  the Instrument overrides when running the conversion function."""
63 
64  biases = []
65  """List dataIds to use to load gen3 biases to test that they exist."""
66 
67  biasName = "bias"
68  """Name of the dataset that the biases are loaded into."""
69 
70  flats = []
71  """List dataIds to use to load gen3 flats to test that they exist."""
72 
73  flatName = "flat"
74  """Name of the dataset that the flats are loaded into."""
75 
76  darks = []
77  """List dataIds to use to load gen3 darks to test that they exist."""
78 
79  darkName = "dark"
80  """Name of the dataset that the darks are loaded into."""
81 
82  kwargs = {}
83  """Other keyword arguments to pass directly to the converter function,
84  as a dict."""
85 
86  refcats = []
87  """Names of the reference catalogs to query for the existence of in the
88  converted gen3 repo."""
89 
90  collections = set()
91  """Additional collections that should appear in the gen3 repo.
92 
93  This will automatically be populated by the base `setUp` to include
94  ``"raw/{instrumentName}"``, ``"refcats"`` (if the ``refcats``
95  class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
96  not `None`).
97  """
98 
99  detectorKey = "ccd"
100  """Key to use in a gen2 dataId to refer to a detector."""
101 
102  exposureKey = "visit"
103  """Key to use in a gen2 dataId to refer to a visit or exposure."""
104 
105  calibFilterType = "physical_filter"
106  """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
107  physical_filter or abstract_filter."""
108 
109  skymapName = None
110  """Name of the Gen3 skymap."""
111 
112  skymapConfig = None
113  """Path to skymap config file defining the new gen3 skymap."""
114 
115  def setUp(self):
116  self.gen3root = tempfile.mkdtemp()
117  self.gen2Butler = lsst.daf.persistence.Butler(root=self.gen2root, calibRoot=self.gen2calib)
118  self.collections = set(type(self).collections)
119  self.collections.add(f"raw/{self.instrumentName}")
120  if len(self.refcats) > 0:
121  self.collections.add("refcats")
122  if self.skymapName is not None:
123  self.collections.add("skymaps")
124 
125  def tearDown(self):
126  shutil.rmtree(self.gen3root, ignore_errors=True)
127 
128  def _run_convert(self):
129  """Convert a gen2 repo to gen3 for testing.
130  """
131 
132  # Turn on logging
133  log = lsst.log.Log.getLogger("convertRepo")
134  log.setLevel(log.INFO)
135  log.info("Converting %s to %s", self.gen2root, self.gen3root)
136 
137  # Run the conversion
138  convertGen2RepoToGen3(self.gen2root, self.gen3root, self.instrumentClass,
139  skymapName=self.skymapName, skymapConfig=self.skymapConfig,
140  config=self.config, calibs=self.gen2calib)
141 
142  def check_raw(self, gen3Butler, exposure, detector):
143  """Check that a raw was converted correctly.
144 
145  Parameters
146  ----------
147  gen3Butler : `lsst.daf.butler.Butler`
148  The Butler to be tested.
149  exposure : `int`
150  The exposure/vist identifier ``get`` from both butlers.
151  detector : `int`
152  The detector identifier to ``get`` from both butlers.
153  """
154  dataIdGen2 = {self.detectorKey: detector, self.exposureKey: exposure}
155  try:
156  gen2Exposure = self.gen2Butler.get("raw", dataId=dataIdGen2)
157  except lsst.daf.persistence.butlerExceptions.NoResults:
158  # ignore datasets that don't actually exist in the gen2 butler.
159  return
160  dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.instrumentName)
161  gen3Exposure = gen3Butler.get("raw", dataId=dataIdGen3)
162  # Check that we got an Exposure, but not what type; there is
163  # inconsistency between different obs packages.
164  self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
165  self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
166  self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
167 
168  def check_calibs(self, calibName, calibIds, gen3Butler):
169  """Test that we can get converted bias/dark/flat from the gen3 repo.
170 
171  Note: because there is no clear way to get calibrations from a gen2
172  repo, we just test that the thing we got is an ExposureF here, and
173  assume that formatter testing is handled properly elsewhere.
174 
175  Parameters
176  ----------
177  calibName : `str`
178  The name of the calibration to attempt to get ("bias", "flat").
179  calibIds : `list` of `dict`
180  The list of calibration dataIds to get.
181  gen3Butler : `lsst.daf.butler.Butler`
182  The Butler to use to get the data.
183  """
184  for dataId in calibIds:
185  with self.subTest(dtype=calibName, dataId=dataId):
186  datasets = list(gen3Butler.registry.queryDatasets(calibName, collections=..., dataId=dataId))
187  gen3Exposure = gen3Butler.getDirect(datasets[0])
188  self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
189 
190  def check_defects(self, gen3Butler, detectors):
191  """Test that we can get converted defects from the gen3 repo.
192 
193  Parameters
194  ----------
195  gen3Butler : `lsst.daf.butler.Butler`
196  The Butler to be tested.
197  detectors : `list` of `int`
198  The detector identifiers to ``get`` from the gen3 butler.
199  """
200  for detector in detectors:
201  dataId = dict(detector=detector, instrument=self.instrumentName)
202  # Fill out the missing parts of the dataId, as we don't a-priori
203  # know e.g. the "calibration_label". Use the first element of the
204  # result because we only need to check one.
205  with self.subTest(dtype="defects", dataId=dataId):
206  datasets = list(gen3Butler.registry.queryDatasets("defects", collections=..., dataId=dataId))
207  if datasets:
208  gen3Defects = gen3Butler.getDirect(datasets[0])
209  self.assertIsInstance(gen3Defects, lsst.meas.algorithms.Defects)
210 
211  def check_refcat(self, gen3Butler):
212  """Test that each expected refcat is in the gen3 repo.
213 
214  Parameters
215  ----------
216  gen3Butler : `lsst.daf.butler.Butler`
217  The Butler to be tested.
218  """
219  if len(self.refcats) > 0:
220  for refcat in self.refcats:
221  query = gen3Butler.registry.queryDatasets(refcat, collections=["refcats"])
222  self.assertGreater(len(list(query)), 0,
223  msg=f"refcat={refcat} has no entries in collection 'refcats'.")
224 
225  def check_collections(self, gen3Butler):
226  """Test that the correct set of collections is in the gen3 repo.
227 
228  Parameters
229  ----------
230  gen3Butler : `lsst.daf.butler.Butler`
231  The Butler to be tested.
232  """
233  self.assertEqual(set(gen3Butler.registry.queryCollections()), self.collections,
234  f"Compare with expected collections ({self.collections})")
235 
236  def test_convert(self):
237  """Test that all data are converted correctly.
238  """
239  self._run_convert()
240  gen3Butler = lsst.daf.butler.Butler(self.gen3root, collections=f"raw/{self.instrumentName}")
241  self.check_collections(gen3Butler)
242 
243  # check every raw detector that the gen2 butler knows about
244  detectors = self.gen2Butler.queryMetadata("raw", self.detectorKey)
245  exposures = self.gen2Butler.queryMetadata("raw", self.exposureKey)
246  for exposure, detector in itertools.product(exposures, detectors):
247  with self.subTest(mode="raw", exposure=exposure, detector=detector):
248  self.check_raw(gen3Butler, exposure, detector)
249 
250  self.check_refcat(gen3Butler)
251  self.check_defects(gen3Butler, detectors)
252  self.check_calibs(self.biasName, self.biases, gen3Butler)
253  self.check_calibs(self.flatName, self.flats, gen3Butler)
254  self.check_calibs(self.darkName, self.darks, gen3Butler)
255 
256 
257 def setup_module(module):
259 
260 
261 if __name__ == "__main__":
263  unittest.main()
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.test_convert
def test_convert(self)
Definition: convertTests.py:236
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_collections
def check_collections(self, gen3Butler)
Definition: convertTests.py:225
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.detectorKey
string detectorKey
Definition: convertTests.py:99
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biases
list biases
Definition: convertTests.py:64
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase._run_convert
def _run_convert(self)
Definition: convertTests.py:128
lsst.obs.base.gen2to3.convertTests.setup_module
def setup_module(module)
Definition: convertTests.py:257
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_raw
def check_raw(self, gen3Butler, exposure, detector)
Definition: convertTests.py:142
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.setUp
def setUp(self)
Definition: convertTests.py:115
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flats
list flats
Definition: convertTests.py:70
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darks
list darks
Definition: convertTests.py:76
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentName
instrumentName
Definition: convertTests.py:53
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen3root
gen3root
Definition: convertTests.py:116
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2calib
gen2calib
Definition: convertTests.py:50
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.tearDown
def tearDown(self)
Definition: convertTests.py:125
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.darkName
string darkName
Definition: convertTests.py:79
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.refcats
list refcats
Definition: convertTests.py:86
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2root
string gen2root
Definition: convertTests.py:47
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.config
config
Definition: convertTests.py:60
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.gen2Butler
gen2Butler
Definition: convertTests.py:117
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.instrumentClass
instrumentClass
Definition: convertTests.py:56
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_defects
def check_defects(self, gen3Butler, detectors)
Definition: convertTests.py:190
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapName
skymapName
Definition: convertTests.py:109
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.flatName
string flatName
Definition: convertTests.py:73
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.biasName
string biasName
Definition: convertTests.py:67
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_calibs
def check_calibs(self, calibName, calibIds, gen3Butler)
Definition: convertTests.py:168
lsst::utils::tests
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.exposureKey
string exposureKey
Definition: convertTests.py:102
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.skymapConfig
skymapConfig
Definition: convertTests.py:112
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase
Definition: convertTests.py:40
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.collections
collections
Definition: convertTests.py:90
lsst.obs.base.gen2to3.convertTests.ConvertGen2To3TestCase.check_refcat
def check_refcat(self, gen3Butler)
Definition: convertTests.py:211
lsst::utils::tests::init
def init()