Coverage for tests/jointcalTestBase.py: 17%
110 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-08 01:04 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-08 01:04 -0700
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["importRepository", "JointcalTestBase"]
24import os
25import shutil
26import tempfile
27import numpy as np
29import lsst.afw.image.utils
30from lsst.ctrl.mpexec import SimplePipelineExecutor
31import lsst.daf.butler
32from lsst.daf.butler.script import ingest_files
33import lsst.pipe.base
34import lsst.obs.base
35import lsst.geom
36from lsst.verify.bin.jobReporter import JobReporter
38import lsst.jointcal
41def importRepository(instrument, exportPath, exportFile, outputDir=None,
42 refcats=None, refcatPath=""):
43 """Import a gen3 test repository into a test path.
45 Parameters
46 ----------
47 instrument : `str`
48 Full string name for the instrument.
49 exportPath : `str`
50 Path to location of repository to export.
51 exportFile : `str`
52 Filename of YAML butler export data, describing the data to import.
53 outputDir : `str`, or `None`
54 Root path to put the test repo in; appended with `testrepo/`.
55 If not supplied, a temporary path is generated with mkdtemp; this path
56 must be deleted by the calling code, it is not automatically removed.
57 refcats : `dict` [`str`, `str`]
58 Mapping of refcat name to relative path to .ecsv ingest file.
59 refcatPath : `str`
60 Path prefix for refcat ingest files and files linked therein.
62 Returns
63 -------
64 repopath : `str`
65 The path to the newly created butler repo.
66 """
67 if outputDir is None:
68 repopath = tempfile.mkdtemp()
69 else:
70 repopath = os.path.join(outputDir, "testrepo")
72 # Make the repo and retrieve a writeable Butler
73 _ = lsst.daf.butler.Butler.makeRepo(repopath)
74 butler = lsst.daf.butler.Butler(repopath, writeable=True)
76 instrInstance = lsst.pipe.base.Instrument.from_string(instrument)
77 instrInstance.register(butler.registry)
79 # Register refcats first, so the `refcats` collection will exist.
80 if refcats is not None:
81 for name, file in refcats.items():
82 graph = butler.registry.dimensions.extract(["htm7"])
83 datasetType = lsst.daf.butler.DatasetType(name, graph, "SimpleCatalog",
84 universe=butler.registry.dimensions)
85 butler.registry.registerDatasetType(datasetType)
86 ingest_files(repopath, name, "refcats", file, prefix=refcatPath)
87 # New butler to get the refcats collections to be visible
88 butler = lsst.daf.butler.Butler(repopath, writeable=True)
90 butler.import_(directory=exportPath, filename=exportFile,
91 transfer='symlink',
92 skip_dimensions={'instrument', 'detector', 'physical_filter'})
93 return repopath
96class JointcalTestBase:
97 """
98 Base class for jointcal tests, to genericize some test running and setup.
100 Derive from this first, then from TestCase.
101 """
103 def set_output_dir(self):
104 """Set the output directory to the name of the test method, in .test/
105 """
106 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1])
108 def setUp_base(self,
109 instrumentClass,
110 instrumentName,
111 input_dir="",
112 all_visits=None,
113 log_level=None,
114 where="",
115 refcats=None,
116 refcatPath="",
117 inputCollections=None,
118 outputDataId=None):
119 """
120 Call from your child classes's setUp() to get the necessary variables built.
122 Parameters
123 ----------
124 instrumentClass : `str`
125 The full module name of the instrument to be registered in the
126 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam"
127 instrumentName : `str`
128 The name of the instrument as it appears in the repo collections.
129 For example, "HSC".
130 input_dir : `str`
131 Directory of input butler repository.
132 all_visits : `list` [`int`]
133 List of the available visits to generate the dataset query from.
134 log_level : `str`
135 Set to the default log level you want jointcal to produce while the
136 tests are running. See the developer docs about logging for valid
137 levels: https://developer.lsst.io/coding/logging.html
138 where : `str`
139 Data ID query for pipetask specifying the data to run on.
140 refcats : `dict` [`str`, `str`]
141 Mapping of refcat name to relative path to .ecsv ingest file.
142 refcatPath : `str`
143 Path prefix for refcat ingest files and files linked therein.
144 inputCollections : `list` [`str`]
145 String to use for "-i" input collections (comma delimited).
146 For example, "refcats,HSC/runs/tests,HSC/calib"
147 outputDataId : `dict`
148 Partial dataIds for testing whether the output files were written.
149 """
150 self.path = os.path.dirname(__file__)
152 self.instrumentClass = instrumentClass
153 self.instrumentName = instrumentName
155 self.input_dir = input_dir
156 self.all_visits = all_visits
157 self.log_level = log_level
159 self.configfiles = []
161 # Make unittest output more verbose failure messages to assert failures.
162 self.longMessage = True
164 self.where = where
165 self.refcats = refcats
166 self.refcatPath = refcatPath
167 self.inputCollections = inputCollections
169 self.outputDataId = outputDataId
171 # Ensure that the filter list is reset for each test so that we avoid
172 # confusion or contamination from other instruments.
173 lsst.obs.base.FilterDefinitionCollection.reset()
175 self.set_output_dir()
177 def tearDown(self):
178 shutil.rmtree(self.output_dir, ignore_errors=True)
180 def _test_metrics(self, result, expect):
181 """Test a dictionary of "metrics" against those returned by jointcal.py
183 Parameters
184 ----------
185 result : `dict`
186 Result metric dictionary from jointcal.py
187 expect : `dict`
188 Expected metric dictionary; set a value to None to not test it.
189 """
190 for key in result:
191 with self.subTest(key.metric):
192 if expect[key.metric] is not None:
193 value = result[key].quantity.value
194 if isinstance(value, float):
195 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5)
196 else:
197 self.assertEqual(value, expect[key.metric], msg=key.metric)
199 def _runJointcal(self, repo,
200 inputCollections, outputCollection,
201 configFiles=None, configOptions=None,
202 registerDatasetTypes=False, whereSuffix=None,
203 nJobs=1):
204 """Run a pipeline via the SimplePipelineExecutor.
206 Parameters
207 ----------
208 repo : `str`
209 Gen3 Butler repository to read from/write to.
210 inputCollections : `list` [`str`]
211 String to use for "-i" input collections (comma delimited).
212 For example, "refcats,HSC/runs/tests,HSC/calib"
213 outputCollection : `str`
214 Name of the output collection to write to. For example,
215 "HSC/testdata/jointcal"
216 configFiles : `list` [`str`], optional
217 List of jointcal config files to use.
218 configOptions : `dict` [`str`], optional
219 Individual jointcal config options (field: value) to override.
220 registerDatasetTypes : bool, optional
221 Set "--register-dataset-types" when running the pipeline.
222 whereSuffix : `str`, optional
223 Additional parameters to the ``where`` pipetask statement.
224 nJobs : `int`, optional
225 Number of quanta expected to be run.
227 Returns
228 -------
229 job : `lsst.verify.Job`
230 Job containing the metric measurements from this test run.
231 """
232 config = lsst.jointcal.JointcalConfig()
233 if configFiles:
234 for file in configFiles:
235 config.load(file)
236 if configOptions:
237 for key, value in configOptions.items():
238 setattr(config, key, value)
239 where = ' '.join((self.where, whereSuffix)) if whereSuffix is not None else self.where
240 lsst.daf.butler.cli.cliLog.CliLog.initLog(False)
241 butler = SimplePipelineExecutor.prep_butler(repo,
242 inputs=inputCollections,
243 output=outputCollection,
244 output_run=outputCollection.replace("all", "jointcal"))
245 executor = SimplePipelineExecutor.from_task_class(lsst.jointcal.JointcalTask,
246 config=config,
247 where=where,
248 butler=butler)
249 executor.run(register_dataset_types=registerDatasetTypes)
250 # JobReporter bundles all metrics in the collection into one job.
251 jobs = JobReporter(repo, outputCollection, "jointcal", "", "jointcal").run()
252 # should only ever get one job output in tests, unless specified
253 self.assertEqual(len(jobs), nJobs)
254 # Sort the jobs, as QuantumGraph ordering is not guaranteed, and some
255 # tests check metrics values for one job while producing two.
256 sorted_jobs = {key: jobs[key] for key in sorted(list(jobs.keys()))}
257 return list(sorted_jobs.values())[0]
259 def _runJointcalTest(self, astrometryOutputs=None, photometryOutputs=None,
260 configFiles=None, configOptions=None, whereSuffix=None,
261 metrics=None, nJobs=1):
262 """Create a Butler repo and run jointcal on it.
264 Parameters
265 ----------
266 atsrometryOutputs, photometryOutputs : `dict` [`int`, `list`]
267 visit: [detectors] dictionary to test that output files were saved.
268 Default None means that there should be no output for that
269 respective dataset type.
270 configFiles : `list` [`str`], optional
271 List of jointcal config files to use.
272 configOptions : `dict` [`str`], optional
273 Individual jointcal config options (field: value) to override.
274 whereSuffix : `str`, optional
275 Additional parameters to the ``where`` pipetask statement.
276 metrics : `dict`, optional
277 Dictionary of 'metricName': value to test jointcal's result.metrics
278 against.
279 nJobs : `int`, optional
280 Number of quanta expected to be run.
282 Returns
283 -------
284 repopath : `str`
285 The path to the newly created butler repo.
286 """
287 repopath = importRepository(self.instrumentClass,
288 os.path.join(self.input_dir, "repo"),
289 os.path.join(self.input_dir, "exports.yaml"),
290 self.output_dir,
291 refcats=self.refcats,
292 refcatPath=self.refcatPath)
293 configs = [os.path.join(self.path, "config/config.py")]
294 configs.extend(self.configfiles or [])
295 configs.extend(configFiles or [])
296 collection = f"{self.instrumentName}/tests/all"
297 job = self._runJointcal(repopath,
298 self.inputCollections,
299 collection,
300 configFiles=configs,
301 configOptions=configOptions,
302 registerDatasetTypes=True,
303 whereSuffix=whereSuffix,
304 nJobs=nJobs)
306 if metrics:
307 self._test_metrics(job.measurements, metrics)
309 butler = lsst.daf.butler.Butler(repopath, collections=collection)
310 if astrometryOutputs is not None:
311 for visit, detectors in astrometryOutputs.items():
312 self._check_astrometry_output(butler, visit, detectors)
313 else:
314 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalSkyWcsCatalog"))), 0)
315 if photometryOutputs is not None:
316 for visit, detectors in photometryOutputs.items():
317 self._check_photometry_output(butler, visit, detectors)
318 else:
319 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalPhotoCalibCatalog"))), 0)
321 return repopath
323 def _check_astrometry_output(self, butler, visit, detectors):
324 """Check that the WCSs were written for each detector, and only for the
325 correct detectors.
326 """
327 dataId = self.outputDataId.copy()
328 dataId['visit'] = visit
330 catalog = butler.get('jointcalSkyWcsCatalog', dataId)
331 for record in catalog:
332 self.assertIsInstance(record.getWcs(), lsst.afw.geom.SkyWcs,
333 msg=f"visit {visit}: {record}")
334 np.testing.assert_array_equal(catalog['id'], detectors)
336 def _check_photometry_output(self, butler, visit, detectors):
337 """Check that the PhotoCalibs were written for each detector, and only
338 for the correct detectors.
339 """
340 dataId = self.outputDataId.copy()
341 dataId['visit'] = visit
343 catalog = butler.get('jointcalPhotoCalibCatalog', dataId)
344 for record in catalog:
345 self.assertIsInstance(record.getPhotoCalib(), lsst.afw.image.PhotoCalib,
346 msg=f"visit {visit}: {record}")
347 np.testing.assert_array_equal(catalog['id'], detectors)