Coverage for tests/jointcalTestBase.py: 16%
109 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-25 03:38 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-25 03:38 -0800
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["importRepository", "JointcalTestBase"]
24import os
25import shutil
26import tempfile
27import numpy as np
29import lsst.afw.image.utils
30from lsst.ctrl.mpexec import SimplePipelineExecutor
31import lsst.daf.butler
32from lsst.daf.butler.script import ingest_files
33import lsst.pipe.base
34import lsst.obs.base
35import lsst.geom
36from lsst.verify.bin.jobReporter import JobReporter
38import lsst.jointcal
41def importRepository(instrument, exportPath, exportFile, outputDir=None,
42 refcats=None, refcatPath=""):
43 """Import a gen3 test repository into a test path.
45 Parameters
46 ----------
47 instrument : `str`
48 Full string name for the instrument.
49 exportPath : `str`
50 Path to location of repository to export.
51 exportFile : `str`
52 Filename of YAML butler export data, describing the data to import.
53 outputDir : `str`, or `None`
54 Root path to put the test repo in; appended with `testrepo/`.
55 If not supplied, a temporary path is generated with mkdtemp; this path
56 must be deleted by the calling code, it is not automatically removed.
57 refcats : `dict` [`str`, `str`]
58 Mapping of refcat name to relative path to .ecsv ingest file.
59 refcatPath : `str`
60 Path prefix for refcat ingest files and files linked therein.
62 Returns
63 -------
64 repopath : `str`
65 The path to the newly created butler repo.
66 """
67 if outputDir is None:
68 repopath = tempfile.mkdtemp()
69 else:
70 repopath = os.path.join(outputDir, "testrepo")
72 # Make the repo and retrieve a writeable Butler
73 _ = lsst.daf.butler.Butler.makeRepo(repopath)
74 butler = lsst.daf.butler.Butler(repopath, writeable=True)
76 instrInstance = lsst.pipe.base.Instrument.from_string(instrument)
77 instrInstance.register(butler.registry)
79 # Register refcats first, so the `refcats` collection will exist.
80 if refcats is not None:
81 for name, file in refcats.items():
82 graph = butler.registry.dimensions.extract(["htm7"])
83 datasetType = lsst.daf.butler.DatasetType(name, graph, "SimpleCatalog",
84 universe=butler.registry.dimensions)
85 butler.registry.registerDatasetType(datasetType)
86 ingest_files(repopath, name, "refcats", file, prefix=refcatPath)
87 # New butler to get the refcats collections to be visible
88 butler = lsst.daf.butler.Butler(repopath, writeable=True)
90 butler.import_(directory=exportPath, filename=exportFile,
91 transfer='symlink',
92 skip_dimensions={'instrument', 'detector', 'physical_filter'})
93 return repopath
96class JointcalTestBase:
97 """
98 Base class for jointcal tests, to genericize some test running and setup.
100 Derive from this first, then from TestCase.
101 """
103 def set_output_dir(self):
104 """Set the output directory to the name of the test method, in .test/
105 """
106 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1])
108 def setUp_base(self,
109 instrumentClass,
110 instrumentName,
111 input_dir="",
112 all_visits=None,
113 log_level=None,
114 where="",
115 refcats=None,
116 refcatPath="",
117 inputCollections=None,
118 outputDataId=None):
119 """
120 Call from your child classes's setUp() to get the necessary variables built.
122 Parameters
123 ----------
124 instrumentClass : `str`
125 The full module name of the instrument to be registered in the
126 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam"
127 instrumentName : `str`
128 The name of the instrument as it appears in the repo collections.
129 For example, "HSC".
130 input_dir : `str`
131 Directory of input butler repository.
132 all_visits : `list` [`int`]
133 List of the available visits to generate the dataset query from.
134 log_level : `str`
135 Set to the default log level you want jointcal to produce while the
136 tests are running. See the developer docs about logging for valid
137 levels: https://developer.lsst.io/coding/logging.html
138 where : `str`
139 Data ID query for pipetask specifying the data to run on.
140 refcats : `dict` [`str`, `str`]
141 Mapping of refcat name to relative path to .ecsv ingest file.
142 refcatPath : `str`
143 Path prefix for refcat ingest files and files linked therein.
144 inputCollections : `list` [`str`]
145 String to use for "-i" input collections (comma delimited).
146 For example, "refcats,HSC/runs/tests,HSC/calib"
147 outputDataId : `dict`
148 Partial dataIds for testing whether the output files were written.
149 """
150 self.path = os.path.dirname(__file__)
152 self.instrumentClass = instrumentClass
153 self.instrumentName = instrumentName
155 self.input_dir = input_dir
156 self.all_visits = all_visits
157 self.log_level = log_level
159 self.configfiles = []
161 # Make unittest output more verbose failure messages to assert failures.
162 self.longMessage = True
164 self.where = where
165 self.refcats = refcats
166 self.refcatPath = refcatPath
167 self.inputCollections = inputCollections
169 self.outputDataId = outputDataId
171 self.set_output_dir()
173 def tearDown(self):
174 shutil.rmtree(self.output_dir, ignore_errors=True)
176 def _test_metrics(self, result, expect):
177 """Test a dictionary of "metrics" against those returned by jointcal.py
179 Parameters
180 ----------
181 result : `dict`
182 Result metric dictionary from jointcal.py
183 expect : `dict`
184 Expected metric dictionary; set a value to None to not test it.
185 """
186 for key in result:
187 with self.subTest(key.metric):
188 if expect[key.metric] is not None:
189 value = result[key].quantity.value
190 if isinstance(value, float):
191 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5)
192 else:
193 self.assertEqual(value, expect[key.metric], msg=key.metric)
195 def _runJointcal(self, repo,
196 inputCollections, outputCollection,
197 configFiles=None, configOptions=None,
198 registerDatasetTypes=False, whereSuffix=None,
199 nJobs=1):
200 """Run a pipeline via the SimplePipelineExecutor.
202 Parameters
203 ----------
204 repo : `str`
205 Gen3 Butler repository to read from/write to.
206 inputCollections : `list` [`str`]
207 String to use for "-i" input collections (comma delimited).
208 For example, "refcats,HSC/runs/tests,HSC/calib"
209 outputCollection : `str`
210 Name of the output collection to write to. For example,
211 "HSC/testdata/jointcal"
212 configFiles : `list` [`str`], optional
213 List of jointcal config files to use.
214 configOptions : `dict` [`str`], optional
215 Individual jointcal config options (field: value) to override.
216 registerDatasetTypes : bool, optional
217 Set "--register-dataset-types" when running the pipeline.
218 whereSuffix : `str`, optional
219 Additional parameters to the ``where`` pipetask statement.
220 nJobs : `int`, optional
221 Number of quanta expected to be run.
223 Returns
224 -------
225 job : `lsst.verify.Job`
226 Job containing the metric measurements from this test run.
227 """
228 config = lsst.jointcal.JointcalConfig()
229 if configFiles:
230 for file in configFiles:
231 config.load(file)
232 if configOptions:
233 for key, value in configOptions.items():
234 setattr(config, key, value)
235 where = ' '.join((self.where, whereSuffix)) if whereSuffix is not None else self.where
236 lsst.daf.butler.cli.cliLog.CliLog.initLog(False)
237 butler = SimplePipelineExecutor.prep_butler(repo,
238 inputs=inputCollections,
239 output=outputCollection,
240 output_run=outputCollection.replace("all", "jointcal"))
241 executor = SimplePipelineExecutor.from_task_class(lsst.jointcal.JointcalTask,
242 config=config,
243 where=where,
244 butler=butler)
245 executor.run(register_dataset_types=registerDatasetTypes)
246 # JobReporter bundles all metrics in the collection into one job.
247 jobs = JobReporter(repo, outputCollection, "jointcal", "", "jointcal").run()
248 # should only ever get one job output in tests, unless specified
249 self.assertEqual(len(jobs), nJobs)
250 # Sort the jobs, as QuantumGraph ordering is not guaranteed, and some
251 # tests check metrics values for one job while producing two.
252 sorted_jobs = {key: jobs[key] for key in sorted(list(jobs.keys()))}
253 return list(sorted_jobs.values())[0]
255 def _runJointcalTest(self, astrometryOutputs=None, photometryOutputs=None,
256 configFiles=None, configOptions=None, whereSuffix=None,
257 metrics=None, nJobs=1):
258 """Create a Butler repo and run jointcal on it.
260 Parameters
261 ----------
262 atsrometryOutputs, photometryOutputs : `dict` [`int`, `list`]
263 visit: [detectors] dictionary to test that output files were saved.
264 Default None means that there should be no output for that
265 respective dataset type.
266 configFiles : `list` [`str`], optional
267 List of jointcal config files to use.
268 configOptions : `dict` [`str`], optional
269 Individual jointcal config options (field: value) to override.
270 whereSuffix : `str`, optional
271 Additional parameters to the ``where`` pipetask statement.
272 metrics : `dict`, optional
273 Dictionary of 'metricName': value to test jointcal's result.metrics
274 against.
275 nJobs : `int`, optional
276 Number of quanta expected to be run.
278 Returns
279 -------
280 repopath : `str`
281 The path to the newly created butler repo.
282 """
283 repopath = importRepository(self.instrumentClass,
284 os.path.join(self.input_dir, "repo"),
285 os.path.join(self.input_dir, "exports.yaml"),
286 self.output_dir,
287 refcats=self.refcats,
288 refcatPath=self.refcatPath)
289 configs = [os.path.join(self.path, "config/config.py")]
290 configs.extend(self.configfiles or [])
291 configs.extend(configFiles or [])
292 collection = f"{self.instrumentName}/tests/all"
293 job = self._runJointcal(repopath,
294 self.inputCollections,
295 collection,
296 configFiles=configs,
297 configOptions=configOptions,
298 registerDatasetTypes=True,
299 whereSuffix=whereSuffix,
300 nJobs=nJobs)
302 if metrics:
303 self._test_metrics(job.measurements, metrics)
305 butler = lsst.daf.butler.Butler(repopath, collections=collection)
306 if astrometryOutputs is not None:
307 for visit, detectors in astrometryOutputs.items():
308 self._check_astrometry_output(butler, visit, detectors)
309 else:
310 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalSkyWcsCatalog"))), 0)
311 if photometryOutputs is not None:
312 for visit, detectors in photometryOutputs.items():
313 self._check_photometry_output(butler, visit, detectors)
314 else:
315 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalPhotoCalibCatalog"))), 0)
317 return repopath
319 def _check_astrometry_output(self, butler, visit, detectors):
320 """Check that the WCSs were written for each detector, and only for the
321 correct detectors.
322 """
323 dataId = self.outputDataId.copy()
324 dataId['visit'] = visit
326 catalog = butler.get('jointcalSkyWcsCatalog', dataId)
327 for record in catalog:
328 self.assertIsInstance(record.getWcs(), lsst.afw.geom.SkyWcs,
329 msg=f"visit {visit}: {record}")
330 np.testing.assert_array_equal(catalog['id'], detectors)
332 def _check_photometry_output(self, butler, visit, detectors):
333 """Check that the PhotoCalibs were written for each detector, and only
334 for the correct detectors.
335 """
336 dataId = self.outputDataId.copy()
337 dataId['visit'] = visit
339 catalog = butler.get('jointcalPhotoCalibCatalog', dataId)
340 for record in catalog:
341 self.assertIsInstance(record.getPhotoCalib(), lsst.afw.image.PhotoCalib,
342 msg=f"visit {visit}: {record}")
343 np.testing.assert_array_equal(catalog['id'], detectors)