Coverage for tests/jointcalTestBase.py: 17%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["importRepository", "JointcalTestBase"]
24import os
25import shutil
26import tempfile
27import numpy as np
29import lsst.afw.image.utils
30from lsst.ctrl.mpexec import SimplePipelineExecutor
31import lsst.daf.butler
32from lsst.daf.butler.script import ingest_files
33import lsst.obs.base
34import lsst.geom
35from lsst.verify.bin.jobReporter import JobReporter
37import lsst.jointcal
40def importRepository(instrument, exportPath, exportFile, outputDir=None,
41 refcats=None, refcatPath=""):
42 """Import a gen3 test repository into a test path.
44 Parameters
45 ----------
46 instrument : `str`
47 Full string name for the instrument.
48 exportPath : `str`
49 Path to location of repository to export.
50 exportFile : `str`
51 Filename of YAML butler export data, describing the data to import.
52 outputDir : `str`, or `None`
53 Root path to put the test repo in; appended with `testrepo/`.
54 If not supplied, a temporary path is generated with mkdtemp; this path
55 must be deleted by the calling code, it is not automatically removed.
56 refcats : `dict` [`str`, `str`]
57 Mapping of refcat name to relative path to .ecsv ingest file.
58 refcatPath : `str`
59 Path prefix for refcat ingest files and files linked therein.
61 Returns
62 -------
63 repopath : `str`
64 The path to the newly created butler repo.
65 """
66 if outputDir is None:
67 repopath = tempfile.mkdtemp()
68 else:
69 repopath = os.path.join(outputDir, "testrepo")
71 # Make the repo and retrieve a writeable Butler
72 _ = lsst.daf.butler.Butler.makeRepo(repopath)
73 butler = lsst.daf.butler.Butler(repopath, writeable=True)
75 instrInstance = lsst.obs.base.utils.getInstrument(instrument)
76 instrInstance.register(butler.registry)
78 # Register refcats first, so the `refcats` collection will exist.
79 if refcats is not None:
80 for name, file in refcats.items():
81 graph = butler.registry.dimensions.extract(["htm7"])
82 datasetType = lsst.daf.butler.DatasetType(name, graph, "SimpleCatalog",
83 universe=butler.registry.dimensions)
84 butler.registry.registerDatasetType(datasetType)
85 ingest_files(repopath, name, "refcats", file, prefix=refcatPath)
86 # New butler to get the refcats collections to be visible
87 butler = lsst.daf.butler.Butler(repopath, writeable=True)
89 butler.import_(directory=exportPath, filename=exportFile,
90 transfer='symlink',
91 skip_dimensions={'instrument', 'detector', 'physical_filter'})
92 return repopath
95class JointcalTestBase:
96 """
97 Base class for jointcal tests, to genericize some test running and setup.
99 Derive from this first, then from TestCase.
100 """
102 def set_output_dir(self):
103 """Set the output directory to the name of the test method, in .test/
104 """
105 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1])
107 def setUp_base(self,
108 instrumentClass,
109 instrumentName,
110 input_dir="",
111 all_visits=None,
112 log_level=None,
113 where="",
114 refcats=None,
115 refcatPath="",
116 inputCollections=None,
117 outputDataId=None):
118 """
119 Call from your child classes's setUp() to get the necessary variables built.
121 Parameters
122 ----------
123 instrumentClass : `str`
124 The full module name of the instrument to be registered in the
125 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam"
126 instrumentName : `str`
127 The name of the instrument as it appears in the repo collections.
128 For example, "HSC".
129 input_dir : `str`
130 Directory of input butler repository.
131 all_visits : `list` [`int`]
132 List of the available visits to generate the dataset query from.
133 log_level : `str`
134 Set to the default log level you want jointcal to produce while the
135 tests are running. See the developer docs about logging for valid
136 levels: https://developer.lsst.io/coding/logging.html
137 where : `str`
138 Data ID query for pipetask specifying the data to run on.
139 refcats : `dict` [`str`, `str`]
140 Mapping of refcat name to relative path to .ecsv ingest file.
141 refcatPath : `str`
142 Path prefix for refcat ingest files and files linked therein.
143 inputCollections : `list` [`str`]
144 String to use for "-i" input collections (comma delimited).
145 For example, "refcats,HSC/runs/tests,HSC/calib"
146 outputDataId : `dict`
147 Partial dataIds for testing whether the output files were written.
148 """
149 self.path = os.path.dirname(__file__)
151 self.instrumentClass = instrumentClass
152 self.instrumentName = instrumentName
154 self.input_dir = input_dir
155 self.all_visits = all_visits
156 self.log_level = log_level
158 self.configfiles = []
160 # Make unittest output more verbose failure messages to assert failures.
161 self.longMessage = True
163 self.where = where
164 self.refcats = refcats
165 self.refcatPath = refcatPath
166 self.inputCollections = inputCollections
168 self.outputDataId = outputDataId
170 # Ensure that the filter list is reset for each test so that we avoid
171 # confusion or contamination from other instruments.
172 lsst.obs.base.FilterDefinitionCollection.reset()
174 self.set_output_dir()
176 def tearDown(self):
177 shutil.rmtree(self.output_dir, ignore_errors=True)
179 def _test_metrics(self, result, expect):
180 """Test a dictionary of "metrics" against those returned by jointcal.py
182 Parameters
183 ----------
184 result : `dict`
185 Result metric dictionary from jointcal.py
186 expect : `dict`
187 Expected metric dictionary; set a value to None to not test it.
188 """
189 for key in result:
190 with self.subTest(key.metric):
191 if expect[key.metric] is not None:
192 value = result[key].quantity.value
193 if isinstance(value, float):
194 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5)
195 else:
196 self.assertEqual(value, expect[key.metric], msg=key.metric)
198 def _runJointcal(self, repo,
199 inputCollections, outputCollection,
200 configFiles=None, configOptions=None,
201 registerDatasetTypes=False, whereSuffix=None,
202 nJobs=1):
203 """Run a pipeline via the SimplePipelineExecutor.
205 Parameters
206 ----------
207 repo : `str`
208 Gen3 Butler repository to read from/write to.
209 inputCollections : `list` [`str`]
210 String to use for "-i" input collections (comma delimited).
211 For example, "refcats,HSC/runs/tests,HSC/calib"
212 outputCollection : `str`
213 Name of the output collection to write to. For example,
214 "HSC/testdata/jointcal"
215 configFiles : `list` [`str`], optional
216 List of jointcal config files to use.
217 configOptions : `dict` [`str`], optional
218 Individual jointcal config options (field: value) to override.
219 registerDatasetTypes : bool, optional
220 Set "--register-dataset-types" when running the pipeline.
221 whereSuffix : `str`, optional
222 Additional parameters to the ``where`` pipetask statement.
223 nJobs : `int`, optional
224 Number of quanta expected to be run.
226 Returns
227 -------
228 job : `lsst.verify.Job`
229 Job containing the metric measurements from this test run.
230 """
231 config = lsst.jointcal.JointcalConfig()
232 if configFiles:
233 for file in configFiles:
234 config.load(file)
235 if configOptions:
236 for key, value in configOptions.items():
237 setattr(config, key, value)
238 where = ' '.join((self.where, whereSuffix)) if whereSuffix is not None else self.where
239 lsst.daf.butler.cli.cliLog.CliLog.initLog(False)
240 butler = SimplePipelineExecutor.prep_butler(repo,
241 inputs=inputCollections,
242 output=outputCollection,
243 output_run=outputCollection.replace("all", "jointcal"))
244 executor = SimplePipelineExecutor.from_task_class(lsst.jointcal.JointcalTask,
245 config=config,
246 where=where,
247 butler=butler)
248 executor.run(register_dataset_types=registerDatasetTypes)
249 # JobReporter bundles all metrics in the collection into one job.
250 jobs = JobReporter(repo, outputCollection, "jointcal", "", "jointcal").run()
251 # should only ever get one job output in tests, unless specified
252 self.assertEqual(len(jobs), nJobs)
253 # Sort the jobs, as QuantumGraph ordering is not guaranteed, and some
254 # tests check metrics values for one job while producing two.
255 sorted_jobs = {key: jobs[key] for key in sorted(list(jobs.keys()))}
256 return list(sorted_jobs.values())[0]
258 def _runJointcalTest(self, astrometryOutputs=None, photometryOutputs=None,
259 configFiles=None, configOptions=None, whereSuffix=None,
260 metrics=None, nJobs=1):
261 """Create a Butler repo and run jointcal on it.
263 Parameters
264 ----------
265 atsrometryOutputs, photometryOutputs : `dict` [`int`, `list`]
266 visit: [detectors] dictionary to test that output files were saved.
267 Default None means that there should be no output for that
268 respective dataset type.
269 configFiles : `list` [`str`], optional
270 List of jointcal config files to use.
271 configOptions : `dict` [`str`], optional
272 Individual jointcal config options (field: value) to override.
273 whereSuffix : `str`, optional
274 Additional parameters to the ``where`` pipetask statement.
275 metrics : `dict`, optional
276 Dictionary of 'metricName': value to test jointcal's result.metrics
277 against.
278 nJobs : `int`, optional
279 Number of quanta expected to be run.
281 Returns
282 -------
283 repopath : `str`
284 The path to the newly created butler repo.
285 """
286 repopath = importRepository(self.instrumentClass,
287 os.path.join(self.input_dir, "repo"),
288 os.path.join(self.input_dir, "exports.yaml"),
289 self.output_dir,
290 refcats=self.refcats,
291 refcatPath=self.refcatPath)
292 configs = [os.path.join(self.path, "config/config.py")]
293 configs.extend(self.configfiles or [])
294 configs.extend(configFiles or [])
295 collection = f"{self.instrumentName}/tests/all"
296 job = self._runJointcal(repopath,
297 self.inputCollections,
298 collection,
299 configFiles=configs,
300 configOptions=configOptions,
301 registerDatasetTypes=True,
302 whereSuffix=whereSuffix,
303 nJobs=nJobs)
305 if metrics:
306 self._test_metrics(job.measurements, metrics)
308 butler = lsst.daf.butler.Butler(repopath, collections=collection)
309 if astrometryOutputs is not None:
310 for visit, detectors in astrometryOutputs.items():
311 self._check_astrometry_output(butler, visit, detectors)
312 else:
313 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalSkyWcsCatalog"))), 0)
314 if photometryOutputs is not None:
315 for visit, detectors in photometryOutputs.items():
316 self._check_photometry_output(butler, visit, detectors)
317 else:
318 self.assertEqual(len(list(butler.registry.queryDatasets("jointcalPhotoCalibCatalog"))), 0)
320 return repopath
322 def _check_astrometry_output(self, butler, visit, detectors):
323 """Check that the WCSs were written for each detector, and only for the
324 correct detectors.
325 """
326 dataId = self.outputDataId.copy()
327 dataId['visit'] = visit
329 catalog = butler.get('jointcalSkyWcsCatalog', dataId)
330 for record in catalog:
331 self.assertIsInstance(record.getWcs(), lsst.afw.geom.SkyWcs,
332 msg=f"visit {visit}: {record}")
333 np.testing.assert_array_equal(catalog['id'], detectors)
335 def _check_photometry_output(self, butler, visit, detectors):
336 """Check that the PhotoCalibs were written for each detector, and only
337 for the correct detectors.
338 """
339 dataId = self.outputDataId.copy()
340 dataId['visit'] = visit
342 catalog = butler.get('jointcalPhotoCalibCatalog', dataId)
343 for record in catalog:
344 self.assertIsInstance(record.getPhotoCalib(), lsst.afw.image.PhotoCalib,
345 msg=f"visit {visit}: {record}")
346 np.testing.assert_array_equal(catalog['id'], detectors)