Coverage for tests/jointcalTestBase.py: 13%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["importRepository", "JointcalTestBase"]
24import copy
25import os
26import shutil
27import tempfile
29import lsst.afw.image.utils
30from lsst.ctrl.mpexec import SimplePipelineExecutor
31import lsst.daf.butler
32from lsst.daf.butler.script import ingest_files
33import lsst.obs.base
34import lsst.geom
35from lsst.verify.bin.jobReporter import JobReporter
37from lsst.jointcal import jointcal, utils
40def importRepository(instrument, exportPath, exportFile, outputDir=None,
41 refcats=None, refcatPath=""):
42 """Import a gen3 test repository into a test path.
44 Parameters
45 ----------
46 instrument : `str`
47 Full string name for the instrument.
48 exportPath : `str`
49 Path to location of repository to export.
50 exportFile : `str`
51 Filename of YAML butler export data, describing the data to import.
52 outputDir : `str`, or `None`
53 Root path to put the test repo in; appended with `testrepo/`.
54 If not supplied, a temporary path is generated with mkdtemp; this path
55 must be deleted by the calling code, it is not automatically removed.
56 refcats : `dict` [`str`, `str`]
57 Mapping of refcat name to relative path to .ecsv ingest file.
58 refcatPath : `str`
59 Path prefix for refcat ingest files and files linked therein.
61 Returns
62 -------
63 repopath : `str`
64 The path to the newly created butler repo.
65 """
66 if outputDir is None:
67 repopath = tempfile.mkdtemp()
68 else:
69 repopath = os.path.join(outputDir, "testrepo")
71 # Make the repo and retrieve a writeable Butler
72 _ = lsst.daf.butler.Butler.makeRepo(repopath)
73 butler = lsst.daf.butler.Butler(repopath, writeable=True)
75 instrInstance = lsst.obs.base.utils.getInstrument(instrument)
76 instrInstance.register(butler.registry)
78 # Register refcats first, so the `refcats` collection will exist.
79 if refcats is not None:
80 for name, file in refcats.items():
81 graph = butler.registry.dimensions.extract(["htm7"])
82 datasetType = lsst.daf.butler.DatasetType(name, graph, "SimpleCatalog",
83 universe=butler.registry.dimensions)
84 butler.registry.registerDatasetType(datasetType)
85 ingest_files(repopath, name, "refcats", file, prefix=refcatPath)
86 # New butler to get the refcats collections to be visible
87 butler = lsst.daf.butler.Butler(repopath, writeable=True)
89 butler.import_(directory=exportPath, filename=exportFile,
90 transfer='symlink',
91 skip_dimensions={'instrument', 'detector', 'physical_filter'})
92 return repopath
95class JointcalTestBase:
96 """
97 Base class for jointcal tests, to genericize some test running and setup.
99 Derive from this first, then from TestCase.
100 """
102 def set_output_dir(self):
103 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1])
105 def setUp_base(self, center, radius,
106 match_radius=0.1*lsst.geom.arcseconds,
107 input_dir="",
108 all_visits=None,
109 other_args=None,
110 do_plot=False,
111 log_level=None,
112 where="",
113 refcats=None,
114 refcatPath="",
115 inputCollections=None):
116 """
117 Call from your child classes's setUp() to get the necessary variables built.
119 Parameters
120 ----------
121 center : `lsst.geom.SpherePoint`
122 Center of the reference catalog.
123 radius : `lsst.geom.Angle`
124 Radius from center to load reference catalog objects inside.
125 match_radius : `lsst.geom.Angle`
126 matching radius when calculating RMS of result.
127 input_dir : `str`
128 Directory of input butler repository.
129 all_visits : `list` [`int`]
130 List of the available visits to generate the parseAndRun arguments.
131 other_args : `list` [`str`]
132 Optional other arguments for the butler dataId.
133 do_plot : `bool`
134 Set to True for a comparison plot and some diagnostic numbers.
135 log_level : `str`
136 Set to the default log level you want jointcal to produce while the
137 tests are running. See the developer docs about logging for valid
138 levels: https://developer.lsst.io/coding/logging.html
139 where : `str`
140 Data ID query for pipetask specifying the data to run on.
141 refcats : `dict` [`str`, `str`]
142 Mapping of refcat name to relative path to .ecsv ingest file.
143 refcatPath : `str`
144 Path prefix for refcat ingest files and files linked therein.
145 inputCollections : `list` [`str`]
146 String to use for "-i" input collections (comma delimited).
147 For example, "refcats,HSC/runs/tests,HSC/calib"
148 """
149 self.path = os.path.dirname(__file__)
151 self.center = center
152 self.radius = radius
153 self.jointcalStatistics = utils.JointcalStatistics(match_radius, verbose=True)
154 self.input_dir = input_dir
155 self.all_visits = all_visits
156 if other_args is None:
157 other_args = []
158 self.other_args = other_args
159 self.do_plot = do_plot
160 self.log_level = log_level
161 # Signal/Noise (flux/fluxErr) for sources to be included in the RMS cross-match.
162 # 100 is a balance between good centroids and enough sources.
163 self.flux_limit = 100
165 # Individual tests may want to tweak the config that is passed to parseAndRun().
166 self.config = None
167 self.configfiles = []
169 # Append `msg` arguments to assert failures.
170 self.longMessage = True
172 self.where = where
173 self.refcats = refcats
174 self.refcatPath = refcatPath
175 self.inputCollections = inputCollections
177 # Ensure that the filter list is reset for each test so that we avoid
178 # confusion or contamination from other instruments.
179 lsst.obs.base.FilterDefinitionCollection.reset()
181 self.set_output_dir()
183 def tearDown(self):
184 shutil.rmtree(self.output_dir, ignore_errors=True)
186 if getattr(self, 'reference', None) is not None:
187 del self.reference
188 if getattr(self, 'oldWcsList', None) is not None:
189 del self.oldWcsList
190 if getattr(self, 'jointcalTask', None) is not None:
191 del self.jointcalTask
192 if getattr(self, 'jointcalStatistics', None) is not None:
193 del self.jointcalStatistics
194 if getattr(self, 'config', None) is not None:
195 del self.config
197 def _testJointcalTask(self, nCatalogs, dist_rms_relative, dist_rms_absolute, pa1,
198 metrics=None):
199 """
200 Test parseAndRun for jointcal on nCatalogs.
202 Checks relative and absolute astrometric error (arcsec) and photometric
203 repeatability (PA1 from the SRD).
205 Parameters
206 ----------
207 nCatalogs : `int`
208 Number of catalogs to run jointcal on. Used to construct the "id"
209 field for parseAndRun.
210 dist_rms_relative : `astropy.Quantity`
211 Minimum relative astrometric rms post-jointcal to pass the test.
212 dist_rms_absolute : `astropy.Quantity`
213 Minimum absolute astrometric rms post-jointcal to pass the test.
214 pa1 : `float`
215 Minimum PA1 (from Table 14 of the Science Requirements Document:
216 https://ls.st/LPM-17) post-jointcal to pass the test.
217 metrics : `dict`, optional
218 Dictionary of 'metricName': value to test jointcal's result.metrics
219 against.
221 Returns
222 -------
223 dataRefs : `list` [`lsst.daf.persistence.ButlerDataRef`]
224 The dataRefs that were processed.
225 """
227 resultFull = self._runJointcalTask(nCatalogs, metrics=metrics)
228 result = resultFull.resultList[0].result # shorten this very long thing
230 def compute_statistics(refObjLoader):
231 refCat = refObjLoader.loadSkyCircle(self.center,
232 self.radius,
233 result.defaultFilter.bandLabel,
234 epoch=result.epoch).refCat
235 rms_result = self.jointcalStatistics.compute_rms(result.dataRefs, refCat)
236 # Make plots before testing, if requested, so we still get plots if tests fail.
237 if self.do_plot:
238 self._plotJointcalTask(result.dataRefs, result.oldWcsList)
239 return rms_result
241 # we now have different astrometry/photometry refcats, so have to
242 # do these calculations separately
243 if self.jointcalStatistics.do_astrometry:
244 refObjLoader = result.astrometryRefObjLoader
245 # preserve do_photometry for the next `if`
246 temp = copy.copy(self.jointcalStatistics.do_photometry)
247 self.jointcalStatistics.do_photometry = False
248 rms_result = compute_statistics(refObjLoader)
249 self.jointcalStatistics.do_photometry = temp # restore do_photometry
251 if dist_rms_relative is not None and dist_rms_absolute is not None:
252 self.assertLess(rms_result.dist_relative, dist_rms_relative)
253 self.assertLess(rms_result.dist_absolute, dist_rms_absolute)
255 if self.jointcalStatistics.do_photometry:
256 refObjLoader = result.photometryRefObjLoader
257 self.jointcalStatistics.do_astrometry = False
258 rms_result = compute_statistics(refObjLoader)
260 if pa1 is not None:
261 self.assertLess(rms_result.pa1, pa1)
263 return result.dataRefs
265 def _runJointcalTask(self, nCatalogs, metrics=None):
266 """
267 Run jointcalTask on nCatalogs, with the most basic tests.
268 Tests for non-empty result list, and that the basic metrics are correct.
270 Parameters
271 ----------
272 nCatalogs : `int`
273 Number of catalogs to test on.
274 metrics : `dict`, optional
275 Dictionary of 'metricName': value to test jointcal's result.metrics
276 against.
278 Returns
279 -------
280 result : `pipe.base.Struct`
281 The structure returned by jointcalTask.run()
282 """
283 visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs])
284 if self.log_level is not None:
285 self.other_args.extend(['--loglevel', 'jointcal=%s'%self.log_level])
287 # Place default configfile first so that specific subclass configfiles are applied after
288 test_config = os.path.join(self.path, 'config/config.py')
289 configfiles = [test_config] + self.configfiles
291 args = [self.input_dir, '--output', self.output_dir,
292 '--clobber-versions', '--clobber-config',
293 '--doraise', '--configfile', *configfiles,
294 '--id', 'visit=%s'%visits]
295 args.extend(self.other_args)
296 result = jointcal.JointcalTask.parseAndRun(args=args, doReturnResults=True, config=self.config)
297 self.assertNotEqual(result.resultList, [], 'resultList should not be empty')
298 self.assertEqual(result.resultList[0].exitStatus, 0)
299 job = result.resultList[0].result.job
300 self._test_metrics(job.measurements, metrics)
302 return result
304 def _plotJointcalTask(self, data_refs, oldWcsList):
305 """
306 Plot the results of a jointcal run.
308 Parameters
309 ----------
310 data_refs : `list` [`lsst.daf.persistence.ButlerDataRef`]
311 The dataRefs that were processed.
312 oldWcsList : `list` [`lsst.afw.image.SkyWcs`]
313 The original WCS from each dataRef.
314 """
315 plot_dir = os.path.join('.test', self.__class__.__name__, 'plots')
316 if not os.path.isdir(plot_dir):
317 os.mkdir(plot_dir)
318 self.jointcalStatistics.make_plots(data_refs, oldWcsList, name=self.id(), outdir=plot_dir)
319 print("Plots saved to: {}".format(plot_dir))
321 def _test_metrics(self, result, expect):
322 """Test a dictionary of "metrics" against those returned by jointcal.py
324 Parameters
325 ----------
326 result : `dict`
327 Result metric dictionary from jointcal.py
328 expect : `dict`
329 Expected metric dictionary; set a value to None to not test it.
330 """
331 for key in result:
332 with self.subTest(key.metric):
333 if expect[key.metric] is not None:
334 value = result[key].quantity.value
335 if isinstance(value, float):
336 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5)
337 else:
338 self.assertEqual(value, expect[key.metric], msg=key.metric)
340 def _runPipeline(self, repo,
341 inputCollections, outputCollection,
342 configFiles=None, configOptions=None,
343 registerDatasetTypes=False, whereSuffix=None,
344 nJobs=1):
345 """Run a pipeline via the SimplePipelineExecutor.
347 Parameters
348 ----------
349 repo : `str`
350 Gen3 Butler repository to read from/write to.
351 inputCollections : `list` [`str`]
352 String to use for "-i" input collections (comma delimited).
353 For example, "refcats,HSC/runs/tests,HSC/calib"
354 outputCollection : `str`
355 Name of the output collection to write to. For example,
356 "HSC/testdata/jointcal"
357 configFiles : `list` [`str`], optional
358 List of jointcal config files to use.
359 configOptions : `dict` [`str`], optional
360 Individual jointcal config options (field: value) to override.
361 registerDatasetTypes : bool, optional
362 Set "--register-dataset-types" when running the pipeline.
363 whereSuffix : `str`, optional
364 Additional parameters to the ``where`` pipetask statement.
365 nJobs : `int`, optional
366 Number of quanta expected to be run.
368 Returns
369 -------
370 job : `lsst.verify.Job`
371 Job containing the metric measurements from this test run.
372 """
373 config = lsst.jointcal.JointcalConfig()
374 if configFiles:
375 for file in configFiles:
376 config.load(file)
377 if configOptions:
378 for key, value in configOptions.items():
379 setattr(config, key, value)
380 where = ' '.join((self.where, whereSuffix)) if whereSuffix is not None else self.where
381 lsst.daf.butler.cli.cliLog.CliLog.initLog(False)
382 butler = SimplePipelineExecutor.prep_butler(repo,
383 inputs=inputCollections,
384 output=outputCollection,
385 output_run=outputCollection.replace("all", "jointcal"))
386 executor = SimplePipelineExecutor.from_task_class(lsst.jointcal.JointcalTask,
387 config=config,
388 where=where,
389 butler=butler)
390 executor.run(register_dataset_types=registerDatasetTypes)
391 # JobReporter bundles all metrics in the collection into one job.
392 jobs = JobReporter(repo, outputCollection, "jointcal", "", "jointcal").run()
393 # should only ever get one job output in tests, unless specified
394 self.assertEqual(len(jobs), nJobs)
395 return list(jobs.values())[0]
397 def _runGen3Jointcal(self, instrumentClass, instrumentName,
398 configFiles=None, configOptions=None, whereSuffix=None,
399 metrics=None, nJobs=1):
400 """Create a Butler repo and run jointcal on it.
402 Parameters
403 ----------
404 instrumentClass : `str`
405 The full module name of the instrument to be registered in the
406 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam"
407 instrumentName : `str`
408 The name of the instrument as it appears in the repo collections.
409 For example, "HSC".
410 configFiles : `list` [`str`], optional
411 List of jointcal config files to use.
412 configOptions : `dict` [`str`], optional
413 Individual jointcal config options (field: value) to override.
414 whereSuffix : `str`, optional
415 Additional parameters to the ``where`` pipetask statement.
416 metrics : `dict`, optional
417 Dictionary of 'metricName': value to test jointcal's result.metrics
418 against.
419 nJobs : `int`, optional
420 Number of quanta expected to be run.
422 Returns
423 -------
424 repopath : `str`
425 The path to the newly created butler repo.
426 """
427 repopath = importRepository(instrumentClass,
428 os.path.join(self.input_dir, "repo"),
429 os.path.join(self.input_dir, "exports.yaml"),
430 self.output_dir,
431 refcats=self.refcats,
432 refcatPath=self.refcatPath)
433 configs = [os.path.join(self.path, "config/config-gen3.py")]
434 configs.extend(self.configfiles or [])
435 configs.extend(configFiles or [])
436 job = self._runPipeline(repopath,
437 self.inputCollections,
438 f"{instrumentName}/tests/all",
439 configFiles=configs,
440 configOptions=configOptions,
441 registerDatasetTypes=True,
442 whereSuffix=whereSuffix,
443 nJobs=nJobs)
445 if metrics:
446 self._test_metrics(job.measurements, metrics)
448 return repopath