Coverage for tests/jointcalTestBase.py: 13%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import copy
23import os
24import shutil
26import click.testing
28import lsst.afw.image.utils
29import lsst.ctrl.mpexec.cli.pipetask
30import lsst.daf.butler
31import lsst.obs.base
32import lsst.geom
34from lsst.jointcal import jointcal, utils
37class JointcalTestBase:
38 """
39 Base class for jointcal tests, to genericize some test running and setup.
41 Derive from this first, then from TestCase.
42 """
44 def set_output_dir(self):
45 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1])
47 def setUp_base(self, center, radius,
48 match_radius=0.1*lsst.geom.arcseconds,
49 input_dir="",
50 all_visits=None,
51 other_args=None,
52 do_plot=False,
53 log_level=None):
54 """
55 Call from your child classes's setUp() to get the necessary variables built.
57 Parameters
58 ----------
59 center : `lsst.geom.SpherePoint`
60 Center of the reference catalog.
61 radius : `lsst.geom.Angle`
62 Radius from center to load reference catalog objects inside.
63 match_radius : `lsst.geom.Angle`
64 matching radius when calculating RMS of result.
65 input_dir : `str`
66 Directory of input butler repository.
67 all_visits : `list` [`int`]
68 List of the available visits to generate the parseAndRun arguments.
69 other_args : `list` [`str`]
70 Optional other arguments for the butler dataId.
71 do_plot : `bool`
72 Set to True for a comparison plot and some diagnostic numbers.
73 log_level : `str`
74 Set to the default log level you want jointcal to produce while the
75 tests are running. See the developer docs about logging for valid
76 levels: https://developer.lsst.io/coding/logging.html
77 """
78 self.path = os.path.dirname(__file__)
80 self.center = center
81 self.radius = radius
82 self.jointcalStatistics = utils.JointcalStatistics(match_radius, verbose=True)
83 self.input_dir = input_dir
84 self.all_visits = all_visits
85 if other_args is None:
86 other_args = []
87 self.other_args = other_args
88 self.do_plot = do_plot
89 self.log_level = log_level
90 # Signal/Noise (flux/fluxErr) for sources to be included in the RMS cross-match.
91 # 100 is a balance between good centroids and enough sources.
92 self.flux_limit = 100
94 # Individual tests may want to tweak the config that is passed to parseAndRun().
95 self.config = None
96 self.configfiles = []
98 # Append `msg` arguments to assert failures.
99 self.longMessage = True
101 # Ensure that the filter list is reset for each test so that we avoid
102 # confusion or contamination from other instruments.
103 lsst.obs.base.FilterDefinitionCollection.reset()
105 self.set_output_dir()
107 def tearDown(self):
108 shutil.rmtree(self.output_dir, ignore_errors=True)
110 if getattr(self, 'reference', None) is not None:
111 del self.reference
112 if getattr(self, 'oldWcsList', None) is not None:
113 del self.oldWcsList
114 if getattr(self, 'jointcalTask', None) is not None:
115 del self.jointcalTask
116 if getattr(self, 'jointcalStatistics', None) is not None:
117 del self.jointcalStatistics
118 if getattr(self, 'config', None) is not None:
119 del self.config
121 def _testJointcalTask(self, nCatalogs, dist_rms_relative, dist_rms_absolute, pa1,
122 metrics=None):
123 """
124 Test parseAndRun for jointcal on nCatalogs.
126 Checks relative and absolute astrometric error (arcsec) and photometric
127 repeatability (PA1 from the SRD).
129 Parameters
130 ----------
131 nCatalogs : `int`
132 Number of catalogs to run jointcal on. Used to construct the "id"
133 field for parseAndRun.
134 dist_rms_relative : `astropy.Quantity`
135 Minimum relative astrometric rms post-jointcal to pass the test.
136 dist_rms_absolute : `astropy.Quantity`
137 Minimum absolute astrometric rms post-jointcal to pass the test.
138 pa1 : `float`
139 Minimum PA1 (from Table 14 of the Science Requirements Document:
140 https://ls.st/LPM-17) post-jointcal to pass the test.
141 metrics : `dict`, optional
142 Dictionary of 'metricName': value to test jointcal's result.metrics
143 against.
145 Returns
146 -------
147 dataRefs : `list` [`lsst.daf.persistence.ButlerDataRef`]
148 The dataRefs that were processed.
149 """
151 resultFull = self._runJointcalTask(nCatalogs, metrics=metrics)
152 result = resultFull.resultList[0].result # shorten this very long thing
154 def compute_statistics(refObjLoader):
155 refCat = refObjLoader.loadSkyCircle(self.center,
156 self.radius,
157 result.defaultFilter.bandLabel,
158 epoch=result.epoch).refCat
159 rms_result = self.jointcalStatistics.compute_rms(result.dataRefs, refCat)
160 # Make plots before testing, if requested, so we still get plots if tests fail.
161 if self.do_plot:
162 self._plotJointcalTask(result.dataRefs, result.oldWcsList)
163 return rms_result
165 # we now have different astrometry/photometry refcats, so have to
166 # do these calculations separately
167 if self.jointcalStatistics.do_astrometry:
168 refObjLoader = result.astrometryRefObjLoader
169 # preserve do_photometry for the next `if`
170 temp = copy.copy(self.jointcalStatistics.do_photometry)
171 self.jointcalStatistics.do_photometry = False
172 rms_result = compute_statistics(refObjLoader)
173 self.jointcalStatistics.do_photometry = temp # restore do_photometry
175 if dist_rms_relative is not None and dist_rms_absolute is not None:
176 self.assertLess(rms_result.dist_relative, dist_rms_relative)
177 self.assertLess(rms_result.dist_absolute, dist_rms_absolute)
179 if self.jointcalStatistics.do_photometry:
180 refObjLoader = result.photometryRefObjLoader
181 self.jointcalStatistics.do_astrometry = False
182 rms_result = compute_statistics(refObjLoader)
184 if pa1 is not None:
185 self.assertLess(rms_result.pa1, pa1)
187 return result.dataRefs
189 def _runJointcalTask(self, nCatalogs, metrics=None):
190 """
191 Run jointcalTask on nCatalogs, with the most basic tests.
192 Tests for non-empty result list, and that the basic metrics are correct.
194 Parameters
195 ----------
196 nCatalogs : `int`
197 Number of catalogs to test on.
198 metrics : `dict`, optional
199 Dictionary of 'metricName': value to test jointcal's result.metrics
200 against.
202 Returns
203 -------
204 result : `pipe.base.Struct`
205 The structure returned by jointcalTask.run()
206 """
207 visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs])
208 if self.log_level is not None:
209 self.other_args.extend(['--loglevel', 'jointcal=%s'%self.log_level])
211 # Place default configfile first so that specific subclass configfiles are applied after
212 test_config = os.path.join(self.path, 'config/config.py')
213 configfiles = [test_config] + self.configfiles
215 args = [self.input_dir, '--output', self.output_dir,
216 '--clobber-versions', '--clobber-config',
217 '--doraise', '--configfile', *configfiles,
218 '--id', 'visit=%s'%visits]
219 args.extend(self.other_args)
220 result = jointcal.JointcalTask.parseAndRun(args=args, doReturnResults=True, config=self.config)
221 self.assertNotEqual(result.resultList, [], 'resultList should not be empty')
222 self.assertEqual(result.resultList[0].exitStatus, 0)
223 job = result.resultList[0].result.job
224 self._test_metrics(job.measurements, metrics)
226 return result
228 def _plotJointcalTask(self, data_refs, oldWcsList):
229 """
230 Plot the results of a jointcal run.
232 Parameters
233 ----------
234 data_refs : `list` [`lsst.daf.persistence.ButlerDataRef`]
235 The dataRefs that were processed.
236 oldWcsList : `list` [`lsst.afw.image.SkyWcs`]
237 The original WCS from each dataRef.
238 """
239 plot_dir = os.path.join('.test', self.__class__.__name__, 'plots')
240 if not os.path.isdir(plot_dir):
241 os.mkdir(plot_dir)
242 self.jointcalStatistics.make_plots(data_refs, oldWcsList, name=self.id(), outdir=plot_dir)
243 print("Plots saved to: {}".format(plot_dir))
245 def _test_metrics(self, result, expect):
246 """Test a dictionary of "metrics" against those returned by jointcal.py
248 Parameters
249 ----------
250 result : `dict`
251 Result metric dictionary from jointcal.py
252 expect : `dict`
253 Expected metric dictionary; set a value to None to not test it.
254 """
255 for key in result:
256 if expect[key.metric] is not None:
257 value = result[key].quantity.value
258 if isinstance(value, float):
259 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5)
260 else:
261 self.assertEqual(value, expect[key.metric], msg=key.metric)
263 def _importRepository(self, instrument, exportPath, exportFile):
264 """Import a gen3 test repository into self.testDir
266 Parameters
267 ----------
268 instrument : `str`
269 Full string name for the instrument.
270 exportPath : `str`
271 Path to location of repository to export.
272 This path must contain an `exports.yaml` file containing the
273 description of the exported gen3 repo that will be imported.
274 exportFile : `str`
275 Filename of export data.
276 """
277 self.repo = os.path.join(self.output_dir, 'testrepo')
279 # Make the repo and retrieve a writeable Butler
280 _ = lsst.daf.butler.Butler.makeRepo(self.repo)
281 butler = lsst.daf.butler.Butler(self.repo, writeable=True)
282 # Register the instrument
283 instrInstance = lsst.obs.base.utils.getInstrument(instrument)
284 instrInstance.register(butler.registry)
285 # Import the exportFile
286 butler.import_(directory=exportPath, filename=exportFile,
287 transfer='symlink',
288 skip_dimensions={'instrument', 'detector', 'physical_filter'})
290 def _runPipeline(self, repo, pipelineFile, queryString=None,
291 inputCollections=None, outputCollection=None,
292 configFiles=None, configOptions=None,
293 registerDatasetTypes=False):
294 """Run a pipeline via pipetask.
296 Parameters
297 ----------
298 repo : `str`
299 Gen3 Butler repository to read from/write to.
300 pipelineFile : `str`
301 The pipeline definition YAML file to execute.
302 queryString : `str`, optional
303 String to use for "-d" data query. For example,
304 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'"
305 inputCollections : `str`, optional
306 String to use for "-i" input collections (comma delimited).
307 For example, "refcats,HSC/runs/tests,HSC/calib"
308 outputCollection : `str`, optional
309 String to use for "-o" output collection. For example,
310 "HSC/testdata/jointcal"
311 configFiles : `list` [`str`], optional
312 List of jointcal config files to use (with "-C").
313 configOptions : `list` [`str`], optional
314 List of individual jointcal config options to use (with "-c").
315 registerDatasetTypes : bool, optional
316 Set "--register-dataset-types" when running the pipeline.
318 Returns
319 -------
320 exit_code: `int`
321 The exit code of the executed `click.testing.CliRunner`
323 Raises
324 ------
325 RuntimeError
326 Raised if the CliRunner executed pipetask failed.
328 Notes
329 -----
330 This approach uses the Click commandline interface, which is not ideal
331 for tests like this. See DM-26239 for the replacement pipetask API, and
332 DM-27940 for the ticket to replace the CliRunner once that API exists.
333 """
334 pipelineArgs = ["run",
335 "-b", repo,
336 "-p", pipelineFile]
338 if queryString is not None:
339 pipelineArgs.extend(["-d", queryString])
340 if inputCollections is not None:
341 pipelineArgs.extend(["-i", inputCollections])
342 if outputCollection is not None:
343 # Each test is in it's own butler, so we don't have to worry about collisions,
344 # so we can use `--output-run` instead of just `-o`.
345 pipelineArgs.extend(["--output-run", outputCollection])
346 if configFiles is not None:
347 for configFile in configFiles:
348 pipelineArgs.extend(["-C", f"jointcal:{configFile}"])
349 if configOptions is not None:
350 for configOption in configOptions:
351 pipelineArgs.extend(["-c", f"jointcal:{configOption}"])
352 if registerDatasetTypes:
353 pipelineArgs.extend(["--register-dataset-types"])
355 # TODO DM-27940: replace CliRunner with pipetask API.
356 runner = click.testing.CliRunner()
357 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs)
358 if results.exception:
359 raise RuntimeError("Pipeline %s failed." % (' '.join(pipelineArgs))) from results.exception
360 return results.exit_code
362 def _runGen3Jointcal(self, instrumentClass, instrumentName, queryString,
363 configFiles=None, configOptions=None):
364 """Create a Butler repo and run jointcal on it.
366 Parameters
367 ----------
368 instrumentClass : `str`
369 The full module name of the instrument to be registered in the
370 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam"
371 instrumentName : `str`
372 The name of the instrument as it appears in the repo collections.
373 For example, "HSC".
374 queryString : `str`
375 The query string to be run for processing. For example,
376 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'".
377 configFiles : `list` [`str`], optional
378 List of jointcal config files to use (with "-C").
379 configOptions : `list` [`str`], optional
380 List of individual jointcal config options to use (with "-c").
381 """
382 self._importRepository(instrumentClass,
383 self.input_dir,
384 os.path.join(self.input_dir, "exports.yaml"))
385 # TODO post-RFC-741: the names of these collections will have to change
386 # once testdata_jointcal is updated to reflect the collection
387 # conventions in RFC-741 (no ticket for that change yet).
388 inputCollections = f"refcats/gen2,{instrumentName}/testdata,{instrumentName}/calib/unbounded"
390 pipelineFile = os.path.join(self.path, f"config/jointcalPipeline-{instrumentName}.yaml")
391 configFiles = [os.path.join(self.path, "config/config-gen3.py")] + self.configfiles
392 self._runPipeline(self.repo, pipelineFile,
393 outputCollection=f"{instrumentName}/testdata/jointcal",
394 inputCollections=inputCollections,
395 queryString=queryString,
396 configFiles=configFiles,
397 configOptions=configOptions,
398 registerDatasetTypes=True)