Coverage for tests/jointcalTestBase.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import copy
23import os
24import shutil
26import click.testing
28import lsst.afw.image.utils
29import lsst.ctrl.mpexec.cli.pipetask
30import lsst.daf.butler
31import lsst.obs.base
32import lsst.geom
34from lsst.jointcal import jointcal, utils
37class JointcalTestBase:
38 """
39 Base class for jointcal tests, to genericize some test running and setup.
41 Derive from this first, then from TestCase.
42 """
44 def set_output_dir(self):
45 self.output_dir = os.path.join('.test', self.__class__.__name__, self.id().split('.')[-1])
47 def setUp_base(self, center, radius,
48 match_radius=0.1*lsst.geom.arcseconds,
49 input_dir="",
50 all_visits=None,
51 other_args=None,
52 do_plot=False,
53 log_level=None):
54 """
55 Call from your child classes's setUp() to get the necessary variables built.
57 Parameters
58 ----------
59 center : lsst.geom.SpherePoint
60 Center of the reference catalog.
61 radius : lsst.geom.Angle
62 Radius from center to load reference catalog objects inside.
63 match_radius : lsst.geom.Angle
64 matching radius when calculating RMS of result.
65 input_dir : str
66 Directory of input butler repository.
67 all_visits : list
68 List of the available visits to generate the parseAndRun arguments.
69 other_args : list
70 Optional other arguments for the butler dataId.
71 do_plot : bool
72 Set to True for a comparison plot and some diagnostic numbers.
73 log_level : str
74 Set to the default log level you want jointcal to produce while the
75 tests are running. See the developer docs about logging for valid
76 levels: https://developer.lsst.io/coding/logging.html
77 """
78 self.center = center
79 self.radius = radius
80 self.jointcalStatistics = utils.JointcalStatistics(match_radius, verbose=True)
81 self.input_dir = input_dir
82 self.all_visits = all_visits
83 if other_args is None:
84 other_args = []
85 self.other_args = other_args
86 self.do_plot = do_plot
87 self.log_level = log_level
88 # Signal/Noise (flux/fluxErr) for sources to be included in the RMS cross-match.
89 # 100 is a balance between good centroids and enough sources.
90 self.flux_limit = 100
92 # Individual tests may want to tweak the config that is passed to parseAndRun().
93 self.config = None
94 self.configfiles = []
96 # Append `msg` arguments to assert failures.
97 self.longMessage = True
99 # Ensure that the filter list is reset for each test so that we avoid
100 # confusion or contamination from other instruments.
101 lsst.obs.base.FilterDefinitionCollection.reset()
103 self.set_output_dir()
105 def tearDown(self):
106 shutil.rmtree(self.output_dir, ignore_errors=True)
108 if getattr(self, 'reference', None) is not None:
109 del self.reference
110 if getattr(self, 'oldWcsList', None) is not None:
111 del self.oldWcsList
112 if getattr(self, 'jointcalTask', None) is not None:
113 del self.jointcalTask
114 if getattr(self, 'jointcalStatistics', None) is not None:
115 del self.jointcalStatistics
116 if getattr(self, 'config', None) is not None:
117 del self.config
119 def _testJointcalTask(self, nCatalogs, dist_rms_relative, dist_rms_absolute, pa1,
120 metrics=None):
121 """
122 Test parseAndRun for jointcal on nCatalogs.
124 Checks relative and absolute astrometric error (arcsec) and photometric
125 repeatability (PA1 from the SRD).
127 Parameters
128 ----------
129 nCatalogs : int
130 Number of catalogs to run jointcal on. Used to construct the "id"
131 field for parseAndRun.
132 dist_rms_relative : astropy.Quantity
133 Minimum relative astrometric rms post-jointcal to pass the test.
134 dist_rms_absolute : astropy.Quantity
135 Minimum absolute astrometric rms post-jointcal to pass the test.
136 pa1 : float
137 Minimum PA1 (from Table 14 of the Science Requirements Document:
138 https://ls.st/LPM-17) post-jointcal to pass the test.
139 metrics : dict, optional
140 Dictionary of 'metricName': value to test jointcal's result.metrics
141 against.
143 Returns
144 -------
145 list of lsst.daf.persistence.ButlerDataRef
146 The dataRefs that were processed.
147 """
149 result = self._runJointcalTask(nCatalogs, metrics=metrics)
151 data_refs = result.resultList[0].result.dataRefs
152 oldWcsList = result.resultList[0].result.oldWcsList
154 defaultBand = result.resultList[0].result.defaultBand
156 def compute_statistics(refObjLoader):
157 refCat = refObjLoader.loadSkyCircle(self.center, self.radius, defaultBand).refCat
158 rms_result = self.jointcalStatistics.compute_rms(data_refs, refCat)
159 # Make plots before testing, if requested, so we still get plots if tests fail.
160 if self.do_plot:
161 self._plotJointcalTask(data_refs, oldWcsList)
162 return rms_result
164 # we now have different astrometry/photometry refcats, so have to
165 # do these calculations separately
166 if self.jointcalStatistics.do_astrometry:
167 refObjLoader = result.resultList[0].result.astrometryRefObjLoader
168 # preserve do_photometry for the next `if`
169 temp = copy.copy(self.jointcalStatistics.do_photometry)
170 self.jointcalStatistics.do_photometry = False
171 rms_result = compute_statistics(refObjLoader)
172 self.jointcalStatistics.do_photometry = temp # restore do_photometry
174 if dist_rms_relative is not None and dist_rms_absolute is not None:
175 self.assertLess(rms_result.dist_relative, dist_rms_relative)
176 self.assertLess(rms_result.dist_absolute, dist_rms_absolute)
178 if self.jointcalStatistics.do_photometry:
179 refObjLoader = result.resultList[0].result.photometryRefObjLoader
180 self.jointcalStatistics.do_astrometry = False
181 rms_result = compute_statistics(refObjLoader)
183 if pa1 is not None:
184 self.assertLess(rms_result.pa1, pa1)
186 return data_refs
188 def _runJointcalTask(self, nCatalogs, metrics=None):
189 """
190 Run jointcalTask on nCatalogs, with the most basic tests.
191 Tests for non-empty result list, and that the basic metrics are correct.
193 Parameters
194 ----------
195 nCatalogs : int
196 Number of catalogs to test on.
197 metrics : dict, optional
198 Dictionary of 'metricName': value to test jointcal's result.metrics
199 against.
201 Returns
202 -------
203 pipe.base.Struct
204 The structure returned by jointcalTask.run()
205 """
206 visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs])
207 if self.log_level is not None:
208 self.other_args.extend(['--loglevel', 'jointcal=%s'%self.log_level])
210 # Place default configfile first so that specific subclass configfiles are applied after
211 test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 'tests/config/config.py')
212 self.configfiles = [test_config] + self.configfiles
214 args = [self.input_dir, '--output', self.output_dir,
215 '--clobber-versions', '--clobber-config',
216 '--doraise', '--configfile', *self.configfiles,
217 '--id', 'visit=%s'%visits]
218 args.extend(self.other_args)
219 result = jointcal.JointcalTask.parseAndRun(args=args, doReturnResults=True, config=self.config)
220 self.assertNotEqual(result.resultList, [], 'resultList should not be empty')
221 self.assertEqual(result.resultList[0].exitStatus, 0)
222 job = result.resultList[0].result.job
223 self._test_metrics(job.measurements, metrics)
225 return result
227 def _plotJointcalTask(self, data_refs, oldWcsList):
228 """
229 Plot the results of a jointcal run.
231 Parameters
232 ----------
233 data_refs : list of lsst.daf.persistence.ButlerDataRef
234 The dataRefs that were processed.
235 oldWcsList : list of lsst.afw.image.Wcs
236 The original WCS from each dataRef.
237 """
238 plot_dir = os.path.join('.test', self.__class__.__name__, 'plots')
239 if not os.path.isdir(plot_dir):
240 os.mkdir(plot_dir)
241 self.jointcalStatistics.make_plots(data_refs, oldWcsList, name=self.id(), outdir=plot_dir)
242 print("Plots saved to: {}".format(plot_dir))
244 def _test_metrics(self, result, expect):
245 """Test a dictionary of "metrics" against those returned by jointcal.py
247 Parameters
248 ----------
249 result : dict
250 Result metric dictionary from jointcal.py
251 expect : dict
252 Expected metric dictionary; set a value to None to not test it.
253 """
254 for key in result:
255 if expect[key.metric] is not None:
256 value = result[key].quantity.value
257 if isinstance(value, float):
258 self.assertFloatsAlmostEqual(value, expect[key.metric], msg=key.metric, rtol=1e-5)
259 else:
260 self.assertEqual(value, expect[key.metric], msg=key.metric)
262 def _importRepository(self, instrument, exportPath, exportFile):
263 """Import a gen3 test repository into self.testDir
265 Parameters
266 ----------
267 instrument : `str`
268 Full string name for the instrument.
269 exportPath : `str`
270 Path to location of repository to export.
271 This path must contain an `exports.yaml` file containing the
272 description of the exported gen3 repo that will be imported.
273 exportFile : `str`
274 Filename of export data.
275 """
276 self.repo = os.path.join(self.output_dir, 'testrepo')
278 # Make the repo and retrieve a writeable Butler
279 _ = lsst.daf.butler.Butler.makeRepo(self.repo)
280 butler = lsst.daf.butler.Butler(self.repo, writeable=True)
281 # Register the instrument
282 instrInstance = lsst.obs.base.utils.getInstrument(instrument)
283 instrInstance.register(butler.registry)
284 # Import the exportFile
285 butler.import_(directory=exportPath, filename=exportFile,
286 transfer='symlink',
287 skip_dimensions={'instrument', 'detector', 'physical_filter'})
289 def _runPipeline(self, repo, queryString=None,
290 inputCollections=None, outputCollection=None,
291 configFiles=None, configOptions=None,
292 registerDatasetTypes=False):
293 """Run a pipeline via pipetask.
295 Parameters
296 ----------
297 repo : `str`
298 Gen3 Butler repository to read from/write to.
299 queryString : `str`, optional
300 String to use for "-d" data query. For example,
301 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'"
302 inputCollections : `str`, optional
303 String to use for "-i" input collections (comma delimited).
304 For example, "refcats,HSC/runs/tests,HSC/calib"
305 outputCollection : `str`, optional
306 String to use for "-o" output collection. For example,
307 "HSC/testdata/jointcal"
308 configFiles : `list` [`str`], optional
309 List of config files to use (with "-C").
310 registerDatasetTypes : bool, optional
311 Set "--register-dataset-types" when running the pipeline.
313 Returns
314 -------
315 exit_code: `int`
316 The exit code of the executed `click.testing.CliRunner`
318 Raises
319 ------
320 RuntimeError
321 Raised if the CliRunner executed pipetask failed.
323 Notes
324 -----
325 This approach uses the Click commandline interface, which is not ideal
326 for tests like this. See DM-26239 for the replacement pipetask API, and
327 DM-27940 for the ticket to replace the CliRunner once that API exists.
328 """
329 pipelineArgs = ["run",
330 "-b", repo,
331 "-t lsst.jointcal.JointcalTask"]
333 if queryString is not None:
334 pipelineArgs.extend(["-d", queryString])
335 if inputCollections is not None:
336 pipelineArgs.extend(["-i", inputCollections])
337 if outputCollection is not None:
338 pipelineArgs.extend(["-o", outputCollection])
339 if configFiles is not None:
340 for configFile in configFiles:
341 pipelineArgs.extend(["-C", configFile])
342 if registerDatasetTypes:
343 pipelineArgs.extend(["--register-dataset-types"])
345 # TODO DM-27940: replace CliRunner with pipetask API.
346 runner = click.testing.CliRunner()
347 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs)
348 if results.exception:
349 raise RuntimeError("Pipeline %s failed." % (' '.join(pipelineArgs))) from results.exception
350 return results.exit_code
352 def _runGen3Jointcal(self, instrumentClass, instrumentName, queryString):
353 """Create a Butler repo and run jointcal on it.
356 Parameters
357 ----------
358 instrumentClass : `str`
359 The full module name of the instrument to be registered in the
360 new repo. For example, "lsst.obs.subaru.HyperSuprimeCam"
361 instrumentName : `str`
362 The name of the instrument as it appears in the repo collections.
363 For example, "HSC".
364 queryString : `str`
365 The query string to be run for processing. For example,
366 "instrument='HSC' and tract=9697 and skymap='hsc_rings_v1'".
367 """
368 self._importRepository(instrumentClass,
369 self.input_dir,
370 os.path.join(self.input_dir, "exports.yaml"))
371 # TODO post-RFC-741: the names of these collections will have to change
372 # once testdata_jointcal is updated to reflect the collection
373 # conventions in RFC-741 (no ticket for that change yet).
374 inputCollections = f"refcats,{instrumentName}/testdata,{instrumentName}/calib/unbounded"
375 self._runPipeline(self.repo,
376 outputCollection=f"{instrumentName}/testdata/jointcal",
377 inputCollections=inputCollections,
378 queryString=queryString,
379 registerDatasetTypes=True)