Coverage for tests/test_PluginLogs.py : 18%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
23import os
24import numpy
26import lsst.geom
27import lsst.afw.table
28import lsst.daf.base
29import lsst.meas.base
30import lsst.utils.tests
31from lsst.meas.base.tests import (AlgorithmTestCase, )
32from lsst.meas.base.sfm import SingleFramePluginConfig, SingleFramePlugin
33from lsst.meas.base.forcedMeasurement import ForcedPlugin
34from lsst.meas.base.pluginRegistry import register
35from lsst.meas.base import FlagDefinitionList, FlagHandler, MeasurementError
37ROOT = os.path.abspath(os.path.dirname(__file__))
40class LoggingPluginConfig(SingleFramePluginConfig):
41 """Configuration for sample plugin.
42 """
43 pass
46@register("test_LoggingPlugin")
47class LoggingPlugin(SingleFramePlugin):
48 """Sample Python plugin which has an associated log name.
50 Notes
51 -----
52 The log name is provided to the plugin by the measurement task which is
53 running it. This requires that the `hasLogName` attribute must be a member
54 of the plugin class, and it must be `True`.
55 """
56 hasLogName = True
57 ConfigClass = LoggingPluginConfig
59 @classmethod
60 def getExecutionOrder(cls):
61 return cls.FLUX_ORDER
63 # The initializer for the class must accept an optional logName parameter.
64 def __init__(self, config, name, schema, metadata, logName=None):
65 SingleFramePlugin.__init__(self, config, name, schema, metadata, logName=logName)
66 flagDefs = FlagDefinitionList()
67 self.FAILURE = flagDefs.addFailureFlag()
68 self.CONTAINS_NAN = flagDefs.add("flag_containsNan", "Measurement area contains a nan")
69 self.flagHandler = FlagHandler.addFields(schema, name, flagDefs)
70 self.instFluxKey = schema.addField(name + "_instFlux", "F", doc="flux")
72 def measure(self, measRecord, exposure):
73 """Perform measurement.
75 Notes
76 -----
77 The `measure` method is called by the measurement framework when `run`
78 is called. If a `MeasurementError` is raised during this method, the
79 `fail` method will be called to set the error flags.
80 """
81 lsst.log.Log.getLogger(self.getLogName()).info("%s plugin measuring."%(self.name,))
82 # Sum the pixels inside the bounding box
83 centerPoint = lsst.geom.Point2I(int(measRecord.getX()), int(measRecord.getY()))
84 bbox = lsst.geom.Box2I(centerPoint, lsst.geom.Extent2I(1, 1))
85 instFlux = lsst.afw.image.ImageF(exposure.getMaskedImage().getImage(), bbox).getArray().sum()
86 measRecord.set(self.instFluxKey, instFlux)
88 # If there was a NaN inside the bounding box, the instFlux will still
89 # be NaN
90 if numpy.isnan(instFlux):
91 raise MeasurementError(self.CONTAINS_NAN.doc, self.CONTAINS_NAN.number)
93 def fail(self, measRecord, error=None):
94 """Handle measurement failures.
96 Notes
97 -----
98 If measurement raises a `MeasurementError`, the error will be passed
99 to the fail method by the measurement framework. If the error is not
100 `None`, ``error.cpp`` should correspond to a specific error and the
101 appropriate error flag will be set.
102 """
103 if error is None:
104 self.flagHandler.handleFailure(measRecord)
105 else:
106 self.flagHandler.handleFailure(measRecord, error.cpp)
109def directLog(log, file=None):
110 """Direct the log given to a file or to the console if ``file`` is `None`.
111 """
112 props = "log4j.rootLogger=INFO, FA\n"
113 if file is None:
114 props += "log4j.appender.FA=ConsoleAppender\n"
115 else:
116 props += "log4j.appender.FA=FileAppender\n"
117 props += "log4j.appender.FA.Append=false\n"
118 props += "log4j.appender.FA.file=%s\n"%(file,)
119 props += "log4j.appender.FA.Append=false\n"
120 props += "log4j.appender.FA.layout=PatternLayout\n"
121 props += "log4j.appender.FA.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c %m %X%n\n"
122 props += "log4j.logger.main.a=DEBUG\n"
123 log.configure_prop(props)
126class RegisteredPluginsTestCase(AlgorithmTestCase, lsst.utils.tests.TestCase):
127 """Test all registered Plugins to see if their logName is set as expected.
129 Those which have the ``hasLogName=True`` attribute will have a ``logName``
130 parameter passed to their ``__init__``, and should set the internal
131 ``_logName`` attribute. If they are wrapped C++ algorithms, the
132 `getLogName` should also return same ``logName`` as the plugin.
133 """
134 def testSingleFramePlugins(self):
135 center = lsst.geom.Point2D(50, 50)
136 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
137 lsst.geom.Extent2I(100, 100))
138 dataset = lsst.meas.base.tests.TestDataset(bbox)
139 dataset.addSource(1000000.0, center)
140 registry = SingleFramePlugin.registry
141 dependencies = registry.keys()
142 task = self.makeSingleFrameMeasurementTask("base_SdssCentroid", dependencies=dependencies)
143 exposure, catalog = dataset.realize(noise=100.0, schema=task.schema, randomSeed=0)
144 task.log.setLevel(lsst.log.ERROR)
145 task.run(catalog, exposure)
146 for pluginName in dependencies:
147 plugin = task.plugins[pluginName]
148 if hasattr(plugin, "hasLogName") and plugin.hasLogName:
149 self.assertEqual(plugin.getLogName(), task.getPluginLogName(pluginName))
150 # if the plugin is cpp, check the cpp Algorithm as well
151 if hasattr(plugin, "cpp"):
152 self.assertEqual(plugin.cpp.getLogName(), plugin.getLogName())
153 else:
154 self.assertEqual(plugin.getLogName(), None)
156 def testForcedPlugins(self):
157 # Test all the ForcedPlugins registered to see if their logName is set
158 # as expected.
159 center = lsst.geom.Point2D(50, 50)
160 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
161 lsst.geom.Extent2I(100, 100))
162 dataset = lsst.meas.base.tests.TestDataset(bbox)
163 dataset.addSource(1000000.0, center)
164 registry = ForcedPlugin.registry
165 dependencies = registry.keys()
167 task = self.makeForcedMeasurementTask("base_SdssCentroid", dependencies=dependencies)
168 measWcs = dataset.makePerturbedWcs(dataset.exposure.getWcs(), randomSeed=1)
169 measDataset = dataset.transform(measWcs)
170 exposure, truthCatalog = measDataset.realize(10.0, measDataset.makeMinimalSchema(), randomSeed=1)
171 refCat = dataset.catalog
172 refWcs = dataset.exposure.getWcs()
173 measCat = task.generateMeasCat(exposure, refCat, refWcs)
174 task.attachTransformedFootprints(measCat, refCat, exposure, refWcs)
176 task.log.setLevel(lsst.log.ERROR)
177 task.run(measCat, exposure, refCat, refWcs)
178 for pluginName in dependencies:
179 plugin = task.plugins[pluginName]
180 if hasattr(plugin, "hasLogName") and plugin.hasLogName:
181 self.assertEqual(plugin.getLogName(), task.getPluginLogName(pluginName))
182 # if the plugin is cpp, check the cpp Algorithm as well
183 if hasattr(plugin, "cpp"):
184 self.assertEqual(plugin.cpp.getLogName(), task.log.getName() + "." + pluginName)
185 else:
186 self.assertEqual(plugin.getLogName(), None)
189class LoggingPythonTestCase(AlgorithmTestCase, lsst.utils.tests.TestCase):
190 """Test one C++ and one Python plugin which have hasLogName=True.
191 """
192 def setUp(self):
193 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(100, 100))
194 self.dataset = lsst.meas.base.tests.TestDataset(bbox)
195 self.dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 25))
196 config = lsst.meas.base.SingleFrameMeasurementConfig()
197 config.slots.centroid = None
198 config.slots.apFlux = None
199 config.slots.calibFlux = None
200 config.slots.gaussianFlux = None
201 config.slots.modelFlux = None
202 config.slots.psfFlux = None
203 config.slots.shape = None
204 config.slots.psfShape = None
205 self.config = config
207 def tearDown(self):
208 del self.config
209 del self.dataset
211 def testLoggingPythonPlugin(self):
212 algName = "test_LoggingPlugin"
213 schema = self.dataset.makeMinimalSchema()
214 self.config.plugins = [algName]
215 task = lsst.meas.base.SingleFrameMeasurementTask(schema=schema, config=self.config)
216 # test that the plugin's logName has been propagated to the plugin
217 self.assertTrue(task.plugins[algName].getLogName(), task.getPluginLogName(algName))
218 log = lsst.log.Log.getLogger(task.getPluginLogName(algName))
219 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName:
220 directLog(log, pluginLogName)
221 exposure, cat = self.dataset.realize(noise=0.0, schema=schema, randomSeed=2)
222 task.run(cat, exposure)
223 directLog(log, None)
224 # direct back to console, closing log files
225 with open(pluginLogName) as fin:
226 lines = fin.read()
227 # test that the sample plugin has correctly logged to where we
228 # expected it to.
229 self.assertTrue(lines.find("measuring") >= 0)
231 def testLoggingCppPlugin(self):
232 # PsfFlux is known to log an ``ERROR`` if a Psf is not attached
233 algName = "base_PsfFlux"
234 self.config.plugins = [algName]
236 schema = self.dataset.makeMinimalSchema()
237 task = lsst.meas.base.SingleFrameMeasurementTask(schema=schema, config=self.config)
238 log = lsst.log.Log.getLogger(task.getPluginLogName(algName))
239 log.setLevel(lsst.log.ERROR)
241 # test that the plugin's logName has been propagated to the plugin
242 self.assertTrue(task.plugins[algName].getLogName(), task.getPluginLogName(algName))
243 self.assertTrue(task.plugins[algName].cpp.getLogName(), task.getPluginLogName(algName))
244 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName:
245 directLog(log, pluginLogName)
246 exposure, cat = self.dataset.realize(noise=0.0, schema=schema, randomSeed=3)
247 exposure.setPsf(None)
248 # This call throws an error, so be prepared for it
249 try:
250 task.run(cat, exposure)
251 except Exception:
252 pass
253 directLog(log, None)
254 # direct back to console, closing log files
255 with open(pluginLogName) as fin:
256 lines = fin.read()
257 # test that the sample plugin has correctly logged to where we
258 # expected it to.
259 self.assertTrue(lines.find("ERROR") >= 0)
262class SingleFrameTestCase(AlgorithmTestCase, lsst.utils.tests.TestCase):
264 def setUp(self):
265 # object in corner to trigger EDGE error
266 self.center = lsst.geom.Point2D(5, 5)
267 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
268 lsst.geom.Extent2I(100, 100))
269 self.dataset = lsst.meas.base.tests.TestDataset(self.bbox)
270 self.dataset.addSource(1000000.0, self.center)
271 self.task = self.makeSingleFrameMeasurementTask("base_SdssCentroid")
272 self.log = lsst.log.Log.getLogger(self.task.getPluginLogName("base_SdssCentroid"))
273 self.exposure, self.catalog = self.dataset.realize(10.0, self.task.schema, randomSeed=4)
275 def tearDown(self):
276 del self.center
277 del self.bbox
278 del self.dataset
279 del self.task
280 del self.log
281 del self.exposure
282 del self.catalog
284 def testSeparatePluginLogs(self):
285 """Check that the task log and the plugin log are truly separate.
286 """
287 taskLogName = os.path.join(ROOT, 'testSeparatePluginLogs-task.log')
288 directLog(self.task.log, taskLogName)
289 self.task.log.info("Testing")
290 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName:
291 directLog(self.log, pluginLogName)
292 self.log.setLevel(lsst.log.DEBUG)
293 self.task.run(self.catalog, self.exposure)
294 # direct back to console, closing log files
295 directLog(self.log, None)
296 directLog(self.task.log, None)
297 with open(taskLogName) as fin:
298 lines = fin.read()
299 os.unlink(taskLogName)
300 self.assertTrue(lines.find("Testing") >= 0)
301 with open(pluginLogName) as fin:
302 lines = fin.read()
303 self.assertTrue(lines.find("MeasurementError") >= 0)
305 def testSetPluginLevel(self):
306 """Test setting the plugin log level.
308 Specifically, we set it to the ``ERROR`` level.
309 """
310 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName:
311 directLog(self.log, pluginLogName)
312 self.log.setLevel(lsst.log.ERROR)
313 self.task.run(self.catalog, self.exposure)
314 # direct back to console, closing log files
315 directLog(self.log, None)
316 with open(pluginLogName) as fin:
317 lines = fin.read()
318 self.assertTrue(lines.find("MeasurementError") < 0)
321class ForcedTestCase(AlgorithmTestCase, lsst.utils.tests.TestCase):
323 def setUp(self):
324 # object in corner to trigger EDGE error
325 self.center = lsst.geom.Point2D(0, 0)
326 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
327 lsst.geom.Extent2I(100, 100))
328 self.dataset = lsst.meas.base.tests.TestDataset(self.bbox)
329 self.dataset.addSource(1000000.0, self.center)
330 self.task = self.makeForcedMeasurementTask("base_SdssCentroid")
331 self.log = lsst.log.Log.getLogger(self.task.getPluginLogName("base_SdssCentroid"))
332 measWcs = self.dataset.makePerturbedWcs(self.dataset.exposure.getWcs(), randomSeed=5)
333 measDataset = self.dataset.transform(measWcs)
334 self.exposure, truthCatalog = measDataset.realize(10.0, measDataset.makeMinimalSchema(), randomSeed=5)
335 self.refCat = self.dataset.catalog
336 self.refWcs = self.dataset.exposure.getWcs()
337 self.measCat = self.task.generateMeasCat(self.exposure, self.refCat, self.refWcs)
338 self.task.attachTransformedFootprints(self.measCat, self.refCat, self.exposure, self.refWcs)
340 def tearDown(self):
341 del self.center
342 del self.bbox
343 del self.dataset
344 del self.task
345 del self.log
346 del self.exposure
347 del self.measCat
348 del self.refCat
349 del self.refWcs
351 def testSeparatePluginLog(self):
352 """Check that the task log and the plugin log are truly separate.
353 """
354 taskLogName = os.path.join(ROOT, 'testSeparatePluginLog-task.log')
355 directLog(self.task.log, taskLogName)
356 self.task.log.info("Testing")
357 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName:
358 directLog(self.log, pluginLogName)
359 self.log.setLevel(lsst.log.DEBUG)
360 self.task.run(self.measCat, self.exposure, self.refCat, self.refWcs)
361 # direct back to console, closing log files
362 directLog(self.log, None)
363 directLog(self.task.log, None)
364 with open(taskLogName) as fin:
365 lines = fin.read()
366 os.unlink(taskLogName)
367 self.assertTrue(lines.find("Testing") >= 0)
368 with open(pluginLogName) as fin:
369 lines = fin.read()
370 self.assertTrue(lines.find("MeasurementError") >= 0)
372 def testSetPluginLevel(self):
373 """Test setting the plugin log level.
375 Specifically, we set it to the ``ERROR`` level.
376 """
377 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName:
378 directLog(self.log, pluginLogName)
379 self.log.setLevel(lsst.log.ERROR)
380 self.task.run(self.measCat, self.exposure, self.refCat, self.refWcs)
381 # direct back to console, closing log files
382 directLog(self.log, None)
383 with open(pluginLogName) as fin:
384 lines = fin.read()
385 self.assertTrue(lines.find("MeasurementError") < 0)
388class TestMemory(lsst.utils.tests.MemoryTestCase):
389 pass
392def setup_module(module):
393 lsst.utils.tests.init()
396if __name__ == "__main__": 396 ↛ 397line 396 didn't jump to line 397, because the condition on line 396 was never true
397 lsst.utils.tests.init()
398 unittest.main()