Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23import os 

24import numpy 

25 

26import lsst.geom 

27import lsst.afw.table 

28import lsst.daf.base 

29import lsst.meas.base 

30import lsst.utils.tests 

31from lsst.meas.base.tests import (AlgorithmTestCase, ) 

32from lsst.meas.base.sfm import SingleFramePluginConfig, SingleFramePlugin 

33from lsst.meas.base.forcedMeasurement import ForcedPlugin 

34from lsst.meas.base.pluginRegistry import register 

35from lsst.meas.base import FlagDefinitionList, FlagHandler, MeasurementError 

36 

37ROOT = os.path.abspath(os.path.dirname(__file__)) 

38 

39 

40class LoggingPluginConfig(SingleFramePluginConfig): 

41 """Configuration for sample plugin. 

42 """ 

43 pass 

44 

45 

46@register("test_LoggingPlugin") 

47class LoggingPlugin(SingleFramePlugin): 

48 """Sample Python plugin which has an associated log name. 

49 

50 Notes 

51 ----- 

52 The log name is provided to the plugin by the measurement task which is 

53 running it. This requires that the `hasLogName` attribute must be a member 

54 of the plugin class, and it must be `True`. 

55 """ 

56 hasLogName = True 

57 ConfigClass = LoggingPluginConfig 

58 

59 @classmethod 

60 def getExecutionOrder(cls): 

61 return cls.FLUX_ORDER 

62 

63 # The initializer for the class must accept an optional logName parameter. 

64 def __init__(self, config, name, schema, metadata, logName=None): 

65 SingleFramePlugin.__init__(self, config, name, schema, metadata, logName=logName) 

66 flagDefs = FlagDefinitionList() 

67 self.FAILURE = flagDefs.addFailureFlag() 

68 self.CONTAINS_NAN = flagDefs.add("flag_containsNan", "Measurement area contains a nan") 

69 self.flagHandler = FlagHandler.addFields(schema, name, flagDefs) 

70 self.instFluxKey = schema.addField(name + "_instFlux", "F", doc="flux") 

71 

72 def measure(self, measRecord, exposure): 

73 """Perform measurement. 

74 

75 Notes 

76 ----- 

77 The `measure` method is called by the measurement framework when `run` 

78 is called. If a `MeasurementError` is raised during this method, the 

79 `fail` method will be called to set the error flags. 

80 """ 

81 lsst.log.Log.getLogger(self.getLogName()).info("%s plugin measuring.", self.name) 

82 # Sum the pixels inside the bounding box 

83 centerPoint = lsst.geom.Point2I(int(measRecord.getX()), int(measRecord.getY())) 

84 bbox = lsst.geom.Box2I(centerPoint, lsst.geom.Extent2I(1, 1)) 

85 instFlux = lsst.afw.image.ImageF(exposure.getMaskedImage().getImage(), bbox).getArray().sum() 

86 measRecord.set(self.instFluxKey, instFlux) 

87 

88 # If there was a NaN inside the bounding box, the instFlux will still 

89 # be NaN 

90 if numpy.isnan(instFlux): 

91 raise MeasurementError(self.CONTAINS_NAN.doc, self.CONTAINS_NAN.number) 

92 

93 def fail(self, measRecord, error=None): 

94 """Handle measurement failures. 

95 

96 Notes 

97 ----- 

98 If measurement raises a `MeasurementError`, the error will be passed 

99 to the fail method by the measurement framework. If the error is not 

100 `None`, ``error.cpp`` should correspond to a specific error and the 

101 appropriate error flag will be set. 

102 """ 

103 if error is None: 

104 self.flagHandler.handleFailure(measRecord) 

105 else: 

106 self.flagHandler.handleFailure(measRecord, error.cpp) 

107 

108 

109def directLog(log, file=None): 

110 """Direct the log given to a file or to the console if ``file`` is `None`. 

111 """ 

112 props = "log4j.rootLogger=INFO, FA\n" 

113 if file is None: 

114 props += "log4j.appender.FA=ConsoleAppender\n" 

115 else: 

116 props += "log4j.appender.FA=FileAppender\n" 

117 props += "log4j.appender.FA.Append=false\n" 

118 props += "log4j.appender.FA.file=%s\n"%(file,) 

119 props += "log4j.appender.FA.Append=false\n" 

120 props += "log4j.appender.FA.layout=PatternLayout\n" 

121 props += "log4j.appender.FA.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c %m %X%n\n" 

122 props += "log4j.logger.main.a=DEBUG\n" 

123 log.configure_prop(props) 

124 

125 

126class RegisteredPluginsTestCase(AlgorithmTestCase, lsst.utils.tests.TestCase): 

127 """Test all registered Plugins to see if their logName is set as expected. 

128 

129 Those which have the ``hasLogName=True`` attribute will have a ``logName`` 

130 parameter passed to their ``__init__``, and should set the internal 

131 ``_logName`` attribute. If they are wrapped C++ algorithms, the 

132 `getLogName` should also return same ``logName`` as the plugin. 

133 """ 

134 def testSingleFramePlugins(self): 

135 center = lsst.geom.Point2D(50, 50) 

136 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), 

137 lsst.geom.Extent2I(100, 100)) 

138 dataset = lsst.meas.base.tests.TestDataset(bbox) 

139 dataset.addSource(1000000.0, center) 

140 registry = SingleFramePlugin.registry 

141 dependencies = registry.keys() 

142 task = self.makeSingleFrameMeasurementTask("base_SdssCentroid", dependencies=dependencies) 

143 exposure, catalog = dataset.realize(noise=100.0, schema=task.schema, randomSeed=0) 

144 task.log.setLevel(lsst.log.ERROR) 

145 task.run(catalog, exposure) 

146 for pluginName in dependencies: 

147 plugin = task.plugins[pluginName] 

148 if hasattr(plugin, "hasLogName") and plugin.hasLogName: 

149 self.assertEqual(plugin.getLogName(), task.log.getChild(pluginName).getName()) 

150 # if the plugin is cpp, check the cpp Algorithm as well 

151 if hasattr(plugin, "cpp"): 

152 self.assertEqual(plugin.cpp.getLogName(), plugin.getLogName()) 

153 else: 

154 self.assertIsNone(plugin.getLogName()) 

155 

156 def testForcedPlugins(self): 

157 # Test all the ForcedPlugins registered to see if their logName is set 

158 # as expected. 

159 center = lsst.geom.Point2D(50, 50) 

160 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), 

161 lsst.geom.Extent2I(100, 100)) 

162 dataset = lsst.meas.base.tests.TestDataset(bbox) 

163 dataset.addSource(1000000.0, center) 

164 registry = ForcedPlugin.registry 

165 dependencies = registry.keys() 

166 

167 task = self.makeForcedMeasurementTask("base_SdssCentroid", dependencies=dependencies) 

168 measWcs = dataset.makePerturbedWcs(dataset.exposure.getWcs(), randomSeed=1) 

169 measDataset = dataset.transform(measWcs) 

170 exposure, truthCatalog = measDataset.realize(10.0, measDataset.makeMinimalSchema(), randomSeed=1) 

171 refCat = dataset.catalog 

172 refWcs = dataset.exposure.getWcs() 

173 measCat = task.generateMeasCat(exposure, refCat, refWcs) 

174 task.attachTransformedFootprints(measCat, refCat, exposure, refWcs) 

175 

176 task.log.setLevel(lsst.log.ERROR) 

177 task.run(measCat, exposure, refCat, refWcs) 

178 for pluginName in dependencies: 

179 plugin = task.plugins[pluginName] 

180 if hasattr(plugin, "hasLogName") and plugin.hasLogName: 

181 self.assertEqual(plugin.getLogName(), task.log.getChild(pluginName).getName()) 

182 # if the plugin is cpp, check the cpp Algorithm as well 

183 if hasattr(plugin, "cpp"): 

184 self.assertEqual(plugin.cpp.getLogName(), task.log.getName() + "." + pluginName) 

185 else: 

186 self.assertIsNone(plugin.getLogName()) 

187 

188 

189class LoggingPythonTestCase(AlgorithmTestCase, lsst.utils.tests.TestCase): 

190 """Test one C++ and one Python plugin which have hasLogName=True. 

191 """ 

192 def setUp(self): 

193 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(100, 100)) 

194 self.dataset = lsst.meas.base.tests.TestDataset(bbox) 

195 self.dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 25)) 

196 config = lsst.meas.base.SingleFrameMeasurementConfig() 

197 config.slots.centroid = None 

198 config.slots.apFlux = None 

199 config.slots.calibFlux = None 

200 config.slots.gaussianFlux = None 

201 config.slots.modelFlux = None 

202 config.slots.psfFlux = None 

203 config.slots.shape = None 

204 config.slots.psfShape = None 

205 self.config = config 

206 

207 def tearDown(self): 

208 del self.config 

209 del self.dataset 

210 

211 def testLoggingPythonPlugin(self): 

212 algName = "test_LoggingPlugin" 

213 schema = self.dataset.makeMinimalSchema() 

214 self.config.plugins = [algName] 

215 task = lsst.meas.base.SingleFrameMeasurementTask(schema=schema, config=self.config) 

216 # test that the plugin's logName has been propagated to the plugin 

217 self.assertEqual(task.plugins[algName].getLogName(), task.log.getChild(algName).getName()) 

218 log = task.log.getChild(algName) 

219 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName: 

220 directLog(log, pluginLogName) 

221 exposure, cat = self.dataset.realize(noise=0.0, schema=schema, randomSeed=2) 

222 task.run(cat, exposure) 

223 directLog(log, None) 

224 # direct back to console, closing log files 

225 with open(pluginLogName) as fin: 

226 lines = fin.read() 

227 # test that the sample plugin has correctly logged to where we 

228 # expected it to. 

229 self.assertGreaterEqual(lines.find("measuring"), 0) 

230 

231 def testLoggingCppPlugin(self): 

232 # PsfFlux is known to log an ``ERROR`` if a Psf is not attached 

233 algName = "base_PsfFlux" 

234 self.config.plugins = [algName] 

235 

236 schema = self.dataset.makeMinimalSchema() 

237 task = lsst.meas.base.SingleFrameMeasurementTask(schema=schema, config=self.config) 

238 log = task.log.getChild(algName) 

239 log.setLevel(lsst.log.ERROR) 

240 

241 # test that the plugin's logName has been propagated to the plugin 

242 self.assertEqual(task.plugins[algName].getLogName(), task.log.getChild(algName).getName()) 

243 self.assertEqual(task.plugins[algName].cpp.getLogName(), task.log.getChild(algName).getName()) 

244 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName: 

245 directLog(log, pluginLogName) 

246 exposure, cat = self.dataset.realize(noise=0.0, schema=schema, randomSeed=3) 

247 exposure.setPsf(None) 

248 # This call throws an error, so be prepared for it 

249 try: 

250 task.run(cat, exposure) 

251 except Exception: 

252 pass 

253 directLog(log, None) 

254 # direct back to console, closing log files 

255 with open(pluginLogName) as fin: 

256 lines = fin.read() 

257 # test that the sample plugin has correctly logged to where we 

258 # expected it to. 

259 self.assertGreaterEqual(lines.find("ERROR"), 0) 

260 

261 

262class SingleFrameTestCase(AlgorithmTestCase, lsst.utils.tests.TestCase): 

263 

264 def setUp(self): 

265 # object in corner to trigger EDGE error 

266 self.center = lsst.geom.Point2D(5, 5) 

267 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), 

268 lsst.geom.Extent2I(100, 100)) 

269 self.dataset = lsst.meas.base.tests.TestDataset(self.bbox) 

270 self.dataset.addSource(1000000.0, self.center) 

271 self.task = self.makeSingleFrameMeasurementTask("base_SdssCentroid") 

272 self.log = self.task.log.getChild("base_SdssCentroid") 

273 self.exposure, self.catalog = self.dataset.realize(10.0, self.task.schema, randomSeed=4) 

274 

275 def tearDown(self): 

276 del self.center 

277 del self.bbox 

278 del self.dataset 

279 del self.task 

280 del self.log 

281 del self.exposure 

282 del self.catalog 

283 

284 def testSeparatePluginLogs(self): 

285 """Check that the task log and the plugin log are truly separate. 

286 """ 

287 taskLogName = os.path.join(ROOT, 'testSeparatePluginLogs-task.log') 

288 directLog(self.task.log, taskLogName) 

289 self.task.log.info("Testing") 

290 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName: 

291 directLog(self.log, pluginLogName) 

292 self.log.setLevel(lsst.log.DEBUG) 

293 self.task.run(self.catalog, self.exposure) 

294 # direct back to console, closing log files 

295 directLog(self.log, None) 

296 directLog(self.task.log, None) 

297 with open(taskLogName) as fin: 

298 lines = fin.read() 

299 os.unlink(taskLogName) 

300 self.assertGreaterEqual(lines.find("Testing"), 0) 

301 with open(pluginLogName) as fin: 

302 lines = fin.read() 

303 self.assertGreaterEqual(lines.find("MeasurementError"), 0) 

304 

305 def testSetPluginLevel(self): 

306 """Test setting the plugin log level. 

307 

308 Specifically, we set it to the ``ERROR`` level. 

309 """ 

310 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName: 

311 directLog(self.log, pluginLogName) 

312 self.log.setLevel(lsst.log.ERROR) 

313 self.task.run(self.catalog, self.exposure) 

314 # direct back to console, closing log files 

315 directLog(self.log, None) 

316 with open(pluginLogName) as fin: 

317 lines = fin.read() 

318 self.assertLess(lines.find("MeasurementError"), 0) 

319 

320 

321class ForcedTestCase(AlgorithmTestCase, lsst.utils.tests.TestCase): 

322 

323 def setUp(self): 

324 # object in corner to trigger EDGE error 

325 self.center = lsst.geom.Point2D(0, 0) 

326 self.bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), 

327 lsst.geom.Extent2I(100, 100)) 

328 self.dataset = lsst.meas.base.tests.TestDataset(self.bbox) 

329 self.dataset.addSource(1000000.0, self.center) 

330 self.task = self.makeForcedMeasurementTask("base_SdssCentroid") 

331 self.log = self.task.log.getChild("base_SdssCentroid") 

332 measWcs = self.dataset.makePerturbedWcs(self.dataset.exposure.getWcs(), randomSeed=5) 

333 measDataset = self.dataset.transform(measWcs) 

334 self.exposure, truthCatalog = measDataset.realize(10.0, measDataset.makeMinimalSchema(), randomSeed=5) 

335 self.refCat = self.dataset.catalog 

336 self.refWcs = self.dataset.exposure.getWcs() 

337 self.measCat = self.task.generateMeasCat(self.exposure, self.refCat, self.refWcs) 

338 self.task.attachTransformedFootprints(self.measCat, self.refCat, self.exposure, self.refWcs) 

339 

340 def tearDown(self): 

341 del self.center 

342 del self.bbox 

343 del self.dataset 

344 del self.task 

345 del self.log 

346 del self.exposure 

347 del self.measCat 

348 del self.refCat 

349 del self.refWcs 

350 

351 def testSeparatePluginLog(self): 

352 """Check that the task log and the plugin log are truly separate. 

353 """ 

354 taskLogName = os.path.join(ROOT, 'testSeparatePluginLog-task.log') 

355 directLog(self.task.log, taskLogName) 

356 self.task.log.info("Testing") 

357 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName: 

358 directLog(self.log, pluginLogName) 

359 self.log.setLevel(lsst.log.DEBUG) 

360 self.task.run(self.measCat, self.exposure, self.refCat, self.refWcs) 

361 # direct back to console, closing log files 

362 directLog(self.log, None) 

363 directLog(self.task.log, None) 

364 with open(taskLogName) as fin: 

365 lines = fin.read() 

366 os.unlink(taskLogName) 

367 self.assertGreaterEqual(lines.find("Testing"), 0) 

368 with open(pluginLogName) as fin: 

369 lines = fin.read() 

370 self.assertGreaterEqual(lines.find("MeasurementError"), 0) 

371 

372 def testSetPluginLevel(self): 

373 """Test setting the plugin log level. 

374 

375 Specifically, we set it to the ``ERROR`` level. 

376 """ 

377 with lsst.utils.tests.getTempFilePath(".log") as pluginLogName: 

378 directLog(self.log, pluginLogName) 

379 self.log.setLevel(lsst.log.ERROR) 

380 self.task.run(self.measCat, self.exposure, self.refCat, self.refWcs) 

381 # direct back to console, closing log files 

382 directLog(self.log, None) 

383 with open(pluginLogName) as fin: 

384 lines = fin.read() 

385 self.assertLess(lines.find("MeasurementError"), 0) 

386 

387 

388class TestMemory(lsst.utils.tests.MemoryTestCase): 

389 pass 

390 

391 

392def setup_module(module): 

393 lsst.utils.tests.init() 

394 

395 

396if __name__ == "__main__": 396 ↛ 397line 396 didn't jump to line 397, because the condition on line 396 was never true

397 lsst.utils.tests.init() 

398 unittest.main()