Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Base measurement task, which subclassed by the single frame and forced 

23measurement tasks. 

24""" 

25 

26import lsst.pipe.base 

27import lsst.pex.config 

28 

29from .pluginRegistry import PluginMap 

30from .exceptions import FatalAlgorithmError, MeasurementError 

31from .pluginsBase import BasePluginConfig, BasePlugin 

32from .noiseReplacer import NoiseReplacerConfig 

33 

34__all__ = ("BaseMeasurementPluginConfig", "BaseMeasurementPlugin", 

35 "BaseMeasurementConfig", "BaseMeasurementTask") 

36 

37# Exceptions that the measurement tasks should always propagate up to their callers 

38FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError) 

39 

40 

41class BaseMeasurementPluginConfig(BasePluginConfig): 

42 """Base config class for all measurement plugins. 

43 

44 Notes 

45 ----- 

46 Most derived classes will want to override `setDefaults` in order to 

47 customize the default `executionOrder`. 

48 

49 A derived class whose corresponding Plugin class implements a do `measureN` 

50 method should additionally add a bool `doMeasureN` field to replace the 

51 bool class attribute defined here. 

52 """ 

53 

54 doMeasure = lsst.pex.config.Field(dtype=bool, default=True, 

55 doc="whether to run this plugin in single-object mode") 

56 

57 doMeasureN = False # replace this class attribute with a Field if measureN-capable 

58 

59 

60class BaseMeasurementPlugin(BasePlugin): 

61 """Base class for all measurement plugins. 

62 

63 Notes 

64 ----- 

65 This is class is a placeholder for future behavior which will be shared 

66 only between measurement plugins and is implemented for symmetry with the 

67 measurement base plugin configuration class 

68 """ 

69 

70 pass 

71 

72 

73class SourceSlotConfig(lsst.pex.config.Config): 

74 """Assign named plugins to measurement slots. 

75 

76 Slot configuration which assigns a particular named plugin to each of a set 

77 of slots. Each slot allows a type of measurement to be fetched from the 

78 `lsst.afw.table.SourceTable` without knowing which algorithm was used to 

79 produced the data. 

80 

81 Notes 

82 ----- 

83 The default algorithm for each slot must be registered, even if the default 

84 is not used. 

85 """ 

86 

87 Field = lsst.pex.config.Field 

88 centroid = Field(dtype=str, default="base_SdssCentroid", optional=True, 

89 doc="the name of the centroiding algorithm used to set source x,y") 

90 shape = Field(dtype=str, default="base_SdssShape", optional=True, 

91 doc="the name of the algorithm used to set source moments parameters") 

92 psfShape = Field(dtype=str, default="base_SdssShape_psf", optional=True, 

93 doc="the name of the algorithm used to set PSF moments parameters") 

94 apFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True, 

95 doc="the name of the algorithm used to set the source aperture instFlux slot") 

96 modelFlux = Field(dtype=str, default="base_GaussianFlux", optional=True, 

97 doc="the name of the algorithm used to set the source model instFlux slot") 

98 psfFlux = Field(dtype=str, default="base_PsfFlux", optional=True, 

99 doc="the name of the algorithm used to set the source psf instFlux slot") 

100 gaussianFlux = Field(dtype=str, default="base_GaussianFlux", optional=True, 

101 doc="the name of the algorithm used to set the source Gaussian instFlux slot") 

102 calibFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True, 

103 doc="the name of the instFlux measurement algorithm used for calibration") 

104 

105 def setupSchema(self, schema): 

106 """Set up a slots in a schema following configuration directives. 

107 

108 Parameters 

109 ---------- 

110 schema : `lsst.afw.table.Schema` 

111 The schema in which slots will be set up. 

112 

113 Notes 

114 ----- 

115 This is defined in this configuration class to support use in unit 

116 tests without needing to construct an `lsst.pipe.base.Task` object. 

117 """ 

118 aliases = schema.getAliasMap() 

119 if self.centroid is not None: 

120 aliases.set("slot_Centroid", self.centroid) 

121 if self.shape is not None: 

122 aliases.set("slot_Shape", self.shape) 

123 if self.psfShape is not None: 

124 aliases.set("slot_PsfShape", self.psfShape) 

125 if self.apFlux is not None: 

126 aliases.set("slot_ApFlux", self.apFlux) 

127 if self.modelFlux is not None: 

128 aliases.set("slot_ModelFlux", self.modelFlux) 

129 if self.psfFlux is not None: 

130 aliases.set("slot_PsfFlux", self.psfFlux) 

131 if self.gaussianFlux is not None: 

132 aliases.set("slot_GaussianFlux", self.gaussianFlux) 

133 if self.calibFlux is not None: 

134 aliases.set("slot_CalibFlux", self.calibFlux) 

135 

136 

137class BaseMeasurementConfig(lsst.pex.config.Config): 

138 """Base configuration for all measurement driver tasks. 

139 

140 Examples 

141 -------- 

142 Subclasses should define the 'plugins' and 'undeblended' registries, e.g. 

143 

144 .. code-block:: py 

145 

146 plugins = PluginBaseClass.registry.makeField( 

147 multi=True, 

148 default=[], 

149 doc="Plugins to be run and their configuration" 

150 ) 

151 undeblended = PluginBaseClass.registry.makeField( 

152 multi=True, 

153 default=[], 

154 doc="Plugins to run on undeblended image" 

155 ) 

156 

157 where ``PluginBaseClass`` is the appropriate base class of the plugin 

158 (e.g., `SingleFramePlugin` or `ForcedPlugin`). 

159 """ 

160 

161 slots = lsst.pex.config.ConfigField( 

162 dtype=SourceSlotConfig, 

163 doc="Mapping from algorithms to special aliases in Source." 

164 ) 

165 

166 doReplaceWithNoise = lsst.pex.config.Field( 

167 dtype=bool, default=True, optional=False, 

168 doc='When measuring, replace other detected footprints with noise?') 

169 

170 noiseReplacer = lsst.pex.config.ConfigField( 

171 dtype=NoiseReplacerConfig, 

172 doc="configuration that sets how to replace neighboring sources with noise" 

173 ) 

174 undeblendedPrefix = lsst.pex.config.Field( 

175 dtype=str, default="undeblended_", 

176 doc="Prefix to give undeblended plugins" 

177 ) 

178 

179 def validate(self): 

180 lsst.pex.config.Config.validate(self) 

181 if self.slots.centroid is not None and self.slots.centroid not in self.plugins.names: 

182 raise ValueError("source centroid slot algorithm is not being run.") 

183 if self.slots.shape is not None and self.slots.shape not in self.plugins.names: 

184 raise ValueError("source shape slot algorithm '%s' is not being run." % self.slots.shape) 

185 for slot in (self.slots.psfFlux, self.slots.apFlux, self.slots.modelFlux, 

186 self.slots.gaussianFlux, self.slots.calibFlux): 

187 if slot is not None: 

188 for name in self.plugins.names: 

189 if len(name) <= len(slot) and name == slot[:len(name)]: 

190 break 

191 else: 

192 raise ValueError("source instFlux slot algorithm '%s' is not being run." % slot) 

193 

194 

195class BaseMeasurementTask(lsst.pipe.base.Task): 

196 """Ultimate base class for all measurement tasks. 

197 

198 Parameters 

199 ---------- 

200 algMetadata : `lsst.daf.base.PropertyList` or `None` 

201 Will be modified in-place to contain metadata about the plugins being 

202 run. If `None`, an empty `~lsst.daf.base.PropertyList` will be 

203 created. 

204 **kwds 

205 Additional arguments passed to `lsst.pipe.base.Task.__init__`. 

206 

207 Notes 

208 ----- 

209 This base class for `SingleFrameMeasurementTask` and 

210 `ForcedMeasurementTask` mostly exists to share code between the two, and 

211 generally should not be used directly. 

212 """ 

213 

214 ConfigClass = BaseMeasurementConfig 

215 _DefaultName = "measurement" 

216 

217 plugins = None 

218 """Plugins to be invoked (`PluginMap`). 

219 

220 Initially empty, this will be populated as plugins are initialized. It 

221 should be considered read-only. 

222 """ 

223 

224 algMetadata = None 

225 """Metadata about active plugins (`lsst.daf.base.PropertyList`). 

226 

227 Contains additional information about active plugins to be saved with 

228 the output catalog. Will be filled by subclasses. 

229 """ 

230 

231 def __init__(self, algMetadata=None, **kwds): 

232 super(BaseMeasurementTask, self).__init__(**kwds) 

233 self.plugins = PluginMap() 

234 self.undeblendedPlugins = PluginMap() 

235 if algMetadata is None: 

236 algMetadata = lsst.daf.base.PropertyList() 

237 self.algMetadata = algMetadata 

238 

239 def getPluginLogName(self, pluginName): 

240 return self.log.getName() + '.' + pluginName 

241 

242 def initializePlugins(self, **kwds): 

243 """Initialize plugins (and slots) according to configuration. 

244 

245 Parameters 

246 ---------- 

247 **kwds 

248 Keyword arguments forwarded directly to plugin constructors. 

249 

250 Notes 

251 ----- 

252 Derived class constructors should call this method to fill the 

253 `plugins` attribute and add corresponding output fields and slot 

254 aliases to the output schema. 

255 

256 In addition to the attributes added by `BaseMeasurementTask.__init__`, 

257 a ``schema``` attribute holding the output schema must be present 

258 before this method is called. 

259 

260 Keyword arguments are forwarded directly to plugin constructors, 

261 allowing derived classes to use plugins with different signatures. 

262 """ 

263 # Make a place at the beginning for the centroid plugin to run first (because it's an OrderedDict, 

264 # adding an empty element in advance means it will get run first when it's reassigned to the 

265 # actual Plugin). 

266 if self.config.slots.centroid is not None: 

267 self.plugins[self.config.slots.centroid] = None 

268 # Init the plugins, sorted by execution order. At the same time add to the schema 

269 for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()): 

270 # Pass logName to the plugin if the plugin is marked as using it 

271 # The task will use this name to log plugin errors, regardless. 

272 if hasattr(PluginClass, "hasLogName") and PluginClass.hasLogName: 

273 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, 

274 logName=self.getPluginLogName(name), **kwds) 

275 else: 

276 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds) 

277 

278 # In rare circumstances (usually tests), the centroid slot not be coming from an algorithm, 

279 # which means we'll have added something we don't want to the plugins map, and we should 

280 # remove it. 

281 if self.config.slots.centroid is not None and self.plugins[self.config.slots.centroid] is None: 

282 del self.plugins[self.config.slots.centroid] 

283 # Initialize the plugins to run on the undeblended image 

284 for executionOrder, name, config, PluginClass in sorted(self.config.undeblended.apply()): 

285 undeblendedName = self.config.undeblendedPrefix + name 

286 self.undeblendedPlugins[name] = PluginClass(config, undeblendedName, metadata=self.algMetadata, 

287 **kwds) 

288 

289 def callMeasure(self, measRecord, *args, **kwds): 

290 """Call ``measure`` on all plugins and consistently handle exceptions. 

291 

292 Parameters 

293 ---------- 

294 measRecord : `lsst.afw.table.SourceRecord` 

295 The record corresponding to the object being measured. Will be 

296 updated in-place with the results of measurement. 

297 *args 

298 Positional arguments forwarded to ``plugin.measure`` 

299 **kwds 

300 Keyword arguments. Two are handled locally: 

301 

302 beginOrder : `int` 

303 Beginning execution order (inclusive). Measurements with 

304 ``executionOrder`` < ``beginOrder`` are not executed. `None` 

305 for no limit. 

306 

307 endOrder : `int` 

308 Ending execution order (exclusive). Measurements with 

309 ``executionOrder`` >= ``endOrder`` are not executed. `None` 

310 for no limit. 

311 

312 Others are forwarded to ``plugin.measure()``. 

313 

314 Notes 

315 ----- 

316 This method can be used with plugins that have different signatures; 

317 the only requirement is that ``measRecord`` be the first argument. 

318 Subsequent positional arguments and keyword arguments are forwarded 

319 directly to the plugin. 

320 

321 This method should be considered "protected": it is intended for use by 

322 derived classes, not users. 

323 """ 

324 beginOrder = kwds.pop("beginOrder", None) 

325 endOrder = kwds.pop("endOrder", None) 

326 for plugin in self.plugins.iter(): 

327 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder: 

328 continue 

329 if endOrder is not None and plugin.getExecutionOrder() >= endOrder: 

330 break 

331 self.doMeasurement(plugin, measRecord, *args, **kwds) 

332 

333 def doMeasurement(self, plugin, measRecord, *args, **kwds): 

334 """Call ``measure`` on the specified plugin. 

335 

336 Exceptions are handled in a consistent way. 

337 

338 Parameters 

339 ---------- 

340 plugin : subclass of `BasePlugin` 

341 Plugin that will be executed. 

342 measRecord : `lsst.afw.table.SourceRecord` 

343 The record corresponding to the object being measured. Will be 

344 updated in-place with the results of measurement. 

345 *args 

346 Positional arguments forwarded to ``plugin.measure()``. 

347 **kwds 

348 Keyword arguments forwarded to ``plugin.measure()``. 

349 

350 Notes 

351 ----- 

352 This method can be used with plugins that have different signatures; 

353 the only requirement is that ``plugin`` and ``measRecord`` be the first 

354 two arguments. Subsequent positional arguments and keyword arguments 

355 are forwarded directly to the plugin. 

356 

357 This method should be considered "protected": it is intended for use by 

358 derived classes, not users. 

359 """ 

360 try: 

361 plugin.measure(measRecord, *args, **kwds) 

362 except FATAL_EXCEPTIONS: 

363 raise 

364 except MeasurementError as error: 

365 lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug( 

366 "MeasurementError in %s.measure on record %s: %s" 

367 % (plugin.name, measRecord.getId(), error)) 

368 plugin.fail(measRecord, error) 

369 except Exception as error: 

370 lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug( 

371 "Exception in %s.measure on record %s: %s" 

372 % (plugin.name, measRecord.getId(), error)) 

373 plugin.fail(measRecord) 

374 

375 def callMeasureN(self, measCat, *args, **kwds): 

376 """Call ``measureN`` on all plugins and consistently handle exceptions. 

377 

378 Parameters 

379 ---------- 

380 measCat : `lsst.afw.table.SourceCatalog` 

381 Catalog containing only the records for the source family to be 

382 measured, and where outputs should be written. 

383 *args 

384 Positional arguments forwarded to ``plugin.measure()`` 

385 **kwds 

386 Keyword arguments. Two are handled locally: 

387 

388 beginOrder: 

389 Beginning execution order (inclusive): Measurements with 

390 ``executionOrder`` < ``beginOrder`` are not executed. `None` 

391 for no limit. 

392 endOrder: 

393 Ending execution order (exclusive): measurements with 

394 ``executionOrder`` >= ``endOrder`` are not executed. `None` for 

395 no ``limit``. 

396 

397 Others are are forwarded to ``plugin.measure()``. 

398 

399 Notes 

400 ----- 

401 This method can be used with plugins that have different signatures; 

402 the only requirement is that ``measRecord`` be the first argument. 

403 Subsequent positional arguments and keyword arguments are forwarded 

404 directly to the plugin. 

405 

406 This method should be considered "protected": it is intended for use by 

407 derived classes, not users. 

408 """ 

409 beginOrder = kwds.pop("beginOrder", None) 

410 endOrder = kwds.pop("endOrder", None) 

411 for plugin in self.plugins.iterN(): 

412 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder: 

413 continue 

414 if endOrder is not None and plugin.getExecutionOrder() >= endOrder: 

415 break 

416 self.doMeasurementN(plugin, measCat, *args, **kwds) 

417 

418 def doMeasurementN(self, plugin, measCat, *args, **kwds): 

419 """Call ``measureN`` on the specified plugin. 

420 

421 Exceptions are handled in a consistent way. 

422 

423 Parameters 

424 ---------- 

425 plugin : subclass of `BasePlugin` 

426 Plugin that will be executed. 

427 measCat : `lsst.afw.table.SourceCatalog` 

428 Catalog containing only the records for the source family to be 

429 measured, and where outputs should be written. 

430 *args 

431 Positional arguments forwarded to ``plugin.measureN()``. 

432 **kwds 

433 Keyword arguments forwarded to ``plugin.measureN()``. 

434 

435 Notes 

436 ----- 

437 This method can be used with plugins that have different signatures; 

438 the only requirement is that the ``plugin`` and ``measCat`` be the 

439 first two arguments. Subsequent positional arguments and keyword 

440 arguments are forwarded directly to the plugin. 

441 

442 This method should be considered "protected": it is intended for use by 

443 derived classes, not users. 

444 """ 

445 try: 

446 plugin.measureN(measCat, *args, **kwds) 

447 except FATAL_EXCEPTIONS: 

448 raise 

449 

450 except MeasurementError as error: 

451 for measRecord in measCat: 

452 lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug( 

453 "MeasurementError in %s.measureN on records %s-%s: %s" 

454 % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error)) 

455 plugin.fail(measRecord, error) 

456 except Exception as error: 

457 for measRecord in measCat: 

458 plugin.fail(measRecord) 

459 lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug( 

460 "Exception in %s.measureN on records %s-%s: %s" 

461 % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))