Coverage for python/lsst/meas/base/baseMeasurement.py: 26%

139 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-15 10:18 +0000

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Base measurement task, which subclassed by the single frame and forced 

23measurement tasks. 

24""" 

25import warnings 

26 

27import lsst.pipe.base 

28import lsst.pex.config 

29 

30from .pluginRegistry import PluginMap 

31from .exceptions import FatalAlgorithmError, MeasurementError 

32from .pluginsBase import BasePluginConfig, BasePlugin 

33from .noiseReplacer import NoiseReplacerConfig 

34 

35__all__ = ("BaseMeasurementPluginConfig", "BaseMeasurementPlugin", 

36 "BaseMeasurementConfig", "BaseMeasurementTask") 

37 

38# Exceptions that the measurement tasks should always propagate up to their 

39# callers 

40FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError) 

41 

42 

43class BaseMeasurementPluginConfig(BasePluginConfig): 

44 """Base config class for all measurement plugins. 

45 

46 Notes 

47 ----- 

48 Most derived classes will want to override `setDefaults` in order to 

49 customize the default `executionOrder`. 

50 

51 A derived class whose corresponding Plugin class implements a do `measureN` 

52 method should additionally add a bool `doMeasureN` field to replace the 

53 bool class attribute defined here. 

54 """ 

55 

56 doMeasure = lsst.pex.config.Field(dtype=bool, default=True, 

57 doc="whether to run this plugin in single-object mode") 

58 

59 doMeasureN = False # replace this class attribute with a Field if measureN-capable 

60 

61 

62class BaseMeasurementPlugin(BasePlugin): 

63 """Base class for all measurement plugins. 

64 

65 Notes 

66 ----- 

67 This is class is a placeholder for future behavior which will be shared 

68 only between measurement plugins and is implemented for symmetry with the 

69 measurement base plugin configuration class 

70 """ 

71 

72 pass 

73 

74 

75class SourceSlotConfig(lsst.pex.config.Config): 

76 """Assign named plugins to measurement slots. 

77 

78 Slot configuration which assigns a particular named plugin to each of a set 

79 of slots. Each slot allows a type of measurement to be fetched from the 

80 `lsst.afw.table.SourceTable` without knowing which algorithm was used to 

81 produced the data. 

82 

83 Notes 

84 ----- 

85 The default algorithm for each slot must be registered, even if the default 

86 is not used. 

87 """ 

88 

89 Field = lsst.pex.config.Field 

90 centroid = Field(dtype=str, default="base_SdssCentroid", optional=True, 

91 doc="the name of the centroiding algorithm used to set source x,y") 

92 shape = Field(dtype=str, default="base_SdssShape", optional=True, 

93 doc="the name of the algorithm used to set source moments parameters") 

94 psfShape = Field(dtype=str, default="base_SdssShape_psf", optional=True, 

95 doc="the name of the algorithm used to set PSF moments parameters") 

96 apFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True, 

97 doc="the name of the algorithm used to set the source aperture instFlux slot") 

98 modelFlux = Field(dtype=str, default="base_GaussianFlux", optional=True, 

99 doc="the name of the algorithm used to set the source model instFlux slot") 

100 psfFlux = Field(dtype=str, default="base_PsfFlux", optional=True, 

101 doc="the name of the algorithm used to set the source psf instFlux slot") 

102 gaussianFlux = Field(dtype=str, default="base_GaussianFlux", optional=True, 

103 doc="the name of the algorithm used to set the source Gaussian instFlux slot") 

104 calibFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True, 

105 doc="the name of the instFlux measurement algorithm used for calibration") 

106 

107 def setupSchema(self, schema): 

108 """Set up a slots in a schema following configuration directives. 

109 

110 Parameters 

111 ---------- 

112 schema : `lsst.afw.table.Schema` 

113 The schema in which slots will be set up. 

114 

115 Notes 

116 ----- 

117 This is defined in this configuration class to support use in unit 

118 tests without needing to construct an `lsst.pipe.base.Task` object. 

119 """ 

120 aliases = schema.getAliasMap() 

121 if self.centroid is not None: 

122 aliases.set("slot_Centroid", self.centroid) 

123 if self.shape is not None: 

124 aliases.set("slot_Shape", self.shape) 

125 if self.psfShape is not None: 

126 aliases.set("slot_PsfShape", self.psfShape) 

127 if self.apFlux is not None: 

128 aliases.set("slot_ApFlux", self.apFlux) 

129 if self.modelFlux is not None: 

130 aliases.set("slot_ModelFlux", self.modelFlux) 

131 if self.psfFlux is not None: 

132 aliases.set("slot_PsfFlux", self.psfFlux) 

133 if self.gaussianFlux is not None: 

134 aliases.set("slot_GaussianFlux", self.gaussianFlux) 

135 if self.calibFlux is not None: 

136 aliases.set("slot_CalibFlux", self.calibFlux) 

137 

138 

139class BaseMeasurementConfig(lsst.pex.config.Config): 

140 """Base configuration for all measurement driver tasks. 

141 

142 Parameters 

143 ---------- 

144 ignoreSlotPluginChecks : `bool`, optional 

145 Do not check that all slots have an associated plugin to run when 

146 validating this config. This is primarily for tests that were written 

147 before we made Tasks always call `config.validate()` on init. 

148 DEPRECATED DM-35949: this is a temporary workaround while we better 

149 define how config/schema validation works for measurement tasks. 

150 

151 Examples 

152 -------- 

153 Subclasses should define the 'plugins' and 'undeblended' registries, e.g. 

154 

155 .. code-block:: py 

156 

157 plugins = PluginBaseClass.registry.makeField( 

158 multi=True, 

159 default=[], 

160 doc="Plugins to be run and their configuration" 

161 ) 

162 undeblended = PluginBaseClass.registry.makeField( 

163 multi=True, 

164 default=[], 

165 doc="Plugins to run on undeblended image" 

166 ) 

167 

168 where ``PluginBaseClass`` is the appropriate base class of the plugin 

169 (e.g., `SingleFramePlugin` or `ForcedPlugin`). 

170 """ 

171 def __new__(cls, *args, ignoreSlotPluginChecks=False, **kwargs): 

172 instance = super().__new__(cls, *args, **kwargs) 

173 if ignoreSlotPluginChecks: 

174 msg = ("ignoreSlotPluginChecks is deprecated and should only be used in tests." 

175 " No removal date has been set; see DM-35949.") 

176 warnings.warn(msg, category=FutureWarning, stacklevel=2) 

177 object.__setattr__(instance, "_ignoreSlotPluginChecks", ignoreSlotPluginChecks) 

178 return instance 

179 

180 slots = lsst.pex.config.ConfigField( 

181 dtype=SourceSlotConfig, 

182 doc="Mapping from algorithms to special aliases in Source." 

183 ) 

184 

185 doReplaceWithNoise = lsst.pex.config.Field( 

186 dtype=bool, default=True, optional=False, 

187 doc='When measuring, replace other detected footprints with noise?') 

188 

189 noiseReplacer = lsst.pex.config.ConfigField( 

190 dtype=NoiseReplacerConfig, 

191 doc="configuration that sets how to replace neighboring sources with noise" 

192 ) 

193 undeblendedPrefix = lsst.pex.config.Field( 

194 dtype=str, default="undeblended_", 

195 doc="Prefix to give undeblended plugins" 

196 ) 

197 

198 def validate(self): 

199 super().validate() 

200 if self._ignoreSlotPluginChecks: 

201 return 

202 if self.slots.centroid is not None and self.slots.centroid not in self.plugins.names: 

203 raise ValueError("source centroid slot algorithm is not being run.") 

204 if self.slots.shape is not None and self.slots.shape not in self.plugins.names: 

205 raise ValueError("source shape slot algorithm '%s' is not being run." % self.slots.shape) 

206 for slot in (self.slots.psfFlux, self.slots.apFlux, self.slots.modelFlux, 

207 self.slots.gaussianFlux, self.slots.calibFlux): 

208 if slot is not None: 

209 for name in self.plugins.names: 

210 if len(name) <= len(slot) and name == slot[:len(name)]: 

211 break 

212 else: 

213 raise ValueError("source instFlux slot algorithm '%s' is not being run." % slot) 

214 

215 

216class BaseMeasurementTask(lsst.pipe.base.Task): 

217 """Ultimate base class for all measurement tasks. 

218 

219 Parameters 

220 ---------- 

221 algMetadata : `lsst.daf.base.PropertyList` or `None` 

222 Will be modified in-place to contain metadata about the plugins being 

223 run. If `None`, an empty `~lsst.daf.base.PropertyList` will be 

224 created. 

225 **kwds 

226 Additional arguments passed to `lsst.pipe.base.Task.__init__`. 

227 

228 Notes 

229 ----- 

230 This base class for `SingleFrameMeasurementTask` and 

231 `ForcedMeasurementTask` mostly exists to share code between the two, and 

232 generally should not be used directly. 

233 """ 

234 

235 ConfigClass = BaseMeasurementConfig 

236 _DefaultName = "measurement" 

237 

238 plugins = None 

239 """Plugins to be invoked (`PluginMap`). 

240 

241 Initially empty, this will be populated as plugins are initialized. It 

242 should be considered read-only. 

243 """ 

244 

245 algMetadata = None 

246 """Metadata about active plugins (`lsst.daf.base.PropertyList`). 

247 

248 Contains additional information about active plugins to be saved with 

249 the output catalog. Will be filled by subclasses. 

250 """ 

251 

252 def __init__(self, algMetadata=None, **kwds): 

253 super(BaseMeasurementTask, self).__init__(**kwds) 

254 self.plugins = PluginMap() 

255 self.undeblendedPlugins = PluginMap() 

256 if algMetadata is None: 

257 algMetadata = lsst.daf.base.PropertyList() 

258 self.algMetadata = algMetadata 

259 

260 def initializePlugins(self, **kwds): 

261 """Initialize plugins (and slots) according to configuration. 

262 

263 Parameters 

264 ---------- 

265 **kwds 

266 Keyword arguments forwarded directly to plugin constructors. 

267 

268 Notes 

269 ----- 

270 Derived class constructors should call this method to fill the 

271 `plugins` attribute and add corresponding output fields and slot 

272 aliases to the output schema. 

273 

274 In addition to the attributes added by `BaseMeasurementTask.__init__`, 

275 a ``schema``` attribute holding the output schema must be present 

276 before this method is called. 

277 

278 Keyword arguments are forwarded directly to plugin constructors, 

279 allowing derived classes to use plugins with different signatures. 

280 """ 

281 # Make a place at the beginning for the centroid plugin to run first 

282 # (because it's an OrderedDict, adding an empty element in advance 

283 # means it will get run first when it's reassigned to the actual 

284 # Plugin). 

285 if self.config.slots.centroid is not None: 

286 self.plugins[self.config.slots.centroid] = None 

287 # Init the plugins, sorted by execution order. At the same time add to 

288 # the schema 

289 for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()): 

290 # Pass logName to the plugin if the plugin is marked as using it 

291 # The task will use this name to log plugin errors, regardless. 

292 if getattr(PluginClass, "hasLogName", False): 

293 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, 

294 logName=self.log.getChild(name).name, **kwds) 

295 else: 

296 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds) 

297 

298 # In rare circumstances (usually tests), the centroid slot not be 

299 # coming from an algorithm, which means we'll have added something we 

300 # don't want to the plugins map, and we should remove it. 

301 if self.config.slots.centroid is not None and self.plugins[self.config.slots.centroid] is None: 

302 del self.plugins[self.config.slots.centroid] 

303 # Initialize the plugins to run on the undeblended image 

304 for executionOrder, name, config, PluginClass in sorted(self.config.undeblended.apply()): 

305 undeblendedName = self.config.undeblendedPrefix + name 

306 if getattr(PluginClass, "hasLogName", False): 

307 self.undeblendedPlugins[name] = PluginClass(config, undeblendedName, 

308 metadata=self.algMetadata, 

309 logName=self.log.getChild(undeblendedName).name, 

310 **kwds) 

311 else: 

312 self.undeblendedPlugins[name] = PluginClass(config, undeblendedName, 

313 metadata=self.algMetadata, **kwds) 

314 

315 def callMeasure(self, measRecord, *args, **kwds): 

316 """Call ``measure`` on all plugins and consistently handle exceptions. 

317 

318 Parameters 

319 ---------- 

320 measRecord : `lsst.afw.table.SourceRecord` 

321 The record corresponding to the object being measured. Will be 

322 updated in-place with the results of measurement. 

323 *args 

324 Positional arguments forwarded to ``plugin.measure`` 

325 **kwds 

326 Keyword arguments. Two are handled locally: 

327 

328 beginOrder : `int` 

329 Beginning execution order (inclusive). Measurements with 

330 ``executionOrder`` < ``beginOrder`` are not executed. `None` 

331 for no limit. 

332 

333 endOrder : `int` 

334 Ending execution order (exclusive). Measurements with 

335 ``executionOrder`` >= ``endOrder`` are not executed. `None` 

336 for no limit. 

337 

338 Others are forwarded to ``plugin.measure()``. 

339 

340 Notes 

341 ----- 

342 This method can be used with plugins that have different signatures; 

343 the only requirement is that ``measRecord`` be the first argument. 

344 Subsequent positional arguments and keyword arguments are forwarded 

345 directly to the plugin. 

346 

347 This method should be considered "protected": it is intended for use by 

348 derived classes, not users. 

349 """ 

350 beginOrder = kwds.pop("beginOrder", None) 

351 endOrder = kwds.pop("endOrder", None) 

352 for plugin in self.plugins.iter(): 

353 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder: 

354 continue 

355 if endOrder is not None and plugin.getExecutionOrder() >= endOrder: 

356 break 

357 self.doMeasurement(plugin, measRecord, *args, **kwds) 

358 

359 def doMeasurement(self, plugin, measRecord, *args, **kwds): 

360 """Call ``measure`` on the specified plugin. 

361 

362 Exceptions are handled in a consistent way. 

363 

364 Parameters 

365 ---------- 

366 plugin : subclass of `BasePlugin` 

367 Plugin that will be executed. 

368 measRecord : `lsst.afw.table.SourceRecord` 

369 The record corresponding to the object being measured. Will be 

370 updated in-place with the results of measurement. 

371 *args 

372 Positional arguments forwarded to ``plugin.measure()``. 

373 **kwds 

374 Keyword arguments forwarded to ``plugin.measure()``. 

375 

376 Notes 

377 ----- 

378 This method can be used with plugins that have different signatures; 

379 the only requirement is that ``plugin`` and ``measRecord`` be the first 

380 two arguments. Subsequent positional arguments and keyword arguments 

381 are forwarded directly to the plugin. 

382 

383 This method should be considered "protected": it is intended for use by 

384 derived classes, not users. 

385 """ 

386 try: 

387 plugin.measure(measRecord, *args, **kwds) 

388 except FATAL_EXCEPTIONS: 

389 raise 

390 except MeasurementError as error: 

391 self.log.getChild(plugin.name).debug( 

392 "MeasurementError in %s.measure on record %s: %s", 

393 plugin.name, measRecord.getId(), error) 

394 plugin.fail(measRecord, error) 

395 except Exception as error: 

396 self.log.getChild(plugin.name).debug( 

397 "Exception in %s.measure on record %s: %s", 

398 plugin.name, measRecord.getId(), error) 

399 plugin.fail(measRecord) 

400 

401 def callMeasureN(self, measCat, *args, **kwds): 

402 """Call ``measureN`` on all plugins and consistently handle exceptions. 

403 

404 Parameters 

405 ---------- 

406 measCat : `lsst.afw.table.SourceCatalog` 

407 Catalog containing only the records for the source family to be 

408 measured, and where outputs should be written. 

409 *args 

410 Positional arguments forwarded to ``plugin.measure()`` 

411 **kwds 

412 Keyword arguments. Two are handled locally: 

413 

414 beginOrder: 

415 Beginning execution order (inclusive): Measurements with 

416 ``executionOrder`` < ``beginOrder`` are not executed. `None` 

417 for no limit. 

418 endOrder: 

419 Ending execution order (exclusive): measurements with 

420 ``executionOrder`` >= ``endOrder`` are not executed. `None` for 

421 no ``limit``. 

422 

423 Others are are forwarded to ``plugin.measure()``. 

424 

425 Notes 

426 ----- 

427 This method can be used with plugins that have different signatures; 

428 the only requirement is that ``measRecord`` be the first argument. 

429 Subsequent positional arguments and keyword arguments are forwarded 

430 directly to the plugin. 

431 

432 This method should be considered "protected": it is intended for use by 

433 derived classes, not users. 

434 """ 

435 beginOrder = kwds.pop("beginOrder", None) 

436 endOrder = kwds.pop("endOrder", None) 

437 for plugin in self.plugins.iterN(): 

438 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder: 

439 continue 

440 if endOrder is not None and plugin.getExecutionOrder() >= endOrder: 

441 break 

442 self.doMeasurementN(plugin, measCat, *args, **kwds) 

443 

444 def doMeasurementN(self, plugin, measCat, *args, **kwds): 

445 """Call ``measureN`` on the specified plugin. 

446 

447 Exceptions are handled in a consistent way. 

448 

449 Parameters 

450 ---------- 

451 plugin : subclass of `BasePlugin` 

452 Plugin that will be executed. 

453 measCat : `lsst.afw.table.SourceCatalog` 

454 Catalog containing only the records for the source family to be 

455 measured, and where outputs should be written. 

456 *args 

457 Positional arguments forwarded to ``plugin.measureN()``. 

458 **kwds 

459 Keyword arguments forwarded to ``plugin.measureN()``. 

460 

461 Notes 

462 ----- 

463 This method can be used with plugins that have different signatures; 

464 the only requirement is that the ``plugin`` and ``measCat`` be the 

465 first two arguments. Subsequent positional arguments and keyword 

466 arguments are forwarded directly to the plugin. 

467 

468 This method should be considered "protected": it is intended for use by 

469 derived classes, not users. 

470 """ 

471 try: 

472 plugin.measureN(measCat, *args, **kwds) 

473 except FATAL_EXCEPTIONS: 

474 raise 

475 

476 except MeasurementError as error: 

477 for measRecord in measCat: 

478 self.log.getChild(plugin.name).debug( 

479 "MeasurementError in %s.measureN on records %s-%s: %s", 

480 plugin.name, measCat[0].getId(), measCat[-1].getId(), error) 

481 plugin.fail(measRecord, error) 

482 except Exception as error: 

483 for measRecord in measCat: 

484 plugin.fail(measRecord) 

485 self.log.getChild(plugin.name).debug( 

486 "Exception in %s.measureN on records %s-%s: %s", 

487 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)