Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

#!/usr/bin/env python 

# 

# LSST Data Management System 

# Copyright 2008-2016 AURA/LSST. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <https://www.lsstcorp.org/LegalNotices/>. 

# 

"""Base measurement task, which subclassed by the single frame and forced measurement tasks. 

""" 

import lsst.pipe.base 

import lsst.pex.config 

 

from .pluginRegistry import PluginMap 

from .exceptions import FatalAlgorithmError, MeasurementError 

from .pluginsBase import BasePluginConfig, BasePlugin 

from .noiseReplacer import NoiseReplacerConfig 

 

__all__ = ("BaseMeasurementPluginConfig", "BaseMeasurementPlugin", 

"BaseMeasurementConfig", "BaseMeasurementTask") 

 

# Exceptions that the measurement tasks should always propagate up to their callers 

FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError) 

 

 

class BaseMeasurementPluginConfig(BasePluginConfig): 

"""! 

Base config class for all measurement plugins 

 

Most derived classes will want to override setDefaults() in order to customize 

the default exceutionOrder. 

 

A derived class whose corresponding Plugin class implements measureN() should 

additionally add a bool doMeasureN field to replace the bool class attribute 

defined here. 

""" 

 

doMeasure = lsst.pex.config.Field(dtype=bool, default=True, 

doc="whether to run this plugin in single-object mode") 

 

doMeasureN = False # replace this class attribute with a Field if measureN-capable 

 

 

class BaseMeasurementPlugin(BasePlugin): 

''' 

Base class for all measurement plugins 

 

This is class is a placeholder for future behavior which will be shared only between 

measurement plugins and is implemented for symmetry with the measurement base plugin 

configuration class 

''' 

pass 

 

 

class SourceSlotConfig(lsst.pex.config.Config): 

"""! 

Slot configuration which assigns a particular named plugin to each of a set of 

slots. Each slot allows a type of measurement to be fetched from the SourceTable 

without knowing which algorithm was used to produced the data. 

 

NOTE: the default algorithm for each slot must be registered, even if the default is not used. 

""" 

 

Field = lsst.pex.config.Field 

centroid = Field(dtype=str, default="base_SdssCentroid", optional=True, 

doc="the name of the centroiding algorithm used to set source x,y") 

shape = Field(dtype=str, default="base_SdssShape", optional=True, 

doc="the name of the algorithm used to set source moments parameters") 

psfShape = Field(dtype=str, default="base_SdssShape_psf", optional=True, 

doc="the name of the algorithm used to set PSF moments parameters") 

apFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True, 

doc="the name of the algorithm used to set the source aperture instFlux slot") 

modelFlux = Field(dtype=str, default="base_GaussianFlux", optional=True, 

doc="the name of the algorithm used to set the source model instFlux slot") 

psfFlux = Field(dtype=str, default="base_PsfFlux", optional=True, 

doc="the name of the algorithm used to set the source psf instFlux slot") 

gaussianFlux = Field(dtype=str, default="base_GaussianFlux", optional=True, 

doc="the name of the algorithm used to set the source Gaussian instFlux slot") 

calibFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True, 

doc="the name of the instFlux measurement algorithm used for calibration") 

 

def setupSchema(self, schema): 

"""Convenience method to setup a Schema's slots according to the config definition. 

 

This is defined in the Config class to support use in unit tests without needing 

to construct a Task object. 

""" 

aliases = schema.getAliasMap() 

if self.centroid is not None: 

aliases.set("slot_Centroid", self.centroid) 

if self.shape is not None: 

aliases.set("slot_Shape", self.shape) 

if self.psfShape is not None: 

aliases.set("slot_PsfShape", self.psfShape) 

if self.apFlux is not None: 

aliases.set("slot_ApFlux", self.apFlux) 

if self.modelFlux is not None: 

aliases.set("slot_ModelFlux", self.modelFlux) 

if self.psfFlux is not None: 

aliases.set("slot_PsfFlux", self.psfFlux) 

if self.gaussianFlux is not None: 

aliases.set("slot_GaussianFlux", self.gaussianFlux) 

if self.calibFlux is not None: 

aliases.set("slot_CalibFlux", self.calibFlux) 

 

 

class BaseMeasurementConfig(lsst.pex.config.Config): 

"""! 

Base config class for all measurement driver tasks. 

 

Subclasses should define the 'plugins' and 'undeblended' registries, e.g.: 

 

plugins = PluginBaseClass.registry.makeField( 

multi=True, 

default=[], 

doc="Plugins to be run and their configuration" 

) 

undeblended = PluginBaseClass.registry.makeField( 

multi=True, 

default=[], 

doc="Plugins to run on undeblended image" 

) 

 

where PluginBaseClass is the appropriate base class of the plugin 

(e.g., SingleFramePlugin or ForcedPlugin). 

""" 

 

slots = lsst.pex.config.ConfigField( 

dtype=SourceSlotConfig, 

doc="Mapping from algorithms to special aliases in Source." 

) 

 

doReplaceWithNoise = lsst.pex.config.Field( 

dtype=bool, default=True, optional=False, 

doc='When measuring, replace other detected footprints with noise?') 

 

noiseReplacer = lsst.pex.config.ConfigField( 

dtype=NoiseReplacerConfig, 

doc="configuration that sets how to replace neighboring sources with noise" 

) 

undeblendedPrefix = lsst.pex.config.Field( 

dtype=str, default="undeblended_", 

doc="Prefix to give undeblended plugins" 

) 

 

def validate(self): 

lsst.pex.config.Config.validate(self) 

162 ↛ 163line 162 didn't jump to line 163, because the condition on line 162 was never true if self.slots.centroid is not None and self.slots.centroid not in self.plugins.names: 

raise ValueError("source centroid slot algorithm is not being run.") 

164 ↛ 165line 164 didn't jump to line 165, because the condition on line 164 was never true if self.slots.shape is not None and self.slots.shape not in self.plugins.names: 

raise ValueError("source shape slot algorithm '%s' is not being run." % self.slots.shape) 

for slot in (self.slots.psfFlux, self.slots.apFlux, self.slots.modelFlux, 

self.slots.gaussianFlux, self.slots.calibFlux): 

168 ↛ 169line 168 didn't jump to line 169, because the condition on line 168 was never true if slot is not None: 

for name in self.plugins.names: 

if len(name) <= len(slot) and name == slot[:len(name)]: 

break 

else: 

raise ValueError("source instFlux slot algorithm '%s' is not being run." % slot) 

 

## @addtogroup LSST_task_documentation 

## @{ 

## @page baseMeasurementTask 

## BaseMeasurementTask @copybrief BaseMeasurementTask 

## @} 

 

 

class BaseMeasurementTask(lsst.pipe.base.Task): 

"""! 

Ultimate base class for all measurement tasks. 

 

This base class for SingleFrameMeasurementTask and ForcedMeasurementTask mostly exists to share 

code between the two, and generally should not be used directly. 

""" 

 

ConfigClass = BaseMeasurementConfig 

_DefaultName = "measurement" 

 

def __init__(self, algMetadata=None, **kwds): 

"""! 

Constructor; only called by derived classes. 

 

@param[in] algMetadata An lsst.daf.base.PropertyList that will be filled with metadata 

about the plugins being run. If None, an empty PropertyList will 

be created. 

@param[in] **kwds Additional arguments passed to lsst.pipe.base.Task.__init__. 

 

This attaches two public attributes to the class for use by derived classes and parent tasks: 

- plugins: an empty PluginMap, which will eventually contain all active plugins that will by 

invoked by the run() method (to be filled by subclasses). This should be considered read-only. 

- algMetadata: a lsst.daf.base.PropertyList that will contain additional information about the 

active plugins to be saved with the output catalog (to be filled by subclasses). 

""" 

super(BaseMeasurementTask, self).__init__(**kwds) 

self.plugins = PluginMap() 

self.undeblendedPlugins = PluginMap() 

if algMetadata is None: 

algMetadata = lsst.daf.base.PropertyList() 

self.algMetadata = algMetadata 

 

def getPluginLogName(self, pluginName): 

return self.log.getName() + '.' + pluginName 

 

def initializePlugins(self, **kwds): 

"""Initialize the plugins (and slots) according to the configuration. 

 

Derived class constructors should call this method to fill the self.plugins 

attribute and add correspond output fields and slot aliases to the output schema. 

 

In addition to the attributes added by BaseMeasurementTask.__init__, a self.schema 

attribute holding the output schema must also be present before this method is called, . 

 

Keyword arguments are forwarded directly to plugin constructors, allowing derived 

classes to use plugins with different signatures. 

""" 

# Make a place at the beginning for the centroid plugin to run first (because it's an OrderedDict, 

# adding an empty element in advance means it will get run first when it's reassigned to the 

# actual Plugin). 

if self.config.slots.centroid is not None: 

self.plugins[self.config.slots.centroid] = None 

# Init the plugins, sorted by execution order. At the same time add to the schema 

for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()): 

# Pass logName to the plugin if the plugin is marked as using it 

# The task will use this name to log plugin errors, regardless. 

if hasattr(PluginClass, "hasLogName") and PluginClass.hasLogName: 

self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, 

logName=self.getPluginLogName(name), **kwds) 

else: 

self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds) 

 

# In rare circumstances (usually tests), the centroid slot not be coming from an algorithm, 

# which means we'll have added something we don't want to the plugins map, and we should 

# remove it. 

if self.config.slots.centroid is not None and self.plugins[self.config.slots.centroid] is None: 

del self.plugins[self.config.slots.centroid] 

# Initialize the plugins to run on the undeblended image 

for executionOrder, name, config, PluginClass in sorted(self.config.undeblended.apply()): 

undeblendedName = self.config.undeblendedPrefix + name 

self.undeblendedPlugins[name] = PluginClass(config, undeblendedName, metadata=self.algMetadata, 

**kwds) 

 

def callMeasure(self, measRecord, *args, **kwds): 

"""! 

Call the measure() method on all plugins, handling exceptions in a consistent way. 

 

@param[in,out] measRecord lsst.afw.table.SourceRecord that corresponds to the object being 

measured, and where outputs should be written. 

@param[in] *args Positional arguments forwarded to Plugin.measure() 

@param[in] **kwds Keyword arguments. Two are handled locally: 

- beginOrder: beginning execution order (inclusive): measurements with 

executionOrder < beginOrder are not executed. None for no limit. 

- endOrder: ending execution order (exclusive): measurements with 

executionOrder >= endOrder are not executed. None for no limit. 

the rest are forwarded to Plugin.measure() 

 

This method can be used with plugins that have different signatures; the only requirement is that 

'measRecord' be the first argument. Subsequent positional arguments and keyword arguments are 

forwarded directly to the plugin. 

 

This method should be considered "protected"; it is intended for use by derived classes, not users. 

""" 

beginOrder = kwds.pop("beginOrder", None) 

endOrder = kwds.pop("endOrder", None) 

for plugin in self.plugins.iter(): 

279 ↛ 280line 279 didn't jump to line 280, because the condition on line 279 was never true if beginOrder is not None and plugin.getExecutionOrder() < beginOrder: 

continue 

281 ↛ 282line 281 didn't jump to line 282, because the condition on line 281 was never true if endOrder is not None and plugin.getExecutionOrder() >= endOrder: 

break 

self.doMeasurement(plugin, measRecord, *args, **kwds) 

 

def doMeasurement(self, plugin, measRecord, *args, **kwds): 

"""! 

Call the measure() method on the nominated plugin, handling exceptions in a consistent way. 

 

@param[in] plugin Plugin that will measure 

@param[in,out] measRecord lsst.afw.table.SourceRecord that corresponds to the object being 

measured, and where outputs should be written. 

@param[in] *args Positional arguments forwarded to plugin.measure() 

@param[in] **kwds Keyword arguments forwarded to plugin.measure() 

 

This method can be used with plugins that have different signatures; the only requirement is that 

the 'plugin' and 'measRecord' be the first two arguments. Subsequent positional arguments and 

keyword arguments are forwarded directly to the plugin. 

 

This method should be considered "protected"; it is intended for use by derived classes, not users. 

""" 

try: 

plugin.measure(measRecord, *args, **kwds) 

except FATAL_EXCEPTIONS: 

raise 

except MeasurementError as error: 

lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug( 

"MeasurementError in %s.measure on record %s: %s" 

% (plugin.name, measRecord.getId(), error)) 

plugin.fail(measRecord, error) 

except Exception as error: 

lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug( 

"Exception in %s.measure on record %s: %s" 

% (plugin.name, measRecord.getId(), error)) 

plugin.fail(measRecord) 

 

def callMeasureN(self, measCat, *args, **kwds): 

"""! 

Call the measureN() method on all plugins, handling exceptions in a consistent way. 

 

@param[in,out] measCat lsst.afw.table.SourceCatalog containing records for just 

the source family to be measured, and where outputs should 

be written. 

@param[in] beginOrder beginning execution order (inclusive): measurements with 

executionOrder < beginOrder are not executed. None for no limit. 

@param[in] endOrder ending execution order (exclusive): measurements with 

executionOrder >= endOrder are not executed. None for no limit. 

@param[in] *args Positional arguments forwarded to Plugin.measure() 

@param[in] **kwds Keyword arguments. Two are handled locally: 

- beginOrder: beginning execution order (inclusive): measurements with 

executionOrder < beginOrder are not executed. None for no limit. 

- endOrder: ending execution order (exclusive): measurements with 

executionOrder >= endOrder are not executed. None for no limit. 

the rest are forwarded to Plugin.measure() 

 

This method can be used with plugins that have different signatures; the only requirement is that 

'measRecord' be the first argument. Subsequent positional arguments and keyword arguments are 

forwarded directly to the plugin. 

 

This method should be considered "protected"; it is intended for use by derived classes, not users. 

""" 

beginOrder = kwds.pop("beginOrder", None) 

endOrder = kwds.pop("endOrder", None) 

343 ↛ 344line 343 didn't jump to line 344, because the loop on line 343 never started for plugin in self.plugins.iterN(): 

if beginOrder is not None and plugin.getExecutionOrder() < beginOrder: 

continue 

if endOrder is not None and plugin.getExecutionOrder() >= endOrder: 

break 

self.doMeasurementN(plugin, measCat, *args, **kwds) 

 

def doMeasurementN(self, plugin, measCat, *args, **kwds): 

"""! 

Call the measureN() method on the nominated plugin, handling exceptions in a consistent way. 

 

@param[in] plugin Plugin that will measure 

@param[in,out] measCat lsst.afw.table.SourceCatalog containing records for just 

the source family to be measured, and where outputs should 

be written. 

@param[in] *args Positional arguments forwarded to plugin.measureN() 

@param[in] **kwds Keyword arguments forwarded to plugin.measureN() 

 

This method can be used with plugins that have different signatures; the only requirement is that 

the 'plugin' and 'measCat' be the first two arguments. Subsequent positional arguments and 

keyword arguments are forwarded directly to the plugin. 

 

This method should be considered "protected"; it is intended for use by derived classes, not users. 

""" 

try: 

plugin.measureN(measCat, *args, **kwds) 

except FATAL_EXCEPTIONS: 

raise 

 

except MeasurementError as error: 

for measRecord in measCat: 

lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug( 

"MeasurementError in %s.measureN on records %s-%s: %s" 

% (plugin.name, measCat[0].getId(), measCat[-1].getId(), error)) 

plugin.fail(measRecord, error) 

except Exception as error: 

for measRecord in measCat: 

plugin.fail(measRecord) 

lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug( 

"Exception in %s.measureN on records %s-%s: %s" 

% (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))