Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24__all__ = ["BaseSourceSelectorConfig", "BaseSourceSelectorTask", "sourceSelectorRegistry", 

25 "ColorLimit", "MagnitudeLimit", "SignalToNoiseLimit", "MagnitudeErrorLimit", 

26 "RequireFlags", "RequireUnresolved", 

27 "ScienceSourceSelectorConfig", "ScienceSourceSelectorTask", 

28 "ReferenceSourceSelectorConfig", "ReferenceSourceSelectorTask", 

29 ] 

30 

31import abc 

32import numpy as np 

33import astropy.units as u 

34 

35import lsst.pex.config as pexConfig 

36import lsst.pipe.base as pipeBase 

37 

38 

39class BaseSourceSelectorConfig(pexConfig.Config): 

40 pass 

41 

42 

43class BaseSourceSelectorTask(pipeBase.Task, metaclass=abc.ABCMeta): 

44 """Base class for source selectors 

45 

46 Source selectors are classes that perform a selection on a catalog 

47 object given a set of criteria or cuts. They return the selected catalog 

48 and can optionally set a specified Flag field in the input catalog to 

49 identify if the source was selected. 

50 

51 Register all source selectors with the sourceSelectorRegistry using: 

52 sourceSelectorRegistry.register(name, class) 

53 

54 Attributes 

55 ---------- 

56 usesMatches : `bool` 

57 A boolean variable specify if the inherited source selector uses 

58 matches to an external catalog, and thus requires the ``matches`` 

59 argument to ``run()``. 

60 """ 

61 

62 ConfigClass = BaseSourceSelectorConfig 

63 _DefaultName = "sourceSelector" 

64 usesMatches = False 

65 

66 def __init__(self, **kwargs): 

67 pipeBase.Task.__init__(self, **kwargs) 

68 

69 def run(self, sourceCat, sourceSelectedField=None, matches=None, exposure=None): 

70 """Select sources and return them. 

71 

72 The input catalog must be contiguous in memory. 

73 

74 Parameters: 

75 ----------- 

76 sourceCat : `lsst.afw.table.SourceCatalog` 

77 Catalog of sources to select from. 

78 sourceSelectedField : `str` or None 

79 Name of flag field in sourceCat to set for selected sources. 

80 If set, will modify sourceCat in-place. 

81 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

82 List of matches to use for source selection. 

83 If usesMatches is set in source selector this field is required. 

84 If not, it is ignored. 

85 exposure : `lsst.afw.image.Exposure` or None 

86 The exposure the catalog was built from; used for debug display. 

87 

88 Return 

89 ------ 

90 struct : `lsst.pipe.base.Struct` 

91 The struct contains the following data: 

92 

93 - sourceCat : `lsst.afw.table.SourceCatalog` 

94 The catalog of sources that were selected. 

95 (may not be memory-contiguous) 

96 - selected : `numpy.ndarray` of `bool`` 

97 Boolean array of sources that were selected, same length as 

98 sourceCat. 

99 

100 Raises 

101 ------ 

102 RuntimeError 

103 Raised if ``sourceCat`` is not contiguous. 

104 """ 

105 if not sourceCat.isContiguous(): 

106 raise RuntimeError("Input catalogs for source selection must be contiguous.") 

107 

108 result = self.selectSources(sourceCat=sourceCat, 

109 exposure=exposure, 

110 matches=matches) 

111 

112 if sourceSelectedField is not None: 

113 source_selected_key = \ 

114 sourceCat.getSchema()[sourceSelectedField].asKey() 

115 # TODO: Remove for loop when DM-6981 is completed. 

116 for source, flag in zip(sourceCat, result.selected): 

117 source.set(source_selected_key, bool(flag)) 

118 return pipeBase.Struct(sourceCat=sourceCat[result.selected], 

119 selected=result.selected) 

120 

121 @abc.abstractmethod 

122 def selectSources(self, sourceCat, matches=None, exposure=None): 

123 """Return a selection of sources selected by some criteria. 

124 

125 Parameters 

126 ---------- 

127 sourceCat : `lsst.afw.table.SourceCatalog` 

128 Catalog of sources to select from. 

129 This catalog must be contiguous in memory. 

130 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

131 A list of lsst.afw.table.ReferenceMatch objects 

132 exposure : `lsst.afw.image.Exposure` or None 

133 The exposure the catalog was built from; used for debug display. 

134 

135 Return 

136 ------ 

137 struct : `lsst.pipe.base.Struct` 

138 The struct contains the following data: 

139 

140 - selected : `numpy.ndarray` of `bool`` 

141 Boolean array of sources that were selected, same length as 

142 sourceCat. 

143 """ 

144 raise NotImplementedError("BaseSourceSelectorTask is abstract") 

145 

146 

147sourceSelectorRegistry = pexConfig.makeRegistry( 

148 doc="A registry of source selectors (subclasses of " 

149 "BaseSourceSelectorTask)", 

150) 

151 

152 

153class BaseLimit(pexConfig.Config): 

154 """Base class for selecting sources by applying a limit 

155 

156 This object can be used as a `lsst.pex.config.Config` for configuring 

157 the limit, and then the `apply` method can be used to identify sources 

158 in the catalog that match the configured limit. 

159 

160 This provides the `maximum` and `minimum` fields in the Config, and 

161 a method to apply the limits to an array of values calculated by the 

162 subclass. 

163 """ 

164 minimum = pexConfig.Field(dtype=float, optional=True, doc="Select objects with value greater than this") 

165 maximum = pexConfig.Field(dtype=float, optional=True, doc="Select objects with value less than this") 

166 

167 def apply(self, values): 

168 """Apply the limits to an array of values 

169 

170 Subclasses should calculate the array of values and then 

171 return the result of calling this method. 

172 

173 Parameters 

174 ---------- 

175 values : `numpy.ndarray` 

176 Array of values to which to apply limits. 

177 

178 Returns 

179 ------- 

180 selected : `numpy.ndarray` 

181 Boolean array indicating for each source whether it is selected 

182 (True means selected). 

183 """ 

184 selected = np.ones(len(values), dtype=bool) 

185 with np.errstate(invalid="ignore"): # suppress NAN warnings 

186 if self.minimum is not None: 

187 selected &= values > self.minimum 

188 if self.maximum is not None: 

189 selected &= values < self.maximum 

190 return selected 

191 

192 

193class ColorLimit(BaseLimit): 

194 """Select sources using a color limit 

195 

196 This object can be used as a `lsst.pex.config.Config` for configuring 

197 the limit, and then the `apply` method can be used to identify sources 

198 in the catalog that match the configured limit. 

199 

200 We refer to 'primary' and 'secondary' flux measurements; these are the 

201 two components of the color, which is: 

202 

203 instFluxToMag(cat[primary]) - instFluxToMag(cat[secondary]) 

204 """ 

205 primary = pexConfig.Field(dtype=str, doc="Name of column with primary flux measurement") 

206 secondary = pexConfig.Field(dtype=str, doc="Name of column with secondary flux measurement") 

207 

208 def apply(self, catalog): 

209 """Apply the color limit to a catalog 

210 

211 Parameters 

212 ---------- 

213 catalog : `lsst.afw.table.SourceCatalog` 

214 Catalog of sources to which the limit will be applied. 

215 

216 Returns 

217 ------- 

218 selected : `numpy.ndarray` 

219 Boolean array indicating for each source whether it is selected 

220 (True means selected). 

221 """ 

222 primary = (catalog[self.primary]*u.nJy).to_value(u.ABmag) 

223 secondary = (catalog[self.secondary]*u.nJy).to_value(u.ABmag) 

224 color = primary - secondary 

225 return BaseLimit.apply(self, color) 

226 

227 

228class FluxLimit(BaseLimit): 

229 """Select sources using a flux limit 

230 

231 This object can be used as a `lsst.pex.config.Config` for configuring 

232 the limit, and then the `apply` method can be used to identify sources 

233 in the catalog that match the configured limit. 

234 """ 

235 fluxField = pexConfig.Field(dtype=str, default="slot_CalibFlux_instFlux", 

236 doc="Name of the source flux field to use.") 

237 

238 def apply(self, catalog): 

239 """Apply the flux limits to a catalog 

240 

241 Parameters 

242 ---------- 

243 catalog : `lsst.afw.table.SourceCatalog` 

244 Catalog of sources to which the limit will be applied. 

245 

246 Returns 

247 ------- 

248 selected : `numpy.ndarray` 

249 Boolean array indicating for each source whether it is selected 

250 (True means selected). 

251 """ 

252 flagField = self.fluxField + "_flag" 

253 if flagField in catalog.schema: 

254 selected = np.logical_not(catalog[flagField]) 

255 else: 

256 selected = np.ones(len(catalog), dtype=bool) 

257 

258 flux = catalog[self.fluxField] 

259 selected &= BaseLimit.apply(self, flux) 

260 return selected 

261 

262 

263class MagnitudeLimit(BaseLimit): 

264 """Select sources using a magnitude limit 

265 

266 Note that this assumes that a zero-point has already been applied and 

267 the fluxes are in AB fluxes in Jansky. It is therefore principally 

268 intended for reference catalogs rather than catalogs extracted from 

269 science images. 

270 

271 This object can be used as a `lsst.pex.config.Config` for configuring 

272 the limit, and then the `apply` method can be used to identify sources 

273 in the catalog that match the configured limit. 

274 """ 

275 fluxField = pexConfig.Field(dtype=str, default="flux", 

276 doc="Name of the source flux field to use.") 

277 

278 def apply(self, catalog): 

279 """Apply the magnitude limits to a catalog 

280 

281 Parameters 

282 ---------- 

283 catalog : `lsst.afw.table.SourceCatalog` 

284 Catalog of sources to which the limit will be applied. 

285 

286 Returns 

287 ------- 

288 selected : `numpy.ndarray` 

289 Boolean array indicating for each source whether it is selected 

290 (True means selected). 

291 """ 

292 flagField = self.fluxField + "_flag" 

293 if flagField in catalog.schema: 

294 selected = np.logical_not(catalog[flagField]) 

295 else: 

296 selected = np.ones(len(catalog), dtype=bool) 

297 

298 magnitude = (catalog[self.fluxField]*u.nJy).to_value(u.ABmag) 

299 selected &= BaseLimit.apply(self, magnitude) 

300 return selected 

301 

302 

303class SignalToNoiseLimit(BaseLimit): 

304 """Select sources using a flux signal-to-noise limit 

305 

306 This object can be used as a `lsst.pex.config.Config` for configuring 

307 the limit, and then the `apply` method can be used to identify sources 

308 in the catalog that match the configured limit. 

309 """ 

310 fluxField = pexConfig.Field(dtype=str, default="flux", 

311 doc="Name of the source flux field to use.") 

312 errField = pexConfig.Field(dtype=str, default="flux_err", 

313 doc="Name of the source flux error field to use.") 

314 

315 def apply(self, catalog): 

316 """Apply the signal-to-noise limits to a catalog 

317 

318 Parameters 

319 ---------- 

320 catalog : `lsst.afw.table.SourceCatalog` 

321 Catalog of sources to which the limit will be applied. 

322 

323 Returns 

324 ------- 

325 selected : `numpy.ndarray` 

326 Boolean array indicating for each source whether it is selected 

327 (True means selected). 

328 """ 

329 flagField = self.fluxField + "_flag" 

330 if flagField in catalog.schema: 

331 selected = np.logical_not(catalog[flagField]) 

332 else: 

333 selected = np.ones(len(catalog), dtype=bool) 

334 

335 signalToNoise = catalog[self.fluxField]/catalog[self.errField] 

336 selected &= BaseLimit.apply(self, signalToNoise) 

337 return selected 

338 

339 

340class MagnitudeErrorLimit(BaseLimit): 

341 """Select sources using a magnitude error limit 

342 

343 Because the magnitude error is the inverse of the signal-to-noise 

344 ratio, this also works to select sources by signal-to-noise when 

345 you only have a magnitude. 

346 

347 This object can be used as a `lsst.pex.config.Config` for configuring 

348 the limit, and then the `apply` method can be used to identify sources 

349 in the catalog that match the configured limit. 

350 """ 

351 magErrField = pexConfig.Field(dtype=str, default="mag_err", 

352 doc="Name of the source flux error field to use.") 

353 

354 def apply(self, catalog): 

355 """Apply the magnitude error limits to a catalog 

356 

357 Parameters 

358 ---------- 

359 catalog : `lsst.afw.table.SourceCatalog` 

360 Catalog of sources to which the limit will be applied. 

361 

362 Returns 

363 ------- 

364 selected : `numpy.ndarray` 

365 Boolean array indicating for each source whether it is selected 

366 (True means selected). 

367 """ 

368 return BaseLimit.apply(self, catalog[self.magErrField]) 

369 

370 

371class RequireFlags(pexConfig.Config): 

372 """Select sources using flags 

373 

374 This object can be used as a `lsst.pex.config.Config` for configuring 

375 the limit, and then the `apply` method can be used to identify sources 

376 in the catalog that match the configured limit. 

377 """ 

378 good = pexConfig.ListField(dtype=str, default=[], 

379 doc="List of source flag fields that must be set for a source to be used.") 

380 bad = pexConfig.ListField(dtype=str, default=[], 

381 doc="List of source flag fields that must NOT be set for a source to be used.") 

382 

383 def apply(self, catalog): 

384 """Apply the flag requirements to a catalog 

385 

386 Returns whether the source is selected. 

387 

388 Parameters 

389 ---------- 

390 catalog : `lsst.afw.table.SourceCatalog` 

391 Catalog of sources to which the requirements will be applied. 

392 

393 Returns 

394 ------- 

395 selected : `numpy.ndarray` 

396 Boolean array indicating for each source whether it is selected 

397 (True means selected). 

398 """ 

399 selected = np.ones(len(catalog), dtype=bool) 

400 for flag in self.good: 

401 selected &= catalog[flag] 

402 for flag in self.bad: 

403 selected &= ~catalog[flag] 

404 return selected 

405 

406 

407class RequireUnresolved(BaseLimit): 

408 """Select sources using star/galaxy separation 

409 

410 This object can be used as a `lsst.pex.config.Config` for configuring 

411 the limit, and then the `apply` method can be used to identify sources 

412 in the catalog that match the configured limit. 

413 """ 

414 name = pexConfig.Field(dtype=str, default="base_ClassificationExtendedness_value", 

415 doc="Name of column for star/galaxy separation") 

416 

417 def setDefaults(self): 

418 """Set default 

419 

420 ``base_ClassificationExtendedness_value < 0.5`` means unresolved. 

421 """ 

422 self.maximum = 0.5 

423 

424 def apply(self, catalog): 

425 """Apply the flag requirements to a catalog 

426 

427 Returns whether the source is selected. 

428 

429 Parameters 

430 ---------- 

431 catalog : `lsst.afw.table.SourceCatalog` 

432 Catalog of sources to which the requirements will be applied. 

433 

434 Returns 

435 ------- 

436 selected : `numpy.ndarray` 

437 Boolean array indicating for each source whether it is selected 

438 (True means selected). 

439 """ 

440 value = catalog[self.name] 

441 return BaseLimit.apply(self, value) 

442 

443 

444class RequireIsolated(pexConfig.Config): 

445 """Select sources based on whether they are isolated 

446 

447 This object can be used as a `lsst.pex.config.Config` for configuring 

448 the column names to check for "parent" and "nChild" keys. 

449 

450 Note that this should only be run on a catalog that has had the 

451 deblender already run (or else deblend_nChild does not exist). 

452 """ 

453 parentName = pexConfig.Field(dtype=str, default="parent", 

454 doc="Name of column for parent") 

455 nChildName = pexConfig.Field(dtype=str, default="deblend_nChild", 

456 doc="Name of column for nChild") 

457 

458 def apply(self, catalog): 

459 """Apply the isolation requirements to a catalog 

460 

461 Returns whether the source is selected. 

462 

463 Parameters 

464 ---------- 

465 catalog : `lsst.afw.table.SourceCatalog` 

466 Catalog of sources to which the requirements will be applied. 

467 

468 Returns 

469 ------- 

470 selected : `numpy.ndarray` 

471 Boolean array indicating for each source whether it is selected 

472 (True means selected). 

473 """ 

474 selected = ((catalog[self.parentName] == 0) 

475 & (catalog[self.nChildName] == 0)) 

476 return selected 

477 

478 

479class ScienceSourceSelectorConfig(pexConfig.Config): 

480 """Configuration for selecting science sources""" 

481 doFluxLimit = pexConfig.Field(dtype=bool, default=False, doc="Apply flux limit?") 

482 doFlags = pexConfig.Field(dtype=bool, default=False, doc="Apply flag limitation?") 

483 doUnresolved = pexConfig.Field(dtype=bool, default=False, doc="Apply unresolved limitation?") 

484 doSignalToNoise = pexConfig.Field(dtype=bool, default=False, doc="Apply signal-to-noise limit?") 

485 doIsolated = pexConfig.Field(dtype=bool, default=False, doc="Apply isolated limitation?") 

486 fluxLimit = pexConfig.ConfigField(dtype=FluxLimit, doc="Flux limit to apply") 

487 flags = pexConfig.ConfigField(dtype=RequireFlags, doc="Flags to require") 

488 unresolved = pexConfig.ConfigField(dtype=RequireUnresolved, doc="Star/galaxy separation to apply") 

489 signalToNoise = pexConfig.ConfigField(dtype=SignalToNoiseLimit, doc="Signal-to-noise limit to apply") 

490 isolated = pexConfig.ConfigField(dtype=RequireIsolated, doc="Isolated criteria to apply") 

491 

492 def setDefaults(self): 

493 pexConfig.Config.setDefaults(self) 

494 self.flags.bad = ["base_PixelFlags_flag_edge", "base_PixelFlags_flag_saturated", "base_PsfFlux_flags"] 

495 self.signalToNoise.fluxField = "base_PsfFlux_instFlux" 

496 self.signalToNoise.errField = "base_PsfFlux_instFluxErr" 

497 

498 

499@pexConfig.registerConfigurable("science", sourceSelectorRegistry) 

500class ScienceSourceSelectorTask(BaseSourceSelectorTask): 

501 """Science source selector 

502 

503 By "science" sources, we mean sources that are on images that we 

504 are processing, as opposed to sources from reference catalogs. 

505 

506 This selects (science) sources by (optionally) applying each of a 

507 magnitude limit, flag requirements and star/galaxy separation. 

508 """ 

509 ConfigClass = ScienceSourceSelectorConfig 

510 

511 def selectSources(self, sourceCat, matches=None, exposure=None): 

512 """Return a selection of sources selected by specified criteria. 

513 

514 Parameters 

515 ---------- 

516 sourceCat : `lsst.afw.table.SourceCatalog` 

517 Catalog of sources to select from. 

518 This catalog must be contiguous in memory. 

519 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

520 Ignored in this SourceSelector. 

521 exposure : `lsst.afw.image.Exposure` or None 

522 The exposure the catalog was built from; used for debug display. 

523 

524 Return 

525 ------ 

526 struct : `lsst.pipe.base.Struct` 

527 The struct contains the following data: 

528 

529 - selected : `array` of `bool`` 

530 Boolean array of sources that were selected, same length as 

531 sourceCat. 

532 """ 

533 selected = np.ones(len(sourceCat), dtype=bool) 

534 if self.config.doFluxLimit: 

535 selected &= self.config.fluxLimit.apply(sourceCat) 

536 if self.config.doFlags: 

537 selected &= self.config.flags.apply(sourceCat) 

538 if self.config.doUnresolved: 

539 selected &= self.config.unresolved.apply(sourceCat) 

540 if self.config.doSignalToNoise: 

541 selected &= self.config.signalToNoise.apply(sourceCat) 

542 if self.config.doIsolated: 

543 selected &= self.config.isolated.apply(sourceCat) 

544 

545 self.log.info("Selected %d/%d sources", selected.sum(), len(sourceCat)) 

546 

547 return pipeBase.Struct(selected=selected) 

548 

549 

550class ReferenceSourceSelectorConfig(pexConfig.Config): 

551 doMagLimit = pexConfig.Field(dtype=bool, default=False, doc="Apply magnitude limit?") 

552 doFlags = pexConfig.Field(dtype=bool, default=False, doc="Apply flag limitation?") 

553 doUnresolved = pexConfig.Field(dtype=bool, default=False, doc="Apply unresolved limitation?") 

554 doSignalToNoise = pexConfig.Field(dtype=bool, default=False, doc="Apply signal-to-noise limit?") 

555 doMagError = pexConfig.Field(dtype=bool, default=False, doc="Apply magnitude error limit?") 

556 magLimit = pexConfig.ConfigField(dtype=MagnitudeLimit, doc="Magnitude limit to apply") 

557 flags = pexConfig.ConfigField(dtype=RequireFlags, doc="Flags to require") 

558 unresolved = pexConfig.ConfigField(dtype=RequireUnresolved, doc="Star/galaxy separation to apply") 

559 signalToNoise = pexConfig.ConfigField(dtype=SignalToNoiseLimit, doc="Signal-to-noise limit to apply") 

560 magError = pexConfig.ConfigField(dtype=MagnitudeErrorLimit, doc="Magnitude error limit to apply") 

561 colorLimits = pexConfig.ConfigDictField(keytype=str, itemtype=ColorLimit, default={}, 

562 doc="Color limits to apply; key is used as a label only") 

563 

564 

565@pexConfig.registerConfigurable("references", sourceSelectorRegistry) 

566class ReferenceSourceSelectorTask(BaseSourceSelectorTask): 

567 """Reference source selector 

568 

569 This selects reference sources by (optionally) applying each of a 

570 magnitude limit, flag requirements and color limits. 

571 """ 

572 ConfigClass = ReferenceSourceSelectorConfig 

573 

574 def selectSources(self, sourceCat, matches=None, exposure=None): 

575 """Return a selection of reference sources selected by some criteria. 

576 

577 Parameters 

578 ---------- 

579 sourceCat : `lsst.afw.table.SourceCatalog` 

580 Catalog of sources to select from. 

581 This catalog must be contiguous in memory. 

582 matches : `list` of `lsst.afw.table.ReferenceMatch` or None 

583 Ignored in this SourceSelector. 

584 exposure : `lsst.afw.image.Exposure` or None 

585 The exposure the catalog was built from; used for debug display. 

586 

587 Return 

588 ------ 

589 struct : `lsst.pipe.base.Struct` 

590 The struct contains the following data: 

591 

592 - selected : `array` of `bool`` 

593 Boolean array of sources that were selected, same length as 

594 sourceCat. 

595 """ 

596 selected = np.ones(len(sourceCat), dtype=bool) 

597 if self.config.doMagLimit: 

598 selected &= self.config.magLimit.apply(sourceCat) 

599 if self.config.doFlags: 

600 selected &= self.config.flags.apply(sourceCat) 

601 if self.config.doUnresolved: 

602 selected &= self.config.unresolved.apply(sourceCat) 

603 if self.config.doSignalToNoise: 

604 selected &= self.config.signalToNoise.apply(sourceCat) 

605 if self.config.doMagError: 

606 selected &= self.config.magError.apply(sourceCat) 

607 for limit in self.config.colorLimits.values(): 

608 selected &= limit.apply(sourceCat) 

609 

610 self.log.info("Selected %d/%d references", selected.sum(), len(sourceCat)) 

611 

612 return pipeBase.Struct(selected=selected)