lsst.meas.algorithms g644241e1a9+00df088e86
sourceSelector.py
Go to the documentation of this file.
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23
24__all__ = ["BaseSourceSelectorConfig", "BaseSourceSelectorTask", "sourceSelectorRegistry",
25 "ColorLimit", "MagnitudeLimit", "SignalToNoiseLimit", "MagnitudeErrorLimit",
26 "RequireFlags", "RequireUnresolved",
27 "ScienceSourceSelectorConfig", "ScienceSourceSelectorTask",
28 "ReferenceSourceSelectorConfig", "ReferenceSourceSelectorTask",
29 ]
30
31import abc
32import numpy as np
33import astropy.units as u
34import pandas
35import astropy.table
36
37import lsst.pex.config as pexConfig
38import lsst.pipe.base as pipeBase
39
40
41class BaseSourceSelectorConfig(pexConfig.Config):
42 pass
43
44
45class BaseSourceSelectorTask(pipeBase.Task, metaclass=abc.ABCMeta):
46 """Base class for source selectors
47
48 Source selectors are classes that perform a selection on a catalog
49 object given a set of criteria or cuts. They return the selected catalog
50 and can optionally set a specified Flag field in the input catalog to
51 identify if the source was selected.
52
53 Register all source selectors with the sourceSelectorRegistry using:
54 sourceSelectorRegistry.register(name, class)
55
56 Attributes
57 ----------
58 usesMatches : `bool`
59 A boolean variable specify if the inherited source selector uses
60 matches to an external catalog, and thus requires the ``matches``
61 argument to ``run()``.
62 """
63
64 ConfigClass = BaseSourceSelectorConfig
65 _DefaultName = "sourceSelector"
66 usesMatches = False
67
68 def __init__(self, **kwargs):
69 pipeBase.Task.__init__(self, **kwargs)
70
71 def run(self, sourceCat, sourceSelectedField=None, matches=None, exposure=None):
72 """Select sources and return them.
73
74 The input catalog must be contiguous in memory.
75
76 Parameters
77 ----------
78 sourceCat : Various table formats
79 Catalog of sources to select from. Can be
80 `lsst.afw.table.SourceCatalog` or `pandas.DataFrame` or
81 `astropy.table.Table`,
82 sourceSelectedField : `str` or None
83 Name of flag field in sourceCat to set for selected sources.
84 If set, will modify sourceCat in-place.
85 matches : `list` of `lsst.afw.table.ReferenceMatch` or None
86 List of matches to use for source selection.
87 If usesMatches is set in source selector this field is required.
88 If not, it is ignored.
89 exposure : `lsst.afw.image.Exposure` or None
90 The exposure the catalog was built from; used for debug display.
91
92 Returns
93 -------
94 struct : `lsst.pipe.base.Struct`
95 The struct contains the following data:
96
97 ``sourceCat``
98 The catalog of sources that were selected.
99 (may not be memory-contiguous)
100 (`lsst.afw.table.SourceCatalog` or `pandas.DataFrame`
101 or `astropy.table.Table`)
102 ``selected``
103 Boolean array of sources that were selected, same length as
104 sourceCat.
105 (`numpy.ndarray` of `bool`)
106
107 Raises
108 ------
109 RuntimeError
110 Raised if ``sourceCat`` is not contiguous.
111 """
112 if hasattr(sourceCat, 'isContiguous'):
113 # Check for continuity on afwTable catalogs
114 if not sourceCat.isContiguous():
115 raise RuntimeError("Input catalogs for source selection must be contiguous.")
116
117 result = self.selectSources(sourceCat=sourceCat,
118 exposure=exposure,
119 matches=matches)
120
121 if sourceSelectedField is not None:
122 if isinstance(sourceCat, (pandas.DataFrame, astropy.table.Table)):
123 sourceCat[sourceSelectedField] = result.selected
124 else:
125 source_selected_key = \
126 sourceCat.getSchema()[sourceSelectedField].asKey()
127 # TODO: Remove for loop when DM-6981 is completed.
128 for source, flag in zip(sourceCat, result.selected):
129 source.set(source_selected_key, bool(flag))
130 return pipeBase.Struct(sourceCat=sourceCat[result.selected],
131 selected=result.selected)
132
133 @abc.abstractmethod
134 def selectSources(self, sourceCat, matches=None, exposure=None):
135 """Return a selection of sources selected by some criteria.
136
137 Parameters
138 ----------
139 sourceCat : Various table formats
140 Catalog of sources to select from. Supports
141 `lsst.afw.table.SourceCatalog` or `pandas.DataFrame`
142 or `astropy.table.Table`
143 This catalog must be contiguous in memory.
144 matches : `list` of `lsst.afw.table.ReferenceMatch` or None
145 A list of lsst.afw.table.ReferenceMatch objects
146 exposure : `lsst.afw.image.Exposure` or None
147 The exposure the catalog was built from; used for debug display.
148
149 Returns
150 -------
151 struct : `lsst.pipe.base.Struct`
152 The struct contains the following data:
153
154 ``selected``
155 Boolean array of sources that were selected, same length as
156 sourceCat.
157 (`numpy.ndarray` of `bool`)
158 """
159 raise NotImplementedError("BaseSourceSelectorTask is abstract")
160
161
162sourceSelectorRegistry = pexConfig.makeRegistry(
163 doc="A registry of source selectors (subclasses of "
164 "BaseSourceSelectorTask)",
165)
166
167
168class BaseLimit(pexConfig.Config):
169 """Base class for selecting sources by applying a limit
170
171 This object can be used as a `lsst.pex.config.Config` for configuring
172 the limit, and then the `apply` method can be used to identify sources
173 in the catalog that match the configured limit.
174
175 This provides the `maximum` and `minimum` fields in the Config, and
176 a method to apply the limits to an array of values calculated by the
177 subclass.
178 """
179 minimum = pexConfig.Field(dtype=float, optional=True, doc="Select objects with value greater than this")
180 maximum = pexConfig.Field(dtype=float, optional=True, doc="Select objects with value less than this")
181
182 def apply(self, values):
183 """Apply the limits to an array of values
184
185 Subclasses should calculate the array of values and then
186 return the result of calling this method.
187
188 Parameters
189 ----------
190 values : `numpy.ndarray`
191 Array of values to which to apply limits.
192
193 Returns
194 -------
195 selected : `numpy.ndarray`
196 Boolean array indicating for each source whether it is selected
197 (True means selected).
198 """
199 selected = np.ones(len(values), dtype=bool)
200 with np.errstate(invalid="ignore"): # suppress NAN warnings
201 if self.minimum is not None:
202 selected &= values > self.minimum
203 if self.maximum is not None:
204 selected &= values < self.maximum
205 return selected
206
207
209 """Select sources using a color limit
210
211 This object can be used as a `lsst.pex.config.Config` for configuring
212 the limit, and then the `apply` method can be used to identify sources
213 in the catalog that match the configured limit.
214
215 We refer to 'primary' and 'secondary' flux measurements; these are the
216 two components of the color, which is:
217
218 instFluxToMag(cat[primary]) - instFluxToMag(cat[secondary])
219 """
220 primary = pexConfig.Field(dtype=str, doc="Name of column with primary flux measurement")
221 secondary = pexConfig.Field(dtype=str, doc="Name of column with secondary flux measurement")
222
223 def apply(self, catalog):
224 """Apply the color limit to a catalog
225
226 Parameters
227 ----------
228 catalog : Various table formats
229 Catalog of sources to which the limit will be applied.
230 Supports `lsst.afw.table.SourceCatalog` or `pandas.DataFrame`
231 or `astropy.table.Table`
232
233 Returns
234 -------
235 selected : `numpy.ndarray`
236 Boolean array indicating for each source whether it is selected
237 (True means selected).
238 """
239 primary = _getFieldFromCatalog(catalog, self.primary)
240 secondary = _getFieldFromCatalog(catalog, self.secondary)
241
242 primary = (primary*u.nJy).to_value(u.ABmag)
243 secondary = (secondary*u.nJy).to_value(u.ABmag)
244 color = primary - secondary
245 return BaseLimit.apply(self, color)
246
247
249 """Select sources using a flux limit
250
251 This object can be used as a `lsst.pex.config.Config` for configuring
252 the limit, and then the `apply` method can be used to identify sources
253 in the catalog that match the configured limit.
254 """
255 fluxField = pexConfig.Field(dtype=str, default="slot_CalibFlux_instFlux",
256 doc="Name of the source flux field to use.")
257
258 def apply(self, catalog):
259 """Apply the flux limits to a catalog
260
261 Parameters
262 ----------
264 Catalog of sources to which the limit will be applied.
265
266 Returns
267 -------
268 selected : `numpy.ndarray`
269 Boolean array indicating for each source whether it is selected
270 (True means selected).
271 """
272 flagField = self.fluxField + "_flag"
273 selected = np.logical_not(_getFieldFromCatalog(catalog, flagField, isFlag=True))
274 flux = _getFieldFromCatalog(catalog, self.fluxField)
275
276 selected &= BaseLimit.apply(self, flux)
277 return selected
278
279
281 """Select sources using a magnitude limit
282
283 Note that this assumes that a zero-point has already been applied and
284 the fluxes are in AB fluxes in Jansky. It is therefore principally
285 intended for reference catalogs rather than catalogs extracted from
286 science images.
287
288 This object can be used as a `lsst.pex.config.Config` for configuring
289 the limit, and then the `apply` method can be used to identify sources
290 in the catalog that match the configured limit.
291 """
292 fluxField = pexConfig.Field(dtype=str, default="flux",
293 doc="Name of the source flux field to use.")
294
295 def apply(self, catalog):
296 """Apply the magnitude limits to a catalog
297
298 Parameters
299 ----------
301 Catalog of sources to which the limit will be applied.
302
303 Returns
304 -------
305 selected : `numpy.ndarray`
306 Boolean array indicating for each source whether it is selected
307 (True means selected).
308 """
309 flagField = self.fluxField + "_flag"
310 selected = np.logical_not(_getFieldFromCatalog(catalog, flagField, isFlag=True))
311 flux = _getFieldFromCatalog(catalog, self.fluxField)
312
313 magnitude = (flux*u.nJy).to_value(u.ABmag)
314 selected &= BaseLimit.apply(self, magnitude)
315 return selected
316
317
319 """Select sources using a flux signal-to-noise limit
320
321 This object can be used as a `lsst.pex.config.Config` for configuring
322 the limit, and then the `apply` method can be used to identify sources
323 in the catalog that match the configured limit.
324 """
325 fluxField = pexConfig.Field(dtype=str, default="flux",
326 doc="Name of the source flux field to use.")
327 errField = pexConfig.Field(dtype=str, default="flux_err",
328 doc="Name of the source flux error field to use.")
329
330 def apply(self, catalog):
331 """Apply the signal-to-noise limits to a catalog
332
333 Parameters
334 ----------
336 Catalog of sources to which the limit will be applied.
337
338 Returns
339 -------
340 selected : `numpy.ndarray`
341 Boolean array indicating for each source whether it is selected
342 (True means selected).
343 """
344 flagField = self.fluxField + "_flag"
345 selected = np.logical_not(_getFieldFromCatalog(catalog, flagField, isFlag=True))
346 flux = _getFieldFromCatalog(catalog, self.fluxField)
347 err = _getFieldFromCatalog(catalog, self.errField)
348
349 signalToNoise = flux/err
350 selected &= BaseLimit.apply(self, signalToNoise)
351 return selected
352
353
355 """Select sources using a magnitude error limit
356
357 Because the magnitude error is the inverse of the signal-to-noise
358 ratio, this also works to select sources by signal-to-noise when
359 you only have a magnitude.
360
361 This object can be used as a `lsst.pex.config.Config` for configuring
362 the limit, and then the `apply` method can be used to identify sources
363 in the catalog that match the configured limit.
364 """
365 magErrField = pexConfig.Field(dtype=str, default="mag_err",
366 doc="Name of the source flux error field to use.")
367
368 def apply(self, catalog):
369 """Apply the magnitude error limits to a catalog
370
371 Parameters
372 ----------
374 Catalog of sources to which the limit will be applied.
375
376 Returns
377 -------
378 selected : `numpy.ndarray`
379 Boolean array indicating for each source whether it is selected
380 (True means selected).
381 """
382 return BaseLimit.apply(self, catalog[self.magErrField])
383
384
385class RequireFlags(pexConfig.Config):
386 """Select sources using flags
387
388 This object can be used as a `lsst.pex.config.Config` for configuring
389 the limit, and then the `apply` method can be used to identify sources
390 in the catalog that match the configured limit.
391 """
392 good = pexConfig.ListField(dtype=str, default=[],
393 doc="List of source flag fields that must be set for a source to be used.")
394 bad = pexConfig.ListField(dtype=str, default=[],
395 doc="List of source flag fields that must NOT be set for a source to be used.")
396
397 def apply(self, catalog):
398 """Apply the flag requirements to a catalog
399
400 Returns whether the source is selected.
401
402 Parameters
403 ----------
405 Catalog of sources to which the requirements will be applied.
406
407 Returns
408 -------
409 selected : `numpy.ndarray`
410 Boolean array indicating for each source whether it is selected
411 (True means selected).
412 """
413 selected = np.ones(len(catalog), dtype=bool)
414 for flag in self.good:
415 selected &= catalog[flag]
416 for flag in self.bad:
417 selected &= ~catalog[flag]
418 return selected
419
420
422 """Select sources using star/galaxy separation
423
424 This object can be used as a `lsst.pex.config.Config` for configuring
425 the limit, and then the `apply` method can be used to identify sources
426 in the catalog that match the configured limit.
427 """
428 name = pexConfig.Field(dtype=str, default="base_ClassificationExtendedness_value",
429 doc="Name of column for star/galaxy separation")
430
431 def setDefaults(self):
432 """Set default
433
434 ``base_ClassificationExtendedness_value < 0.5`` means unresolved.
435 """
436 self.maximummaximum = 0.5
437
438 def apply(self, catalog):
439 """Apply the flag requirements to a catalog
440
441 Returns whether the source is selected.
442
443 Parameters
444 ----------
446 Catalog of sources to which the requirements will be applied.
447
448 Returns
449 -------
450 selected : `numpy.ndarray`
451 Boolean array indicating for each source whether it is selected
452 (True means selected).
453 """
454 value = catalog[self.name]
455 return BaseLimit.apply(self, value)
456
457
458class RequireIsolated(pexConfig.Config):
459 """Select sources based on whether they are isolated
460
461 This object can be used as a `lsst.pex.config.Config` for configuring
462 the column names to check for "parent" and "nChild" keys.
463
464 Note that this should only be run on a catalog that has had the
465 deblender already run (or else deblend_nChild does not exist).
466 """
467 parentName = pexConfig.Field(dtype=str, default="parent",
468 doc="Name of column for parent")
469 nChildName = pexConfig.Field(dtype=str, default="deblend_nChild",
470 doc="Name of column for nChild")
471
472 def apply(self, catalog):
473 """Apply the isolation requirements to a catalog
474
475 Returns whether the source is selected.
476
477 Parameters
478 ----------
480 Catalog of sources to which the requirements will be applied.
481
482 Returns
483 -------
484 selected : `numpy.ndarray`
485 Boolean array indicating for each source whether it is selected
486 (True means selected).
487 """
488 selected = ((catalog[self.parentName] == 0)
489 & (catalog[self.nChildName] == 0))
490 return selected
491
492
493class ScienceSourceSelectorConfig(pexConfig.Config):
494 """Configuration for selecting science sources"""
495 doFluxLimit = pexConfig.Field(dtype=bool, default=False, doc="Apply flux limit?")
496 doFlags = pexConfig.Field(dtype=bool, default=False, doc="Apply flag limitation?")
497 doUnresolved = pexConfig.Field(dtype=bool, default=False, doc="Apply unresolved limitation?")
498 doSignalToNoise = pexConfig.Field(dtype=bool, default=False, doc="Apply signal-to-noise limit?")
499 doIsolated = pexConfig.Field(dtype=bool, default=False, doc="Apply isolated limitation?")
500 fluxLimit = pexConfig.ConfigField(dtype=FluxLimit, doc="Flux limit to apply")
501 flags = pexConfig.ConfigField(dtype=RequireFlags, doc="Flags to require")
502 unresolved = pexConfig.ConfigField(dtype=RequireUnresolved, doc="Star/galaxy separation to apply")
503 signalToNoise = pexConfig.ConfigField(dtype=SignalToNoiseLimit, doc="Signal-to-noise limit to apply")
504 isolated = pexConfig.ConfigField(dtype=RequireIsolated, doc="Isolated criteria to apply")
505
506 def setDefaults(self):
507 pexConfig.Config.setDefaults(self)
508 self.flags.bad = ["base_PixelFlags_flag_edge", "base_PixelFlags_flag_saturated", "base_PsfFlux_flags"]
509 self.signalToNoise.fluxField = "base_PsfFlux_instFlux"
510 self.signalToNoise.errField = "base_PsfFlux_instFluxErr"
511
512
513@pexConfig.registerConfigurable("science", sourceSelectorRegistry)
515 """Science source selector
516
517 By "science" sources, we mean sources that are on images that we
518 are processing, as opposed to sources from reference catalogs.
519
520 This selects (science) sources by (optionally) applying each of a
521 magnitude limit, flag requirements and star/galaxy separation.
522 """
523 ConfigClass = ScienceSourceSelectorConfig
524
525 def selectSources(self, sourceCat, matches=None, exposure=None):
526 """Return a selection of sources selected by specified criteria.
527
528 Parameters
529 ----------
530 sourceCat : `lsst.afw.table.SourceCatalog`
531 Catalog of sources to select from.
532 This catalog must be contiguous in memory.
533 matches : `list` of `lsst.afw.table.ReferenceMatch` or None
534 Ignored in this SourceSelector.
535 exposure : `lsst.afw.image.Exposure` or None
536 The exposure the catalog was built from; used for debug display.
537
538 Returns
539 -------
540 struct : `lsst.pipe.base.Struct`
541 The struct contains the following data:
542
543 ``selected``
544 Boolean array of sources that were selected, same length as
545 sourceCat.
546 (`numpy.ndarray` of `bool`)
547 """
548 selected = np.ones(len(sourceCat), dtype=bool)
549 if self.config.doFluxLimit:
550 selected &= self.config.fluxLimit.apply(sourceCat)
551 if self.config.doFlags:
552 selected &= self.config.flags.apply(sourceCat)
553 if self.config.doUnresolved:
554 selected &= self.config.unresolved.apply(sourceCat)
555 if self.config.doSignalToNoise:
556 selected &= self.config.signalToNoise.apply(sourceCat)
557 if self.config.doIsolated:
558 selected &= self.config.isolated.apply(sourceCat)
559
560 self.log.info("Selected %d/%d sources", selected.sum(), len(sourceCat))
561
562 return pipeBase.Struct(selected=selected)
563
564
565class ReferenceSourceSelectorConfig(pexConfig.Config):
566 doMagLimit = pexConfig.Field(dtype=bool, default=False, doc="Apply magnitude limit?")
567 doFlags = pexConfig.Field(dtype=bool, default=False, doc="Apply flag limitation?")
568 doUnresolved = pexConfig.Field(dtype=bool, default=False, doc="Apply unresolved limitation?")
569 doSignalToNoise = pexConfig.Field(dtype=bool, default=False, doc="Apply signal-to-noise limit?")
570 doMagError = pexConfig.Field(dtype=bool, default=False, doc="Apply magnitude error limit?")
571 magLimit = pexConfig.ConfigField(dtype=MagnitudeLimit, doc="Magnitude limit to apply")
572 flags = pexConfig.ConfigField(dtype=RequireFlags, doc="Flags to require")
573 unresolved = pexConfig.ConfigField(dtype=RequireUnresolved, doc="Star/galaxy separation to apply")
574 signalToNoise = pexConfig.ConfigField(dtype=SignalToNoiseLimit, doc="Signal-to-noise limit to apply")
575 magError = pexConfig.ConfigField(dtype=MagnitudeErrorLimit, doc="Magnitude error limit to apply")
576 colorLimits = pexConfig.ConfigDictField(keytype=str, itemtype=ColorLimit, default={},
577 doc="Color limits to apply; key is used as a label only")
578
579
580@pexConfig.registerConfigurable("references", sourceSelectorRegistry)
582 """Reference source selector
583
584 This selects reference sources by (optionally) applying each of a
585 magnitude limit, flag requirements and color limits.
586 """
587 ConfigClass = ReferenceSourceSelectorConfig
588
589 def selectSources(self, sourceCat, matches=None, exposure=None):
590 """Return a selection of reference sources selected by some criteria.
591
592 Parameters
593 ----------
594 sourceCat : `lsst.afw.table.SourceCatalog`
595 Catalog of sources to select from.
596 This catalog must be contiguous in memory.
597 matches : `list` of `lsst.afw.table.ReferenceMatch` or None
598 Ignored in this SourceSelector.
599 exposure : `lsst.afw.image.Exposure` or None
600 The exposure the catalog was built from; used for debug display.
601
602 Returns
603 -------
604 struct : `lsst.pipe.base.Struct`
605 The struct contains the following data:
606
607 ``selected``
608 Boolean array of sources that were selected, same length as
609 sourceCat.
610 (`numpy.ndarray` of `bool`)
611 """
612 selected = np.ones(len(sourceCat), dtype=bool)
613 if self.config.doMagLimit:
614 selected &= self.config.magLimit.apply(sourceCat)
615 if self.config.doFlags:
616 selected &= self.config.flags.apply(sourceCat)
617 if self.config.doUnresolved:
618 selected &= self.config.unresolved.apply(sourceCat)
619 if self.config.doSignalToNoise:
620 selected &= self.config.signalToNoise.apply(sourceCat)
621 if self.config.doMagError:
622 selected &= self.config.magError.apply(sourceCat)
623 for limit in self.config.colorLimits.values():
624 selected &= limit.apply(sourceCat)
625
626 self.log.info("Selected %d/%d references", selected.sum(), len(sourceCat))
627
628 return pipeBase.Struct(selected=selected)
629
630
631def _getFieldFromCatalog(catalog, field, isFlag=False):
632 """
633 Get a field from a catalog, for `lsst.afw.table` catalogs or
634 `pandas.DataFrame` or `astropy.table.Table` catalogs.
635
636 Parameters
637 ----------
638 catalog : `lsst.afw.table.SourceCatalog` or `pandas.DataFrame`
639 or `astropy.table.Table`
640 Catalog of sources to extract field array
641 field : `str`
642 Name of field
643 isFlag : `bool`, optional
644 Is this a flag column? If it does not exist, return array
645 of False.
646
647 Returns
648 -------
649 array : `np.ndarray`
650 Array of field values from the catalog.
651 """
652 found = False
653 if isinstance(catalog, (pandas.DataFrame, astropy.table.Table)):
654 if field in catalog.columns:
655 found = True
656 # Sequences must be converted to numpy arrays
657 arr = np.array(catalog[field])
658 else:
659 if field in catalog.schema:
660 found = True
661 arr = catalog[field]
662
663 if isFlag and not found:
664 arr = np.zeros(len(catalog), dtype=bool)
665 elif not found:
666 raise KeyError(f"Could not find field {field} in catalog.")
667
668 return arr
def selectSources(self, sourceCat, matches=None, exposure=None)
def run(self, sourceCat, sourceSelectedField=None, matches=None, exposure=None)
def selectSources(self, sourceCat, matches=None, exposure=None)
def selectSources(self, sourceCat, matches=None, exposure=None)