Coverage for python/lsst/ap/association/filterDiaSourceCatalog.py: 37%
59 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-09 11:35 +0000
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-09 11:35 +0000
1# This file is part of ap_association
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = (
23 "FilterDiaSourceCatalogConfig",
24 "FilterDiaSourceCatalogTask",
25)
27import numpy as np
29from lsst.afw.table import SourceCatalog
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32import lsst.pipe.base.connectionTypes as connTypes
33from lsst.utils.timer import timeMethod
36class FilterDiaSourceCatalogConnections(
37 pipeBase.PipelineTaskConnections,
38 dimensions=("instrument", "visit", "detector"),
39 defaultTemplates={"coaddName": "deep", "fakesType": ""},
40):
41 """Connections class for FilterDiaSourceCatalogTask."""
43 diaSourceCat = connTypes.Input(
44 doc="Catalog of DiaSources produced during image differencing.",
45 name="{fakesType}{coaddName}Diff_diaSrc",
46 storageClass="SourceCatalog",
47 dimensions=("instrument", "visit", "detector"),
48 )
50 filteredDiaSourceCat = connTypes.Output(
51 doc="Output catalog of DiaSources after filtering.",
52 name="{fakesType}{coaddName}Diff_candidateDiaSrc",
53 storageClass="SourceCatalog",
54 dimensions=("instrument", "visit", "detector"),
55 )
57 rejectedDiaSources = connTypes.Output(
58 doc="Optional output storing all the rejected DiaSources.",
59 name="{fakesType}{coaddName}Diff_rejectedDiaSrc",
60 storageClass="SourceCatalog",
61 dimensions={"instrument", "visit", "detector"},
62 )
64 diffImVisitInfo = connTypes.Input(
65 doc="VisitInfo of diffIm.",
66 name="{fakesType}{coaddName}Diff_differenceExp.visitInfo",
67 storageClass="VisitInfo",
68 dimensions=("instrument", "visit", "detector"),
69 )
71 longTrailedSources = connTypes.Output(
72 doc="Optional output temporarily storing long trailed diaSources.",
73 dimensions=("instrument", "visit", "detector"),
74 storageClass="ArrowAstropy",
75 name="{fakesType}{coaddName}Diff_longTrailedSrc",
76 )
78 def __init__(self, *, config=None):
79 super().__init__(config=config)
80 if not self.config.doWriteRejectedSkySources:
81 self.outputs.remove("rejectedDiaSources")
82 if not self.config.doTrailedSourceFilter:
83 self.outputs.remove("longTrailedSources")
84 if not self.config.doWriteTrailedSources:
85 self.outputs.remove("longTrailedSources")
88class FilterDiaSourceCatalogConfig(
89 pipeBase.PipelineTaskConfig, pipelineConnections=FilterDiaSourceCatalogConnections
90):
91 """Config class for FilterDiaSourceCatalogTask."""
93 doRemoveSkySources = pexConfig.Field(
94 dtype=bool,
95 default=False,
96 doc="Input DiaSource catalog contains SkySources that should be "
97 "removed before storing the output DiaSource catalog.",
98 )
100 doWriteRejectedSkySources = pexConfig.Field(
101 dtype=bool,
102 default=True,
103 doc="Store the output DiaSource catalog containing all the rejected "
104 "sky sources."
105 )
107 doTrailedSourceFilter = pexConfig.Field(
108 doc="Run trailedSourceFilter to remove long trailed sources from the"
109 "diaSource output catalog.",
110 dtype=bool,
111 default=True,
112 )
114 doWriteTrailedSources = pexConfig.Field(
115 doc="Write trailed diaSources sources to a table.",
116 dtype=bool,
117 default=True,
118 deprecated="Trailed sources will not be written out during production."
119 )
121 max_trail_length = pexConfig.Field(
122 dtype=float,
123 doc="Length of long trailed sources to remove from the input catalog, "
124 "in arcseconds per second. Default comes from DMTN-199, which "
125 "requires removal of sources with trails longer than 10 "
126 "degrees/day, which is 36000/3600/24 arcsec/second, or roughly"
127 "0.416 arcseconds per second.",
128 default=36000/3600.0/24.0,
129 )
132class FilterDiaSourceCatalogTask(pipeBase.PipelineTask):
133 """Filter out sky sources from a DiaSource catalog."""
135 ConfigClass = FilterDiaSourceCatalogConfig
136 _DefaultName = "filterDiaSourceCatalog"
138 @timeMethod
139 def run(self, diaSourceCat, diffImVisitInfo):
140 """Filter sky sources from the supplied DiaSource catalog.
142 Parameters
143 ----------
144 diaSourceCat : `lsst.afw.table.SourceCatalog`
145 Catalog of sources measured on the difference image.
146 diffImVisitInfo: `lsst.afw.image.VisitInfo`
147 VisitInfo for the difference image corresponding to diaSourceCat.
149 Returns
150 -------
151 filterResults : `lsst.pipe.base.Struct`
153 ``filteredDiaSourceCat`` : `lsst.afw.table.SourceCatalog`
154 The catalog of filtered sources.
155 ``rejectedDiaSources`` : `lsst.afw.table.SourceCatalog`
156 The catalog of rejected sky sources.
157 ``longTrailedDiaSources`` : `astropy.table.Table`
158 DiaSources which have trail lengths greater than
159 max_trail_length*exposure_time.
160 """
161 rejectedSkySources = None
162 exposure_time = diffImVisitInfo.exposureTime
163 if self.config.doRemoveSkySources:
164 sky_source_column = diaSourceCat["sky_source"]
165 num_sky_sources = np.sum(sky_source_column)
166 rejectedSkySources = diaSourceCat[sky_source_column].copy(deep=True)
167 diaSourceCat = diaSourceCat[~sky_source_column].copy(deep=True)
168 self.log.info(f"Filtered {num_sky_sources} sky sources.")
169 if not rejectedSkySources:
170 rejectedSkySources = SourceCatalog(diaSourceCat.getSchema())
172 if self.config.doTrailedSourceFilter:
173 trail_mask = self._check_dia_source_trail(diaSourceCat, exposure_time)
174 longTrailedDiaSources = diaSourceCat[trail_mask].copy(deep=True)
175 diaSourceCat = diaSourceCat[~trail_mask]
177 self.log.info("%i DiaSources exceed max_trail_length %f arcseconds per second, "
178 "dropping from source catalog."
179 % (self.config.max_trail_length, len(diaSourceCat)))
180 self.metadata.add("num_filtered", len(longTrailedDiaSources))
182 if self.config.doWriteTrailedSources:
183 filterResults = pipeBase.Struct(filteredDiaSourceCat=diaSourceCat,
184 rejectedDiaSources=rejectedSkySources,
185 longTrailedSources=longTrailedDiaSources.asAstropy())
186 else:
187 filterResults = pipeBase.Struct(filteredDiaSourceCat=diaSourceCat,
188 rejectedDiaSources=rejectedSkySources)
190 else:
191 filterResults = pipeBase.Struct(filteredDiaSourceCat=diaSourceCat,
192 rejectedDiaSources=rejectedSkySources)
194 return filterResults
196 def _check_dia_source_trail(self, dia_sources, exposure_time):
197 """Find DiaSources that have long trails or trails with indeterminant
198 end points.
200 Return a mask of sources with lengths greater than
201 (``config.max_trail_length`` multiplied by the exposure time)
202 arcseconds.
203 Additionally, set mask if
204 ``ext_trailedSources_Naive_flag_off_image`` is set or if
205 ``ext_trailedSources_Naive_flag_suspect_long_trail`` and
206 ``ext_trailedSources_Naive_flag_edge`` are both set.
208 Parameters
209 ----------
210 dia_sources : `pandas.DataFrame`
211 Input diaSources to check for trail lengths.
212 exposure_time : `float`
213 Exposure time from difference image.
215 Returns
216 -------
217 trail_mask : `pandas.DataFrame`
218 Boolean mask for diaSources which are greater than the
219 Boolean mask for diaSources which are greater than the
220 cutoff length or have trails which extend beyond the edge of the
221 detector (off_image set). Also checks if both
222 suspect_long_trail and edge are set and masks those sources out.
223 """
224 print(dia_sources.getSchema())
225 trail_mask = (dia_sources["ext_trailedSources_Naive_length"]
226 >= (self.config.max_trail_length*exposure_time))
227 trail_mask |= dia_sources['ext_trailedSources_Naive_flag_off_image']
228 trail_mask |= (dia_sources['ext_trailedSources_Naive_flag_suspect_long_trail']
229 & dia_sources['ext_trailedSources_Naive_flag_edge'])
231 return trail_mask