Coverage for python/lsst/faro/utils/matcher.py : 5%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of faro.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from lsst.afw.table import (
23 SchemaMapper,
24 Field,
25 MultiMatch,
26 SimpleRecord,
27 SourceCatalog,
28 updateSourceCoords,
29)
31import numpy as np
32from astropy.table import join, Table
34__all__ = (
35 "matchCatalogs",
36 "ellipticityFromCat",
37 "ellipticity",
38 "makeMatchedPhotom",
39 "mergeCatalogs",
40)
43def matchCatalogs(inputs, photoCalibs, astromCalibs, dataIds, matchRadius, logger=None):
44 schema = inputs[0].schema
45 mapper = SchemaMapper(schema)
46 mapper.addMinimalSchema(schema)
47 mapper.addOutputField(Field[float]("base_PsfFlux_snr", "PSF flux SNR"))
48 mapper.addOutputField(Field[float]("base_PsfFlux_mag", "PSF magnitude"))
49 mapper.addOutputField(
50 Field[float]("base_PsfFlux_magErr", "PSF magnitude uncertainty")
51 )
52 # Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name
53 aliasMap = schema.getAliasMap()
54 # Possibly not needed since base_GaussianFlux is the default, but this ought to be safe
55 modelName = (
56 aliasMap["slot_ModelFlux"]
57 if "slot_ModelFlux" in aliasMap.keys()
58 else "base_GaussianFlux"
59 )
60 mapper.addOutputField(Field[float](f"{modelName}_mag", "Model magnitude"))
61 mapper.addOutputField(
62 Field[float](f"{modelName}_magErr", "Model magnitude uncertainty")
63 )
64 mapper.addOutputField(Field[float](f"{modelName}_snr", "Model flux snr"))
65 mapper.addOutputField(Field[float]("e1", "Source Ellipticity 1"))
66 mapper.addOutputField(Field[float]("e2", "Source Ellipticity 1"))
67 mapper.addOutputField(Field[float]("psf_e1", "PSF Ellipticity 1"))
68 mapper.addOutputField(Field[float]("psf_e2", "PSF Ellipticity 1"))
69 mapper.addOutputField(Field[np.int32]("filt", "filter code"))
70 newSchema = mapper.getOutputSchema()
71 newSchema.setAliasMap(schema.getAliasMap())
73 # Create an object that matches multiple catalogs with same schema
74 mmatch = MultiMatch(
75 newSchema,
76 dataIdFormat={"visit": np.int32, "detector": np.int32},
77 radius=matchRadius,
78 RecordClass=SimpleRecord,
79 )
81 # create the new extended source catalog
82 srcVis = SourceCatalog(newSchema)
84 filter_dict = {
85 "u": 1,
86 "g": 2,
87 "r": 3,
88 "i": 4,
89 "z": 5,
90 "y": 6,
91 "HSC-U": 1,
92 "HSC-G": 2,
93 "HSC-R": 3,
94 "HSC-I": 4,
95 "HSC-Z": 5,
96 "HSC-Y": 6,
97 }
99 # Sort by visit, detector, then filter
100 vislist = [v["visit"] for v in dataIds]
101 ccdlist = [v["detector"] for v in dataIds]
102 filtlist = [v["band"] for v in dataIds]
103 tab_vids = Table([vislist, ccdlist, filtlist], names=["vis", "ccd", "filt"])
104 sortinds = np.argsort(tab_vids, order=("vis", "ccd", "filt"))
106 for ind in sortinds:
107 oldSrc = inputs[ind]
108 photoCalib = photoCalibs[ind]
109 wcs = astromCalibs[ind]
110 dataId = dataIds[ind]
112 if logger:
113 logger.debug(
114 "%d sources in ccd %s visit %s",
115 len(oldSrc),
116 dataId["detector"],
117 dataId["visit"],
118 )
120 # create temporary catalog
121 tmpCat = SourceCatalog(SourceCatalog(newSchema).table)
122 tmpCat.extend(oldSrc, mapper=mapper)
124 filtnum = filter_dict[dataId["band"]]
125 tmpCat["filt"] = np.repeat(filtnum, len(oldSrc))
127 tmpCat["base_PsfFlux_snr"][:] = (
128 tmpCat["base_PsfFlux_instFlux"] / tmpCat["base_PsfFlux_instFluxErr"]
129 )
131 updateSourceCoords(wcs, tmpCat)
133 photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
134 tmpCat["slot_ModelFlux_snr"][:] = (
135 tmpCat["slot_ModelFlux_instFlux"] / tmpCat["slot_ModelFlux_instFluxErr"]
136 )
137 photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux", "slot_ModelFlux")
139 _, psf_e1, psf_e2 = ellipticityFromCat(oldSrc, slot_shape="slot_PsfShape")
140 _, star_e1, star_e2 = ellipticityFromCat(oldSrc, slot_shape="slot_Shape")
141 tmpCat["e1"][:] = star_e1
142 tmpCat["e2"][:] = star_e2
143 tmpCat["psf_e1"][:] = psf_e1
144 tmpCat["psf_e2"][:] = psf_e2
146 srcVis.extend(tmpCat, False)
147 mmatch.add(catalog=tmpCat, dataId=dataId)
149 # Complete the match, returning a catalog that includes
150 # all matched sources with object IDs that can be used to group them.
151 matchCat = mmatch.finish()
153 # Create a mapping object that allows the matches to be manipulated
154 # as a mapping of object ID to catalog of sources.
156 # I don't think I can persist a group view, so this may need to be called in a subsequent task
157 # allMatches = GroupView.build(matchCat)
159 return srcVis, matchCat
162def ellipticityFromCat(cat, slot_shape="slot_Shape"):
163 """Calculate the ellipticity of the Shapes in a catalog from the 2nd moments.
164 Parameters
165 ----------
166 cat : `lsst.afw.table.BaseCatalog`
167 A catalog with 'slot_Shape' defined and '_xx', '_xy', '_yy'
168 entries for the target of 'slot_Shape'.
169 E.g., 'slot_shape' defined as 'base_SdssShape'
170 And 'base_SdssShape_xx', 'base_SdssShape_xy', 'base_SdssShape_yy' defined.
171 slot_shape : str, optional
172 Specify what slot shape requested. Intended use is to get the PSF shape
173 estimates by specifying 'slot_shape=slot_PsfShape'
174 instead of the default 'slot_shape=slot_Shape'.
175 Returns
176 -------
177 e, e1, e2 : complex, float, float
178 Complex ellipticity, real part, imaginary part
179 """
180 i_xx, i_xy, i_yy = (
181 cat.get(slot_shape + "_xx"),
182 cat.get(slot_shape + "_xy"),
183 cat.get(slot_shape + "_yy"),
184 )
185 return ellipticity(i_xx, i_xy, i_yy)
188def ellipticity(i_xx, i_xy, i_yy):
189 """Calculate ellipticity from second moments.
190 Parameters
191 ----------
192 i_xx : float or `numpy.array`
193 i_xy : float or `numpy.array`
194 i_yy : float or `numpy.array`
195 Returns
196 -------
197 e, e1, e2 : (float, float, float) or (numpy.array, numpy.array, numpy.array)
198 Complex ellipticity, real component, imaginary component
199 """
200 e = (i_xx - i_yy + 2j * i_xy) / (i_xx + i_yy)
201 e1 = np.real(e)
202 e2 = np.imag(e)
203 return e, e1, e2
206def makeMatchedPhotom(dataIds, catalogs, photoCalibs):
207 # inputs: dataIds, catalogs, photoCalibs
209 # Match all input bands:
210 bands = list(set([f["band"] for f in dataIds]))
212 # Should probably add an "assert" that requires bands>1...
214 empty_cat = catalogs[0].copy()
215 empty_cat.clear()
217 cat_dict = {}
218 mags_dict = {}
219 magerrs_dict = {}
220 for band in bands:
221 cat_dict[band] = empty_cat.copy()
222 mags_dict[band] = []
223 magerrs_dict[band] = []
225 for i in range(len(catalogs)):
226 for band in bands:
227 if dataIds[i]["band"] in band:
228 cat_dict[band].extend(catalogs[i].copy(deep=True))
229 mags = photoCalibs[i].instFluxToMagnitude(catalogs[i], "base_PsfFlux")
230 mags_dict[band] = np.append(mags_dict[band], mags[:, 0])
231 magerrs_dict[band] = np.append(magerrs_dict[band], mags[:, 1])
233 for band in bands:
234 cat_tmp = cat_dict[band]
235 if cat_tmp:
236 if not cat_tmp.isContiguous():
237 cat_tmp = cat_tmp.copy(deep=True)
238 cat_tmp_final = cat_tmp.asAstropy()
239 cat_tmp_final["base_PsfFlux_mag"] = mags_dict[band]
240 cat_tmp_final["base_PsfFlux_magErr"] = magerrs_dict[band]
241 # Put the bandpass name in the column names:
242 for c in cat_tmp_final.colnames:
243 if c not in "id":
244 cat_tmp_final[c].name = c + "_" + str(band)
245 # Write the new catalog to the dict of catalogs:
246 cat_dict[band] = cat_tmp_final
248 cat_combined = join(cat_dict[bands[1]], cat_dict[bands[0]], keys="id")
249 if len(bands) > 2:
250 for i in range(2, len(bands)):
251 cat_combined = join(cat_combined, cat_dict[bands[i]], keys="id")
253 qual_cuts = (
254 (cat_combined["base_ClassificationExtendedness_value_g"] < 0.5)
255 & (cat_combined["base_PixelFlags_flag_saturated_g"] is False)
256 & (cat_combined["base_PixelFlags_flag_cr_g"] is False)
257 & (cat_combined["base_PixelFlags_flag_bad_g"] is False)
258 & (cat_combined["base_PixelFlags_flag_edge_g"] is False)
259 & (cat_combined["base_ClassificationExtendedness_value_r"] < 0.5)
260 & (cat_combined["base_PixelFlags_flag_saturated_r"] is False)
261 & (cat_combined["base_PixelFlags_flag_cr_r"] is False)
262 & (cat_combined["base_PixelFlags_flag_bad_r"] is False)
263 & (cat_combined["base_PixelFlags_flag_edge_r"] is False)
264 & (cat_combined["base_ClassificationExtendedness_value_i"] < 0.5)
265 & (cat_combined["base_PixelFlags_flag_saturated_i"] is False)
266 & (cat_combined["base_PixelFlags_flag_cr_i"] is False)
267 & (cat_combined["base_PixelFlags_flag_bad_i"] is False)
268 & (cat_combined["base_PixelFlags_flag_edge_i"] is False)
269 ) # noqa: E712
271 # Return the astropy table of matched catalogs:
272 return cat_combined[qual_cuts]
275def mergeCatalogs(
276 catalogs,
277 photoCalibs=None,
278 astromCalibs=None,
279 models=["slot_PsfFlux"],
280 applyExternalWcs=False,
281):
282 """Merge catalogs and optionally apply photometric and astrometric calibrations.
283 """
285 schema = catalogs[0].schema
286 mapper = SchemaMapper(schema)
287 mapper.addMinimalSchema(schema)
288 aliasMap = schema.getAliasMap()
289 for model in models:
290 modelName = aliasMap[model] if model in aliasMap.keys() else model
291 mapper.addOutputField(
292 Field[float](f"{modelName}_mag", f"{modelName} magnitude")
293 )
294 mapper.addOutputField(
295 Field[float](f"{modelName}_magErr", f"{modelName} magnitude uncertainty")
296 )
297 newSchema = mapper.getOutputSchema()
298 newSchema.setAliasMap(schema.getAliasMap())
300 size = sum([len(cat) for cat in catalogs])
301 catalog = SourceCatalog(newSchema)
302 catalog.reserve(size)
304 for ii in range(0, len(catalogs)):
305 cat = catalogs[ii]
307 # Create temporary catalog. Is this step needed?
308 tempCat = SourceCatalog(SourceCatalog(newSchema).table)
309 tempCat.extend(cat, mapper=mapper)
311 if applyExternalWcs and astromCalibs is not None:
312 wcs = astromCalibs[ii]
313 updateSourceCoords(wcs, tempCat)
315 if photoCalibs is not None:
316 photoCalib = photoCalibs[ii]
317 for model in models:
318 modelName = aliasMap[model] if model in aliasMap.keys() else model
319 photoCalib.instFluxToMagnitude(tempCat, modelName, modelName)
321 catalog.extend(tempCat)
323 return catalog