Coverage for tests/test_functors.py : 18%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import astropy.units as u
23import copy
24import functools
25import numpy as np
26import os
27import pandas as pd
28import unittest
30import lsst.utils.tests
32# TODO: Remove skipUnless and this try block DM-22256
33try:
34 from lsst.pipe.tasks.parquetTable import MultilevelParquetTable
35 from lsst.pipe.tasks.functors import (CompositeFunctor, CustomFunctor, Column, RAColumn,
36 DecColumn, Mag, MagDiff, Color, StarGalaxyLabeller,
37 DeconvolvedMoments, SdssTraceSize, PsfSdssTraceSizeDiff,
38 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm,
39 LocalPhotometry, LocalNanojansky, LocalNanojanskyErr,
40 LocalMagnitude, LocalMagnitudeErr)
41 havePyArrow = True
42except ImportError:
43 havePyArrow = False
45ROOT = os.path.abspath(os.path.dirname(__file__))
48@unittest.skipUnless(havePyArrow, "Requires pyarrow")
49class FunctorTestCase(unittest.TestCase):
51 def simulateMultiParquet(self, dataDict):
52 """Create a simple test MultilevelParquetTable
53 """
54 simpleDF = pd.DataFrame(dataDict)
55 dfFilterDSCombos = []
56 for ds in self.datasets:
57 for filterName in self.filters:
58 df = copy.copy(simpleDF)
59 df.reindex(sorted(df.columns), axis=1)
60 df['dataset'] = ds
61 df['filter'] = filterName
62 df.columns = pd.MultiIndex.from_tuples(
63 [(ds, filterName, c) for c in df.columns],
64 names=('dataset', 'filter', 'column'))
65 dfFilterDSCombos.append(df)
67 df = functools.reduce(lambda d1, d2: d1.join(d2), dfFilterDSCombos)
69 return MultilevelParquetTable(dataFrame=df)
71 def setUp(self):
72 np.random.seed(1234)
73 self.datasets = ['forced_src', 'meas', 'ref']
74 self.filters = ['HSC-G', 'HSC-R']
75 self.columns = ['coord_ra', 'coord_dec']
76 self.nRecords = 5
77 self.dataDict = {
78 "coord_ra": [3.77654137, 3.77643059, 3.77621148, 3.77611944, 3.77610396],
79 "coord_dec": [0.01127624, 0.01127787, 0.01127543, 0.01127543, 0.01127543]}
81 def _funcVal(self, functor, parq):
82 self.assertIsInstance(functor.name, str)
83 self.assertIsInstance(functor.shortname, str)
85 val = functor(parq)
86 self.assertIsInstance(val, pd.Series)
88 val = functor(parq, dropna=True)
89 self.assertEqual(val.isnull().sum(), 0)
91 return val
93 def testColumn(self):
94 self.columns.append("base_FootprintArea_value")
95 self.dataDict["base_FootprintArea_value"] = \
96 np.full(self.nRecords, 1)
97 parq = self.simulateMultiParquet(self.dataDict)
98 func = Column('base_FootprintArea_value', filt='HSC-G')
99 self._funcVal(func, parq)
101 def testCustom(self):
102 self.columns.append("base_FootprintArea_value")
103 self.dataDict["base_FootprintArea_value"] = \
104 np.random.rand(self.nRecords)
105 parq = self.simulateMultiParquet(self.dataDict)
106 func = CustomFunctor('2*base_FootprintArea_value', filt='HSC-G')
107 val = self._funcVal(func, parq)
109 func2 = Column('base_FootprintArea_value', filt='HSC-G')
111 np.allclose(val.values, 2*func2(parq).values, atol=1e-13, rtol=0)
113 def testCoords(self):
114 parq = self.simulateMultiParquet(self.dataDict)
115 ra = self._funcVal(RAColumn(), parq)
116 dec = self._funcVal(DecColumn(), parq)
118 columnDict = {'dataset': 'ref', 'filter': 'HSC-G',
119 'column': ['coord_ra', 'coord_dec']}
120 coords = parq.toDataFrame(columns=columnDict, droplevels=True) / np.pi * 180.
122 self.assertTrue(np.allclose(ra, coords[('ref', 'HSC-G', 'coord_ra')], atol=1e-13, rtol=0))
123 self.assertTrue(np.allclose(dec, coords[('ref', 'HSC-G', 'coord_dec')], atol=1e-13, rtol=0))
125 def testMag(self):
126 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr"])
127 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
128 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10)
129 parq = self.simulateMultiParquet(self.dataDict)
130 # Change one dataset filter combinations value.
131 parq._df[("meas", "HSC-G", "base_PsfFlux_instFlux")] -= 1
133 fluxName = 'base_PsfFlux'
135 # Check that things work when you provide dataset explicitly
136 for dataset in ['forced_src', 'meas']:
137 psfMag_G = self._funcVal(Mag(fluxName, dataset=dataset,
138 filt='HSC-G'),
139 parq)
140 psfMag_R = self._funcVal(Mag(fluxName, dataset=dataset,
141 filt='HSC-R'),
142 parq)
144 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R',
145 dataset=dataset),
146 parq)
148 self.assertTrue(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR, rtol=0, atol=1e-13))
150 # Check that behavior as expected when dataset not provided;
151 # that is, that the color comes from forced and default Mag is meas
152 psfMag_G = self._funcVal(Mag(fluxName, filt='HSC-G'), parq)
153 psfMag_R = self._funcVal(Mag(fluxName, filt='HSC-R'), parq)
155 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R'), parq)
157 # These should *not* be equal.
158 self.assertFalse(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR))
160 def testMagDiff(self):
161 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr",
162 "modelfit_CModel_instFlux", "modelfit_CModel_instFluxErr"])
163 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
164 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10)
165 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1000)
166 self.dataDict["modelfit_CModel_instFluxErr"] = np.full(self.nRecords, 10)
167 parq = self.simulateMultiParquet(self.dataDict)
169 for filt in self.filters:
170 filt = 'HSC-G'
171 val = self._funcVal(MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt), parq)
173 mag1 = self._funcVal(Mag('modelfit_CModel', filt=filt), parq)
174 mag2 = self._funcVal(Mag('base_PsfFlux', filt=filt), parq)
175 self.assertTrue(np.allclose((mag2 - mag1).dropna(), val, rtol=0, atol=1e-13))
177 def testLabeller(self):
178 # Covering the code is better than nothing
179 self.columns.append("base_ClassificationExtendedness_value")
180 self.dataDict["base_ClassificationExtendedness_value"] = np.full(self.nRecords, 1)
181 parq = self.simulateMultiParquet(self.dataDict)
182 labels = self._funcVal(StarGalaxyLabeller(), parq) # noqa
184 def testOther(self):
185 self.columns.extend(["ext_shapeHSM_HsmSourceMoments_xx", "ext_shapeHSM_HsmSourceMoments_yy",
186 "base_SdssShape_xx", "base_SdssShape_yy",
187 "ext_shapeHSM_HsmPsfMoments_xx", "ext_shapeHSM_HsmPsfMoments_yy",
188 "base_SdssShape_psf_xx", "base_SdssShape_psf_yy"])
189 self.dataDict["ext_shapeHSM_HsmSourceMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2))
190 self.dataDict["ext_shapeHSM_HsmSourceMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2))
191 self.dataDict["base_SdssShape_xx"] = np.full(self.nRecords, 1 / np.sqrt(2))
192 self.dataDict["base_SdssShape_yy"] = np.full(self.nRecords, 1 / np.sqrt(2))
193 self.dataDict["ext_shapeHSM_HsmPsfMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2))
194 self.dataDict["ext_shapeHSM_HsmPsfMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2))
195 self.dataDict["base_SdssShape_psf_xx"] = np.full(self.nRecords, 1)
196 self.dataDict["base_SdssShape_psf_yy"] = np.full(self.nRecords, 1)
197 parq = self.simulateMultiParquet(self.dataDict)
198 # Covering the code is better than nothing
199 for filt in self.filters:
200 for Func in [DeconvolvedMoments,
201 SdssTraceSize,
202 PsfSdssTraceSizeDiff,
203 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm]:
204 val = self._funcVal(Func(filt=filt), parq) # noqa
206 def _compositeFuncVal(self, functor, parq):
207 self.assertIsInstance(functor, CompositeFunctor)
209 df = functor(parq)
211 self.assertIsInstance(df, pd.DataFrame)
212 self.assertTrue(np.all([k in df.columns for k in functor.funcDict.keys()]))
214 df = functor(parq, dropna=True)
216 # Check that there are no nulls
217 self.assertFalse(df.isnull().any(axis=None))
219 return df
221 def testComposite(self):
222 self.columns.extend(["modelfit_CModel_instFlux", "base_PsfFlux_instFlux"])
223 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1)
224 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1)
225 parq = self.simulateMultiParquet(self.dataDict)
226 # Modify r band value slightly.
227 parq._df[("meas", "HSC-R", "base_PsfFlux_instFlux")] -= 0.1
229 filt = 'HSC-G'
230 funcDict = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'),
231 'ra': RAColumn(),
232 'dec': DecColumn(),
233 'psfMag': Mag('base_PsfFlux', filt=filt),
234 'cmodel_magDiff': MagDiff('base_PsfFlux',
235 'modelfit_CModel', filt=filt)}
236 func = CompositeFunctor(funcDict)
237 df = self._compositeFuncVal(func, parq)
239 # Repeat same, but define filter globally instead of individually
240 funcDict2 = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'),
241 'ra': RAColumn(),
242 'dec': DecColumn(),
243 'psfMag': Mag('base_PsfFlux'),
244 'cmodel_magDiff': MagDiff('base_PsfFlux',
245 'modelfit_CModel')}
247 func2 = CompositeFunctor(funcDict2, filt=filt)
248 df2 = self._compositeFuncVal(func2, parq)
249 self.assertTrue(df.equals(df2))
251 func2.filt = 'HSC-R'
252 df3 = self._compositeFuncVal(func2, parq)
253 # Because we modified the R filter this should fail.
254 self.assertFalse(df2.equals(df3))
256 # Make sure things work with passing list instead of dict
257 funcs = [Mag('base_PsfFlux', dataset='ref'),
258 RAColumn(),
259 DecColumn(),
260 Mag('base_PsfFlux', filt=filt),
261 MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt)]
263 df = self._compositeFuncVal(CompositeFunctor(funcs), parq)
265 def testCompositeColor(self):
266 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
267 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10)
268 parq = self.simulateMultiParquet(self.dataDict)
269 funcDict = {'a': Mag('base_PsfFlux', dataset='meas', filt='HSC-G'),
270 'b': Mag('base_PsfFlux', dataset='forced_src', filt='HSC-G'),
271 'c': Color('base_PsfFlux', 'HSC-G', 'HSC-R')}
272 # Covering the code is better than nothing
273 df = self._compositeFuncVal(CompositeFunctor(funcDict), parq) # noqa
275 def testLocalPhotometry(self):
276 """Test the local photometry functors.
277 """
278 flux = 1000
279 fluxErr = 10
280 calib = 10
281 calibErr = 1
282 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, flux)
283 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords,
284 fluxErr)
285 self.dataDict["base_LocalPhotoCalib"] = np.full(self.nRecords, calib)
286 self.dataDict["base_LocalPhotoCalibErr"] = np.full(self.nRecords,
287 calibErr)
288 parq = self.simulateMultiParquet(self.dataDict)
289 func = LocalPhotometry("base_PsfFlux_instFlux",
290 "base_PsfFlux_instFluxErr",
291 "base_LocalPhotoCalib",
292 "base_LocalPhotoCalibErr")
293 df = parq.toDataFrame(columns={"dataset": "meas",
294 "filter": "HSC-G",
295 "columns": ["base_PsfFlux_instFlux",
296 "base_PsfFlux_instFluxErr",
297 "base_LocalPhotoCalib",
298 "base_LocalPhotoCalibErr"]})
299 nanoJansky = func.instFluxToNanojansky(
300 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
301 df[("meas", "HSC-G", "base_LocalPhotoCalib")])
302 mag = func.instFluxToMagnitude(
303 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
304 df[("meas", "HSC-G", "base_LocalPhotoCalib")])
305 nanoJanskyErr = func.instFluxErrToNanojanskyErr(
306 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
307 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")],
308 df[("meas", "HSC-G", "base_LocalPhotoCalib")],
309 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")])
310 magErr = func.instFluxErrToMagnitudeErr(
311 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
312 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")],
313 df[("meas", "HSC-G", "base_LocalPhotoCalib")],
314 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")])
316 self.assertTrue(np.allclose(nanoJansky.values,
317 flux * calib,
318 atol=1e-13,
319 rtol=0))
320 self.assertTrue(np.allclose(mag.values,
321 (flux * calib * u.nJy).to_value(u.ABmag),
322 atol=1e-13,
323 rtol=0))
324 self.assertTrue(np.allclose(nanoJanskyErr.values,
325 np.hypot(fluxErr * calib, flux * calibErr),
326 atol=1e-13,
327 rtol=0))
328 self.assertTrue(np.allclose(
329 magErr.values,
330 2.5 / np.log(10) * nanoJanskyErr.values / nanoJansky.values,
331 atol=1e-13,
332 rtol=0))
334 # Test functors against the values computed above.
335 self._testLocalPhotometryFunctors(LocalNanojansky,
336 parq,
337 nanoJansky)
338 self._testLocalPhotometryFunctors(LocalNanojanskyErr,
339 parq,
340 nanoJanskyErr)
341 self._testLocalPhotometryFunctors(LocalMagnitude,
342 parq,
343 mag)
344 self._testLocalPhotometryFunctors(LocalMagnitudeErr,
345 parq,
346 magErr)
348 def _testLocalPhotometryFunctors(self, functor, parq, testValues):
349 func = functor("base_PsfFlux_instFlux",
350 "base_PsfFlux_instFluxErr",
351 "base_LocalPhotoCalib",
352 "base_LocalPhotoCalibErr")
353 val = self._funcVal(func, parq)
354 self.assertTrue(np.allclose(testValues.values,
355 val.values,
356 atol=1e-13,
357 rtol=0))
360class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
361 pass
364def setup_module(module):
365 lsst.utils.tests.init()
368if __name__ == "__main__": 368 ↛ 369line 368 didn't jump to line 369, because the condition on line 368 was never true
369 lsst.utils.tests.init()
370 unittest.main()