Coverage for tests/test_functors.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import astropy.units as u
23import copy
24import functools
25import numpy as np
26import os
27import pandas as pd
28import unittest
30import lsst.daf.base as dafBase
31import lsst.afw.geom as afwGeom
32import lsst.utils.tests
34# TODO: Remove skipUnless and this try block DM-22256
35try:
36 from lsst.pipe.tasks.parquetTable import MultilevelParquetTable
37 from lsst.pipe.tasks.functors import (CompositeFunctor, CustomFunctor, Column, RAColumn,
38 DecColumn, Mag, MagDiff, Color, StarGalaxyLabeller,
39 DeconvolvedMoments, SdssTraceSize, PsfSdssTraceSizeDiff,
40 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm,
41 LocalPhotometry, LocalNanojansky, LocalNanojanskyErr,
42 LocalMagnitude, LocalMagnitudeErr,
43 ComputePixelScale, ConvertPixelToArcseconds)
44 havePyArrow = True
45except ImportError:
46 havePyArrow = False
48ROOT = os.path.abspath(os.path.dirname(__file__))
51@unittest.skipUnless(havePyArrow, "Requires pyarrow")
52class FunctorTestCase(unittest.TestCase):
54 def simulateMultiParquet(self, dataDict):
55 """Create a simple test MultilevelParquetTable
56 """
57 simpleDF = pd.DataFrame(dataDict)
58 dfFilterDSCombos = []
59 for ds in self.datasets:
60 for filterName in self.filters:
61 df = copy.copy(simpleDF)
62 df.reindex(sorted(df.columns), axis=1)
63 df['dataset'] = ds
64 df['filter'] = filterName
65 df.columns = pd.MultiIndex.from_tuples(
66 [(ds, filterName, c) for c in df.columns],
67 names=('dataset', 'filter', 'column'))
68 dfFilterDSCombos.append(df)
70 df = functools.reduce(lambda d1, d2: d1.join(d2), dfFilterDSCombos)
72 return MultilevelParquetTable(dataFrame=df)
74 def setUp(self):
75 np.random.seed(1234)
76 self.datasets = ['forced_src', 'meas', 'ref']
77 self.filters = ['HSC-G', 'HSC-R']
78 self.columns = ['coord_ra', 'coord_dec']
79 self.nRecords = 5
80 self.dataDict = {
81 "coord_ra": [3.77654137, 3.77643059, 3.77621148, 3.77611944, 3.77610396],
82 "coord_dec": [0.01127624, 0.01127787, 0.01127543, 0.01127543, 0.01127543]}
84 def _funcVal(self, functor, parq):
85 self.assertIsInstance(functor.name, str)
86 self.assertIsInstance(functor.shortname, str)
88 val = functor(parq)
89 self.assertIsInstance(val, pd.Series)
91 val = functor(parq, dropna=True)
92 self.assertEqual(val.isnull().sum(), 0)
94 return val
96 def testColumn(self):
97 self.columns.append("base_FootprintArea_value")
98 self.dataDict["base_FootprintArea_value"] = \
99 np.full(self.nRecords, 1)
100 parq = self.simulateMultiParquet(self.dataDict)
101 func = Column('base_FootprintArea_value', filt='HSC-G')
102 self._funcVal(func, parq)
104 def testCustom(self):
105 self.columns.append("base_FootprintArea_value")
106 self.dataDict["base_FootprintArea_value"] = \
107 np.random.rand(self.nRecords)
108 parq = self.simulateMultiParquet(self.dataDict)
109 func = CustomFunctor('2*base_FootprintArea_value', filt='HSC-G')
110 val = self._funcVal(func, parq)
112 func2 = Column('base_FootprintArea_value', filt='HSC-G')
114 np.allclose(val.values, 2*func2(parq).values, atol=1e-13, rtol=0)
116 def testCoords(self):
117 parq = self.simulateMultiParquet(self.dataDict)
118 ra = self._funcVal(RAColumn(), parq)
119 dec = self._funcVal(DecColumn(), parq)
121 columnDict = {'dataset': 'ref', 'filter': 'HSC-G',
122 'column': ['coord_ra', 'coord_dec']}
123 coords = parq.toDataFrame(columns=columnDict, droplevels=True) / np.pi * 180.
125 self.assertTrue(np.allclose(ra, coords[('ref', 'HSC-G', 'coord_ra')], atol=1e-13, rtol=0))
126 self.assertTrue(np.allclose(dec, coords[('ref', 'HSC-G', 'coord_dec')], atol=1e-13, rtol=0))
128 def testMag(self):
129 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr"])
130 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
131 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10)
132 parq = self.simulateMultiParquet(self.dataDict)
133 # Change one dataset filter combinations value.
134 parq._df[("meas", "HSC-G", "base_PsfFlux_instFlux")] -= 1
136 fluxName = 'base_PsfFlux'
138 # Check that things work when you provide dataset explicitly
139 for dataset in ['forced_src', 'meas']:
140 psfMag_G = self._funcVal(Mag(fluxName, dataset=dataset,
141 filt='HSC-G'),
142 parq)
143 psfMag_R = self._funcVal(Mag(fluxName, dataset=dataset,
144 filt='HSC-R'),
145 parq)
147 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R',
148 dataset=dataset),
149 parq)
151 self.assertTrue(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR, rtol=0, atol=1e-13))
153 # Check that behavior as expected when dataset not provided;
154 # that is, that the color comes from forced and default Mag is meas
155 psfMag_G = self._funcVal(Mag(fluxName, filt='HSC-G'), parq)
156 psfMag_R = self._funcVal(Mag(fluxName, filt='HSC-R'), parq)
158 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R'), parq)
160 # These should *not* be equal.
161 self.assertFalse(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR))
163 def testMagDiff(self):
164 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr",
165 "modelfit_CModel_instFlux", "modelfit_CModel_instFluxErr"])
166 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
167 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10)
168 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1000)
169 self.dataDict["modelfit_CModel_instFluxErr"] = np.full(self.nRecords, 10)
170 parq = self.simulateMultiParquet(self.dataDict)
172 for filt in self.filters:
173 filt = 'HSC-G'
174 val = self._funcVal(MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt), parq)
176 mag1 = self._funcVal(Mag('modelfit_CModel', filt=filt), parq)
177 mag2 = self._funcVal(Mag('base_PsfFlux', filt=filt), parq)
178 self.assertTrue(np.allclose((mag2 - mag1).dropna(), val, rtol=0, atol=1e-13))
180 def testLabeller(self):
181 # Covering the code is better than nothing
182 self.columns.append("base_ClassificationExtendedness_value")
183 self.dataDict["base_ClassificationExtendedness_value"] = np.full(self.nRecords, 1)
184 parq = self.simulateMultiParquet(self.dataDict)
185 labels = self._funcVal(StarGalaxyLabeller(), parq) # noqa
187 def testPixelScale(self):
188 # Test that the pixel scale and pix->arcsec calculations perform as
189 # expected.
190 pass
192 def testOther(self):
193 self.columns.extend(["ext_shapeHSM_HsmSourceMoments_xx", "ext_shapeHSM_HsmSourceMoments_yy",
194 "base_SdssShape_xx", "base_SdssShape_yy",
195 "ext_shapeHSM_HsmPsfMoments_xx", "ext_shapeHSM_HsmPsfMoments_yy",
196 "base_SdssShape_psf_xx", "base_SdssShape_psf_yy"])
197 self.dataDict["ext_shapeHSM_HsmSourceMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2))
198 self.dataDict["ext_shapeHSM_HsmSourceMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2))
199 self.dataDict["base_SdssShape_xx"] = np.full(self.nRecords, 1 / np.sqrt(2))
200 self.dataDict["base_SdssShape_yy"] = np.full(self.nRecords, 1 / np.sqrt(2))
201 self.dataDict["ext_shapeHSM_HsmPsfMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2))
202 self.dataDict["ext_shapeHSM_HsmPsfMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2))
203 self.dataDict["base_SdssShape_psf_xx"] = np.full(self.nRecords, 1)
204 self.dataDict["base_SdssShape_psf_yy"] = np.full(self.nRecords, 1)
205 parq = self.simulateMultiParquet(self.dataDict)
206 # Covering the code is better than nothing
207 for filt in self.filters:
208 for Func in [DeconvolvedMoments,
209 SdssTraceSize,
210 PsfSdssTraceSizeDiff,
211 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm]:
212 val = self._funcVal(Func(filt=filt), parq) # noqa
214 def _compositeFuncVal(self, functor, parq):
215 self.assertIsInstance(functor, CompositeFunctor)
217 df = functor(parq)
219 self.assertIsInstance(df, pd.DataFrame)
220 self.assertTrue(np.all([k in df.columns for k in functor.funcDict.keys()]))
222 df = functor(parq, dropna=True)
224 # Check that there are no nulls
225 self.assertFalse(df.isnull().any(axis=None))
227 return df
229 def testComposite(self):
230 self.columns.extend(["modelfit_CModel_instFlux", "base_PsfFlux_instFlux"])
231 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1)
232 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1)
233 parq = self.simulateMultiParquet(self.dataDict)
234 # Modify r band value slightly.
235 parq._df[("meas", "HSC-R", "base_PsfFlux_instFlux")] -= 0.1
237 filt = 'HSC-G'
238 funcDict = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'),
239 'ra': RAColumn(),
240 'dec': DecColumn(),
241 'psfMag': Mag('base_PsfFlux', filt=filt),
242 'cmodel_magDiff': MagDiff('base_PsfFlux',
243 'modelfit_CModel', filt=filt)}
244 func = CompositeFunctor(funcDict)
245 df = self._compositeFuncVal(func, parq)
247 # Repeat same, but define filter globally instead of individually
248 funcDict2 = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'),
249 'ra': RAColumn(),
250 'dec': DecColumn(),
251 'psfMag': Mag('base_PsfFlux'),
252 'cmodel_magDiff': MagDiff('base_PsfFlux',
253 'modelfit_CModel')}
255 func2 = CompositeFunctor(funcDict2, filt=filt)
256 df2 = self._compositeFuncVal(func2, parq)
257 self.assertTrue(df.equals(df2))
259 func2.filt = 'HSC-R'
260 df3 = self._compositeFuncVal(func2, parq)
261 # Because we modified the R filter this should fail.
262 self.assertFalse(df2.equals(df3))
264 # Make sure things work with passing list instead of dict
265 funcs = [Mag('base_PsfFlux', dataset='ref'),
266 RAColumn(),
267 DecColumn(),
268 Mag('base_PsfFlux', filt=filt),
269 MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt)]
271 df = self._compositeFuncVal(CompositeFunctor(funcs), parq)
273 def testCompositeColor(self):
274 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
275 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10)
276 parq = self.simulateMultiParquet(self.dataDict)
277 funcDict = {'a': Mag('base_PsfFlux', dataset='meas', filt='HSC-G'),
278 'b': Mag('base_PsfFlux', dataset='forced_src', filt='HSC-G'),
279 'c': Color('base_PsfFlux', 'HSC-G', 'HSC-R')}
280 # Covering the code is better than nothing
281 df = self._compositeFuncVal(CompositeFunctor(funcDict), parq) # noqa
283 def testLocalPhotometry(self):
284 """Test the local photometry functors.
285 """
286 flux = 1000
287 fluxErr = 10
288 calib = 10
289 calibErr = 1
290 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, flux)
291 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords,
292 fluxErr)
293 self.dataDict["base_LocalPhotoCalib"] = np.full(self.nRecords, calib)
294 self.dataDict["base_LocalPhotoCalibErr"] = np.full(self.nRecords,
295 calibErr)
296 parq = self.simulateMultiParquet(self.dataDict)
297 func = LocalPhotometry("base_PsfFlux_instFlux",
298 "base_PsfFlux_instFluxErr",
299 "base_LocalPhotoCalib",
300 "base_LocalPhotoCalibErr")
301 df = parq.toDataFrame(columns={"dataset": "meas",
302 "filter": "HSC-G",
303 "columns": ["base_PsfFlux_instFlux",
304 "base_PsfFlux_instFluxErr",
305 "base_LocalPhotoCalib",
306 "base_LocalPhotoCalibErr"]})
307 nanoJansky = func.instFluxToNanojansky(
308 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
309 df[("meas", "HSC-G", "base_LocalPhotoCalib")])
310 mag = func.instFluxToMagnitude(
311 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
312 df[("meas", "HSC-G", "base_LocalPhotoCalib")])
313 nanoJanskyErr = func.instFluxErrToNanojanskyErr(
314 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
315 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")],
316 df[("meas", "HSC-G", "base_LocalPhotoCalib")],
317 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")])
318 magErr = func.instFluxErrToMagnitudeErr(
319 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
320 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")],
321 df[("meas", "HSC-G", "base_LocalPhotoCalib")],
322 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")])
324 self.assertTrue(np.allclose(nanoJansky.values,
325 flux * calib,
326 atol=1e-13,
327 rtol=0))
328 self.assertTrue(np.allclose(mag.values,
329 (flux * calib * u.nJy).to_value(u.ABmag),
330 atol=1e-13,
331 rtol=0))
332 self.assertTrue(np.allclose(nanoJanskyErr.values,
333 np.hypot(fluxErr * calib, flux * calibErr),
334 atol=1e-13,
335 rtol=0))
336 self.assertTrue(np.allclose(
337 magErr.values,
338 2.5 / np.log(10) * nanoJanskyErr.values / nanoJansky.values,
339 atol=1e-13,
340 rtol=0))
342 # Test functors against the values computed above.
343 self._testLocalPhotometryFunctors(LocalNanojansky,
344 parq,
345 nanoJansky)
346 self._testLocalPhotometryFunctors(LocalNanojanskyErr,
347 parq,
348 nanoJanskyErr)
349 self._testLocalPhotometryFunctors(LocalMagnitude,
350 parq,
351 mag)
352 self._testLocalPhotometryFunctors(LocalMagnitudeErr,
353 parq,
354 magErr)
356 def _testLocalPhotometryFunctors(self, functor, parq, testValues):
357 func = functor("base_PsfFlux_instFlux",
358 "base_PsfFlux_instFluxErr",
359 "base_LocalPhotoCalib",
360 "base_LocalPhotoCalibErr")
361 val = self._funcVal(func, parq)
362 self.assertTrue(np.allclose(testValues.values,
363 val.values,
364 atol=1e-13,
365 rtol=0))
367 def testConvertPixelToArcseconds(self):
368 """Test calculations of the pixel scale and conversions of pixel to
369 arcseconds.
370 """
371 wcs = self._makeWcs()
372 cdMatrix = wcs.getCdMatrix(wcs.getPixelOrigin())
373 self.dataDict["dipoleSep"] = np.full(self.nRecords, 10)
374 self.dataDict["base_LocalWcs_CDMatrix_1_1"] = np.full(self.nRecords,
375 cdMatrix[0, 0])
376 self.dataDict["base_LocalWcs_CDMatrix_1_2"] = np.full(self.nRecords,
377 cdMatrix[0, 1])
378 self.dataDict["base_LocalWcs_CDMatrix_2_1"] = np.full(self.nRecords,
379 cdMatrix[1, 0])
380 self.dataDict["base_LocalWcs_CDMatrix_2_2"] = np.full(self.nRecords,
381 cdMatrix[1, 1])
382 parq = self.simulateMultiParquet(self.dataDict)
383 func = ComputePixelScale("base_LocalWcs_CDMatrix_1_1",
384 "base_LocalWcs_CDMatrix_1_2",
385 "base_LocalWcs_CDMatrix_2_1",
386 "base_LocalWcs_CDMatrix_2_2")
387 df = parq.toDataFrame(columns={"dataset": "meas",
388 "filter": "HSC-G",
389 "columns": ["dipoleSep",
390 "base_LocalWcs_CDMatrix_1_1",
391 "base_LocalWcs_CDMatrix_1_2",
392 "base_LocalWcs_CDMatrix_2_1",
393 "base_LocalWcs_CDMatrix_2_2"]})
394 pixelScale = func.pixelScale(df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_1_1")],
395 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_1_2")],
396 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_2_1")],
397 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_2_2")])
398 self.assertTrue(np.allclose(pixelScale.values,
399 wcs.getPixelScale().asArcseconds(),
400 rtol=0,
401 atol=1e-10))
403 # Test functors
404 val = self._funcVal(func, parq)
405 self.assertTrue(np.allclose(pixelScale.values,
406 val.values,
407 atol=1e-13,
408 rtol=0))
410 func = ConvertPixelToArcseconds("dipoleSep",
411 "base_LocalWcs_CDMatrix_1_1",
412 "base_LocalWcs_CDMatrix_1_2",
413 "base_LocalWcs_CDMatrix_2_1",
414 "base_LocalWcs_CDMatrix_2_2")
415 val = self._funcVal(func, parq)
416 self.assertTrue(np.allclose(pixelScale.values * 10,
417 val.values,
418 atol=1e-13,
419 rtol=0))
421 def _makeWcs(self):
422 """Create a wcs from real CFHT values.
424 Returns
425 -------
426 wcs : `lsst.afw.geom`
427 Created wcs.
428 """
429 metadata = dafBase.PropertySet()
431 metadata.set("SIMPLE", "T")
432 metadata.set("BITPIX", -32)
433 metadata.set("NAXIS", 2)
434 metadata.set("NAXIS1", 1024)
435 metadata.set("NAXIS2", 1153)
436 metadata.set("RADECSYS", 'FK5')
437 metadata.set("EQUINOX", 2000.)
439 metadata.setDouble("CRVAL1", 215.604025685476)
440 metadata.setDouble("CRVAL2", 53.1595451514076)
441 metadata.setDouble("CRPIX1", 1109.99981456774)
442 metadata.setDouble("CRPIX2", 560.018167811613)
443 metadata.set("CTYPE1", 'RA---SIN')
444 metadata.set("CTYPE2", 'DEC--SIN')
446 metadata.setDouble("CD1_1", 5.10808596133527E-05)
447 metadata.setDouble("CD1_2", 1.85579539217196E-07)
448 metadata.setDouble("CD2_2", -5.10281493481982E-05)
449 metadata.setDouble("CD2_1", -8.27440751733828E-07)
451 return afwGeom.makeSkyWcs(metadata)
454class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
455 pass
458def setup_module(module):
459 lsst.utils.tests.init()
462if __name__ == "__main__": 462 ↛ 463line 462 didn't jump to line 463, because the condition on line 462 was never true
463 lsst.utils.tests.init()
464 unittest.main()