Coverage for tests/test_functors.py : 99%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import astropy.units as u
23import copy
24import functools
25import numpy as np
26import os
27import pandas as pd
28import unittest
30import lsst.daf.base as dafBase
31import lsst.afw.geom as afwGeom
32import lsst.geom as geom
33import lsst.meas.base as measBase
34import lsst.utils.tests
35from lsst.pipe.tasks.parquetTable import MultilevelParquetTable
36from lsst.pipe.tasks.functors import (CompositeFunctor, CustomFunctor, Column, RAColumn,
37 DecColumn, Mag, MagDiff, Color, StarGalaxyLabeller,
38 DeconvolvedMoments, SdssTraceSize, PsfSdssTraceSizeDiff,
39 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm,
40 LocalPhotometry, LocalNanojansky, LocalNanojanskyErr,
41 LocalMagnitude, LocalMagnitudeErr,
42 LocalWcs, ComputePixelScale, ConvertPixelToArcseconds)
44ROOT = os.path.abspath(os.path.dirname(__file__))
47class FunctorTestCase(unittest.TestCase):
49 def simulateMultiParquet(self, dataDict):
50 """Create a simple test MultilevelParquetTable
51 """
52 simpleDF = pd.DataFrame(dataDict)
53 dfFilterDSCombos = []
54 for ds in self.datasets:
55 for filterName in self.filters:
56 df = copy.copy(simpleDF)
57 df.reindex(sorted(df.columns), axis=1)
58 df['dataset'] = ds
59 df['filter'] = filterName
60 df.columns = pd.MultiIndex.from_tuples(
61 [(ds, filterName, c) for c in df.columns],
62 names=('dataset', 'filter', 'column'))
63 dfFilterDSCombos.append(df)
65 df = functools.reduce(lambda d1, d2: d1.join(d2), dfFilterDSCombos)
67 return MultilevelParquetTable(dataFrame=df)
69 def setUp(self):
70 np.random.seed(1234)
71 self.datasets = ['forced_src', 'meas', 'ref']
72 self.filters = ['HSC-G', 'HSC-R']
73 self.columns = ['coord_ra', 'coord_dec']
74 self.nRecords = 5
75 self.dataDict = {
76 "coord_ra": [3.77654137, 3.77643059, 3.77621148, 3.77611944, 3.77610396],
77 "coord_dec": [0.01127624, 0.01127787, 0.01127543, 0.01127543, 0.01127543]}
79 def _funcVal(self, functor, parq):
80 self.assertIsInstance(functor.name, str)
81 self.assertIsInstance(functor.shortname, str)
83 val = functor(parq)
84 self.assertIsInstance(val, pd.Series)
86 val = functor(parq, dropna=True)
87 self.assertEqual(val.isnull().sum(), 0)
89 return val
91 def testColumn(self):
92 self.columns.append("base_FootprintArea_value")
93 self.dataDict["base_FootprintArea_value"] = \
94 np.full(self.nRecords, 1)
95 parq = self.simulateMultiParquet(self.dataDict)
96 func = Column('base_FootprintArea_value', filt='HSC-G')
97 self._funcVal(func, parq)
99 def testCustom(self):
100 self.columns.append("base_FootprintArea_value")
101 self.dataDict["base_FootprintArea_value"] = \
102 np.random.rand(self.nRecords)
103 parq = self.simulateMultiParquet(self.dataDict)
104 func = CustomFunctor('2*base_FootprintArea_value', filt='HSC-G')
105 val = self._funcVal(func, parq)
107 func2 = Column('base_FootprintArea_value', filt='HSC-G')
109 np.allclose(val.values, 2*func2(parq).values, atol=1e-13, rtol=0)
111 def testCoords(self):
112 parq = self.simulateMultiParquet(self.dataDict)
113 ra = self._funcVal(RAColumn(), parq)
114 dec = self._funcVal(DecColumn(), parq)
116 columnDict = {'dataset': 'ref', 'filter': 'HSC-G',
117 'column': ['coord_ra', 'coord_dec']}
118 coords = parq.toDataFrame(columns=columnDict, droplevels=True) / np.pi * 180.
120 self.assertTrue(np.allclose(ra, coords[('ref', 'HSC-G', 'coord_ra')], atol=1e-13, rtol=0))
121 self.assertTrue(np.allclose(dec, coords[('ref', 'HSC-G', 'coord_dec')], atol=1e-13, rtol=0))
123 def testMag(self):
124 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr"])
125 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
126 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10)
127 parq = self.simulateMultiParquet(self.dataDict)
128 # Change one dataset filter combinations value.
129 parq._df[("meas", "HSC-G", "base_PsfFlux_instFlux")] -= 1
131 fluxName = 'base_PsfFlux'
133 # Check that things work when you provide dataset explicitly
134 for dataset in ['forced_src', 'meas']:
135 psfMag_G = self._funcVal(Mag(fluxName, dataset=dataset,
136 filt='HSC-G'),
137 parq)
138 psfMag_R = self._funcVal(Mag(fluxName, dataset=dataset,
139 filt='HSC-R'),
140 parq)
142 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R',
143 dataset=dataset),
144 parq)
146 self.assertTrue(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR, rtol=0, atol=1e-13))
148 # Check that behavior as expected when dataset not provided;
149 # that is, that the color comes from forced and default Mag is meas
150 psfMag_G = self._funcVal(Mag(fluxName, filt='HSC-G'), parq)
151 psfMag_R = self._funcVal(Mag(fluxName, filt='HSC-R'), parq)
153 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R'), parq)
155 # These should *not* be equal.
156 self.assertFalse(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR))
158 def testMagDiff(self):
159 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr",
160 "modelfit_CModel_instFlux", "modelfit_CModel_instFluxErr"])
161 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
162 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10)
163 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1000)
164 self.dataDict["modelfit_CModel_instFluxErr"] = np.full(self.nRecords, 10)
165 parq = self.simulateMultiParquet(self.dataDict)
167 for filt in self.filters:
168 filt = 'HSC-G'
169 val = self._funcVal(MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt), parq)
171 mag1 = self._funcVal(Mag('modelfit_CModel', filt=filt), parq)
172 mag2 = self._funcVal(Mag('base_PsfFlux', filt=filt), parq)
173 self.assertTrue(np.allclose((mag2 - mag1).dropna(), val, rtol=0, atol=1e-13))
175 def testLabeller(self):
176 # Covering the code is better than nothing
177 self.columns.append("base_ClassificationExtendedness_value")
178 self.dataDict["base_ClassificationExtendedness_value"] = np.full(self.nRecords, 1)
179 parq = self.simulateMultiParquet(self.dataDict)
180 labels = self._funcVal(StarGalaxyLabeller(), parq) # noqa
182 def testPixelScale(self):
183 # Test that the pixel scale and pix->arcsec calculations perform as
184 # expected.
185 pass
187 def testOther(self):
188 self.columns.extend(["ext_shapeHSM_HsmSourceMoments_xx", "ext_shapeHSM_HsmSourceMoments_yy",
189 "base_SdssShape_xx", "base_SdssShape_yy",
190 "ext_shapeHSM_HsmPsfMoments_xx", "ext_shapeHSM_HsmPsfMoments_yy",
191 "base_SdssShape_psf_xx", "base_SdssShape_psf_yy"])
192 self.dataDict["ext_shapeHSM_HsmSourceMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2))
193 self.dataDict["ext_shapeHSM_HsmSourceMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2))
194 self.dataDict["base_SdssShape_xx"] = np.full(self.nRecords, 1 / np.sqrt(2))
195 self.dataDict["base_SdssShape_yy"] = np.full(self.nRecords, 1 / np.sqrt(2))
196 self.dataDict["ext_shapeHSM_HsmPsfMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2))
197 self.dataDict["ext_shapeHSM_HsmPsfMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2))
198 self.dataDict["base_SdssShape_psf_xx"] = np.full(self.nRecords, 1)
199 self.dataDict["base_SdssShape_psf_yy"] = np.full(self.nRecords, 1)
200 parq = self.simulateMultiParquet(self.dataDict)
201 # Covering the code is better than nothing
202 for filt in self.filters:
203 for Func in [DeconvolvedMoments,
204 SdssTraceSize,
205 PsfSdssTraceSizeDiff,
206 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm]:
207 val = self._funcVal(Func(filt=filt), parq) # noqa
209 def _compositeFuncVal(self, functor, parq):
210 self.assertIsInstance(functor, CompositeFunctor)
212 df = functor(parq)
214 self.assertIsInstance(df, pd.DataFrame)
215 self.assertTrue(np.all([k in df.columns for k in functor.funcDict.keys()]))
217 df = functor(parq, dropna=True)
219 # Check that there are no nulls
220 self.assertFalse(df.isnull().any(axis=None))
222 return df
224 def testComposite(self):
225 self.columns.extend(["modelfit_CModel_instFlux", "base_PsfFlux_instFlux"])
226 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1)
227 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1)
228 parq = self.simulateMultiParquet(self.dataDict)
229 # Modify r band value slightly.
230 parq._df[("meas", "HSC-R", "base_PsfFlux_instFlux")] -= 0.1
232 filt = 'HSC-G'
233 funcDict = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'),
234 'ra': RAColumn(),
235 'dec': DecColumn(),
236 'psfMag': Mag('base_PsfFlux', filt=filt),
237 'cmodel_magDiff': MagDiff('base_PsfFlux',
238 'modelfit_CModel', filt=filt)}
239 func = CompositeFunctor(funcDict)
240 df = self._compositeFuncVal(func, parq)
242 # Repeat same, but define filter globally instead of individually
243 funcDict2 = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'),
244 'ra': RAColumn(),
245 'dec': DecColumn(),
246 'psfMag': Mag('base_PsfFlux'),
247 'cmodel_magDiff': MagDiff('base_PsfFlux',
248 'modelfit_CModel')}
250 func2 = CompositeFunctor(funcDict2, filt=filt)
251 df2 = self._compositeFuncVal(func2, parq)
252 self.assertTrue(df.equals(df2))
254 func2.filt = 'HSC-R'
255 df3 = self._compositeFuncVal(func2, parq)
256 # Because we modified the R filter this should fail.
257 self.assertFalse(df2.equals(df3))
259 # Make sure things work with passing list instead of dict
260 funcs = [Mag('base_PsfFlux', dataset='ref'),
261 RAColumn(),
262 DecColumn(),
263 Mag('base_PsfFlux', filt=filt),
264 MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt)]
266 df = self._compositeFuncVal(CompositeFunctor(funcs), parq)
268 def testCompositeColor(self):
269 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000)
270 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10)
271 parq = self.simulateMultiParquet(self.dataDict)
272 funcDict = {'a': Mag('base_PsfFlux', dataset='meas', filt='HSC-G'),
273 'b': Mag('base_PsfFlux', dataset='forced_src', filt='HSC-G'),
274 'c': Color('base_PsfFlux', 'HSC-G', 'HSC-R')}
275 # Covering the code is better than nothing
276 df = self._compositeFuncVal(CompositeFunctor(funcDict), parq) # noqa
278 def testLocalPhotometry(self):
279 """Test the local photometry functors.
280 """
281 flux = 1000
282 fluxErr = 10
283 calib = 10
284 calibErr = 1
285 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, flux)
286 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords,
287 fluxErr)
288 self.dataDict["base_LocalPhotoCalib"] = np.full(self.nRecords, calib)
289 self.dataDict["base_LocalPhotoCalibErr"] = np.full(self.nRecords,
290 calibErr)
291 parq = self.simulateMultiParquet(self.dataDict)
292 func = LocalPhotometry("base_PsfFlux_instFlux",
293 "base_PsfFlux_instFluxErr",
294 "base_LocalPhotoCalib",
295 "base_LocalPhotoCalibErr")
296 df = parq.toDataFrame(columns={"dataset": "meas",
297 "filter": "HSC-G",
298 "columns": ["base_PsfFlux_instFlux",
299 "base_PsfFlux_instFluxErr",
300 "base_LocalPhotoCalib",
301 "base_LocalPhotoCalibErr"]})
302 nanoJansky = func.instFluxToNanojansky(
303 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
304 df[("meas", "HSC-G", "base_LocalPhotoCalib")])
305 mag = func.instFluxToMagnitude(
306 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
307 df[("meas", "HSC-G", "base_LocalPhotoCalib")])
308 nanoJanskyErr = func.instFluxErrToNanojanskyErr(
309 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
310 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")],
311 df[("meas", "HSC-G", "base_LocalPhotoCalib")],
312 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")])
313 magErr = func.instFluxErrToMagnitudeErr(
314 df[("meas", "HSC-G", "base_PsfFlux_instFlux")],
315 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")],
316 df[("meas", "HSC-G", "base_LocalPhotoCalib")],
317 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")])
319 self.assertTrue(np.allclose(nanoJansky.values,
320 flux * calib,
321 atol=1e-13,
322 rtol=0))
323 self.assertTrue(np.allclose(mag.values,
324 (flux * calib * u.nJy).to_value(u.ABmag),
325 atol=1e-13,
326 rtol=0))
327 self.assertTrue(np.allclose(nanoJanskyErr.values,
328 np.hypot(fluxErr * calib, flux * calibErr),
329 atol=1e-13,
330 rtol=0))
331 self.assertTrue(np.allclose(
332 magErr.values,
333 2.5 / np.log(10) * nanoJanskyErr.values / nanoJansky.values,
334 atol=1e-13,
335 rtol=0))
337 # Test functors against the values computed above.
338 self._testLocalPhotometryFunctors(LocalNanojansky,
339 parq,
340 nanoJansky)
341 self._testLocalPhotometryFunctors(LocalNanojanskyErr,
342 parq,
343 nanoJanskyErr)
344 self._testLocalPhotometryFunctors(LocalMagnitude,
345 parq,
346 mag)
347 self._testLocalPhotometryFunctors(LocalMagnitudeErr,
348 parq,
349 magErr)
351 def _testLocalPhotometryFunctors(self, functor, parq, testValues):
352 func = functor("base_PsfFlux_instFlux",
353 "base_PsfFlux_instFluxErr",
354 "base_LocalPhotoCalib",
355 "base_LocalPhotoCalibErr")
356 val = self._funcVal(func, parq)
357 self.assertTrue(np.allclose(testValues.values,
358 val.values,
359 atol=1e-13,
360 rtol=0))
362 def testConvertPixelToArcseconds(self):
363 """Test calculations of the pixel scale and conversions of pixel to
364 arcseconds.
365 """
366 dipoleSep = 10
367 np.random.seed(1234)
368 testPixelDeltas = np.random.uniform(-100, 100, size=(self.nRecords, 2))
369 import lsst.afw.table as afwTable
370 localWcsPlugin = measBase.EvaluateLocalWcsPlugin(
371 None,
372 "base_LocalWcs",
373 afwTable.SourceTable.makeMinimalSchema(),
374 None)
375 for dec in np.linspace(-90, 90, 10):
376 for x, y in zip(np.random.uniform(2 * 1109.99981456774, size=10),
377 np.random.uniform(2 * 560.018167811613, size=10)):
379 center = geom.Point2D(x, y)
380 wcs = self._makeWcs(dec)
381 skyOrigin = wcs.pixelToSky(center)
383 linAffMatrix = localWcsPlugin.makeLocalTransformMatrix(wcs,
384 center)
385 self.dataDict["dipoleSep"] = np.full(self.nRecords, dipoleSep)
386 self.dataDict["slot_Centroid_x"] = np.full(self.nRecords, x)
387 self.dataDict["slot_Centroid_y"] = np.full(self.nRecords, y)
388 self.dataDict["someCentroid_x"] = x + testPixelDeltas[:, 0]
389 self.dataDict["someCentroid_y"] = y + testPixelDeltas[:, 1]
390 self.dataDict["base_LocalWcs_CDMatrix_1_1"] = np.full(self.nRecords,
391 linAffMatrix[0, 0])
392 self.dataDict["base_LocalWcs_CDMatrix_1_2"] = np.full(self.nRecords,
393 linAffMatrix[0, 1])
394 self.dataDict["base_LocalWcs_CDMatrix_2_1"] = np.full(self.nRecords,
395 linAffMatrix[1, 0])
396 self.dataDict["base_LocalWcs_CDMatrix_2_2"] = np.full(self.nRecords,
397 linAffMatrix[1, 1])
398 parq = self.simulateMultiParquet(self.dataDict)
399 func = LocalWcs("base_LocalWcs_CDMatrix_1_1",
400 "base_LocalWcs_CDMatrix_1_2",
401 "base_LocalWcs_CDMatrix_2_1",
402 "base_LocalWcs_CDMatrix_2_2")
403 df = parq.toDataFrame(columns={"dataset": "meas",
404 "filter": "HSC-G",
405 "columns": ["dipoleSep",
406 "slot_Centroid_x",
407 "slot_Centroid_y",
408 "someCentroid_x",
409 "someCentroid_y",
410 "base_LocalWcs_CDMatrix_1_1",
411 "base_LocalWcs_CDMatrix_1_2",
412 "base_LocalWcs_CDMatrix_2_1",
413 "base_LocalWcs_CDMatrix_2_2"]})
415 # Exercise the full set of functions in LocalWcs.
416 sepRadians = func.getSkySeperationFromPixel(
417 df[("meas", "HSC-G", "someCentroid_x")] - df[("meas", "HSC-G", "slot_Centroid_x")],
418 df[("meas", "HSC-G", "someCentroid_y")] - df[("meas", "HSC-G", "slot_Centroid_y")],
419 0.0,
420 0.0,
421 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_1_1")],
422 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_1_2")],
423 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_2_1")],
424 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_2_2")])
426 # Test functor values against afw SkyWcs computations.
427 for centX, centY, sep in zip(testPixelDeltas[:, 0],
428 testPixelDeltas[:, 1],
429 sepRadians.values):
430 afwSepRadians = skyOrigin.separation(
431 wcs.pixelToSky(x + centX, y + centY)).asRadians()
432 self.assertAlmostEqual(1 - sep / afwSepRadians, 0, places=6)
434 # Test the pixel scale computation.
435 func = ComputePixelScale("base_LocalWcs_CDMatrix_1_1",
436 "base_LocalWcs_CDMatrix_1_2",
437 "base_LocalWcs_CDMatrix_2_1",
438 "base_LocalWcs_CDMatrix_2_2")
439 pixelScale = self._funcVal(func, parq)
440 self.assertTrue(np.allclose(
441 wcs.getPixelScale(center).asArcseconds(),
442 pixelScale.values,
443 rtol=1e-8,
444 atol=0))
446 func = ConvertPixelToArcseconds("dipoleSep",
447 "base_LocalWcs_CDMatrix_1_1",
448 "base_LocalWcs_CDMatrix_1_2",
449 "base_LocalWcs_CDMatrix_2_1",
450 "base_LocalWcs_CDMatrix_2_2")
451 val = self._funcVal(func, parq)
452 self.assertTrue(np.allclose(pixelScale.values * dipoleSep,
453 val.values,
454 atol=1e-16,
455 rtol=1e-16))
457 def _makeWcs(self, dec=53.1595451514076):
458 """Create a wcs from real CFHT values.
460 Returns
461 -------
462 wcs : `lsst.afw.geom`
463 Created wcs.
464 """
465 metadata = dafBase.PropertySet()
467 metadata.set("SIMPLE", "T")
468 metadata.set("BITPIX", -32)
469 metadata.set("NAXIS", 2)
470 metadata.set("NAXIS1", 1024)
471 metadata.set("NAXIS2", 1153)
472 metadata.set("RADECSYS", 'FK5')
473 metadata.set("EQUINOX", 2000.)
475 metadata.setDouble("CRVAL1", 215.604025685476)
476 metadata.setDouble("CRVAL2", dec)
477 metadata.setDouble("CRPIX1", 1109.99981456774)
478 metadata.setDouble("CRPIX2", 560.018167811613)
479 metadata.set("CTYPE1", 'RA---SIN')
480 metadata.set("CTYPE2", 'DEC--SIN')
482 metadata.setDouble("CD1_1", 5.10808596133527E-05)
483 metadata.setDouble("CD1_2", 1.85579539217196E-07)
484 metadata.setDouble("CD2_2", -5.10281493481982E-05)
485 metadata.setDouble("CD2_1", -8.27440751733828E-07)
487 return afwGeom.makeSkyWcs(metadata)
490class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase):
491 pass
494def setup_module(module):
495 lsst.utils.tests.init()
498if __name__ == "__main__": 498 ↛ 499line 498 didn't jump to line 499, because the condition on line 498 was never true
499 lsst.utils.tests.init()
500 unittest.main()