Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24import pandas as pd 

25 

26import lsst.utils.tests 

27 

28# TODO: Remove skipUnless and this try block DM-22256 

29try: 

30 import pyarrow as pa 

31 import pyarrow.parquet as pq 

32 from lsst.pipe.tasks.parquetTable import MultilevelParquetTable 

33 from lsst.pipe.tasks.functors import HsmFwhm 

34 from lsst.pipe.tasks.postprocess import TransformObjectCatalogTask, TransformObjectCatalogConfig 

35 havePyArrow = True 

36except ImportError: 

37 havePyArrow = False 

38 

39ROOT = os.path.abspath(os.path.dirname(__file__)) 

40 

41 

42def setup_module(module): 

43 lsst.utils.tests.init() 

44 

45 

46@unittest.skipUnless(havePyArrow, "Requires pyarrow") 

47class TransformObjectCatalogTestCase(unittest.TestCase): 

48 def setUp(self): 

49 # Note that this test input includes HSC-G, HSC-R, and HSC-I data 

50 df = pd.read_csv(os.path.join(ROOT, 'data', 'test_multilevel_parq.csv.gz'), 

51 header=[0, 1, 2], index_col=0) 

52 with lsst.utils.tests.getTempFilePath('*.parq') as filename: 

53 table = pa.Table.from_pandas(df) 

54 pq.write_table(table, filename, compression='none') 

55 self.parq = MultilevelParquetTable(filename) 

56 

57 self.dataId = {"tract": 9615, "patch": "4,4"} 

58 

59 def testNullFilter(self): 

60 """Test that columns for all filters are created despite they may not 

61 exist in the input data. 

62 """ 

63 config = TransformObjectCatalogConfig() 

64 # Want y band columns despite the input data do not have them 

65 # Exclude g band columns despite the input data have them 

66 filterMap = {"HSC-R": "r", "HSC-I": "i", "HSC-Y": "y"} 

67 config.filterMap = filterMap 

68 task = TransformObjectCatalogTask(config=config) 

69 funcs = {'Fwhm': HsmFwhm(dataset='meas')} 

70 df = task.run(self.parq, funcs=funcs, dataId=self.dataId) 

71 self.assertIsInstance(df, pd.DataFrame) 

72 for column in ('coord_ra', 'coord_dec'): 

73 self.assertIn(column, df.columns) 

74 

75 for filt in filterMap.values(): 

76 self.assertIn(filt + 'Fwhm', df.columns) 

77 

78 self.assertNotIn('gFwhm', df.columns) 

79 self.assertTrue(df['yFwhm'].isnull().all()) 

80 self.assertTrue(df['iFwhm'].notnull().all()) 

81 

82 def testUnderscoreColumnFormat(self): 

83 """Test the per-filter column format with an underscore""" 

84 config = TransformObjectCatalogConfig() 

85 filterMap = {"HSC-G": "g", "HSC-R": "r", "HSC-I": "i"} 

86 config.filterMap = filterMap 

87 config.camelCase = False 

88 task = TransformObjectCatalogTask(config=config) 

89 funcs = {'Fwhm': HsmFwhm(dataset='meas')} 

90 df = task.run(self.parq, funcs=funcs, dataId=self.dataId) 

91 self.assertIsInstance(df, pd.DataFrame) 

92 for filt in filterMap.values(): 

93 self.assertIn(filt + '_Fwhm', df.columns) 

94 

95 def testMultilevelOutput(self): 

96 """Test the non-flattened result dataframe with a multilevel column index""" 

97 config = TransformObjectCatalogConfig() 

98 filterMap = {"HSC-R": "r", "HSC-I": "i"} 

99 config.filterMap = filterMap 

100 config.multilevelOutput = True 

101 task = TransformObjectCatalogTask(config=config) 

102 funcs = {'Fwhm': HsmFwhm(dataset='meas')} 

103 df = task.run(self.parq, funcs=funcs, dataId=self.dataId) 

104 self.assertIsInstance(df, pd.DataFrame) 

105 self.assertNotIn('HSC-G', df) 

106 for filt in filterMap: 

107 self.assertIsInstance(df[filt], pd.DataFrame) 

108 self.assertIn('Fwhm', df[filt].columns)