Coverage for tests/test_transformObject.py : 28%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import unittest
24import pandas as pd
26import lsst.utils.tests
28# TODO: Remove skipUnless and this try block DM-22256
29try:
30 import pyarrow as pa
31 import pyarrow.parquet as pq
32 from lsst.pipe.tasks.parquetTable import MultilevelParquetTable
33 from lsst.pipe.tasks.functors import HsmFwhm
34 from lsst.pipe.tasks.postprocess import TransformObjectCatalogTask, TransformObjectCatalogConfig
35 havePyArrow = True
36except ImportError:
37 havePyArrow = False
39ROOT = os.path.abspath(os.path.dirname(__file__))
42def setup_module(module):
43 lsst.utils.tests.init()
46@unittest.skipUnless(havePyArrow, "Requires pyarrow")
47class TransformObjectCatalogTestCase(unittest.TestCase):
48 def setUp(self):
49 # Note that this test input includes HSC-G, HSC-R, and HSC-I data
50 df = pd.read_csv(os.path.join(ROOT, 'data', 'test_multilevel_parq.csv.gz'),
51 header=[0, 1, 2], index_col=0)
52 with lsst.utils.tests.getTempFilePath('*.parq') as filename:
53 table = pa.Table.from_pandas(df)
54 pq.write_table(table, filename, compression='none')
55 self.parq = MultilevelParquetTable(filename)
57 self.dataId = {"tract": 9615, "patch": "4,4"}
59 def testNullFilter(self):
60 """Test that columns for all filters are created despite they may not
61 exist in the input data.
62 """
63 config = TransformObjectCatalogConfig()
64 # Want y band columns despite the input data do not have them
65 # Exclude g band columns despite the input data have them
66 filterMap = {"HSC-R": "r", "HSC-I": "i", "HSC-Y": "y"}
67 config.filterMap = filterMap
68 task = TransformObjectCatalogTask(config=config)
69 funcs = {'Fwhm': HsmFwhm(dataset='meas')}
70 df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
71 self.assertIsInstance(df, pd.DataFrame)
72 for column in ('coord_ra', 'coord_dec'):
73 self.assertIn(column, df.columns)
75 for filt in filterMap.values():
76 self.assertIn(filt + 'Fwhm', df.columns)
78 self.assertNotIn('gFwhm', df.columns)
79 self.assertTrue(df['yFwhm'].isnull().all())
80 self.assertTrue(df['iFwhm'].notnull().all())
82 def testUnderscoreColumnFormat(self):
83 """Test the per-filter column format with an underscore"""
84 config = TransformObjectCatalogConfig()
85 filterMap = {"HSC-G": "g", "HSC-R": "r", "HSC-I": "i"}
86 config.filterMap = filterMap
87 config.camelCase = False
88 task = TransformObjectCatalogTask(config=config)
89 funcs = {'Fwhm': HsmFwhm(dataset='meas')}
90 df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
91 self.assertIsInstance(df, pd.DataFrame)
92 for filt in filterMap.values():
93 self.assertIn(filt + '_Fwhm', df.columns)
95 def testMultilevelOutput(self):
96 """Test the non-flattened result dataframe with a multilevel column index"""
97 config = TransformObjectCatalogConfig()
98 filterMap = {"HSC-R": "r", "HSC-I": "i"}
99 config.filterMap = filterMap
100 config.multilevelOutput = True
101 task = TransformObjectCatalogTask(config=config)
102 funcs = {'Fwhm': HsmFwhm(dataset='meas')}
103 df = task.run(self.parq, funcs=funcs, dataId=self.dataId)
104 self.assertIsInstance(df, pd.DataFrame)
105 self.assertNotIn('HSC-G', df)
106 for filt in filterMap:
107 self.assertIsInstance(df[filt], pd.DataFrame)
108 self.assertIn('Fwhm', df[filt].columns)