Coverage for tests/test_diaPipe.py: 25%

1# This file is part of ap_association. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23import numpy as np 

24import pandas as pd 

25 

26import lsst.afw.image as afwImage 

27import lsst.afw.table as afwTable 

28from lsst.pipe.base.testUtils import assertValidOutput 

29import lsst.utils.tests 

30import lsst.utils.timer 

31from unittest.mock import patch, Mock, MagicMock, DEFAULT 

32 

33from lsst.ap.association import DiaPipelineTask 

34 

35 

36class TestDiaPipelineTask(unittest.TestCase): 

37 

38 @classmethod 

39 def _makeDefaultConfig(cls, 

40 doPackageAlerts=False, 

41 doSolarSystemAssociation=False): 

42 config = DiaPipelineTask.ConfigClass() 

43 config.apdb.db_url = "sqlite://" 

44 config.doPackageAlerts = doPackageAlerts 

45 config.doSolarSystemAssociation = doSolarSystemAssociation 

46 return config 

47 

48 def setUp(self): 

49 # schemas are persisted in both Gen 2 and Gen 3 butler as prototypical catalogs 

50 srcSchema = afwTable.SourceTable.makeMinimalSchema() 

51 srcSchema.addField("base_PixelFlags_flag", type="Flag") 

52 srcSchema.addField("base_PixelFlags_flag_offimage", type="Flag") 

53 self.srcSchema = afwTable.SourceCatalog(srcSchema) 

54 

55 def tearDown(self): 

56 pass 

57 

58 def testRun(self): 

59 """Test running while creating and packaging alerts. 

60 """ 

61 self._testRun(doPackageAlerts=True, doSolarSystemAssociation=True) 

62 

63 def testRunWithSolarSystemAssociation(self): 

64 """Test running while creating and packaging alerts. 

65 """ 

66 self._testRun(doPackageAlerts=False, doSolarSystemAssociation=True) 

67 

68 def testRunWithAlerts(self): 

69 """Test running while creating and packaging alerts. 

70 """ 

71 self._testRun(doPackageAlerts=True, doSolarSystemAssociation=False) 

72 

73 def testRunWithoutAlertsOrSolarSystem(self): 

74 """Test running without creating and packaging alerts. 

75 """ 

76 self._testRun(doPackageAlerts=False, doSolarSystemAssociation=False) 

77 

78 def _testRun(self, doPackageAlerts=False, doSolarSystemAssociation=False): 

79 """Test the normal workflow of each ap_pipe step. 

80 """ 

81 config = self._makeDefaultConfig( 

82 doPackageAlerts=doPackageAlerts, 

83 doSolarSystemAssociation=doSolarSystemAssociation) 

84 task = DiaPipelineTask(config=config) 

85 # Set DataFrame index testing to always return False. Mocks return 

86 # true for this check otherwise. 

87 task.testDataFrameIndex = lambda x: False 

88 diffIm = Mock(spec=afwImage.ExposureF) 

89 exposure = Mock(spec=afwImage.ExposureF) 

90 template = Mock(spec=afwImage.ExposureF) 

91 diaSrc = MagicMock(spec=pd.DataFrame()) 

92 ssObjects = MagicMock(spec=pd.DataFrame()) 

93 ccdExposureIdBits = 32 

94 

95 # Each of these subtasks should be called once during diaPipe 

96 # execution. We use mocks here to check they are being executed 

97 # appropriately. 

98 subtasksToMock = [ 

99 "diaCatalogLoader", 

100 "diaCalculation", 

101 "diaForcedSource", 

102 ] 

103 if doPackageAlerts: 

104 subtasksToMock.append("alertPackager") 

105 else: 

106 self.assertFalse(hasattr(task, "alertPackager")) 

107 

108 if not doSolarSystemAssociation: 

109 self.assertFalse(hasattr(task, "solarSystemAssociator")) 

110 

111 def concatMock(_data, **_kwargs): 

112 return MagicMock(spec=pd.DataFrame) 

113 

114 # Mock out the run() methods of these two Tasks to ensure they 

115 # return data in the correct form. 

116 @lsst.utils.timer.timeMethod 

117 def solarSystemAssociator_run(self, unAssocDiaSources, solarSystemObjectTable, diffIm): 

118 return lsst.pipe.base.Struct(nTotalSsObjects=42, 

119 nAssociatedSsObjects=30, 

120 ssoAssocDiaSources=MagicMock(spec=pd.DataFrame()), 

121 unAssocDiaSources=MagicMock(spec=pd.DataFrame())) 

122 

123 @lsst.utils.timer.timeMethod 

124 def associator_run(self, table, diaObjects, exposure_time=None): 

125 return lsst.pipe.base.Struct(nUpdatedDiaObjects=2, nUnassociatedDiaObjects=3, 

126 matchedDiaSources=MagicMock(spec=pd.DataFrame()), 

127 unAssocDiaSources=MagicMock(spec=pd.DataFrame())) 

128 

129 # apdb isn't a subtask, but still needs to be mocked out for correct 

130 # execution in the test environment. 

131 with patch.multiple( 

132 task, **{task: DEFAULT for task in subtasksToMock + ["apdb"]} 

133 ): 

134 with patch('lsst.ap.association.diaPipe.pd.concat', new=concatMock), \ 

135 patch('lsst.ap.association.association.AssociationTask.run', new=associator_run), \ 

136 patch('lsst.ap.association.ssoAssociation.SolarSystemAssociationTask.run', 

137 new=solarSystemAssociator_run): 

138 

139 result = task.run(diaSrc, 

140 ssObjects, 

141 diffIm, 

142 exposure, 

143 template, 

144 ccdExposureIdBits, 

145 "g") 

146 for subtaskName in subtasksToMock: 

147 getattr(task, subtaskName).run.assert_called_once() 

148 assertValidOutput(task, result) 

149 self.assertEqual(result.apdbMarker.db_url, "sqlite://") 

150 meta = task.getFullMetadata() 

151 # Check that the expected metadata has been set. 

152 self.assertEqual(meta["diaPipe.numUpdatedDiaObjects"], 2) 

153 self.assertEqual(meta["diaPipe.numUnassociatedDiaObjects"], 3) 

154 # and that associators ran once or not at all. 

155 self.assertEqual(len(meta.getArray("diaPipe:associator.associator_runEndUtc")), 1) 

156 if doSolarSystemAssociation: 

157 self.assertEqual(len(meta.getArray("diaPipe:solarSystemAssociator." 

158 "solarSystemAssociator_runEndUtc")), 1) 

159 else: 

160 self.assertNotIn("diaPipe:solarSystemAssociator", meta) 

161 

162 def test_createDiaObjects(self): 

163 """Test that creating new DiaObjects works as expected. 

164 """ 

165 nSources = 5 

166 diaSources = pd.DataFrame(data=[ 

167 {"ra": 0.04*idx, "dec": 0.04*idx, 

168 "diaSourceId": idx + 1 + nSources, "diaObjectId": 0, 

169 "ssObjectId": 0} 

170 for idx in range(nSources)]) 

171 

172 config = self._makeDefaultConfig(doPackageAlerts=False) 

173 task = DiaPipelineTask(config=config) 

174 result = task.createNewDiaObjects(diaSources) 

175 self.assertEqual(nSources, len(result.newDiaObjects)) 

176 self.assertTrue(np.all(np.equal( 

177 result.diaSources["diaObjectId"].to_numpy(), 

178 result.diaSources["diaSourceId"].to_numpy()))) 

179 self.assertTrue(np.all(np.equal( 

180 result.newDiaObjects["diaObjectId"].to_numpy(), 

181 result.diaSources["diaSourceId"].to_numpy()))) 

182 

183 

184class MemoryTester(lsst.utils.tests.MemoryTestCase): 

185 pass 

186 

187 

188def setup_module(module): 

189 lsst.utils.tests.init() 

190 

191 

192if __name__ == "__main__": 192 ↛ 193line 192 didn't jump to line 193, because the condition on line 192 was never true

193 lsst.utils.tests.init() 

194 unittest.main()