Coverage for tests/test_efdUtils.py: 27%

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Test cases for utils.""" 

23 

24import asyncio 

25import datetime 

26import unittest 

27 

28import astropy 

29import pandas as pd 

30from astropy.time import Time 

31from utils import getVcr 

32 

33import lsst.utils.tests 

34from lsst.summit.utils.efdUtils import ( 

35 astropyToEfdTimestamp, 

36 clipDataToEvent, 

37 efdTimestampToAstropy, 

38 getDayObsEndTime, 

39 getDayObsForTime, 

40 getDayObsStartTime, 

41 getEfdData, 

42 getMostRecentRowWithDataBefore, 

43 getTopics, 

44 makeEfdClient, 

45) 

46from lsst.summit.utils.tmaUtils import TMAEvent, TMAState 

47 

48HAS_EFD_CLIENT = True 

49try: 

50 import lsst_efd_client 

51except ImportError: 

52 HAS_EFD_CLIENT = False 

53 

54vcr = getVcr() 

55 

56 

57@unittest.skipIf(not HAS_EFD_CLIENT, "No EFD client available") 

58@vcr.use_cassette() 

59class EfdUtilsTestCase(lsst.utils.tests.TestCase): 

60 @classmethod 

61 @vcr.use_cassette() 

62 def setUpClass(cls): 

63 try: 

64 cls.client = makeEfdClient(testing=True) 

65 except RuntimeError: 

66 raise unittest.SkipTest("Could not instantiate an EFD client") 

67 cls.dayObs = 20230531 

68 # get a sample expRecord here to test expRecordToTimespan 

69 cls.axisTopic = "lsst.sal.MTMount.logevent_azimuthMotionState" 

70 cls.timeSeriesTopic = "lsst.sal.MTMount.azimuth" 

71 cls.event = TMAEvent( 

72 dayObs=20230531, 

73 seqNum=27, 

74 type=TMAState.TRACKING, 

75 endReason=TMAState.SLEWING, 

76 duration=0.47125244140625, 

77 begin=Time(1685578353.2265284, scale="utc", format="unix"), 

78 end=Time(1685578353.6977808, scale="utc", format="unix"), 

79 blockInfos=None, 

80 version=0, 

81 _startRow=254, 

82 _endRow=255, 

83 ) 

84 

85 @vcr.use_cassette() 

86 def tearDown(self): 

87 loop = asyncio.get_event_loop() 

88 if self.client.influx_client is not None: 

89 loop.run_until_complete(self.client.influx_client.close()) 

90 

91 @vcr.use_cassette() 

92 def test_makeEfdClient(self): 

93 self.assertIsInstance(self.client, lsst_efd_client.efd_helper.EfdClient) 

94 

95 def test_getDayObsAsTimes(self): 

96 """This tests getDayObsStartTime and getDayObsEndTime explicitly, 

97 but the days we loop over are chosen to test calcNextDay() which is 

98 called by getDayObsEndTime(). 

99 """ 

100 for dayObs in ( 

101 self.dayObs, # the nominal value 

102 20200228, # day before end of Feb on a leap year 

103 20200229, # end of Feb on a leap year 

104 20210227, # day before end of Feb on a non-leap year 

105 20200228, # end of Feb on a non-leap year 

106 20200430, # end of a month with 30 days 

107 20200530, # end of a month with 31 days 

108 20201231, # year rollover 

109 ): 

110 dayStart = getDayObsStartTime(dayObs) 

111 self.assertIsInstance(dayStart, astropy.time.Time) 

112 

113 dayEnd = getDayObsEndTime(dayObs) 

114 self.assertIsInstance(dayStart, astropy.time.Time) 

115 

116 self.assertGreater(dayEnd, dayStart) 

117 self.assertEqual(dayEnd.jd, dayStart.jd + 1) 

118 

119 @vcr.use_cassette() 

120 def test_getTopics(self): 

121 topics = getTopics(self.client, "lsst.sal.MTMount*") 

122 self.assertIsInstance(topics, list) 

123 self.assertGreater(len(topics), 0) 

124 

125 topics = getTopics(self.client, "*fake.topics.does.not.exist*") 

126 self.assertIsInstance(topics, list) 

127 self.assertEqual(len(topics), 0) 

128 

129 # check we can find the mount with a preceding wildcard 

130 topics = getTopics(self.client, "*mTmoUnt*") 

131 self.assertIsInstance(topics, list) 

132 self.assertGreater(len(topics), 0) 

133 

134 # check it fails if we don't allow case insensitivity 

135 topics = getTopics(self.client, "*mTmoUnt*", caseSensitive=True) 

136 self.assertIsInstance(topics, list) 

137 self.assertEqual(len(topics), 0) 

138 

139 @vcr.use_cassette() 

140 def test_getEfdData(self): 

141 dayStart = getDayObsStartTime(self.dayObs) 

142 dayEnd = getDayObsEndTime(self.dayObs) 

143 oneDay = datetime.timedelta(hours=24) 

144 # twelveHours = datetime.timedelta(hours=12) 

145 

146 # test the dayObs interface 

147 dayObsData = getEfdData(self.client, self.axisTopic, dayObs=self.dayObs) 

148 self.assertIsInstance(dayObsData, pd.DataFrame) 

149 

150 # test the starttime interface 

151 dayStartData = getEfdData(self.client, self.axisTopic, begin=dayStart, timespan=oneDay) 

152 self.assertIsInstance(dayStartData, pd.DataFrame) 

153 

154 # check they're equal 

155 self.assertTrue(dayObsData.equals(dayStartData)) 

156 

157 # test the starttime interface with an endtime 

158 dayEnd = getDayObsEndTime(self.dayObs) 

159 dayStartEndData = getEfdData(self.client, self.axisTopic, begin=dayStart, end=dayEnd) 

160 self.assertTrue(dayObsData.equals(dayStartEndData)) 

161 

162 # test event 

163 # note that here we're going to clip to an event and pad things, so 

164 # we want to use the timeSeriesTopic not the states, so that there's 

165 # plenty of rows to test the padding is actually working 

166 eventData = getEfdData(self.client, self.timeSeriesTopic, event=self.event) 

167 self.assertIsInstance(dayObsData, pd.DataFrame) 

168 

169 # test padding options 

170 padded = getEfdData(self.client, self.timeSeriesTopic, event=self.event, prePadding=1, postPadding=2) 

171 self.assertGreater(len(padded), len(eventData)) 

172 startTimeDiff = efdTimestampToAstropy(eventData.iloc[0]["private_efdStamp"]) - efdTimestampToAstropy( 

173 padded.iloc[0]["private_efdStamp"] 

174 ) 

175 endTimeDiff = efdTimestampToAstropy(padded.iloc[-1]["private_efdStamp"]) - efdTimestampToAstropy( 

176 eventData.iloc[-1]["private_efdStamp"] 

177 ) 

178 

179 self.assertGreater(startTimeDiff.sec, 0) 

180 self.assertLess(startTimeDiff.sec, 1.1) # padding isn't super exact, so give a little wiggle room 

181 self.assertGreater(endTimeDiff.sec, 0) 

182 self.assertLess(endTimeDiff.sec, 2.1) # padding isn't super exact, so give a little wiggle room 

183 

184 with self.assertRaises(ValueError): 

185 # not enough info to constrain 

186 _ = getEfdData(self.client, self.axisTopic) 

187 # dayObs supplied and a start time is not allowed 

188 _ = getEfdData(self.client, self.axisTopic, dayObs=self.dayObs, begin=dayStart) 

189 # dayObs supplied and a stop time is not allowed 

190 _ = getEfdData(self.client, self.axisTopic, dayObs=self.dayObs, end=dayEnd) 

191 # dayObs supplied and timespan is not allowed 

192 _ = getEfdData(self.client, self.axisTopic, dayObs=self.dayObs, timespan=oneDay) 

193 # being alone is not allowed 

194 _ = getEfdData(self.client, self.axisTopic, begin=self.dayObs) 

195 # good query, except the topic doesn't exist 

196 _ = getEfdData(self.client, "badTopic", begin=dayStart, end=dayEnd) 

197 

198 @vcr.use_cassette() 

199 def test_raiseIfTopicNotInSchema(self): 

200 dayStart = getDayObsStartTime(self.dayObs) 

201 dayEnd = getDayObsEndTime(self.dayObs) 

202 

203 badTopic = "lsst.sal.nonExistentTopic" 

204 # test this does not raise 

205 _ = getEfdData(self.client, badTopic, begin=dayStart, end=dayEnd, raiseIfTopicNotInSchema=False) 

206 

207 with self.assertRaises(ValueError): 

208 # test this does raise, as raiseIfTopicNotInSchema defaults to True 

209 _ = getEfdData(self.client, badTopic, begin=dayStart, end=dayEnd) 

210 

211 @vcr.use_cassette() 

212 def test_getMostRecentRowWithDataBefore(self): 

213 time = Time(1687845854.736784, scale="utc", format="unix") 

214 rowData = getMostRecentRowWithDataBefore( 

215 self.client, "lsst.sal.MTM1M3.logevent_forceActuatorState", time 

216 ) 

217 self.assertIsInstance(rowData, pd.Series) 

218 

219 stateTime = efdTimestampToAstropy(rowData["private_efdStamp"]) 

220 self.assertLess(stateTime, time) 

221 

222 def test_efdTimestampToAstropy(self): 

223 time = efdTimestampToAstropy(1687845854.736784) 

224 self.assertIsInstance(time, astropy.time.Time) 

225 return 

226 

227 def test_astropyToEfdTimestamp(self): 

228 time = Time(1687845854.736784, scale="utc", format="unix") 

229 efdTimestamp = astropyToEfdTimestamp(time) 

230 self.assertIsInstance(efdTimestamp, float) 

231 return 

232 

233 @vcr.use_cassette() 

234 def test_clipDataToEvent(self): 

235 # get 10 mins of data either side of the event we'll clip to 

236 duration = datetime.timedelta(seconds=10 * 60) 

237 queryBegin = self.event.begin - duration 

238 queryEnd = self.event.end + duration 

239 dayObsData = getEfdData(self.client, "lsst.sal.MTMount.azimuth", begin=queryBegin, end=queryEnd) 

240 

241 # clip the data, and check it's shorter, non-zero, and falls in the 

242 # right time range 

243 clippedData = clipDataToEvent(dayObsData, self.event) 

244 

245 self.assertIsInstance(clippedData, pd.DataFrame) 

246 self.assertGreater(len(clippedData), 0) 

247 self.assertLess(len(clippedData), len(dayObsData)) 

248 

249 dataStart = efdTimestampToAstropy(clippedData.iloc[0]["private_efdStamp"]) 

250 dataEnd = efdTimestampToAstropy(clippedData.iloc[-1]["private_efdStamp"]) 

251 

252 self.assertGreaterEqual(dataStart, self.event.begin) 

253 self.assertLessEqual(dataEnd, self.event.end) 

254 

255 # test the pre/post padding options 

256 clippedPaddedData = clipDataToEvent(dayObsData, self.event, prePadding=1, postPadding=2) 

257 self.assertIsInstance(clippedPaddedData, pd.DataFrame) 

258 self.assertGreater(len(clippedPaddedData), 0) 

259 self.assertLess(len(clippedPaddedData), len(dayObsData)) 

260 self.assertGreater(len(clippedPaddedData), len(clippedData)) 

261 

262 paddedDataStart = efdTimestampToAstropy(clippedPaddedData.iloc[0]["private_efdStamp"]) 

263 paddedDataEnd = efdTimestampToAstropy(clippedPaddedData.iloc[-1]["private_efdStamp"]) 

264 self.assertLessEqual(paddedDataStart, dataStart) 

265 self.assertGreaterEqual(paddedDataEnd, dataEnd) 

266 

267 # Get the minimum and maximum timestamps before padding 

268 startTimeUnpadded = clippedData["private_efdStamp"].min() 

269 endTimeUnpadded = clippedData["private_efdStamp"].max() 

270 

271 # Get the minimum and maximum timestamps after padding 

272 startTimePadded = clippedPaddedData["private_efdStamp"].min() 

273 endTimePadded = clippedPaddedData["private_efdStamp"].max() 

274 

275 # Check that the difference between the min times and max times is 

276 # approximately equal to the padding. Not exact as data sampling is 

277 # not infinite. 

278 self.assertAlmostEqual(startTimeUnpadded - startTimePadded, 1, delta=0.1) 

279 self.assertAlmostEqual(endTimePadded - endTimeUnpadded, 2, delta=0.1) 

280 return 

281 

282 def test_getDayObsForTime(self): 

283 pydate = datetime.datetime(2023, 2, 5, 13, 30, 1) 

284 time = Time(pydate) 

285 dayObs = getDayObsForTime(time) 

286 self.assertEqual(dayObs, 20230205) 

287 

288 pydate = datetime.datetime(2023, 2, 5, 11, 30, 1) 

289 time = Time(pydate) 

290 dayObs = getDayObsForTime(time) 

291 self.assertEqual(dayObs, 20230204) 

292 return 

293 

294 

295class TestMemory(lsst.utils.tests.MemoryTestCase): 

296 pass 

297 

298 

299def setup_module(module): 

300 lsst.utils.tests.init() 

301 

302 

303if __name__ == "__main__": 303 ↛ 304line 303 didn't jump to line 304 because the condition on line 303 was never true

304 lsst.utils.tests.init() 

305 unittest.main()