Coverage for python/lsst/summit/utils/efdUtils.py: 17%

150 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-23 14:24 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import asyncio 

23from astropy.time import Time, TimeDelta 

24from astropy import units as u 

25import datetime 

26import logging 

27import pandas as pd 

28import re 

29from deprecated.sphinx import deprecated 

30 

31from .utils import getSite 

32 

33HAS_EFD_CLIENT = True 

34try: 

35 from lsst_efd_client import EfdClient 

36except ImportError: 

37 HAS_EFD_CLIENT = False 

38 

39__all__ = [ 

40 'getEfdData', 

41 'getMostRecentRowWithDataBefore', 

42 'makeEfdClient', 

43 'expRecordToTimespan', 

44 'efdTimestampToAstropy', 

45 'astropyToEfdTimestamp', 

46 'clipDataToEvent', 

47 'calcNextDay', 

48 'getDayObsStartTime', 

49 'getDayObsEndTime', 

50 'getDayObsForTime', 

51 'getSubTopics', # deprecated, being removed in w_2023_50 

52 'getTopics', 

53] 

54 

55 

56COMMAND_ALIASES = { 

57 'raDecTarget': 'lsst.sal.MTPtg.command_raDecTarget', 

58 'moveToTarget': 'lsst.sal.MTMount.command_moveToTarget', 

59 'startTracking': 'lsst.sal.MTMount.command_startTracking', 

60 'stopTracking': 'lsst.sal.MTMount.command_stopTracking', 

61 'trackTarget': 'lsst.sal.MTMount.command_trackTarget', # issued at 20Hz - don't plot 

62} 

63 

64# When looking backwards in time to find the most recent state event, look back 

65# in chunks of this size. Too small, and there will be too many queries, too 

66# large and there will be too much data returned unnecessarily, as we only need 

67# one row by definition. Will tune this parameters in consultation with SQuaRE. 

68TIME_CHUNKING = datetime.timedelta(minutes=15) 

69 

70 

71def _getBeginEnd(dayObs=None, begin=None, end=None, timespan=None, event=None, expRecord=None): 

72 """Calculate the begin and end times to pass to _getEfdData, given the 

73 kwargs passed to getEfdData. 

74 

75 Parameters 

76 ---------- 

77 dayObs : `int` 

78 The dayObs to query. If specified, this is used to determine the begin 

79 and end times. 

80 begin : `astropy.Time` 

81 The begin time for the query. If specified, either an end time or a 

82 timespan must be supplied. 

83 end : `astropy.Time` 

84 The end time for the query. If specified, a begin time must also be 

85 supplied. 

86 timespan : `astropy.TimeDelta` 

87 The timespan for the query. If specified, a begin time must also be 

88 supplied. 

89 event : `lsst.summit.utils.efdUtils.TmaEvent` 

90 The event to query. If specified, this is used to determine the begin 

91 and end times, and all other options are disallowed. 

92 expRecord : `lsst.daf.butler.dimensions.DimensionRecord` 

93 The exposure record containing the timespan to query. If specified, all 

94 other options are disallowed. 

95 

96 Returns 

97 ------- 

98 begin : `astropy.Time` 

99 The begin time for the query. 

100 end : `astropy.Time` 

101 The end time for the query. 

102 """ 

103 if expRecord is not None: 

104 forbiddenOpts = [event, begin, end, timespan, dayObs] 

105 if any(x is not None for x in forbiddenOpts): 

106 raise ValueError("You can't specify both an expRecord and a begin/end or timespan or dayObs") 

107 begin = expRecord.timespan.begin 

108 end = expRecord.timespan.end 

109 return begin, end 

110 

111 if event is not None: 

112 forbiddenOpts = [begin, end, timespan, dayObs] 

113 if any(x is not None for x in forbiddenOpts): 

114 raise ValueError("You can't specify both an event and a begin/end or timespan or dayObs") 

115 begin = event.begin 

116 end = event.end 

117 return begin, end 

118 

119 # check for dayObs, and that other options aren't inconsistently specified 

120 if dayObs is not None: 

121 forbiddenOpts = [begin, end, timespan] 

122 if any(x is not None for x in forbiddenOpts): 

123 raise ValueError("You can't specify both a dayObs and a begin/end or timespan") 

124 begin = getDayObsStartTime(dayObs) 

125 end = getDayObsEndTime(dayObs) 

126 return begin, end 

127 # can now disregard dayObs entirely 

128 

129 if begin is None: 

130 raise ValueError("You must specify either a dayObs or a begin/end or begin/timespan") 

131 # can now rely on begin, so just need to deal with end/timespan 

132 

133 if end is None and timespan is None: 

134 raise ValueError("If you specify a begin, you must specify either a end or a timespan") 

135 if end is not None and timespan is not None: 

136 raise ValueError("You can't specify both a end and a timespan") 

137 if end is None: 

138 if timespan > datetime.timedelta(minutes=0): 

139 end = begin + timespan # the normal case 

140 else: 

141 end = begin # the case where timespan is negative 

142 begin = begin + timespan # adding the negative to the start, i.e. subtracting it to bring back 

143 

144 assert begin is not None 

145 assert end is not None 

146 return begin, end 

147 

148 

149def getEfdData(client, topic, *, 

150 columns=None, 

151 prePadding=0, 

152 postPadding=0, 

153 dayObs=None, 

154 begin=None, 

155 end=None, 

156 timespan=None, 

157 event=None, 

158 expRecord=None, 

159 warn=True, 

160 ): 

161 """Get one or more EFD topics over a time range, synchronously. 

162 

163 The time range can be specified as either: 

164 * a dayObs, in which case the full 24 hour period is used, 

165 * a begin point and a end point, 

166 * a begin point and a timespan. 

167 * a mount event 

168 * an exposure record 

169 If it is desired to use an end time with a timespan, just specify it as the 

170 begin time and use a negative timespan. 

171 

172 The results from all topics are merged into a single dataframe. 

173 

174 Parameters 

175 ---------- 

176 client : `lsst_efd_client.efd_helper.EfdClient` 

177 The EFD client to use. 

178 topic : `str` 

179 The topic to query. 

180 columns : `list` of `str`, optional 

181 The columns to query. If not specified, all columns are queried. 

182 prePadding : `float` 

183 The amount of time before the nominal start of the query to include, in 

184 seconds. 

185 postPadding : `float` 

186 The amount of extra time after the nominal end of the query to include, 

187 in seconds. 

188 dayObs : `int`, optional 

189 The dayObs to query. If specified, this is used to determine the begin 

190 and end times. 

191 begin : `astropy.Time`, optional 

192 The begin time for the query. If specified, either a end time or a 

193 timespan must be supplied. 

194 end : `astropy.Time`, optional 

195 The end time for the query. If specified, a begin time must also be 

196 supplied. 

197 timespan : `astropy.TimeDelta`, optional 

198 The timespan for the query. If specified, a begin time must also be 

199 supplied. 

200 event : `lsst.summit.utils.efdUtils.TmaEvent`, optional 

201 The event to query. If specified, this is used to determine the begin 

202 and end times, and all other options are disallowed. 

203 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`, optional 

204 The exposure record containing the timespan to query. If specified, all 

205 other options are disallowed. 

206 warn : bool, optional 

207 If ``True``, warn when no data is found. Exists so that utility code 

208 can disable warnings when checking for data, and therefore defaults to 

209 ``True``. 

210 

211 Returns 

212 ------- 

213 data : `pd.DataFrame` 

214 The merged data from all topics. 

215 

216 Raises 

217 ------ 

218 ValueError: 

219 If the topics are not in the EFD schema. 

220 ValueError: 

221 If both a dayObs and a begin/end or timespan are specified. 

222 ValueError: 

223 If a begin time is specified but no end time or timespan. 

224 

225 """ 

226 # TODO: DM-40100 ideally should calls mpts as necessary so that users 

227 # needn't care if things are packed 

228 

229 # supports aliases so that you can query with them. If there is no entry in 

230 # the alias dict then it queries with the supplied key. The fact the schema 

231 # is now being checked means this shouldn't be a problem now. 

232 

233 # TODO: RFC-948 Move this import back to top of file once is implemented. 

234 import nest_asyncio 

235 

236 begin, end = _getBeginEnd(dayObs, begin, end, timespan, event, expRecord) 

237 begin -= TimeDelta(prePadding, format='sec') 

238 end += TimeDelta(postPadding, format='sec') 

239 

240 nest_asyncio.apply() 

241 loop = asyncio.get_event_loop() 

242 ret = loop.run_until_complete(_getEfdData(client=client, 

243 topic=topic, 

244 begin=begin, 

245 end=end, 

246 columns=columns)) 

247 if ret.empty and warn: 

248 log = logging.getLogger(__name__) 

249 log.warning(f"Topic {topic} is in the schema, but no data was returned by the query for the specified" 

250 " time range") 

251 return ret 

252 

253 

254async def _getEfdData(client, topic, begin, end, columns=None): 

255 """Get data for a topic from the EFD over the specified time range. 

256 

257 Parameters 

258 ---------- 

259 client : `lsst_efd_client.efd_helper.EfdClient` 

260 The EFD client to use. 

261 topic : `str` 

262 The topic to query. 

263 begin : `astropy.Time` 

264 The begin time for the query. 

265 end : `astropy.Time` 

266 The end time for the query. 

267 columns : `list` of `str`, optional 

268 The columns to query. If not specified, all columns are returned. 

269 

270 Returns 

271 ------- 

272 data : `pd.DataFrame` 

273 The data from the query. 

274 """ 

275 if columns is None: 

276 columns = ['*'] 

277 

278 availableTopics = await client.get_topics() 

279 

280 if topic not in availableTopics: 

281 raise ValueError(f"Topic {topic} not in EFD schema") 

282 

283 data = await client.select_time_series(topic, columns, begin.utc, end.utc) 

284 

285 return data 

286 

287 

288def getMostRecentRowWithDataBefore(client, topic, timeToLookBefore, warnStaleAfterNMinutes=60*12): 

289 """Get the most recent row of data for a topic before a given time. 

290 

291 Parameters 

292 ---------- 

293 client : `lsst_efd_client.efd_helper.EfdClient` 

294 The EFD client to use. 

295 topic : `str` 

296 The topic to query. 

297 timeToLookBefore : `astropy.Time` 

298 The time to look before. 

299 warnStaleAfterNMinutes : `float`, optional 

300 The number of minutes after which to consider the data stale and issue 

301 a warning. 

302 

303 Returns 

304 ------- 

305 row : `pd.Series` 

306 The row of data from the EFD containing the most recent data before the 

307 specified time. 

308 

309 Raises 

310 ------ 

311 ValueError: 

312 If the topic is not in the EFD schema. 

313 """ 

314 staleAge = datetime.timedelta(warnStaleAfterNMinutes) 

315 

316 firstDayPossible = getDayObsStartTime(20190101) 

317 

318 if timeToLookBefore < firstDayPossible: 

319 raise ValueError(f"Requested time {timeToLookBefore} is before any data was put in the EFD") 

320 

321 df = pd.DataFrame() 

322 beginTime = timeToLookBefore 

323 while df.empty and beginTime > firstDayPossible: 

324 df = getEfdData(client, topic, begin=beginTime, timespan=-TIME_CHUNKING, warn=False) 

325 beginTime -= TIME_CHUNKING 

326 

327 if beginTime < firstDayPossible and df.empty: # we ran all the way back to the beginning of time 

328 raise ValueError(f"The entire EFD was searched backwards from {timeToLookBefore} and no data was " 

329 f"found in {topic=}") 

330 

331 lastRow = df.iloc[-1] 

332 commandTime = efdTimestampToAstropy(lastRow['private_efdStamp']) 

333 

334 commandAge = timeToLookBefore - commandTime 

335 if commandAge > staleAge: 

336 log = logging.getLogger(__name__) 

337 log.warning(f"Component {topic} was last set {commandAge.sec/60:.1} minutes" 

338 " before the requested time") 

339 

340 return lastRow 

341 

342 

343def makeEfdClient(testing=False): 

344 """Automatically create an EFD client based on the site. 

345 

346 Parameters 

347 ---------- 

348 testing : `bool`, optional 

349 Set to ``True`` if running in a test suite. This will default to using 

350 the USDF EFD, for which data has been recorded for replay by the ``vcr` 

351 package. Note data must be re-recorded to ``vcr`` from both inside and 

352 outside the USDF when the package/data changes, due to the use of a 

353 proxy meaning that the web requests are different depending on whether 

354 the EFD is being contacted from inside and outside the USDF. 

355 

356 Returns 

357 ------- 

358 efdClient : `lsst_efd_client.efd_helper.EfdClient` 

359 The EFD client to use for the current site. 

360 """ 

361 if not HAS_EFD_CLIENT: 

362 raise RuntimeError("Could not create EFD client because importing lsst_efd_client failed.") 

363 

364 if testing: 

365 return EfdClient('usdf_efd') 

366 

367 try: 

368 site = getSite() 

369 except ValueError as e: 

370 raise RuntimeError("Could not create EFD client as the site could not be determined") from e 

371 

372 if site == 'summit': 

373 return EfdClient('summit_efd') 

374 if site == 'tucson': 

375 return EfdClient('tucson_teststand_efd') 

376 if site == 'base': 

377 return EfdClient('base_efd') 

378 if site in ['staff-rsp', 'rubin-devl', 'usdf-k8s']: 

379 return EfdClient('usdf_efd') 

380 if site == 'jenkins': 

381 return EfdClient('usdf_efd') 

382 

383 raise RuntimeError(f"Could not create EFD client as the {site=} is not recognized") 

384 

385 

386def expRecordToTimespan(expRecord): 

387 """Get the timespan from an exposure record. 

388 

389 Returns the timespan in a format where it can be used to directly unpack 

390 into a efdClient.select_time_series() call. 

391 

392 Parameters 

393 ---------- 

394 expRecord : `lsst.daf.butler.dimensions.ExposureRecord` 

395 The exposure record. 

396 

397 Returns 

398 ------- 

399 timespanDict : `dict` 

400 The timespan in a format that can be used to directly unpack into a 

401 efdClient.select_time_series() call. 

402 """ 

403 return {'begin': expRecord.timespan.begin.utc, 

404 'end': expRecord.timespan.end.utc, 

405 } 

406 

407 

408def efdTimestampToAstropy(timestamp): 

409 """Get an efd timestamp as an astropy.time.Time object. 

410 

411 Parameters 

412 ---------- 

413 timestamp : `float` 

414 The timestamp, as a float. 

415 

416 Returns 

417 ------- 

418 time : `astropy.time.Time` 

419 The timestamp as an astropy.time.Time object. 

420 """ 

421 return Time(timestamp, format='unix') 

422 

423 

424def astropyToEfdTimestamp(time): 

425 """Get astropy Time object as an efd timestamp 

426 

427 Parameters 

428 ---------- 

429 time : `astropy.time.Time` 

430 The time as an astropy.time.Time object. 

431 

432 Returns 

433 ------- 

434 timestamp : `float` 

435 The timestamp, in UTC, in unix seconds. 

436 """ 

437 

438 return time.utc.unix 

439 

440 

441def clipDataToEvent(df, event): 

442 """Clip a padded dataframe to an event. 

443 

444 Parameters 

445 ---------- 

446 df : `pd.DataFrame` 

447 The dataframe to clip. 

448 event : `lsst.summit.utils.efdUtils.TmaEvent` 

449 The event to clip to. 

450 

451 Returns 

452 ------- 

453 clipped : `pd.DataFrame` 

454 The clipped dataframe. 

455 """ 

456 mask = (df['private_efdStamp'] >= event.begin.value) & (df['private_efdStamp'] <= event.end.value) 

457 clipped_df = df.loc[mask].copy() 

458 return clipped_df 

459 

460 

461def calcNextDay(dayObs): 

462 """Given an integer dayObs, calculate the next integer dayObs. 

463 

464 Integers are used for dayObs, but dayObs values are therefore not 

465 contiguous due to month/year ends etc, so this utility provides a robust 

466 way to get the integer dayObs which follows the one specified. 

467 

468 Parameters 

469 ---------- 

470 dayObs : `int` 

471 The dayObs, as an integer, e.g. 20231231 

472 

473 Returns 

474 ------- 

475 nextDayObs : `int` 

476 The next dayObs, as an integer, e.g. 20240101 

477 """ 

478 d1 = datetime.datetime.strptime(str(dayObs), '%Y%m%d') 

479 oneDay = datetime.timedelta(days=1) 

480 return int((d1 + oneDay).strftime('%Y%m%d')) 

481 

482 

483def getDayObsStartTime(dayObs): 

484 """Get the start of the given dayObs as an astropy.time.Time object. 

485 

486 The observatory rolls the date over at UTC-12. 

487 

488 Parameters 

489 ---------- 

490 dayObs : `int` 

491 The dayObs, as an integer, e.g. 20231225 

492 

493 Returns 

494 ------- 

495 time : `astropy.time.Time` 

496 The start of the dayObs as an astropy.time.Time object. 

497 """ 

498 pythonDateTime = datetime.datetime.strptime(str(dayObs), "%Y%m%d") 

499 return Time(pythonDateTime) + 12 * u.hour 

500 

501 

502def getDayObsEndTime(dayObs): 

503 """Get the end of the given dayObs as an astropy.time.Time object. 

504 

505 Parameters 

506 ---------- 

507 dayObs : `int` 

508 The dayObs, as an integer, e.g. 20231225 

509 

510 Returns 

511 ------- 

512 time : `astropy.time.Time` 

513 The end of the dayObs as an astropy.time.Time object. 

514 """ 

515 return getDayObsStartTime(dayObs) + 24 * u.hour 

516 

517 

518def getDayObsForTime(time): 

519 """Get the dayObs in which an astropy.time.Time object falls. 

520 

521 Parameters 

522 ---------- 

523 time : `astropy.time.Time` 

524 The time. 

525 

526 Returns 

527 ------- 

528 dayObs : `int` 

529 The dayObs, as an integer, e.g. 20231225 

530 """ 

531 twelveHours = datetime.timedelta(hours=-12) 

532 offset = TimeDelta(twelveHours, format='datetime') 

533 return int((time + offset).utc.isot[:10].replace('-', '')) 

534 

535 

536@deprecated( 

537 reason="getSubTopics() has been replaced by getTopics() and using wildcards. " 

538 "Will be removed after w_2023_50.", 

539 version="w_2023_40", 

540 category=FutureWarning, 

541) 

542def getSubTopics(client, topic): 

543 """Get all the sub topics within a given topic. 

544 

545 Note that the topic need not be a complete one, for example, rather than 

546 doing `getSubTopics(client, 'lsst.sal.ATMCS')` to get all the topics for 

547 the AuxTel Mount Control System, you can do `getSubTopics(client, 

548 'lsst.sal.AT')` to get all which relate to the AuxTel in general. 

549 

550 Parameters 

551 ---------- 

552 client : `lsst_efd_client.efd_helper.EfdClient` 

553 The EFD client to use. 

554 topic : `str` 

555 The topic to query. 

556 

557 Returns 

558 ------- 

559 subTopics : `list` of `str` 

560 The sub topics. 

561 """ 

562 loop = asyncio.get_event_loop() 

563 topics = loop.run_until_complete(client.get_topics()) 

564 return sorted([t for t in topics if t.startswith(topic)]) 

565 

566 

567def getTopics(client, toFind, caseSensitive=False): 

568 """Return all the strings in topics which match the topic query string. 

569 

570 Supports wildcards, which are denoted as `*``, as per shell globs. 

571 

572 Example: 

573 >>> # assume topics are ['apple', 'banana', 'grape'] 

574 >>> getTopics(, 'a*p*') 

575 ['apple', 'grape'] 

576 

577 Parameters 

578 ---------- 

579 client : `lsst_efd_client.efd_helper.EfdClient` 

580 The EFD client to use. 

581 toFind : `str` 

582 The query string, with optional wildcards denoted as *. 

583 caseSensitive : `bool`, optional 

584 If ``True``, the query is case sensitive. Defaults to ``False``. 

585 

586 Returns 

587 ------- 

588 matches : `list` of `str` 

589 The list of matching topics. 

590 """ 

591 loop = asyncio.get_event_loop() 

592 topics = loop.run_until_complete(client.get_topics()) 

593 

594 # Replace wildcard with regex equivalent 

595 pattern = toFind.replace('*', '.*') 

596 flags = re.IGNORECASE if not caseSensitive else 0 

597 

598 matches = [] 

599 for topic in topics: 

600 if re.match(pattern, topic, flags): 

601 matches.append(topic) 

602 

603 return matches