Coverage for python/lsst/summit/utils/efdUtils.py: 16%

164 statements  

« prev     ^ index     » next       coverage.py v7.4.2, created at 2024-02-23 15:47 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import asyncio 

23from astropy.time import Time, TimeDelta 

24from astropy import units as u 

25import datetime 

26import logging 

27import pandas as pd 

28import re 

29from deprecated.sphinx import deprecated 

30 

31from lsst.utils.iteration import ensure_iterable 

32 

33from .utils import getSite 

34 

35HAS_EFD_CLIENT = True 

36try: 

37 from lsst_efd_client import EfdClient 

38except ImportError: 

39 HAS_EFD_CLIENT = False 

40 

41__all__ = [ 

42 'getEfdData', 

43 'getMostRecentRowWithDataBefore', 

44 'makeEfdClient', 

45 'expRecordToTimespan', 

46 'efdTimestampToAstropy', 

47 'astropyToEfdTimestamp', 

48 'clipDataToEvent', 

49 'calcNextDay', 

50 'getDayObsStartTime', 

51 'getDayObsEndTime', 

52 'getDayObsForTime', 

53 'getSubTopics', # deprecated, being removed in w_2023_50 

54 'getTopics', 

55] 

56 

57 

58COMMAND_ALIASES = { 

59 'raDecTarget': 'lsst.sal.MTPtg.command_raDecTarget', 

60 'moveToTarget': 'lsst.sal.MTMount.command_moveToTarget', 

61 'startTracking': 'lsst.sal.MTMount.command_startTracking', 

62 'stopTracking': 'lsst.sal.MTMount.command_stopTracking', 

63 'trackTarget': 'lsst.sal.MTMount.command_trackTarget', # issued at 20Hz - don't plot 

64} 

65 

66# When looking backwards in time to find the most recent state event, look back 

67# in chunks of this size. Too small, and there will be too many queries, too 

68# large and there will be too much data returned unnecessarily, as we only need 

69# one row by definition. Will tune this parameters in consultation with SQuaRE. 

70TIME_CHUNKING = datetime.timedelta(minutes=15) 

71 

72 

73def _getBeginEnd(dayObs=None, begin=None, end=None, timespan=None, event=None, expRecord=None): 

74 """Calculate the begin and end times to pass to _getEfdData, given the 

75 kwargs passed to getEfdData. 

76 

77 Parameters 

78 ---------- 

79 dayObs : `int` 

80 The dayObs to query. If specified, this is used to determine the begin 

81 and end times. 

82 begin : `astropy.Time` 

83 The begin time for the query. If specified, either an end time or a 

84 timespan must be supplied. 

85 end : `astropy.Time` 

86 The end time for the query. If specified, a begin time must also be 

87 supplied. 

88 timespan : `astropy.TimeDelta` 

89 The timespan for the query. If specified, a begin time must also be 

90 supplied. 

91 event : `lsst.summit.utils.efdUtils.TmaEvent` 

92 The event to query. If specified, this is used to determine the begin 

93 and end times, and all other options are disallowed. 

94 expRecord : `lsst.daf.butler.dimensions.DimensionRecord` 

95 The exposure record containing the timespan to query. If specified, all 

96 other options are disallowed. 

97 

98 Returns 

99 ------- 

100 begin : `astropy.Time` 

101 The begin time for the query. 

102 end : `astropy.Time` 

103 The end time for the query. 

104 """ 

105 if expRecord is not None: 

106 forbiddenOpts = [event, begin, end, timespan, dayObs] 

107 if any(x is not None for x in forbiddenOpts): 

108 raise ValueError("You can't specify both an expRecord and a begin/end or timespan or dayObs") 

109 begin = expRecord.timespan.begin 

110 end = expRecord.timespan.end 

111 return begin, end 

112 

113 if event is not None: 

114 forbiddenOpts = [begin, end, timespan, dayObs] 

115 if any(x is not None for x in forbiddenOpts): 

116 raise ValueError("You can't specify both an event and a begin/end or timespan or dayObs") 

117 begin = event.begin 

118 end = event.end 

119 return begin, end 

120 

121 # check for dayObs, and that other options aren't inconsistently specified 

122 if dayObs is not None: 

123 forbiddenOpts = [begin, end, timespan] 

124 if any(x is not None for x in forbiddenOpts): 

125 raise ValueError("You can't specify both a dayObs and a begin/end or timespan") 

126 begin = getDayObsStartTime(dayObs) 

127 end = getDayObsEndTime(dayObs) 

128 return begin, end 

129 # can now disregard dayObs entirely 

130 

131 if begin is None: 

132 raise ValueError("You must specify either a dayObs or a begin/end or begin/timespan") 

133 # can now rely on begin, so just need to deal with end/timespan 

134 

135 if end is None and timespan is None: 

136 raise ValueError("If you specify a begin, you must specify either a end or a timespan") 

137 if end is not None and timespan is not None: 

138 raise ValueError("You can't specify both a end and a timespan") 

139 if end is None: 

140 if timespan > datetime.timedelta(minutes=0): 

141 end = begin + timespan # the normal case 

142 else: 

143 end = begin # the case where timespan is negative 

144 begin = begin + timespan # adding the negative to the start, i.e. subtracting it to bring back 

145 

146 assert begin is not None 

147 assert end is not None 

148 return begin, end 

149 

150 

151def getEfdData(client, topic, *, 

152 columns=None, 

153 prePadding=0, 

154 postPadding=0, 

155 dayObs=None, 

156 begin=None, 

157 end=None, 

158 timespan=None, 

159 event=None, 

160 expRecord=None, 

161 warn=True, 

162 ): 

163 """Get one or more EFD topics over a time range, synchronously. 

164 

165 The time range can be specified as either: 

166 * a dayObs, in which case the full 24 hour period is used, 

167 * a begin point and a end point, 

168 * a begin point and a timespan. 

169 * a mount event 

170 * an exposure record 

171 If it is desired to use an end time with a timespan, just specify it as the 

172 begin time and use a negative timespan. 

173 

174 The results from all topics are merged into a single dataframe. 

175 

176 Parameters 

177 ---------- 

178 client : `lsst_efd_client.efd_helper.EfdClient` 

179 The EFD client to use. 

180 topic : `str` 

181 The topic to query. 

182 columns : `list` of `str`, optional 

183 The columns to query. If not specified, all columns are queried. 

184 prePadding : `float` 

185 The amount of time before the nominal start of the query to include, in 

186 seconds. 

187 postPadding : `float` 

188 The amount of extra time after the nominal end of the query to include, 

189 in seconds. 

190 dayObs : `int`, optional 

191 The dayObs to query. If specified, this is used to determine the begin 

192 and end times. 

193 begin : `astropy.Time`, optional 

194 The begin time for the query. If specified, either a end time or a 

195 timespan must be supplied. 

196 end : `astropy.Time`, optional 

197 The end time for the query. If specified, a begin time must also be 

198 supplied. 

199 timespan : `astropy.TimeDelta`, optional 

200 The timespan for the query. If specified, a begin time must also be 

201 supplied. 

202 event : `lsst.summit.utils.efdUtils.TmaEvent`, optional 

203 The event to query. If specified, this is used to determine the begin 

204 and end times, and all other options are disallowed. 

205 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`, optional 

206 The exposure record containing the timespan to query. If specified, all 

207 other options are disallowed. 

208 warn : bool, optional 

209 If ``True``, warn when no data is found. Exists so that utility code 

210 can disable warnings when checking for data, and therefore defaults to 

211 ``True``. 

212 

213 Returns 

214 ------- 

215 data : `pd.DataFrame` 

216 The merged data from all topics. 

217 

218 Raises 

219 ------ 

220 ValueError: 

221 If the topics are not in the EFD schema. 

222 ValueError: 

223 If both a dayObs and a begin/end or timespan are specified. 

224 ValueError: 

225 If a begin time is specified but no end time or timespan. 

226 

227 """ 

228 # TODO: DM-40100 ideally should calls mpts as necessary so that users 

229 # needn't care if things are packed 

230 

231 # supports aliases so that you can query with them. If there is no entry in 

232 # the alias dict then it queries with the supplied key. The fact the schema 

233 # is now being checked means this shouldn't be a problem now. 

234 

235 # TODO: RFC-948 Move this import back to top of file once is implemented. 

236 import nest_asyncio 

237 

238 begin, end = _getBeginEnd(dayObs, begin, end, timespan, event, expRecord) 

239 begin -= TimeDelta(prePadding, format='sec') 

240 end += TimeDelta(postPadding, format='sec') 

241 

242 nest_asyncio.apply() 

243 loop = asyncio.get_event_loop() 

244 ret = loop.run_until_complete(_getEfdData(client=client, 

245 topic=topic, 

246 begin=begin, 

247 end=end, 

248 columns=columns)) 

249 if ret.empty and warn: 

250 log = logging.getLogger(__name__) 

251 log.warning(f"Topic {topic} is in the schema, but no data was returned by the query for the specified" 

252 " time range") 

253 return ret 

254 

255 

256async def _getEfdData(client, topic, begin, end, columns=None): 

257 """Get data for a topic from the EFD over the specified time range. 

258 

259 Parameters 

260 ---------- 

261 client : `lsst_efd_client.efd_helper.EfdClient` 

262 The EFD client to use. 

263 topic : `str` 

264 The topic to query. 

265 begin : `astropy.Time` 

266 The begin time for the query. 

267 end : `astropy.Time` 

268 The end time for the query. 

269 columns : `list` of `str`, optional 

270 The columns to query. If not specified, all columns are returned. 

271 

272 Returns 

273 ------- 

274 data : `pd.DataFrame` 

275 The data from the query. 

276 """ 

277 if columns is None: 

278 columns = ['*'] 

279 columns = list(ensure_iterable(columns)) 

280 

281 availableTopics = await client.get_topics() 

282 

283 if topic not in availableTopics: 

284 raise ValueError(f"Topic {topic} not in EFD schema") 

285 

286 data = await client.select_time_series(topic, columns, begin.utc, end.utc) 

287 

288 return data 

289 

290 

291def getMostRecentRowWithDataBefore(client, topic, timeToLookBefore, warnStaleAfterNMinutes=60*12): 

292 """Get the most recent row of data for a topic before a given time. 

293 

294 Parameters 

295 ---------- 

296 client : `lsst_efd_client.efd_helper.EfdClient` 

297 The EFD client to use. 

298 topic : `str` 

299 The topic to query. 

300 timeToLookBefore : `astropy.Time` 

301 The time to look before. 

302 warnStaleAfterNMinutes : `float`, optional 

303 The number of minutes after which to consider the data stale and issue 

304 a warning. 

305 

306 Returns 

307 ------- 

308 row : `pd.Series` 

309 The row of data from the EFD containing the most recent data before the 

310 specified time. 

311 

312 Raises 

313 ------ 

314 ValueError: 

315 If the topic is not in the EFD schema. 

316 """ 

317 staleAge = datetime.timedelta(warnStaleAfterNMinutes) 

318 

319 firstDayPossible = getDayObsStartTime(20190101) 

320 

321 if timeToLookBefore < firstDayPossible: 

322 raise ValueError(f"Requested time {timeToLookBefore} is before any data was put in the EFD") 

323 

324 df = pd.DataFrame() 

325 beginTime = timeToLookBefore 

326 while df.empty and beginTime > firstDayPossible: 

327 df = getEfdData(client, topic, begin=beginTime, timespan=-TIME_CHUNKING, warn=False) 

328 beginTime -= TIME_CHUNKING 

329 

330 if beginTime < firstDayPossible and df.empty: # we ran all the way back to the beginning of time 

331 raise ValueError(f"The entire EFD was searched backwards from {timeToLookBefore} and no data was " 

332 f"found in {topic=}") 

333 

334 lastRow = df.iloc[-1] 

335 commandTime = efdTimestampToAstropy(lastRow['private_efdStamp']) 

336 

337 commandAge = timeToLookBefore - commandTime 

338 if commandAge > staleAge: 

339 log = logging.getLogger(__name__) 

340 log.warning(f"Component {topic} was last set {commandAge.sec/60:.1} minutes" 

341 " before the requested time") 

342 

343 return lastRow 

344 

345 

346def makeEfdClient(testing=False): 

347 """Automatically create an EFD client based on the site. 

348 

349 Parameters 

350 ---------- 

351 testing : `bool`, optional 

352 Set to ``True`` if running in a test suite. This will default to using 

353 the USDF EFD, for which data has been recorded for replay by the ``vcr` 

354 package. Note data must be re-recorded to ``vcr`` from both inside and 

355 outside the USDF when the package/data changes, due to the use of a 

356 proxy meaning that the web requests are different depending on whether 

357 the EFD is being contacted from inside and outside the USDF. 

358 

359 Returns 

360 ------- 

361 efdClient : `lsst_efd_client.efd_helper.EfdClient` 

362 The EFD client to use for the current site. 

363 """ 

364 if not HAS_EFD_CLIENT: 

365 raise RuntimeError("Could not create EFD client because importing lsst_efd_client failed.") 

366 

367 if testing: 

368 return EfdClient('usdf_efd') 

369 

370 try: 

371 site = getSite() 

372 except ValueError as e: 

373 raise RuntimeError("Could not create EFD client as the site could not be determined") from e 

374 

375 if site == 'summit': 

376 return EfdClient('summit_efd') 

377 if site == 'tucson': 

378 return EfdClient('tucson_teststand_efd') 

379 if site == 'base': 

380 return EfdClient('base_efd') 

381 if site in ['staff-rsp', 'rubin-devl', 'usdf-k8s']: 

382 return EfdClient('usdf_efd') 

383 if site == 'jenkins': 

384 return EfdClient('usdf_efd') 

385 

386 raise RuntimeError(f"Could not create EFD client as the {site=} is not recognized") 

387 

388 

389def expRecordToTimespan(expRecord): 

390 """Get the timespan from an exposure record. 

391 

392 Returns the timespan in a format where it can be used to directly unpack 

393 into a efdClient.select_time_series() call. 

394 

395 Parameters 

396 ---------- 

397 expRecord : `lsst.daf.butler.dimensions.ExposureRecord` 

398 The exposure record. 

399 

400 Returns 

401 ------- 

402 timespanDict : `dict` 

403 The timespan in a format that can be used to directly unpack into a 

404 efdClient.select_time_series() call. 

405 """ 

406 return {'begin': expRecord.timespan.begin.utc, 

407 'end': expRecord.timespan.end.utc, 

408 } 

409 

410 

411def efdTimestampToAstropy(timestamp): 

412 """Get an efd timestamp as an astropy.time.Time object. 

413 

414 Parameters 

415 ---------- 

416 timestamp : `float` 

417 The timestamp, as a float. 

418 

419 Returns 

420 ------- 

421 time : `astropy.time.Time` 

422 The timestamp as an astropy.time.Time object. 

423 """ 

424 return Time(timestamp, format='unix') 

425 

426 

427def astropyToEfdTimestamp(time): 

428 """Get astropy Time object as an efd timestamp 

429 

430 Parameters 

431 ---------- 

432 time : `astropy.time.Time` 

433 The time as an astropy.time.Time object. 

434 

435 Returns 

436 ------- 

437 timestamp : `float` 

438 The timestamp, in UTC, in unix seconds. 

439 """ 

440 

441 return time.utc.unix 

442 

443 

444def clipDataToEvent(df, event, prePadding=0, postPadding=0, logger=None): 

445 """Clip a padded dataframe to an event. 

446 

447 Parameters 

448 ---------- 

449 df : `pd.DataFrame` 

450 The dataframe to clip. 

451 event : `lsst.summit.utils.efdUtils.TmaEvent` 

452 The event to clip to. 

453 prePadding : `float`, optional 

454 The amount of time before the nominal start of the event to include, in 

455 seconds. 

456 postPadding : `float`, optional 

457 The amount of extra time after the nominal end of the event to include, 

458 in seconds. 

459 logger : `logging.Logger`, optional 

460 The logger to use. If not specified, a new one is created. 

461 

462 Returns 

463 ------- 

464 clipped : `pd.DataFrame` 

465 The clipped dataframe. 

466 """ 

467 begin = event.begin.value - prePadding 

468 end = event.end.value + postPadding 

469 

470 if logger is None: 

471 logger = logging.getLogger(__name__) 

472 

473 if begin < df['private_efdStamp'].min(): 

474 logger.warning(f"Requested begin time {begin} is before the start of the data") 

475 if end > df['private_efdStamp'].max(): 

476 logger.warning(f"Requested end time {end} is after the end of the data") 

477 

478 mask = (df['private_efdStamp'] >= begin) & (df['private_efdStamp'] <= end) 

479 clipped_df = df.loc[mask].copy() 

480 return clipped_df 

481 

482 

483def offsetDayObs(dayObs, nDays): 

484 """Offset a dayObs by a given number of days. 

485 

486 Parameters 

487 ---------- 

488 dayObs : `int` 

489 The dayObs, as an integer, e.g. 20231225 

490 nDays : `int` 

491 The number of days to offset the dayObs by. 

492 

493 Returns 

494 ------- 

495 newDayObs : `int` 

496 The new dayObs, as an integer, e.g. 20231225 

497 """ 

498 d1 = datetime.datetime.strptime(str(dayObs), '%Y%m%d') 

499 oneDay = datetime.timedelta(days=nDays) 

500 return int((d1 + oneDay).strftime('%Y%m%d')) 

501 

502 

503def calcNextDay(dayObs): 

504 """Given an integer dayObs, calculate the next integer dayObs. 

505 

506 Integers are used for dayObs, but dayObs values are therefore not 

507 contiguous due to month/year ends etc, so this utility provides a robust 

508 way to get the integer dayObs which follows the one specified. 

509 

510 Parameters 

511 ---------- 

512 dayObs : `int` 

513 The dayObs, as an integer, e.g. 20231231 

514 

515 Returns 

516 ------- 

517 nextDayObs : `int` 

518 The next dayObs, as an integer, e.g. 20240101 

519 """ 

520 return offsetDayObs(dayObs, 1) 

521 

522 

523def calcPreviousDay(dayObs): 

524 """Given an integer dayObs, calculate the next integer dayObs. 

525 

526 Integers are used for dayObs, but dayObs values are therefore not 

527 contiguous due to month/year ends etc, so this utility provides a robust 

528 way to get the integer dayObs which follows the one specified. 

529 

530 Parameters 

531 ---------- 

532 dayObs : `int` 

533 The dayObs, as an integer, e.g. 20231231 

534 

535 Returns 

536 ------- 

537 nextDayObs : `int` 

538 The next dayObs, as an integer, e.g. 20240101 

539 """ 

540 return offsetDayObs(dayObs, -1) 

541 

542 

543def getDayObsStartTime(dayObs): 

544 """Get the start of the given dayObs as an astropy.time.Time object. 

545 

546 The observatory rolls the date over at UTC-12. 

547 

548 Parameters 

549 ---------- 

550 dayObs : `int` 

551 The dayObs, as an integer, e.g. 20231225 

552 

553 Returns 

554 ------- 

555 time : `astropy.time.Time` 

556 The start of the dayObs as an astropy.time.Time object. 

557 """ 

558 pythonDateTime = datetime.datetime.strptime(str(dayObs), "%Y%m%d") 

559 return Time(pythonDateTime) + 12 * u.hour 

560 

561 

562def getDayObsEndTime(dayObs): 

563 """Get the end of the given dayObs as an astropy.time.Time object. 

564 

565 Parameters 

566 ---------- 

567 dayObs : `int` 

568 The dayObs, as an integer, e.g. 20231225 

569 

570 Returns 

571 ------- 

572 time : `astropy.time.Time` 

573 The end of the dayObs as an astropy.time.Time object. 

574 """ 

575 return getDayObsStartTime(dayObs) + 24 * u.hour 

576 

577 

578def getDayObsForTime(time): 

579 """Get the dayObs in which an astropy.time.Time object falls. 

580 

581 Parameters 

582 ---------- 

583 time : `astropy.time.Time` 

584 The time. 

585 

586 Returns 

587 ------- 

588 dayObs : `int` 

589 The dayObs, as an integer, e.g. 20231225 

590 """ 

591 twelveHours = datetime.timedelta(hours=-12) 

592 offset = TimeDelta(twelveHours, format='datetime') 

593 return int((time + offset).utc.isot[:10].replace('-', '')) 

594 

595 

596@deprecated( 

597 reason="getSubTopics() has been replaced by getTopics() and using wildcards. " 

598 "Will be removed after w_2023_50.", 

599 version="w_2023_40", 

600 category=FutureWarning, 

601) 

602def getSubTopics(client, topic): 

603 """Get all the sub topics within a given topic. 

604 

605 Note that the topic need not be a complete one, for example, rather than 

606 doing `getSubTopics(client, 'lsst.sal.ATMCS')` to get all the topics for 

607 the AuxTel Mount Control System, you can do `getSubTopics(client, 

608 'lsst.sal.AT')` to get all which relate to the AuxTel in general. 

609 

610 Parameters 

611 ---------- 

612 client : `lsst_efd_client.efd_helper.EfdClient` 

613 The EFD client to use. 

614 topic : `str` 

615 The topic to query. 

616 

617 Returns 

618 ------- 

619 subTopics : `list` of `str` 

620 The sub topics. 

621 """ 

622 loop = asyncio.get_event_loop() 

623 topics = loop.run_until_complete(client.get_topics()) 

624 return sorted([t for t in topics if t.startswith(topic)]) 

625 

626 

627def getTopics(client, toFind, caseSensitive=False): 

628 """Return all the strings in topics which match the topic query string. 

629 

630 Supports wildcards, which are denoted as `*``, as per shell globs. 

631 

632 Example: 

633 >>> # assume topics are ['apple', 'banana', 'grape'] 

634 >>> getTopics(, 'a*p*') 

635 ['apple', 'grape'] 

636 

637 Parameters 

638 ---------- 

639 client : `lsst_efd_client.efd_helper.EfdClient` 

640 The EFD client to use. 

641 toFind : `str` 

642 The query string, with optional wildcards denoted as *. 

643 caseSensitive : `bool`, optional 

644 If ``True``, the query is case sensitive. Defaults to ``False``. 

645 

646 Returns 

647 ------- 

648 matches : `list` of `str` 

649 The list of matching topics. 

650 """ 

651 loop = asyncio.get_event_loop() 

652 topics = loop.run_until_complete(client.get_topics()) 

653 

654 # Replace wildcard with regex equivalent 

655 pattern = toFind.replace('*', '.*') 

656 flags = re.IGNORECASE if not caseSensitive else 0 

657 

658 matches = [] 

659 for topic in topics: 

660 if re.match(pattern, topic, flags): 

661 matches.append(topic) 

662 

663 return matches