Coverage for python/lsst/summit/utils/tmaUtils.py: 21%

594 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-17 12:43 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import re 

23import enum 

24import itertools 

25import logging 

26import pandas as pd 

27import numpy as np 

28import humanize 

29from dataclasses import dataclass 

30from astropy.time import Time 

31from matplotlib.ticker import FuncFormatter 

32import matplotlib.dates as mdates 

33import matplotlib.pyplot as plt 

34from lsst.utils.iteration import ensure_iterable 

35 

36from .enums import ScriptState, AxisMotionState, PowerState 

37from .utils import getCurrentDayObs_int, dayObsIntToString 

38from .efdUtils import (getEfdData, 

39 makeEfdClient, 

40 efdTimestampToAstropy, 

41 COMMAND_ALIASES, 

42 getDayObsForTime, 

43 getDayObsStartTime, 

44 getDayObsEndTime, 

45 clipDataToEvent, 

46 ) 

47 

48__all__ = ( 

49 'TMAStateMachine', 

50 'TMAEvent', 

51 'TMAEventMaker', 

52 'TMAState', 

53 'AxisMotionState', 

54 'PowerState', 

55 'getSlewsFromEventList', 

56 'getTracksFromEventList', 

57 'getTorqueMaxima', 

58) 

59 

60# we don't want to use `None` for a no data sentinel because dict.get('key') 

61# returns None if the key isn't present, and also we need to mark that the data 

62# was queried for and no data was found, whereas the key not being present 

63# means that we've not yet looked for the data. 

64NO_DATA_SENTINEL = "NODATA" 

65 

66 

67def getSlewsFromEventList(events): 

68 """Get the slew events from a list of TMAEvents. 

69 

70 Parameters 

71 ---------- 

72 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent` 

73 The list of events to filter. 

74 

75 Returns 

76 ------- 

77 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent` 

78 The filtered list of events. 

79 """ 

80 return [e for e in events if e.type == TMAState.SLEWING] 

81 

82 

83def getTracksFromEventList(events): 

84 """Get the tracking events from a list of TMAEvents. 

85 

86 Parameters 

87 ---------- 

88 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent` 

89 The list of events to filter. 

90 

91 Returns 

92 ------- 

93 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent` 

94 The filtered list of events. 

95 """ 

96 return [e for e in events if e.type == TMAState.TRACKING] 

97 

98 

99def getTorqueMaxima(table): 

100 """Print the maximum positive and negative azimuth and elevation torques. 

101 

102 Designed to be used with the table as downloaded from RubinTV. 

103 

104 Parameters 

105 ---------- 

106 table : `pd.DataFrame` 

107 The table of data to use, as generated by Rapid Analysis. 

108 """ 

109 for axis in ['elevation', 'azimuth']: 

110 col = f'Largest {axis} torque' 

111 maxPos = np.argmax(table[col]) 

112 maxVal = table[col].iloc[maxPos] 

113 print(f"Max positive {axis:9} torque during seqNum {maxPos:>4}: {maxVal/1000:>7.1f}kNm") 

114 minPos = np.argmin(table[col]) 

115 minVal = table[col].iloc[minPos] 

116 print(f"Max negative {axis:9} torque during seqNum {minPos:>4}: {minVal/1000:>7.1f}kNm") 

117 

118 

119def getAzimuthElevationDataForEvent(client, event, prePadding=0, postPadding=0): 

120 """Get the data for the az/el telemetry topics for a given TMAEvent. 

121 

122 Parameters 

123 ---------- 

124 client : `lsst_efd_client.efd_helper.EfdClient` 

125 The EFD client to use. 

126 event : `lsst.summit.utils.tmaUtils.TMAEvent` 

127 The event to get the data for. 

128 prePadding : `float`, optional 

129 The amount of time to pad the event with before the start time, in 

130 seconds. 

131 postPadding : `float`, optional 

132 The amount of time to pad the event with after the end time, in 

133 seconds. 

134 

135 Returns 

136 ------- 

137 azimuthData : `pd.DataFrame` 

138 The azimuth data for the specified event. 

139 elevationData : `pd.DataFrame` 

140 The elevation data for the specified event. 

141 """ 

142 azimuthData = getEfdData(client, 

143 'lsst.sal.MTMount.azimuth', 

144 event=event, 

145 prePadding=prePadding, 

146 postPadding=postPadding) 

147 elevationData = getEfdData(client, 

148 'lsst.sal.MTMount.elevation', 

149 event=event, 

150 prePadding=prePadding, 

151 postPadding=postPadding) 

152 

153 return azimuthData, elevationData 

154 

155 

156def plotEvent(client, event, fig=None, prePadding=0, postPadding=0, commands={}, 

157 azimuthData=None, elevationData=None): 

158 """Plot the TMA axis positions over the course of a given TMAEvent. 

159 

160 Plots the axis motion profiles for the given event, with optional padding 

161 at the start and end of the event. If the data is provided via the 

162 azimuthData and elevationData parameters, it will be used, otherwise it 

163 will be queried from the EFD. 

164 

165 Optionally plots any commands issued during or around the event, if these 

166 are supplied. Commands are supplied as a dictionary of the command topic 

167 strings, with values as astro.time.Time objects at which the command was 

168 issued. 

169 

170 Parameters 

171 ---------- 

172 client : `lsst_efd_client.efd_helper.EfdClient` 

173 The EFD client to use. 

174 event : `lsst.summit.utils.tmaUtils.TMAEvent` 

175 The event to plot. 

176 fig : `matplotlib.figure.Figure`, optional 

177 The figure to plot on. If not specified, a new figure will be created. 

178 prePadding : `float`, optional 

179 The amount of time to pad the event with before the start time, in 

180 seconds. 

181 postPadding : `float`, optional 

182 The amount of time to pad the event with after the end time, in 

183 seconds. 

184 commands : `dict` of `str` : `astropy.time.Time`, optional 

185 A dictionary of commands to plot on the figure. The keys are the topic 

186 names, and the values are the times at which the commands were sent. 

187 azimuthData : `pd.DataFrame`, optional 

188 The azimuth data to plot. If not specified, it will be queried from the 

189 EFD. 

190 elevationData : `pd.DataFrame`, optional 

191 The elevation data to plot. If not specified, it will be queried from 

192 the EFD. 

193 

194 Returns 

195 ------- 

196 fig : `matplotlib.figure.Figure` 

197 The figure on which the plot was made. 

198 """ 

199 def tickFormatter(value, tick_number): 

200 # Convert the value to a string without subtracting large numbers 

201 # tick_number is unused. 

202 return f"{value:.2f}" 

203 

204 # plot any commands we might have 

205 if not isinstance(commands, dict): 

206 raise TypeError('commands must be a dict of command names with values as' 

207 ' astropy.time.Time values') 

208 

209 if fig is None: 

210 fig = plt.figure(figsize=(10, 8)) 

211 log = logging.getLogger(__name__) 

212 log.warning("Making new matplotlib figure - if this is in a loop you're going to have a bad time." 

213 " Pass in a figure with fig = plt.figure(figsize=(10, 8)) to avoid this warning.") 

214 

215 fig.clear() 

216 ax1, ax2 = fig.subplots(2, 

217 sharex=True, 

218 gridspec_kw={'wspace': 0, 

219 'hspace': 0, 

220 'height_ratios': [2.5, 1]}) 

221 

222 if azimuthData is None or elevationData is None: 

223 azimuthData, elevationData = getAzimuthElevationDataForEvent(client, 

224 event, 

225 prePadding=prePadding, 

226 postPadding=postPadding) 

227 

228 # Use the native color cycle for the lines. Because they're on different 

229 # axes they don't cycle by themselves 

230 lineColors = [p['color'] for p in plt.rcParams['axes.prop_cycle']] 

231 colorCounter = 0 

232 

233 ax1.plot(azimuthData['actualPosition'], label='Azimuth position', c=lineColors[colorCounter]) 

234 colorCounter += 1 

235 ax1.yaxis.set_major_formatter(FuncFormatter(tickFormatter)) 

236 ax1.set_ylabel('Azimuth (degrees)') 

237 

238 ax1_twin = ax1.twinx() 

239 ax1_twin.plot(elevationData['actualPosition'], label='Elevation position', c=lineColors[colorCounter]) 

240 colorCounter += 1 

241 ax1_twin.yaxis.set_major_formatter(FuncFormatter(tickFormatter)) 

242 ax1_twin.set_ylabel('Elevation (degrees)') 

243 ax1.set_xticks([]) # remove x tick labels on the hidden upper x-axis 

244 

245 ax2_twin = ax2.twinx() 

246 ax2.plot(azimuthData['actualTorque'], label='Azimuth torque', c=lineColors[colorCounter]) 

247 colorCounter += 1 

248 ax2_twin.plot(elevationData['actualTorque'], label='Elevation torque', c=lineColors[colorCounter]) 

249 colorCounter += 1 

250 ax2.set_ylabel('Azimuth torque (Nm)') 

251 ax2_twin.set_ylabel('Elevation torque (Nm)') 

252 ax2.set_xlabel('Time (UTC)') # yes, it really is UTC, matplotlib converts this automatically! 

253 

254 # put the ticks at an angle, and right align with the tick marks 

255 ax2.set_xticks(ax2.get_xticks()) # needed to supress a user warning 

256 xlabels = ax2.get_xticks() 

257 ax2.set_xticklabels(xlabels, rotation=40, ha='right') 

258 ax2.xaxis.set_major_locator(mdates.AutoDateLocator()) 

259 ax2.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M:%S')) 

260 

261 if prePadding or postPadding: 

262 # note the conversion to utc because the x-axis from the dataframe 

263 # already got automagically converted when plotting before, so this is 

264 # necessary for things to line up 

265 ax1_twin.axvline(event.begin.utc.datetime, c='k', ls='--', alpha=0.5, label='Event begin/end') 

266 ax1_twin.axvline(event.end.utc.datetime, c='k', ls='--', alpha=0.5) 

267 # extend lines down across lower plot, but do not re-add label 

268 ax2_twin.axvline(event.begin.utc.datetime, c='k', ls='--', alpha=0.5) 

269 ax2_twin.axvline(event.end.utc.datetime, c='k', ls='--', alpha=0.5) 

270 

271 for command, commandTime in commands.items(): 

272 # if commands weren't found, the item is set to None. This is common 

273 # for events so handle it gracefully and silently. The command finding 

274 # code logs about lack of commands found so no need to mention here. 

275 if commandTime is None: 

276 continue 

277 ax1_twin.axvline(commandTime.utc.datetime, c=lineColors[colorCounter], 

278 ls='--', alpha=0.75, label=f'{command}') 

279 # extend lines down across lower plot, but do not re-add label 

280 ax2_twin.axvline(commandTime.utc.datetime, c=lineColors[colorCounter], 

281 ls='--', alpha=0.75) 

282 colorCounter += 1 

283 

284 # combine the legends and put inside the plot 

285 handles1a, labels1a = ax1.get_legend_handles_labels() 

286 handles1b, labels1b = ax1_twin.get_legend_handles_labels() 

287 handles2a, labels2a = ax2.get_legend_handles_labels() 

288 handles2b, labels2b = ax2_twin.get_legend_handles_labels() 

289 

290 handles = handles1a + handles1b + handles2a + handles2b 

291 labels = labels1a + labels1b + labels2a + labels2b 

292 # ax2 is "in front" of ax1 because it has the vlines plotted on it, and 

293 # vlines are on ax2 so that they appear at the bottom of the legend, so 

294 # make sure to plot the legend on ax2, otherwise the vlines will go on top 

295 # of the otherwise-opaque legend. 

296 ax1_twin.legend(handles, labels, facecolor='white', framealpha=1) 

297 

298 # Add title with the event name, type etc 

299 dayObsStr = dayObsIntToString(event.dayObs) 

300 title = (f"{dayObsStr} - seqNum {event.seqNum} (version {event.version})" # top line, rest below 

301 f"\nDuration = {event.duration:.2f}s" 

302 f" Event type: {event.type.name}" 

303 f" End reason: {event.endReason.name}" 

304 ) 

305 ax1_twin.set_title(title) 

306 return fig 

307 

308 

309def getCommandsDuringEvent(client, event, commands=('raDecTarget'), log=None, doLog=True): 

310 """Get the commands issued during an event. 

311 

312 Get the times at which the specified commands were issued during the event. 

313 

314 Parameters 

315 ---------- 

316 client : `lsst_efd_client.efd_helper.EfdClient` 

317 The EFD client to use. 

318 event : `lsst.summit.utils.tmaUtils.TMAEvent` 

319 The event to plot. 

320 commands : `list` of `str`, optional 

321 The commands or command aliases to look for. Defaults to 

322 ['raDecTarget']. 

323 log : `logging.Logger`, optional 

324 The logger to use. If not specified, a new logger will be created if 

325 needed. 

326 doLog : `bool`, optional 

327 Whether to log messages. Defaults to True. 

328 

329 Returns 

330 ------- 

331 commands : `dict` of `str` : `astropy.time.Time` 

332 A dictionary of the commands and the times at which they were issued. 

333 """ 

334 # TODO: DM-40100 Add support for padding the event here to allow looking 

335 # for triggering commands before the event 

336 

337 # TODO: DM-40100 Change this to always return a list of times, and remove 

338 # warning about finding multiple commands. Remember to update docs and 

339 # plotting code. 

340 if log is None and doLog: 

341 log = logging.getLogger(__name__) 

342 

343 commands = ensure_iterable(commands) 

344 fullCommands = [c if c not in COMMAND_ALIASES else COMMAND_ALIASES[c] for c in commands] 

345 del commands # make sure we always use their full names 

346 

347 ret = {} 

348 for command in fullCommands: 

349 data = getEfdData(client, command, event=event, warn=False) 

350 if data.empty: 

351 if doLog: 

352 log.info(f'Found no command issued for {command} during event') 

353 ret[command] = None 

354 elif len(data) > 1: 

355 if doLog: 

356 log.warning(f'Found multiple commands issued for {command} during event, returning None') 

357 ret[command] = None 

358 else: 

359 assert len(data) == 1 # this must be true now 

360 commandTime = data.private_efdStamp 

361 ret[command] = Time(commandTime, format='unix') 

362 

363 return ret 

364 

365 

366def _initializeTma(tma): 

367 """Helper function to turn a TMA into a valid state for testing. 

368 

369 Do not call directly in normal usage or code, as this just arbitrarily 

370 sets values to make the TMA valid. 

371 

372 Parameters 

373 ---------- 

374 tma : `lsst.summit.utils.tmaUtils.TMAStateMachine` 

375 The TMA state machine model to initialize. 

376 """ 

377 tma._parts['azimuthInPosition'] = False 

378 tma._parts['azimuthMotionState'] = AxisMotionState.STOPPED 

379 tma._parts['azimuthSystemState'] = PowerState.ON 

380 tma._parts['elevationInPosition'] = False 

381 tma._parts['elevationMotionState'] = AxisMotionState.STOPPED 

382 tma._parts['elevationSystemState'] = PowerState.ON 

383 

384 

385@dataclass(slots=True, kw_only=True, frozen=True) 

386class BlockInfo: 

387 """The block info relating to a TMAEvent. 

388 

389 Parameters 

390 ---------- 

391 blockNumber : `int` 

392 The block number, as an integer. 

393 blockId : `str` 

394 The block ID, as a string. 

395 salIndices : `list` of `int` 

396 One or more SAL indices, relating to the block. 

397 tickets : `list` of `str` 

398 One or more SITCOM tickets, relating to the block. 

399 states : `list` of `ScriptStatePoint` 

400 The states of the script during the block. Each element is a 

401 ``ScriptStatePoint`` which contains: 

402 - the time, as an astropy.time.Time 

403 - the state, as a ``ScriptState`` enum 

404 - the reason for state change, as a string 

405 """ 

406 blockNumber: int 

407 blockId: str 

408 salIndices: int 

409 tickets: list 

410 states: list 

411 

412 def __repr__(self): 

413 return ( 

414 f"BlockInfo(blockNumber={self.blockNumber}, blockId={self.blockId}, salIndices={self.salIndices}," 

415 f" tickets={self.tickets}, states={self.states!r}" 

416 ) 

417 

418 def _ipython_display_(self): 

419 print(self.__str__()) 

420 

421 def __str__(self): 

422 # You can't put the characters '\n' directly into the evaluated part of 

423 # an f-string i.e. inside the {} part, until py 3.12, so this must go 

424 # in via a variable until then. 

425 newline = ' \n' 

426 

427 return ( 

428 f"blockNumber: {self.blockNumber}\n" 

429 f"blockId: {self.blockId}\n" 

430 f"salIndices: {self.salIndices}\n" 

431 f"tickets: {self.tickets}\n" 

432 f"states: \n{newline.join([str(state) for state in self.states])}" 

433 ) 

434 

435 

436@dataclass(slots=True, kw_only=True, frozen=True) 

437class ScriptStatePoint: 

438 time: Time 

439 state: ScriptState 

440 reason: str 

441 

442 def __repr__(self): 

443 return ( 

444 f"ScriptStatePoint(time={self.time!r}, state={self.state!r}, reason={self.reason!r})" 

445 ) 

446 

447 def _ipython_display_(self): 

448 print(self.__str__()) 

449 

450 def __str__(self): 

451 reasonStr = f" - {self.reason}" if self.reason else "" 

452 return (f"{self.state.name:>10} @ {self.time.isot}{reasonStr}") 

453 

454 

455@dataclass(slots=True, kw_only=True, frozen=True) 

456class TMAEvent: 

457 """A movement event for the TMA. 

458 

459 Contains the dayObs on which the event occured, using the standard 

460 observatory definition of the dayObs, and the sequence number of the event, 

461 which is unique for each event on a given dayObs. 

462 

463 The event type can be either 'SLEWING' or 'TRACKING', defined as: 

464 - SLEWING: some part of the TMA is in motion 

465 - TRACKING: both axes are in position and tracking the sky 

466 

467 The end reason can be 'STOPPED', 'TRACKING', 'FAULT', 'SLEWING', or 'OFF'. 

468 - SLEWING: The previous event was a TRACKING event, and one or more of 

469 the TMA components either stopped being in position, or stopped 

470 moving, or went into fault, or was turned off, and hence we are now 

471 only slewing and no longer tracking the sky. 

472 - TRACKING: the TMA started tracking the sky when it wasn't previously. 

473 Usualy this would always be preceded by directly by a SLEWING 

474 event, but this is not strictly true, as the EUI seems to be able 

475 to make the TMA start tracking the sky without slewing first. 

476 - STOPPED: the components of the TMA transitioned to the STOPPED state. 

477 - FAULT: the TMA went into fault. 

478 - OFF: the TMA components were turned off. 

479 

480 Note that this class is not intended to be instantiated directly, but 

481 rather to be returned by the ``TMAEventMaker.getEvents()`` function. 

482 

483 Parameters 

484 ---------- 

485 dayObs : `int` 

486 The dayObs on which the event occured. 

487 seqNum : `int` 

488 The sequence number of the event, 

489 type : `lsst.summit.utils.tmaUtils.TMAState` 

490 The type of the event, either 'SLEWING' or 'TRACKING'. 

491 endReason : `lsst.summit.utils.tmaUtils.TMAState` 

492 The reason the event ended, either 'STOPPED', 'TRACKING', 'FAULT', 

493 'SLEWING', or 'OFF'. 

494 duration : `float` 

495 The duration of the event, in seconds. 

496 begin : `astropy.time.Time` 

497 The time the event began. 

498 end : `astropy.time.Time` 

499 The time the event ended. 

500 blockInfo : `lsst.summit.utils.tmaUtils.BlockInfo` 

501 The block info relating to the event. 

502 version : `int` 

503 The version of the TMAEvent class. Equality between events is only 

504 valid for a given version of the class. If the class definition 

505 changes, the time ranges can change, and hence the equality between 

506 events is ``False``. 

507 _startRow : `int` 

508 The first row in the merged EFD data which is part of the event. 

509 _endRow : `int` 

510 The last row in the merged EFD data which is part of the event. 

511 """ 

512 dayObs: int 

513 seqNum: int 

514 type: str # can be 'SLEWING', 'TRACKING' 

515 endReason: str # can be 'STOPPED', 'TRACKING', 'FAULT', 'SLEWING', 'OFF' 

516 duration: float # seconds 

517 begin: Time 

518 end: Time 

519 blockInfo: BlockInfo = None 

520 version: int = 0 # update this number any time a code change which could change event definitions is made 

521 _startRow: int 

522 _endRow: int 

523 

524 def __lt__(self, other): 

525 if self.version != other.version: 

526 raise ValueError( 

527 f"Cannot compare TMAEvents with different versions: {self.version} != {other.version}" 

528 ) 

529 if self.dayObs < other.dayObs: 

530 return True 

531 elif self.dayObs == other.dayObs: 

532 return self.seqNum < other.seqNum 

533 return False 

534 

535 def __repr__(self): 

536 return ( 

537 f"TMAEvent(dayObs={self.dayObs}, seqNum={self.seqNum}, type={self.type!r}," 

538 f" endReason={self.endReason!r}, duration={self.duration}, begin={self.begin!r}," 

539 f" end={self.end!r}" 

540 ) 

541 

542 def _ipython_display_(self): 

543 print(self.__str__()) 

544 

545 def __str__(self): 

546 return ( 

547 f"dayObs: {self.dayObs}\nseqNum: {self.seqNum}\ntype: {self.type.name}" 

548 f"\nendReason: {self.endReason.name}\nduration: {self.duration}\nbegin: {self.begin!r}," 

549 f"\nend: {self.end!r}" 

550 ) 

551 

552 

553class TMAState(enum.IntEnum): 

554 """Overall state of the TMA. 

555 

556 States are defined as follows: 

557 

558 UNINITIALIZED 

559 We have not yet got data for all relevant components, so the overall 

560 state is undefined. 

561 STOPPED 

562 All components are on, and none are moving. 

563 TRACKING 

564 We are tracking the sky. 

565 SLEWING 

566 One or more components are moving, and one or more are not tracking the 

567 sky. This should probably be called MOVING, as it includes: slewing, 

568 MOVING_POINT_TO_POINT, and JOGGING. 

569 FAULT 

570 All (if engineeringMode) or any (if not engineeringMode) components are 

571 in fault. 

572 OFF 

573 All components are off. 

574 """ 

575 UNINITIALIZED = -1 

576 STOPPED = 0 

577 TRACKING = 1 

578 SLEWING = 2 

579 FAULT = 3 

580 OFF = 4 

581 

582 def __repr__(self): 

583 return f"TMAState.{self.name}" 

584 

585 

586def getAxisAndType(rowFor): 

587 """Get the axis the data relates to, and the type of data it contains. 

588 

589 Parameters 

590 ---------- 

591 rowFor : `str` 

592 The column in the dataframe denoting what this row is for, e.g. 

593 "elevationMotionState" or "azimuthInPosition", etc. 

594 

595 Returns 

596 ------- 

597 axis : `str` 

598 The axis the row is for, e.g. "azimuth", "elevation". 

599 rowType : `str` 

600 The type of the row, e.g. "MotionState", "SystemState", "InPosition". 

601 """ 

602 regex = r'(azimuth|elevation)(InPosition|MotionState|SystemState)$' # matches the end of the line 

603 matches = re.search(regex, rowFor) 

604 if matches is None: 

605 raise ValueError(f"Could not parse axis and rowType from {rowFor=}") 

606 axis = matches.group(1) 

607 rowType = matches.group(2) 

608 

609 assert rowFor.endswith(f"{axis}{rowType}") 

610 return axis, rowType 

611 

612 

613class ListViewOfDict: 

614 """A class to allow making lists which contain references to an underlying 

615 dictionary. 

616 

617 Normally, making a list of items from a dictionary would make a copy of the 

618 items, but this class allows making a list which contains references to the 

619 underlying dictionary items themselves. This is useful for making a list of 

620 components, such that they can be manipulated in their logical sets. 

621 """ 

622 def __init__(self, underlyingDictionary, keysToLink): 

623 self.dictionary = underlyingDictionary 

624 self.keys = keysToLink 

625 

626 def __getitem__(self, index): 

627 return self.dictionary[self.keys[index]] 

628 

629 def __setitem__(self, index, value): 

630 self.dictionary[self.keys[index]] = value 

631 

632 def __len__(self): 

633 return len(self.keys) 

634 

635 

636class TMAStateMachine: 

637 """A state machine model of the TMA. 

638 

639 Note that this is currently only implemented for the azimuth and elevation 

640 axes, but will be extended to include the rotator in the future. 

641 

642 Note that when used for event generation, changing ``engineeringMode`` to 

643 False might change the resulting list of events, and that if the TMA moves 

644 with some axis in fault, then these events will be missed. It is therefore 

645 thought that ``engineeringMode=True`` should always be used when generating 

646 events. The option, however, is there for completeness, as this will be 

647 useful for knowing is the CSC would consider the TMA to be in fault in the 

648 general case. 

649 

650 Parameters 

651 ---------- 

652 engineeringMode : `bool`, optional 

653 Whether the TMA is in engineering mode. Defaults to True. If False, 

654 then the TMA will be in fault if any component is in fault. If True, 

655 then the TMA will be in fault only if all components are in fault. 

656 debug : `bool`, optional 

657 Whether to log debug messages. Defaults to False. 

658 """ 

659 _UNINITIALIZED_VALUE: int = -999 

660 

661 def __init__(self, engineeringMode=True, debug=False): 

662 self.engineeringMode = engineeringMode 

663 self.log = logging.getLogger('lsst.summit.utils.tmaUtils.TMA') 

664 if debug: 

665 self.log.level = logging.DEBUG 

666 self._mostRecentRowTime = -1 

667 

668 # the actual components of the TMA 

669 self._parts = {'azimuthInPosition': self._UNINITIALIZED_VALUE, 

670 'azimuthMotionState': self._UNINITIALIZED_VALUE, 

671 'azimuthSystemState': self._UNINITIALIZED_VALUE, 

672 'elevationInPosition': self._UNINITIALIZED_VALUE, 

673 'elevationMotionState': self._UNINITIALIZED_VALUE, 

674 'elevationSystemState': self._UNINITIALIZED_VALUE, 

675 } 

676 systemKeys = ['azimuthSystemState', 'elevationSystemState'] 

677 positionKeys = ['azimuthInPosition', 'elevationInPosition'] 

678 motionKeys = ['azimuthMotionState', 'elevationMotionState'] 

679 

680 # references to the _parts as conceptual groupings 

681 self.system = ListViewOfDict(self._parts, systemKeys) 

682 self.motion = ListViewOfDict(self._parts, motionKeys) 

683 self.inPosition = ListViewOfDict(self._parts, positionKeys) 

684 

685 # tuples of states for state collapsing. Note that STOP_LIKE + 

686 # MOVING_LIKE must cover the full set of AxisMotionState enums 

687 self.STOP_LIKE = (AxisMotionState.STOPPING, 

688 AxisMotionState.STOPPED, 

689 AxisMotionState.TRACKING_PAUSED) 

690 self.MOVING_LIKE = (AxisMotionState.MOVING_POINT_TO_POINT, 

691 AxisMotionState.JOGGING, 

692 AxisMotionState.TRACKING) 

693 # Likewise, ON_LIKE + OFF_LIKE must cover the full set of PowerState 

694 # enums 

695 self.OFF_LIKE = (PowerState.OFF, PowerState.TURNING_OFF) 

696 self.ON_LIKE = (PowerState.ON, PowerState.TURNING_ON) 

697 self.FAULT_LIKE = (PowerState.FAULT,) # note the trailing comma - this must be an iterable 

698 

699 def apply(self, row): 

700 """Apply a row of data to the TMA state. 

701 

702 Checks that the row contains data for a later time than any data 

703 previously applied, and applies the relevant column entry to the 

704 relevant component. 

705 

706 Parameters 

707 ---------- 

708 row : `pd.Series` 

709 The row of data to apply to the state machine. 

710 """ 

711 timestamp = row['private_efdStamp'] 

712 if timestamp < self._mostRecentRowTime: # NB equals is OK, technically, though it never happens 

713 raise ValueError('TMA evolution must be monotonic increasing in time, tried to apply a row which' 

714 ' predates the most previous one') 

715 self._mostRecentRowTime = timestamp 

716 

717 rowFor = row['rowFor'] # e.g. elevationMotionState 

718 axis, rowType = getAxisAndType(rowFor) # e.g. elevation, MotionState 

719 value = self._getRowPayload(row, rowType, rowFor) 

720 self.log.debug(f"Setting {rowFor} to {repr(value)}") 

721 self._parts[rowFor] = value 

722 try: 

723 # touch the state property as this executes the sieving, to make 

724 # sure we don't fall through the sieve at any point in time 

725 _ = self.state 

726 except RuntimeError as e: 

727 # improve error reporting, but always reraise this, as this is a 

728 # full-blown failure 

729 raise RuntimeError(f'Failed to apply {value} to {axis}{rowType} with state {self._parts}') from e 

730 

731 def _getRowPayload(self, row, rowType, rowFor): 

732 """Get the relevant value from the row. 

733 

734 Given the row, and which component it relates to, get the relevant 

735 value, as a bool or cast to the appropriate enum class. 

736 

737 Parameters 

738 ---------- 

739 row : `pd.Series` 

740 The row of data from the dataframe. 

741 rowType : `str` 

742 The type of the row, e.g. "MotionState", "SystemState", 

743 "InPosition". 

744 rowFor : `str` 

745 The component the row is for, e.g. "azimuth", "elevation". 

746 

747 Returns 

748 ------- 

749 value : `bool` or `enum` 

750 The value of the row, as a bool or enum, depending on the 

751 component, cast to the appropriate enum class or bool. 

752 """ 

753 match rowType: 

754 case 'MotionState': 

755 value = row[f'state_{rowFor}'] 

756 return AxisMotionState(value) 

757 case 'SystemState': 

758 value = row[f'powerState_{rowFor}'] 

759 return PowerState(value) 

760 case 'InPosition': 

761 value = row[f'inPosition_{rowFor}'] 

762 return bool(value) 

763 case _: 

764 raise ValueError(f'Failed to get row payload with {rowType=} and {row=}') 

765 

766 @property 

767 def _isValid(self): 

768 """Has the TMA had a value applied to all its components? 

769 

770 If any component has not yet had a value applied, the TMA is not valid, 

771 as those components will be in an unknown state. 

772 

773 Returns 

774 ------- 

775 isValid : `bool` 

776 Whether the TMA is fully initialized. 

777 """ 

778 return not any([v == self._UNINITIALIZED_VALUE for v in self._parts.values()]) 

779 

780 # state inspection properties - a high level way of inspecting the state as 

781 # an API 

782 @property 

783 def isMoving(self): 

784 return self.state in [TMAState.TRACKING, TMAState.SLEWING] 

785 

786 @property 

787 def isNotMoving(self): 

788 return not self.isMoving 

789 

790 @property 

791 def isTracking(self): 

792 return self.state == TMAState.TRACKING 

793 

794 @property 

795 def isSlewing(self): 

796 return self.state == TMAState.SLEWING 

797 

798 @property 

799 def canMove(self): 

800 badStates = [PowerState.OFF, PowerState.TURNING_OFF, PowerState.FAULT, PowerState.UNKNOWN] 

801 return bool( 

802 self._isValid and 

803 self._parts['azimuthSystemState'] not in badStates and 

804 self._parts['elevationSystemState'] not in badStates 

805 ) 

806 

807 # Axis inspection properties, designed for internal use. These return 

808 # iterables so that they can be used in any() and all() calls, which make 

809 # the logic much easier to read, e.g. to see if anything is moving, we can 

810 # write `if not any(_axisInMotion):` 

811 @property 

812 def _axesInFault(self): 

813 return [x in self.FAULT_LIKE for x in self.system] 

814 

815 @property 

816 def _axesOff(self): 

817 return [x in self.OFF_LIKE for x in self.system] 

818 

819 @property 

820 def _axesOn(self): 

821 return [not x for x in self._axesOn] 

822 

823 @property 

824 def _axesInMotion(self): 

825 return [x in self.MOVING_LIKE for x in self.motion] 

826 

827 @property 

828 def _axesTRACKING(self): 

829 """Note this is deliberately named _axesTRACKING and not _axesTracking 

830 to make it clear that this is the AxisMotionState type of TRACKING and 

831 not the normal conceptual notion of tracking (the sky, i.e. as opposed 

832 to slewing). 

833 """ 

834 return [x == AxisMotionState.TRACKING for x in self.motion] 

835 

836 @property 

837 def _axesInPosition(self): 

838 return [x is True for x in self.inPosition] 

839 

840 @property 

841 def state(self): 

842 """The overall state of the TMA. 

843 

844 Note that this is both a property, and also the method which applies 

845 the logic sieve to determine the state at a given point in time. 

846 

847 Returns 

848 ------- 

849 state : `lsst.summit.utils.tmaUtils.TMAState` 

850 The overall state of the TMA. 

851 """ 

852 # first, check we're valid, and if not, return UNINITIALIZED state, as 

853 # things are unknown 

854 if not self._isValid: 

855 return TMAState.UNINITIALIZED 

856 

857 # if we're not in engineering mode, i.e. we're under normal CSC 

858 # control, then if anything is in fault, we're in fault. If we're 

859 # engineering then some axes will move when others are in fault 

860 if not self.engineeringMode: 

861 if any(self._axesInFault): 

862 return TMAState.FAULT 

863 else: 

864 # we're in engineering mode, so return fault state if ALL are in 

865 # fault 

866 if all(self._axesInFault): 

867 return TMAState.FAULT 

868 

869 # if all axes are off, the TMA is OFF 

870 if all(self._axesOff): 

871 return TMAState.OFF 

872 

873 # we know we're valid and at least some axes are not off, so see if 

874 # we're in motion if no axes are moving, we're stopped 

875 if not any(self._axesInMotion): 

876 return TMAState.STOPPED 

877 

878 # now we know we're initialized, and that at least one axis is moving 

879 # so check axes for motion and in position. If all axes are tracking 

880 # and all are in position, we're tracking the sky 

881 if (all(self._axesTRACKING) and all(self._axesInPosition)): 

882 return TMAState.TRACKING 

883 

884 # we now know explicitly that not everything is in position, so we no 

885 # longer need to check that. We do actually know that something is in 

886 # motion, but confirm that's the case and return SLEWING 

887 if (any(self._axesInMotion)): 

888 return TMAState.SLEWING 

889 

890 # if we want to differentiate between MOVING_POINT_TO_POINT moves, 

891 # JOGGING moves and regular slews, the logic in the step above needs to 

892 # be changed and the new steps added here. 

893 

894 raise RuntimeError('State error: fell through the state sieve - rewrite your logic!') 

895 

896 

897class TMAEventMaker: 

898 """A class to create per-dayObs TMAEvents for the TMA's movements. 

899 

900 Example usage: 

901 >>> dayObs = 20230630 

902 >>> eventMaker = TMAEventMaker() 

903 >>> events = eventMaker.getEvents(dayObs) 

904 >>> print(f'Found {len(events)} for {dayObs=}') 

905 

906 Parameters 

907 ---------- 

908 client : `lsst_efd_client.efd_helper.EfdClient`, optional 

909 The EFD client to use, created if not provided. 

910 """ 

911 # the topics which need logical combination to determine the overall mount 

912 # state. Will need updating as new components are added to the system. 

913 

914 # relevant column: 'state' 

915 _movingComponents = [ 

916 'lsst.sal.MTMount.logevent_azimuthMotionState', 

917 'lsst.sal.MTMount.logevent_elevationMotionState', 

918 ] 

919 

920 # relevant column: 'inPosition' 

921 _inPositionComponents = [ 

922 'lsst.sal.MTMount.logevent_azimuthInPosition', 

923 'lsst.sal.MTMount.logevent_elevationInPosition', 

924 ] 

925 

926 # the components which, if in fault, put the TMA into fault 

927 # relevant column: 'powerState' 

928 _stateComponents = [ 

929 'lsst.sal.MTMount.logevent_azimuthSystemState', 

930 'lsst.sal.MTMount.logevent_elevationSystemState', 

931 ] 

932 

933 def __init__(self, client=None): 

934 if client is not None: 

935 self.client = client 

936 else: 

937 self.client = makeEfdClient() 

938 self.log = logging.getLogger(__name__) 

939 self._data = {} 

940 

941 @dataclass(frozen=True) 

942 class ParsedState: 

943 eventStart: Time 

944 eventEnd: int 

945 previousState: TMAState 

946 state: TMAState 

947 

948 @staticmethod 

949 def isToday(dayObs): 

950 """Find out if the specified dayObs is today, or in the past. 

951 

952 If the day is today, the function returns ``True``, if it is in the 

953 past it returns ``False``. If the day is in the future, a 

954 ``ValueError`` is raised, as this indicates there is likely an 

955 off-by-one type error somewhere in the logic. 

956 

957 Parameters 

958 ---------- 

959 dayObs : `int` 

960 The dayObs to check, in the format YYYYMMDD. 

961 

962 Returns 

963 ------- 

964 isToday : `bool` 

965 ``True`` if the dayObs is today, ``False`` if it is in the past. 

966 

967 Raises 

968 ValueError: if the dayObs is in the future. 

969 """ 

970 todayDayObs = getCurrentDayObs_int() 

971 if dayObs == todayDayObs: 

972 return True 

973 if dayObs > todayDayObs: 

974 raise ValueError("dayObs is in the future") 

975 return False 

976 

977 @staticmethod 

978 def _shortName(topic): 

979 """Get the short name of a topic. 

980 

981 Parameters 

982 ---------- 

983 topic : `str` 

984 The topic to get the short name of. 

985 

986 Returns 

987 ------- 

988 shortName : `str` 

989 The short name of the topic, e.g. 'azimuthInPosition' 

990 """ 

991 # get, for example 'azimuthInPosition' from 

992 # lsst.sal.MTMount.logevent_azimuthInPosition 

993 return topic.split('_')[-1] 

994 

995 def _mergeData(self, data): 

996 """Merge a dict of dataframes based on private_efdStamp, recording 

997 where each row came from. 

998 

999 Given a dict or dataframes, keyed by topic, merge them into a single 

1000 dataframe, adding a column to record which topic each row came from. 

1001 

1002 Parameters 

1003 ---------- 

1004 data : `dict` of `str` : `pd.DataFrame` 

1005 The dataframes to merge. 

1006 

1007 Returns 

1008 ------- 

1009 merged : `pd.DataFrame` 

1010 The merged dataframe. 

1011 """ 

1012 excludeColumns = ['private_efdStamp', 'rowFor'] 

1013 

1014 mergeArgs = { 

1015 'how': 'outer', 

1016 'sort': True, 

1017 } 

1018 

1019 merged = None 

1020 originalRowCounter = 0 

1021 

1022 # Iterate over the keys and merge the corresponding DataFrames 

1023 for key, df in data.items(): 

1024 if df.empty: 

1025 # Must skip the df if it's empty, otherwise the merge will fail 

1026 # due to lack of private_efdStamp. Because other axes might 

1027 # still be in motion, so we still want to merge what we have 

1028 continue 

1029 

1030 originalRowCounter += len(df) 

1031 component = self._shortName(key) # Add suffix to column names to identify the source 

1032 suffix = '_' + component 

1033 

1034 df['rowFor'] = component 

1035 

1036 columnsToSuffix = [col for col in df.columns if col not in excludeColumns] 

1037 df_to_suffix = df[columnsToSuffix].add_suffix(suffix) 

1038 df = pd.concat([df[excludeColumns], df_to_suffix], axis=1) 

1039 

1040 if merged is None: 

1041 merged = df.copy() 

1042 else: 

1043 merged = pd.merge(merged, df, **mergeArgs) 

1044 

1045 merged = merged.loc[:, ~merged.columns.duplicated()] # Remove duplicate columns after merge 

1046 

1047 if len(merged) != originalRowCounter: 

1048 self.log.warning("Merged data has a different number of rows to the original data, some" 

1049 " timestamps (rows) will contain more than one piece of actual information.") 

1050 return merged 

1051 

1052 def getEvents(self, dayObs): 

1053 """Get the TMA events for the specified dayObs. 

1054 

1055 Gets the required mount data from the cache or the EFD as required, 

1056 handling whether we're working with live vs historical data. The 

1057 dataframes from the EFD is merged and applied to the TMAStateMachine, 

1058 and that series of state changes is used to generate a list of 

1059 TmaEvents for the day's data. 

1060 

1061 If the data is for the current day, i.e. if new events can potentially 

1062 land, then if the last event is "open" (meaning that the TMA appears to 

1063 be in motion and thus the event is growing with time), then that event 

1064 is excluded from the event list as it is expected to be changing with 

1065 time, and will likely close eventually. However, if that situation 

1066 occurs on a day in the past, then that event can never close, and the 

1067 event is therefore included, but a warning about the open event is 

1068 logged. 

1069 

1070 Parameters 

1071 ---------- 

1072 dayObs : `int` 

1073 The dayObs for which to get the events. 

1074 

1075 Returns 

1076 ------- 

1077 events : `list` of `lsst.summit.utils.tmaUtils.TMAState` 

1078 The events for the specified dayObs. 

1079 """ 

1080 workingLive = self.isToday(dayObs) 

1081 data = None 

1082 

1083 if workingLive: 

1084 # it's potentially updating data, so we must update the date 

1085 # regarless of whether we have it already or not 

1086 self.log.info(f'Updating mount data for {dayObs} from the EFD') 

1087 self._getEfdDataForDayObs(dayObs) 

1088 data = self._data[dayObs] 

1089 elif dayObs in self._data: 

1090 # data is in the cache and it's not being updated, so use it 

1091 data = self._data[dayObs] 

1092 elif dayObs not in self._data: 

1093 # we don't have the data yet, but it's not growing, so put it in 

1094 # the cache and use it from there 

1095 self.log.info(f'Retrieving mount data for {dayObs} from the EFD') 

1096 self._getEfdDataForDayObs(dayObs) 

1097 data = self._data[dayObs] 

1098 else: 

1099 raise RuntimeError("This should never happen") 

1100 

1101 # if we don't have something to work with, log a warning and return 

1102 if not self.dataFound(data): 

1103 self.log.warning(f"No EFD data found for {dayObs=}") 

1104 return [] 

1105 

1106 # applies the data to the state machine, and generates events from the 

1107 # series of states which results 

1108 events = self._calculateEventsFromMergedData(data, dayObs, dataIsForCurrentDay=workingLive) 

1109 if not events: 

1110 self.log.warning(f"Failed to calculate any events for {dayObs=} despite EFD data existing!") 

1111 return events 

1112 

1113 @staticmethod 

1114 def dataFound(data): 

1115 """Check if any data was found. 

1116 

1117 Parameters 

1118 ---------- 

1119 data : `pd.DataFrame` 

1120 The merged dataframe to check. 

1121 

1122 Returns 

1123 ------- 

1124 dataFound : `bool` 

1125 Whether data was found. 

1126 """ 

1127 # You can't just compare to with data == NO_DATA_SENTINEL because 

1128 # `data` is usually a dataframe, and you can't compare a dataframe to a 

1129 # string directly. 

1130 return not (isinstance(data, str) and data == NO_DATA_SENTINEL) 

1131 

1132 def _getEfdDataForDayObs(self, dayObs): 

1133 """Get the EFD data for the specified dayObs and store it in the cache. 

1134 

1135 Gets the EFD data for all components, as a dict of dataframes keyed by 

1136 component name. These are then merged into a single dataframe in time 

1137 order, based on each row's `private_efdStamp`. This is then stored in 

1138 self._data[dayObs]. 

1139 

1140 If no data is found, the value is set to ``NO_DATA_SENTINEL`` to 

1141 differentiate this from ``None``, as this is what you'd get if you 

1142 queried the cache with `self._data.get(dayObs)`. It also marks that we 

1143 have already queried this day. 

1144 

1145 Parameters 

1146 ---------- 

1147 dayObs : `int` 

1148 The dayObs to query. 

1149 """ 

1150 data = {} 

1151 for component in itertools.chain( 

1152 self._movingComponents, 

1153 self._inPositionComponents, 

1154 self._stateComponents 

1155 ): 

1156 data[component] = getEfdData(self.client, component, dayObs=dayObs, warn=False) 

1157 self.log.debug(f"Found {len(data[component])} for {component}") 

1158 

1159 if all(dataframe.empty for dataframe in data.values()): 

1160 # if every single dataframe is empty, set the sentinel and don't 

1161 # try to merge anything, otherwise merge all the data we found 

1162 self.log.debug(f"No data found for {dayObs=}") 

1163 # a sentinel value that's not None 

1164 self._data[dayObs] = NO_DATA_SENTINEL 

1165 else: 

1166 merged = self._mergeData(data) 

1167 self._data[dayObs] = merged 

1168 

1169 def _calculateEventsFromMergedData(self, data, dayObs, dataIsForCurrentDay): 

1170 """Calculate the list of events from the merged data. 

1171 

1172 Runs the merged data, row by row, through the TMA state machine (with 

1173 ``tma.apply``) to get the overall TMA state at each row, building a 

1174 dict of these states, keyed by row number. 

1175 

1176 This time-series of TMA states are then looped over (in 

1177 `_statesToEventTuples`), building a list of tuples representing the 

1178 start and end of each event, the type of the event, and the reason for 

1179 the event ending. 

1180 

1181 This list of tuples is then passed to ``_makeEventsFromStateTuples``, 

1182 which actually creates the ``TMAEvent`` objects. 

1183 

1184 Parameters 

1185 ---------- 

1186 data : `pd.DataFrame` 

1187 The merged dataframe to use. 

1188 dayObs : `int` 

1189 The dayObs for the data. 

1190 dataIsForCurrentDay : `bool` 

1191 Whether the data is for the current day. Determines whether to 

1192 allow an open last event or not. 

1193 

1194 Returns 

1195 ------- 

1196 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent` 

1197 The events for the specified dayObs. 

1198 """ 

1199 engineeringMode = True 

1200 tma = TMAStateMachine(engineeringMode=engineeringMode) 

1201 

1202 # For now, we assume that the TMA starts each day able to move, but 

1203 # stationary. If this turns out to cause problems, we will need to 

1204 # change to loading data from the previous day(s), and looking back 

1205 # through it in time until a state change has been found for every 

1206 # axis. For now though, Bruno et. al think this is acceptable and 

1207 # preferable. 

1208 _initializeTma(tma) 

1209 

1210 tmaStates = {} 

1211 for rowNum, row in data.iterrows(): 

1212 tma.apply(row) 

1213 tmaStates[rowNum] = tma.state 

1214 

1215 stateTuples = self._statesToEventTuples(tmaStates, dataIsForCurrentDay) 

1216 events = self._makeEventsFromStateTuples(stateTuples, dayObs, data) 

1217 self.addBlockDataToEvents(events) 

1218 return events 

1219 

1220 def _statesToEventTuples(self, states, dataIsForCurrentDay): 

1221 """Get the event-tuples from the dictionary of TMAStates. 

1222 

1223 Chunks the states into blocks of the same state, so that we can create 

1224 an event for each block in `_makeEventsFromStateTuples`. Off-type 

1225 states are skipped over, with each event starting when the telescope 

1226 next resumes motion or changes to a different type of motion state, 

1227 i.e. from non-tracking type movement (MOVE_POINT_TO_POINT, JOGGING, 

1228 TRACKING-but-not-in-position, i.e. slewing) to a tracking type 

1229 movement, or vice versa. 

1230 

1231 Parameters 

1232 ---------- 

1233 states : `dict` of `int` : `lsst.summit.utils.tmaUtils.TMAState` 

1234 The states of the TMA, keyed by row number. 

1235 dataIsForCurrentDay : `bool` 

1236 Whether the data is for the current day. Determines whether to 

1237 allow and open last event or not. 

1238 

1239 Returns 

1240 ------- 

1241 parsedStates : `list` of `tuple` 

1242 The parsed states, as a list of tuples of the form: 

1243 ``(eventStart, eventEnd, eventType, endReason)`` 

1244 """ 

1245 # Consider rewriting this with states as a list and using pop(0)? 

1246 skipStates = (TMAState.STOPPED, TMAState.OFF, TMAState.FAULT) 

1247 

1248 parsedStates = [] 

1249 eventStart = None 

1250 rowNum = 0 

1251 nRows = len(states) 

1252 while rowNum < nRows: 

1253 previousState = None 

1254 state = states[rowNum] 

1255 # if we're not in an event, fast forward through off-like rows 

1256 # until a new event starts 

1257 if eventStart is None and state in skipStates: 

1258 rowNum += 1 

1259 continue 

1260 

1261 # we've started a new event, so walk through it and find the end 

1262 eventStart = rowNum 

1263 previousState = state 

1264 rowNum += 1 # move to the next row before starting the while loop 

1265 if rowNum == nRows: 

1266 # we've reached the end of the data, and we're still in an 

1267 # event, so don't return this presumably in-progress event 

1268 self.log.warning('Reached the end of the data while starting a new event') 

1269 break 

1270 state = states[rowNum] 

1271 while state == previousState: 

1272 rowNum += 1 

1273 if rowNum == nRows: 

1274 break 

1275 state = states[rowNum] 

1276 parsedStates.append( 

1277 self.ParsedState( 

1278 eventStart=eventStart, 

1279 eventEnd=rowNum, 

1280 previousState=previousState, 

1281 state=state 

1282 ) 

1283 ) 

1284 if state in skipStates: 

1285 eventStart = None 

1286 

1287 # done parsing, just check the last event is valid 

1288 if parsedStates: # ensure we have at least one event 

1289 lastEvent = parsedStates[-1] 

1290 if lastEvent.eventEnd == nRows: 

1291 # Generally, you *want* the timespan for an event to be the 

1292 # first row of the next event, because you were in that state 

1293 # right up until that state change. However, if that event is 

1294 # a) the last one of the day and b) runs right up until the end 

1295 # of the dataframe, then there isn't another row, so this will 

1296 # overrun the array. 

1297 # 

1298 # If the data is for the current day then this isn't a worry, 

1299 # as we're likely still taking data, and this event will likely 

1300 # close yet, so we don't issue a warning, and simply drop the 

1301 # event from the list. 

1302 

1303 # However, if the data is for a past day then no new data will 

1304 # come to close the event, so allow the event to be "open", and 

1305 # issue a warning 

1306 if dataIsForCurrentDay: 

1307 self.log.info("Discarding open (likely in-progess) final event from current day's events") 

1308 parsedStates = parsedStates[:-1] 

1309 else: 

1310 self.log.warning("Last event ends open, forcing it to end at end of the day's data") 

1311 # it's a tuple, so (deliberately) awkward to modify 

1312 parsedStates[-1] = self.ParsedState( 

1313 eventStart=lastEvent.eventStart, 

1314 eventEnd=lastEvent.eventEnd - 1, 

1315 previousState=lastEvent.previousState, 

1316 state=lastEvent.state 

1317 ) 

1318 

1319 return parsedStates 

1320 

1321 def addBlockDataToEvents(self, events): 

1322 """Find all the block data in the EFD for the specified events. 

1323 

1324 Finds all the block data in the EFD relating to the events, parses it, 

1325 from the rows of the dataframe, and adds it to the events in place. 

1326 

1327 Parameters 

1328 ---------- 

1329 events : `lsst.summit.utils.tmaUtils.TMAEvent` or 

1330 `list` of `lsst.summit.utils.tmaUtils.TMAEvent` 

1331 One or more events to get the block data for. 

1332 """ 

1333 events = ensure_iterable(events) 

1334 events = sorted(events) 

1335 

1336 # Get all the data in one go and then clip to the events in the loop. 

1337 # This is orders of magnitude faster than querying individually. 

1338 allData = getEfdData(self.client, 

1339 "lsst.sal.Script.logevent_state", 

1340 begin=events[0].begin, # time ordered, so this is the start of the window 

1341 end=events[-1].end, # and this is the end 

1342 warn=False) 

1343 if allData.empty: 

1344 self.log.info('No block data found for the specified events') 

1345 return {} 

1346 

1347 blockPattern = r"BLOCK-(\d+)" 

1348 blockIdPattern = r"BL\d+(?:_\w+)+" 

1349 sitcomPattern = r"SITCOM-(\d+)" 

1350 

1351 for event in events: 

1352 eventData = clipDataToEvent(allData, event) 

1353 if eventData.empty: 

1354 continue 

1355 

1356 blockNums = set() 

1357 blockIds = set() 

1358 tickets = set() 

1359 salIndices = set() 

1360 stateList = [] 

1361 

1362 # for each for in the data which corresponds to the event, extract 

1363 # the block number, block id, sitcom tickets, sal index and state 

1364 # some may have multiple values, some may be None, so collect in 

1365 # sets and then remove None, and validate on ones which must not 

1366 # contain duplicate values 

1367 for rowNum, row in eventData.iterrows(): 

1368 # the lastCheckpoint column contains the block number, blockId, 

1369 # and any sitcom tickets. 

1370 rowStr = row['lastCheckpoint'] 

1371 

1372 blockMatch = re.search(blockPattern, rowStr) 

1373 blockNumber = int(blockMatch.group(1)) if blockMatch else None 

1374 blockNums.add(blockNumber) 

1375 

1376 blockIdMatch = re.search(blockIdPattern, rowStr) 

1377 blockId = blockIdMatch.group(0) if blockIdMatch else None 

1378 blockIds.add(blockId) 

1379 

1380 sitcomMatches = re.findall(sitcomPattern, rowStr) 

1381 sitcomTicketNumbers = [int(match) for match in sitcomMatches] 

1382 tickets.update(sitcomTicketNumbers) 

1383 

1384 salIndices.add(row['salIndex']) 

1385 

1386 state = row['state'] 

1387 state = ScriptState(state) # cast this back to its native enum 

1388 stateReason = row['reason'] # might be empty, might contain useful error messages 

1389 stateTimestamp = efdTimestampToAstropy(row['private_efdStamp']) 

1390 scriptStatePoint = ScriptStatePoint(time=stateTimestamp, 

1391 state=state, 

1392 reason=stateReason) 

1393 stateList.append(scriptStatePoint) 

1394 

1395 # remove all the Nones from the sets, and then check the lengths 

1396 for fieldSet in (blockNums, blockIds, salIndices): 

1397 if None in fieldSet: 

1398 fieldSet.remove(None) 

1399 

1400 # if we didn't find any block numbers at all, that is fine, just 

1401 # continue as this event doesn't relate to a BLOCK 

1402 if not blockNums: 

1403 continue 

1404 

1405 # but if it does related to a BLOCK then it must not have more than 

1406 # one. If this is the case something is wrong with a SAL script, so 

1407 # raise here to indicate the something needs debugging in the 

1408 # scriptQueue or something like that. 

1409 if len(blockNums) > 1: 

1410 raise RuntimeError(f"Found multiple BLOCK values ({blockNums}) for {event}") 

1411 blockNumber = blockNums.pop() 

1412 

1413 # likewise for the blockIds 

1414 if len(blockIds) > 1: 

1415 raise RuntimeError(f"Found multiple blockIds ({blockIds}) for {event}") 

1416 blockId = blockIds.pop() 

1417 

1418 blockInfo = BlockInfo( 

1419 blockNumber=blockNumber, 

1420 blockId=blockId, 

1421 salIndices=sorted([i for i in salIndices]), 

1422 tickets=[f'SITCOM-{ticket}' for ticket in tickets], 

1423 states=stateList, 

1424 ) 

1425 

1426 # Add the blockInfo to the TMAEvent. Because this is a frozen 

1427 # dataclass, use object.__setattr__ to set the attribute. This is 

1428 # the correct way to set a frozen dataclass attribute after 

1429 # creation. 

1430 object.__setattr__(event, 'blockInfo', blockInfo) 

1431 

1432 def _makeEventsFromStateTuples(self, states, dayObs, data): 

1433 """For the list of state-tuples, create a list of ``TMAEvent`` objects. 

1434 

1435 Given the underlying data, and the start/stop points for each event, 

1436 create the TMAEvent objects for the dayObs. 

1437 

1438 Parameters 

1439 ---------- 

1440 states : `list` of `tuple` 

1441 The parsed states, as a list of tuples of the form: 

1442 ``(eventStart, eventEnd, eventType, endReason)`` 

1443 dayObs : `int` 

1444 The dayObs for the data. 

1445 data : `pd.DataFrame` 

1446 The merged dataframe. 

1447 

1448 Returns 

1449 ------- 

1450 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent` 

1451 The events for the specified dayObs. 

1452 """ 

1453 seqNum = 0 

1454 events = [] 

1455 for parsedState in states: 

1456 begin = data.iloc[parsedState.eventStart]['private_efdStamp'] 

1457 end = data.iloc[parsedState.eventEnd]['private_efdStamp'] 

1458 beginAstropy = efdTimestampToAstropy(begin) 

1459 endAstropy = efdTimestampToAstropy(end) 

1460 duration = end - begin 

1461 event = TMAEvent( 

1462 dayObs=dayObs, 

1463 seqNum=seqNum, 

1464 type=parsedState.previousState, 

1465 endReason=parsedState.state, 

1466 duration=duration, 

1467 begin=beginAstropy, 

1468 end=endAstropy, 

1469 blockInfo=None, # this is added later 

1470 _startRow=parsedState.eventStart, 

1471 _endRow=parsedState.eventEnd, 

1472 ) 

1473 events.append(event) 

1474 seqNum += 1 

1475 return events 

1476 

1477 @staticmethod 

1478 def printTmaDetailedState(tma): 

1479 """Print the full state of all the components of the TMA. 

1480 

1481 Currently this is the azimuth and elevation axes' power and motion 

1482 states, and their respective inPosition statuses. 

1483 

1484 Parameters 

1485 ---------- 

1486 tma : `lsst.summit.utils.tmaUtils.TMAStateMachine` 

1487 The TMA state machine in the state we want to print. 

1488 """ 

1489 axes = ['azimuth', 'elevation'] 

1490 p = tma._parts 

1491 axisPad = len(max(axes, key=len)) # length of the longest axis string == 9 here, but this is general 

1492 motionPad = max(len(s.name) for s in AxisMotionState) 

1493 powerPad = max(len(s.name) for s in PowerState) 

1494 

1495 # example output to show what's being done with the padding: 

1496 # azimuth - Power: ON Motion: STOPPED InPosition: True # noqa: W505 

1497 # elevation - Power: ON Motion: MOVING_POINT_TO_POINT InPosition: False # noqa: W505 

1498 for axis in axes: 

1499 print(f"{axis:>{axisPad}} - " 

1500 f"Power: {p[f'{axis}SystemState'].name:>{powerPad}} " 

1501 f"Motion: {p[f'{axis}MotionState'].name:>{motionPad}} " 

1502 f"InPosition: {p[f'{axis}InPosition']}") 

1503 print(f"Overall system state: {tma.state.name}") 

1504 

1505 def printFullDayStateEvolution(self, dayObs, taiOrUtc='utc'): 

1506 """Print the full TMA state evolution for the specified dayObs. 

1507 

1508 Replays all the data from the EFD for the specified dayObs through 

1509 the TMA state machine, and prints both the overall and detailed state 

1510 of the TMA for each row. 

1511 

1512 Parameters 

1513 ---------- 

1514 dayObs : `int` 

1515 The dayObs for which to print the state evolution. 

1516 taiOrUtc : `str`, optional 

1517 Whether to print the timestamps in TAI or UTC. Default is UTC. 

1518 """ 

1519 # create a fake event which spans the whole day, and then use 

1520 # printEventDetails code while skipping the header to print the 

1521 # evolution. 

1522 _ = self.getEvents(dayObs) # ensure the data has been retrieved from the EFD 

1523 data = self._data[dayObs] 

1524 lastRowNum = len(data) - 1 

1525 

1526 fakeEvent = TMAEvent( 

1527 dayObs=dayObs, 

1528 seqNum=-1, # anything will do 

1529 type=TMAState.OFF, # anything will do 

1530 endReason=TMAState.OFF, # anything will do 

1531 duration=-1, # anything will do 

1532 begin=efdTimestampToAstropy(data.iloc[0]['private_efdStamp']), 

1533 end=efdTimestampToAstropy(data.iloc[-1]['private_efdStamp']), 

1534 _startRow=0, 

1535 _endRow=lastRowNum 

1536 ) 

1537 self.printEventDetails(fakeEvent, taiOrUtc=taiOrUtc, printHeader=False) 

1538 

1539 def printEventDetails(self, event, taiOrUtc='tai', printHeader=True): 

1540 """Print a detailed breakdown of all state transitions during an event. 

1541 

1542 Note: this is not the most efficient way to do this, but it is much the 

1543 cleanest with respect to the actual state machine application and event 

1544 generation code, and is easily fast enough for the cases it will be 

1545 used for. It is not worth complicating the normal state machine logic 

1546 to try to use this code. 

1547 

1548 Parameters 

1549 ---------- 

1550 event : `lsst.summit.utils.tmaUtils.TMAEvent` 

1551 The event to display the details of. 

1552 taiOrUtc : `str`, optional 

1553 Whether to display time strings in TAI or UTC. Defaults to TAI. 

1554 Case insensitive. 

1555 printHeader : `bool`, optional 

1556 Whether to print the event summary. Defaults to True. The primary 

1557 reason for the existence of this option is so that this same 

1558 printing function can be used to show the evolution of a whole day 

1559 by supplying a fake event which spans the whole day, but this event 

1560 necessarily has a meaningless summary, and so needs suppressing. 

1561 """ 

1562 taiOrUtc = taiOrUtc.lower() 

1563 if taiOrUtc not in ['tai', 'utc']: 

1564 raise ValueError(f'Got unsuppoted value for {taiOrUtc=}') 

1565 useUtc = taiOrUtc == 'utc' 

1566 

1567 if printHeader: 

1568 print(f"Details for {event.duration:.2f}s {event.type.name} event dayObs={event.dayObs}" 

1569 f" seqNum={event.seqNum}:") 

1570 print(f"- Event began at: {event.begin.utc.isot if useUtc else event.begin.isot}") 

1571 print(f"- Event ended at: {event.end.utc.isot if useUtc else event.end.isot}") 

1572 

1573 dayObs = event.dayObs 

1574 data = self._data[dayObs] 

1575 startRow = event._startRow 

1576 endRow = event._endRow 

1577 nRowsToApply = endRow - startRow + 1 

1578 print(f"\nTotal number of rows in the merged dataframe: {len(data)}") 

1579 if printHeader: 

1580 print(f"of which rows {startRow} to {endRow} (inclusive) relate to this event.") 

1581 

1582 # reconstruct all the states 

1583 tma = TMAStateMachine(engineeringMode=True) 

1584 _initializeTma(tma) 

1585 

1586 tmaStates = {} 

1587 firstAppliedRow = True # flag to print a header on the first row that's applied 

1588 for rowNum, row in data.iterrows(): # must replay rows right from start to get full correct state 

1589 if rowNum == startRow: 

1590 # we've not yet applied this row, so this is the state just 

1591 # before event 

1592 print(f"\nBefore the event the TMA was in state {tma.state.name}:") 

1593 self.printTmaDetailedState(tma) 

1594 

1595 if rowNum >= startRow and rowNum <= endRow: 

1596 if firstAppliedRow: # only print this intro on the first row we're applying 

1597 print(f"\nThen, applying the {nRowsToApply} rows of data for this event, the state" 

1598 " evolved as follows:\n") 

1599 firstAppliedRow = False 

1600 

1601 # break the row down and print its details 

1602 rowFor = row['rowFor'] 

1603 axis, rowType = getAxisAndType(rowFor) # e.g. elevation, MotionState 

1604 value = tma._getRowPayload(row, rowType, rowFor) 

1605 valueStr = f"{str(value) if isinstance(value, bool) else value.name}" 

1606 rowTime = efdTimestampToAstropy(row['private_efdStamp']) 

1607 print(f"On row {rowNum} the {axis} axis had the {rowType} set to {valueStr} at" 

1608 f" {rowTime.utc.isot if useUtc else rowTime.isot}") 

1609 

1610 # then apply it as usual, printing the state right afterwards 

1611 tma.apply(row) 

1612 tmaStates[rowNum] = tma.state 

1613 self.printTmaDetailedState(tma) 

1614 print() 

1615 

1616 else: 

1617 # if it's not in the range of interest then just apply it 

1618 # silently as usual 

1619 tma.apply(row) 

1620 tmaStates[rowNum] = tma.state 

1621 

1622 def findEvent(self, time): 

1623 """Find the event which contains the specified time. 

1624 

1625 If the specified time lies within an event, that event is returned. If 

1626 it is at the exact start, that is logged, and if that start point is 

1627 shared by the end of the previous event, that is logged too. If the 

1628 event lies between events, the events either side are logged, but 

1629 ``None`` is returned. If the time lies before the first event of the 

1630 day a warning is logged, as for times after the last event of the day. 

1631 

1632 Parameters 

1633 ---------- 

1634 time : `astropy.time.Time` 

1635 The time. 

1636 

1637 Returns 

1638 ------- 

1639 event : `lsst.summit.utils.tmaUtils.TMAEvent` or `None` 

1640 The event which contains the specified time, or ``None`` if the 

1641 time doesn't fall during an event. 

1642 """ 

1643 # there are five possible cases: 

1644 # 1) the time lies before the first event of the day 

1645 # 2) the time lies after the last event of the day 

1646 # 3) the time lies within an event 

1647 # 3a) the time is exactly at the start of an event 

1648 # 3b) if so, time can be shared by the end of the previous event if 

1649 # they are contiguous 

1650 # 4) the time lies between two events 

1651 # 5) the time is exactly at end of the last event of the day. This is 

1652 # an issue because event end times are exclusive, so this time is 

1653 # not technically in that event, it's the moment it closes (and if 

1654 # there *was* an event which followed contiguously, it would be in 

1655 # that event instead, which is what motivates this definition of 

1656 # lies within what event) 

1657 

1658 dayObs = getDayObsForTime(time) 

1659 # we know this is on the right day, and definitely before the specified 

1660 # time, but sanity check this before continuing as this needs to be 

1661 # true for this to give the correct answer 

1662 assert getDayObsStartTime(dayObs) <= time 

1663 assert getDayObsEndTime(dayObs) > time 

1664 

1665 # command start to many log messages so define once here 

1666 logStart = f"Specified time {time.isot} falls on {dayObs=}" 

1667 

1668 events = self.getEvents(dayObs) 

1669 if len(events) == 0: 

1670 self.log.warning(f'There are no events found for {dayObs}') 

1671 return None 

1672 

1673 # check case 1) 

1674 if time < events[0].begin: 

1675 self.log.warning(f'{logStart} and is before the first event of the day') 

1676 return None 

1677 

1678 # check case 2) 

1679 if time > events[-1].end: 

1680 self.log.warning(f'{logStart} and is after the last event of the day') 

1681 return None 

1682 

1683 # check case 5) 

1684 if time == events[-1].end: 

1685 self.log.warning(f'{logStart} and is exactly at the end of the last event of the day' 

1686 f' (seqnum={events[-1].seqNum}). Because event intervals are half-open, this' 

1687 ' time does not technically lie in any event') 

1688 return None 

1689 

1690 # we are now either in an event, or between events. Walk through the 

1691 # events, and if the end of the event is after the specified time, then 

1692 # we're either in it or past it, so check if we're in. 

1693 for eventNum, event in enumerate(events): 

1694 if event.end > time: # case 3) we are now into or past the right event 

1695 # the event end encloses the time, so note the > and not >=, 

1696 # this must be strictly greater, we check the overlap case 

1697 # later 

1698 if time >= event.begin: # we're fully inside the event, so return it. 

1699 # 3a) before returning, check if we're exactly at the start 

1700 # of the event, and if so, log it. Then 3b) also check if 

1701 # we're at the exact end of the previous event, and if so, 

1702 # log that too. 

1703 if time == event.begin: 

1704 self.log.info(f"{logStart} and is exactly at the start of event" 

1705 f" {eventNum}") 

1706 if eventNum == 0: # I think this is actually impossible, but check anyway 

1707 return event # can't check the previous event so return here 

1708 previousEvent = events[eventNum - 1] 

1709 if previousEvent.end == time: 

1710 self.log.info("Previous event is contiguous, so this time is also at the exact" 

1711 f" end of {eventNum - 1}") 

1712 return event 

1713 else: # case 4) 

1714 # the event end is past the time, but it's not inside the 

1715 # event, so we're between events. Log which we're between 

1716 # and return None 

1717 previousEvent = events[eventNum - 1] 

1718 timeAfterPrev = (time - previousEvent.end).to_datetime() 

1719 naturalTimeAfterPrev = humanize.naturaldelta(timeAfterPrev, minimum_unit='MICROSECONDS') 

1720 timeBeforeCurrent = (event.begin - time).to_datetime() 

1721 naturalTimeBeforeCurrent = humanize.naturaldelta(timeBeforeCurrent, 

1722 minimum_unit='MICROSECONDS') 

1723 self.log.info(f"{logStart} and lies" 

1724 f" {naturalTimeAfterPrev} after the end of event {previousEvent.seqNum}" 

1725 f" and {naturalTimeBeforeCurrent} before the start of event {event.seqNum}." 

1726 ) 

1727 return None 

1728 

1729 raise RuntimeError('Event finding logic fundamentally failed, which should never happen - the code' 

1730 ' needs fixing')