Coverage for python/lsst/summit/utils/tmaUtils.py: 21%
594 statements
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-29 10:24 +0000
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-29 10:24 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import re
23import enum
24import itertools
25import logging
26import pandas as pd
27import numpy as np
28import humanize
29from dataclasses import dataclass
30from astropy.time import Time
31from matplotlib.ticker import FuncFormatter
32import matplotlib.dates as mdates
33import matplotlib.pyplot as plt
34from lsst.utils.iteration import ensure_iterable
36from .enums import ScriptState, AxisMotionState, PowerState
37from .utils import getCurrentDayObs_int, dayObsIntToString
38from .efdUtils import (getEfdData,
39 makeEfdClient,
40 efdTimestampToAstropy,
41 COMMAND_ALIASES,
42 getDayObsForTime,
43 getDayObsStartTime,
44 getDayObsEndTime,
45 clipDataToEvent,
46 )
48__all__ = (
49 'TMAStateMachine',
50 'TMAEvent',
51 'TMAEventMaker',
52 'TMAState',
53 'AxisMotionState',
54 'PowerState',
55 'getSlewsFromEventList',
56 'getTracksFromEventList',
57 'getTorqueMaxima',
58)
60# we don't want to use `None` for a no data sentinel because dict.get('key')
61# returns None if the key isn't present, and also we need to mark that the data
62# was queried for and no data was found, whereas the key not being present
63# means that we've not yet looked for the data.
64NO_DATA_SENTINEL = "NODATA"
67def getSlewsFromEventList(events):
68 """Get the slew events from a list of TMAEvents.
70 Parameters
71 ----------
72 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent`
73 The list of events to filter.
75 Returns
76 -------
77 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent`
78 The filtered list of events.
79 """
80 return [e for e in events if e.type == TMAState.SLEWING]
83def getTracksFromEventList(events):
84 """Get the tracking events from a list of TMAEvents.
86 Parameters
87 ----------
88 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent`
89 The list of events to filter.
91 Returns
92 -------
93 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent`
94 The filtered list of events.
95 """
96 return [e for e in events if e.type == TMAState.TRACKING]
99def getTorqueMaxima(table):
100 """Print the maximum positive and negative azimuth and elevation torques.
102 Designed to be used with the table as downloaded from RubinTV.
104 Parameters
105 ----------
106 table : `pd.DataFrame`
107 The table of data to use, as generated by Rapid Analysis.
108 """
109 for axis in ['elevation', 'azimuth']:
110 col = f'Largest {axis} torque'
111 maxPos = np.argmax(table[col])
112 maxVal = table[col].iloc[maxPos]
113 print(f"Max positive {axis:9} torque during seqNum {maxPos:>4}: {maxVal/1000:>7.1f}kNm")
114 minPos = np.argmin(table[col])
115 minVal = table[col].iloc[minPos]
116 print(f"Max negative {axis:9} torque during seqNum {minPos:>4}: {minVal/1000:>7.1f}kNm")
119def getAzimuthElevationDataForEvent(client, event, prePadding=0, postPadding=0):
120 """Get the data for the az/el telemetry topics for a given TMAEvent.
122 Parameters
123 ----------
124 client : `lsst_efd_client.efd_helper.EfdClient`
125 The EFD client to use.
126 event : `lsst.summit.utils.tmaUtils.TMAEvent`
127 The event to get the data for.
128 prePadding : `float`, optional
129 The amount of time to pad the event with before the start time, in
130 seconds.
131 postPadding : `float`, optional
132 The amount of time to pad the event with after the end time, in
133 seconds.
135 Returns
136 -------
137 azimuthData : `pd.DataFrame`
138 The azimuth data for the specified event.
139 elevationData : `pd.DataFrame`
140 The elevation data for the specified event.
141 """
142 azimuthData = getEfdData(client,
143 'lsst.sal.MTMount.azimuth',
144 event=event,
145 prePadding=prePadding,
146 postPadding=postPadding)
147 elevationData = getEfdData(client,
148 'lsst.sal.MTMount.elevation',
149 event=event,
150 prePadding=prePadding,
151 postPadding=postPadding)
153 return azimuthData, elevationData
156def plotEvent(client, event, fig=None, prePadding=0, postPadding=0, commands={},
157 azimuthData=None, elevationData=None):
158 """Plot the TMA axis positions over the course of a given TMAEvent.
160 Plots the axis motion profiles for the given event, with optional padding
161 at the start and end of the event. If the data is provided via the
162 azimuthData and elevationData parameters, it will be used, otherwise it
163 will be queried from the EFD.
165 Optionally plots any commands issued during or around the event, if these
166 are supplied. Commands are supplied as a dictionary of the command topic
167 strings, with values as astro.time.Time objects at which the command was
168 issued.
170 Parameters
171 ----------
172 client : `lsst_efd_client.efd_helper.EfdClient`
173 The EFD client to use.
174 event : `lsst.summit.utils.tmaUtils.TMAEvent`
175 The event to plot.
176 fig : `matplotlib.figure.Figure`, optional
177 The figure to plot on. If not specified, a new figure will be created.
178 prePadding : `float`, optional
179 The amount of time to pad the event with before the start time, in
180 seconds.
181 postPadding : `float`, optional
182 The amount of time to pad the event with after the end time, in
183 seconds.
184 commands : `dict` of `str` : `astropy.time.Time`, optional
185 A dictionary of commands to plot on the figure. The keys are the topic
186 names, and the values are the times at which the commands were sent.
187 azimuthData : `pd.DataFrame`, optional
188 The azimuth data to plot. If not specified, it will be queried from the
189 EFD.
190 elevationData : `pd.DataFrame`, optional
191 The elevation data to plot. If not specified, it will be queried from
192 the EFD.
194 Returns
195 -------
196 fig : `matplotlib.figure.Figure`
197 The figure on which the plot was made.
198 """
199 def tickFormatter(value, tick_number):
200 # Convert the value to a string without subtracting large numbers
201 # tick_number is unused.
202 return f"{value:.2f}"
204 # plot any commands we might have
205 if not isinstance(commands, dict):
206 raise TypeError('commands must be a dict of command names with values as'
207 ' astropy.time.Time values')
209 if fig is None:
210 fig = plt.figure(figsize=(10, 8))
211 log = logging.getLogger(__name__)
212 log.warning("Making new matplotlib figure - if this is in a loop you're going to have a bad time."
213 " Pass in a figure with fig = plt.figure(figsize=(10, 8)) to avoid this warning.")
215 fig.clear()
216 ax1, ax2 = fig.subplots(2,
217 sharex=True,
218 gridspec_kw={'wspace': 0,
219 'hspace': 0,
220 'height_ratios': [2.5, 1]})
222 if azimuthData is None or elevationData is None:
223 azimuthData, elevationData = getAzimuthElevationDataForEvent(client,
224 event,
225 prePadding=prePadding,
226 postPadding=postPadding)
228 # Use the native color cycle for the lines. Because they're on different
229 # axes they don't cycle by themselves
230 lineColors = [p['color'] for p in plt.rcParams['axes.prop_cycle']]
231 colorCounter = 0
233 ax1.plot(azimuthData['actualPosition'], label='Azimuth position', c=lineColors[colorCounter])
234 colorCounter += 1
235 ax1.yaxis.set_major_formatter(FuncFormatter(tickFormatter))
236 ax1.set_ylabel('Azimuth (degrees)')
238 ax1_twin = ax1.twinx()
239 ax1_twin.plot(elevationData['actualPosition'], label='Elevation position', c=lineColors[colorCounter])
240 colorCounter += 1
241 ax1_twin.yaxis.set_major_formatter(FuncFormatter(tickFormatter))
242 ax1_twin.set_ylabel('Elevation (degrees)')
243 ax1.set_xticks([]) # remove x tick labels on the hidden upper x-axis
245 ax2_twin = ax2.twinx()
246 ax2.plot(azimuthData['actualTorque'], label='Azimuth torque', c=lineColors[colorCounter])
247 colorCounter += 1
248 ax2_twin.plot(elevationData['actualTorque'], label='Elevation torque', c=lineColors[colorCounter])
249 colorCounter += 1
250 ax2.set_ylabel('Azimuth torque (Nm)')
251 ax2_twin.set_ylabel('Elevation torque (Nm)')
252 ax2.set_xlabel('Time (UTC)') # yes, it really is UTC, matplotlib converts this automatically!
254 # put the ticks at an angle, and right align with the tick marks
255 ax2.set_xticks(ax2.get_xticks()) # needed to supress a user warning
256 xlabels = ax2.get_xticks()
257 ax2.set_xticklabels(xlabels, rotation=40, ha='right')
258 ax2.xaxis.set_major_locator(mdates.AutoDateLocator())
259 ax2.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M:%S'))
261 if prePadding or postPadding:
262 # note the conversion to utc because the x-axis from the dataframe
263 # already got automagically converted when plotting before, so this is
264 # necessary for things to line up
265 ax1_twin.axvline(event.begin.utc.datetime, c='k', ls='--', alpha=0.5, label='Event begin/end')
266 ax1_twin.axvline(event.end.utc.datetime, c='k', ls='--', alpha=0.5)
267 # extend lines down across lower plot, but do not re-add label
268 ax2_twin.axvline(event.begin.utc.datetime, c='k', ls='--', alpha=0.5)
269 ax2_twin.axvline(event.end.utc.datetime, c='k', ls='--', alpha=0.5)
271 for command, commandTime in commands.items():
272 # if commands weren't found, the item is set to None. This is common
273 # for events so handle it gracefully and silently. The command finding
274 # code logs about lack of commands found so no need to mention here.
275 if commandTime is None:
276 continue
277 ax1_twin.axvline(commandTime.utc.datetime, c=lineColors[colorCounter],
278 ls='--', alpha=0.75, label=f'{command}')
279 # extend lines down across lower plot, but do not re-add label
280 ax2_twin.axvline(commandTime.utc.datetime, c=lineColors[colorCounter],
281 ls='--', alpha=0.75)
282 colorCounter += 1
284 # combine the legends and put inside the plot
285 handles1a, labels1a = ax1.get_legend_handles_labels()
286 handles1b, labels1b = ax1_twin.get_legend_handles_labels()
287 handles2a, labels2a = ax2.get_legend_handles_labels()
288 handles2b, labels2b = ax2_twin.get_legend_handles_labels()
290 handles = handles1a + handles1b + handles2a + handles2b
291 labels = labels1a + labels1b + labels2a + labels2b
292 # ax2 is "in front" of ax1 because it has the vlines plotted on it, and
293 # vlines are on ax2 so that they appear at the bottom of the legend, so
294 # make sure to plot the legend on ax2, otherwise the vlines will go on top
295 # of the otherwise-opaque legend.
296 ax1_twin.legend(handles, labels, facecolor='white', framealpha=1)
298 # Add title with the event name, type etc
299 dayObsStr = dayObsIntToString(event.dayObs)
300 title = (f"{dayObsStr} - seqNum {event.seqNum} (version {event.version})" # top line, rest below
301 f"\nDuration = {event.duration:.2f}s"
302 f" Event type: {event.type.name}"
303 f" End reason: {event.endReason.name}"
304 )
305 ax1_twin.set_title(title)
306 return fig
309def getCommandsDuringEvent(client, event, commands=('raDecTarget'), log=None, doLog=True):
310 """Get the commands issued during an event.
312 Get the times at which the specified commands were issued during the event.
314 Parameters
315 ----------
316 client : `lsst_efd_client.efd_helper.EfdClient`
317 The EFD client to use.
318 event : `lsst.summit.utils.tmaUtils.TMAEvent`
319 The event to plot.
320 commands : `list` of `str`, optional
321 The commands or command aliases to look for. Defaults to
322 ['raDecTarget'].
323 log : `logging.Logger`, optional
324 The logger to use. If not specified, a new logger will be created if
325 needed.
326 doLog : `bool`, optional
327 Whether to log messages. Defaults to True.
329 Returns
330 -------
331 commands : `dict` of `str` : `astropy.time.Time`
332 A dictionary of the commands and the times at which they were issued.
333 """
334 # TODO: DM-40100 Add support for padding the event here to allow looking
335 # for triggering commands before the event
337 # TODO: DM-40100 Change this to always return a list of times, and remove
338 # warning about finding multiple commands. Remember to update docs and
339 # plotting code.
340 if log is None and doLog:
341 log = logging.getLogger(__name__)
343 commands = ensure_iterable(commands)
344 fullCommands = [c if c not in COMMAND_ALIASES else COMMAND_ALIASES[c] for c in commands]
345 del commands # make sure we always use their full names
347 ret = {}
348 for command in fullCommands:
349 data = getEfdData(client, command, event=event, warn=False)
350 if data.empty:
351 if doLog:
352 log.info(f'Found no command issued for {command} during event')
353 ret[command] = None
354 elif len(data) > 1:
355 if doLog:
356 log.warning(f'Found multiple commands issued for {command} during event, returning None')
357 ret[command] = None
358 else:
359 assert len(data) == 1 # this must be true now
360 commandTime = data.private_efdStamp
361 ret[command] = Time(commandTime, format='unix')
363 return ret
366def _initializeTma(tma):
367 """Helper function to turn a TMA into a valid state for testing.
369 Do not call directly in normal usage or code, as this just arbitrarily
370 sets values to make the TMA valid.
372 Parameters
373 ----------
374 tma : `lsst.summit.utils.tmaUtils.TMAStateMachine`
375 The TMA state machine model to initialize.
376 """
377 tma._parts['azimuthInPosition'] = False
378 tma._parts['azimuthMotionState'] = AxisMotionState.STOPPED
379 tma._parts['azimuthSystemState'] = PowerState.ON
380 tma._parts['elevationInPosition'] = False
381 tma._parts['elevationMotionState'] = AxisMotionState.STOPPED
382 tma._parts['elevationSystemState'] = PowerState.ON
385@dataclass(slots=True, kw_only=True, frozen=True)
386class BlockInfo:
387 """The block info relating to a TMAEvent.
389 Parameters
390 ----------
391 blockNumber : `int`
392 The block number, as an integer.
393 blockId : `str`
394 The block ID, as a string.
395 salIndices : `list` of `int`
396 One or more SAL indices, relating to the block.
397 tickets : `list` of `str`
398 One or more SITCOM tickets, relating to the block.
399 states : `list` of `ScriptStatePoint`
400 The states of the script during the block. Each element is a
401 ``ScriptStatePoint`` which contains:
402 - the time, as an astropy.time.Time
403 - the state, as a ``ScriptState`` enum
404 - the reason for state change, as a string
405 """
406 blockNumber: int
407 blockId: str
408 salIndices: int
409 tickets: list
410 states: list
412 def __repr__(self):
413 return (
414 f"BlockInfo(blockNumber={self.blockNumber}, blockId={self.blockId}, salIndices={self.salIndices},"
415 f" tickets={self.tickets}, states={self.states!r}"
416 )
418 def _ipython_display_(self):
419 print(self.__str__())
421 def __str__(self):
422 # You can't put the characters '\n' directly into the evaluated part of
423 # an f-string i.e. inside the {} part, until py 3.12, so this must go
424 # in via a variable until then.
425 newline = ' \n'
427 return (
428 f"blockNumber: {self.blockNumber}\n"
429 f"blockId: {self.blockId}\n"
430 f"salIndices: {self.salIndices}\n"
431 f"tickets: {self.tickets}\n"
432 f"states: \n{newline.join([str(state) for state in self.states])}"
433 )
436@dataclass(slots=True, kw_only=True, frozen=True)
437class ScriptStatePoint:
438 time: Time
439 state: ScriptState
440 reason: str
442 def __repr__(self):
443 return (
444 f"ScriptStatePoint(time={self.time!r}, state={self.state!r}, reason={self.reason!r})"
445 )
447 def _ipython_display_(self):
448 print(self.__str__())
450 def __str__(self):
451 reasonStr = f" - {self.reason}" if self.reason else ""
452 return (f"{self.state.name:>10} @ {self.time.isot}{reasonStr}")
455@dataclass(slots=True, kw_only=True, frozen=True)
456class TMAEvent:
457 """A movement event for the TMA.
459 Contains the dayObs on which the event occured, using the standard
460 observatory definition of the dayObs, and the sequence number of the event,
461 which is unique for each event on a given dayObs.
463 The event type can be either 'SLEWING' or 'TRACKING', defined as:
464 - SLEWING: some part of the TMA is in motion
465 - TRACKING: both axes are in position and tracking the sky
467 The end reason can be 'STOPPED', 'TRACKING', 'FAULT', 'SLEWING', or 'OFF'.
468 - SLEWING: The previous event was a TRACKING event, and one or more of
469 the TMA components either stopped being in position, or stopped
470 moving, or went into fault, or was turned off, and hence we are now
471 only slewing and no longer tracking the sky.
472 - TRACKING: the TMA started tracking the sky when it wasn't previously.
473 Usualy this would always be preceded by directly by a SLEWING
474 event, but this is not strictly true, as the EUI seems to be able
475 to make the TMA start tracking the sky without slewing first.
476 - STOPPED: the components of the TMA transitioned to the STOPPED state.
477 - FAULT: the TMA went into fault.
478 - OFF: the TMA components were turned off.
480 Note that this class is not intended to be instantiated directly, but
481 rather to be returned by the ``TMAEventMaker.getEvents()`` function.
483 Parameters
484 ----------
485 dayObs : `int`
486 The dayObs on which the event occured.
487 seqNum : `int`
488 The sequence number of the event,
489 type : `lsst.summit.utils.tmaUtils.TMAState`
490 The type of the event, either 'SLEWING' or 'TRACKING'.
491 endReason : `lsst.summit.utils.tmaUtils.TMAState`
492 The reason the event ended, either 'STOPPED', 'TRACKING', 'FAULT',
493 'SLEWING', or 'OFF'.
494 duration : `float`
495 The duration of the event, in seconds.
496 begin : `astropy.time.Time`
497 The time the event began.
498 end : `astropy.time.Time`
499 The time the event ended.
500 blockInfo : `lsst.summit.utils.tmaUtils.BlockInfo`
501 The block info relating to the event.
502 version : `int`
503 The version of the TMAEvent class. Equality between events is only
504 valid for a given version of the class. If the class definition
505 changes, the time ranges can change, and hence the equality between
506 events is ``False``.
507 _startRow : `int`
508 The first row in the merged EFD data which is part of the event.
509 _endRow : `int`
510 The last row in the merged EFD data which is part of the event.
511 """
512 dayObs: int
513 seqNum: int
514 type: str # can be 'SLEWING', 'TRACKING'
515 endReason: str # can be 'STOPPED', 'TRACKING', 'FAULT', 'SLEWING', 'OFF'
516 duration: float # seconds
517 begin: Time
518 end: Time
519 blockInfo: BlockInfo = None
520 version: int = 0 # update this number any time a code change which could change event definitions is made
521 _startRow: int
522 _endRow: int
524 def __lt__(self, other):
525 if self.version != other.version:
526 raise ValueError(
527 f"Cannot compare TMAEvents with different versions: {self.version} != {other.version}"
528 )
529 if self.dayObs < other.dayObs:
530 return True
531 elif self.dayObs == other.dayObs:
532 return self.seqNum < other.seqNum
533 return False
535 def __repr__(self):
536 return (
537 f"TMAEvent(dayObs={self.dayObs}, seqNum={self.seqNum}, type={self.type!r},"
538 f" endReason={self.endReason!r}, duration={self.duration}, begin={self.begin!r},"
539 f" end={self.end!r}"
540 )
542 def _ipython_display_(self):
543 print(self.__str__())
545 def __str__(self):
546 return (
547 f"dayObs: {self.dayObs}\nseqNum: {self.seqNum}\ntype: {self.type.name}"
548 f"\nendReason: {self.endReason.name}\nduration: {self.duration}\nbegin: {self.begin!r},"
549 f"\nend: {self.end!r}"
550 )
553class TMAState(enum.IntEnum):
554 """Overall state of the TMA.
556 States are defined as follows:
558 UNINITIALIZED
559 We have not yet got data for all relevant components, so the overall
560 state is undefined.
561 STOPPED
562 All components are on, and none are moving.
563 TRACKING
564 We are tracking the sky.
565 SLEWING
566 One or more components are moving, and one or more are not tracking the
567 sky. This should probably be called MOVING, as it includes: slewing,
568 MOVING_POINT_TO_POINT, and JOGGING.
569 FAULT
570 All (if engineeringMode) or any (if not engineeringMode) components are
571 in fault.
572 OFF
573 All components are off.
574 """
575 UNINITIALIZED = -1
576 STOPPED = 0
577 TRACKING = 1
578 SLEWING = 2
579 FAULT = 3
580 OFF = 4
582 def __repr__(self):
583 return f"TMAState.{self.name}"
586def getAxisAndType(rowFor):
587 """Get the axis the data relates to, and the type of data it contains.
589 Parameters
590 ----------
591 rowFor : `str`
592 The column in the dataframe denoting what this row is for, e.g.
593 "elevationMotionState" or "azimuthInPosition", etc.
595 Returns
596 -------
597 axis : `str`
598 The axis the row is for, e.g. "azimuth", "elevation".
599 rowType : `str`
600 The type of the row, e.g. "MotionState", "SystemState", "InPosition".
601 """
602 regex = r'(azimuth|elevation)(InPosition|MotionState|SystemState)$' # matches the end of the line
603 matches = re.search(regex, rowFor)
604 if matches is None:
605 raise ValueError(f"Could not parse axis and rowType from {rowFor=}")
606 axis = matches.group(1)
607 rowType = matches.group(2)
609 assert rowFor.endswith(f"{axis}{rowType}")
610 return axis, rowType
613class ListViewOfDict:
614 """A class to allow making lists which contain references to an underlying
615 dictionary.
617 Normally, making a list of items from a dictionary would make a copy of the
618 items, but this class allows making a list which contains references to the
619 underlying dictionary items themselves. This is useful for making a list of
620 components, such that they can be manipulated in their logical sets.
621 """
622 def __init__(self, underlyingDictionary, keysToLink):
623 self.dictionary = underlyingDictionary
624 self.keys = keysToLink
626 def __getitem__(self, index):
627 return self.dictionary[self.keys[index]]
629 def __setitem__(self, index, value):
630 self.dictionary[self.keys[index]] = value
632 def __len__(self):
633 return len(self.keys)
636class TMAStateMachine:
637 """A state machine model of the TMA.
639 Note that this is currently only implemented for the azimuth and elevation
640 axes, but will be extended to include the rotator in the future.
642 Note that when used for event generation, changing ``engineeringMode`` to
643 False might change the resulting list of events, and that if the TMA moves
644 with some axis in fault, then these events will be missed. It is therefore
645 thought that ``engineeringMode=True`` should always be used when generating
646 events. The option, however, is there for completeness, as this will be
647 useful for knowing is the CSC would consider the TMA to be in fault in the
648 general case.
650 Parameters
651 ----------
652 engineeringMode : `bool`, optional
653 Whether the TMA is in engineering mode. Defaults to True. If False,
654 then the TMA will be in fault if any component is in fault. If True,
655 then the TMA will be in fault only if all components are in fault.
656 debug : `bool`, optional
657 Whether to log debug messages. Defaults to False.
658 """
659 _UNINITIALIZED_VALUE: int = -999
661 def __init__(self, engineeringMode=True, debug=False):
662 self.engineeringMode = engineeringMode
663 self.log = logging.getLogger('lsst.summit.utils.tmaUtils.TMA')
664 if debug:
665 self.log.level = logging.DEBUG
666 self._mostRecentRowTime = -1
668 # the actual components of the TMA
669 self._parts = {'azimuthInPosition': self._UNINITIALIZED_VALUE,
670 'azimuthMotionState': self._UNINITIALIZED_VALUE,
671 'azimuthSystemState': self._UNINITIALIZED_VALUE,
672 'elevationInPosition': self._UNINITIALIZED_VALUE,
673 'elevationMotionState': self._UNINITIALIZED_VALUE,
674 'elevationSystemState': self._UNINITIALIZED_VALUE,
675 }
676 systemKeys = ['azimuthSystemState', 'elevationSystemState']
677 positionKeys = ['azimuthInPosition', 'elevationInPosition']
678 motionKeys = ['azimuthMotionState', 'elevationMotionState']
680 # references to the _parts as conceptual groupings
681 self.system = ListViewOfDict(self._parts, systemKeys)
682 self.motion = ListViewOfDict(self._parts, motionKeys)
683 self.inPosition = ListViewOfDict(self._parts, positionKeys)
685 # tuples of states for state collapsing. Note that STOP_LIKE +
686 # MOVING_LIKE must cover the full set of AxisMotionState enums
687 self.STOP_LIKE = (AxisMotionState.STOPPING,
688 AxisMotionState.STOPPED,
689 AxisMotionState.TRACKING_PAUSED)
690 self.MOVING_LIKE = (AxisMotionState.MOVING_POINT_TO_POINT,
691 AxisMotionState.JOGGING,
692 AxisMotionState.TRACKING)
693 # Likewise, ON_LIKE + OFF_LIKE must cover the full set of PowerState
694 # enums
695 self.OFF_LIKE = (PowerState.OFF, PowerState.TURNING_OFF)
696 self.ON_LIKE = (PowerState.ON, PowerState.TURNING_ON)
697 self.FAULT_LIKE = (PowerState.FAULT,) # note the trailing comma - this must be an iterable
699 def apply(self, row):
700 """Apply a row of data to the TMA state.
702 Checks that the row contains data for a later time than any data
703 previously applied, and applies the relevant column entry to the
704 relevant component.
706 Parameters
707 ----------
708 row : `pd.Series`
709 The row of data to apply to the state machine.
710 """
711 timestamp = row['private_efdStamp']
712 if timestamp < self._mostRecentRowTime: # NB equals is OK, technically, though it never happens
713 raise ValueError('TMA evolution must be monotonic increasing in time, tried to apply a row which'
714 ' predates the most previous one')
715 self._mostRecentRowTime = timestamp
717 rowFor = row['rowFor'] # e.g. elevationMotionState
718 axis, rowType = getAxisAndType(rowFor) # e.g. elevation, MotionState
719 value = self._getRowPayload(row, rowType, rowFor)
720 self.log.debug(f"Setting {rowFor} to {repr(value)}")
721 self._parts[rowFor] = value
722 try:
723 # touch the state property as this executes the sieving, to make
724 # sure we don't fall through the sieve at any point in time
725 _ = self.state
726 except RuntimeError as e:
727 # improve error reporting, but always reraise this, as this is a
728 # full-blown failure
729 raise RuntimeError(f'Failed to apply {value} to {axis}{rowType} with state {self._parts}') from e
731 def _getRowPayload(self, row, rowType, rowFor):
732 """Get the relevant value from the row.
734 Given the row, and which component it relates to, get the relevant
735 value, as a bool or cast to the appropriate enum class.
737 Parameters
738 ----------
739 row : `pd.Series`
740 The row of data from the dataframe.
741 rowType : `str`
742 The type of the row, e.g. "MotionState", "SystemState",
743 "InPosition".
744 rowFor : `str`
745 The component the row is for, e.g. "azimuth", "elevation".
747 Returns
748 -------
749 value : `bool` or `enum`
750 The value of the row, as a bool or enum, depending on the
751 component, cast to the appropriate enum class or bool.
752 """
753 match rowType:
754 case 'MotionState':
755 value = row[f'state_{rowFor}']
756 return AxisMotionState(value)
757 case 'SystemState':
758 value = row[f'powerState_{rowFor}']
759 return PowerState(value)
760 case 'InPosition':
761 value = row[f'inPosition_{rowFor}']
762 return bool(value)
763 case _:
764 raise ValueError(f'Failed to get row payload with {rowType=} and {row=}')
766 @property
767 def _isValid(self):
768 """Has the TMA had a value applied to all its components?
770 If any component has not yet had a value applied, the TMA is not valid,
771 as those components will be in an unknown state.
773 Returns
774 -------
775 isValid : `bool`
776 Whether the TMA is fully initialized.
777 """
778 return not any([v == self._UNINITIALIZED_VALUE for v in self._parts.values()])
780 # state inspection properties - a high level way of inspecting the state as
781 # an API
782 @property
783 def isMoving(self):
784 return self.state in [TMAState.TRACKING, TMAState.SLEWING]
786 @property
787 def isNotMoving(self):
788 return not self.isMoving
790 @property
791 def isTracking(self):
792 return self.state == TMAState.TRACKING
794 @property
795 def isSlewing(self):
796 return self.state == TMAState.SLEWING
798 @property
799 def canMove(self):
800 badStates = [PowerState.OFF, PowerState.TURNING_OFF, PowerState.FAULT, PowerState.UNKNOWN]
801 return bool(
802 self._isValid and
803 self._parts['azimuthSystemState'] not in badStates and
804 self._parts['elevationSystemState'] not in badStates
805 )
807 # Axis inspection properties, designed for internal use. These return
808 # iterables so that they can be used in any() and all() calls, which make
809 # the logic much easier to read, e.g. to see if anything is moving, we can
810 # write `if not any(_axisInMotion):`
811 @property
812 def _axesInFault(self):
813 return [x in self.FAULT_LIKE for x in self.system]
815 @property
816 def _axesOff(self):
817 return [x in self.OFF_LIKE for x in self.system]
819 @property
820 def _axesOn(self):
821 return [not x for x in self._axesOn]
823 @property
824 def _axesInMotion(self):
825 return [x in self.MOVING_LIKE for x in self.motion]
827 @property
828 def _axesTRACKING(self):
829 """Note this is deliberately named _axesTRACKING and not _axesTracking
830 to make it clear that this is the AxisMotionState type of TRACKING and
831 not the normal conceptual notion of tracking (the sky, i.e. as opposed
832 to slewing).
833 """
834 return [x == AxisMotionState.TRACKING for x in self.motion]
836 @property
837 def _axesInPosition(self):
838 return [x is True for x in self.inPosition]
840 @property
841 def state(self):
842 """The overall state of the TMA.
844 Note that this is both a property, and also the method which applies
845 the logic sieve to determine the state at a given point in time.
847 Returns
848 -------
849 state : `lsst.summit.utils.tmaUtils.TMAState`
850 The overall state of the TMA.
851 """
852 # first, check we're valid, and if not, return UNINITIALIZED state, as
853 # things are unknown
854 if not self._isValid:
855 return TMAState.UNINITIALIZED
857 # if we're not in engineering mode, i.e. we're under normal CSC
858 # control, then if anything is in fault, we're in fault. If we're
859 # engineering then some axes will move when others are in fault
860 if not self.engineeringMode:
861 if any(self._axesInFault):
862 return TMAState.FAULT
863 else:
864 # we're in engineering mode, so return fault state if ALL are in
865 # fault
866 if all(self._axesInFault):
867 return TMAState.FAULT
869 # if all axes are off, the TMA is OFF
870 if all(self._axesOff):
871 return TMAState.OFF
873 # we know we're valid and at least some axes are not off, so see if
874 # we're in motion if no axes are moving, we're stopped
875 if not any(self._axesInMotion):
876 return TMAState.STOPPED
878 # now we know we're initialized, and that at least one axis is moving
879 # so check axes for motion and in position. If all axes are tracking
880 # and all are in position, we're tracking the sky
881 if (all(self._axesTRACKING) and all(self._axesInPosition)):
882 return TMAState.TRACKING
884 # we now know explicitly that not everything is in position, so we no
885 # longer need to check that. We do actually know that something is in
886 # motion, but confirm that's the case and return SLEWING
887 if (any(self._axesInMotion)):
888 return TMAState.SLEWING
890 # if we want to differentiate between MOVING_POINT_TO_POINT moves,
891 # JOGGING moves and regular slews, the logic in the step above needs to
892 # be changed and the new steps added here.
894 raise RuntimeError('State error: fell through the state sieve - rewrite your logic!')
897class TMAEventMaker:
898 """A class to create per-dayObs TMAEvents for the TMA's movements.
900 Example usage:
901 >>> dayObs = 20230630
902 >>> eventMaker = TMAEventMaker()
903 >>> events = eventMaker.getEvents(dayObs)
904 >>> print(f'Found {len(events)} for {dayObs=}')
906 Parameters
907 ----------
908 client : `lsst_efd_client.efd_helper.EfdClient`, optional
909 The EFD client to use, created if not provided.
910 """
911 # the topics which need logical combination to determine the overall mount
912 # state. Will need updating as new components are added to the system.
914 # relevant column: 'state'
915 _movingComponents = [
916 'lsst.sal.MTMount.logevent_azimuthMotionState',
917 'lsst.sal.MTMount.logevent_elevationMotionState',
918 ]
920 # relevant column: 'inPosition'
921 _inPositionComponents = [
922 'lsst.sal.MTMount.logevent_azimuthInPosition',
923 'lsst.sal.MTMount.logevent_elevationInPosition',
924 ]
926 # the components which, if in fault, put the TMA into fault
927 # relevant column: 'powerState'
928 _stateComponents = [
929 'lsst.sal.MTMount.logevent_azimuthSystemState',
930 'lsst.sal.MTMount.logevent_elevationSystemState',
931 ]
933 def __init__(self, client=None):
934 if client is not None:
935 self.client = client
936 else:
937 self.client = makeEfdClient()
938 self.log = logging.getLogger(__name__)
939 self._data = {}
941 @dataclass(frozen=True)
942 class ParsedState:
943 eventStart: Time
944 eventEnd: int
945 previousState: TMAState
946 state: TMAState
948 @staticmethod
949 def isToday(dayObs):
950 """Find out if the specified dayObs is today, or in the past.
952 If the day is today, the function returns ``True``, if it is in the
953 past it returns ``False``. If the day is in the future, a
954 ``ValueError`` is raised, as this indicates there is likely an
955 off-by-one type error somewhere in the logic.
957 Parameters
958 ----------
959 dayObs : `int`
960 The dayObs to check, in the format YYYYMMDD.
962 Returns
963 -------
964 isToday : `bool`
965 ``True`` if the dayObs is today, ``False`` if it is in the past.
967 Raises
968 ValueError: if the dayObs is in the future.
969 """
970 todayDayObs = getCurrentDayObs_int()
971 if dayObs == todayDayObs:
972 return True
973 if dayObs > todayDayObs:
974 raise ValueError("dayObs is in the future")
975 return False
977 @staticmethod
978 def _shortName(topic):
979 """Get the short name of a topic.
981 Parameters
982 ----------
983 topic : `str`
984 The topic to get the short name of.
986 Returns
987 -------
988 shortName : `str`
989 The short name of the topic, e.g. 'azimuthInPosition'
990 """
991 # get, for example 'azimuthInPosition' from
992 # lsst.sal.MTMount.logevent_azimuthInPosition
993 return topic.split('_')[-1]
995 def _mergeData(self, data):
996 """Merge a dict of dataframes based on private_efdStamp, recording
997 where each row came from.
999 Given a dict or dataframes, keyed by topic, merge them into a single
1000 dataframe, adding a column to record which topic each row came from.
1002 Parameters
1003 ----------
1004 data : `dict` of `str` : `pd.DataFrame`
1005 The dataframes to merge.
1007 Returns
1008 -------
1009 merged : `pd.DataFrame`
1010 The merged dataframe.
1011 """
1012 excludeColumns = ['private_efdStamp', 'rowFor']
1014 mergeArgs = {
1015 'how': 'outer',
1016 'sort': True,
1017 }
1019 merged = None
1020 originalRowCounter = 0
1022 # Iterate over the keys and merge the corresponding DataFrames
1023 for key, df in data.items():
1024 if df.empty:
1025 # Must skip the df if it's empty, otherwise the merge will fail
1026 # due to lack of private_efdStamp. Because other axes might
1027 # still be in motion, so we still want to merge what we have
1028 continue
1030 originalRowCounter += len(df)
1031 component = self._shortName(key) # Add suffix to column names to identify the source
1032 suffix = '_' + component
1034 df['rowFor'] = component
1036 columnsToSuffix = [col for col in df.columns if col not in excludeColumns]
1037 df_to_suffix = df[columnsToSuffix].add_suffix(suffix)
1038 df = pd.concat([df[excludeColumns], df_to_suffix], axis=1)
1040 if merged is None:
1041 merged = df.copy()
1042 else:
1043 merged = pd.merge(merged, df, **mergeArgs)
1045 merged = merged.loc[:, ~merged.columns.duplicated()] # Remove duplicate columns after merge
1047 if len(merged) != originalRowCounter:
1048 self.log.warning("Merged data has a different number of rows to the original data, some"
1049 " timestamps (rows) will contain more than one piece of actual information.")
1050 return merged
1052 def getEvents(self, dayObs):
1053 """Get the TMA events for the specified dayObs.
1055 Gets the required mount data from the cache or the EFD as required,
1056 handling whether we're working with live vs historical data. The
1057 dataframes from the EFD is merged and applied to the TMAStateMachine,
1058 and that series of state changes is used to generate a list of
1059 TmaEvents for the day's data.
1061 If the data is for the current day, i.e. if new events can potentially
1062 land, then if the last event is "open" (meaning that the TMA appears to
1063 be in motion and thus the event is growing with time), then that event
1064 is excluded from the event list as it is expected to be changing with
1065 time, and will likely close eventually. However, if that situation
1066 occurs on a day in the past, then that event can never close, and the
1067 event is therefore included, but a warning about the open event is
1068 logged.
1070 Parameters
1071 ----------
1072 dayObs : `int`
1073 The dayObs for which to get the events.
1075 Returns
1076 -------
1077 events : `list` of `lsst.summit.utils.tmaUtils.TMAState`
1078 The events for the specified dayObs.
1079 """
1080 workingLive = self.isToday(dayObs)
1081 data = None
1083 if workingLive:
1084 # it's potentially updating data, so we must update the date
1085 # regarless of whether we have it already or not
1086 self.log.info(f'Updating mount data for {dayObs} from the EFD')
1087 self._getEfdDataForDayObs(dayObs)
1088 data = self._data[dayObs]
1089 elif dayObs in self._data:
1090 # data is in the cache and it's not being updated, so use it
1091 data = self._data[dayObs]
1092 elif dayObs not in self._data:
1093 # we don't have the data yet, but it's not growing, so put it in
1094 # the cache and use it from there
1095 self.log.info(f'Retrieving mount data for {dayObs} from the EFD')
1096 self._getEfdDataForDayObs(dayObs)
1097 data = self._data[dayObs]
1098 else:
1099 raise RuntimeError("This should never happen")
1101 # if we don't have something to work with, log a warning and return
1102 if not self.dataFound(data):
1103 self.log.warning(f"No EFD data found for {dayObs=}")
1104 return []
1106 # applies the data to the state machine, and generates events from the
1107 # series of states which results
1108 events = self._calculateEventsFromMergedData(data, dayObs, dataIsForCurrentDay=workingLive)
1109 if not events:
1110 self.log.warning(f"Failed to calculate any events for {dayObs=} despite EFD data existing!")
1111 return events
1113 @staticmethod
1114 def dataFound(data):
1115 """Check if any data was found.
1117 Parameters
1118 ----------
1119 data : `pd.DataFrame`
1120 The merged dataframe to check.
1122 Returns
1123 -------
1124 dataFound : `bool`
1125 Whether data was found.
1126 """
1127 # You can't just compare to with data == NO_DATA_SENTINEL because
1128 # `data` is usually a dataframe, and you can't compare a dataframe to a
1129 # string directly.
1130 return not (isinstance(data, str) and data == NO_DATA_SENTINEL)
1132 def _getEfdDataForDayObs(self, dayObs):
1133 """Get the EFD data for the specified dayObs and store it in the cache.
1135 Gets the EFD data for all components, as a dict of dataframes keyed by
1136 component name. These are then merged into a single dataframe in time
1137 order, based on each row's `private_efdStamp`. This is then stored in
1138 self._data[dayObs].
1140 If no data is found, the value is set to ``NO_DATA_SENTINEL`` to
1141 differentiate this from ``None``, as this is what you'd get if you
1142 queried the cache with `self._data.get(dayObs)`. It also marks that we
1143 have already queried this day.
1145 Parameters
1146 ----------
1147 dayObs : `int`
1148 The dayObs to query.
1149 """
1150 data = {}
1151 for component in itertools.chain(
1152 self._movingComponents,
1153 self._inPositionComponents,
1154 self._stateComponents
1155 ):
1156 data[component] = getEfdData(self.client, component, dayObs=dayObs, warn=False)
1157 self.log.debug(f"Found {len(data[component])} for {component}")
1159 if all(dataframe.empty for dataframe in data.values()):
1160 # if every single dataframe is empty, set the sentinel and don't
1161 # try to merge anything, otherwise merge all the data we found
1162 self.log.debug(f"No data found for {dayObs=}")
1163 # a sentinel value that's not None
1164 self._data[dayObs] = NO_DATA_SENTINEL
1165 else:
1166 merged = self._mergeData(data)
1167 self._data[dayObs] = merged
1169 def _calculateEventsFromMergedData(self, data, dayObs, dataIsForCurrentDay):
1170 """Calculate the list of events from the merged data.
1172 Runs the merged data, row by row, through the TMA state machine (with
1173 ``tma.apply``) to get the overall TMA state at each row, building a
1174 dict of these states, keyed by row number.
1176 This time-series of TMA states are then looped over (in
1177 `_statesToEventTuples`), building a list of tuples representing the
1178 start and end of each event, the type of the event, and the reason for
1179 the event ending.
1181 This list of tuples is then passed to ``_makeEventsFromStateTuples``,
1182 which actually creates the ``TMAEvent`` objects.
1184 Parameters
1185 ----------
1186 data : `pd.DataFrame`
1187 The merged dataframe to use.
1188 dayObs : `int`
1189 The dayObs for the data.
1190 dataIsForCurrentDay : `bool`
1191 Whether the data is for the current day. Determines whether to
1192 allow an open last event or not.
1194 Returns
1195 -------
1196 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent`
1197 The events for the specified dayObs.
1198 """
1199 engineeringMode = True
1200 tma = TMAStateMachine(engineeringMode=engineeringMode)
1202 # For now, we assume that the TMA starts each day able to move, but
1203 # stationary. If this turns out to cause problems, we will need to
1204 # change to loading data from the previous day(s), and looking back
1205 # through it in time until a state change has been found for every
1206 # axis. For now though, Bruno et. al think this is acceptable and
1207 # preferable.
1208 _initializeTma(tma)
1210 tmaStates = {}
1211 for rowNum, row in data.iterrows():
1212 tma.apply(row)
1213 tmaStates[rowNum] = tma.state
1215 stateTuples = self._statesToEventTuples(tmaStates, dataIsForCurrentDay)
1216 events = self._makeEventsFromStateTuples(stateTuples, dayObs, data)
1217 self.addBlockDataToEvents(events)
1218 return events
1220 def _statesToEventTuples(self, states, dataIsForCurrentDay):
1221 """Get the event-tuples from the dictionary of TMAStates.
1223 Chunks the states into blocks of the same state, so that we can create
1224 an event for each block in `_makeEventsFromStateTuples`. Off-type
1225 states are skipped over, with each event starting when the telescope
1226 next resumes motion or changes to a different type of motion state,
1227 i.e. from non-tracking type movement (MOVE_POINT_TO_POINT, JOGGING,
1228 TRACKING-but-not-in-position, i.e. slewing) to a tracking type
1229 movement, or vice versa.
1231 Parameters
1232 ----------
1233 states : `dict` of `int` : `lsst.summit.utils.tmaUtils.TMAState`
1234 The states of the TMA, keyed by row number.
1235 dataIsForCurrentDay : `bool`
1236 Whether the data is for the current day. Determines whether to
1237 allow and open last event or not.
1239 Returns
1240 -------
1241 parsedStates : `list` of `tuple`
1242 The parsed states, as a list of tuples of the form:
1243 ``(eventStart, eventEnd, eventType, endReason)``
1244 """
1245 # Consider rewriting this with states as a list and using pop(0)?
1246 skipStates = (TMAState.STOPPED, TMAState.OFF, TMAState.FAULT)
1248 parsedStates = []
1249 eventStart = None
1250 rowNum = 0
1251 nRows = len(states)
1252 while rowNum < nRows:
1253 previousState = None
1254 state = states[rowNum]
1255 # if we're not in an event, fast forward through off-like rows
1256 # until a new event starts
1257 if eventStart is None and state in skipStates:
1258 rowNum += 1
1259 continue
1261 # we've started a new event, so walk through it and find the end
1262 eventStart = rowNum
1263 previousState = state
1264 rowNum += 1 # move to the next row before starting the while loop
1265 if rowNum == nRows:
1266 # we've reached the end of the data, and we're still in an
1267 # event, so don't return this presumably in-progress event
1268 self.log.warning('Reached the end of the data while starting a new event')
1269 break
1270 state = states[rowNum]
1271 while state == previousState:
1272 rowNum += 1
1273 if rowNum == nRows:
1274 break
1275 state = states[rowNum]
1276 parsedStates.append(
1277 self.ParsedState(
1278 eventStart=eventStart,
1279 eventEnd=rowNum,
1280 previousState=previousState,
1281 state=state
1282 )
1283 )
1284 if state in skipStates:
1285 eventStart = None
1287 # done parsing, just check the last event is valid
1288 if parsedStates: # ensure we have at least one event
1289 lastEvent = parsedStates[-1]
1290 if lastEvent.eventEnd == nRows:
1291 # Generally, you *want* the timespan for an event to be the
1292 # first row of the next event, because you were in that state
1293 # right up until that state change. However, if that event is
1294 # a) the last one of the day and b) runs right up until the end
1295 # of the dataframe, then there isn't another row, so this will
1296 # overrun the array.
1297 #
1298 # If the data is for the current day then this isn't a worry,
1299 # as we're likely still taking data, and this event will likely
1300 # close yet, so we don't issue a warning, and simply drop the
1301 # event from the list.
1303 # However, if the data is for a past day then no new data will
1304 # come to close the event, so allow the event to be "open", and
1305 # issue a warning
1306 if dataIsForCurrentDay:
1307 self.log.info("Discarding open (likely in-progess) final event from current day's events")
1308 parsedStates = parsedStates[:-1]
1309 else:
1310 self.log.warning("Last event ends open, forcing it to end at end of the day's data")
1311 # it's a tuple, so (deliberately) awkward to modify
1312 parsedStates[-1] = self.ParsedState(
1313 eventStart=lastEvent.eventStart,
1314 eventEnd=lastEvent.eventEnd - 1,
1315 previousState=lastEvent.previousState,
1316 state=lastEvent.state
1317 )
1319 return parsedStates
1321 def addBlockDataToEvents(self, events):
1322 """Find all the block data in the EFD for the specified events.
1324 Finds all the block data in the EFD relating to the events, parses it,
1325 from the rows of the dataframe, and adds it to the events in place.
1327 Parameters
1328 ----------
1329 events : `lsst.summit.utils.tmaUtils.TMAEvent` or
1330 `list` of `lsst.summit.utils.tmaUtils.TMAEvent`
1331 One or more events to get the block data for.
1332 """
1333 events = ensure_iterable(events)
1334 events = sorted(events)
1336 # Get all the data in one go and then clip to the events in the loop.
1337 # This is orders of magnitude faster than querying individually.
1338 allData = getEfdData(self.client,
1339 "lsst.sal.Script.logevent_state",
1340 begin=events[0].begin, # time ordered, so this is the start of the window
1341 end=events[-1].end, # and this is the end
1342 warn=False)
1343 if allData.empty:
1344 self.log.info('No block data found for the specified events')
1345 return {}
1347 blockPattern = r"BLOCK-(\d+)"
1348 blockIdPattern = r"BL\d+(?:_\w+)+"
1349 sitcomPattern = r"SITCOM-(\d+)"
1351 for event in events:
1352 eventData = clipDataToEvent(allData, event)
1353 if eventData.empty:
1354 continue
1356 blockNums = set()
1357 blockIds = set()
1358 tickets = set()
1359 salIndices = set()
1360 stateList = []
1362 # for each for in the data which corresponds to the event, extract
1363 # the block number, block id, sitcom tickets, sal index and state
1364 # some may have multiple values, some may be None, so collect in
1365 # sets and then remove None, and validate on ones which must not
1366 # contain duplicate values
1367 for rowNum, row in eventData.iterrows():
1368 # the lastCheckpoint column contains the block number, blockId,
1369 # and any sitcom tickets.
1370 rowStr = row['lastCheckpoint']
1372 blockMatch = re.search(blockPattern, rowStr)
1373 blockNumber = int(blockMatch.group(1)) if blockMatch else None
1374 blockNums.add(blockNumber)
1376 blockIdMatch = re.search(blockIdPattern, rowStr)
1377 blockId = blockIdMatch.group(0) if blockIdMatch else None
1378 blockIds.add(blockId)
1380 sitcomMatches = re.findall(sitcomPattern, rowStr)
1381 sitcomTicketNumbers = [int(match) for match in sitcomMatches]
1382 tickets.update(sitcomTicketNumbers)
1384 salIndices.add(row['salIndex'])
1386 state = row['state']
1387 state = ScriptState(state) # cast this back to its native enum
1388 stateReason = row['reason'] # might be empty, might contain useful error messages
1389 stateTimestamp = efdTimestampToAstropy(row['private_efdStamp'])
1390 scriptStatePoint = ScriptStatePoint(time=stateTimestamp,
1391 state=state,
1392 reason=stateReason)
1393 stateList.append(scriptStatePoint)
1395 # remove all the Nones from the sets, and then check the lengths
1396 for fieldSet in (blockNums, blockIds, salIndices):
1397 if None in fieldSet:
1398 fieldSet.remove(None)
1400 # if we didn't find any block numbers at all, that is fine, just
1401 # continue as this event doesn't relate to a BLOCK
1402 if not blockNums:
1403 continue
1405 # but if it does related to a BLOCK then it must not have more than
1406 # one. If this is the case something is wrong with a SAL script, so
1407 # raise here to indicate the something needs debugging in the
1408 # scriptQueue or something like that.
1409 if len(blockNums) > 1:
1410 raise RuntimeError(f"Found multiple BLOCK values ({blockNums}) for {event}")
1411 blockNumber = blockNums.pop()
1413 # likewise for the blockIds
1414 if len(blockIds) > 1:
1415 raise RuntimeError(f"Found multiple blockIds ({blockIds}) for {event}")
1416 blockId = blockIds.pop()
1418 blockInfo = BlockInfo(
1419 blockNumber=blockNumber,
1420 blockId=blockId,
1421 salIndices=sorted([i for i in salIndices]),
1422 tickets=[f'SITCOM-{ticket}' for ticket in tickets],
1423 states=stateList,
1424 )
1426 # Add the blockInfo to the TMAEvent. Because this is a frozen
1427 # dataclass, use object.__setattr__ to set the attribute. This is
1428 # the correct way to set a frozen dataclass attribute after
1429 # creation.
1430 object.__setattr__(event, 'blockInfo', blockInfo)
1432 def _makeEventsFromStateTuples(self, states, dayObs, data):
1433 """For the list of state-tuples, create a list of ``TMAEvent`` objects.
1435 Given the underlying data, and the start/stop points for each event,
1436 create the TMAEvent objects for the dayObs.
1438 Parameters
1439 ----------
1440 states : `list` of `tuple`
1441 The parsed states, as a list of tuples of the form:
1442 ``(eventStart, eventEnd, eventType, endReason)``
1443 dayObs : `int`
1444 The dayObs for the data.
1445 data : `pd.DataFrame`
1446 The merged dataframe.
1448 Returns
1449 -------
1450 events : `list` of `lsst.summit.utils.tmaUtils.TMAEvent`
1451 The events for the specified dayObs.
1452 """
1453 seqNum = 0
1454 events = []
1455 for parsedState in states:
1456 begin = data.iloc[parsedState.eventStart]['private_efdStamp']
1457 end = data.iloc[parsedState.eventEnd]['private_efdStamp']
1458 beginAstropy = efdTimestampToAstropy(begin)
1459 endAstropy = efdTimestampToAstropy(end)
1460 duration = end - begin
1461 event = TMAEvent(
1462 dayObs=dayObs,
1463 seqNum=seqNum,
1464 type=parsedState.previousState,
1465 endReason=parsedState.state,
1466 duration=duration,
1467 begin=beginAstropy,
1468 end=endAstropy,
1469 blockInfo=None, # this is added later
1470 _startRow=parsedState.eventStart,
1471 _endRow=parsedState.eventEnd,
1472 )
1473 events.append(event)
1474 seqNum += 1
1475 return events
1477 @staticmethod
1478 def printTmaDetailedState(tma):
1479 """Print the full state of all the components of the TMA.
1481 Currently this is the azimuth and elevation axes' power and motion
1482 states, and their respective inPosition statuses.
1484 Parameters
1485 ----------
1486 tma : `lsst.summit.utils.tmaUtils.TMAStateMachine`
1487 The TMA state machine in the state we want to print.
1488 """
1489 axes = ['azimuth', 'elevation']
1490 p = tma._parts
1491 axisPad = len(max(axes, key=len)) # length of the longest axis string == 9 here, but this is general
1492 motionPad = max(len(s.name) for s in AxisMotionState)
1493 powerPad = max(len(s.name) for s in PowerState)
1495 # example output to show what's being done with the padding:
1496 # azimuth - Power: ON Motion: STOPPED InPosition: True # noqa: W505
1497 # elevation - Power: ON Motion: MOVING_POINT_TO_POINT InPosition: False # noqa: W505
1498 for axis in axes:
1499 print(f"{axis:>{axisPad}} - "
1500 f"Power: {p[f'{axis}SystemState'].name:>{powerPad}} "
1501 f"Motion: {p[f'{axis}MotionState'].name:>{motionPad}} "
1502 f"InPosition: {p[f'{axis}InPosition']}")
1503 print(f"Overall system state: {tma.state.name}")
1505 def printFullDayStateEvolution(self, dayObs, taiOrUtc='utc'):
1506 """Print the full TMA state evolution for the specified dayObs.
1508 Replays all the data from the EFD for the specified dayObs through
1509 the TMA state machine, and prints both the overall and detailed state
1510 of the TMA for each row.
1512 Parameters
1513 ----------
1514 dayObs : `int`
1515 The dayObs for which to print the state evolution.
1516 taiOrUtc : `str`, optional
1517 Whether to print the timestamps in TAI or UTC. Default is UTC.
1518 """
1519 # create a fake event which spans the whole day, and then use
1520 # printEventDetails code while skipping the header to print the
1521 # evolution.
1522 _ = self.getEvents(dayObs) # ensure the data has been retrieved from the EFD
1523 data = self._data[dayObs]
1524 lastRowNum = len(data) - 1
1526 fakeEvent = TMAEvent(
1527 dayObs=dayObs,
1528 seqNum=-1, # anything will do
1529 type=TMAState.OFF, # anything will do
1530 endReason=TMAState.OFF, # anything will do
1531 duration=-1, # anything will do
1532 begin=efdTimestampToAstropy(data.iloc[0]['private_efdStamp']),
1533 end=efdTimestampToAstropy(data.iloc[-1]['private_efdStamp']),
1534 _startRow=0,
1535 _endRow=lastRowNum
1536 )
1537 self.printEventDetails(fakeEvent, taiOrUtc=taiOrUtc, printHeader=False)
1539 def printEventDetails(self, event, taiOrUtc='tai', printHeader=True):
1540 """Print a detailed breakdown of all state transitions during an event.
1542 Note: this is not the most efficient way to do this, but it is much the
1543 cleanest with respect to the actual state machine application and event
1544 generation code, and is easily fast enough for the cases it will be
1545 used for. It is not worth complicating the normal state machine logic
1546 to try to use this code.
1548 Parameters
1549 ----------
1550 event : `lsst.summit.utils.tmaUtils.TMAEvent`
1551 The event to display the details of.
1552 taiOrUtc : `str`, optional
1553 Whether to display time strings in TAI or UTC. Defaults to TAI.
1554 Case insensitive.
1555 printHeader : `bool`, optional
1556 Whether to print the event summary. Defaults to True. The primary
1557 reason for the existence of this option is so that this same
1558 printing function can be used to show the evolution of a whole day
1559 by supplying a fake event which spans the whole day, but this event
1560 necessarily has a meaningless summary, and so needs suppressing.
1561 """
1562 taiOrUtc = taiOrUtc.lower()
1563 if taiOrUtc not in ['tai', 'utc']:
1564 raise ValueError(f'Got unsuppoted value for {taiOrUtc=}')
1565 useUtc = taiOrUtc == 'utc'
1567 if printHeader:
1568 print(f"Details for {event.duration:.2f}s {event.type.name} event dayObs={event.dayObs}"
1569 f" seqNum={event.seqNum}:")
1570 print(f"- Event began at: {event.begin.utc.isot if useUtc else event.begin.isot}")
1571 print(f"- Event ended at: {event.end.utc.isot if useUtc else event.end.isot}")
1573 dayObs = event.dayObs
1574 data = self._data[dayObs]
1575 startRow = event._startRow
1576 endRow = event._endRow
1577 nRowsToApply = endRow - startRow + 1
1578 print(f"\nTotal number of rows in the merged dataframe: {len(data)}")
1579 if printHeader:
1580 print(f"of which rows {startRow} to {endRow} (inclusive) relate to this event.")
1582 # reconstruct all the states
1583 tma = TMAStateMachine(engineeringMode=True)
1584 _initializeTma(tma)
1586 tmaStates = {}
1587 firstAppliedRow = True # flag to print a header on the first row that's applied
1588 for rowNum, row in data.iterrows(): # must replay rows right from start to get full correct state
1589 if rowNum == startRow:
1590 # we've not yet applied this row, so this is the state just
1591 # before event
1592 print(f"\nBefore the event the TMA was in state {tma.state.name}:")
1593 self.printTmaDetailedState(tma)
1595 if rowNum >= startRow and rowNum <= endRow:
1596 if firstAppliedRow: # only print this intro on the first row we're applying
1597 print(f"\nThen, applying the {nRowsToApply} rows of data for this event, the state"
1598 " evolved as follows:\n")
1599 firstAppliedRow = False
1601 # break the row down and print its details
1602 rowFor = row['rowFor']
1603 axis, rowType = getAxisAndType(rowFor) # e.g. elevation, MotionState
1604 value = tma._getRowPayload(row, rowType, rowFor)
1605 valueStr = f"{str(value) if isinstance(value, bool) else value.name}"
1606 rowTime = efdTimestampToAstropy(row['private_efdStamp'])
1607 print(f"On row {rowNum} the {axis} axis had the {rowType} set to {valueStr} at"
1608 f" {rowTime.utc.isot if useUtc else rowTime.isot}")
1610 # then apply it as usual, printing the state right afterwards
1611 tma.apply(row)
1612 tmaStates[rowNum] = tma.state
1613 self.printTmaDetailedState(tma)
1614 print()
1616 else:
1617 # if it's not in the range of interest then just apply it
1618 # silently as usual
1619 tma.apply(row)
1620 tmaStates[rowNum] = tma.state
1622 def findEvent(self, time):
1623 """Find the event which contains the specified time.
1625 If the specified time lies within an event, that event is returned. If
1626 it is at the exact start, that is logged, and if that start point is
1627 shared by the end of the previous event, that is logged too. If the
1628 event lies between events, the events either side are logged, but
1629 ``None`` is returned. If the time lies before the first event of the
1630 day a warning is logged, as for times after the last event of the day.
1632 Parameters
1633 ----------
1634 time : `astropy.time.Time`
1635 The time.
1637 Returns
1638 -------
1639 event : `lsst.summit.utils.tmaUtils.TMAEvent` or `None`
1640 The event which contains the specified time, or ``None`` if the
1641 time doesn't fall during an event.
1642 """
1643 # there are five possible cases:
1644 # 1) the time lies before the first event of the day
1645 # 2) the time lies after the last event of the day
1646 # 3) the time lies within an event
1647 # 3a) the time is exactly at the start of an event
1648 # 3b) if so, time can be shared by the end of the previous event if
1649 # they are contiguous
1650 # 4) the time lies between two events
1651 # 5) the time is exactly at end of the last event of the day. This is
1652 # an issue because event end times are exclusive, so this time is
1653 # not technically in that event, it's the moment it closes (and if
1654 # there *was* an event which followed contiguously, it would be in
1655 # that event instead, which is what motivates this definition of
1656 # lies within what event)
1658 dayObs = getDayObsForTime(time)
1659 # we know this is on the right day, and definitely before the specified
1660 # time, but sanity check this before continuing as this needs to be
1661 # true for this to give the correct answer
1662 assert getDayObsStartTime(dayObs) <= time
1663 assert getDayObsEndTime(dayObs) > time
1665 # command start to many log messages so define once here
1666 logStart = f"Specified time {time.isot} falls on {dayObs=}"
1668 events = self.getEvents(dayObs)
1669 if len(events) == 0:
1670 self.log.warning(f'There are no events found for {dayObs}')
1671 return None
1673 # check case 1)
1674 if time < events[0].begin:
1675 self.log.warning(f'{logStart} and is before the first event of the day')
1676 return None
1678 # check case 2)
1679 if time > events[-1].end:
1680 self.log.warning(f'{logStart} and is after the last event of the day')
1681 return None
1683 # check case 5)
1684 if time == events[-1].end:
1685 self.log.warning(f'{logStart} and is exactly at the end of the last event of the day'
1686 f' (seqnum={events[-1].seqNum}). Because event intervals are half-open, this'
1687 ' time does not technically lie in any event')
1688 return None
1690 # we are now either in an event, or between events. Walk through the
1691 # events, and if the end of the event is after the specified time, then
1692 # we're either in it or past it, so check if we're in.
1693 for eventNum, event in enumerate(events):
1694 if event.end > time: # case 3) we are now into or past the right event
1695 # the event end encloses the time, so note the > and not >=,
1696 # this must be strictly greater, we check the overlap case
1697 # later
1698 if time >= event.begin: # we're fully inside the event, so return it.
1699 # 3a) before returning, check if we're exactly at the start
1700 # of the event, and if so, log it. Then 3b) also check if
1701 # we're at the exact end of the previous event, and if so,
1702 # log that too.
1703 if time == event.begin:
1704 self.log.info(f"{logStart} and is exactly at the start of event"
1705 f" {eventNum}")
1706 if eventNum == 0: # I think this is actually impossible, but check anyway
1707 return event # can't check the previous event so return here
1708 previousEvent = events[eventNum - 1]
1709 if previousEvent.end == time:
1710 self.log.info("Previous event is contiguous, so this time is also at the exact"
1711 f" end of {eventNum - 1}")
1712 return event
1713 else: # case 4)
1714 # the event end is past the time, but it's not inside the
1715 # event, so we're between events. Log which we're between
1716 # and return None
1717 previousEvent = events[eventNum - 1]
1718 timeAfterPrev = (time - previousEvent.end).to_datetime()
1719 naturalTimeAfterPrev = humanize.naturaldelta(timeAfterPrev, minimum_unit='MICROSECONDS')
1720 timeBeforeCurrent = (event.begin - time).to_datetime()
1721 naturalTimeBeforeCurrent = humanize.naturaldelta(timeBeforeCurrent,
1722 minimum_unit='MICROSECONDS')
1723 self.log.info(f"{logStart} and lies"
1724 f" {naturalTimeAfterPrev} after the end of event {previousEvent.seqNum}"
1725 f" and {naturalTimeBeforeCurrent} before the start of event {event.seqNum}."
1726 )
1727 return None
1729 raise RuntimeError('Event finding logic fundamentally failed, which should never happen - the code'
1730 ' needs fixing')