Coverage for python/lsst/summit/utils/efdUtils.py: 16%
131 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-06 04:33 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-06 04:33 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import asyncio
23from astropy.time import Time, TimeDelta
24from astropy import units as u
25import datetime
26import logging
27import pandas as pd
29from .utils import getSite
31HAS_EFD_CLIENT = True
32try:
33 from lsst_efd_client import EfdClient
34except ImportError:
35 HAS_EFD_CLIENT = False
37__all__ = [
38 'getEfdData',
39 'getMostRecentRowWithDataBefore',
40 'makeEfdClient',
41 'expRecordToTimespan',
42 'efdTimestampToAstropy',
43 'astropyToEfdTimestamp',
44 'clipDataToEvent',
45 'calcNextDay',
46 'getDayObsStartTime',
47 'getDayObsEndTime',
48 'getDayObsForTime',
49 'getSubTopics',
50]
53COMMAND_ALIASES = {
54 'raDecTarget': 'lsst.sal.MTPtg.command_raDecTarget',
55 'moveToTarget': 'lsst.sal.MTMount.command_moveToTarget',
56 'startTracking': 'lsst.sal.MTMount.command_startTracking',
57 'stopTracking': 'lsst.sal.MTMount.command_stopTracking',
58 'trackTarget': 'lsst.sal.MTMount.command_trackTarget', # issued at 20Hz - don't plot
59}
61# When looking backwards in time to find the most recent state event, look back
62# in chunks of this size. Too small, and there will be too many queries, too
63# large and there will be too much data returned unnecessarily, as we only need
64# one row by definition. Will tune this parameters in consultation with SQuaRE.
65TIME_CHUNKING = datetime.timedelta(minutes=15)
68def _getBeginEnd(dayObs=None, begin=None, end=None, timespan=None, event=None, expRecord=None):
69 """Calculate the begin and end times to pass to _getEfdData, given the
70 kwargs passed to getEfdData.
72 Parameters
73 ----------
74 dayObs : `int`
75 The dayObs to query. If specified, this is used to determine the begin
76 and end times.
77 begin : `astropy.Time`
78 The begin time for the query. If specified, either an end time or a
79 timespan must be supplied.
80 end : `astropy.Time`
81 The end time for the query. If specified, a begin time must also be
82 supplied.
83 timespan : `astropy.TimeDelta`
84 The timespan for the query. If specified, a begin time must also be
85 supplied.
86 event : `lsst.summit.utils.efdUtils.TmaEvent`
87 The event to query. If specified, this is used to determine the begin
88 and end times, and all other options are disallowed.
89 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`
90 The exposure record containing the timespan to query. If specified, all
91 other options are disallowed.
93 Returns
94 -------
95 begin : `astropy.Time`
96 The begin time for the query.
97 end : `astropy.Time`
98 The end time for the query.
99 """
100 if expRecord is not None:
101 forbiddenOpts = [event, begin, end, timespan, dayObs]
102 if any(x is not None for x in forbiddenOpts):
103 raise ValueError("You can't specify both an expRecord and a begin/end or timespan or dayObs")
104 begin = expRecord.timespan.begin
105 end = expRecord.timespan.end
106 return begin, end
108 if event is not None:
109 forbiddenOpts = [begin, end, timespan, dayObs]
110 if any(x is not None for x in forbiddenOpts):
111 raise ValueError("You can't specify both an event and a begin/end or timespan or dayObs")
112 begin = event.begin
113 end = event.end
114 return begin, end
116 # check for dayObs, and that other options aren't inconsistently specified
117 if dayObs is not None:
118 forbiddenOpts = [begin, end, timespan]
119 if any(x is not None for x in forbiddenOpts):
120 raise ValueError("You can't specify both a dayObs and a begin/end or timespan")
121 begin = getDayObsStartTime(dayObs)
122 end = getDayObsEndTime(dayObs)
123 return begin, end
124 # can now disregard dayObs entirely
126 if begin is None:
127 raise ValueError("You must specify either a dayObs or a begin/end or begin/timespan")
128 # can now rely on begin, so just need to deal with end/timespan
130 if end is None and timespan is None:
131 raise ValueError("If you specify a begin, you must specify either a end or a timespan")
132 if end is not None and timespan is not None:
133 raise ValueError("You can't specify both a end and a timespan")
134 if end is None:
135 if timespan > datetime.timedelta(minutes=0):
136 end = begin + timespan # the normal case
137 else:
138 end = begin # the case where timespan is negative
139 begin = begin + timespan # adding the negative to the start, i.e. subtracting it to bring back
141 assert begin is not None
142 assert end is not None
143 return begin, end
146def getEfdData(client, topic, *,
147 columns=None,
148 prePadding=0,
149 postPadding=0,
150 dayObs=None,
151 begin=None,
152 end=None,
153 timespan=None,
154 event=None,
155 expRecord=None,
156 warn=True,
157 ):
158 """Get one or more EFD topics over a time range, synchronously.
160 The time range can be specified as either:
161 * a dayObs, in which case the full 24 hour period is used,
162 * a begin point and a end point,
163 * a begin point and a timespan.
164 * a mount event
165 * an exposure record
166 If it is desired to use an end time with a timespan, just specify it as the
167 begin time and use a negative timespan.
169 The results from all topics are merged into a single dataframe.
171 Parameters
172 ----------
173 client : `lsst_efd_client.efd_helper.EfdClient`
174 The EFD client to use.
175 topic : `str`
176 The topic to query.
177 columns : `list` of `str`, optional
178 The columns to query. If not specified, all columns are queried.
179 prePadding : `float`
180 The amount of time before the nominal start of the query to include, in
181 seconds.
182 postPadding : `float`
183 The amount of extra time after the nominal end of the query to include,
184 in seconds.
185 dayObs : `int`, optional
186 The dayObs to query. If specified, this is used to determine the begin
187 and end times.
188 begin : `astropy.Time`, optional
189 The begin time for the query. If specified, either a end time or a
190 timespan must be supplied.
191 end : `astropy.Time`, optional
192 The end time for the query. If specified, a begin time must also be
193 supplied.
194 timespan : `astropy.TimeDelta`, optional
195 The timespan for the query. If specified, a begin time must also be
196 supplied.
197 event : `lsst.summit.utils.efdUtils.TmaEvent`, optional
198 The event to query. If specified, this is used to determine the begin
199 and end times, and all other options are disallowed.
200 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`, optional
201 The exposure record containing the timespan to query. If specified, all
202 other options are disallowed.
203 warn : bool, optional
204 If ``True``, warn when no data is found. Exists so that utility code
205 can disable warnings when checking for data, and therefore defaults to
206 ``True``.
208 Returns
209 -------
210 data : `pd.DataFrame`
211 The merged data from all topics.
213 Raises
214 ------
215 ValueError:
216 If the topics are not in the EFD schema.
217 ValueError:
218 If both a dayObs and a begin/end or timespan are specified.
219 ValueError:
220 If a begin time is specified but no end time or timespan.
222 """
223 # TODO: DM-40100 ideally should calls mpts as necessary so that users
224 # needn't care if things are packed
226 # supports aliases so that you can query with them. If there is no entry in
227 # the alias dict then it queries with the supplied key. The fact the schema
228 # is now being checked means this shouldn't be a problem now.
230 # TODO: RFC-948 Move this import back to top of file once is implemented.
231 import nest_asyncio
233 begin, end = _getBeginEnd(dayObs, begin, end, timespan, event, expRecord)
234 begin -= TimeDelta(prePadding, format='sec')
235 end += TimeDelta(postPadding, format='sec')
237 nest_asyncio.apply()
238 loop = asyncio.get_event_loop()
239 ret = loop.run_until_complete(_getEfdData(client=client,
240 topic=topic,
241 begin=begin,
242 end=end,
243 columns=columns))
244 if ret.empty and warn:
245 log = logging.getLogger(__name__)
246 log.warning(f"Topic {topic} is in the schema, but no data was returned by the query for the specified"
247 " time range")
248 return ret
251async def _getEfdData(client, topic, begin, end, columns=None):
252 """Get data for a topic from the EFD over the specified time range.
254 Parameters
255 ----------
256 client : `lsst_efd_client.efd_helper.EfdClient`
257 The EFD client to use.
258 topic : `str`
259 The topic to query.
260 begin : `astropy.Time`
261 The begin time for the query.
262 end : `astropy.Time`
263 The end time for the query.
264 columns : `list` of `str`, optional
265 The columns to query. If not specified, all columns are returned.
267 Returns
268 -------
269 data : `pd.DataFrame`
270 The data from the query.
271 """
272 if columns is None:
273 columns = ['*']
275 availableTopics = await client.get_topics()
277 if topic not in availableTopics:
278 raise ValueError(f"Topic {topic} not in EFD schema")
280 data = await client.select_time_series(topic, columns, begin.utc, end.utc)
282 return data
285def getMostRecentRowWithDataBefore(client, topic, timeToLookBefore, warnStaleAfterNMinutes=60*12):
286 """Get the most recent row of data for a topic before a given time.
288 Parameters
289 ----------
290 client : `lsst_efd_client.efd_helper.EfdClient`
291 The EFD client to use.
292 topic : `str`
293 The topic to query.
294 timeToLookBefore : `astropy.Time`
295 The time to look before.
296 warnStaleAfterNMinutes : `float`, optional
297 The number of minutes after which to consider the data stale and issue
298 a warning.
300 Returns
301 -------
302 row : `pd.Series`
303 The row of data from the EFD containing the most recent data before the
304 specified time.
306 Raises
307 ------
308 ValueError:
309 If the topic is not in the EFD schema.
310 """
311 staleAge = datetime.timedelta(warnStaleAfterNMinutes)
313 firstDayPossible = getDayObsStartTime(20190101)
315 if timeToLookBefore < firstDayPossible:
316 raise ValueError(f"Requested time {timeToLookBefore} is before any data was put in the EFD")
318 df = pd.DataFrame()
319 beginTime = timeToLookBefore
320 while df.empty and beginTime > firstDayPossible:
321 df = getEfdData(client, topic, begin=beginTime, timespan=-TIME_CHUNKING, warn=False)
322 beginTime -= TIME_CHUNKING
324 if beginTime < firstDayPossible and df.empty: # we ran all the way back to the beginning of time
325 raise ValueError(f"The entire EFD was searched backwards from {timeToLookBefore} and no data was "
326 f"found in {topic=}")
328 lastRow = df.iloc[-1]
329 commandTime = efdTimestampToAstropy(lastRow['private_efdStamp'])
331 commandAge = timeToLookBefore - commandTime
332 if commandAge > staleAge:
333 log = logging.getLogger(__name__)
334 log.warning(f"Component {topic} was last set {commandAge.sec/60:.1} minutes"
335 " before the requested time")
337 return lastRow
340def makeEfdClient():
341 """Automatically create an EFD client based on the site.
343 Returns
344 -------
345 efdClient : `lsst_efd_client.efd_helper.EfdClient`
346 The EFD client to use for the current site.
347 """
348 if not HAS_EFD_CLIENT:
349 raise RuntimeError("Could not create EFD client because importing lsst_efd_client failed.")
351 try:
352 site = getSite()
353 except ValueError as e:
354 raise RuntimeError("Could not create EFD client as the site could not be determined") from e
356 if site == 'summit':
357 return EfdClient('summit_efd')
358 if site == 'base':
359 return EfdClient('summit_efd_copy')
360 if site in ['staff-rsp', 'rubin-devl']:
361 return EfdClient('usdf_efd')
363 raise RuntimeError(f"Could not create EFD client as the {site=} is not recognized")
366def expRecordToTimespan(expRecord):
367 """Get the timespan from an exposure record.
369 Returns the timespan in a format where it can be used to directly unpack
370 into a efdClient.select_time_series() call.
372 Parameters
373 ----------
374 expRecord : `lsst.daf.butler.dimensions.ExposureRecord`
375 The exposure record.
377 Returns
378 -------
379 timespanDict : `dict`
380 The timespan in a format that can be used to directly unpack into a
381 efdClient.select_time_series() call.
382 """
383 return {'begin': expRecord.timespan.begin.utc,
384 'end': expRecord.timespan.end.utc,
385 }
388def efdTimestampToAstropy(timestamp):
389 """Get an efd timestamp as an astropy.time.Time object.
391 Parameters
392 ----------
393 timestamp : `float`
394 The timestamp, as a float.
396 Returns
397 -------
398 time : `astropy.time.Time`
399 The timestamp as an astropy.time.Time object.
400 """
401 return Time(timestamp, format='unix')
404def astropyToEfdTimestamp(time):
405 """Get astropy Time object as an efd timestamp
407 Parameters
408 ----------
409 time : `astropy.time.Time`
410 The time as an astropy.time.Time object.
412 Returns
413 -------
414 timestamp : `float`
415 The timestamp, in UTC, in unix seconds.
416 """
418 return time.utc.unix
421def clipDataToEvent(df, event):
422 """Clip a padded dataframe to an event.
424 Parameters
425 ----------
426 df : `pd.DataFrame`
427 The dataframe to clip.
428 event : `lsst.summit.utils.efdUtils.TmaEvent`
429 The event to clip to.
431 Returns
432 -------
433 clipped : `pd.DataFrame`
434 The clipped dataframe.
435 """
436 mask = (df['private_efdStamp'] >= event.begin.value) & (df['private_efdStamp'] <= event.end.value)
437 clipped_df = df.loc[mask].copy()
438 return clipped_df
441def calcNextDay(dayObs):
442 """Given an integer dayObs, calculate the next integer dayObs.
444 Integers are used for dayObs, but dayObs values are therefore not
445 contiguous due to month/year ends etc, so this utility provides a robust
446 way to get the integer dayObs which follows the one specified.
448 Parameters
449 ----------
450 dayObs : `int`
451 The dayObs, as an integer, e.g. 20231231
453 Returns
454 -------
455 nextDayObs : `int`
456 The next dayObs, as an integer, e.g. 20240101
457 """
458 d1 = datetime.datetime.strptime(str(dayObs), '%Y%m%d')
459 oneDay = datetime.timedelta(days=1)
460 return int((d1 + oneDay).strftime('%Y%m%d'))
463def getDayObsStartTime(dayObs):
464 """Get the start of the given dayObs as an astropy.time.Time object.
466 The observatory rolls the date over at UTC-12.
468 Parameters
469 ----------
470 dayObs : `int`
471 The dayObs, as an integer, e.g. 20231225
473 Returns
474 -------
475 time : `astropy.time.Time`
476 The start of the dayObs as an astropy.time.Time object.
477 """
478 pythonDateTime = datetime.datetime.strptime(str(dayObs), "%Y%m%d")
479 return Time(pythonDateTime) + 12 * u.hour
482def getDayObsEndTime(dayObs):
483 """Get the end of the given dayObs as an astropy.time.Time object.
485 Parameters
486 ----------
487 dayObs : `int`
488 The dayObs, as an integer, e.g. 20231225
490 Returns
491 -------
492 time : `astropy.time.Time`
493 The end of the dayObs as an astropy.time.Time object.
494 """
495 return getDayObsStartTime(dayObs) + 24 * u.hour
498def getDayObsForTime(time):
499 """Get the dayObs in which an astropy.time.Time object falls.
501 Parameters
502 ----------
503 time : `astropy.time.Time`
504 The time.
506 Returns
507 -------
508 dayObs : `int`
509 The dayObs, as an integer, e.g. 20231225
510 """
511 twelveHours = datetime.timedelta(hours=-12)
512 offset = TimeDelta(twelveHours, format='datetime')
513 return int((time + offset).utc.isot[:10].replace('-', ''))
516def getSubTopics(client, topic):
517 """Get all the sub topics within a given topic.
519 Note that the topic need not be a complete one, for example, rather than
520 doing `getSubTopics(client, 'lsst.sal.ATMCS')` to get all the topics for
521 the AuxTel Mount Control System, you can do `getSubTopics(client,
522 'lsst.sal.AT')` to get all which relate to the AuxTel in general.
524 Parameters
525 ----------
526 client : `lsst_efd_client.efd_helper.EfdClient`
527 The EFD client to use.
528 topic : `str`
529 The topic to query.
531 Returns
532 -------
533 subTopics : `list` of `str`
534 The sub topics.
535 """
536 loop = asyncio.get_event_loop()
537 topics = loop.run_until_complete(client.get_topics())
538 return sorted([t for t in topics if t.startswith(topic)])