Coverage for python/lsst/summit/utils/efdUtils.py: 17%
150 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-24 12:17 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-24 12:17 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import asyncio
23from astropy.time import Time, TimeDelta
24from astropy import units as u
25import datetime
26import logging
27import pandas as pd
28import re
29from deprecated.sphinx import deprecated
31from .utils import getSite
33HAS_EFD_CLIENT = True
34try:
35 from lsst_efd_client import EfdClient
36except ImportError:
37 HAS_EFD_CLIENT = False
39__all__ = [
40 'getEfdData',
41 'getMostRecentRowWithDataBefore',
42 'makeEfdClient',
43 'expRecordToTimespan',
44 'efdTimestampToAstropy',
45 'astropyToEfdTimestamp',
46 'clipDataToEvent',
47 'calcNextDay',
48 'getDayObsStartTime',
49 'getDayObsEndTime',
50 'getDayObsForTime',
51 'getSubTopics', # deprecated, being removed in w_2023_50
52 'getTopics',
53]
56COMMAND_ALIASES = {
57 'raDecTarget': 'lsst.sal.MTPtg.command_raDecTarget',
58 'moveToTarget': 'lsst.sal.MTMount.command_moveToTarget',
59 'startTracking': 'lsst.sal.MTMount.command_startTracking',
60 'stopTracking': 'lsst.sal.MTMount.command_stopTracking',
61 'trackTarget': 'lsst.sal.MTMount.command_trackTarget', # issued at 20Hz - don't plot
62}
64# When looking backwards in time to find the most recent state event, look back
65# in chunks of this size. Too small, and there will be too many queries, too
66# large and there will be too much data returned unnecessarily, as we only need
67# one row by definition. Will tune this parameters in consultation with SQuaRE.
68TIME_CHUNKING = datetime.timedelta(minutes=15)
71def _getBeginEnd(dayObs=None, begin=None, end=None, timespan=None, event=None, expRecord=None):
72 """Calculate the begin and end times to pass to _getEfdData, given the
73 kwargs passed to getEfdData.
75 Parameters
76 ----------
77 dayObs : `int`
78 The dayObs to query. If specified, this is used to determine the begin
79 and end times.
80 begin : `astropy.Time`
81 The begin time for the query. If specified, either an end time or a
82 timespan must be supplied.
83 end : `astropy.Time`
84 The end time for the query. If specified, a begin time must also be
85 supplied.
86 timespan : `astropy.TimeDelta`
87 The timespan for the query. If specified, a begin time must also be
88 supplied.
89 event : `lsst.summit.utils.efdUtils.TmaEvent`
90 The event to query. If specified, this is used to determine the begin
91 and end times, and all other options are disallowed.
92 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`
93 The exposure record containing the timespan to query. If specified, all
94 other options are disallowed.
96 Returns
97 -------
98 begin : `astropy.Time`
99 The begin time for the query.
100 end : `astropy.Time`
101 The end time for the query.
102 """
103 if expRecord is not None:
104 forbiddenOpts = [event, begin, end, timespan, dayObs]
105 if any(x is not None for x in forbiddenOpts):
106 raise ValueError("You can't specify both an expRecord and a begin/end or timespan or dayObs")
107 begin = expRecord.timespan.begin
108 end = expRecord.timespan.end
109 return begin, end
111 if event is not None:
112 forbiddenOpts = [begin, end, timespan, dayObs]
113 if any(x is not None for x in forbiddenOpts):
114 raise ValueError("You can't specify both an event and a begin/end or timespan or dayObs")
115 begin = event.begin
116 end = event.end
117 return begin, end
119 # check for dayObs, and that other options aren't inconsistently specified
120 if dayObs is not None:
121 forbiddenOpts = [begin, end, timespan]
122 if any(x is not None for x in forbiddenOpts):
123 raise ValueError("You can't specify both a dayObs and a begin/end or timespan")
124 begin = getDayObsStartTime(dayObs)
125 end = getDayObsEndTime(dayObs)
126 return begin, end
127 # can now disregard dayObs entirely
129 if begin is None:
130 raise ValueError("You must specify either a dayObs or a begin/end or begin/timespan")
131 # can now rely on begin, so just need to deal with end/timespan
133 if end is None and timespan is None:
134 raise ValueError("If you specify a begin, you must specify either a end or a timespan")
135 if end is not None and timespan is not None:
136 raise ValueError("You can't specify both a end and a timespan")
137 if end is None:
138 if timespan > datetime.timedelta(minutes=0):
139 end = begin + timespan # the normal case
140 else:
141 end = begin # the case where timespan is negative
142 begin = begin + timespan # adding the negative to the start, i.e. subtracting it to bring back
144 assert begin is not None
145 assert end is not None
146 return begin, end
149def getEfdData(client, topic, *,
150 columns=None,
151 prePadding=0,
152 postPadding=0,
153 dayObs=None,
154 begin=None,
155 end=None,
156 timespan=None,
157 event=None,
158 expRecord=None,
159 warn=True,
160 ):
161 """Get one or more EFD topics over a time range, synchronously.
163 The time range can be specified as either:
164 * a dayObs, in which case the full 24 hour period is used,
165 * a begin point and a end point,
166 * a begin point and a timespan.
167 * a mount event
168 * an exposure record
169 If it is desired to use an end time with a timespan, just specify it as the
170 begin time and use a negative timespan.
172 The results from all topics are merged into a single dataframe.
174 Parameters
175 ----------
176 client : `lsst_efd_client.efd_helper.EfdClient`
177 The EFD client to use.
178 topic : `str`
179 The topic to query.
180 columns : `list` of `str`, optional
181 The columns to query. If not specified, all columns are queried.
182 prePadding : `float`
183 The amount of time before the nominal start of the query to include, in
184 seconds.
185 postPadding : `float`
186 The amount of extra time after the nominal end of the query to include,
187 in seconds.
188 dayObs : `int`, optional
189 The dayObs to query. If specified, this is used to determine the begin
190 and end times.
191 begin : `astropy.Time`, optional
192 The begin time for the query. If specified, either a end time or a
193 timespan must be supplied.
194 end : `astropy.Time`, optional
195 The end time for the query. If specified, a begin time must also be
196 supplied.
197 timespan : `astropy.TimeDelta`, optional
198 The timespan for the query. If specified, a begin time must also be
199 supplied.
200 event : `lsst.summit.utils.efdUtils.TmaEvent`, optional
201 The event to query. If specified, this is used to determine the begin
202 and end times, and all other options are disallowed.
203 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`, optional
204 The exposure record containing the timespan to query. If specified, all
205 other options are disallowed.
206 warn : bool, optional
207 If ``True``, warn when no data is found. Exists so that utility code
208 can disable warnings when checking for data, and therefore defaults to
209 ``True``.
211 Returns
212 -------
213 data : `pd.DataFrame`
214 The merged data from all topics.
216 Raises
217 ------
218 ValueError:
219 If the topics are not in the EFD schema.
220 ValueError:
221 If both a dayObs and a begin/end or timespan are specified.
222 ValueError:
223 If a begin time is specified but no end time or timespan.
225 """
226 # TODO: DM-40100 ideally should calls mpts as necessary so that users
227 # needn't care if things are packed
229 # supports aliases so that you can query with them. If there is no entry in
230 # the alias dict then it queries with the supplied key. The fact the schema
231 # is now being checked means this shouldn't be a problem now.
233 # TODO: RFC-948 Move this import back to top of file once is implemented.
234 import nest_asyncio
236 begin, end = _getBeginEnd(dayObs, begin, end, timespan, event, expRecord)
237 begin -= TimeDelta(prePadding, format='sec')
238 end += TimeDelta(postPadding, format='sec')
240 nest_asyncio.apply()
241 loop = asyncio.get_event_loop()
242 ret = loop.run_until_complete(_getEfdData(client=client,
243 topic=topic,
244 begin=begin,
245 end=end,
246 columns=columns))
247 if ret.empty and warn:
248 log = logging.getLogger(__name__)
249 log.warning(f"Topic {topic} is in the schema, but no data was returned by the query for the specified"
250 " time range")
251 return ret
254async def _getEfdData(client, topic, begin, end, columns=None):
255 """Get data for a topic from the EFD over the specified time range.
257 Parameters
258 ----------
259 client : `lsst_efd_client.efd_helper.EfdClient`
260 The EFD client to use.
261 topic : `str`
262 The topic to query.
263 begin : `astropy.Time`
264 The begin time for the query.
265 end : `astropy.Time`
266 The end time for the query.
267 columns : `list` of `str`, optional
268 The columns to query. If not specified, all columns are returned.
270 Returns
271 -------
272 data : `pd.DataFrame`
273 The data from the query.
274 """
275 if columns is None:
276 columns = ['*']
278 availableTopics = await client.get_topics()
280 if topic not in availableTopics:
281 raise ValueError(f"Topic {topic} not in EFD schema")
283 data = await client.select_time_series(topic, columns, begin.utc, end.utc)
285 return data
288def getMostRecentRowWithDataBefore(client, topic, timeToLookBefore, warnStaleAfterNMinutes=60*12):
289 """Get the most recent row of data for a topic before a given time.
291 Parameters
292 ----------
293 client : `lsst_efd_client.efd_helper.EfdClient`
294 The EFD client to use.
295 topic : `str`
296 The topic to query.
297 timeToLookBefore : `astropy.Time`
298 The time to look before.
299 warnStaleAfterNMinutes : `float`, optional
300 The number of minutes after which to consider the data stale and issue
301 a warning.
303 Returns
304 -------
305 row : `pd.Series`
306 The row of data from the EFD containing the most recent data before the
307 specified time.
309 Raises
310 ------
311 ValueError:
312 If the topic is not in the EFD schema.
313 """
314 staleAge = datetime.timedelta(warnStaleAfterNMinutes)
316 firstDayPossible = getDayObsStartTime(20190101)
318 if timeToLookBefore < firstDayPossible:
319 raise ValueError(f"Requested time {timeToLookBefore} is before any data was put in the EFD")
321 df = pd.DataFrame()
322 beginTime = timeToLookBefore
323 while df.empty and beginTime > firstDayPossible:
324 df = getEfdData(client, topic, begin=beginTime, timespan=-TIME_CHUNKING, warn=False)
325 beginTime -= TIME_CHUNKING
327 if beginTime < firstDayPossible and df.empty: # we ran all the way back to the beginning of time
328 raise ValueError(f"The entire EFD was searched backwards from {timeToLookBefore} and no data was "
329 f"found in {topic=}")
331 lastRow = df.iloc[-1]
332 commandTime = efdTimestampToAstropy(lastRow['private_efdStamp'])
334 commandAge = timeToLookBefore - commandTime
335 if commandAge > staleAge:
336 log = logging.getLogger(__name__)
337 log.warning(f"Component {topic} was last set {commandAge.sec/60:.1} minutes"
338 " before the requested time")
340 return lastRow
343def makeEfdClient(testing=False):
344 """Automatically create an EFD client based on the site.
346 Parameters
347 ----------
348 testing : `bool`, optional
349 Set to ``True`` if running in a test suite. This will default to using
350 the USDF EFD, for which data has been recorded for replay by the ``vcr`
351 package. Note data must be re-recorded to ``vcr`` from both inside and
352 outside the USDF when the package/data changes, due to the use of a
353 proxy meaning that the web requests are different depending on whether
354 the EFD is being contacted from inside and outside the USDF.
356 Returns
357 -------
358 efdClient : `lsst_efd_client.efd_helper.EfdClient`
359 The EFD client to use for the current site.
360 """
361 if not HAS_EFD_CLIENT:
362 raise RuntimeError("Could not create EFD client because importing lsst_efd_client failed.")
364 if testing:
365 return EfdClient('usdf_efd')
367 try:
368 site = getSite()
369 except ValueError as e:
370 raise RuntimeError("Could not create EFD client as the site could not be determined") from e
372 if site == 'summit':
373 return EfdClient('summit_efd')
374 if site == 'tucson':
375 return EfdClient('tucson_teststand_efd')
376 if site == 'base':
377 return EfdClient('base_efd')
378 if site in ['staff-rsp', 'rubin-devl', 'usdf-k8s']:
379 return EfdClient('usdf_efd')
380 if site == 'jenkins':
381 return EfdClient('usdf_efd')
383 raise RuntimeError(f"Could not create EFD client as the {site=} is not recognized")
386def expRecordToTimespan(expRecord):
387 """Get the timespan from an exposure record.
389 Returns the timespan in a format where it can be used to directly unpack
390 into a efdClient.select_time_series() call.
392 Parameters
393 ----------
394 expRecord : `lsst.daf.butler.dimensions.ExposureRecord`
395 The exposure record.
397 Returns
398 -------
399 timespanDict : `dict`
400 The timespan in a format that can be used to directly unpack into a
401 efdClient.select_time_series() call.
402 """
403 return {'begin': expRecord.timespan.begin.utc,
404 'end': expRecord.timespan.end.utc,
405 }
408def efdTimestampToAstropy(timestamp):
409 """Get an efd timestamp as an astropy.time.Time object.
411 Parameters
412 ----------
413 timestamp : `float`
414 The timestamp, as a float.
416 Returns
417 -------
418 time : `astropy.time.Time`
419 The timestamp as an astropy.time.Time object.
420 """
421 return Time(timestamp, format='unix')
424def astropyToEfdTimestamp(time):
425 """Get astropy Time object as an efd timestamp
427 Parameters
428 ----------
429 time : `astropy.time.Time`
430 The time as an astropy.time.Time object.
432 Returns
433 -------
434 timestamp : `float`
435 The timestamp, in UTC, in unix seconds.
436 """
438 return time.utc.unix
441def clipDataToEvent(df, event):
442 """Clip a padded dataframe to an event.
444 Parameters
445 ----------
446 df : `pd.DataFrame`
447 The dataframe to clip.
448 event : `lsst.summit.utils.efdUtils.TmaEvent`
449 The event to clip to.
451 Returns
452 -------
453 clipped : `pd.DataFrame`
454 The clipped dataframe.
455 """
456 mask = (df['private_efdStamp'] >= event.begin.value) & (df['private_efdStamp'] <= event.end.value)
457 clipped_df = df.loc[mask].copy()
458 return clipped_df
461def calcNextDay(dayObs):
462 """Given an integer dayObs, calculate the next integer dayObs.
464 Integers are used for dayObs, but dayObs values are therefore not
465 contiguous due to month/year ends etc, so this utility provides a robust
466 way to get the integer dayObs which follows the one specified.
468 Parameters
469 ----------
470 dayObs : `int`
471 The dayObs, as an integer, e.g. 20231231
473 Returns
474 -------
475 nextDayObs : `int`
476 The next dayObs, as an integer, e.g. 20240101
477 """
478 d1 = datetime.datetime.strptime(str(dayObs), '%Y%m%d')
479 oneDay = datetime.timedelta(days=1)
480 return int((d1 + oneDay).strftime('%Y%m%d'))
483def getDayObsStartTime(dayObs):
484 """Get the start of the given dayObs as an astropy.time.Time object.
486 The observatory rolls the date over at UTC-12.
488 Parameters
489 ----------
490 dayObs : `int`
491 The dayObs, as an integer, e.g. 20231225
493 Returns
494 -------
495 time : `astropy.time.Time`
496 The start of the dayObs as an astropy.time.Time object.
497 """
498 pythonDateTime = datetime.datetime.strptime(str(dayObs), "%Y%m%d")
499 return Time(pythonDateTime) + 12 * u.hour
502def getDayObsEndTime(dayObs):
503 """Get the end of the given dayObs as an astropy.time.Time object.
505 Parameters
506 ----------
507 dayObs : `int`
508 The dayObs, as an integer, e.g. 20231225
510 Returns
511 -------
512 time : `astropy.time.Time`
513 The end of the dayObs as an astropy.time.Time object.
514 """
515 return getDayObsStartTime(dayObs) + 24 * u.hour
518def getDayObsForTime(time):
519 """Get the dayObs in which an astropy.time.Time object falls.
521 Parameters
522 ----------
523 time : `astropy.time.Time`
524 The time.
526 Returns
527 -------
528 dayObs : `int`
529 The dayObs, as an integer, e.g. 20231225
530 """
531 twelveHours = datetime.timedelta(hours=-12)
532 offset = TimeDelta(twelveHours, format='datetime')
533 return int((time + offset).utc.isot[:10].replace('-', ''))
536@deprecated(
537 reason="getSubTopics() has been replaced by getTopics() and using wildcards. "
538 "Will be removed after w_2023_50.",
539 version="w_2023_40",
540 category=FutureWarning,
541)
542def getSubTopics(client, topic):
543 """Get all the sub topics within a given topic.
545 Note that the topic need not be a complete one, for example, rather than
546 doing `getSubTopics(client, 'lsst.sal.ATMCS')` to get all the topics for
547 the AuxTel Mount Control System, you can do `getSubTopics(client,
548 'lsst.sal.AT')` to get all which relate to the AuxTel in general.
550 Parameters
551 ----------
552 client : `lsst_efd_client.efd_helper.EfdClient`
553 The EFD client to use.
554 topic : `str`
555 The topic to query.
557 Returns
558 -------
559 subTopics : `list` of `str`
560 The sub topics.
561 """
562 loop = asyncio.get_event_loop()
563 topics = loop.run_until_complete(client.get_topics())
564 return sorted([t for t in topics if t.startswith(topic)])
567def getTopics(client, toFind, caseSensitive=False):
568 """Return all the strings in topics which match the topic query string.
570 Supports wildcards, which are denoted as `*``, as per shell globs.
572 Example:
573 >>> # assume topics are ['apple', 'banana', 'grape']
574 >>> getTopics(, 'a*p*')
575 ['apple', 'grape']
577 Parameters
578 ----------
579 client : `lsst_efd_client.efd_helper.EfdClient`
580 The EFD client to use.
581 toFind : `str`
582 The query string, with optional wildcards denoted as *.
583 caseSensitive : `bool`, optional
584 If ``True``, the query is case sensitive. Defaults to ``False``.
586 Returns
587 -------
588 matches : `list` of `str`
589 The list of matching topics.
590 """
591 loop = asyncio.get_event_loop()
592 topics = loop.run_until_complete(client.get_topics())
594 # Replace wildcard with regex equivalent
595 pattern = toFind.replace('*', '.*')
596 flags = re.IGNORECASE if not caseSensitive else 0
598 matches = []
599 for topic in topics:
600 if re.match(pattern, topic, flags):
601 matches.append(topic)
603 return matches