Coverage for python/lsst/summit/utils/efdUtils.py: 16%
160 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-06 14:08 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-06 14:08 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import asyncio
23from astropy.time import Time, TimeDelta
24from astropy import units as u
25import datetime
26import logging
27import pandas as pd
28import re
29from deprecated.sphinx import deprecated
31from lsst.utils.iteration import ensure_iterable
33from .utils import getSite
35HAS_EFD_CLIENT = True
36try:
37 from lsst_efd_client import EfdClient
38except ImportError:
39 HAS_EFD_CLIENT = False
41__all__ = [
42 'getEfdData',
43 'getMostRecentRowWithDataBefore',
44 'makeEfdClient',
45 'expRecordToTimespan',
46 'efdTimestampToAstropy',
47 'astropyToEfdTimestamp',
48 'clipDataToEvent',
49 'calcNextDay',
50 'getDayObsStartTime',
51 'getDayObsEndTime',
52 'getDayObsForTime',
53 'getSubTopics', # deprecated, being removed in w_2023_50
54 'getTopics',
55]
58COMMAND_ALIASES = {
59 'raDecTarget': 'lsst.sal.MTPtg.command_raDecTarget',
60 'moveToTarget': 'lsst.sal.MTMount.command_moveToTarget',
61 'startTracking': 'lsst.sal.MTMount.command_startTracking',
62 'stopTracking': 'lsst.sal.MTMount.command_stopTracking',
63 'trackTarget': 'lsst.sal.MTMount.command_trackTarget', # issued at 20Hz - don't plot
64}
66# When looking backwards in time to find the most recent state event, look back
67# in chunks of this size. Too small, and there will be too many queries, too
68# large and there will be too much data returned unnecessarily, as we only need
69# one row by definition. Will tune this parameters in consultation with SQuaRE.
70TIME_CHUNKING = datetime.timedelta(minutes=15)
73def _getBeginEnd(dayObs=None, begin=None, end=None, timespan=None, event=None, expRecord=None):
74 """Calculate the begin and end times to pass to _getEfdData, given the
75 kwargs passed to getEfdData.
77 Parameters
78 ----------
79 dayObs : `int`
80 The dayObs to query. If specified, this is used to determine the begin
81 and end times.
82 begin : `astropy.Time`
83 The begin time for the query. If specified, either an end time or a
84 timespan must be supplied.
85 end : `astropy.Time`
86 The end time for the query. If specified, a begin time must also be
87 supplied.
88 timespan : `astropy.TimeDelta`
89 The timespan for the query. If specified, a begin time must also be
90 supplied.
91 event : `lsst.summit.utils.efdUtils.TmaEvent`
92 The event to query. If specified, this is used to determine the begin
93 and end times, and all other options are disallowed.
94 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`
95 The exposure record containing the timespan to query. If specified, all
96 other options are disallowed.
98 Returns
99 -------
100 begin : `astropy.Time`
101 The begin time for the query.
102 end : `astropy.Time`
103 The end time for the query.
104 """
105 if expRecord is not None:
106 forbiddenOpts = [event, begin, end, timespan, dayObs]
107 if any(x is not None for x in forbiddenOpts):
108 raise ValueError("You can't specify both an expRecord and a begin/end or timespan or dayObs")
109 begin = expRecord.timespan.begin
110 end = expRecord.timespan.end
111 return begin, end
113 if event is not None:
114 forbiddenOpts = [begin, end, timespan, dayObs]
115 if any(x is not None for x in forbiddenOpts):
116 raise ValueError("You can't specify both an event and a begin/end or timespan or dayObs")
117 begin = event.begin
118 end = event.end
119 return begin, end
121 # check for dayObs, and that other options aren't inconsistently specified
122 if dayObs is not None:
123 forbiddenOpts = [begin, end, timespan]
124 if any(x is not None for x in forbiddenOpts):
125 raise ValueError("You can't specify both a dayObs and a begin/end or timespan")
126 begin = getDayObsStartTime(dayObs)
127 end = getDayObsEndTime(dayObs)
128 return begin, end
129 # can now disregard dayObs entirely
131 if begin is None:
132 raise ValueError("You must specify either a dayObs or a begin/end or begin/timespan")
133 # can now rely on begin, so just need to deal with end/timespan
135 if end is None and timespan is None:
136 raise ValueError("If you specify a begin, you must specify either a end or a timespan")
137 if end is not None and timespan is not None:
138 raise ValueError("You can't specify both a end and a timespan")
139 if end is None:
140 if timespan > datetime.timedelta(minutes=0):
141 end = begin + timespan # the normal case
142 else:
143 end = begin # the case where timespan is negative
144 begin = begin + timespan # adding the negative to the start, i.e. subtracting it to bring back
146 assert begin is not None
147 assert end is not None
148 return begin, end
151def getEfdData(client, topic, *,
152 columns=None,
153 prePadding=0,
154 postPadding=0,
155 dayObs=None,
156 begin=None,
157 end=None,
158 timespan=None,
159 event=None,
160 expRecord=None,
161 warn=True,
162 ):
163 """Get one or more EFD topics over a time range, synchronously.
165 The time range can be specified as either:
166 * a dayObs, in which case the full 24 hour period is used,
167 * a begin point and a end point,
168 * a begin point and a timespan.
169 * a mount event
170 * an exposure record
171 If it is desired to use an end time with a timespan, just specify it as the
172 begin time and use a negative timespan.
174 The results from all topics are merged into a single dataframe.
176 Parameters
177 ----------
178 client : `lsst_efd_client.efd_helper.EfdClient`
179 The EFD client to use.
180 topic : `str`
181 The topic to query.
182 columns : `list` of `str`, optional
183 The columns to query. If not specified, all columns are queried.
184 prePadding : `float`
185 The amount of time before the nominal start of the query to include, in
186 seconds.
187 postPadding : `float`
188 The amount of extra time after the nominal end of the query to include,
189 in seconds.
190 dayObs : `int`, optional
191 The dayObs to query. If specified, this is used to determine the begin
192 and end times.
193 begin : `astropy.Time`, optional
194 The begin time for the query. If specified, either a end time or a
195 timespan must be supplied.
196 end : `astropy.Time`, optional
197 The end time for the query. If specified, a begin time must also be
198 supplied.
199 timespan : `astropy.TimeDelta`, optional
200 The timespan for the query. If specified, a begin time must also be
201 supplied.
202 event : `lsst.summit.utils.efdUtils.TmaEvent`, optional
203 The event to query. If specified, this is used to determine the begin
204 and end times, and all other options are disallowed.
205 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`, optional
206 The exposure record containing the timespan to query. If specified, all
207 other options are disallowed.
208 warn : bool, optional
209 If ``True``, warn when no data is found. Exists so that utility code
210 can disable warnings when checking for data, and therefore defaults to
211 ``True``.
213 Returns
214 -------
215 data : `pd.DataFrame`
216 The merged data from all topics.
218 Raises
219 ------
220 ValueError:
221 If the topics are not in the EFD schema.
222 ValueError:
223 If both a dayObs and a begin/end or timespan are specified.
224 ValueError:
225 If a begin time is specified but no end time or timespan.
227 """
228 # TODO: DM-40100 ideally should calls mpts as necessary so that users
229 # needn't care if things are packed
231 # supports aliases so that you can query with them. If there is no entry in
232 # the alias dict then it queries with the supplied key. The fact the schema
233 # is now being checked means this shouldn't be a problem now.
235 # TODO: RFC-948 Move this import back to top of file once is implemented.
236 import nest_asyncio
238 begin, end = _getBeginEnd(dayObs, begin, end, timespan, event, expRecord)
239 begin -= TimeDelta(prePadding, format='sec')
240 end += TimeDelta(postPadding, format='sec')
242 nest_asyncio.apply()
243 loop = asyncio.get_event_loop()
244 ret = loop.run_until_complete(_getEfdData(client=client,
245 topic=topic,
246 begin=begin,
247 end=end,
248 columns=columns))
249 if ret.empty and warn:
250 log = logging.getLogger(__name__)
251 log.warning(f"Topic {topic} is in the schema, but no data was returned by the query for the specified"
252 " time range")
253 return ret
256async def _getEfdData(client, topic, begin, end, columns=None):
257 """Get data for a topic from the EFD over the specified time range.
259 Parameters
260 ----------
261 client : `lsst_efd_client.efd_helper.EfdClient`
262 The EFD client to use.
263 topic : `str`
264 The topic to query.
265 begin : `astropy.Time`
266 The begin time for the query.
267 end : `astropy.Time`
268 The end time for the query.
269 columns : `list` of `str`, optional
270 The columns to query. If not specified, all columns are returned.
272 Returns
273 -------
274 data : `pd.DataFrame`
275 The data from the query.
276 """
277 if columns is None:
278 columns = ['*']
279 columns = list(ensure_iterable(columns))
281 availableTopics = await client.get_topics()
283 if topic not in availableTopics:
284 raise ValueError(f"Topic {topic} not in EFD schema")
286 data = await client.select_time_series(topic, columns, begin.utc, end.utc)
288 return data
291def getMostRecentRowWithDataBefore(client, topic, timeToLookBefore, warnStaleAfterNMinutes=60*12):
292 """Get the most recent row of data for a topic before a given time.
294 Parameters
295 ----------
296 client : `lsst_efd_client.efd_helper.EfdClient`
297 The EFD client to use.
298 topic : `str`
299 The topic to query.
300 timeToLookBefore : `astropy.Time`
301 The time to look before.
302 warnStaleAfterNMinutes : `float`, optional
303 The number of minutes after which to consider the data stale and issue
304 a warning.
306 Returns
307 -------
308 row : `pd.Series`
309 The row of data from the EFD containing the most recent data before the
310 specified time.
312 Raises
313 ------
314 ValueError:
315 If the topic is not in the EFD schema.
316 """
317 staleAge = datetime.timedelta(warnStaleAfterNMinutes)
319 firstDayPossible = getDayObsStartTime(20190101)
321 if timeToLookBefore < firstDayPossible:
322 raise ValueError(f"Requested time {timeToLookBefore} is before any data was put in the EFD")
324 df = pd.DataFrame()
325 beginTime = timeToLookBefore
326 while df.empty and beginTime > firstDayPossible:
327 df = getEfdData(client, topic, begin=beginTime, timespan=-TIME_CHUNKING, warn=False)
328 beginTime -= TIME_CHUNKING
330 if beginTime < firstDayPossible and df.empty: # we ran all the way back to the beginning of time
331 raise ValueError(f"The entire EFD was searched backwards from {timeToLookBefore} and no data was "
332 f"found in {topic=}")
334 lastRow = df.iloc[-1]
335 commandTime = efdTimestampToAstropy(lastRow['private_efdStamp'])
337 commandAge = timeToLookBefore - commandTime
338 if commandAge > staleAge:
339 log = logging.getLogger(__name__)
340 log.warning(f"Component {topic} was last set {commandAge.sec/60:.1} minutes"
341 " before the requested time")
343 return lastRow
346def makeEfdClient(testing=False):
347 """Automatically create an EFD client based on the site.
349 Parameters
350 ----------
351 testing : `bool`, optional
352 Set to ``True`` if running in a test suite. This will default to using
353 the USDF EFD, for which data has been recorded for replay by the ``vcr`
354 package. Note data must be re-recorded to ``vcr`` from both inside and
355 outside the USDF when the package/data changes, due to the use of a
356 proxy meaning that the web requests are different depending on whether
357 the EFD is being contacted from inside and outside the USDF.
359 Returns
360 -------
361 efdClient : `lsst_efd_client.efd_helper.EfdClient`
362 The EFD client to use for the current site.
363 """
364 if not HAS_EFD_CLIENT:
365 raise RuntimeError("Could not create EFD client because importing lsst_efd_client failed.")
367 if testing:
368 return EfdClient('usdf_efd')
370 try:
371 site = getSite()
372 except ValueError as e:
373 raise RuntimeError("Could not create EFD client as the site could not be determined") from e
375 if site == 'summit':
376 return EfdClient('summit_efd')
377 if site == 'tucson':
378 return EfdClient('tucson_teststand_efd')
379 if site == 'base':
380 return EfdClient('base_efd')
381 if site in ['staff-rsp', 'rubin-devl', 'usdf-k8s']:
382 return EfdClient('usdf_efd')
383 if site == 'jenkins':
384 return EfdClient('usdf_efd')
386 raise RuntimeError(f"Could not create EFD client as the {site=} is not recognized")
389def expRecordToTimespan(expRecord):
390 """Get the timespan from an exposure record.
392 Returns the timespan in a format where it can be used to directly unpack
393 into a efdClient.select_time_series() call.
395 Parameters
396 ----------
397 expRecord : `lsst.daf.butler.dimensions.ExposureRecord`
398 The exposure record.
400 Returns
401 -------
402 timespanDict : `dict`
403 The timespan in a format that can be used to directly unpack into a
404 efdClient.select_time_series() call.
405 """
406 return {'begin': expRecord.timespan.begin.utc,
407 'end': expRecord.timespan.end.utc,
408 }
411def efdTimestampToAstropy(timestamp):
412 """Get an efd timestamp as an astropy.time.Time object.
414 Parameters
415 ----------
416 timestamp : `float`
417 The timestamp, as a float.
419 Returns
420 -------
421 time : `astropy.time.Time`
422 The timestamp as an astropy.time.Time object.
423 """
424 return Time(timestamp, format='unix')
427def astropyToEfdTimestamp(time):
428 """Get astropy Time object as an efd timestamp
430 Parameters
431 ----------
432 time : `astropy.time.Time`
433 The time as an astropy.time.Time object.
435 Returns
436 -------
437 timestamp : `float`
438 The timestamp, in UTC, in unix seconds.
439 """
441 return time.utc.unix
444def clipDataToEvent(df, event, prePadding=0, postPadding=0, logger=None):
445 """Clip a padded dataframe to an event.
447 Parameters
448 ----------
449 df : `pd.DataFrame`
450 The dataframe to clip.
451 event : `lsst.summit.utils.efdUtils.TmaEvent`
452 The event to clip to.
453 prePadding : `float`, optional
454 The amount of time before the nominal start of the event to include, in
455 seconds.
456 postPadding : `float`, optional
457 The amount of extra time after the nominal end of the event to include,
458 in seconds.
459 logger : `logging.Logger`, optional
460 The logger to use. If not specified, a new one is created.
462 Returns
463 -------
464 clipped : `pd.DataFrame`
465 The clipped dataframe.
466 """
467 begin = event.begin.value - prePadding
468 end = event.end.value + postPadding
470 if logger is None:
471 logger = logging.getLogger(__name__)
473 if begin < df['private_efdStamp'].min():
474 logger.warning(f"Requested begin time {begin} is before the start of the data")
475 if end > df['private_efdStamp'].max():
476 logger.warning(f"Requested end time {end} is after the end of the data")
478 mask = (df['private_efdStamp'] >= begin) & (df['private_efdStamp'] <= end)
479 clipped_df = df.loc[mask].copy()
480 return clipped_df
483def calcNextDay(dayObs):
484 """Given an integer dayObs, calculate the next integer dayObs.
486 Integers are used for dayObs, but dayObs values are therefore not
487 contiguous due to month/year ends etc, so this utility provides a robust
488 way to get the integer dayObs which follows the one specified.
490 Parameters
491 ----------
492 dayObs : `int`
493 The dayObs, as an integer, e.g. 20231231
495 Returns
496 -------
497 nextDayObs : `int`
498 The next dayObs, as an integer, e.g. 20240101
499 """
500 d1 = datetime.datetime.strptime(str(dayObs), '%Y%m%d')
501 oneDay = datetime.timedelta(days=1)
502 return int((d1 + oneDay).strftime('%Y%m%d'))
505def getDayObsStartTime(dayObs):
506 """Get the start of the given dayObs as an astropy.time.Time object.
508 The observatory rolls the date over at UTC-12.
510 Parameters
511 ----------
512 dayObs : `int`
513 The dayObs, as an integer, e.g. 20231225
515 Returns
516 -------
517 time : `astropy.time.Time`
518 The start of the dayObs as an astropy.time.Time object.
519 """
520 pythonDateTime = datetime.datetime.strptime(str(dayObs), "%Y%m%d")
521 return Time(pythonDateTime) + 12 * u.hour
524def getDayObsEndTime(dayObs):
525 """Get the end of the given dayObs as an astropy.time.Time object.
527 Parameters
528 ----------
529 dayObs : `int`
530 The dayObs, as an integer, e.g. 20231225
532 Returns
533 -------
534 time : `astropy.time.Time`
535 The end of the dayObs as an astropy.time.Time object.
536 """
537 return getDayObsStartTime(dayObs) + 24 * u.hour
540def getDayObsForTime(time):
541 """Get the dayObs in which an astropy.time.Time object falls.
543 Parameters
544 ----------
545 time : `astropy.time.Time`
546 The time.
548 Returns
549 -------
550 dayObs : `int`
551 The dayObs, as an integer, e.g. 20231225
552 """
553 twelveHours = datetime.timedelta(hours=-12)
554 offset = TimeDelta(twelveHours, format='datetime')
555 return int((time + offset).utc.isot[:10].replace('-', ''))
558@deprecated(
559 reason="getSubTopics() has been replaced by getTopics() and using wildcards. "
560 "Will be removed after w_2023_50.",
561 version="w_2023_40",
562 category=FutureWarning,
563)
564def getSubTopics(client, topic):
565 """Get all the sub topics within a given topic.
567 Note that the topic need not be a complete one, for example, rather than
568 doing `getSubTopics(client, 'lsst.sal.ATMCS')` to get all the topics for
569 the AuxTel Mount Control System, you can do `getSubTopics(client,
570 'lsst.sal.AT')` to get all which relate to the AuxTel in general.
572 Parameters
573 ----------
574 client : `lsst_efd_client.efd_helper.EfdClient`
575 The EFD client to use.
576 topic : `str`
577 The topic to query.
579 Returns
580 -------
581 subTopics : `list` of `str`
582 The sub topics.
583 """
584 loop = asyncio.get_event_loop()
585 topics = loop.run_until_complete(client.get_topics())
586 return sorted([t for t in topics if t.startswith(topic)])
589def getTopics(client, toFind, caseSensitive=False):
590 """Return all the strings in topics which match the topic query string.
592 Supports wildcards, which are denoted as `*``, as per shell globs.
594 Example:
595 >>> # assume topics are ['apple', 'banana', 'grape']
596 >>> getTopics(, 'a*p*')
597 ['apple', 'grape']
599 Parameters
600 ----------
601 client : `lsst_efd_client.efd_helper.EfdClient`
602 The EFD client to use.
603 toFind : `str`
604 The query string, with optional wildcards denoted as *.
605 caseSensitive : `bool`, optional
606 If ``True``, the query is case sensitive. Defaults to ``False``.
608 Returns
609 -------
610 matches : `list` of `str`
611 The list of matching topics.
612 """
613 loop = asyncio.get_event_loop()
614 topics = loop.run_until_complete(client.get_topics())
616 # Replace wildcard with regex equivalent
617 pattern = toFind.replace('*', '.*')
618 flags = re.IGNORECASE if not caseSensitive else 0
620 matches = []
621 for topic in topics:
622 if re.match(pattern, topic, flags):
623 matches.append(topic)
625 return matches