Coverage for python/lsst/summit/utils/efdUtils.py: 15%
137 statements
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-17 12:43 +0000
« prev ^ index » next coverage.py v7.3.0, created at 2023-08-17 12:43 +0000
1# This file is part of summit_utils.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import asyncio
23from astropy.time import Time, TimeDelta
24from astropy import units as u
25import datetime
26import logging
27import pandas as pd
29from .utils import getSite
31HAS_EFD_CLIENT = True
32try:
33 from lsst_efd_client import EfdClient
34except ImportError:
35 HAS_EFD_CLIENT = False
37__all__ = [
38 'getEfdData',
39 'getMostRecentRowWithDataBefore',
40 'makeEfdClient',
41 'expRecordToTimespan',
42 'efdTimestampToAstropy',
43 'astropyToEfdTimestamp',
44 'clipDataToEvent',
45 'calcNextDay',
46 'getDayObsStartTime',
47 'getDayObsEndTime',
48 'getDayObsForTime',
49 'getSubTopics',
50]
53COMMAND_ALIASES = {
54 'raDecTarget': 'lsst.sal.MTPtg.command_raDecTarget',
55 'moveToTarget': 'lsst.sal.MTMount.command_moveToTarget',
56 'startTracking': 'lsst.sal.MTMount.command_startTracking',
57 'stopTracking': 'lsst.sal.MTMount.command_stopTracking',
58 'trackTarget': 'lsst.sal.MTMount.command_trackTarget', # issued at 20Hz - don't plot
59}
61# When looking backwards in time to find the most recent state event, look back
62# in chunks of this size. Too small, and there will be too many queries, too
63# large and there will be too much data returned unnecessarily, as we only need
64# one row by definition. Will tune this parameters in consultation with SQuaRE.
65TIME_CHUNKING = datetime.timedelta(minutes=15)
68def _getBeginEnd(dayObs=None, begin=None, end=None, timespan=None, event=None, expRecord=None):
69 """Calculate the begin and end times to pass to _getEfdData, given the
70 kwargs passed to getEfdData.
72 Parameters
73 ----------
74 dayObs : `int`
75 The dayObs to query. If specified, this is used to determine the begin
76 and end times.
77 begin : `astropy.Time`
78 The begin time for the query. If specified, either an end time or a
79 timespan must be supplied.
80 end : `astropy.Time`
81 The end time for the query. If specified, a begin time must also be
82 supplied.
83 timespan : `astropy.TimeDelta`
84 The timespan for the query. If specified, a begin time must also be
85 supplied.
86 event : `lsst.summit.utils.efdUtils.TmaEvent`
87 The event to query. If specified, this is used to determine the begin
88 and end times, and all other options are disallowed.
89 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`
90 The exposure record containing the timespan to query. If specified, all
91 other options are disallowed.
93 Returns
94 -------
95 begin : `astropy.Time`
96 The begin time for the query.
97 end : `astropy.Time`
98 The end time for the query.
99 """
100 if expRecord is not None:
101 forbiddenOpts = [event, begin, end, timespan, dayObs]
102 if any(x is not None for x in forbiddenOpts):
103 raise ValueError("You can't specify both an expRecord and a begin/end or timespan or dayObs")
104 begin = expRecord.timespan.begin
105 end = expRecord.timespan.end
106 return begin, end
108 if event is not None:
109 forbiddenOpts = [begin, end, timespan, dayObs]
110 if any(x is not None for x in forbiddenOpts):
111 raise ValueError("You can't specify both an event and a begin/end or timespan or dayObs")
112 begin = event.begin
113 end = event.end
114 return begin, end
116 # check for dayObs, and that other options aren't inconsistently specified
117 if dayObs is not None:
118 forbiddenOpts = [begin, end, timespan]
119 if any(x is not None for x in forbiddenOpts):
120 raise ValueError("You can't specify both a dayObs and a begin/end or timespan")
121 begin = getDayObsStartTime(dayObs)
122 end = getDayObsEndTime(dayObs)
123 return begin, end
124 # can now disregard dayObs entirely
126 if begin is None:
127 raise ValueError("You must specify either a dayObs or a begin/end or begin/timespan")
128 # can now rely on begin, so just need to deal with end/timespan
130 if end is None and timespan is None:
131 raise ValueError("If you specify a begin, you must specify either a end or a timespan")
132 if end is not None and timespan is not None:
133 raise ValueError("You can't specify both a end and a timespan")
134 if end is None:
135 if timespan > datetime.timedelta(minutes=0):
136 end = begin + timespan # the normal case
137 else:
138 end = begin # the case where timespan is negative
139 begin = begin + timespan # adding the negative to the start, i.e. subtracting it to bring back
141 assert begin is not None
142 assert end is not None
143 return begin, end
146def getEfdData(client, topic, *,
147 columns=None,
148 prePadding=0,
149 postPadding=0,
150 dayObs=None,
151 begin=None,
152 end=None,
153 timespan=None,
154 event=None,
155 expRecord=None,
156 warn=True,
157 ):
158 """Get one or more EFD topics over a time range, synchronously.
160 The time range can be specified as either:
161 * a dayObs, in which case the full 24 hour period is used,
162 * a begin point and a end point,
163 * a begin point and a timespan.
164 * a mount event
165 * an exposure record
166 If it is desired to use an end time with a timespan, just specify it as the
167 begin time and use a negative timespan.
169 The results from all topics are merged into a single dataframe.
171 Parameters
172 ----------
173 client : `lsst_efd_client.efd_helper.EfdClient`
174 The EFD client to use.
175 topic : `str`
176 The topic to query.
177 columns : `list` of `str`, optional
178 The columns to query. If not specified, all columns are queried.
179 prePadding : `float`
180 The amount of time before the nominal start of the query to include, in
181 seconds.
182 postPadding : `float`
183 The amount of extra time after the nominal end of the query to include,
184 in seconds.
185 dayObs : `int`, optional
186 The dayObs to query. If specified, this is used to determine the begin
187 and end times.
188 begin : `astropy.Time`, optional
189 The begin time for the query. If specified, either a end time or a
190 timespan must be supplied.
191 end : `astropy.Time`, optional
192 The end time for the query. If specified, a begin time must also be
193 supplied.
194 timespan : `astropy.TimeDelta`, optional
195 The timespan for the query. If specified, a begin time must also be
196 supplied.
197 event : `lsst.summit.utils.efdUtils.TmaEvent`, optional
198 The event to query. If specified, this is used to determine the begin
199 and end times, and all other options are disallowed.
200 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`, optional
201 The exposure record containing the timespan to query. If specified, all
202 other options are disallowed.
203 warn : bool, optional
204 If ``True``, warn when no data is found. Exists so that utility code
205 can disable warnings when checking for data, and therefore defaults to
206 ``True``.
208 Returns
209 -------
210 data : `pd.DataFrame`
211 The merged data from all topics.
213 Raises
214 ------
215 ValueError:
216 If the topics are not in the EFD schema.
217 ValueError:
218 If both a dayObs and a begin/end or timespan are specified.
219 ValueError:
220 If a begin time is specified but no end time or timespan.
222 """
223 # TODO: DM-40100 ideally should calls mpts as necessary so that users
224 # needn't care if things are packed
226 # supports aliases so that you can query with them. If there is no entry in
227 # the alias dict then it queries with the supplied key. The fact the schema
228 # is now being checked means this shouldn't be a problem now.
230 # TODO: RFC-948 Move this import back to top of file once is implemented.
231 import nest_asyncio
233 begin, end = _getBeginEnd(dayObs, begin, end, timespan, event, expRecord)
234 begin -= TimeDelta(prePadding, format='sec')
235 end += TimeDelta(postPadding, format='sec')
237 nest_asyncio.apply()
238 loop = asyncio.get_event_loop()
239 ret = loop.run_until_complete(_getEfdData(client=client,
240 topic=topic,
241 begin=begin,
242 end=end,
243 columns=columns))
244 if ret.empty and warn:
245 log = logging.getLogger(__name__)
246 log.warning(f"Topic {topic} is in the schema, but no data was returned by the query for the specified"
247 " time range")
248 return ret
251async def _getEfdData(client, topic, begin, end, columns=None):
252 """Get data for a topic from the EFD over the specified time range.
254 Parameters
255 ----------
256 client : `lsst_efd_client.efd_helper.EfdClient`
257 The EFD client to use.
258 topic : `str`
259 The topic to query.
260 begin : `astropy.Time`
261 The begin time for the query.
262 end : `astropy.Time`
263 The end time for the query.
264 columns : `list` of `str`, optional
265 The columns to query. If not specified, all columns are returned.
267 Returns
268 -------
269 data : `pd.DataFrame`
270 The data from the query.
271 """
272 if columns is None:
273 columns = ['*']
275 availableTopics = await client.get_topics()
277 if topic not in availableTopics:
278 raise ValueError(f"Topic {topic} not in EFD schema")
280 data = await client.select_time_series(topic, columns, begin.utc, end.utc)
282 return data
285def getMostRecentRowWithDataBefore(client, topic, timeToLookBefore, warnStaleAfterNMinutes=60*12):
286 """Get the most recent row of data for a topic before a given time.
288 Parameters
289 ----------
290 client : `lsst_efd_client.efd_helper.EfdClient`
291 The EFD client to use.
292 topic : `str`
293 The topic to query.
294 timeToLookBefore : `astropy.Time`
295 The time to look before.
296 warnStaleAfterNMinutes : `float`, optional
297 The number of minutes after which to consider the data stale and issue
298 a warning.
300 Returns
301 -------
302 row : `pd.Series`
303 The row of data from the EFD containing the most recent data before the
304 specified time.
306 Raises
307 ------
308 ValueError:
309 If the topic is not in the EFD schema.
310 """
311 staleAge = datetime.timedelta(warnStaleAfterNMinutes)
313 firstDayPossible = getDayObsStartTime(20190101)
315 if timeToLookBefore < firstDayPossible:
316 raise ValueError(f"Requested time {timeToLookBefore} is before any data was put in the EFD")
318 df = pd.DataFrame()
319 beginTime = timeToLookBefore
320 while df.empty and beginTime > firstDayPossible:
321 df = getEfdData(client, topic, begin=beginTime, timespan=-TIME_CHUNKING, warn=False)
322 beginTime -= TIME_CHUNKING
324 if beginTime < firstDayPossible and df.empty: # we ran all the way back to the beginning of time
325 raise ValueError(f"The entire EFD was searched backwards from {timeToLookBefore} and no data was "
326 f"found in {topic=}")
328 lastRow = df.iloc[-1]
329 commandTime = efdTimestampToAstropy(lastRow['private_efdStamp'])
331 commandAge = timeToLookBefore - commandTime
332 if commandAge > staleAge:
333 log = logging.getLogger(__name__)
334 log.warning(f"Component {topic} was last set {commandAge.sec/60:.1} minutes"
335 " before the requested time")
337 return lastRow
340def makeEfdClient(testing=False):
341 """Automatically create an EFD client based on the site.
343 Parameters
344 ----------
345 testing : `bool`, optional
346 Set to ``True`` if running in a test suite. This will default to using
347 the USDF EFD, for which data has been recorded for replay by the ``vcr`
348 package. Note data must be re-recorded to ``vcr`` from both inside and
349 outside the USDF when the package/data changes, due to the use of a
350 proxy meaning that the web requests are different depending on whether
351 the EFD is being contacted from inside and outside the USDF.
353 Returns
354 -------
355 efdClient : `lsst_efd_client.efd_helper.EfdClient`
356 The EFD client to use for the current site.
357 """
358 if not HAS_EFD_CLIENT:
359 raise RuntimeError("Could not create EFD client because importing lsst_efd_client failed.")
361 if testing:
362 return EfdClient('usdf_efd')
364 try:
365 site = getSite()
366 except ValueError as e:
367 raise RuntimeError("Could not create EFD client as the site could not be determined") from e
369 if site == 'summit':
370 return EfdClient('summit_efd')
371 if site == 'tucson':
372 return EfdClient('tucson_teststand_efd')
373 if site == 'base':
374 return EfdClient('base_efd')
375 if site in ['staff-rsp', 'rubin-devl']:
376 return EfdClient('usdf_efd')
377 if site == 'jenkins':
378 return EfdClient('usdf_efd')
380 raise RuntimeError(f"Could not create EFD client as the {site=} is not recognized")
383def expRecordToTimespan(expRecord):
384 """Get the timespan from an exposure record.
386 Returns the timespan in a format where it can be used to directly unpack
387 into a efdClient.select_time_series() call.
389 Parameters
390 ----------
391 expRecord : `lsst.daf.butler.dimensions.ExposureRecord`
392 The exposure record.
394 Returns
395 -------
396 timespanDict : `dict`
397 The timespan in a format that can be used to directly unpack into a
398 efdClient.select_time_series() call.
399 """
400 return {'begin': expRecord.timespan.begin.utc,
401 'end': expRecord.timespan.end.utc,
402 }
405def efdTimestampToAstropy(timestamp):
406 """Get an efd timestamp as an astropy.time.Time object.
408 Parameters
409 ----------
410 timestamp : `float`
411 The timestamp, as a float.
413 Returns
414 -------
415 time : `astropy.time.Time`
416 The timestamp as an astropy.time.Time object.
417 """
418 return Time(timestamp, format='unix')
421def astropyToEfdTimestamp(time):
422 """Get astropy Time object as an efd timestamp
424 Parameters
425 ----------
426 time : `astropy.time.Time`
427 The time as an astropy.time.Time object.
429 Returns
430 -------
431 timestamp : `float`
432 The timestamp, in UTC, in unix seconds.
433 """
435 return time.utc.unix
438def clipDataToEvent(df, event):
439 """Clip a padded dataframe to an event.
441 Parameters
442 ----------
443 df : `pd.DataFrame`
444 The dataframe to clip.
445 event : `lsst.summit.utils.efdUtils.TmaEvent`
446 The event to clip to.
448 Returns
449 -------
450 clipped : `pd.DataFrame`
451 The clipped dataframe.
452 """
453 mask = (df['private_efdStamp'] >= event.begin.value) & (df['private_efdStamp'] <= event.end.value)
454 clipped_df = df.loc[mask].copy()
455 return clipped_df
458def calcNextDay(dayObs):
459 """Given an integer dayObs, calculate the next integer dayObs.
461 Integers are used for dayObs, but dayObs values are therefore not
462 contiguous due to month/year ends etc, so this utility provides a robust
463 way to get the integer dayObs which follows the one specified.
465 Parameters
466 ----------
467 dayObs : `int`
468 The dayObs, as an integer, e.g. 20231231
470 Returns
471 -------
472 nextDayObs : `int`
473 The next dayObs, as an integer, e.g. 20240101
474 """
475 d1 = datetime.datetime.strptime(str(dayObs), '%Y%m%d')
476 oneDay = datetime.timedelta(days=1)
477 return int((d1 + oneDay).strftime('%Y%m%d'))
480def getDayObsStartTime(dayObs):
481 """Get the start of the given dayObs as an astropy.time.Time object.
483 The observatory rolls the date over at UTC-12.
485 Parameters
486 ----------
487 dayObs : `int`
488 The dayObs, as an integer, e.g. 20231225
490 Returns
491 -------
492 time : `astropy.time.Time`
493 The start of the dayObs as an astropy.time.Time object.
494 """
495 pythonDateTime = datetime.datetime.strptime(str(dayObs), "%Y%m%d")
496 return Time(pythonDateTime) + 12 * u.hour
499def getDayObsEndTime(dayObs):
500 """Get the end of the given dayObs as an astropy.time.Time object.
502 Parameters
503 ----------
504 dayObs : `int`
505 The dayObs, as an integer, e.g. 20231225
507 Returns
508 -------
509 time : `astropy.time.Time`
510 The end of the dayObs as an astropy.time.Time object.
511 """
512 return getDayObsStartTime(dayObs) + 24 * u.hour
515def getDayObsForTime(time):
516 """Get the dayObs in which an astropy.time.Time object falls.
518 Parameters
519 ----------
520 time : `astropy.time.Time`
521 The time.
523 Returns
524 -------
525 dayObs : `int`
526 The dayObs, as an integer, e.g. 20231225
527 """
528 twelveHours = datetime.timedelta(hours=-12)
529 offset = TimeDelta(twelveHours, format='datetime')
530 return int((time + offset).utc.isot[:10].replace('-', ''))
533def getSubTopics(client, topic):
534 """Get all the sub topics within a given topic.
536 Note that the topic need not be a complete one, for example, rather than
537 doing `getSubTopics(client, 'lsst.sal.ATMCS')` to get all the topics for
538 the AuxTel Mount Control System, you can do `getSubTopics(client,
539 'lsst.sal.AT')` to get all which relate to the AuxTel in general.
541 Parameters
542 ----------
543 client : `lsst_efd_client.efd_helper.EfdClient`
544 The EFD client to use.
545 topic : `str`
546 The topic to query.
548 Returns
549 -------
550 subTopics : `list` of `str`
551 The sub topics.
552 """
553 loop = asyncio.get_event_loop()
554 topics = loop.run_until_complete(client.get_topics())
555 return sorted([t for t in topics if t.startswith(topic)])