Coverage for python/lsst/summit/extras/nightReport.py: 15%
236 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-15 02:51 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-15 02:51 +0000
1# This file is part of summit_extras.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from dataclasses import dataclass
23import logging
24import os
25import pickle
27import numpy as np
28import matplotlib.pyplot as plt
29from matplotlib.pyplot import cm
31from lsst.obs.lsst.translators.lsst import FILTER_DELIMITER
32from astro_metadata_translator import ObservationInfo
33from lsst.summit.utils.butlerUtils import makeDefaultLatissButler, getSeqNumsForDayObs, sanitize_day_obs
35__all__ = ['NightReporter', 'saveReport', 'loadReport']
37CALIB_VALUES = ['FlatField position', 'Park position', 'azel_target']
38N_STARS_PER_SYMBOL = 6
39MARKER_SEQUENCE = ['*', 'o', "D", 'P', 'v', "^", 's', '.', ',', 'o', 'v', '^',
40 '<', '>', '1', '2', '3', '4', '8', 's', 'p', 'P', '*', 'h',
41 'H', '+', 'x', 'X', 'D', 'd', '|', '_']
42SOUTHPOLESTAR = 'HD 185975'
44PICKLE_TEMPLATE = "%s.pickle"
46KEY_MAPPER = {'OBJECT': 'object',
47 'EXPTIME': 'exposure_time',
48 'IMGTYPE': 'observation_type',
49 'MJD-BEG': 'datetime_begin',
50 }
52# TODO: DM-34250 rewrite (and document) this whole file.
55def getValue(key, header, stripUnits=True):
56 """Get a header value the Right Way.
58 If it is available from the ObservationInfo, use that, either directly or
59 via the KEY_MAPPER dict.
60 If not, try to get it from the header.
61 If it is not in the header, return None.
62 """
63 if key in KEY_MAPPER:
64 key = KEY_MAPPER[key]
66 if hasattr(header['ObservationInfo'], key):
67 val = getattr(header['ObservationInfo'], key)
68 if hasattr(val, 'value') and stripUnits:
69 return val.value
70 else:
71 return val
73 return header.get(key, None)
76# wanted these to be on the class but it doesn't pickle itself nicely
77def saveReport(reporter, savePath):
78 # the reporter.butler seems to pickle OK but perhaps it should be
79 # removed for saving?
80 filename = os.path.join(savePath, PICKLE_TEMPLATE % reporter.dayObs)
81 with open(filename, "wb") as dumpFile:
82 pickle.dump(reporter, dumpFile)
85def loadReport(loadPath, dayObs):
86 filename = os.path.join(loadPath, PICKLE_TEMPLATE % dayObs)
87 if not os.path.exists(filename):
88 raise FileNotFoundError(f"{filename} not found")
89 with open(filename, "rb") as input_file:
90 reporter = pickle.load(input_file)
91 return reporter
94@dataclass
95class ColorAndMarker:
96 '''Class for holding colors and marker symbols'''
97 color: list
98 marker: str = '*'
101class NightReporter():
103 def __init__(self, dayObs, deferLoadingData=False):
104 self._supressAstroMetadataTranslatorWarnings() # call early
106 self.butler = makeDefaultLatissButler()
107 if isinstance(dayObs, str):
108 dayObs = sanitize_day_obs(dayObs)
109 print('Converted string-format dayObs to integer for Gen3')
110 self.dayObs = dayObs
111 self.data = {}
112 self.stars = None
113 self.cMap = None
114 if not deferLoadingData:
115 self.rebuild()
117 def _supressAstroMetadataTranslatorWarnings(self):
118 """NB: must be called early"""
119 logging.basicConfig()
120 _astroLogger = logging.getLogger("lsst.obs.lsst.translators.latiss")
121 _astroLogger.setLevel(logging.ERROR)
123 def rebuild(self, dayObs=None):
124 """Reload new observations, or load a different night"""
125 dayToUse = self.dayObs
126 if dayObs:
127 # new day, so blow away old data
128 # as scraping skips seqNums we've loaded!
129 if dayObs != self.dayObs:
130 self.data = {}
131 self.dayObs = dayObs
132 dayToUse = dayObs
133 self._scrapeData(dayToUse)
134 self.stars = self.getObservedObjects()
135 self.cMap = self.makeStarColorAndMarkerMap(self.stars)
137 def _scrapeData(self, dayObs):
138 """Load data into self.data skipping as necessary. Don't call directly!
140 Don't call directly as the rebuild() function zeros out data for when
141 it's a new dayObs."""
142 seqNums = getSeqNumsForDayObs(self.butler, dayObs)
143 for seqNum in sorted(seqNums):
144 if seqNum in self.data.keys():
145 continue
146 dataId = {'day_obs': dayObs, 'seq_num': seqNum, 'detector': 0}
147 md = self.butler.get('raw.metadata', dataId)
148 self.data[seqNum] = md.toDict()
149 self.data[seqNum]['ObservationInfo'] = ObservationInfo(md)
150 print(f"Loaded data for seqNums {sorted(seqNums)[0]} to {sorted(seqNums)[-1]}")
152 def getUniqueValuesForKey(self, key, ignoreCalibs=True):
153 values = []
154 for seqNum in self.data.keys():
155 v = getValue(key, self.data[seqNum])
156 if ignoreCalibs is True and v in CALIB_VALUES:
157 continue
158 values.append(v)
159 return list(set(values))
161 def _makePolarPlot(self, azimuthsInDegrees, zenithAngles, marker="*-",
162 title=None, makeFig=True, color=None, objName=None):
163 if makeFig:
164 _ = plt.figure(figsize=(10, 10))
165 ax = plt.subplot(111, polar=True)
166 ax.plot([a*np.pi/180 for a in azimuthsInDegrees], zenithAngles, marker, c=color, label=objName)
167 if title:
168 ax.set_title(title, va='bottom')
169 ax.set_theta_zero_location("N")
170 ax.set_theta_direction(-1)
171 ax.set_rlim(0, 90)
172 return ax
174 def makePolarPlotForObjects(self, objects=None, withLines=True):
175 if not objects:
176 objects = self.stars
177 objects = self._safeListArg(objects)
179 _ = plt.figure(figsize=(10, 10))
181 for i, obj in enumerate(objects):
182 azs = self.getAllValuesForKVPair('AZSTART', ("OBJECT", obj))
183 els = self.getAllValuesForKVPair('ELSTART', ("OBJECT", obj))
184 assert(len(azs) == len(els))
185 if len(azs) == 0:
186 print(f"WARNING: found no alt/az data for {obj}")
187 zens = [90 - el for el in els]
188 color = self.cMap[obj].color
189 marker = self.cMap[obj].marker
190 if withLines:
191 marker += '-'
193 ax = self._makePolarPlot(azs, zens, marker=marker, title=None, makeFig=False,
194 color=color, objName=obj)
195 lgnd = ax.legend(bbox_to_anchor=(1.05, 1), prop={'size': 15}, loc='upper left')
196 for h in lgnd.legendHandles:
197 size = 14
198 if '-' in marker:
199 size += 5
200 h.set_markersize(size)
202 def getAllValuesForKVPair(self, keyToGet, keyValPairAsTuple, uniqueOnly=False):
203 """e.g. all the RA values for OBJECT=='HD 123'"""
204 ret = []
205 for seqNum in self.data.keys():
206 if getValue(keyValPairAsTuple[0], self.data[seqNum]) == keyValPairAsTuple[1]:
207 ret.append(getValue(keyToGet, self.data[seqNum]))
208 if uniqueOnly:
209 return list(set(ret))
210 return ret
212 @staticmethod
213 def makeStarColorAndMarkerMap(stars):
214 markerMap = {}
215 colors = cm.rainbow(np.linspace(0, 1, N_STARS_PER_SYMBOL))
216 for i, star in enumerate(stars):
217 markerIndex = i//(N_STARS_PER_SYMBOL)
218 colorIndex = i%(N_STARS_PER_SYMBOL)
219 markerMap[star] = ColorAndMarker(colors[colorIndex], MARKER_SEQUENCE[markerIndex])
220 return markerMap
222 def getObjectValues(self, key, objName):
223 return self.getAllValuesForKVPair(key, ('OBJECT', objName), uniqueOnly=False)
225 def getAllHeaderKeys(self):
226 return list(list(self.data.items())[0][1].keys())
228 @staticmethod # designed for use in place of user-provided filter callbacks so gets self via call
229 def isDispersed(self, seqNum):
230 filt = self.data[seqNum]['ObservationInfo'].physical_filter
231 grating = filt.split(FILTER_DELIMITER)[1]
232 if "EMPTY" not in grating.upper():
233 return True
234 return False
236 def _calcObjectAirmasses(self, objects, filterFunc=None):
237 if filterFunc is None:
238 def noopFilter(*args, **kwargs):
239 return True
240 filterFunc = noopFilter
241 airMasses = {}
242 for star in objects:
243 seqNums = self.getObjectValues('SEQNUM', star)
244 airMasses[star] = [(self.data[seqNum]['ObservationInfo'].boresight_airmass,
245 getValue('MJD-BEG', self.data[seqNum]))
246 for seqNum in sorted(seqNums) if filterFunc(self, seqNum)]
247 return airMasses
249 def getSeqNums(self, filterFunc, *args, **kwargs):
250 """Get seqNums for a corresponding filtering function.
252 filterFunc is called with (self, seqNum) and must return a bool."""
253 seqNums = []
254 for seqNum in self.data.keys():
255 if filterFunc(self, seqNum, *args, **kwargs):
256 seqNums.append(seqNum)
257 return seqNums
259 def getObservedObjects(self):
260 return self.getUniqueValuesForKey('OBJECT')
262 def plotPerObjectAirMass(self, objects=None, airmassOneAtTop=True, filterFunc=None):
263 """filterFunc is self as the first argument and seqNum as second."""
264 if not objects:
265 objects = self.stars
267 objects = self._safeListArg(objects)
269 # lazy to always recalculate but it's not *that* slow
270 # and optionally passing around can be messy
271 # TODO: keep some of this in class state
272 airMasses = self._calcObjectAirmasses(objects, filterFunc=filterFunc)
274 _ = plt.figure(figsize=(10, 6))
275 for star in objects:
276 if airMasses[star]: # skip stars fully filtered out by callbacks
277 ams, times = np.asarray(airMasses[star])[:, 0], np.asarray(airMasses[star])[:, 1]
278 else:
279 continue
280 color = self.cMap[star].color
281 marker = self.cMap[star].marker
282 plt.plot(times, ams, color=color, marker=marker, label=star, ms=10, ls='')
284 plt.ylabel('Airmass', fontsize=20)
285 if airmassOneAtTop:
286 ax = plt.gca()
287 ax.set_ylim(ax.get_ylim()[::-1])
288 _ = plt.legend(bbox_to_anchor=(1, 1.025), prop={'size': 15}, loc='upper left')
290 def printObsTable(self, imageType=None, tailNumber=0):
291 """Print a table of the days observations.
293 Parameters
294 ----------
295 imageType : str
296 Only consider images with this image type
297 tailNumber : int
298 Only print out the last n entries in the night
299 """
300 lines = []
301 if not imageType:
302 seqNums = self.data.keys()
303 else:
304 seqNums = [s for s in self.data.keys()
305 if self.data[s]['ObservationInfo'].observation_type == imageType]
307 seqNums = sorted(seqNums)
308 for i, seqNum in enumerate(seqNums):
309 try:
310 expTime = self.data[seqNum]['ObservationInfo'].exposure_time.value
311 filt = self.data[seqNum]['ObservationInfo'].physical_filter
312 imageType = self.data[seqNum]['ObservationInfo'].observation_type
313 d1 = self.data[seqNum]['ObservationInfo'].datetime_begin
314 obj = self.data[seqNum]['ObservationInfo'].object
315 if i == 0:
316 d0 = d1
317 dt = (d1-d0)
318 d0 = d1
319 timeOfDay = d1.isot.split('T')[1]
320 msg = f'{seqNum:4} {imageType:9} {obj:10} {timeOfDay} {filt:25} {dt.sec:6.1f} {expTime:2.2f}'
321 except KeyError:
322 msg = f'{seqNum:4} - error parsing headers/observation info! Check the file'
323 lines.append(msg)
325 print(r"{seqNum} {imageType} {obj} {timeOfDay} {filt} {timeSinceLastExp} {expTime}")
326 for line in lines[-tailNumber:]:
327 print(line)
329 def calcShutterOpenEfficiency(self, seqMin=0, seqMax=0):
330 if seqMin == 0:
331 seqMin = min(self.data.keys())
332 if seqMax == 0:
333 seqMax = max(self.data.keys())
334 assert seqMax > seqMin
335 assert (seqMin in self.data.keys())
336 assert (seqMax in self.data.keys())
338 timeStart = self.data[seqMin]['ObservationInfo'].datetime_begin
339 timeEnd = self.data[seqMax]['ObservationInfo'].datetime_end
340 expTimeSum = 0
341 for seqNum in range(seqMin, seqMax+1):
342 if seqNum not in self.data.keys():
343 print(f"Warning! No data found for seqNum {seqNum}")
344 continue
345 expTimeSum += self.data[seqNum]['ObservationInfo'].exposure_time.value
347 timeOnSky = (timeEnd - timeStart).sec
348 efficiency = expTimeSum/timeOnSky
349 print(f"{100*efficiency:.2f}% shutter open in seqNum range {seqMin} and {seqMax}")
350 print(f"Total integration time = {expTimeSum:.1f}s")
351 return efficiency
353 @staticmethod
354 def _safeListArg(arg):
355 if type(arg) == str:
356 return [arg]
357 assert(type(arg) == list), f"Expect list, got {arg}"
358 return arg