Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from lsst.meas.algorithms import Defects 

2from lsst.meas.algorithms.simple_curve import Curve 

3 

4import os 

5import glob 

6import dateutil.parser 

7 

8 

9def read_one_chip(root, chip_name, chip_id): 

10 """Read data for a particular sensor from the standard format at a particular root. 

11 

12 Parameters 

13 ---------- 

14 root : `str` 

15 Path to the top level of the data tree. This is expected to hold directories 

16 named after the sensor names. They are expected to be lower case. 

17 chip_name : `str` 

18 The name of the sensor for which to read data. 

19 chip_id : `int` 

20 The identifier for the sensor in question. 

21 

22 Returns 

23 ------- 

24 `dict` 

25 A dictionary of objects constructed from the appropriate factory class. 

26 The key is the validity start time as a `datetime` object. 

27 """ 

28 factory_map = {'qe_curve': Curve, 'defects': Defects} 

29 files = glob.glob(os.path.join(root, chip_name, '*.ecsv')) 

30 parts = os.path.split(root) 

31 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name> 

32 data_name = parts[1] 

33 if data_name not in factory_map: 

34 raise ValueError(f"Unknown calibration data type, '{data_name}' found. " 

35 f"Only understand {','.join(k for k in factory_map)}") 

36 factory = factory_map[data_name] 

37 data_dict = {} 

38 for f in files: 

39 date_str = os.path.splitext(os.path.basename(f))[0] 

40 valid_start = dateutil.parser.parse(date_str) 

41 data_dict[valid_start] = factory.readText(f) 

42 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name) 

43 return data_dict, data_name 

44 

45 

46def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name): 

47 """Check that the metadata is complete and self consistent 

48 

49 Parameters 

50 ---------- 

51 obj : object of same type as the factory 

52 Object to retrieve metadata from in order to compare with 

53 metadata inferred from the path. 

54 valid_start : `datetime` 

55 Start of the validity range for data 

56 instrument : `str` 

57 Name of the instrument in question 

58 chip_id : `int` 

59 Identifier of the sensor in question 

60 filepath : `str` 

61 Path of the file read to construct the data 

62 data_name : `str` 

63 Name of the type of data being read 

64 

65 Returns 

66 ------- 

67 None 

68 

69 Raises 

70 ------ 

71 ValueError 

72 If the metadata from the path and the metadata encoded 

73 in the path do not match for any reason. 

74 """ 

75 md = obj.getMetadata() 

76 finst = md['INSTRUME'] 

77 fchip_id = md['DETECTOR'] 

78 fcalib_date = md['CALIBDATE'] 

79 fdata_name = md['OBSTYPE'] 

80 if not ((finst.lower(), int(fchip_id), fcalib_date, fdata_name) == 

81 (instrument.lower(), chip_id, valid_start.isoformat(), data_name)): 

82 st_time = valid_start.isoformat() 

83 raise ValueError(f"Path and file metadata do not agree:\n" 

84 f"Path metadata: {instrument} {chip_id} {st_time} {data_name}\n" 

85 f"File metadata: {finst} {fchip_id} {fcalib_date} {fdata_name}\n" 

86 f"File read from : %s\n"%(filepath) 

87 ) 

88 

89 

90def read_all(root, camera): 

91 """Read all data from the standard format at a particular root. 

92 

93 Parameters 

94 ---------- 

95 root : `str` 

96 Path to the top level of the data tree. This is expected to hold directories 

97 named after the sensor names. They are expected to be lower case. 

98 camera : `lsst.afw.cameraGeom.Camera` 

99 The camera that goes with the data being read. 

100 

101 Returns 

102 ------- 

103 dict 

104 A dictionary of dictionaries of objects constructed with the appropriate factory class. 

105 The first key is the sensor name lowered, and the second is the validity 

106 start time as a `datetime` object. 

107 

108 Notes 

109 ----- 

110 Each leaf object in the constructed dictionary has metadata associated with it. 

111 The detector ID may be retrieved from the DETECTOR entry of that metadata. 

112 """ 

113 root = os.path.normpath(root) 

114 dirs = os.listdir(root) # assumes all directories contain data 

115 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))] 

116 data_by_chip = {} 

117 name_map = {det.getName().lower(): det.getName() for 

118 det in camera} # we assume the directories have been lowered 

119 

120 if not dirs: 

121 raise RuntimeError(f"No data found on path {root}") 

122 

123 calib_types = set() 

124 for d in dirs: 

125 chip_name = os.path.basename(d) 

126 chip_id = camera[name_map[chip_name]].getId() 

127 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id) 

128 calib_types.add(calib_type) 

129 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here. 

130 raise ValueError(f'Error mixing calib types: {calib_types}') 

131 

132 no_data = all([v == {} for v in data_by_chip.values()]) 

133 if no_data: 

134 raise RuntimeError(f'No data to ingest') 

135 

136 return data_by_chip, calib_type