Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from astropy.table import Table 

23import logging 

24import os 

25from .. import Butler, CollectionType 

26 

27 

28log = logging.getLogger(__name__) 

29 

30 

31def parseCalibrationCollection(registry, collection, datasetTypes): 

32 """Search a calibration collection for calibration datasets. 

33 

34 Parameters 

35 ---------- 

36 registry : `lsst.daf.butler.Registry` 

37 Butler registry to use. 

38 collection : `str` 

39 Collection to search. This should be a CALIBRATION 

40 collection. 

41 datasetTypes : `list` [`lsst.daf.Butler.DatasetType`] 

42 List of calibration dataset types. 

43 

44 Returns 

45 ------- 

46 exportCollections : `list` [`str`] 

47 List of collections to save on export. 

48 exportDatasets : `list` [`lsst.daf.butler.queries.DatasetQueryResults`] 

49 Datasets to save on export. 

50 

51 Raises 

52 ------ 

53 RuntimeError 

54 Raised if the collection to search is not a CALIBRATION collection. 

55 """ 

56 if registry.getCollectionType(collection) != CollectionType.CALIBRATION: 

57 raise RuntimeError(f"Collection {collection} is not a CALIBRATION collection.") 

58 

59 exportCollections = [] 

60 exportDatasets = [] 

61 for calibType in datasetTypes: 

62 associations = registry.queryDatasetAssociations(calibType, collections=collection, 

63 collectionTypes=[CollectionType.CALIBRATION]) 

64 for result in associations: 

65 exportDatasets.append(result.ref) 

66 exportCollections.append(result.ref.run) 

67 return exportCollections, exportDatasets 

68 

69 

70def exportCalibs(repo, directory, collections): 

71 """Certify a set of calibrations with a validity range. 

72 

73 Parameters 

74 ---------- 

75 repo : `str` 

76 URI to the location of the repo or URI to a config file 

77 describing the repo and its location. 

78 directory : `str` 

79 URI string of the directory to write the exported 

80 calibrations. 

81 collections : `list` [`str`] 

82 Data collections to pull calibrations from. Must be an 

83 existing `~CollectionType.CHAINED` or 

84 `~CollectionType.CALIBRATION` collection. 

85 

86 Returns 

87 ------- 

88 datasetTable : `astropy.table.Table` 

89 A table containing relevant information about the calibrations 

90 exported. 

91 

92 Raises 

93 ------ 

94 RuntimeError : 

95 Raised if the output directory already exists. 

96 """ 

97 butler = Butler(repo, writeable=False) 

98 

99 calibTypes = [datasetType for datasetType in butler.registry.queryDatasetTypes(...) 

100 if datasetType.isCalibration()] 

101 

102 collectionsToExport = [] 

103 datasetsToExport = [] 

104 

105 for collection in butler.registry.queryCollections(collections, flattenChains=True, includeChains=True, 

106 collectionTypes={CollectionType.CALIBRATION, 

107 CollectionType.CHAINED}): 

108 log.info("Checking collection: %s", collection) 

109 

110 # Get collection information. 

111 collectionsToExport.append(collection) 

112 collectionType = butler.registry.getCollectionType(collection) 

113 if collectionType == CollectionType.CALIBRATION: 

114 exportCollections, exportDatasets = parseCalibrationCollection(butler.registry, 

115 collection, 

116 calibTypes) 

117 collectionsToExport.extend(exportCollections) 

118 datasetsToExport.extend(exportDatasets) 

119 

120 if os.path.exists(directory): 

121 raise RuntimeError(f"Export directory exists: {directory}") 

122 os.makedirs(directory) 

123 with butler.export(directory=directory, format="yaml", transfer="auto") as export: 

124 collectionsToExport = list(set(collectionsToExport)) 

125 datasetsToExport = list(set(datasetsToExport)) 

126 

127 for exportable in collectionsToExport: 

128 try: 

129 export.saveCollection(exportable) 

130 except Exception as e: 

131 log.warning("Did not save collection %s due to %s.", exportable, e) 

132 

133 log.info("Saving %d dataset(s)", len(datasetsToExport)) 

134 export.saveDatasets(datasetsToExport) 

135 

136 sortedDatasets = sorted(datasetsToExport, key=lambda x: x.datasetType.name) 

137 

138 requiredDimensions = set() 

139 for ref in sortedDatasets: 

140 requiredDimensions.update(ref.dimensions.names) 

141 dimensionColumns = {dimensionName: [ref.dataId.get(dimensionName, "") for ref in sortedDatasets] 

142 for dimensionName in requiredDimensions} 

143 

144 return Table({"calibrationType": [ref.datasetType.name for ref in sortedDatasets], 

145 "run": [ref.run for ref in sortedDatasets], 

146 **dimensionColumns 

147 })