Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software=you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ["RegistryTablesTuple", "makeRegistryTableSpecs"] 

24 

25from collections import namedtuple 

26from typing import Type 

27 

28import sqlalchemy 

29 

30from ..core import ( 

31 ddl, 

32 DimensionUniverse, 

33) 

34from .interfaces import CollectionManager, DatasetRecordStorageManager 

35 

36 

37RegistryTablesTuple = namedtuple( 

38 "RegistryTablesTuple", 

39 [ 

40 "quantum", 

41 "dataset_consumers", 

42 "dataset_location", 

43 "dataset_location_trash", 

44 ] 

45) 

46 

47 

48def makeRegistryTableSpecs(universe: DimensionUniverse, 

49 collections: Type[CollectionManager], 

50 datasets: Type[DatasetRecordStorageManager], 

51 ) -> RegistryTablesTuple: 

52 """Construct descriptions tables in the Registry that are not (yet) 

53 managed by helper classes. 

54 

55 Parameters 

56 ---------- 

57 universe : `DimensionUniverse` 

58 All dimensions known to the `Registry`. 

59 collections : `Collection` 

60 The `CollectionManager` that will be used for this `Registry`; used to 

61 create foreign keys to the run and collection tables. 

62 

63 Returns 

64 ------- 

65 specs : `RegistryTablesTuple` 

66 A named tuple containing `ddl.TableSpec` instances. 

67 """ 

68 quantum = ddl.TableSpec( 

69 doc="A table used to capture fine-grained provenance for datasets produced by PipelineTasks.", 

70 fields=[ 

71 ddl.FieldSpec( 

72 name="id", 

73 dtype=sqlalchemy.BigInteger, 

74 primaryKey=True, 

75 autoincrement=True, 

76 doc="A unique autoincrement integer identifier for this quantum.", 

77 ), 

78 ddl.FieldSpec( 

79 name="task", 

80 dtype=sqlalchemy.String, 

81 length=256, 

82 doc="Fully qualified name of the SuperTask that executed this quantum.", 

83 ), 

84 ddl.FieldSpec( 

85 name="start_time", 

86 dtype=ddl.AstropyTimeNsecTai, 

87 nullable=True, 

88 doc="The start time for the quantum.", 

89 ), 

90 ddl.FieldSpec( 

91 name="end_time", 

92 dtype=ddl.AstropyTimeNsecTai, 

93 nullable=True, 

94 doc="The end time for the quantum.", 

95 ), 

96 ddl.FieldSpec( 

97 name="host", 

98 dtype=sqlalchemy.String, 

99 length=64, 

100 nullable=True, 

101 doc="The system on which the quantum was executed.", 

102 ), 

103 ], 

104 ) 

105 collections.addRunForeignKey(quantum, onDelete="CASCADE", nullable=False) 

106 

107 dataset_consumers = ddl.TableSpec( 

108 doc="A table relating Quantum records to the datasets they used as inputs.", 

109 fields=[ 

110 ddl.FieldSpec( 

111 name="quantum_id", 

112 dtype=sqlalchemy.BigInteger, 

113 nullable=False, 

114 doc="A link to the associated Quantum.", 

115 ), 

116 ddl.FieldSpec( 

117 name="actual", 

118 dtype=sqlalchemy.Boolean, 

119 nullable=False, 

120 doc=( 

121 "Whether the Dataset was actually used as an input by the Quantum " 

122 "(as opposed to just predicted to be used during preflight)." 

123 ), 

124 ), 

125 ], 

126 foreignKeys=[ 

127 ddl.ForeignKeySpec( 

128 table="quantum", 

129 source=("quantum_id",), 

130 target=("id",), 

131 onDelete="CASCADE", 

132 ), 

133 ] 

134 ) 

135 datasets.addDatasetForeignKey(dataset_consumers, nullable=True, onDelete="SET NULL") 

136 

137 # We want the dataset_location and dataset_location_trash tables 

138 # to have the same definition, aside from the behavior of their link 

139 # to the dataset table: the trash table has no foreign key constraint. 

140 dataset_location_spec = dict( 

141 doc=( 

142 "A table that provides information on whether a dataset is stored in " 

143 "one or more Datastores. The presence or absence of a record in this " 

144 "table itself indicates whether the dataset is present in that " 

145 "Datastore. " 

146 ), 

147 fields=[ 

148 ddl.FieldSpec( 

149 name="datastore_name", 

150 dtype=sqlalchemy.String, 

151 length=256, 

152 primaryKey=True, 

153 nullable=False, 

154 doc="Name of the Datastore this entry corresponds to.", 

155 ), 

156 ], 

157 ) 

158 dataset_location = ddl.TableSpec(**dataset_location_spec) 

159 datasets.addDatasetForeignKey(dataset_location, primaryKey=True) 

160 dataset_location_trash = ddl.TableSpec(**dataset_location_spec) 

161 datasets.addDatasetForeignKey(dataset_location_trash, primaryKey=True, constraint=False) 

162 

163 return RegistryTablesTuple( 

164 quantum=quantum, 

165 dataset_consumers=dataset_consumers, 

166 dataset_location=dataset_location, 

167 dataset_location_trash=dataset_location_trash, 

168 )