Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

# This file is part of daf_butler. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

 

__all__ = ("FitsCatalogDatasetsHelper", "DatasetTestHelper", "DatastoreTestHelper", 

"BadWriteFormatter", "BadNoWriteFormatter", "MultiDetectorFormatter") 

 

import os 

from lsst.daf.butler import DatasetType, DatasetRef 

from lsst.daf.butler.formatters.yamlFormatter import YamlFormatter 

 

 

class FitsCatalogDatasetsHelper: 

 

def makeExampleCatalog(self): 

import lsst.afw.table 

catalogPath = os.path.join(self.testDir, "data", "basic", "source_catalog.fits") 

return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

 

def assertCatalogEqual(self, inputCatalog, outputCatalog): 

import lsst.afw.table 

self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

inputTable = inputCatalog.getTable() 

inputRecord = inputCatalog[0] 

outputTable = outputCatalog.getTable() 

outputRecord = outputCatalog[0] 

self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

outputTable.getSchema().getAliasMap().get("slot_Centroid")) 

self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

self.assertFloatsAlmostEqual( 

inputRecord.getCentroidErr()[0, 0], 

outputRecord.getCentroidErr()[0, 0], rtol=1e-6) 

self.assertFloatsAlmostEqual( 

inputRecord.getCentroidErr()[1, 1], 

outputRecord.getCentroidErr()[1, 1], rtol=1e-6) 

self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"), 

outputTable.getSchema().getAliasMap().get("slot_Shape")) 

self.assertFloatsAlmostEqual( 

inputRecord.getShapeErr()[0, 0], 

outputRecord.getShapeErr()[0, 0], rtol=1e-6) 

self.assertFloatsAlmostEqual( 

inputRecord.getShapeErr()[1, 1], 

outputRecord.getShapeErr()[1, 1], rtol=1e-6) 

self.assertFloatsAlmostEqual( 

inputRecord.getShapeErr()[2, 2], 

outputRecord.getShapeErr()[2, 2], rtol=1e-6) 

 

 

class DatasetTestHelper: 

"""Helper methods for Datasets""" 

 

def makeDatasetRef(self, datasetTypeName, dimensions, storageClass, dataId, *, id=None, run=None, 

conform=True): 

"""Make a DatasetType and wrap it in a DatasetRef for a test""" 

datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

if id is None: 

self.id += 1 

id = self.id 

if run is None: 

run = "dummy" 

return DatasetRef(datasetType, dataId, id=id, run=run, conform=conform) 

 

 

class DatastoreTestHelper: 

"""Helper methods for Datastore tests""" 

 

def setUpDatastoreTests(self, registryClass, configClass): 

"""Shared setUp code for all Datastore tests""" 

self.registry = registryClass() 

 

# Need to keep ID for each datasetRef since we have no butler 

# for these tests 

self.id = 1 

 

self.config = configClass(self.configFile) 

 

# Some subclasses override the working root directory 

if self.root is not None: 

self.datastoreType.setConfigRoot(self.root, self.config, self.config.copy()) 

 

def makeDatastore(self, sub=None): 

"""Make a new Datastore instance of the appropriate type. 

 

Parameters 

---------- 

sub : str, optional 

If not None, the returned Datastore will be distinct from any 

Datastore constructed with a different value of ``sub``. For 

PosixDatastore, for example, the converse is also true, and ``sub`` 

is used as a subdirectory to form the new root. 

 

Returns 

------- 

datastore : `Datastore` 

Datastore constructed by this routine using the supplied 

optional subdirectory if supported. 

""" 

config = self.config.copy() 

if sub is not None and self.root is not None: 

self.datastoreType.setConfigRoot(os.path.join(self.root, sub), config, self.config) 

if sub is not None: 

# Ensure that each datastore gets its own registry 

registryClass = type(self.registry) 

registry = registryClass() 

else: 

registry = self.registry 

return self.datastoreType(config=config, registry=registry) 

 

 

class BadWriteFormatter(YamlFormatter): 

"""A formatter that never works but does leave a file behind.""" 

 

def _readFile(self, path, pytype=None): 

raise NotImplementedError("This formatter can not read anything") 

 

def _writeFile(self, inMemoryDataset): 

"""Write an empty file and then raise an exception.""" 

with open(self.fileDescriptor.location.path, "wb"): 

pass 

raise RuntimeError("Did not succeed in writing file") 

 

 

class BadNoWriteFormatter(BadWriteFormatter): 

"""A formatter that always fails without writing anything.""" 

 

def _writeFile(self, inMemoryDataset): 

raise RuntimeError("Did not writing anything at all") 

 

 

class MultiDetectorFormatter(YamlFormatter): 

 

def _writeFile(self, inMemoryDataset): 

raise NotImplementedError("Can not write") 

 

def _fromBytes(self, serializedDataset, pytype=None): 

data = super()._fromBytes(serializedDataset) 

if self.dataId is None: 

raise RuntimeError("This formatter requires a dataId") 

if "detector" not in self.dataId: 

raise RuntimeError("This formatter requires detector to be present in dataId") 

key = f"detector{self.dataId['detector']}" 

if key in data: 

return pytype(data[key]) 

raise RuntimeError(f"Could not find '{key}' in data file")