Coverage for python/lsst/fgcmcal/fgcmCalibrateTractTable.py: 24%

88 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-14 02:53 -0700

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Class for running fgcmcal on a single tract using sourceTable_visit tables. 

24""" 

25import numpy as np 

26 

27import lsst.pipe.base as pipeBase 

28from lsst.pipe.base import connectionTypes 

29from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

30import lsst.afw.table as afwTable 

31 

32from .fgcmBuildStarsTable import FgcmBuildStarsTableTask 

33from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, 

34 FgcmCalibrateTractBaseTask) 

35 

36__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask'] 

37 

38 

39class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections, 

40 dimensions=("instrument", 

41 "tract",)): 

42 camera = connectionTypes.PrerequisiteInput( 

43 doc="Camera instrument", 

44 name="camera", 

45 storageClass="Camera", 

46 dimensions=("instrument",), 

47 isCalibration=True, 

48 ) 

49 

50 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

51 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

52 "chromatic corrections."), 

53 name="fgcmLookUpTable", 

54 storageClass="Catalog", 

55 dimensions=("instrument",), 

56 deferLoad=True, 

57 ) 

58 

59 sourceSchema = connectionTypes.InitInput( 

60 doc="Schema for source catalogs", 

61 name="src_schema", 

62 storageClass="SourceCatalog", 

63 ) 

64 

65 refCat = connectionTypes.PrerequisiteInput( 

66 doc="Reference catalog to use for photometric calibration", 

67 name="cal_ref_cat", 

68 storageClass="SimpleCatalog", 

69 dimensions=("skypix",), 

70 deferLoad=True, 

71 multiple=True, 

72 ) 

73 

74 source_catalogs = connectionTypes.Input( 

75 doc="Source table in parquet format, per visit", 

76 name="sourceTable_visit", 

77 storageClass="DataFrame", 

78 dimensions=("instrument", "visit"), 

79 deferLoad=True, 

80 multiple=True, 

81 ) 

82 

83 visitSummary = connectionTypes.Input( 

84 doc="Per-visit summary statistics table", 

85 name="visitSummary", 

86 storageClass="ExposureCatalog", 

87 dimensions=("instrument", "visit"), 

88 deferLoad=True, 

89 multiple=True, 

90 ) 

91 

92 background = connectionTypes.Input( 

93 doc="Calexp background model", 

94 name="calexpBackground", 

95 storageClass="Background", 

96 dimensions=("instrument", "visit", "detector"), 

97 deferLoad=True, 

98 multiple=True, 

99 ) 

100 

101 fgcmPhotoCalib = connectionTypes.Output( 

102 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration", 

103 name="fgcmPhotoCalibTractCatalog", 

104 storageClass="ExposureCatalog", 

105 dimensions=("instrument", "tract", "visit",), 

106 multiple=True, 

107 ) 

108 

109 fgcmTransmissionAtmosphere = connectionTypes.Output( 

110 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

111 name="transmission_atmosphere_fgcm_tract", 

112 storageClass="TransmissionCurve", 

113 dimensions=("instrument", "tract", "visit",), 

114 multiple=True, 

115 ) 

116 

117 fgcmRepeatability = connectionTypes.Output( 

118 doc="Per-band raw repeatability numbers in the fgcm tract calibration", 

119 name="fgcmRawRepeatability", 

120 storageClass="Catalog", 

121 dimensions=("instrument", "tract",), 

122 multiple=False, 

123 ) 

124 

125 def __init__(self, *, config=None): 

126 super().__init__(config=config) 

127 

128 if not config.fgcmBuildStars.doModelErrorsWithBackground: 

129 self.inputs.remove("background") 

130 

131 if not config.fgcmOutputProducts.doAtmosphereOutput: 

132 self.prerequisiteInputs.remove("fgcmAtmosphereParameters") 

133 if not config.fgcmOutputProducts.doZeropointOutput: 

134 self.prerequisiteInputs.remove("fgcmZeropoints") 

135 

136 def getSpatialBoundsConnections(self): 

137 return ("visitSummary",) 

138 

139 

140class FgcmCalibrateTractTableConfig(FgcmCalibrateTractConfigBase, pipeBase.PipelineTaskConfig, 

141 pipelineConnections=FgcmCalibrateTractTableConnections): 

142 """Config for FgcmCalibrateTractTable task""" 

143 def setDefaults(self): 

144 super().setDefaults() 

145 

146 # For the Table version of CalibrateTract, use the associated 

147 # Table version of the BuildStars task. 

148 self.fgcmBuildStars.retarget(FgcmBuildStarsTableTask) 

149 # For tract mode, we set a very high effective density cut. 

150 self.fgcmBuildStars.densityCutMaxPerPixel = 10000 

151 

152 

153class FgcmCalibrateTractTableTask(FgcmCalibrateTractBaseTask): 

154 """ 

155 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet) 

156 input catalogs. 

157 """ 

158 ConfigClass = FgcmCalibrateTractTableConfig 

159 _DefaultName = "fgcmCalibrateTractTable" 

160 

161 canMultiprocess = False 

162 

163 def __init__(self, initInputs=None, **kwargs): 

164 super().__init__(initInputs=initInputs, **kwargs) 

165 if initInputs is not None: 

166 self.sourceSchema = initInputs["sourceSchema"].schema 

167 

168 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

169 handleDict = butlerQC.get(inputRefs) 

170 

171 self.log.info("Running with %d sourceTable_visit handles", (len(handleDict['source_catalogs']))) 

172 

173 # Run the build stars tasks 

174 tract = butlerQC.quantum.dataId['tract'] 

175 

176 handleDict['sourceSchema'] = self.sourceSchema 

177 

178 sourceTableHandles = handleDict['source_catalogs'] 

179 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

180 sourceTableHandle in sourceTableHandles} 

181 

182 visitSummaryHandles = handleDict['visitSummary'] 

183 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

184 visitSummaryHandle in visitSummaryHandles} 

185 

186 handleDict['sourceTableHandleDict'] = sourceTableHandleDict 

187 handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict 

188 

189 # And the outputs 

190 if self.config.fgcmOutputProducts.doZeropointOutput: 

191 photoCalibRefDict = {photoCalibRef.dataId['visit']: 

192 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

193 handleDict['fgcmPhotoCalibs'] = photoCalibRefDict 

194 

195 if self.config.fgcmOutputProducts.doAtmosphereOutput: 

196 atmRefDict = {atmRef.dataId['visit']: atmRef for 

197 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

198 handleDict['fgcmTransmissionAtmospheres'] = atmRefDict 

199 

200 if self.config.fgcmBuildStars.doReferenceMatches: 

201 refConfig = LoadReferenceObjectsConfig() 

202 refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap 

203 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

204 for ref in inputRefs.refCat], 

205 refCats=butlerQC.get(inputRefs.refCat), 

206 name=self.config.connections.refCat, 

207 config=refConfig, 

208 log=self.log) 

209 buildStarsRefObjLoader = loader 

210 else: 

211 buildStarsRefObjLoader = None 

212 

213 if self.config.fgcmOutputProducts.doReferenceCalibration: 

214 refConfig = self.config.fgcmOutputProducts.refObjLoader 

215 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

216 for ref in inputRefs.refCat], 

217 refCats=butlerQC.get(inputRefs.refCat), 

218 name=self.config.connections.refCat, 

219 config=refConfig, 

220 log=self.log) 

221 self.fgcmOutputProducts.refObjLoader = loader 

222 

223 struct = self.run(handleDict, tract, 

224 buildStarsRefObjLoader=buildStarsRefObjLoader) 

225 

226 if struct.photoCalibCatalogs is not None: 

227 self.log.info("Outputting photoCalib catalogs.") 

228 for visit, expCatalog in struct.photoCalibCatalogs: 

229 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

230 self.log.info("Done outputting photoCalib catalogs.") 

231 

232 if struct.atmospheres is not None: 

233 self.log.info("Outputting atmosphere transmission files.") 

234 for visit, atm in struct.atmospheres: 

235 butlerQC.put(atm, atmRefDict[visit]) 

236 self.log.info("Done outputting atmosphere files.") 

237 

238 # Turn raw repeatability into simple catalog for persistence 

239 schema = afwTable.Schema() 

240 schema.addField('rawRepeatability', type=np.float64, 

241 doc="Per-band raw repeatability in FGCM calibration.") 

242 repeatabilityCat = afwTable.BaseCatalog(schema) 

243 repeatabilityCat.resize(len(struct.repeatability)) 

244 repeatabilityCat['rawRepeatability'][:] = struct.repeatability 

245 

246 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability) 

247 

248 return