Coverage for python/lsst/fgcmcal/fgcmCalibrateTractTable.py: 24%

87 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-24 11:18 +0000

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Class for running fgcmcal on a single tract using sourceTable_visit tables. 

24""" 

25import numpy as np 

26 

27import lsst.pipe.base as pipeBase 

28from lsst.pipe.base import connectionTypes 

29from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

30import lsst.afw.table as afwTable 

31 

32from .fgcmBuildStarsTable import FgcmBuildStarsTableTask 

33from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, 

34 FgcmCalibrateTractBaseTask) 

35from .utilities import lookupStaticCalibrations 

36 

37__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask'] 

38 

39 

40class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections, 

41 dimensions=("instrument", 

42 "tract",)): 

43 camera = connectionTypes.PrerequisiteInput( 

44 doc="Camera instrument", 

45 name="camera", 

46 storageClass="Camera", 

47 dimensions=("instrument",), 

48 lookupFunction=lookupStaticCalibrations, 

49 isCalibration=True, 

50 ) 

51 

52 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

53 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

54 "chromatic corrections."), 

55 name="fgcmLookUpTable", 

56 storageClass="Catalog", 

57 dimensions=("instrument",), 

58 deferLoad=True, 

59 ) 

60 

61 sourceSchema = connectionTypes.InitInput( 

62 doc="Schema for source catalogs", 

63 name="src_schema", 

64 storageClass="SourceCatalog", 

65 ) 

66 

67 refCat = connectionTypes.PrerequisiteInput( 

68 doc="Reference catalog to use for photometric calibration", 

69 name="cal_ref_cat", 

70 storageClass="SimpleCatalog", 

71 dimensions=("skypix",), 

72 deferLoad=True, 

73 multiple=True, 

74 ) 

75 

76 source_catalogs = connectionTypes.Input( 

77 doc="Source table in parquet format, per visit", 

78 name="sourceTable_visit", 

79 storageClass="DataFrame", 

80 dimensions=("instrument", "visit"), 

81 deferLoad=True, 

82 multiple=True, 

83 ) 

84 

85 visitSummary = connectionTypes.Input( 

86 doc="Per-visit summary statistics table", 

87 name="visitSummary", 

88 storageClass="ExposureCatalog", 

89 dimensions=("instrument", "visit"), 

90 deferLoad=True, 

91 multiple=True, 

92 ) 

93 

94 background = connectionTypes.Input( 

95 doc="Calexp background model", 

96 name="calexpBackground", 

97 storageClass="Background", 

98 dimensions=("instrument", "visit", "detector"), 

99 deferLoad=True, 

100 multiple=True, 

101 ) 

102 

103 fgcmPhotoCalib = connectionTypes.Output( 

104 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration", 

105 name="fgcmPhotoCalibTractCatalog", 

106 storageClass="ExposureCatalog", 

107 dimensions=("instrument", "tract", "visit",), 

108 multiple=True, 

109 ) 

110 

111 fgcmTransmissionAtmosphere = connectionTypes.Output( 

112 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

113 name="transmission_atmosphere_fgcm_tract", 

114 storageClass="TransmissionCurve", 

115 dimensions=("instrument", "tract", "visit",), 

116 multiple=True, 

117 ) 

118 

119 fgcmRepeatability = connectionTypes.Output( 

120 doc="Per-band raw repeatability numbers in the fgcm tract calibration", 

121 name="fgcmRawRepeatability", 

122 storageClass="Catalog", 

123 dimensions=("instrument", "tract",), 

124 multiple=False, 

125 ) 

126 

127 def __init__(self, *, config=None): 

128 super().__init__(config=config) 

129 

130 if not config.fgcmBuildStars.doModelErrorsWithBackground: 

131 self.inputs.remove("background") 

132 

133 if not config.fgcmOutputProducts.doAtmosphereOutput: 

134 self.prerequisiteInputs.remove("fgcmAtmosphereParameters") 

135 if not config.fgcmOutputProducts.doZeropointOutput: 

136 self.prerequisiteInputs.remove("fgcmZeropoints") 

137 

138 

139class FgcmCalibrateTractTableConfig(FgcmCalibrateTractConfigBase, pipeBase.PipelineTaskConfig, 

140 pipelineConnections=FgcmCalibrateTractTableConnections): 

141 """Config for FgcmCalibrateTractTable task""" 

142 def setDefaults(self): 

143 super().setDefaults() 

144 

145 # For the Table version of CalibrateTract, use the associated 

146 # Table version of the BuildStars task. 

147 self.fgcmBuildStars.retarget(FgcmBuildStarsTableTask) 

148 # For tract mode, we set a very high effective density cut. 

149 self.fgcmBuildStars.densityCutMaxPerPixel = 10000 

150 

151 

152class FgcmCalibrateTractTableTask(FgcmCalibrateTractBaseTask): 

153 """ 

154 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet) 

155 input catalogs. 

156 """ 

157 ConfigClass = FgcmCalibrateTractTableConfig 

158 _DefaultName = "fgcmCalibrateTractTable" 

159 

160 canMultiprocess = False 

161 

162 def __init__(self, initInputs=None, **kwargs): 

163 super().__init__(initInputs=initInputs, **kwargs) 

164 if initInputs is not None: 

165 self.sourceSchema = initInputs["sourceSchema"].schema 

166 

167 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

168 handleDict = butlerQC.get(inputRefs) 

169 

170 self.log.info("Running with %d sourceTable_visit handles", (len(handleDict['source_catalogs']))) 

171 

172 # Run the build stars tasks 

173 tract = butlerQC.quantum.dataId['tract'] 

174 

175 handleDict['sourceSchema'] = self.sourceSchema 

176 

177 sourceTableHandles = handleDict['source_catalogs'] 

178 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

179 sourceTableHandle in sourceTableHandles} 

180 

181 visitSummaryHandles = handleDict['visitSummary'] 

182 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

183 visitSummaryHandle in visitSummaryHandles} 

184 

185 handleDict['sourceTableHandleDict'] = sourceTableHandleDict 

186 handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict 

187 

188 # And the outputs 

189 if self.config.fgcmOutputProducts.doZeropointOutput: 

190 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

191 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

192 handleDict['fgcmPhotoCalibs'] = photoCalibRefDict 

193 

194 if self.config.fgcmOutputProducts.doAtmosphereOutput: 

195 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

196 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

197 handleDict['fgcmTransmissionAtmospheres'] = atmRefDict 

198 

199 if self.config.fgcmBuildStars.doReferenceMatches: 

200 refConfig = LoadReferenceObjectsConfig() 

201 refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap 

202 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

203 for ref in inputRefs.refCat], 

204 refCats=butlerQC.get(inputRefs.refCat), 

205 name=self.config.connections.refCat, 

206 config=refConfig, 

207 log=self.log) 

208 buildStarsRefObjLoader = loader 

209 else: 

210 buildStarsRefObjLoader = None 

211 

212 if self.config.fgcmOutputProducts.doReferenceCalibration: 

213 refConfig = self.config.fgcmOutputProducts.refObjLoader 

214 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

215 for ref in inputRefs.refCat], 

216 refCats=butlerQC.get(inputRefs.refCat), 

217 name=self.config.connections.refCat, 

218 config=refConfig, 

219 log=self.log) 

220 self.fgcmOutputProducts.refObjLoader = loader 

221 

222 struct = self.run(handleDict, tract, 

223 buildStarsRefObjLoader=buildStarsRefObjLoader) 

224 

225 if struct.photoCalibCatalogs is not None: 

226 self.log.info("Outputting photoCalib catalogs.") 

227 for visit, expCatalog in struct.photoCalibCatalogs: 

228 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

229 self.log.info("Done outputting photoCalib catalogs.") 

230 

231 if struct.atmospheres is not None: 

232 self.log.info("Outputting atmosphere transmission files.") 

233 for visit, atm in struct.atmospheres: 

234 butlerQC.put(atm, atmRefDict[visit]) 

235 self.log.info("Done outputting atmosphere files.") 

236 

237 # Turn raw repeatability into simple catalog for persistence 

238 schema = afwTable.Schema() 

239 schema.addField('rawRepeatability', type=np.float64, 

240 doc="Per-band raw repeatability in FGCM calibration.") 

241 repeatabilityCat = afwTable.BaseCatalog(schema) 

242 repeatabilityCat.resize(len(struct.repeatability)) 

243 repeatabilityCat['rawRepeatability'][:] = struct.repeatability 

244 

245 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability) 

246 

247 return