Coverage for python/lsst/fgcmcal/fgcmCalibrateTractTable.py: 25%

89 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-25 12:04 +0000

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Class for running fgcmcal on a single tract using sourceTable_visit tables. 

24""" 

25import numpy as np 

26 

27import lsst.pipe.base as pipeBase 

28from lsst.pipe.base import connectionTypes 

29from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

30import lsst.afw.table as afwTable 

31 

32from .fgcmBuildStarsTable import FgcmBuildStarsTableTask 

33from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, 

34 FgcmCalibrateTractBaseTask) 

35from .utilities import lookupStaticCalibrations 

36 

37__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask'] 

38 

39 

40class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections, 

41 dimensions=("instrument", 

42 "tract",)): 

43 camera = connectionTypes.PrerequisiteInput( 

44 doc="Camera instrument", 

45 name="camera", 

46 storageClass="Camera", 

47 dimensions=("instrument",), 

48 lookupFunction=lookupStaticCalibrations, 

49 isCalibration=True, 

50 ) 

51 

52 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

53 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

54 "chromatic corrections."), 

55 name="fgcmLookUpTable", 

56 storageClass="Catalog", 

57 dimensions=("instrument",), 

58 deferLoad=True, 

59 ) 

60 

61 sourceSchema = connectionTypes.InitInput( 

62 doc="Schema for source catalogs", 

63 name="src_schema", 

64 storageClass="SourceCatalog", 

65 ) 

66 

67 refCat = connectionTypes.PrerequisiteInput( 

68 doc="Reference catalog to use for photometric calibration", 

69 name="cal_ref_cat", 

70 storageClass="SimpleCatalog", 

71 dimensions=("skypix",), 

72 deferLoad=True, 

73 multiple=True, 

74 ) 

75 

76 source_catalogs = connectionTypes.Input( 

77 doc="Source table in parquet format, per visit", 

78 name="sourceTable_visit", 

79 storageClass="DataFrame", 

80 dimensions=("instrument", "visit"), 

81 deferLoad=True, 

82 multiple=True, 

83 ) 

84 

85 visitSummary = connectionTypes.Input( 

86 doc="Per-visit summary statistics table", 

87 name="visitSummary", 

88 storageClass="ExposureCatalog", 

89 dimensions=("instrument", "visit"), 

90 deferLoad=True, 

91 multiple=True, 

92 ) 

93 

94 background = connectionTypes.Input( 

95 doc="Calexp background model", 

96 name="calexpBackground", 

97 storageClass="Background", 

98 dimensions=("instrument", "visit", "detector"), 

99 deferLoad=True, 

100 multiple=True, 

101 ) 

102 

103 fgcmPhotoCalib = connectionTypes.Output( 

104 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration", 

105 name="fgcmPhotoCalibTractCatalog", 

106 storageClass="ExposureCatalog", 

107 dimensions=("instrument", "tract", "visit",), 

108 multiple=True, 

109 ) 

110 

111 fgcmTransmissionAtmosphere = connectionTypes.Output( 

112 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

113 name="transmission_atmosphere_fgcm_tract", 

114 storageClass="TransmissionCurve", 

115 dimensions=("instrument", "tract", "visit",), 

116 multiple=True, 

117 ) 

118 

119 fgcmRepeatability = connectionTypes.Output( 

120 doc="Per-band raw repeatability numbers in the fgcm tract calibration", 

121 name="fgcmRawRepeatability", 

122 storageClass="Catalog", 

123 dimensions=("instrument", "tract",), 

124 multiple=False, 

125 ) 

126 

127 def __init__(self, *, config=None): 

128 super().__init__(config=config) 

129 

130 if not config.fgcmBuildStars.doModelErrorsWithBackground: 

131 self.inputs.remove("background") 

132 

133 if not config.fgcmOutputProducts.doAtmosphereOutput: 

134 self.prerequisiteInputs.remove("fgcmAtmosphereParameters") 

135 if not config.fgcmOutputProducts.doZeropointOutput: 

136 self.prerequisiteInputs.remove("fgcmZeropoints") 

137 

138 def getSpatialBoundsConnections(self): 

139 return ("visitSummary",) 

140 

141 

142class FgcmCalibrateTractTableConfig(FgcmCalibrateTractConfigBase, pipeBase.PipelineTaskConfig, 

143 pipelineConnections=FgcmCalibrateTractTableConnections): 

144 """Config for FgcmCalibrateTractTable task""" 

145 def setDefaults(self): 

146 super().setDefaults() 

147 

148 # For the Table version of CalibrateTract, use the associated 

149 # Table version of the BuildStars task. 

150 self.fgcmBuildStars.retarget(FgcmBuildStarsTableTask) 

151 # For tract mode, we set a very high effective density cut. 

152 self.fgcmBuildStars.densityCutMaxPerPixel = 10000 

153 

154 

155class FgcmCalibrateTractTableTask(FgcmCalibrateTractBaseTask): 

156 """ 

157 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet) 

158 input catalogs. 

159 """ 

160 ConfigClass = FgcmCalibrateTractTableConfig 

161 _DefaultName = "fgcmCalibrateTractTable" 

162 

163 canMultiprocess = False 

164 

165 def __init__(self, initInputs=None, **kwargs): 

166 super().__init__(initInputs=initInputs, **kwargs) 

167 if initInputs is not None: 

168 self.sourceSchema = initInputs["sourceSchema"].schema 

169 

170 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

171 handleDict = butlerQC.get(inputRefs) 

172 

173 self.log.info("Running with %d sourceTable_visit handles", (len(handleDict['source_catalogs']))) 

174 

175 # Run the build stars tasks 

176 tract = butlerQC.quantum.dataId['tract'] 

177 

178 handleDict['sourceSchema'] = self.sourceSchema 

179 

180 sourceTableHandles = handleDict['source_catalogs'] 

181 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

182 sourceTableHandle in sourceTableHandles} 

183 

184 visitSummaryHandles = handleDict['visitSummary'] 

185 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

186 visitSummaryHandle in visitSummaryHandles} 

187 

188 handleDict['sourceTableHandleDict'] = sourceTableHandleDict 

189 handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict 

190 

191 # And the outputs 

192 if self.config.fgcmOutputProducts.doZeropointOutput: 

193 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

194 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

195 handleDict['fgcmPhotoCalibs'] = photoCalibRefDict 

196 

197 if self.config.fgcmOutputProducts.doAtmosphereOutput: 

198 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

199 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

200 handleDict['fgcmTransmissionAtmospheres'] = atmRefDict 

201 

202 if self.config.fgcmBuildStars.doReferenceMatches: 

203 refConfig = LoadReferenceObjectsConfig() 

204 refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap 

205 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

206 for ref in inputRefs.refCat], 

207 refCats=butlerQC.get(inputRefs.refCat), 

208 name=self.config.connections.refCat, 

209 config=refConfig, 

210 log=self.log) 

211 buildStarsRefObjLoader = loader 

212 else: 

213 buildStarsRefObjLoader = None 

214 

215 if self.config.fgcmOutputProducts.doReferenceCalibration: 

216 refConfig = self.config.fgcmOutputProducts.refObjLoader 

217 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

218 for ref in inputRefs.refCat], 

219 refCats=butlerQC.get(inputRefs.refCat), 

220 name=self.config.connections.refCat, 

221 config=refConfig, 

222 log=self.log) 

223 self.fgcmOutputProducts.refObjLoader = loader 

224 

225 struct = self.run(handleDict, tract, 

226 buildStarsRefObjLoader=buildStarsRefObjLoader) 

227 

228 if struct.photoCalibCatalogs is not None: 

229 self.log.info("Outputting photoCalib catalogs.") 

230 for visit, expCatalog in struct.photoCalibCatalogs: 

231 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

232 self.log.info("Done outputting photoCalib catalogs.") 

233 

234 if struct.atmospheres is not None: 

235 self.log.info("Outputting atmosphere transmission files.") 

236 for visit, atm in struct.atmospheres: 

237 butlerQC.put(atm, atmRefDict[visit]) 

238 self.log.info("Done outputting atmosphere files.") 

239 

240 # Turn raw repeatability into simple catalog for persistence 

241 schema = afwTable.Schema() 

242 schema.addField('rawRepeatability', type=np.float64, 

243 doc="Per-band raw repeatability in FGCM calibration.") 

244 repeatabilityCat = afwTable.BaseCatalog(schema) 

245 repeatabilityCat.resize(len(struct.repeatability)) 

246 repeatabilityCat['rawRepeatability'][:] = struct.repeatability 

247 

248 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability) 

249 

250 return