Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Class for running fgcmcal on a single tract using sourceTable_visit tables. 

24""" 

25import numpy as np 

26 

27import lsst.pipe.base as pipeBase 

28from lsst.pipe.base import connectionTypes 

29from lsst.meas.algorithms import ReferenceObjectLoader 

30import lsst.afw.table as afwTable 

31 

32from .dataIds import TractCheckDataIdContainer 

33from .fgcmBuildStarsTable import FgcmBuildStarsTableTask 

34from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, FgcmCalibrateTractRunner, 

35 FgcmCalibrateTractBaseTask) 

36from .utilities import lookupStaticCalibrations 

37 

38__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask'] 

39 

40 

41class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections, 

42 dimensions=("instrument", 

43 "tract",)): 

44 camera = connectionTypes.PrerequisiteInput( 

45 doc="Camera instrument", 

46 name="camera", 

47 storageClass="Camera", 

48 dimensions=("instrument",), 

49 lookupFunction=lookupStaticCalibrations, 

50 isCalibration=True, 

51 ) 

52 

53 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

54 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

55 "chromatic corrections."), 

56 name="fgcmLookUpTable", 

57 storageClass="Catalog", 

58 dimensions=("instrument",), 

59 deferLoad=True, 

60 ) 

61 

62 sourceSchema = connectionTypes.PrerequisiteInput( 

63 doc="Schema for source catalogs", 

64 name="src_schema", 

65 storageClass="SourceCatalog", 

66 deferLoad=True, 

67 ) 

68 

69 refCat = connectionTypes.PrerequisiteInput( 

70 doc="Reference catalog to use for photometric calibration", 

71 name="cal_ref_cat", 

72 storageClass="SimpleCatalog", 

73 dimensions=("skypix",), 

74 deferLoad=True, 

75 multiple=True, 

76 ) 

77 

78 source_catalogs = connectionTypes.Input( 

79 doc="Source table in parquet format, per visit", 

80 name="sourceTable_visit", 

81 storageClass="DataFrame", 

82 dimensions=("instrument", "visit"), 

83 deferLoad=True, 

84 multiple=True, 

85 ) 

86 

87 visitSummary = connectionTypes.Input( 

88 doc="Per-visit summary statistics table", 

89 name="visitSummary", 

90 storageClass="ExposureCatalog", 

91 dimensions=("instrument", "visit"), 

92 deferLoad=True, 

93 multiple=True, 

94 ) 

95 

96 background = connectionTypes.Input( 

97 doc="Calexp background model", 

98 name="calexpBackground", 

99 storageClass="Background", 

100 dimensions=("instrument", "visit", "detector"), 

101 deferLoad=True, 

102 multiple=True, 

103 ) 

104 

105 fgcmPhotoCalib = connectionTypes.Output( 

106 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration", 

107 name="fgcmPhotoCalibTractCatalog", 

108 storageClass="ExposureCatalog", 

109 dimensions=("instrument", "tract", "visit",), 

110 multiple=True, 

111 ) 

112 

113 fgcmTransmissionAtmosphere = connectionTypes.Output( 

114 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

115 name="transmission_atmosphere_fgcm_tract", 

116 storageClass="TransmissionCurve", 

117 dimensions=("instrument", "tract", "visit",), 

118 multiple=True, 

119 ) 

120 

121 fgcmRepeatability = connectionTypes.Output( 

122 doc="Per-band raw repeatability numbers in the fgcm tract calibration", 

123 name="fgcmRawRepeatability", 

124 storageClass="Catalog", 

125 dimensions=("instrument", "tract",), 

126 multiple=False, 

127 ) 

128 

129 def __init__(self, *, config=None): 

130 super().__init__(config=config) 

131 

132 # The ref_dataset_name will be deprecated with Gen2 

133 loaderName = config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name 

134 if config.connections.refCat != loaderName: 

135 raise ValueError("connections.refCat must be the same as " 

136 "config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name") 

137 if config.fgcmOutputProducts.doReferenceCalibration: 

138 loaderName = config.fgcmOutputProducts.refObjLoader.ref_dataset_name 

139 if config.connections.refCat != loaderName: 

140 raise ValueError("connections.refCat must be the same as " 

141 "config.fgcmOutputProducts.refObjLoader.ref_dataset_name") 

142 

143 if not config.fgcmBuildStars.doModelErrorsWithBackground: 

144 self.inputs.remove("background") 

145 

146 if config.fgcmOutputProducts.doRefcatOutput: 

147 raise ValueError("FgcmCalibrateTractTableTask (Gen3) does not support doRefcatOutput") 

148 if not config.fgcmOutputProducts.doAtmosphereOutput: 

149 self.prerequisiteInputs.remove("fgcmAtmosphereParameters") 

150 if not config.fgcmOutputProducts.doZeropointOutput: 

151 self.prerequisiteInputs.remove("fgcmZeropoints") 

152 

153 

154class FgcmCalibrateTractTableConfig(FgcmCalibrateTractConfigBase, pipeBase.PipelineTaskConfig, 

155 pipelineConnections=FgcmCalibrateTractTableConnections): 

156 """Config for FgcmCalibrateTractTable task""" 

157 def setDefaults(self): 

158 super().setDefaults() 

159 

160 # For the Table version of CalibrateTract, use the associated 

161 # Table version of the BuildStars task. 

162 self.fgcmBuildStars.retarget(FgcmBuildStarsTableTask) 

163 # For tract mode, we set a very high effective density cut. 

164 self.fgcmBuildStars.densityCutMaxPerPixel = 10000 

165 

166 

167class FgcmCalibrateTractTableTask(FgcmCalibrateTractBaseTask): 

168 """ 

169 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet) 

170 input catalogs. 

171 """ 

172 ConfigClass = FgcmCalibrateTractTableConfig 

173 RunnerClass = FgcmCalibrateTractRunner 

174 _DefaultName = "fgcmCalibrateTractTable" 

175 

176 canMultiprocess = False 

177 

178 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

179 dataRefDict = butlerQC.get(inputRefs) 

180 

181 self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefDict['source_catalogs']))) 

182 

183 # Run the build stars tasks 

184 tract = butlerQC.quantum.dataId['tract'] 

185 

186 sourceTableRefs = dataRefDict['source_catalogs'] 

187 sourceTableDataRefDict = {sourceTableRef.dataId['visit']: sourceTableRef for 

188 sourceTableRef in sourceTableRefs} 

189 

190 visitSummaryRefs = dataRefDict['visitSummary'] 

191 visitSummaryDataRefDict = {visitSummaryRef.dataId['visit']: visitSummaryRef for 

192 visitSummaryRef in visitSummaryRefs} 

193 

194 dataRefDict['sourceTableDataRefDict'] = sourceTableDataRefDict 

195 dataRefDict['visitSummaryDataRefDict'] = visitSummaryDataRefDict 

196 

197 # And the outputs 

198 if self.config.fgcmOutputProducts.doZeropointOutput: 

199 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

200 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

201 dataRefDict['fgcmPhotoCalibs'] = photoCalibRefDict 

202 

203 if self.config.fgcmOutputProducts.doAtmosphereOutput: 

204 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

205 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

206 dataRefDict['fgcmTransmissionAtmospheres'] = atmRefDict 

207 

208 if self.config.fgcmBuildStars.doReferenceMatches: 

209 refConfig = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader 

210 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

211 for ref in inputRefs.refCat], 

212 refCats=butlerQC.get(inputRefs.refCat), 

213 config=refConfig, 

214 log=self.log) 

215 buildStarsRefObjLoader = loader 

216 else: 

217 buildStarsRefObjLoader = None 

218 

219 if self.config.fgcmOutputProducts.doReferenceCalibration: 

220 refConfig = self.config.fgcmOutputProducts.refObjLoader 

221 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

222 for ref in inputRefs.refCat], 

223 refCats=butlerQC.get(inputRefs.refCat), 

224 config=refConfig, 

225 log=self.log) 

226 self.fgcmOutputProducts.refObjLoader = loader 

227 

228 struct = self.run(dataRefDict, tract, 

229 buildStarsRefObjLoader=buildStarsRefObjLoader) 

230 

231 if struct.photoCalibCatalogs is not None: 

232 self.log.info("Outputting photoCalib catalogs.") 

233 for visit, expCatalog in struct.photoCalibCatalogs: 

234 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

235 self.log.info("Done outputting photoCalib catalogs.") 

236 

237 if struct.atmospheres is not None: 

238 self.log.info("Outputting atmosphere transmission files.") 

239 for visit, atm in struct.atmospheres: 

240 butlerQC.put(atm, atmRefDict[visit]) 

241 self.log.info("Done outputting atmosphere files.") 

242 

243 # Turn raw repeatability into simple catalog for persistence 

244 schema = afwTable.Schema() 

245 schema.addField('rawRepeatability', type=np.float64, 

246 doc="Per-band raw repeatability in FGCM calibration.") 

247 repeatabilityCat = afwTable.BaseCatalog(schema) 

248 repeatabilityCat.resize(len(struct.repeatability)) 

249 repeatabilityCat['rawRepeatability'][:] = struct.repeatability 

250 

251 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability) 

252 

253 return 

254 

255 @classmethod 

256 def _makeArgumentParser(cls): 

257 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

258 parser.add_id_argument("--id", "sourceTable_visit", 

259 help="Data ID, e.g. --id visit=6789 tract=9617", 

260 ContainerClass=TractCheckDataIdContainer) 

261 

262 return parser