Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Class for running fgcmcal on a single tract using sourceTable_visit tables. 

24""" 

25import numpy as np 

26 

27import lsst.pipe.base as pipeBase 

28from lsst.pipe.base import connectionTypes 

29from lsst.meas.algorithms import ReferenceObjectLoader 

30import lsst.afw.table as afwTable 

31 

32from .dataIds import TractCheckDataIdContainer 

33from .fgcmBuildStarsTable import FgcmBuildStarsTableTask 

34from .fgcmCalibrateTractBase import (FgcmCalibrateTractConfigBase, FgcmCalibrateTractRunner, 

35 FgcmCalibrateTractBaseTask) 

36from .utilities import lookupStaticCalibrations 

37 

38__all__ = ['FgcmCalibrateTractTableConfig', 'FgcmCalibrateTractTableTask'] 

39 

40 

41class FgcmCalibrateTractTableConnections(pipeBase.PipelineTaskConnections, 

42 dimensions=("instrument", 

43 "tract",)): 

44 camera = connectionTypes.PrerequisiteInput( 

45 doc="Camera instrument", 

46 name="camera", 

47 storageClass="Camera", 

48 dimensions=("instrument",), 

49 lookupFunction=lookupStaticCalibrations, 

50 isCalibration=True, 

51 ) 

52 

53 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

54 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

55 "chromatic corrections."), 

56 name="fgcmLookUpTable", 

57 storageClass="Catalog", 

58 dimensions=("instrument",), 

59 deferLoad=True, 

60 ) 

61 

62 sourceSchema = connectionTypes.InitInput( 

63 doc="Schema for source catalogs", 

64 name="src_schema", 

65 storageClass="SourceCatalog", 

66 ) 

67 

68 refCat = connectionTypes.PrerequisiteInput( 

69 doc="Reference catalog to use for photometric calibration", 

70 name="cal_ref_cat", 

71 storageClass="SimpleCatalog", 

72 dimensions=("skypix",), 

73 deferLoad=True, 

74 multiple=True, 

75 ) 

76 

77 source_catalogs = connectionTypes.Input( 

78 doc="Source table in parquet format, per visit", 

79 name="sourceTable_visit", 

80 storageClass="DataFrame", 

81 dimensions=("instrument", "visit"), 

82 deferLoad=True, 

83 multiple=True, 

84 ) 

85 

86 visitSummary = connectionTypes.Input( 

87 doc="Per-visit summary statistics table", 

88 name="visitSummary", 

89 storageClass="ExposureCatalog", 

90 dimensions=("instrument", "visit"), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 background = connectionTypes.Input( 

96 doc="Calexp background model", 

97 name="calexpBackground", 

98 storageClass="Background", 

99 dimensions=("instrument", "visit", "detector"), 

100 deferLoad=True, 

101 multiple=True, 

102 ) 

103 

104 fgcmPhotoCalib = connectionTypes.Output( 

105 doc="Per-tract, per-visit photoCalib exposure catalogs produced from fgcm calibration", 

106 name="fgcmPhotoCalibTractCatalog", 

107 storageClass="ExposureCatalog", 

108 dimensions=("instrument", "tract", "visit",), 

109 multiple=True, 

110 ) 

111 

112 fgcmTransmissionAtmosphere = connectionTypes.Output( 

113 doc="Per-visit atmosphere transmission files produced from fgcm calibration", 

114 name="transmission_atmosphere_fgcm_tract", 

115 storageClass="TransmissionCurve", 

116 dimensions=("instrument", "tract", "visit",), 

117 multiple=True, 

118 ) 

119 

120 fgcmRepeatability = connectionTypes.Output( 

121 doc="Per-band raw repeatability numbers in the fgcm tract calibration", 

122 name="fgcmRawRepeatability", 

123 storageClass="Catalog", 

124 dimensions=("instrument", "tract",), 

125 multiple=False, 

126 ) 

127 

128 def __init__(self, *, config=None): 

129 super().__init__(config=config) 

130 

131 # The ref_dataset_name will be deprecated with Gen2 

132 loaderName = config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name 

133 if config.connections.refCat != loaderName: 

134 raise ValueError("connections.refCat must be the same as " 

135 "config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader.ref_dataset_name") 

136 if config.fgcmOutputProducts.doReferenceCalibration: 

137 loaderName = config.fgcmOutputProducts.refObjLoader.ref_dataset_name 

138 if config.connections.refCat != loaderName: 

139 raise ValueError("connections.refCat must be the same as " 

140 "config.fgcmOutputProducts.refObjLoader.ref_dataset_name") 

141 

142 if not config.fgcmBuildStars.doModelErrorsWithBackground: 

143 self.inputs.remove("background") 

144 

145 if config.fgcmOutputProducts.doRefcatOutput: 

146 raise ValueError("FgcmCalibrateTractTableTask (Gen3) does not support doRefcatOutput") 

147 if not config.fgcmOutputProducts.doAtmosphereOutput: 

148 self.prerequisiteInputs.remove("fgcmAtmosphereParameters") 

149 if not config.fgcmOutputProducts.doZeropointOutput: 

150 self.prerequisiteInputs.remove("fgcmZeropoints") 

151 

152 

153class FgcmCalibrateTractTableConfig(FgcmCalibrateTractConfigBase, pipeBase.PipelineTaskConfig, 

154 pipelineConnections=FgcmCalibrateTractTableConnections): 

155 """Config for FgcmCalibrateTractTable task""" 

156 def setDefaults(self): 

157 super().setDefaults() 

158 

159 # For the Table version of CalibrateTract, use the associated 

160 # Table version of the BuildStars task. 

161 self.fgcmBuildStars.retarget(FgcmBuildStarsTableTask) 

162 # For tract mode, we set a very high effective density cut. 

163 self.fgcmBuildStars.densityCutMaxPerPixel = 10000 

164 

165 

166class FgcmCalibrateTractTableTask(FgcmCalibrateTractBaseTask): 

167 """ 

168 Calibrate a single tract using fgcmcal, using sourceTable_visit (parquet) 

169 input catalogs. 

170 """ 

171 ConfigClass = FgcmCalibrateTractTableConfig 

172 RunnerClass = FgcmCalibrateTractRunner 

173 _DefaultName = "fgcmCalibrateTractTable" 

174 

175 canMultiprocess = False 

176 

177 def __init__(self, initInputs=None, **kwargs): 

178 super().__init__(initInputs=initInputs, **kwargs) 

179 if initInputs is not None: 

180 self.sourceSchema = initInputs["sourceSchema"].schema 

181 

182 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

183 dataRefDict = butlerQC.get(inputRefs) 

184 

185 self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefDict['source_catalogs']))) 

186 

187 # Run the build stars tasks 

188 tract = butlerQC.quantum.dataId['tract'] 

189 

190 dataRefDict['sourceSchema'] = self.sourceSchema 

191 

192 sourceTableRefs = dataRefDict['source_catalogs'] 

193 sourceTableDataRefDict = {sourceTableRef.dataId['visit']: sourceTableRef for 

194 sourceTableRef in sourceTableRefs} 

195 

196 visitSummaryRefs = dataRefDict['visitSummary'] 

197 visitSummaryDataRefDict = {visitSummaryRef.dataId['visit']: visitSummaryRef for 

198 visitSummaryRef in visitSummaryRefs} 

199 

200 dataRefDict['sourceTableDataRefDict'] = sourceTableDataRefDict 

201 dataRefDict['visitSummaryDataRefDict'] = visitSummaryDataRefDict 

202 

203 # And the outputs 

204 if self.config.fgcmOutputProducts.doZeropointOutput: 

205 photoCalibRefDict = {photoCalibRef.dataId.byName()['visit']: 

206 photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib} 

207 dataRefDict['fgcmPhotoCalibs'] = photoCalibRefDict 

208 

209 if self.config.fgcmOutputProducts.doAtmosphereOutput: 

210 atmRefDict = {atmRef.dataId.byName()['visit']: atmRef for 

211 atmRef in outputRefs.fgcmTransmissionAtmosphere} 

212 dataRefDict['fgcmTransmissionAtmospheres'] = atmRefDict 

213 

214 if self.config.fgcmBuildStars.doReferenceMatches: 

215 refConfig = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.refObjLoader 

216 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

217 for ref in inputRefs.refCat], 

218 refCats=butlerQC.get(inputRefs.refCat), 

219 config=refConfig, 

220 log=self.log) 

221 buildStarsRefObjLoader = loader 

222 else: 

223 buildStarsRefObjLoader = None 

224 

225 if self.config.fgcmOutputProducts.doReferenceCalibration: 

226 refConfig = self.config.fgcmOutputProducts.refObjLoader 

227 loader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

228 for ref in inputRefs.refCat], 

229 refCats=butlerQC.get(inputRefs.refCat), 

230 config=refConfig, 

231 log=self.log) 

232 self.fgcmOutputProducts.refObjLoader = loader 

233 

234 struct = self.run(dataRefDict, tract, 

235 buildStarsRefObjLoader=buildStarsRefObjLoader) 

236 

237 if struct.photoCalibCatalogs is not None: 

238 self.log.info("Outputting photoCalib catalogs.") 

239 for visit, expCatalog in struct.photoCalibCatalogs: 

240 butlerQC.put(expCatalog, photoCalibRefDict[visit]) 

241 self.log.info("Done outputting photoCalib catalogs.") 

242 

243 if struct.atmospheres is not None: 

244 self.log.info("Outputting atmosphere transmission files.") 

245 for visit, atm in struct.atmospheres: 

246 butlerQC.put(atm, atmRefDict[visit]) 

247 self.log.info("Done outputting atmosphere files.") 

248 

249 # Turn raw repeatability into simple catalog for persistence 

250 schema = afwTable.Schema() 

251 schema.addField('rawRepeatability', type=np.float64, 

252 doc="Per-band raw repeatability in FGCM calibration.") 

253 repeatabilityCat = afwTable.BaseCatalog(schema) 

254 repeatabilityCat.resize(len(struct.repeatability)) 

255 repeatabilityCat['rawRepeatability'][:] = struct.repeatability 

256 

257 butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability) 

258 

259 return 

260 

261 @classmethod 

262 def _makeArgumentParser(cls): 

263 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

264 parser.add_id_argument("--id", "sourceTable_visit", 

265 help="Data ID, e.g. --id visit=6789 tract=9617", 

266 ContainerClass=TractCheckDataIdContainer) 

267 

268 return parser