Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM. 

24 

25This task finds all the visits and calexps in a repository (or a subset 

26based on command line parameters) and extract all the potential calibration 

27stars for input into fgcm. This task additionally uses fgcm to match 

28star observations into unique stars, and performs as much cleaning of 

29the input catalog as possible. 

30""" 

31 

32import time 

33 

34import numpy as np 

35 

36import lsst.pex.config as pexConfig 

37import lsst.pipe.base as pipeBase 

38import lsst.afw.table as afwTable 

39 

40from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask 

41from .utilities import computeApproxPixelAreaFields 

42 

43__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask'] 

44 

45 

46class FgcmBuildStarsConfig(FgcmBuildStarsConfigBase): 

47 """Config for FgcmBuildStarsTask""" 

48 

49 referenceCCD = pexConfig.Field( 

50 doc="Reference CCD for scanning visits", 

51 dtype=int, 

52 default=13, 

53 ) 

54 checkAllCcds = pexConfig.Field( 

55 doc=("Check repo for all CCDs for each visit specified. To be used when the " 

56 "full set of ids (visit/ccd) are not specified on the command line. For " 

57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly " 

58 "faster than the alternatives."), 

59 dtype=bool, 

60 default=True, 

61 ) 

62 

63 def setDefaults(self): 

64 super().setDefaults() 

65 

66 sourceSelector = self.sourceSelector["science"] 

67 

68 # The names here correspond to raw src catalogs, which differ 

69 # from the post-transformed sourceTable_visit catalogs. 

70 # Therefore, field and flag names cannot be easily 

71 # derived from the base config class. 

72 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

73 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge', 

74 'base_PixelFlags_flag_interpolatedCenter', 

75 'base_PixelFlags_flag_saturatedCenter', 

76 'base_PixelFlags_flag_crCenter', 

77 'base_PixelFlags_flag_bad', 

78 'base_PixelFlags_flag_interpolated', 

79 'base_PixelFlags_flag_saturated', 

80 'slot_Centroid_flag', 

81 fluxFlagName] 

82 

83 if self.doSubtractLocalBackground: 

84 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

85 sourceSelector.flags.bad.append(localBackgroundFlagName) 

86 

87 sourceSelector.signalToNoise.fluxField = self.instFluxField 

88 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

89 

90 

91class FgcmBuildStarsTask(FgcmBuildStarsBaseTask): 

92 """ 

93 Build stars for the FGCM global calibration, using src catalogs. 

94 """ 

95 ConfigClass = FgcmBuildStarsConfig 

96 RunnerClass = FgcmBuildStarsRunner 

97 _DefaultName = "fgcmBuildStars" 

98 

99 @classmethod 

100 def _makeArgumentParser(cls): 

101 """Create an argument parser""" 

102 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

103 parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789") 

104 

105 return parser 

106 

107 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs): 

108 self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName)) 

109 

110 ccdIds = [] 

111 for detector in camera: 

112 ccdIds.append(detector.getId()) 

113 

114 # TODO: related to DM-13730, this dance of looking for source visits 

115 # will be unnecessary with Gen3 Butler. This should be part of 

116 # DM-13730. 

117 

118 nVisits = 0 

119 

120 groupedDataRefs = {} 

121 for dataRef in dataRefs: 

122 visit = dataRef.dataId[self.config.visitDataRefName] 

123 # If we don't have the dataset, just continue 

124 if not dataRef.datasetExists(datasetType='src'): 

125 continue 

126 # If we need to check all ccds, do it here 

127 if self.config.checkAllCcds: 

128 if visit in groupedDataRefs: 

129 # We already have found this visit 

130 continue 

131 dataId = dataRef.dataId.copy() 

132 # For each ccd we must check that a valid source catalog exists. 

133 for ccdId in ccdIds: 

134 dataId[self.config.ccdDataRefName] = ccdId 

135 if butler.datasetExists('src', dataId=dataId): 

136 goodDataRef = butler.dataRef('src', dataId=dataId) 

137 if visit in groupedDataRefs: 

138 if (goodDataRef.dataId[self.config.ccdDataRefName] not in 

139 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]): 

140 groupedDataRefs[visit].append(goodDataRef) 

141 else: 

142 # This is a new visit 

143 nVisits += 1 

144 groupedDataRefs[visit] = [goodDataRef] 

145 else: 

146 # We have already confirmed that the dataset exists, so no need 

147 # to check here. 

148 if visit in groupedDataRefs: 

149 if (dataRef.dataId[self.config.ccdDataRefName] not in 

150 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]): 

151 groupedDataRefs[visit].append(dataRef) 

152 else: 

153 # This is a new visit 

154 nVisits += 1 

155 groupedDataRefs[visit] = [dataRef] 

156 

157 if (nVisits % 100) == 0 and nVisits > 0: 

158 self.log.info("Found %d unique %ss..." % (nVisits, 

159 self.config.visitDataRefName)) 

160 

161 self.log.info("Found %d unique %ss total." % (nVisits, 

162 self.config.visitDataRefName)) 

163 

164 # Put them in ccd order, with the reference ccd first (if available) 

165 def ccdSorter(dataRef): 

166 ccdId = dataRef.dataId[self.config.ccdDataRefName] 

167 if ccdId == self.config.referenceCCD: 

168 return -100 

169 else: 

170 return ccdId 

171 

172 # If we did not check all ccds, put them in ccd order 

173 if not self.config.checkAllCcds: 

174 for visit in groupedDataRefs: 

175 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter) 

176 

177 # This should be sorted by visit (the key) 

178 return dict(sorted(groupedDataRefs.items())) 

179 

180 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

181 srcSchemaDataRef, 

182 camera, 

183 calibFluxApertureRadius=None, 

184 visitCatDataRef=None, 

185 starObsDataRef=None, 

186 inStarObsCat=None): 

187 startTime = time.time() 

188 

189 # If both dataRefs are None, then we assume the caller does not 

190 # want to store checkpoint files. If both are set, we will 

191 # do checkpoint files. And if only one is set, this is potentially 

192 # unintentional and we will warn. 

193 if (visitCatDataRef is not None and starObsDataRef is None 

194 or visitCatDataRef is None and starObsDataRef is not None): 

195 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so " 

196 "no checkpoint files will be persisted.") 

197 

198 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

199 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

200 

201 # create our source schema. Use the first valid dataRef 

202 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0] 

203 sourceSchema = dataRef.get('src_schema', immediate=True).schema 

204 

205 # Construct a mapping from ccd number to index 

206 ccdMapping = {} 

207 for ccdIndex, detector in enumerate(camera): 

208 ccdMapping[detector.getId()] = ccdIndex 

209 

210 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

211 

212 sourceMapper = self._makeSourceMapper(sourceSchema) 

213 

214 # We also have a temporary catalog that will accumulate aperture measurements 

215 aperMapper = self._makeAperMapper(sourceSchema) 

216 

217 outputSchema = sourceMapper.getOutputSchema() 

218 

219 if inStarObsCat is not None: 

220 fullCatalog = inStarObsCat 

221 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS) 

222 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES) 

223 if not comp1 or not comp2: 

224 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.") 

225 else: 

226 fullCatalog = afwTable.BaseCatalog(outputSchema) 

227 

228 # FGCM will provide relative calibration for the flux in config.instFluxField 

229 

230 instFluxKey = sourceSchema[self.config.instFluxField].asKey() 

231 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey() 

232 visitKey = outputSchema['visit'].asKey() 

233 ccdKey = outputSchema['ccd'].asKey() 

234 instMagKey = outputSchema['instMag'].asKey() 

235 instMagErrKey = outputSchema['instMagErr'].asKey() 

236 deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey() 

237 

238 # Prepare local background if desired 

239 if self.config.doSubtractLocalBackground: 

240 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey() 

241 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

242 

243 aperOutputSchema = aperMapper.getOutputSchema() 

244 

245 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey() 

246 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey() 

247 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey() 

248 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey() 

249 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey() 

250 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey() 

251 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey() 

252 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey() 

253 

254 k = 2.5/np.log(10.) 

255 

256 # loop over visits 

257 for ctr, visit in enumerate(visitCat): 

258 if visit['sources_read']: 

259 continue 

260 

261 expTime = visit['exptime'] 

262 

263 nStarInVisit = 0 

264 

265 # Reset the aperture catalog (per visit) 

266 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema) 

267 

268 for dataRef in groupedDataRefs[visit['visit']]: 

269 

270 ccdId = dataRef.dataId[self.config.ccdDataRefName] 

271 

272 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) 

273 goodSrc = self.sourceSelector.selectSources(sources) 

274 

275 # Need to add a selection based on the local background correction 

276 # if necessary 

277 if self.config.doSubtractLocalBackground: 

278 localBackground = localBackgroundArea*sources[localBackgroundFluxKey] 

279 

280 bad, = np.where((sources[instFluxKey] - localBackground) <= 0.0) 

281 goodSrc.selected[bad] = False 

282 

283 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

284 tempCat.reserve(goodSrc.selected.sum()) 

285 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper) 

286 tempCat[visitKey][:] = visit['visit'] 

287 tempCat[ccdKey][:] = ccdId 

288 

289 # Compute "instrumental magnitude" by scaling flux with exposure time. 

290 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] 

291 * visit['scaling'][ccdMapping[ccdId]]) 

292 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

293 

294 # Compute the change in magnitude from the background offset 

295 if self.config.doSubtractLocalBackground: 

296 # At the moment we only adjust the flux and not the flux 

297 # error by the background because the error on 

298 # base_LocalBackground_instFlux is the rms error in the 

299 # background annulus, not the error on the mean in the 

300 # background estimate (which is much smaller, by sqrt(n) 

301 # pixels used to estimate the background, which we do not 

302 # have access to in this task). In the default settings, 

303 # the annulus is sufficiently large such that these 

304 # additional errors are are negligibly small (much less 

305 # than a mmag in quadrature). 

306 

307 # This is the difference between the mag with background correction 

308 # and the mag without background correction. 

309 tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected] 

310 - localBackground[goodSrc.selected]) - 

311 -2.5*np.log10(sources[instFluxKey][goodSrc.selected])) 

312 else: 

313 tempCat[deltaMagBkgKey][:] = 0.0 

314 

315 # Compute instMagErr from instFluxErr/instFlux, any scaling 

316 # will cancel out. 

317 

318 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] 

319 / sources[instFluxKey][goodSrc.selected]) 

320 

321 # Compute the jacobian from an approximate PixelAreaBoundedField 

322 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'], 

323 tempCat['y']) 

324 

325 # Apply the jacobian if configured 

326 if self.config.doApplyWcsJacobian: 

327 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

328 

329 fullCatalog.extend(tempCat) 

330 

331 # And the aperture information 

332 # This does not need the jacobian because it is all locally relative 

333 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema) 

334 tempAperCat.reserve(goodSrc.selected.sum()) 

335 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper) 

336 

337 with np.warnings.catch_warnings(): 

338 # Ignore warnings, we will filter infinities and 

339 # nans below. 

340 np.warnings.simplefilter("ignore") 

341 

342 tempAperCat[instMagInKey][:] = -2.5*np.log10( 

343 sources[instFluxAperInKey][goodSrc.selected]) 

344 tempAperCat[instMagErrInKey][:] = k*( 

345 sources[instFluxErrAperInKey][goodSrc.selected] 

346 / sources[instFluxAperInKey][goodSrc.selected]) 

347 tempAperCat[instMagOutKey][:] = -2.5*np.log10( 

348 sources[instFluxAperOutKey][goodSrc.selected]) 

349 tempAperCat[instMagErrOutKey][:] = k*( 

350 sources[instFluxErrAperOutKey][goodSrc.selected] 

351 / sources[instFluxAperOutKey][goodSrc.selected]) 

352 

353 aperVisitCatalog.extend(tempAperCat) 

354 

355 nStarInVisit += len(tempCat) 

356 

357 # Compute the median delta-aper 

358 if not aperVisitCatalog.isContiguous(): 

359 aperVisitCatalog = aperVisitCatalog.copy(deep=True) 

360 

361 instMagIn = aperVisitCatalog[instMagInKey] 

362 instMagErrIn = aperVisitCatalog[instMagErrInKey] 

363 instMagOut = aperVisitCatalog[instMagOutKey] 

364 instMagErrOut = aperVisitCatalog[instMagErrOutKey] 

365 

366 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) 

367 & np.isfinite(instMagOut) & np.isfinite(instMagErrOut)) 

368 

369 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok]) 

370 visit['sources_read'] = True 

371 

372 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" % 

373 (nStarInVisit, visit['visit'], visit['deltaAper'])) 

374 

375 if ((ctr % self.config.nVisitsPerCheckpoint) == 0 

376 and starObsDataRef is not None and visitCatDataRef is not None): 

377 # We need to persist both the stars and the visit catalog which gets 

378 # additional metadata from each visit. 

379 starObsDataRef.put(fullCatalog) 

380 visitCatDataRef.put(visitCat) 

381 

382 self.log.info("Found all good star observations in %.2f s" % 

383 (time.time() - startTime)) 

384 

385 return fullCatalog 

386 

387 def _makeAperMapper(self, sourceSchema): 

388 """ 

389 Make a schema mapper for fgcm aperture measurements 

390 

391 Parameters 

392 ---------- 

393 sourceSchema: `afwTable.Schema` 

394 Default source schema from the butler 

395 

396 Returns 

397 ------- 

398 aperMapper: `afwTable.schemaMapper` 

399 Mapper to the FGCM aperture schema 

400 """ 

401 

402 aperMapper = afwTable.SchemaMapper(sourceSchema) 

403 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

404 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

405 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64, 

406 doc="Magnitude at inner aperture") 

407 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64, 

408 doc="Magnitude error at inner aperture") 

409 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64, 

410 doc="Magnitude at outer aperture") 

411 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64, 

412 doc="Magnitude error at outer aperture") 

413 

414 return aperMapper