Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM. 

24 

25This task finds all the visits and calexps in a repository (or a subset 

26based on command line parameters) and extract all the potential calibration 

27stars for input into fgcm. This task additionally uses fgcm to match 

28star observations into unique stars, and performs as much cleaning of 

29the input catalog as possible. 

30""" 

31 

32import time 

33 

34import numpy as np 

35 

36import lsst.pex.config as pexConfig 

37import lsst.pipe.base as pipeBase 

38import lsst.afw.table as afwTable 

39 

40from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask 

41from .utilities import computeApproxPixelAreaFields 

42 

43__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask'] 

44 

45 

46class FgcmBuildStarsConfig(FgcmBuildStarsConfigBase): 

47 """Config for FgcmBuildStarsTask""" 

48 

49 referenceCCD = pexConfig.Field( 

50 doc="Reference CCD for scanning visits", 

51 dtype=int, 

52 default=13, 

53 ) 

54 checkAllCcds = pexConfig.Field( 

55 doc=("Check repo for all CCDs for each visit specified. To be used when the " 

56 "full set of ids (visit/ccd) are not specified on the command line. For " 

57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly " 

58 "faster than the alternatives."), 

59 dtype=bool, 

60 default=True, 

61 ) 

62 

63 def setDefaults(self): 

64 super().setDefaults() 

65 

66 sourceSelector = self.sourceSelector["science"] 

67 

68 # The names here correspond to raw src catalogs, which differ 

69 # from the post-transformed sourceTable_visit catalogs. 

70 # Therefore, field and flag names cannot be easily 

71 # derived from the base config class. 

72 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

73 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge', 

74 'base_PixelFlags_flag_interpolatedCenter', 

75 'base_PixelFlags_flag_saturatedCenter', 

76 'base_PixelFlags_flag_crCenter', 

77 'base_PixelFlags_flag_bad', 

78 'base_PixelFlags_flag_interpolated', 

79 'base_PixelFlags_flag_saturated', 

80 'slot_Centroid_flag', 

81 fluxFlagName] 

82 

83 if self.doSubtractLocalBackground: 

84 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

85 sourceSelector.flags.bad.append(localBackgroundFlagName) 

86 

87 sourceSelector.signalToNoise.fluxField = self.instFluxField 

88 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

89 

90 

91class FgcmBuildStarsTask(FgcmBuildStarsBaseTask): 

92 """ 

93 Build stars for the FGCM global calibration, using src catalogs. 

94 """ 

95 ConfigClass = FgcmBuildStarsConfig 

96 RunnerClass = FgcmBuildStarsRunner 

97 _DefaultName = "fgcmBuildStars" 

98 

99 @classmethod 

100 def _makeArgumentParser(cls): 

101 """Create an argument parser""" 

102 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

103 parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789") 

104 

105 return parser 

106 

107 def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None): 

108 if butler is None: 

109 raise RuntimeError("Gen2 _findAndGroupDataRefs must be called with a butler.") 

110 if calexpDataRefDict is not None: 

111 self.log.warn("Ignoring calexpDataRefDict in gen2 _findAndGroupDataRefs") 

112 

113 self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName)) 

114 

115 ccdIds = [] 

116 for detector in camera: 

117 ccdIds.append(detector.getId()) 

118 

119 # TODO: related to DM-13730, this dance of looking for source visits 

120 # will be unnecessary with Gen3 Butler. This should be part of 

121 # DM-13730. 

122 

123 nVisits = 0 

124 

125 groupedDataRefs = {} 

126 for dataRef in dataRefs: 

127 visit = dataRef.dataId[self.config.visitDataRefName] 

128 # If we don't have the dataset, just continue 

129 if not dataRef.datasetExists(datasetType='src'): 

130 continue 

131 # If we need to check all ccds, do it here 

132 if self.config.checkAllCcds: 

133 if visit in groupedDataRefs: 

134 # We already have found this visit 

135 continue 

136 dataId = dataRef.dataId.copy() 

137 # For each ccd we must check that a valid source catalog exists. 

138 for ccdId in ccdIds: 

139 dataId[self.config.ccdDataRefName] = ccdId 

140 if butler.datasetExists('src', dataId=dataId): 

141 goodDataRef = butler.dataRef('src', dataId=dataId) 

142 if visit in groupedDataRefs: 

143 if (goodDataRef.dataId[self.config.ccdDataRefName] not in 

144 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]): 

145 groupedDataRefs[visit].append(goodDataRef) 

146 else: 

147 # This is a new visit 

148 nVisits += 1 

149 groupedDataRefs[visit] = [goodDataRef] 

150 else: 

151 # We have already confirmed that the dataset exists, so no need 

152 # to check here. 

153 if visit in groupedDataRefs: 

154 if (dataRef.dataId[self.config.ccdDataRefName] not in 

155 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]): 

156 groupedDataRefs[visit].append(dataRef) 

157 else: 

158 # This is a new visit 

159 nVisits += 1 

160 groupedDataRefs[visit] = [dataRef] 

161 

162 if (nVisits % 100) == 0 and nVisits > 0: 

163 self.log.info("Found %d unique %ss..." % (nVisits, 

164 self.config.visitDataRefName)) 

165 

166 self.log.info("Found %d unique %ss total." % (nVisits, 

167 self.config.visitDataRefName)) 

168 

169 # Put them in ccd order, with the reference ccd first (if available) 

170 def ccdSorter(dataRef): 

171 ccdId = dataRef.dataId[self.config.ccdDataRefName] 

172 if ccdId == self.config.referenceCCD: 

173 return -100 

174 else: 

175 return ccdId 

176 

177 # If we did not check all ccds, put them in ccd order 

178 if not self.config.checkAllCcds: 

179 for visit in groupedDataRefs: 

180 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter) 

181 

182 # This should be sorted by visit (the key) 

183 return dict(sorted(groupedDataRefs.items())) 

184 

185 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

186 srcSchemaDataRef, 

187 camera, 

188 calibFluxApertureRadius=None, 

189 visitCatDataRef=None, 

190 starObsDataRef=None, 

191 inStarObsCat=None): 

192 startTime = time.time() 

193 

194 # If both dataRefs are None, then we assume the caller does not 

195 # want to store checkpoint files. If both are set, we will 

196 # do checkpoint files. And if only one is set, this is potentially 

197 # unintentional and we will warn. 

198 if (visitCatDataRef is not None and starObsDataRef is None 

199 or visitCatDataRef is None and starObsDataRef is not None): 

200 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so " 

201 "no checkpoint files will be persisted.") 

202 

203 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

204 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

205 

206 # create our source schema. Use the first valid dataRef 

207 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0] 

208 sourceSchema = dataRef.get('src_schema', immediate=True).schema 

209 

210 # Construct a mapping from ccd number to index 

211 ccdMapping = {} 

212 for ccdIndex, detector in enumerate(camera): 

213 ccdMapping[detector.getId()] = ccdIndex 

214 

215 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

216 

217 sourceMapper = self._makeSourceMapper(sourceSchema) 

218 

219 # We also have a temporary catalog that will accumulate aperture measurements 

220 aperMapper = self._makeAperMapper(sourceSchema) 

221 

222 outputSchema = sourceMapper.getOutputSchema() 

223 

224 if inStarObsCat is not None: 

225 fullCatalog = inStarObsCat 

226 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS) 

227 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES) 

228 if not comp1 or not comp2: 

229 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.") 

230 else: 

231 fullCatalog = afwTable.BaseCatalog(outputSchema) 

232 

233 # FGCM will provide relative calibration for the flux in config.instFluxField 

234 

235 instFluxKey = sourceSchema[self.config.instFluxField].asKey() 

236 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey() 

237 visitKey = outputSchema['visit'].asKey() 

238 ccdKey = outputSchema['ccd'].asKey() 

239 instMagKey = outputSchema['instMag'].asKey() 

240 instMagErrKey = outputSchema['instMagErr'].asKey() 

241 deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey() 

242 

243 # Prepare local background if desired 

244 if self.config.doSubtractLocalBackground: 

245 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey() 

246 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

247 

248 aperOutputSchema = aperMapper.getOutputSchema() 

249 

250 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey() 

251 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey() 

252 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey() 

253 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey() 

254 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey() 

255 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey() 

256 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey() 

257 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey() 

258 

259 k = 2.5/np.log(10.) 

260 

261 # loop over visits 

262 for ctr, visit in enumerate(visitCat): 

263 if visit['sources_read']: 

264 continue 

265 

266 expTime = visit['exptime'] 

267 

268 nStarInVisit = 0 

269 

270 # Reset the aperture catalog (per visit) 

271 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema) 

272 

273 for dataRef in groupedDataRefs[visit['visit']]: 

274 

275 ccdId = dataRef.dataId[self.config.ccdDataRefName] 

276 

277 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) 

278 goodSrc = self.sourceSelector.selectSources(sources) 

279 

280 # Need to add a selection based on the local background correction 

281 # if necessary 

282 if self.config.doSubtractLocalBackground: 

283 localBackground = localBackgroundArea*sources[localBackgroundFluxKey] 

284 

285 bad, = np.where((sources[instFluxKey] - localBackground) <= 0.0) 

286 goodSrc.selected[bad] = False 

287 

288 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

289 tempCat.reserve(goodSrc.selected.sum()) 

290 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper) 

291 tempCat[visitKey][:] = visit['visit'] 

292 tempCat[ccdKey][:] = ccdId 

293 

294 # Compute "instrumental magnitude" by scaling flux with exposure time. 

295 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] 

296 * visit['scaling'][ccdMapping[ccdId]]) 

297 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

298 

299 # Compute the change in magnitude from the background offset 

300 if self.config.doSubtractLocalBackground: 

301 # At the moment we only adjust the flux and not the flux 

302 # error by the background because the error on 

303 # base_LocalBackground_instFlux is the rms error in the 

304 # background annulus, not the error on the mean in the 

305 # background estimate (which is much smaller, by sqrt(n) 

306 # pixels used to estimate the background, which we do not 

307 # have access to in this task). In the default settings, 

308 # the annulus is sufficiently large such that these 

309 # additional errors are are negligibly small (much less 

310 # than a mmag in quadrature). 

311 

312 # This is the difference between the mag with background correction 

313 # and the mag without background correction. 

314 tempCat[deltaMagBkgKey][:] = (-2.5*np.log10(sources[instFluxKey][goodSrc.selected] 

315 - localBackground[goodSrc.selected]) - 

316 -2.5*np.log10(sources[instFluxKey][goodSrc.selected])) 

317 else: 

318 tempCat[deltaMagBkgKey][:] = 0.0 

319 

320 # Compute instMagErr from instFluxErr/instFlux, any scaling 

321 # will cancel out. 

322 

323 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] 

324 / sources[instFluxKey][goodSrc.selected]) 

325 

326 # Compute the jacobian from an approximate PixelAreaBoundedField 

327 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'], 

328 tempCat['y']) 

329 

330 # Apply the jacobian if configured 

331 if self.config.doApplyWcsJacobian: 

332 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

333 

334 fullCatalog.extend(tempCat) 

335 

336 # And the aperture information 

337 # This does not need the jacobian because it is all locally relative 

338 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema) 

339 tempAperCat.reserve(goodSrc.selected.sum()) 

340 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper) 

341 

342 with np.warnings.catch_warnings(): 

343 # Ignore warnings, we will filter infinities and 

344 # nans below. 

345 np.warnings.simplefilter("ignore") 

346 

347 tempAperCat[instMagInKey][:] = -2.5*np.log10( 

348 sources[instFluxAperInKey][goodSrc.selected]) 

349 tempAperCat[instMagErrInKey][:] = k*( 

350 sources[instFluxErrAperInKey][goodSrc.selected] 

351 / sources[instFluxAperInKey][goodSrc.selected]) 

352 tempAperCat[instMagOutKey][:] = -2.5*np.log10( 

353 sources[instFluxAperOutKey][goodSrc.selected]) 

354 tempAperCat[instMagErrOutKey][:] = k*( 

355 sources[instFluxErrAperOutKey][goodSrc.selected] 

356 / sources[instFluxAperOutKey][goodSrc.selected]) 

357 

358 aperVisitCatalog.extend(tempAperCat) 

359 

360 nStarInVisit += len(tempCat) 

361 

362 # Compute the median delta-aper 

363 if not aperVisitCatalog.isContiguous(): 

364 aperVisitCatalog = aperVisitCatalog.copy(deep=True) 

365 

366 instMagIn = aperVisitCatalog[instMagInKey] 

367 instMagErrIn = aperVisitCatalog[instMagErrInKey] 

368 instMagOut = aperVisitCatalog[instMagOutKey] 

369 instMagErrOut = aperVisitCatalog[instMagErrOutKey] 

370 

371 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) 

372 & np.isfinite(instMagOut) & np.isfinite(instMagErrOut)) 

373 

374 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok]) 

375 visit['sources_read'] = True 

376 

377 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" % 

378 (nStarInVisit, visit['visit'], visit['deltaAper'])) 

379 

380 if ((ctr % self.config.nVisitsPerCheckpoint) == 0 

381 and starObsDataRef is not None and visitCatDataRef is not None): 

382 # We need to persist both the stars and the visit catalog which gets 

383 # additional metadata from each visit. 

384 starObsDataRef.put(fullCatalog) 

385 visitCatDataRef.put(visitCat) 

386 

387 self.log.info("Found all good star observations in %.2f s" % 

388 (time.time() - startTime)) 

389 

390 return fullCatalog 

391 

392 def _makeAperMapper(self, sourceSchema): 

393 """ 

394 Make a schema mapper for fgcm aperture measurements 

395 

396 Parameters 

397 ---------- 

398 sourceSchema: `afwTable.Schema` 

399 Default source schema from the butler 

400 

401 Returns 

402 ------- 

403 aperMapper: `afwTable.schemaMapper` 

404 Mapper to the FGCM aperture schema 

405 """ 

406 

407 aperMapper = afwTable.SchemaMapper(sourceSchema) 

408 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

409 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

410 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64, 

411 doc="Magnitude at inner aperture") 

412 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64, 

413 doc="Magnitude error at inner aperture") 

414 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64, 

415 doc="Magnitude at outer aperture") 

416 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64, 

417 doc="Magnitude error at outer aperture") 

418 

419 return aperMapper