Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM. 

24 

25This task finds all the visits and calexps in a repository (or a subset 

26based on command line parameters) and extract all the potential calibration 

27stars for input into fgcm. This task additionally uses fgcm to match 

28star observations into unique stars, and performs as much cleaning of 

29the input catalog as possible. 

30""" 

31 

32import time 

33 

34import numpy as np 

35 

36import lsst.pex.config as pexConfig 

37import lsst.pipe.base as pipeBase 

38import lsst.afw.table as afwTable 

39 

40from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask 

41from .utilities import computeApproxPixelAreaFields 

42 

43__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask'] 

44 

45 

46class FgcmBuildStarsConfig(FgcmBuildStarsConfigBase): 

47 """Config for FgcmBuildStarsTask""" 

48 

49 referenceCCD = pexConfig.Field( 

50 doc="Reference CCD for scanning visits", 

51 dtype=int, 

52 default=13, 

53 ) 

54 checkAllCcds = pexConfig.Field( 

55 doc=("Check repo for all CCDs for each visit specified. To be used when the " 

56 "full set of ids (visit/ccd) are not specified on the command line. For " 

57 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly " 

58 "faster than the alternatives."), 

59 dtype=bool, 

60 default=True, 

61 ) 

62 

63 def setDefaults(self): 

64 super().setDefaults() 

65 

66 sourceSelector = self.sourceSelector["science"] 

67 

68 # The names here correspond to raw src catalogs, which differ 

69 # from the post-transformed sourceTable_visit catalogs. 

70 # Therefore, field and flag names cannot be easily 

71 # derived from the base config class. 

72 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

73 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge', 

74 'base_PixelFlags_flag_interpolatedCenter', 

75 'base_PixelFlags_flag_saturatedCenter', 

76 'base_PixelFlags_flag_crCenter', 

77 'base_PixelFlags_flag_bad', 

78 'base_PixelFlags_flag_interpolated', 

79 'base_PixelFlags_flag_saturated', 

80 'slot_Centroid_flag', 

81 fluxFlagName] 

82 

83 if self.doSubtractLocalBackground: 

84 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

85 sourceSelector.flags.bad.append(localBackgroundFlagName) 

86 

87 sourceSelector.signalToNoise.fluxField = self.instFluxField 

88 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

89 

90 

91class FgcmBuildStarsTask(FgcmBuildStarsBaseTask): 

92 """ 

93 Build stars for the FGCM global calibration, using src catalogs. 

94 """ 

95 ConfigClass = FgcmBuildStarsConfig 

96 RunnerClass = FgcmBuildStarsRunner 

97 _DefaultName = "fgcmBuildStars" 

98 

99 @classmethod 

100 def _makeArgumentParser(cls): 

101 """Create an argument parser""" 

102 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

103 parser.add_id_argument("--id", "src", help="Data ID, e.g. --id visit=6789") 

104 

105 return parser 

106 

107 def findAndGroupDataRefs(self, butler, dataRefs): 

108 self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName)) 

109 

110 camera = butler.get('camera') 

111 

112 ccdIds = [] 

113 for detector in camera: 

114 ccdIds.append(detector.getId()) 

115 

116 # TODO: related to DM-13730, this dance of looking for source visits 

117 # will be unnecessary with Gen3 Butler. This should be part of 

118 # DM-13730. 

119 

120 nVisits = 0 

121 

122 groupedDataRefs = {} 

123 for dataRef in dataRefs: 

124 visit = dataRef.dataId[self.config.visitDataRefName] 

125 # If we don't have the dataset, just continue 

126 if not dataRef.datasetExists(datasetType='src'): 

127 continue 

128 # If we need to check all ccds, do it here 

129 if self.config.checkAllCcds: 

130 if visit in groupedDataRefs: 

131 # We already have found this visit 

132 continue 

133 dataId = dataRef.dataId.copy() 

134 # For each ccd we must check that a valid source catalog exists. 

135 for ccdId in ccdIds: 

136 dataId[self.config.ccdDataRefName] = ccdId 

137 if butler.datasetExists('src', dataId=dataId): 

138 goodDataRef = butler.dataRef('src', dataId=dataId) 

139 if visit in groupedDataRefs: 

140 if (goodDataRef.dataId[self.config.ccdDataRefName] not in 

141 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]): 

142 groupedDataRefs[visit].append(goodDataRef) 

143 else: 

144 # This is a new visit 

145 nVisits += 1 

146 groupedDataRefs[visit] = [goodDataRef] 

147 else: 

148 # We have already confirmed that the dataset exists, so no need 

149 # to check here. 

150 if visit in groupedDataRefs: 

151 if (dataRef.dataId[self.config.ccdDataRefName] not in 

152 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]): 

153 groupedDataRefs[visit].append(dataRef) 

154 else: 

155 # This is a new visit 

156 nVisits += 1 

157 groupedDataRefs[visit] = [dataRef] 

158 

159 if (nVisits % 100) == 0 and nVisits > 0: 

160 self.log.info("Found %d unique %ss..." % (nVisits, 

161 self.config.visitDataRefName)) 

162 

163 self.log.info("Found %d unique %ss total." % (nVisits, 

164 self.config.visitDataRefName)) 

165 

166 # Put them in ccd order, with the reference ccd first (if available) 

167 def ccdSorter(dataRef): 

168 ccdId = dataRef.dataId[self.config.ccdDataRefName] 

169 if ccdId == self.config.referenceCCD: 

170 return -100 

171 else: 

172 return ccdId 

173 

174 # If we did not check all ccds, put them in ccd order 

175 if not self.config.checkAllCcds: 

176 for visit in groupedDataRefs: 

177 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter) 

178 

179 return groupedDataRefs 

180 

181 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

182 calibFluxApertureRadius=None, 

183 visitCatDataRef=None, 

184 starObsDataRef=None, 

185 inStarObsCat=None): 

186 startTime = time.time() 

187 

188 # If both dataRefs are None, then we assume the caller does not 

189 # want to store checkpoint files. If both are set, we will 

190 # do checkpoint files. And if only one is set, this is potentially 

191 # unintentional and we will warn. 

192 if (visitCatDataRef is not None and starObsDataRef is None or 

193 visitCatDataRef is None and starObsDataRef is not None): 

194 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so " 

195 "no checkpoint files will be persisted.") 

196 

197 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

198 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

199 

200 # create our source schema. Use the first valid dataRef 

201 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0] 

202 sourceSchema = dataRef.get('src_schema', immediate=True).schema 

203 

204 # Construct a mapping from ccd number to index 

205 camera = dataRef.get('camera') 

206 ccdMapping = {} 

207 for ccdIndex, detector in enumerate(camera): 

208 ccdMapping[detector.getId()] = ccdIndex 

209 

210 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

211 

212 sourceMapper = self._makeSourceMapper(sourceSchema) 

213 

214 # We also have a temporary catalog that will accumulate aperture measurements 

215 aperMapper = self._makeAperMapper(sourceSchema) 

216 

217 outputSchema = sourceMapper.getOutputSchema() 

218 

219 if inStarObsCat is not None: 

220 fullCatalog = inStarObsCat 

221 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS) 

222 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES) 

223 if not comp1 or not comp2: 

224 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.") 

225 else: 

226 fullCatalog = afwTable.BaseCatalog(outputSchema) 

227 

228 # FGCM will provide relative calibration for the flux in config.instFluxField 

229 

230 instFluxKey = sourceSchema[self.config.instFluxField].asKey() 

231 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey() 

232 visitKey = outputSchema['visit'].asKey() 

233 ccdKey = outputSchema['ccd'].asKey() 

234 instMagKey = outputSchema['instMag'].asKey() 

235 instMagErrKey = outputSchema['instMagErr'].asKey() 

236 

237 # Prepare local background if desired 

238 if self.config.doSubtractLocalBackground: 

239 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey() 

240 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

241 else: 

242 localBackground = 0.0 

243 

244 aperOutputSchema = aperMapper.getOutputSchema() 

245 

246 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey() 

247 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey() 

248 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey() 

249 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey() 

250 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey() 

251 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey() 

252 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey() 

253 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey() 

254 

255 k = 2.5/np.log(10.) 

256 

257 # loop over visits 

258 for ctr, visit in enumerate(visitCat): 

259 if visit['sources_read']: 

260 continue 

261 

262 expTime = visit['exptime'] 

263 

264 nStarInVisit = 0 

265 

266 # Reset the aperture catalog (per visit) 

267 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema) 

268 

269 for dataRef in groupedDataRefs[visit['visit']]: 

270 

271 ccdId = dataRef.dataId[self.config.ccdDataRefName] 

272 

273 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) 

274 

275 # If we are subtracting the local background, then correct here 

276 # before we do the s/n selection. This ensures we do not have 

277 # bad stars after local background subtraction. 

278 

279 if self.config.doSubtractLocalBackground: 

280 # At the moment we only adjust the flux and not the flux 

281 # error by the background because the error on 

282 # base_LocalBackground_instFlux is the rms error in the 

283 # background annulus, not the error on the mean in the 

284 # background estimate (which is much smaller, by sqrt(n) 

285 # pixels used to estimate the background, which we do not 

286 # have access to in this task). In the default settings, 

287 # the annulus is sufficiently large such that these 

288 # additional errors are are negligibly small (much less 

289 # than a mmag in quadrature). 

290 

291 localBackground = localBackgroundArea*sources[localBackgroundFluxKey] 

292 sources[instFluxKey] -= localBackground 

293 

294 goodSrc = self.sourceSelector.selectSources(sources) 

295 

296 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

297 tempCat.reserve(goodSrc.selected.sum()) 

298 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper) 

299 tempCat[visitKey][:] = visit['visit'] 

300 tempCat[ccdKey][:] = ccdId 

301 

302 # Compute "instrumental magnitude" by scaling flux with exposure time. 

303 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] * 

304 visit['scaling'][ccdMapping[ccdId]]) 

305 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

306 

307 # Compute instMagErr from instFluxErr/instFlux, any scaling 

308 # will cancel out. 

309 

310 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] / 

311 sources[instFluxKey][goodSrc.selected]) 

312 

313 # Compute the jacobian from an approximate PixelAreaBoundedField 

314 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'], 

315 tempCat['y']) 

316 

317 # Apply the jacobian if configured 

318 if self.config.doApplyWcsJacobian: 

319 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

320 

321 fullCatalog.extend(tempCat) 

322 

323 # And the aperture information 

324 # This does not need the jacobian because it is all locally relative 

325 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema) 

326 tempAperCat.reserve(goodSrc.selected.sum()) 

327 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper) 

328 

329 with np.warnings.catch_warnings(): 

330 # Ignore warnings, we will filter infinities and 

331 # nans below. 

332 np.warnings.simplefilter("ignore") 

333 

334 tempAperCat[instMagInKey][:] = -2.5*np.log10( 

335 sources[instFluxAperInKey][goodSrc.selected]) 

336 tempAperCat[instMagErrInKey][:] = k*( 

337 sources[instFluxErrAperInKey][goodSrc.selected] / 

338 sources[instFluxAperInKey][goodSrc.selected]) 

339 tempAperCat[instMagOutKey][:] = -2.5*np.log10( 

340 sources[instFluxAperOutKey][goodSrc.selected]) 

341 tempAperCat[instMagErrOutKey][:] = k*( 

342 sources[instFluxErrAperOutKey][goodSrc.selected] / 

343 sources[instFluxAperOutKey][goodSrc.selected]) 

344 

345 aperVisitCatalog.extend(tempAperCat) 

346 

347 nStarInVisit += len(tempCat) 

348 

349 # Compute the median delta-aper 

350 if not aperVisitCatalog.isContiguous(): 

351 aperVisitCatalog = aperVisitCatalog.copy(deep=True) 

352 

353 instMagIn = aperVisitCatalog[instMagInKey] 

354 instMagErrIn = aperVisitCatalog[instMagErrInKey] 

355 instMagOut = aperVisitCatalog[instMagOutKey] 

356 instMagErrOut = aperVisitCatalog[instMagErrOutKey] 

357 

358 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) & 

359 np.isfinite(instMagOut) & np.isfinite(instMagErrOut)) 

360 

361 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok]) 

362 visit['sources_read'] = True 

363 

364 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" % 

365 (nStarInVisit, visit['visit'], visit['deltaAper'])) 

366 

367 if ((ctr % self.config.nVisitsPerCheckpoint) == 0 and 

368 starObsDataRef is not None and visitCatDataRef is not None): 

369 # We need to persist both the stars and the visit catalog which gets 

370 # additional metadata from each visit. 

371 starObsDataRef.put(fullCatalog) 

372 visitCatDataRef.put(visitCat) 

373 

374 self.log.info("Found all good star observations in %.2f s" % 

375 (time.time() - startTime)) 

376 

377 return fullCatalog 

378 

379 def _makeAperMapper(self, sourceSchema): 

380 """ 

381 Make a schema mapper for fgcm aperture measurements 

382 

383 Parameters 

384 ---------- 

385 sourceSchema: `afwTable.Schema` 

386 Default source schema from the butler 

387 

388 Returns 

389 ------- 

390 aperMapper: `afwTable.schemaMapper` 

391 Mapper to the FGCM aperture schema 

392 """ 

393 

394 aperMapper = afwTable.SchemaMapper(sourceSchema) 

395 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

396 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

397 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64, 

398 doc="Magnitude at inner aperture") 

399 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64, 

400 doc="Magnitude error at inner aperture") 

401 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64, 

402 doc="Magnitude at outer aperture") 

403 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64, 

404 doc="Magnitude error at outer aperture") 

405 

406 return aperMapper