Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM. 

24 

25This task finds all the visits and calexps in a repository (or a subset 

26based on command line parameters) and extract all the potential calibration 

27stars for input into fgcm. This task additionally uses fgcm to match 

28star observations into unique stars, and performs as much cleaning of 

29the input catalog as possible. 

30""" 

31 

32import sys 

33import time 

34import traceback 

35 

36import numpy as np 

37 

38import lsst.pex.config as pexConfig 

39import lsst.pipe.base as pipeBase 

40import lsst.afw.table as afwTable 

41import lsst.geom as geom 

42from lsst.daf.base import PropertyList 

43from lsst.daf.base.dateTime import DateTime 

44from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

45 

46from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask 

47from .utilities import computeApproxPixelAreaFields, computeApertureRadius 

48 

49import fgcm 

50 

51REFSTARS_FORMAT_VERSION = 1 

52 

53__all__ = ['FgcmBuildStarsConfig', 'FgcmBuildStarsTask', 'FgcmBuildStarsRunner'] 

54 

55 

56class FgcmBuildStarsConfig(pexConfig.Config): 

57 """Config for FgcmBuildStarsTask""" 

58 

59 instFluxField = pexConfig.Field( 

60 doc=("Name of the source instFlux field to use. The associated flag field " 

61 "('<name>_flag') will be implicitly included in badFlags"), 

62 dtype=str, 

63 default='slot_CalibFlux_instFlux', 

64 ) 

65 minPerBand = pexConfig.Field( 

66 doc="Minimum observations per band", 

67 dtype=int, 

68 default=2, 

69 ) 

70 matchRadius = pexConfig.Field( 

71 doc="Match radius (arcseconds)", 

72 dtype=float, 

73 default=1.0, 

74 ) 

75 isolationRadius = pexConfig.Field( 

76 doc="Isolation radius (arcseconds)", 

77 dtype=float, 

78 default=2.0, 

79 ) 

80 densityCutNside = pexConfig.Field( 

81 doc="Density cut healpix nside", 

82 dtype=int, 

83 default=128, 

84 ) 

85 densityCutMaxPerPixel = pexConfig.Field( 

86 doc="Density cut number of stars per pixel", 

87 dtype=int, 

88 default=1000, 

89 ) 

90 matchNside = pexConfig.Field( 

91 doc="Healpix Nside for matching", 

92 dtype=int, 

93 default=4096, 

94 ) 

95 coarseNside = pexConfig.Field( 

96 doc="Healpix coarse Nside for partitioning matches", 

97 dtype=int, 

98 default=8, 

99 ) 

100 filterMap = pexConfig.DictField( 

101 doc="Mapping from 'filterName' to band.", 

102 keytype=str, 

103 itemtype=str, 

104 default={}, 

105 ) 

106 requiredBands = pexConfig.ListField( 

107 doc="Bands required for each star", 

108 dtype=str, 

109 default=(), 

110 ) 

111 primaryBands = pexConfig.ListField( 

112 doc=("Bands for 'primary' star matches. " 

113 "A star must be observed in one of these bands to be considered " 

114 "as a calibration star."), 

115 dtype=str, 

116 default=None 

117 ) 

118 referenceCCD = pexConfig.Field( 

119 doc="Reference CCD for scanning visits", 

120 dtype=int, 

121 default=13, 

122 ) 

123 checkAllCcds = pexConfig.Field( 

124 doc=("Check repo for all CCDs for each visit specified. To be used when the " 

125 "full set of ids (visit/ccd) are not specified on the command line. For " 

126 "Gen2, specifying one ccd and setting checkAllCcds=True is significantly " 

127 "faster than the alternatives."), 

128 dtype=bool, 

129 default=True, 

130 ) 

131 visitDataRefName = pexConfig.Field( 

132 doc="dataRef name for the 'visit' field", 

133 dtype=str, 

134 default="visit" 

135 ) 

136 ccdDataRefName = pexConfig.Field( 

137 doc="dataRef name for the 'ccd' field", 

138 dtype=str, 

139 default="ccd" 

140 ) 

141 applyJacobian = pexConfig.Field( 

142 doc="Apply Jacobian correction?", 

143 dtype=bool, 

144 deprecated=("This field is no longer used, and has been deprecated by DM-20163. " 

145 "It will be removed after v20."), 

146 default=False 

147 ) 

148 jacobianName = pexConfig.Field( 

149 doc="Name of field with jacobian correction", 

150 dtype=str, 

151 deprecated=("This field is no longer used, and has been deprecated by DM-20163. " 

152 "It will be removed after v20."), 

153 default="base_Jacobian_value" 

154 ) 

155 doApplyWcsJacobian = pexConfig.Field( 

156 doc="Apply the jacobian of the WCS to the star observations prior to fit?", 

157 dtype=bool, 

158 default=True 

159 ) 

160 psfCandidateName = pexConfig.Field( 

161 doc="Name of field with psf candidate flag for propagation", 

162 dtype=str, 

163 default="calib_psf_candidate" 

164 ) 

165 doSubtractLocalBackground = pexConfig.Field( 

166 doc=("Subtract the local background before performing calibration? " 

167 "This is only supported for circular aperture calibration fluxes."), 

168 dtype=bool, 

169 default=False 

170 ) 

171 localBackgroundFluxField = pexConfig.Field( 

172 doc="Name of the local background instFlux field to use.", 

173 dtype=str, 

174 default='base_LocalBackground_instFlux' 

175 ) 

176 sourceSelector = sourceSelectorRegistry.makeField( 

177 doc="How to select sources", 

178 default="science" 

179 ) 

180 apertureInnerInstFluxField = pexConfig.Field( 

181 doc="Field that contains inner aperture for aperture correction proxy", 

182 dtype=str, 

183 default='base_CircularApertureFlux_12_0_instFlux' 

184 ) 

185 apertureOuterInstFluxField = pexConfig.Field( 

186 doc="Field that contains outer aperture for aperture correction proxy", 

187 dtype=str, 

188 default='base_CircularApertureFlux_17_0_instFlux' 

189 ) 

190 doReferenceMatches = pexConfig.Field( 

191 doc="Match reference catalog as additional constraint on calibration", 

192 dtype=bool, 

193 default=True, 

194 ) 

195 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField( 

196 target=FgcmLoadReferenceCatalogTask, 

197 doc="FGCM reference object loader", 

198 ) 

199 

200 def setDefaults(self): 

201 sourceSelector = self.sourceSelector["science"] 

202 sourceSelector.setDefaults() 

203 

204 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

205 

206 sourceSelector.flags.bad = ['base_PixelFlags_flag_edge', 

207 'base_PixelFlags_flag_interpolatedCenter', 

208 'base_PixelFlags_flag_saturatedCenter', 

209 'base_PixelFlags_flag_crCenter', 

210 'base_PixelFlags_flag_bad', 

211 'base_PixelFlags_flag_interpolated', 

212 'base_PixelFlags_flag_saturated', 

213 'slot_Centroid_flag', 

214 fluxFlagName] 

215 

216 if self.doSubtractLocalBackground: 

217 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

218 sourceSelector.flags.bad.append(localBackgroundFlagName) 

219 

220 sourceSelector.doFlags = True 

221 sourceSelector.doUnresolved = True 

222 sourceSelector.doSignalToNoise = True 

223 sourceSelector.doIsolated = True 

224 

225 sourceSelector.signalToNoise.fluxField = self.instFluxField 

226 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

227 sourceSelector.signalToNoise.minimum = 10.0 

228 sourceSelector.signalToNoise.maximum = 1000.0 

229 

230 # FGCM operates on unresolved sources, and this setting is 

231 # appropriate for the current base_ClassificationExtendedness 

232 sourceSelector.unresolved.maximum = 0.5 

233 

234 

235class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner): 

236 """Subclass of TaskRunner for fgcmBuildStarsTask 

237 

238 fgcmBuildStarsTask.run() takes a number of arguments, one of which is the 

239 butler (for persistence and mapper data), and a list of dataRefs 

240 extracted from the command line. Note that FGCM runs on a large set of 

241 dataRefs, and not on single dataRef/tract/patch. 

242 This class transforms the process arguments generated by the ArgumentParser 

243 into the arguments expected by FgcmBuildStarsTask.run(). 

244 This runner does not use any parallelization. 

245 

246 """ 

247 

248 @staticmethod 

249 def getTargetList(parsedCmd): 

250 """ 

251 Return a list with one element: a tuple with the butler and 

252 list of dataRefs 

253 """ 

254 # we want to combine the butler with any (or no!) dataRefs 

255 return [(parsedCmd.butler, parsedCmd.id.refList)] 

256 

257 def __call__(self, args): 

258 """ 

259 Parameters 

260 ---------- 

261 args: `tuple` with (butler, dataRefList) 

262 

263 Returns 

264 ------- 

265 exitStatus: `list` with `lsst.pipe.base.Struct` 

266 exitStatus (0: success; 1: failure) 

267 """ 

268 butler, dataRefList = args 

269 

270 task = self.TaskClass(config=self.config, log=self.log) 

271 

272 exitStatus = 0 

273 if self.doRaise: 

274 task.runDataRef(butler, dataRefList) 

275 else: 

276 try: 

277 task.runDataRef(butler, dataRefList) 

278 except Exception as e: 

279 exitStatus = 1 

280 task.log.fatal("Failed: %s" % e) 

281 if not isinstance(e, pipeBase.TaskError): 

282 traceback.print_exc(file=sys.stderr) 

283 

284 task.writeMetadata(butler) 

285 

286 # The task does not return any results: 

287 return [pipeBase.Struct(exitStatus=exitStatus)] 

288 

289 def run(self, parsedCmd): 

290 """ 

291 Run the task, with no multiprocessing 

292 

293 Parameters 

294 ---------- 

295 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

296 """ 

297 

298 resultList = [] 

299 

300 if self.precall(parsedCmd): 

301 targetList = self.getTargetList(parsedCmd) 

302 resultList = self(targetList[0]) 

303 

304 return resultList 

305 

306 

307class FgcmBuildStarsTask(pipeBase.CmdLineTask): 

308 """ 

309 Build stars for the FGCM global calibration 

310 """ 

311 

312 ConfigClass = FgcmBuildStarsConfig 

313 RunnerClass = FgcmBuildStarsRunner 

314 _DefaultName = "fgcmBuildStars" 

315 

316 def __init__(self, butler=None, **kwargs): 

317 """ 

318 Instantiate an `FgcmBuildStarsTask`. 

319 

320 Parameters 

321 ---------- 

322 butler : `lsst.daf.persistence.Butler` 

323 """ 

324 

325 pipeBase.CmdLineTask.__init__(self, **kwargs) 

326 self.makeSubtask("sourceSelector") 

327 # Only log warning and fatal errors from the sourceSelector 

328 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN) 

329 

330 @classmethod 

331 def _makeArgumentParser(cls): 

332 """Create an argument parser""" 

333 

334 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

335 parser.add_id_argument("--id", "calexp", help="Data ID, e.g. --id visit=6789") 

336 

337 return parser 

338 

339 # no saving of metadata for now 

340 def _getMetadataName(self): 

341 return None 

342 

343 @pipeBase.timeMethod 

344 def runDataRef(self, butler, dataRefs): 

345 """ 

346 Cross-match and make star list for FGCM Input 

347 

348 Parameters 

349 ---------- 

350 butler: `lsst.daf.persistence.Butler` 

351 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

352 Data references for the input visits. 

353 If this is an empty list, all visits with src catalogs in 

354 the repository are used. 

355 Only one individual dataRef from a visit need be specified 

356 and the code will find the other source catalogs from 

357 each visit. 

358 

359 Raises 

360 ------ 

361 RuntimeErrror: Raised if `config.doReferenceMatches` is set and 

362 an fgcmLookUpTable is not available, or if computeFluxApertureRadius() 

363 fails if the calibFlux is not a CircularAperture flux. 

364 """ 

365 

366 if self.config.doReferenceMatches: 

367 # Ensure that we have a LUT 

368 if not butler.datasetExists('fgcmLookUpTable'): 

369 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches") 

370 # Compute aperture radius if necessary. This is useful to do now before 

371 # any heavy lifting has happened (fail early). 

372 calibFluxApertureRadius = None 

373 if self.config.doSubtractLocalBackground: 

374 sourceSchema = butler.get('src_schema').schema 

375 try: 

376 calibFluxApertureRadius = computeApertureRadius(sourceSchema, 

377 self.config.instFluxField) 

378 except (RuntimeError, LookupError): 

379 raise RuntimeError("Could not determine aperture radius from %s. " 

380 "Cannot use doSubtractLocalBackground." % 

381 (self.config.instFluxField)) 

382 

383 groupedDataRefs = self.findAndGroupDataRefs(butler, dataRefs) 

384 

385 camera = butler.get('camera') 

386 

387 # Make the visit catalog if necessary 

388 if not butler.datasetExists('fgcmVisitCatalog'): 

389 # we need to build visitCat 

390 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs) 

391 else: 

392 self.log.info("Found fgcmVisitCatalog.") 

393 visitCat = butler.get('fgcmVisitCatalog') 

394 

395 # Compile all the stars 

396 if not butler.datasetExists('fgcmStarObservations'): 

397 rad = calibFluxApertureRadius 

398 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs, 

399 visitCat, 

400 calibFluxApertureRadius=rad) 

401 else: 

402 self.log.info("Found fgcmStarObservations") 

403 fgcmStarObservationCat = butler.get('fgcmStarObservations') 

404 

405 if not butler.datasetExists('fgcmStarIds') or not butler.datasetExists('fgcmStarIndices'): 

406 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(butler, 

407 visitCat, 

408 fgcmStarObservationCat) 

409 else: 

410 self.log.info("Found fgcmStarIds and fgcmStarIndices") 

411 

412 # Persist catalogs via the butler 

413 butler.put(visitCat, 'fgcmVisitCatalog') 

414 butler.put(fgcmStarObservationCat, 'fgcmStarObservations') 

415 butler.put(fgcmStarIdCat, 'fgcmStarIds') 

416 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') 

417 if fgcmRefCat is not None: 

418 butler.put(fgcmRefCat, 'fgcmReferenceStars') 

419 

420 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs): 

421 """ 

422 Make a visit catalog with all the keys from each visit 

423 

424 Parameters 

425 ---------- 

426 camera: `lsst.afw.cameraGeom.Camera` 

427 Camera from the butler 

428 groupedDataRefs: `dict` 

429 Dictionary with visit keys, and `list`s of 

430 `lsst.daf.persistence.ButlerDataRef` 

431 

432 Returns 

433 ------- 

434 visitCat: `afw.table.BaseCatalog` 

435 """ 

436 

437 nCcd = len(camera) 

438 

439 schema = self._makeFgcmVisitSchema(nCcd) 

440 

441 visitCat = afwTable.BaseCatalog(schema) 

442 visitCat.reserve(len(groupedDataRefs)) 

443 

444 self._fillVisitCatalog(visitCat, groupedDataRefs) 

445 

446 return visitCat 

447 

448 def _fillVisitCatalog(self, visitCat, groupedDataRefs): 

449 """ 

450 Fill the visit catalog with visit metadata 

451 

452 Parameters 

453 ---------- 

454 visitCat: `afw.table.BaseCatalog` 

455 Catalog with schema from _makeFgcmVisitSchema() 

456 groupedDataRefs: `dict` 

457 Dictionary with visit keys, and `list`s of 

458 `lsst.daf.persistence.ButlerDataRef` 

459 """ 

460 

461 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1)) 

462 

463 for i, visit in enumerate(sorted(groupedDataRefs)): 

464 # We don't use the bypasses since we need the psf info which does 

465 # not have a bypass 

466 # TODO: When DM-15500 is implemented in the Gen3 Butler, this 

467 # can be fixed 

468 

469 # Note that the reference ccd is first in the list (if available). 

470 

471 # The first dataRef in the group will be the reference ccd (if available) 

472 dataRef = groupedDataRefs[visit][0] 

473 

474 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox, 

475 flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) 

476 

477 visitInfo = exp.getInfo().getVisitInfo() 

478 f = exp.getFilter() 

479 psf = exp.getPsf() 

480 

481 rec = visitCat.addNew() 

482 rec['visit'] = visit 

483 rec['filtername'] = f.getName() 

484 radec = visitInfo.getBoresightRaDec() 

485 rec['telra'] = radec.getRa().asDegrees() 

486 rec['teldec'] = radec.getDec().asDegrees() 

487 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees() 

488 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees() 

489 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD) 

490 rec['exptime'] = visitInfo.getExposureTime() 

491 # convert from Pa to millibar 

492 # Note that I don't know if this unit will need to be per-camera config 

493 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100 

494 # Flag to signify if this is a "deep" field. Not currently used 

495 rec['deepFlag'] = 0 

496 # Relative flat scaling (1.0 means no relative scaling) 

497 rec['scaling'][:] = 1.0 

498 # Median delta aperture, to be measured from stars 

499 rec['deltaAper'] = 0.0 

500 

501 rec['psfSigma'] = psf.computeShape().getDeterminantRadius() 

502 

503 if dataRef.datasetExists(datasetType='calexpBackground'): 

504 # Get background for reference CCD 

505 # This approximation is good enough for now 

506 bgStats = (bg[0].getStatsImage().getImage().array 

507 for bg in dataRef.get(datasetType='calexpBackground')) 

508 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats) 

509 else: 

510 self.log.warn('Sky background not found for visit %d / ccd %d' % 

511 (visit, dataRef.dataId[self.config.ccdDataRefName])) 

512 rec['skyBackground'] = -1.0 

513 

514 def findAndGroupDataRefs(self, butler, dataRefs): 

515 """ 

516 Find and group dataRefs (by visit). If dataRefs is an empty list, 

517 this will look for all source catalogs in a given repo. 

518 

519 Parameters 

520 ---------- 

521 butler: `lsst.daf.persistence.Butler` 

522 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

523 Data references for the input visits. 

524 If this is an empty list, all visits with src catalogs in 

525 the repository are used. 

526 

527 Returns 

528 ------- 

529 groupedDataRefs: `dict` 

530 Dictionary with visit keys, and `list`s of `lsst.daf.persistence.ButlerDataRef` 

531 """ 

532 

533 camera = butler.get('camera') 

534 

535 ccdIds = [] 

536 for detector in camera: 

537 ccdIds.append(detector.getId()) 

538 

539 # TODO: related to DM-13730, this dance of looking for source visits 

540 # will be unnecessary with Gen3 Butler. This should be part of 

541 # DM-13730. 

542 

543 groupedDataRefs = {} 

544 for dataRef in dataRefs: 

545 visit = dataRef.dataId[self.config.visitDataRefName] 

546 # If we don't have the dataset, just continue 

547 if not dataRef.datasetExists(datasetType='src'): 

548 continue 

549 # If we need to check all ccds, do it here 

550 if self.config.checkAllCcds: 

551 dataId = dataRef.dataId.copy() 

552 # For each ccd we must check that a valid source catalog exists. 

553 for ccdId in ccdIds: 

554 dataId[self.config.ccdDataRefName] = ccdId 

555 if butler.datasetExists('src', dataId=dataId): 

556 goodDataRef = butler.dataRef('src', dataId=dataId) 

557 if visit in groupedDataRefs: 

558 if (goodDataRef.dataId[self.config.ccdDataRefName] not in 

559 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]): 

560 groupedDataRefs[visit].append(goodDataRef) 

561 else: 

562 groupedDataRefs[visit] = [goodDataRef] 

563 else: 

564 # We have already confirmed that the dataset exists, so no need 

565 # to check here. 

566 if visit in groupedDataRefs: 

567 if (dataRef.dataId[self.config.ccdDataRefName] not in 

568 [d.dataId[self.config.ccdDataRefName] for d in groupedDataRefs[visit]]): 

569 groupedDataRefs[visit].append(dataRef) 

570 else: 

571 groupedDataRefs[visit] = [dataRef] 

572 

573 # Put them in ccd order, with the reference ccd first (if available) 

574 def ccdSorter(dataRef): 

575 ccdId = dataRef.dataId[self.config.ccdDataRefName] 

576 if ccdId == self.config.referenceCCD: 

577 return -100 

578 else: 

579 return ccdId 

580 

581 # If we did not check all ccds, put them in ccd order 

582 if not self.config.checkAllCcds: 

583 for visit in groupedDataRefs: 

584 groupedDataRefs[visit] = sorted(groupedDataRefs[visit], key=ccdSorter) 

585 

586 return groupedDataRefs 

587 

588 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

589 calibFluxApertureRadius=None): 

590 """ 

591 Compile all good star observations from visits in visitCat. 

592 

593 Parameters 

594 ---------- 

595 groupedDataRefs: `dict` of `list`s 

596 Lists of `lsst.daf.persistence.ButlerDataRef`, grouped by visit. 

597 visitCat: `afw.table.BaseCatalog` 

598 Catalog with visit data for FGCM 

599 calibFluxApertureRadius: `float`, optional 

600 Aperture radius for calibration flux. Default is None. 

601 

602 Returns 

603 ------- 

604 fgcmStarObservations: `afw.table.BaseCatalog` 

605 Full catalog of good observations. 

606 

607 Raises 

608 ------ 

609 RuntimeError: Raised if doSubtractLocalBackground is True and 

610 calibFluxApertureRadius is not set. 

611 """ 

612 startTime = time.time() 

613 

614 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

615 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

616 

617 # create our source schema. Use the first valid dataRef 

618 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0] 

619 sourceSchema = dataRef.get('src_schema', immediate=True).schema 

620 

621 # Construct a mapping from ccd number to index 

622 camera = dataRef.get('camera') 

623 ccdMapping = {} 

624 for ccdIndex, detector in enumerate(camera): 

625 ccdMapping[detector.getId()] = ccdIndex 

626 

627 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

628 

629 sourceMapper = self._makeSourceMapper(sourceSchema) 

630 

631 # We also have a temporary catalog that will accumulate aperture measurements 

632 aperMapper = self._makeAperMapper(sourceSchema) 

633 

634 outputSchema = sourceMapper.getOutputSchema() 

635 fullCatalog = afwTable.BaseCatalog(outputSchema) 

636 

637 # FGCM will provide relative calibration for the flux in config.instFluxField 

638 

639 instFluxKey = sourceSchema[self.config.instFluxField].asKey() 

640 instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey() 

641 visitKey = outputSchema['visit'].asKey() 

642 ccdKey = outputSchema['ccd'].asKey() 

643 instMagKey = outputSchema['instMag'].asKey() 

644 instMagErrKey = outputSchema['instMagErr'].asKey() 

645 

646 # Prepare local background if desired 

647 if self.config.doSubtractLocalBackground: 

648 localBackgroundFluxKey = sourceSchema[self.config.localBackgroundFluxField].asKey() 

649 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

650 else: 

651 localBackground = 0.0 

652 

653 aperOutputSchema = aperMapper.getOutputSchema() 

654 

655 instFluxAperInKey = sourceSchema[self.config.apertureInnerInstFluxField].asKey() 

656 instFluxErrAperInKey = sourceSchema[self.config.apertureInnerInstFluxField + 'Err'].asKey() 

657 instFluxAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField].asKey() 

658 instFluxErrAperOutKey = sourceSchema[self.config.apertureOuterInstFluxField + 'Err'].asKey() 

659 instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey() 

660 instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey() 

661 instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey() 

662 instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey() 

663 

664 k = 2.5 / np.log(10.) 

665 

666 # loop over visits 

667 for visit in visitCat: 

668 expTime = visit['exptime'] 

669 

670 nStarInVisit = 0 

671 

672 # Reset the aperture catalog (per visit) 

673 aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema) 

674 

675 for dataRef in groupedDataRefs[visit['visit']]: 

676 

677 ccdId = dataRef.dataId[self.config.ccdDataRefName] 

678 

679 sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) 

680 

681 # If we are subtracting the local background, then correct here 

682 # before we do the s/n selection. This ensures we do not have 

683 # bad stars after local background subtraction. 

684 

685 if self.config.doSubtractLocalBackground: 

686 # At the moment we only adjust the flux and not the flux 

687 # error by the background because the error on 

688 # base_LocalBackground_instFlux is the rms error in the 

689 # background annulus, not the error on the mean in the 

690 # background estimate (which is much smaller, by sqrt(n) 

691 # pixels used to estimate the background, which we do not 

692 # have access to in this task). In the default settings, 

693 # the annulus is sufficiently large such that these 

694 # additional errors are are negligibly small (much less 

695 # than a mmag in quadrature). 

696 

697 localBackground = localBackgroundArea*sources[localBackgroundFluxKey] 

698 sources[instFluxKey] -= localBackground 

699 

700 goodSrc = self.sourceSelector.selectSources(sources) 

701 

702 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

703 tempCat.reserve(goodSrc.selected.sum()) 

704 tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper) 

705 tempCat[visitKey][:] = visit['visit'] 

706 tempCat[ccdKey][:] = ccdId 

707 

708 # Compute "instrumental magnitude" by scaling flux with exposure time. 

709 scaledInstFlux = (sources[instFluxKey][goodSrc.selected] * 

710 visit['scaling'][ccdMapping[ccdId]]) 

711 tempCat[instMagKey][:] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

712 

713 # Compute instMagErr from instFluxErr / instFlux, any scaling 

714 # will cancel out. 

715 

716 tempCat[instMagErrKey][:] = k*(sources[instFluxErrKey][goodSrc.selected] / 

717 sources[instFluxKey][goodSrc.selected]) 

718 

719 # Compute the jacobian from an approximate PixelAreaBoundedField 

720 tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'], 

721 tempCat['y']) 

722 

723 # Apply the jacobian if configured 

724 if self.config.doApplyWcsJacobian: 

725 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

726 

727 fullCatalog.extend(tempCat) 

728 

729 # And the aperture information 

730 # This does not need the jacobian because it is all locally relative 

731 tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema) 

732 tempAperCat.reserve(goodSrc.selected.sum()) 

733 tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper) 

734 

735 with np.warnings.catch_warnings(): 

736 # Ignore warnings, we will filter infinities and 

737 # nans below. 

738 np.warnings.simplefilter("ignore") 

739 

740 tempAperCat[instMagInKey][:] = -2.5*np.log10( 

741 sources[instFluxAperInKey][goodSrc.selected]) 

742 tempAperCat[instMagErrInKey][:] = (2.5/np.log(10.))*( 

743 sources[instFluxErrAperInKey][goodSrc.selected] / 

744 sources[instFluxAperInKey][goodSrc.selected]) 

745 tempAperCat[instMagOutKey][:] = -2.5*np.log10( 

746 sources[instFluxAperOutKey][goodSrc.selected]) 

747 tempAperCat[instMagErrOutKey][:] = (2.5/np.log(10.))*( 

748 sources[instFluxErrAperOutKey][goodSrc.selected] / 

749 sources[instFluxAperOutKey][goodSrc.selected]) 

750 

751 aperVisitCatalog.extend(tempAperCat) 

752 

753 nStarInVisit += len(tempCat) 

754 

755 # Compute the median delta-aper 

756 if not aperVisitCatalog.isContiguous(): 

757 aperVisitCatalog = aperVisitCatalog.copy(deep=True) 

758 

759 instMagIn = aperVisitCatalog[instMagInKey] 

760 instMagErrIn = aperVisitCatalog[instMagErrInKey] 

761 instMagOut = aperVisitCatalog[instMagOutKey] 

762 instMagErrOut = aperVisitCatalog[instMagErrOutKey] 

763 

764 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) & 

765 np.isfinite(instMagOut) & np.isfinite(instMagErrOut)) 

766 

767 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok]) 

768 

769 self.log.info(" Found %d good stars in visit %d (deltaAper = %.3f)" % 

770 (nStarInVisit, visit['visit'], visit['deltaAper'])) 

771 

772 self.log.info("Found all good star observations in %.2f s" % 

773 (time.time() - startTime)) 

774 

775 return fullCatalog 

776 

777 def fgcmMatchStars(self, butler, visitCat, obsCat): 

778 """ 

779 Use FGCM code to match observations into unique stars. 

780 

781 Parameters 

782 ---------- 

783 butler: `lsst.daf.persistence.Butler` 

784 visitCat: `afw.table.BaseCatalog` 

785 Catalog with visit data for fgcm 

786 obsCat: `afw.table.BaseCatalog` 

787 Full catalog of star observations for fgcm 

788 

789 Returns 

790 ------- 

791 fgcmStarIdCat: `afw.table.BaseCatalog` 

792 Catalog of unique star identifiers and index keys 

793 fgcmStarIndicesCat: `afwTable.BaseCatalog` 

794 Catalog of unique star indices 

795 fgcmRefCat: `afw.table.BaseCatalog` 

796 Catalog of matched reference stars. 

797 Will be None if `config.doReferenceMatches` is False. 

798 """ 

799 

800 if self.config.doReferenceMatches: 

801 # Make a subtask for reference loading 

802 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler) 

803 

804 # get filter names into a numpy array... 

805 # This is the type that is expected by the fgcm code 

806 visitFilterNames = np.zeros(len(visitCat), dtype='a10') 

807 for i in range(len(visitCat)): 

808 visitFilterNames[i] = visitCat[i]['filtername'] 

809 

810 # match to put filterNames with observations 

811 visitIndex = np.searchsorted(visitCat['visit'], 

812 obsCat['visit']) 

813 

814 obsFilterNames = visitFilterNames[visitIndex] 

815 

816 if self.config.doReferenceMatches: 

817 # Get the reference filter names, using the LUT 

818 lutCat = butler.get('fgcmLookUpTable') 

819 

820 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in 

821 zip(lutCat[0]['filterNames'].split(','), 

822 lutCat[0]['stdFilterNames'].split(','))} 

823 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in 

824 zip(lutCat[0]['stdFilterNames'].split(','), 

825 lutCat[0]['lambdaStdFilter'])} 

826 

827 del lutCat 

828 

829 referenceFilterNames = self._getReferenceFilterNames(visitCat, 

830 stdFilterDict, 

831 stdLambdaDict) 

832 self.log.info("Using the following reference filters: %s" % 

833 (', '.join(referenceFilterNames))) 

834 

835 else: 

836 # This should be an empty list 

837 referenceFilterNames = [] 

838 

839 # make the fgcm starConfig dict 

840 

841 starConfig = {'logger': self.log, 

842 'filterToBand': self.config.filterMap, 

843 'requiredBands': self.config.requiredBands, 

844 'minPerBand': self.config.minPerBand, 

845 'matchRadius': self.config.matchRadius, 

846 'isolationRadius': self.config.isolationRadius, 

847 'matchNSide': self.config.matchNside, 

848 'coarseNSide': self.config.coarseNside, 

849 'densNSide': self.config.densityCutNside, 

850 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 

851 'primaryBands': self.config.primaryBands, 

852 'referenceFilterNames': referenceFilterNames} 

853 

854 # initialize the FgcmMakeStars object 

855 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) 

856 

857 # make the primary stars 

858 # note that the ra/dec native Angle format is radians 

859 # We determine the conversion from the native units (typically 

860 # radians) to degrees for the first observation. This allows us 

861 # to treate ra/dec as numpy arrays rather than Angles, which would 

862 # be approximately 600x slower. 

863 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) 

864 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, 

865 obsCat['dec'] * conv, 

866 filterNameArray=obsFilterNames, 

867 bandSelected=False) 

868 

869 # and match all the stars 

870 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, 

871 obsCat['dec'] * conv, 

872 obsFilterNames) 

873 

874 if self.config.doReferenceMatches: 

875 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) 

876 

877 # now persist 

878 

879 objSchema = self._makeFgcmObjSchema() 

880 

881 # make catalog and records 

882 fgcmStarIdCat = afwTable.BaseCatalog(objSchema) 

883 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) 

884 for i in range(fgcmMakeStars.objIndexCat.size): 

885 fgcmStarIdCat.addNew() 

886 

887 # fill the catalog 

888 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] 

889 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] 

890 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] 

891 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex'] 

892 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] 

893 

894 obsSchema = self._makeFgcmObsSchema() 

895 

896 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) 

897 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) 

898 for i in range(fgcmMakeStars.obsIndexCat.size): 

899 fgcmStarIndicesCat.addNew() 

900 

901 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex'] 

902 

903 if self.config.doReferenceMatches: 

904 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) 

905 

906 fgcmRefCat = afwTable.BaseCatalog(refSchema) 

907 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) 

908 

909 for i in range(fgcmMakeStars.referenceCat.size): 

910 fgcmRefCat.addNew() 

911 

912 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] 

913 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] 

914 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr'] 

915 

916 md = PropertyList() 

917 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION) 

918 md.set("FILTERNAMES", referenceFilterNames) 

919 fgcmRefCat.setMetadata(md) 

920 

921 else: 

922 fgcmRefCat = None 

923 

924 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat 

925 

926 def _makeFgcmVisitSchema(self, nCcd): 

927 """ 

928 Make a schema for an fgcmVisitCatalog 

929 

930 Parameters 

931 ---------- 

932 nCcd: `int` 

933 Number of CCDs in the camera 

934 

935 Returns 

936 ------- 

937 schema: `afwTable.Schema` 

938 """ 

939 

940 schema = afwTable.Schema() 

941 schema.addField('visit', type=np.int32, doc="Visit number") 

942 # Note that the FGCM code currently handles filternames up to 2 characters long 

943 schema.addField('filtername', type=str, size=10, doc="Filter name") 

944 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") 

945 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") 

946 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") 

947 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)") 

948 schema.addField('mjd', type=np.float64, doc="MJD of visit") 

949 schema.addField('exptime', type=np.float32, doc="Exposure time") 

950 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") 

951 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") 

952 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") 

953 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") 

954 # the following field is not used yet 

955 schema.addField('deepFlag', type=np.int32, doc="Deep observation") 

956 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", 

957 size=nCcd) 

958 

959 return schema 

960 

961 def _makeSourceMapper(self, sourceSchema): 

962 """ 

963 Make a schema mapper for fgcm sources 

964 

965 Parameters 

966 ---------- 

967 sourceSchema: `afwTable.Schema` 

968 Default source schema from the butler 

969 

970 Returns 

971 ------- 

972 sourceMapper: `afwTable.schemaMapper` 

973 Mapper to the FGCM source schema 

974 """ 

975 

976 # create a mapper to the preferred output 

977 sourceMapper = afwTable.SchemaMapper(sourceSchema) 

978 

979 # map to ra/dec 

980 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

981 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

982 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x') 

983 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y') 

984 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(), 

985 'psf_candidate') 

986 

987 # and add the fields we want 

988 sourceMapper.editOutputSchema().addField( 

989 "visit", type=np.int32, doc="Visit number") 

990 sourceMapper.editOutputSchema().addField( 

991 "ccd", type=np.int32, doc="CCD number") 

992 sourceMapper.editOutputSchema().addField( 

993 "instMag", type=np.float32, doc="Instrumental magnitude") 

994 sourceMapper.editOutputSchema().addField( 

995 "instMagErr", type=np.float32, doc="Instrumental magnitude error") 

996 sourceMapper.editOutputSchema().addField( 

997 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian") 

998 

999 return sourceMapper 

1000 

1001 def _makeAperMapper(self, sourceSchema): 

1002 """ 

1003 Make a schema mapper for fgcm aperture measurements 

1004 

1005 Parameters 

1006 ---------- 

1007 sourceSchema: `afwTable.Schema` 

1008 Default source schema from the butler 

1009 

1010 Returns 

1011 ------- 

1012 aperMapper: `afwTable.schemaMapper` 

1013 Mapper to the FGCM aperture schema 

1014 """ 

1015 

1016 aperMapper = afwTable.SchemaMapper(sourceSchema) 

1017 aperMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

1018 aperMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

1019 aperMapper.editOutputSchema().addField('instMag_aper_inner', type=np.float64, 

1020 doc="Magnitude at inner aperture") 

1021 aperMapper.editOutputSchema().addField('instMagErr_aper_inner', type=np.float64, 

1022 doc="Magnitude error at inner aperture") 

1023 aperMapper.editOutputSchema().addField('instMag_aper_outer', type=np.float64, 

1024 doc="Magnitude at outer aperture") 

1025 aperMapper.editOutputSchema().addField('instMagErr_aper_outer', type=np.float64, 

1026 doc="Magnitude error at outer aperture") 

1027 

1028 return aperMapper 

1029 

1030 def _makeFgcmObjSchema(self): 

1031 """ 

1032 Make a schema for the objIndexCat from fgcmMakeStars 

1033 

1034 Returns 

1035 ------- 

1036 schema: `afwTable.Schema` 

1037 """ 

1038 

1039 objSchema = afwTable.Schema() 

1040 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

1041 # Will investigate making these angles... 

1042 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)') 

1043 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)') 

1044 objSchema.addField('obsArrIndex', type=np.int32, 

1045 doc='Index in obsIndexTable for first observation') 

1046 objSchema.addField('nObs', type=np.int32, doc='Total number of observations') 

1047 

1048 return objSchema 

1049 

1050 def _makeFgcmObsSchema(self): 

1051 """ 

1052 Make a schema for the obsIndexCat from fgcmMakeStars 

1053 

1054 Returns 

1055 ------- 

1056 schema: `afwTable.Schema` 

1057 """ 

1058 

1059 obsSchema = afwTable.Schema() 

1060 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table') 

1061 

1062 return obsSchema 

1063 

1064 def _makeFgcmRefSchema(self, nReferenceBands): 

1065 """ 

1066 Make a schema for the referenceCat from fgcmMakeStars 

1067 

1068 Parameters 

1069 ---------- 

1070 nReferenceBands: `int` 

1071 Number of reference bands 

1072 

1073 Returns 

1074 ------- 

1075 schema: `afwTable.Schema` 

1076 """ 

1077 

1078 refSchema = afwTable.Schema() 

1079 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

1080 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)', 

1081 size=nReferenceBands) 

1082 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array', 

1083 size=nReferenceBands) 

1084 

1085 return refSchema 

1086 

1087 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict): 

1088 """ 

1089 Get the reference filter names, in wavelength order, from the visitCat and 

1090 information from the look-up-table. 

1091 

1092 Parameters 

1093 ---------- 

1094 visitCat: `afw.table.BaseCatalog` 

1095 Catalog with visit data for FGCM 

1096 stdFilterDict: `dict` 

1097 Mapping of filterName to stdFilterName from LUT 

1098 stdLambdaDict: `dict` 

1099 Mapping of stdFilterName to stdLambda from LUT 

1100 

1101 Returns 

1102 ------- 

1103 referenceFilterNames: `list` 

1104 Wavelength-ordered list of reference filter names 

1105 """ 

1106 

1107 # Find the unique list of filter names in visitCat 

1108 filterNames = np.unique(visitCat.asAstropy()['filtername']) 

1109 

1110 # Find the unique list of "standard" filters 

1111 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames} 

1112 

1113 # And sort these by wavelength 

1114 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get) 

1115 

1116 return referenceFilterNames