Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for BuildStars using src tables or sourceTable_visit tables. 

22""" 

23 

24import os 

25import sys 

26import traceback 

27import abc 

28 

29import numpy as np 

30 

31import lsst.daf.persistence as dafPersist 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34import lsst.afw.table as afwTable 

35import lsst.geom as geom 

36from lsst.daf.base import PropertyList 

37from lsst.daf.base.dateTime import DateTime 

38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

39 

40from .utilities import computeApertureRadiusFromDataRef 

41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask 

42 

43import fgcm 

44 

45REFSTARS_FORMAT_VERSION = 1 

46 

47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask'] 

48 

49 

50class FgcmBuildStarsConfigBase(pexConfig.Config): 

51 """Base config for FgcmBuildStars tasks""" 

52 

53 instFluxField = pexConfig.Field( 

54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. " 

55 "The associated flag will be implicitly included in badFlags"), 

56 dtype=str, 

57 default='slot_CalibFlux_instFlux', 

58 ) 

59 minPerBand = pexConfig.Field( 

60 doc="Minimum observations per band", 

61 dtype=int, 

62 default=2, 

63 ) 

64 matchRadius = pexConfig.Field( 

65 doc="Match radius (arcseconds)", 

66 dtype=float, 

67 default=1.0, 

68 ) 

69 isolationRadius = pexConfig.Field( 

70 doc="Isolation radius (arcseconds)", 

71 dtype=float, 

72 default=2.0, 

73 ) 

74 densityCutNside = pexConfig.Field( 

75 doc="Density cut healpix nside", 

76 dtype=int, 

77 default=128, 

78 ) 

79 densityCutMaxPerPixel = pexConfig.Field( 

80 doc="Density cut number of stars per pixel", 

81 dtype=int, 

82 default=1000, 

83 ) 

84 randomSeed = pexConfig.Field( 

85 doc="Random seed for high density down-sampling.", 

86 dtype=int, 

87 default=None, 

88 optional=True, 

89 ) 

90 matchNside = pexConfig.Field( 

91 doc="Healpix Nside for matching", 

92 dtype=int, 

93 default=4096, 

94 ) 

95 coarseNside = pexConfig.Field( 

96 doc="Healpix coarse Nside for partitioning matches", 

97 dtype=int, 

98 default=8, 

99 ) 

100 # The following config will not be necessary after Gen2 retirement. 

101 # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band' 

102 # which is easiest to access in the config file. 

103 physicalFilterMap = pexConfig.DictField( 

104 doc="Mapping from 'physicalFilter' to band.", 

105 keytype=str, 

106 itemtype=str, 

107 default={}, 

108 ) 

109 requiredBands = pexConfig.ListField( 

110 doc="Bands required for each star", 

111 dtype=str, 

112 default=(), 

113 ) 

114 primaryBands = pexConfig.ListField( 

115 doc=("Bands for 'primary' star matches. " 

116 "A star must be observed in one of these bands to be considered " 

117 "as a calibration star."), 

118 dtype=str, 

119 default=None 

120 ) 

121 visitDataRefName = pexConfig.Field( 

122 doc="dataRef name for the 'visit' field, usually 'visit'.", 

123 dtype=str, 

124 default="visit" 

125 ) 

126 ccdDataRefName = pexConfig.Field( 

127 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.", 

128 dtype=str, 

129 default="ccd" 

130 ) 

131 doApplyWcsJacobian = pexConfig.Field( 

132 doc="Apply the jacobian of the WCS to the star observations prior to fit?", 

133 dtype=bool, 

134 default=True 

135 ) 

136 doModelErrorsWithBackground = pexConfig.Field( 

137 doc="Model flux errors with background term?", 

138 dtype=bool, 

139 default=True 

140 ) 

141 psfCandidateName = pexConfig.Field( 

142 doc="Name of field with psf candidate flag for propagation", 

143 dtype=str, 

144 default="calib_psf_candidate" 

145 ) 

146 doSubtractLocalBackground = pexConfig.Field( 

147 doc=("Subtract the local background before performing calibration? " 

148 "This is only supported for circular aperture calibration fluxes."), 

149 dtype=bool, 

150 default=False 

151 ) 

152 localBackgroundFluxField = pexConfig.Field( 

153 doc="Full name of the local background instFlux field to use.", 

154 dtype=str, 

155 default='base_LocalBackground_instFlux' 

156 ) 

157 sourceSelector = sourceSelectorRegistry.makeField( 

158 doc="How to select sources", 

159 default="science" 

160 ) 

161 apertureInnerInstFluxField = pexConfig.Field( 

162 doc=("Full name of instFlux field that contains inner aperture " 

163 "flux for aperture correction proxy"), 

164 dtype=str, 

165 default='base_CircularApertureFlux_12_0_instFlux' 

166 ) 

167 apertureOuterInstFluxField = pexConfig.Field( 

168 doc=("Full name of instFlux field that contains outer aperture " 

169 "flux for aperture correction proxy"), 

170 dtype=str, 

171 default='base_CircularApertureFlux_17_0_instFlux' 

172 ) 

173 doReferenceMatches = pexConfig.Field( 

174 doc="Match reference catalog as additional constraint on calibration", 

175 dtype=bool, 

176 default=True, 

177 ) 

178 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField( 

179 target=FgcmLoadReferenceCatalogTask, 

180 doc="FGCM reference object loader", 

181 ) 

182 nVisitsPerCheckpoint = pexConfig.Field( 

183 doc="Number of visits read between checkpoints", 

184 dtype=int, 

185 default=500, 

186 ) 

187 

188 def setDefaults(self): 

189 sourceSelector = self.sourceSelector["science"] 

190 sourceSelector.setDefaults() 

191 

192 sourceSelector.doFlags = True 

193 sourceSelector.doUnresolved = True 

194 sourceSelector.doSignalToNoise = True 

195 sourceSelector.doIsolated = True 

196 

197 sourceSelector.signalToNoise.minimum = 10.0 

198 sourceSelector.signalToNoise.maximum = 1000.0 

199 

200 # FGCM operates on unresolved sources, and this setting is 

201 # appropriate for the current base_ClassificationExtendedness 

202 sourceSelector.unresolved.maximum = 0.5 

203 

204 

205class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner): 

206 """Subclass of TaskRunner for FgcmBuildStars tasks 

207 

208 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of 

209 arguments, one of which is the butler (for persistence and mapper data), 

210 and a list of dataRefs extracted from the command line. Note that FGCM 

211 runs on a large set of dataRefs, and not on single dataRef/tract/patch. 

212 This class transforms the process arguments generated by the ArgumentParser 

213 into the arguments expected by FgcmBuildStarsTask.run(). This runner does 

214 not use any parallelization. 

215 """ 

216 @staticmethod 

217 def getTargetList(parsedCmd): 

218 """ 

219 Return a list with one element: a tuple with the butler and 

220 list of dataRefs 

221 """ 

222 # we want to combine the butler with any (or no!) dataRefs 

223 return [(parsedCmd.butler, parsedCmd.id.refList)] 

224 

225 def __call__(self, args): 

226 """ 

227 Parameters 

228 ---------- 

229 args: `tuple` with (butler, dataRefList) 

230 

231 Returns 

232 ------- 

233 exitStatus: `list` with `lsst.pipe.base.Struct` 

234 exitStatus (0: success; 1: failure) 

235 """ 

236 butler, dataRefList = args 

237 

238 task = self.TaskClass(config=self.config, log=self.log) 

239 

240 exitStatus = 0 

241 if self.doRaise: 

242 task.runDataRef(butler, dataRefList) 

243 else: 

244 try: 

245 task.runDataRef(butler, dataRefList) 

246 except Exception as e: 

247 exitStatus = 1 

248 task.log.fatal("Failed: %s" % e) 

249 if not isinstance(e, pipeBase.TaskError): 

250 traceback.print_exc(file=sys.stderr) 

251 

252 task.writeMetadata(butler) 

253 

254 # The task does not return any results: 

255 return [pipeBase.Struct(exitStatus=exitStatus)] 

256 

257 def run(self, parsedCmd): 

258 """ 

259 Run the task, with no multiprocessing 

260 

261 Parameters 

262 ---------- 

263 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

264 """ 

265 

266 resultList = [] 

267 

268 if self.precall(parsedCmd): 

269 targetList = self.getTargetList(parsedCmd) 

270 resultList = self(targetList[0]) 

271 

272 return resultList 

273 

274 

275class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC): 

276 """ 

277 Base task to build stars for FGCM global calibration 

278 

279 Parameters 

280 ---------- 

281 butler : `lsst.daf.persistence.Butler` 

282 """ 

283 def __init__(self, initInputs=None, butler=None, **kwargs): 

284 super().__init__(**kwargs) 

285 

286 self.makeSubtask("sourceSelector") 

287 # Only log warning and fatal errors from the sourceSelector 

288 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN) 

289 

290 # no saving of metadata for now 

291 def _getMetadataName(self): 

292 return None 

293 

294 @pipeBase.timeMethod 

295 def runDataRef(self, butler, dataRefs): 

296 """ 

297 Cross-match and make star list for FGCM Input 

298 

299 Parameters 

300 ---------- 

301 butler: `lsst.daf.persistence.Butler` 

302 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

303 Source data references for the input visits. 

304 

305 Raises 

306 ------ 

307 RuntimeErrror: Raised if `config.doReferenceMatches` is set and 

308 an fgcmLookUpTable is not available, or if computeFluxApertureRadius() 

309 fails if the calibFlux is not a CircularAperture flux. 

310 """ 

311 datasetType = dataRefs[0].butlerSubset.datasetType 

312 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType) 

313 

314 if self.config.doReferenceMatches: 

315 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler) 

316 # Ensure that we have a LUT 

317 if not butler.datasetExists('fgcmLookUpTable'): 

318 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches") 

319 # Compute aperture radius if necessary. This is useful to do now before 

320 # any heavy lifting has happened (fail early). 

321 calibFluxApertureRadius = None 

322 if self.config.doSubtractLocalBackground: 

323 try: 

324 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

325 self.config.instFluxField) 

326 except RuntimeError as e: 

327 raise RuntimeError("Could not determine aperture radius from %s. " 

328 "Cannot use doSubtractLocalBackground." % 

329 (self.config.instFluxField)) from e 

330 

331 camera = butler.get('camera') 

332 groupedDataRefs = self._findAndGroupDataRefsGen2(butler, camera, dataRefs) 

333 

334 # Make the visit catalog if necessary 

335 # First check if the visit catalog is in the _current_ path 

336 # We cannot use Gen2 datasetExists() because that checks all parent 

337 # directories as well, which would make recovering from faults 

338 # and fgcmcal reruns impossible. 

339 visitCatDataRef = butler.dataRef('fgcmVisitCatalog') 

340 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0] 

341 if os.path.exists(filename): 

342 # This file exists and we should continue processing 

343 inVisitCat = visitCatDataRef.get() 

344 if len(inVisitCat) != len(groupedDataRefs): 

345 raise RuntimeError("Existing visitCatalog found, but has an inconsistent " 

346 "number of visits. Cannot continue.") 

347 else: 

348 inVisitCat = None 

349 

350 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs, 

351 visitCatDataRef=visitCatDataRef, 

352 inVisitCat=inVisitCat) 

353 

354 # Persist the visitCat as a checkpoint file. 

355 visitCatDataRef.put(visitCat) 

356 

357 starObsDataRef = butler.dataRef('fgcmStarObservations') 

358 filename = starObsDataRef.get('fgcmStarObservations_filename')[0] 

359 if os.path.exists(filename): 

360 inStarObsCat = starObsDataRef.get() 

361 else: 

362 inStarObsCat = None 

363 

364 rad = calibFluxApertureRadius 

365 sourceSchemaDataRef = butler.dataRef('src_schema') 

366 sourceSchema = sourceSchemaDataRef.get('src_schema', immediate=True).schema 

367 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs, 

368 visitCat, 

369 sourceSchema, 

370 camera, 

371 calibFluxApertureRadius=rad, 

372 starObsDataRef=starObsDataRef, 

373 visitCatDataRef=visitCatDataRef, 

374 inStarObsCat=inStarObsCat) 

375 visitCatDataRef.put(visitCat) 

376 starObsDataRef.put(fgcmStarObservationCat) 

377 

378 # Always do the matching. 

379 if self.config.doReferenceMatches: 

380 lutDataRef = butler.dataRef('fgcmLookUpTable') 

381 else: 

382 lutDataRef = None 

383 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

384 fgcmStarObservationCat, 

385 lutDataRef=lutDataRef) 

386 

387 # Persist catalogs via the butler 

388 butler.put(fgcmStarIdCat, 'fgcmStarIds') 

389 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') 

390 if fgcmRefCat is not None: 

391 butler.put(fgcmRefCat, 'fgcmReferenceStars') 

392 

393 @abc.abstractmethod 

394 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs): 

395 """ 

396 Find and group dataRefs (by visit); Gen2 only. 

397 

398 Parameters 

399 ---------- 

400 butler : `lsst.daf.persistence.Butler` 

401 Gen2 butler. 

402 camera : `lsst.afw.cameraGeom.Camera` 

403 Camera from the butler. 

404 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

405 Data references for the input visits. 

406 

407 Returns 

408 ------- 

409 groupedDataRefs : `dict` [`int`, `list`] 

410 Dictionary with sorted visit keys, and `list`s of 

411 `lsst.daf.persistence.ButlerDataRef` 

412 """ 

413 raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.") 

414 

415 @abc.abstractmethod 

416 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

417 sourceSchema, 

418 camera, 

419 calibFluxApertureRadius=None, 

420 visitCatDataRef=None, 

421 starObsDataRef=None, 

422 inStarObsCat=None): 

423 """ 

424 Compile all good star observations from visits in visitCat. Checkpoint files 

425 will be stored if both visitCatDataRef and starObsDataRef are not None. 

426 

427 Parameters 

428 ---------- 

429 groupedDataRefs : `dict` of `list`s 

430 Lists of `~lsst.daf.persistence.ButlerDataRef` or 

431 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit. 

432 visitCat : `~afw.table.BaseCatalog` 

433 Catalog with visit data for FGCM 

434 sourceSchema : `~lsst.afw.table.Schema` 

435 Schema for the input src catalogs. 

436 camera : `~lsst.afw.cameraGeom.Camera` 

437 calibFluxApertureRadius : `float`, optional 

438 Aperture radius for calibration flux. 

439 visitCatDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional 

440 Dataref to write visitCat for checkpoints 

441 starObsDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional 

442 Dataref to write the star observation catalog for checkpoints. 

443 inStarObsCat : `~afw.table.BaseCatalog` 

444 Input observation catalog. If this is incomplete, observations 

445 will be appended from when it was cut off. 

446 

447 Returns 

448 ------- 

449 fgcmStarObservations : `afw.table.BaseCatalog` 

450 Full catalog of good observations. 

451 

452 Raises 

453 ------ 

454 RuntimeError: Raised if doSubtractLocalBackground is True and 

455 calibFluxApertureRadius is not set. 

456 """ 

457 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.") 

458 

459 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, 

460 visitCatDataRef=None, inVisitCat=None): 

461 """ 

462 Make a visit catalog with all the keys from each visit 

463 

464 Parameters 

465 ---------- 

466 camera: `lsst.afw.cameraGeom.Camera` 

467 Camera from the butler 

468 groupedDataRefs: `dict` 

469 Dictionary with visit keys, and `list`s of 

470 `lsst.daf.persistence.ButlerDataRef` 

471 bkgDataRefDict: `dict`, optional 

472 Dictionary of gen3 dataRefHandles for background info. 

473 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

474 Dataref to write visitCat for checkpoints 

475 inVisitCat: `afw.table.BaseCatalog`, optional 

476 Input (possibly incomplete) visit catalog 

477 

478 Returns 

479 ------- 

480 visitCat: `afw.table.BaseCatalog` 

481 """ 

482 

483 self.log.info("Assembling visitCatalog from %d %ss" % 

484 (len(groupedDataRefs), self.config.visitDataRefName)) 

485 

486 nCcd = len(camera) 

487 

488 if inVisitCat is None: 

489 schema = self._makeFgcmVisitSchema(nCcd) 

490 

491 visitCat = afwTable.BaseCatalog(schema) 

492 visitCat.reserve(len(groupedDataRefs)) 

493 visitCat.resize(len(groupedDataRefs)) 

494 

495 visitCat['visit'] = list(groupedDataRefs.keys()) 

496 visitCat['used'] = 0 

497 visitCat['sources_read'] = False 

498 else: 

499 visitCat = inVisitCat 

500 

501 # No matter what, fill the catalog. This will check if it was 

502 # already read. 

503 self._fillVisitCatalog(visitCat, groupedDataRefs, 

504 bkgDataRefDict=bkgDataRefDict, 

505 visitCatDataRef=visitCatDataRef) 

506 

507 return visitCat 

508 

509 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, 

510 visitCatDataRef=None): 

511 """ 

512 Fill the visit catalog with visit metadata 

513 

514 Parameters 

515 ---------- 

516 visitCat : `afw.table.BaseCatalog` 

517 Visit catalog. See _makeFgcmVisitSchema() for schema definition. 

518 groupedDataRefs : `dict` 

519 Dictionary with visit keys, and `list`s of 

520 `lsst.daf.persistence.ButlerDataRef` or 

521 `lsst.daf.butler.DeferredDatasetHandle` 

522 visitCatDataRef : `lsst.daf.persistence.ButlerDataRef`, optional 

523 Dataref to write ``visitCat`` for checkpoints. Gen2 only. 

524 bkgDataRefDict : `dict`, optional 

525 Dictionary of Gen3 `lsst.daf.butler.DeferredDatasetHandle` 

526 for background info. 

527 """ 

528 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1)) 

529 

530 for i, visit in enumerate(groupedDataRefs): 

531 # We don't use the bypasses since we need the psf info which does 

532 # not have a bypass 

533 # TODO: When DM-15500 is implemented in the Gen3 Butler, this 

534 # can be fixed 

535 

536 # Do not read those that have already been read 

537 if visitCat['used'][i]: 

538 continue 

539 

540 if (i % self.config.nVisitsPerCheckpoint) == 0: 

541 self.log.info("Retrieving metadata for %s %d (%d/%d)" % 

542 (self.config.visitDataRefName, visit, i, len(groupedDataRefs))) 

543 # Save checkpoint if desired 

544 if visitCatDataRef is not None: 

545 visitCatDataRef.put(visitCat) 

546 

547 dataRef = groupedDataRefs[visit][0] 

548 if isinstance(dataRef, dafPersist.ButlerDataRef): 

549 # Gen2: calexp dataRef 

550 # The first dataRef in the group will be the reference ccd (if available) 

551 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox) 

552 visitInfo = exp.getInfo().getVisitInfo() 

553 label = dataRef.get(datasetType='calexp_filterLabel') 

554 physicalFilter = label.physicalLabel 

555 psf = exp.getPsf() 

556 psfSigma = psf.computeShape().getDeterminantRadius() 

557 else: 

558 # Gen3: use the visitSummary dataRef 

559 summary = dataRef.get() 

560 

561 summaryRow = summary.find(self.config.referenceCCD) 

562 if summaryRow is None: 

563 # Take the first available ccd if reference isn't available 

564 summaryRow = summary[0] 

565 

566 summaryDetector = summaryRow['id'] 

567 visitInfo = summaryRow.getVisitInfo() 

568 physicalFilter = summaryRow['physical_filter'] 

569 # Compute the median psf sigma if possible 

570 goodSigma, = np.where(summary['psfSigma'] > 0) 

571 if goodSigma.size > 2: 

572 psfSigma = np.median(summary['psfSigma'][goodSigma]) 

573 elif goodSigma > 0: 

574 psfSigma = np.mean(summary['psfSigma'][goodSigma]) 

575 else: 

576 psfSigma = 0.0 

577 

578 rec = visitCat[i] 

579 rec['visit'] = visit 

580 rec['physicalFilter'] = physicalFilter 

581 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it 

582 # much easier to get the wcs's necessary to recompute the pointing 

583 # ra/dec at the center of the camera. 

584 radec = visitInfo.getBoresightRaDec() 

585 rec['telra'] = radec.getRa().asDegrees() 

586 rec['teldec'] = radec.getDec().asDegrees() 

587 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees() 

588 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees() 

589 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD) 

590 rec['exptime'] = visitInfo.getExposureTime() 

591 # convert from Pa to millibar 

592 # Note that I don't know if this unit will need to be per-camera config 

593 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100 

594 # Flag to signify if this is a "deep" field. Not currently used 

595 rec['deepFlag'] = 0 

596 # Relative flat scaling (1.0 means no relative scaling) 

597 rec['scaling'][:] = 1.0 

598 # Median delta aperture, to be measured from stars 

599 rec['deltaAper'] = 0.0 

600 rec['psfSigma'] = psfSigma 

601 

602 if self.config.doModelErrorsWithBackground: 

603 foundBkg = False 

604 if isinstance(dataRef, dafPersist.ButlerDataRef): 

605 # Gen2-style dataRef 

606 det = dataRef.dataId[self.config.ccdDataRefName] 

607 if dataRef.datasetExists(datasetType='calexpBackground'): 

608 bgList = dataRef.get(datasetType='calexpBackground') 

609 foundBkg = True 

610 else: 

611 # Gen3-style dataRef 

612 try: 

613 # Use the same detector used from the summary. 

614 bkgRef = bkgDataRefDict[(visit, summaryDetector)] 

615 bgList = bkgRef.get() 

616 foundBkg = True 

617 except KeyError: 

618 pass 

619 

620 if foundBkg: 

621 bgStats = (bg[0].getStatsImage().getImage().array 

622 for bg in bgList) 

623 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats) 

624 else: 

625 self.log.warning('Sky background not found for visit %d / ccd %d' % 

626 (visit, det)) 

627 rec['skyBackground'] = -1.0 

628 else: 

629 rec['skyBackground'] = -1.0 

630 

631 rec['used'] = 1 

632 

633 def _makeSourceMapper(self, sourceSchema): 

634 """ 

635 Make a schema mapper for fgcm sources 

636 

637 Parameters 

638 ---------- 

639 sourceSchema: `afwTable.Schema` 

640 Default source schema from the butler 

641 

642 Returns 

643 ------- 

644 sourceMapper: `afwTable.schemaMapper` 

645 Mapper to the FGCM source schema 

646 """ 

647 

648 # create a mapper to the preferred output 

649 sourceMapper = afwTable.SchemaMapper(sourceSchema) 

650 

651 # map to ra/dec 

652 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

653 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

654 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x') 

655 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y') 

656 # Add the mapping if the field exists in the input catalog. 

657 # If the field does not exist, simply add it (set to False). 

658 # This field is not required for calibration, but is useful 

659 # to collate if available. 

660 try: 

661 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(), 

662 'psf_candidate') 

663 except LookupError: 

664 sourceMapper.editOutputSchema().addField( 

665 "psf_candidate", type='Flag', 

666 doc=("Flag set if the source was a candidate for PSF determination, " 

667 "as determined by the star selector.")) 

668 

669 # and add the fields we want 

670 sourceMapper.editOutputSchema().addField( 

671 "visit", type=np.int32, doc="Visit number") 

672 sourceMapper.editOutputSchema().addField( 

673 "ccd", type=np.int32, doc="CCD number") 

674 sourceMapper.editOutputSchema().addField( 

675 "instMag", type=np.float32, doc="Instrumental magnitude") 

676 sourceMapper.editOutputSchema().addField( 

677 "instMagErr", type=np.float32, doc="Instrumental magnitude error") 

678 sourceMapper.editOutputSchema().addField( 

679 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian") 

680 sourceMapper.editOutputSchema().addField( 

681 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset") 

682 

683 return sourceMapper 

684 

685 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None): 

686 """ 

687 Use FGCM code to match observations into unique stars. 

688 

689 Parameters 

690 ---------- 

691 visitCat: `afw.table.BaseCatalog` 

692 Catalog with visit data for fgcm 

693 obsCat: `afw.table.BaseCatalog` 

694 Full catalog of star observations for fgcm 

695 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or 

696 `lsst.daf.butler.DeferredDatasetHandle`, optional 

697 Data reference to fgcm look-up table (used if matching reference stars). 

698 

699 Returns 

700 ------- 

701 fgcmStarIdCat: `afw.table.BaseCatalog` 

702 Catalog of unique star identifiers and index keys 

703 fgcmStarIndicesCat: `afwTable.BaseCatalog` 

704 Catalog of unique star indices 

705 fgcmRefCat: `afw.table.BaseCatalog` 

706 Catalog of matched reference stars. 

707 Will be None if `config.doReferenceMatches` is False. 

708 """ 

709 # get filter names into a numpy array... 

710 # This is the type that is expected by the fgcm code 

711 visitFilterNames = np.zeros(len(visitCat), dtype='a30') 

712 for i in range(len(visitCat)): 

713 visitFilterNames[i] = visitCat[i]['physicalFilter'] 

714 

715 # match to put filterNames with observations 

716 visitIndex = np.searchsorted(visitCat['visit'], 

717 obsCat['visit']) 

718 

719 obsFilterNames = visitFilterNames[visitIndex] 

720 

721 if self.config.doReferenceMatches: 

722 # Get the reference filter names, using the LUT 

723 lutCat = lutDataRef.get() 

724 

725 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in 

726 zip(lutCat[0]['physicalFilters'].split(','), 

727 lutCat[0]['stdPhysicalFilters'].split(','))} 

728 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in 

729 zip(lutCat[0]['stdPhysicalFilters'].split(','), 

730 lutCat[0]['lambdaStdFilter'])} 

731 

732 del lutCat 

733 

734 referenceFilterNames = self._getReferenceFilterNames(visitCat, 

735 stdFilterDict, 

736 stdLambdaDict) 

737 self.log.info("Using the following reference filters: %s" % 

738 (', '.join(referenceFilterNames))) 

739 

740 else: 

741 # This should be an empty list 

742 referenceFilterNames = [] 

743 

744 # make the fgcm starConfig dict 

745 starConfig = {'logger': self.log, 

746 'filterToBand': self.config.physicalFilterMap, 

747 'requiredBands': self.config.requiredBands, 

748 'minPerBand': self.config.minPerBand, 

749 'matchRadius': self.config.matchRadius, 

750 'isolationRadius': self.config.isolationRadius, 

751 'matchNSide': self.config.matchNside, 

752 'coarseNSide': self.config.coarseNside, 

753 'densNSide': self.config.densityCutNside, 

754 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 

755 'randomSeed': self.config.randomSeed, 

756 'primaryBands': self.config.primaryBands, 

757 'referenceFilterNames': referenceFilterNames} 

758 

759 # initialize the FgcmMakeStars object 

760 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) 

761 

762 # make the primary stars 

763 # note that the ra/dec native Angle format is radians 

764 # We determine the conversion from the native units (typically 

765 # radians) to degrees for the first observation. This allows us 

766 # to treate ra/dec as numpy arrays rather than Angles, which would 

767 # be approximately 600x slower. 

768 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) 

769 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, 

770 obsCat['dec'] * conv, 

771 filterNameArray=obsFilterNames, 

772 bandSelected=False) 

773 

774 # and match all the stars 

775 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, 

776 obsCat['dec'] * conv, 

777 obsFilterNames) 

778 

779 if self.config.doReferenceMatches: 

780 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) 

781 

782 # now persist 

783 

784 objSchema = self._makeFgcmObjSchema() 

785 

786 # make catalog and records 

787 fgcmStarIdCat = afwTable.BaseCatalog(objSchema) 

788 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) 

789 for i in range(fgcmMakeStars.objIndexCat.size): 

790 fgcmStarIdCat.addNew() 

791 

792 # fill the catalog 

793 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] 

794 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] 

795 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] 

796 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex'] 

797 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] 

798 

799 obsSchema = self._makeFgcmObsSchema() 

800 

801 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) 

802 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) 

803 for i in range(fgcmMakeStars.obsIndexCat.size): 

804 fgcmStarIndicesCat.addNew() 

805 

806 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex'] 

807 

808 if self.config.doReferenceMatches: 

809 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) 

810 

811 fgcmRefCat = afwTable.BaseCatalog(refSchema) 

812 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) 

813 

814 for i in range(fgcmMakeStars.referenceCat.size): 

815 fgcmRefCat.addNew() 

816 

817 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] 

818 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] 

819 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr'] 

820 

821 md = PropertyList() 

822 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION) 

823 md.set("FILTERNAMES", referenceFilterNames) 

824 fgcmRefCat.setMetadata(md) 

825 

826 else: 

827 fgcmRefCat = None 

828 

829 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat 

830 

831 def _makeFgcmVisitSchema(self, nCcd): 

832 """ 

833 Make a schema for an fgcmVisitCatalog 

834 

835 Parameters 

836 ---------- 

837 nCcd: `int` 

838 Number of CCDs in the camera 

839 

840 Returns 

841 ------- 

842 schema: `afwTable.Schema` 

843 """ 

844 

845 schema = afwTable.Schema() 

846 schema.addField('visit', type=np.int32, doc="Visit number") 

847 schema.addField('physicalFilter', type=str, size=30, doc="Physical filter") 

848 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") 

849 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") 

850 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") 

851 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)") 

852 schema.addField('mjd', type=np.float64, doc="MJD of visit") 

853 schema.addField('exptime', type=np.float32, doc="Exposure time") 

854 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") 

855 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") 

856 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") 

857 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") 

858 # the following field is not used yet 

859 schema.addField('deepFlag', type=np.int32, doc="Deep observation") 

860 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", 

861 size=nCcd) 

862 schema.addField('used', type=np.int32, doc="This visit has been ingested.") 

863 schema.addField('sources_read', type='Flag', doc="This visit had sources read.") 

864 

865 return schema 

866 

867 def _makeFgcmObjSchema(self): 

868 """ 

869 Make a schema for the objIndexCat from fgcmMakeStars 

870 

871 Returns 

872 ------- 

873 schema: `afwTable.Schema` 

874 """ 

875 

876 objSchema = afwTable.Schema() 

877 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

878 # Will investigate making these angles... 

879 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)') 

880 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)') 

881 objSchema.addField('obsArrIndex', type=np.int32, 

882 doc='Index in obsIndexTable for first observation') 

883 objSchema.addField('nObs', type=np.int32, doc='Total number of observations') 

884 

885 return objSchema 

886 

887 def _makeFgcmObsSchema(self): 

888 """ 

889 Make a schema for the obsIndexCat from fgcmMakeStars 

890 

891 Returns 

892 ------- 

893 schema: `afwTable.Schema` 

894 """ 

895 

896 obsSchema = afwTable.Schema() 

897 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table') 

898 

899 return obsSchema 

900 

901 def _makeFgcmRefSchema(self, nReferenceBands): 

902 """ 

903 Make a schema for the referenceCat from fgcmMakeStars 

904 

905 Parameters 

906 ---------- 

907 nReferenceBands: `int` 

908 Number of reference bands 

909 

910 Returns 

911 ------- 

912 schema: `afwTable.Schema` 

913 """ 

914 

915 refSchema = afwTable.Schema() 

916 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

917 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)', 

918 size=nReferenceBands) 

919 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array', 

920 size=nReferenceBands) 

921 

922 return refSchema 

923 

924 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict): 

925 """ 

926 Get the reference filter names, in wavelength order, from the visitCat and 

927 information from the look-up-table. 

928 

929 Parameters 

930 ---------- 

931 visitCat: `afw.table.BaseCatalog` 

932 Catalog with visit data for FGCM 

933 stdFilterDict: `dict` 

934 Mapping of filterName to stdFilterName from LUT 

935 stdLambdaDict: `dict` 

936 Mapping of stdFilterName to stdLambda from LUT 

937 

938 Returns 

939 ------- 

940 referenceFilterNames: `list` 

941 Wavelength-ordered list of reference filter names 

942 """ 

943 

944 # Find the unique list of filter names in visitCat 

945 filterNames = np.unique(visitCat.asAstropy()['physicalFilter']) 

946 

947 # Find the unique list of "standard" filters 

948 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames} 

949 

950 # And sort these by wavelength 

951 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get) 

952 

953 return referenceFilterNames