Coverage for python/lsst/fgcmcal/fgcmBuildStarsBase.py: 18%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

328 statements  

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for BuildStars using src tables or sourceTable_visit tables. 

22""" 

23 

24import os 

25import sys 

26import traceback 

27import abc 

28 

29import numpy as np 

30 

31import lsst.daf.persistence as dafPersist 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34import lsst.afw.table as afwTable 

35import lsst.geom as geom 

36from lsst.daf.base import PropertyList 

37from lsst.daf.base.dateTime import DateTime 

38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

39 

40from .utilities import computeApertureRadiusFromDataRef 

41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask 

42 

43import fgcm 

44 

45REFSTARS_FORMAT_VERSION = 1 

46 

47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask'] 

48 

49 

50class FgcmBuildStarsConfigBase(pexConfig.Config): 

51 """Base config for FgcmBuildStars tasks""" 

52 

53 instFluxField = pexConfig.Field( 

54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. " 

55 "The associated flag will be implicitly included in badFlags"), 

56 dtype=str, 

57 default='slot_CalibFlux_instFlux', 

58 ) 

59 minPerBand = pexConfig.Field( 

60 doc="Minimum observations per band", 

61 dtype=int, 

62 default=2, 

63 ) 

64 matchRadius = pexConfig.Field( 

65 doc="Match radius (arcseconds)", 

66 dtype=float, 

67 default=1.0, 

68 ) 

69 isolationRadius = pexConfig.Field( 

70 doc="Isolation radius (arcseconds)", 

71 dtype=float, 

72 default=2.0, 

73 ) 

74 densityCutNside = pexConfig.Field( 

75 doc="Density cut healpix nside", 

76 dtype=int, 

77 default=128, 

78 ) 

79 densityCutMaxPerPixel = pexConfig.Field( 

80 doc="Density cut number of stars per pixel", 

81 dtype=int, 

82 default=1000, 

83 ) 

84 randomSeed = pexConfig.Field( 

85 doc="Random seed for high density down-sampling.", 

86 dtype=int, 

87 default=None, 

88 optional=True, 

89 ) 

90 matchNside = pexConfig.Field( 

91 doc="Healpix Nside for matching", 

92 dtype=int, 

93 default=4096, 

94 ) 

95 coarseNside = pexConfig.Field( 

96 doc="Healpix coarse Nside for partitioning matches", 

97 dtype=int, 

98 default=8, 

99 ) 

100 # The following config will not be necessary after Gen2 retirement. 

101 # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band' 

102 # which is easiest to access in the config file. 

103 physicalFilterMap = pexConfig.DictField( 

104 doc="Mapping from 'physicalFilter' to band.", 

105 keytype=str, 

106 itemtype=str, 

107 default={}, 

108 ) 

109 requiredBands = pexConfig.ListField( 

110 doc="Bands required for each star", 

111 dtype=str, 

112 default=(), 

113 ) 

114 primaryBands = pexConfig.ListField( 

115 doc=("Bands for 'primary' star matches. " 

116 "A star must be observed in one of these bands to be considered " 

117 "as a calibration star."), 

118 dtype=str, 

119 default=None 

120 ) 

121 visitDataRefName = pexConfig.Field( 

122 doc="dataRef name for the 'visit' field, usually 'visit'.", 

123 dtype=str, 

124 default="visit" 

125 ) 

126 ccdDataRefName = pexConfig.Field( 

127 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.", 

128 dtype=str, 

129 default="ccd" 

130 ) 

131 doApplyWcsJacobian = pexConfig.Field( 

132 doc="Apply the jacobian of the WCS to the star observations prior to fit?", 

133 dtype=bool, 

134 default=True 

135 ) 

136 doModelErrorsWithBackground = pexConfig.Field( 

137 doc="Model flux errors with background term?", 

138 dtype=bool, 

139 default=True 

140 ) 

141 psfCandidateName = pexConfig.Field( 

142 doc="Name of field with psf candidate flag for propagation", 

143 dtype=str, 

144 default="calib_psf_candidate" 

145 ) 

146 doSubtractLocalBackground = pexConfig.Field( 

147 doc=("Subtract the local background before performing calibration? " 

148 "This is only supported for circular aperture calibration fluxes."), 

149 dtype=bool, 

150 default=False 

151 ) 

152 localBackgroundFluxField = pexConfig.Field( 

153 doc="Full name of the local background instFlux field to use.", 

154 dtype=str, 

155 default='base_LocalBackground_instFlux' 

156 ) 

157 sourceSelector = sourceSelectorRegistry.makeField( 

158 doc="How to select sources", 

159 default="science" 

160 ) 

161 apertureInnerInstFluxField = pexConfig.Field( 

162 doc=("Full name of instFlux field that contains inner aperture " 

163 "flux for aperture correction proxy"), 

164 dtype=str, 

165 default='base_CircularApertureFlux_12_0_instFlux' 

166 ) 

167 apertureOuterInstFluxField = pexConfig.Field( 

168 doc=("Full name of instFlux field that contains outer aperture " 

169 "flux for aperture correction proxy"), 

170 dtype=str, 

171 default='base_CircularApertureFlux_17_0_instFlux' 

172 ) 

173 doReferenceMatches = pexConfig.Field( 

174 doc="Match reference catalog as additional constraint on calibration", 

175 dtype=bool, 

176 default=True, 

177 ) 

178 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField( 

179 target=FgcmLoadReferenceCatalogTask, 

180 doc="FGCM reference object loader", 

181 ) 

182 nVisitsPerCheckpoint = pexConfig.Field( 

183 doc="Number of visits read between checkpoints", 

184 dtype=int, 

185 default=500, 

186 ) 

187 

188 def setDefaults(self): 

189 sourceSelector = self.sourceSelector["science"] 

190 sourceSelector.setDefaults() 

191 

192 sourceSelector.doFlags = True 

193 sourceSelector.doUnresolved = True 

194 sourceSelector.doSignalToNoise = True 

195 sourceSelector.doIsolated = True 

196 

197 sourceSelector.signalToNoise.minimum = 10.0 

198 sourceSelector.signalToNoise.maximum = 1000.0 

199 

200 # FGCM operates on unresolved sources, and this setting is 

201 # appropriate for the current base_ClassificationExtendedness 

202 sourceSelector.unresolved.maximum = 0.5 

203 

204 

205class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner): 

206 """Subclass of TaskRunner for FgcmBuildStars tasks 

207 

208 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of 

209 arguments, one of which is the butler (for persistence and mapper data), 

210 and a list of dataRefs extracted from the command line. Note that FGCM 

211 runs on a large set of dataRefs, and not on single dataRef/tract/patch. 

212 This class transforms the process arguments generated by the ArgumentParser 

213 into the arguments expected by FgcmBuildStarsTask.run(). This runner does 

214 not use any parallelization. 

215 """ 

216 @staticmethod 

217 def getTargetList(parsedCmd): 

218 """ 

219 Return a list with one element: a tuple with the butler and 

220 list of dataRefs 

221 """ 

222 # we want to combine the butler with any (or no!) dataRefs 

223 return [(parsedCmd.butler, parsedCmd.id.refList)] 

224 

225 def __call__(self, args): 

226 """ 

227 Parameters 

228 ---------- 

229 args: `tuple` with (butler, dataRefList) 

230 

231 Returns 

232 ------- 

233 exitStatus: `list` with `lsst.pipe.base.Struct` 

234 exitStatus (0: success; 1: failure) 

235 """ 

236 butler, dataRefList = args 

237 

238 task = self.TaskClass(config=self.config, log=self.log) 

239 

240 exitStatus = 0 

241 if self.doRaise: 

242 task.runDataRef(butler, dataRefList) 

243 else: 

244 try: 

245 task.runDataRef(butler, dataRefList) 

246 except Exception as e: 

247 exitStatus = 1 

248 task.log.fatal("Failed: %s" % e) 

249 if not isinstance(e, pipeBase.TaskError): 

250 traceback.print_exc(file=sys.stderr) 

251 

252 task.writeMetadata(butler) 

253 

254 # The task does not return any results: 

255 return [pipeBase.Struct(exitStatus=exitStatus)] 

256 

257 def run(self, parsedCmd): 

258 """ 

259 Run the task, with no multiprocessing 

260 

261 Parameters 

262 ---------- 

263 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

264 """ 

265 

266 resultList = [] 

267 

268 if self.precall(parsedCmd): 

269 targetList = self.getTargetList(parsedCmd) 

270 resultList = self(targetList[0]) 

271 

272 return resultList 

273 

274 

275class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC): 

276 """ 

277 Base task to build stars for FGCM global calibration 

278 

279 Parameters 

280 ---------- 

281 butler : `lsst.daf.persistence.Butler` 

282 """ 

283 def __init__(self, initInputs=None, butler=None, **kwargs): 

284 super().__init__(**kwargs) 

285 

286 self.makeSubtask("sourceSelector") 

287 # Only log warning and fatal errors from the sourceSelector 

288 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN) 

289 

290 # no saving of metadata for now 

291 def _getMetadataName(self): 

292 return None 

293 

294 @pipeBase.timeMethod 

295 def runDataRef(self, butler, dataRefs): 

296 """ 

297 Cross-match and make star list for FGCM Input 

298 

299 Parameters 

300 ---------- 

301 butler: `lsst.daf.persistence.Butler` 

302 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

303 Source data references for the input visits. 

304 

305 Raises 

306 ------ 

307 RuntimeErrror: Raised if `config.doReferenceMatches` is set and 

308 an fgcmLookUpTable is not available, or if computeFluxApertureRadius() 

309 fails if the calibFlux is not a CircularAperture flux. 

310 """ 

311 datasetType = dataRefs[0].butlerSubset.datasetType 

312 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType) 

313 

314 if self.config.doReferenceMatches: 

315 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler) 

316 # Ensure that we have a LUT 

317 if not butler.datasetExists('fgcmLookUpTable'): 

318 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches") 

319 # Compute aperture radius if necessary. This is useful to do now before 

320 # any heavy lifting has happened (fail early). 

321 calibFluxApertureRadius = None 

322 if self.config.doSubtractLocalBackground: 

323 try: 

324 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

325 self.config.instFluxField) 

326 except RuntimeError as e: 

327 raise RuntimeError("Could not determine aperture radius from %s. " 

328 "Cannot use doSubtractLocalBackground." % 

329 (self.config.instFluxField)) from e 

330 

331 camera = butler.get('camera') 

332 groupedDataRefs = self._findAndGroupDataRefsGen2(butler, camera, dataRefs) 

333 

334 # Make the visit catalog if necessary 

335 # First check if the visit catalog is in the _current_ path 

336 # We cannot use Gen2 datasetExists() because that checks all parent 

337 # directories as well, which would make recovering from faults 

338 # and fgcmcal reruns impossible. 

339 visitCatDataRef = butler.dataRef('fgcmVisitCatalog') 

340 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0] 

341 if os.path.exists(filename): 

342 # This file exists and we should continue processing 

343 inVisitCat = visitCatDataRef.get() 

344 if len(inVisitCat) != len(groupedDataRefs): 

345 raise RuntimeError("Existing visitCatalog found, but has an inconsistent " 

346 "number of visits. Cannot continue.") 

347 else: 

348 inVisitCat = None 

349 

350 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs, 

351 visitCatDataRef=visitCatDataRef, 

352 inVisitCat=inVisitCat) 

353 

354 # Persist the visitCat as a checkpoint file. 

355 visitCatDataRef.put(visitCat) 

356 

357 starObsDataRef = butler.dataRef('fgcmStarObservations') 

358 filename = starObsDataRef.get('fgcmStarObservations_filename')[0] 

359 if os.path.exists(filename): 

360 inStarObsCat = starObsDataRef.get() 

361 else: 

362 inStarObsCat = None 

363 

364 rad = calibFluxApertureRadius 

365 sourceSchemaDataRef = butler.dataRef('src_schema') 

366 sourceSchema = sourceSchemaDataRef.get('src_schema', immediate=True).schema 

367 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs, 

368 visitCat, 

369 sourceSchema, 

370 camera, 

371 calibFluxApertureRadius=rad, 

372 starObsDataRef=starObsDataRef, 

373 visitCatDataRef=visitCatDataRef, 

374 inStarObsCat=inStarObsCat) 

375 visitCatDataRef.put(visitCat) 

376 starObsDataRef.put(fgcmStarObservationCat) 

377 

378 # Always do the matching. 

379 if self.config.doReferenceMatches: 

380 lutDataRef = butler.dataRef('fgcmLookUpTable') 

381 else: 

382 lutDataRef = None 

383 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

384 fgcmStarObservationCat, 

385 lutDataRef=lutDataRef) 

386 

387 # Persist catalogs via the butler 

388 butler.put(fgcmStarIdCat, 'fgcmStarIds') 

389 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') 

390 if fgcmRefCat is not None: 

391 butler.put(fgcmRefCat, 'fgcmReferenceStars') 

392 

393 @abc.abstractmethod 

394 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs): 

395 """ 

396 Find and group dataRefs (by visit); Gen2 only. 

397 

398 Parameters 

399 ---------- 

400 butler : `lsst.daf.persistence.Butler` 

401 Gen2 butler. 

402 camera : `lsst.afw.cameraGeom.Camera` 

403 Camera from the butler. 

404 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

405 Data references for the input visits. 

406 

407 Returns 

408 ------- 

409 groupedDataRefs : `dict` [`int`, `list`] 

410 Dictionary with sorted visit keys, and `list`s of 

411 `lsst.daf.persistence.ButlerDataRef` 

412 """ 

413 raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.") 

414 

415 @abc.abstractmethod 

416 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

417 sourceSchema, 

418 camera, 

419 calibFluxApertureRadius=None, 

420 visitCatDataRef=None, 

421 starObsDataRef=None, 

422 inStarObsCat=None): 

423 """ 

424 Compile all good star observations from visits in visitCat. Checkpoint files 

425 will be stored if both visitCatDataRef and starObsDataRef are not None. 

426 

427 Parameters 

428 ---------- 

429 groupedDataRefs : `dict` of `list`s 

430 Lists of `~lsst.daf.persistence.ButlerDataRef` or 

431 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit. 

432 visitCat : `~afw.table.BaseCatalog` 

433 Catalog with visit data for FGCM 

434 sourceSchema : `~lsst.afw.table.Schema` 

435 Schema for the input src catalogs. 

436 camera : `~lsst.afw.cameraGeom.Camera` 

437 calibFluxApertureRadius : `float`, optional 

438 Aperture radius for calibration flux. 

439 visitCatDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional 

440 Dataref to write visitCat for checkpoints 

441 starObsDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional 

442 Dataref to write the star observation catalog for checkpoints. 

443 inStarObsCat : `~afw.table.BaseCatalog` 

444 Input observation catalog. If this is incomplete, observations 

445 will be appended from when it was cut off. 

446 

447 Returns 

448 ------- 

449 fgcmStarObservations : `afw.table.BaseCatalog` 

450 Full catalog of good observations. 

451 

452 Raises 

453 ------ 

454 RuntimeError: Raised if doSubtractLocalBackground is True and 

455 calibFluxApertureRadius is not set. 

456 """ 

457 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.") 

458 

459 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, 

460 visitCatDataRef=None, inVisitCat=None): 

461 """ 

462 Make a visit catalog with all the keys from each visit 

463 

464 Parameters 

465 ---------- 

466 camera: `lsst.afw.cameraGeom.Camera` 

467 Camera from the butler 

468 groupedDataRefs: `dict` 

469 Dictionary with visit keys, and `list`s of 

470 `lsst.daf.persistence.ButlerDataRef` 

471 bkgDataRefDict: `dict`, optional 

472 Dictionary of gen3 dataRefHandles for background info. 

473 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

474 Dataref to write visitCat for checkpoints 

475 inVisitCat: `afw.table.BaseCatalog`, optional 

476 Input (possibly incomplete) visit catalog 

477 

478 Returns 

479 ------- 

480 visitCat: `afw.table.BaseCatalog` 

481 """ 

482 

483 self.log.info("Assembling visitCatalog from %d %ss" % 

484 (len(groupedDataRefs), self.config.visitDataRefName)) 

485 

486 nCcd = len(camera) 

487 

488 if inVisitCat is None: 

489 schema = self._makeFgcmVisitSchema(nCcd) 

490 

491 visitCat = afwTable.BaseCatalog(schema) 

492 visitCat.reserve(len(groupedDataRefs)) 

493 visitCat.resize(len(groupedDataRefs)) 

494 

495 visitCat['visit'] = list(groupedDataRefs.keys()) 

496 visitCat['used'] = 0 

497 visitCat['sources_read'] = False 

498 else: 

499 visitCat = inVisitCat 

500 

501 # No matter what, fill the catalog. This will check if it was 

502 # already read. 

503 self._fillVisitCatalog(visitCat, groupedDataRefs, 

504 bkgDataRefDict=bkgDataRefDict, 

505 visitCatDataRef=visitCatDataRef) 

506 

507 return visitCat 

508 

509 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, 

510 visitCatDataRef=None): 

511 """ 

512 Fill the visit catalog with visit metadata 

513 

514 Parameters 

515 ---------- 

516 visitCat : `afw.table.BaseCatalog` 

517 Visit catalog. See _makeFgcmVisitSchema() for schema definition. 

518 groupedDataRefs : `dict` 

519 Dictionary with visit keys, and `list`s of 

520 `lsst.daf.persistence.ButlerDataRef` or 

521 `lsst.daf.butler.DeferredDatasetHandle` 

522 visitCatDataRef : `lsst.daf.persistence.ButlerDataRef`, optional 

523 Dataref to write ``visitCat`` for checkpoints. Gen2 only. 

524 bkgDataRefDict : `dict`, optional 

525 Dictionary of Gen3 `lsst.daf.butler.DeferredDatasetHandle` 

526 for background info. 

527 """ 

528 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1)) 

529 

530 for i, visit in enumerate(groupedDataRefs): 

531 # We don't use the bypasses since we need the psf info which does 

532 # not have a bypass 

533 # TODO: When DM-15500 is implemented in the Gen3 Butler, this 

534 # can be fixed 

535 

536 # Do not read those that have already been read 

537 if visitCat['used'][i]: 

538 continue 

539 

540 if (i % self.config.nVisitsPerCheckpoint) == 0: 

541 self.log.info("Retrieving metadata for %s %d (%d/%d)" % 

542 (self.config.visitDataRefName, visit, i, len(groupedDataRefs))) 

543 # Save checkpoint if desired 

544 if visitCatDataRef is not None: 

545 visitCatDataRef.put(visitCat) 

546 

547 dataRef = groupedDataRefs[visit][0] 

548 if isinstance(dataRef, dafPersist.ButlerDataRef): 

549 # Gen2: calexp dataRef 

550 # The first dataRef in the group will be the reference ccd (if available) 

551 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox) 

552 visitInfo = exp.getInfo().getVisitInfo() 

553 label = dataRef.get(datasetType='calexp_filterLabel') 

554 physicalFilter = label.physicalLabel 

555 psf = exp.getPsf() 

556 bbox = dataRef.get(datasetType='calexp_bbox') 

557 psfSigma = psf.computeShape(bbox.getCenter()).getDeterminantRadius() 

558 else: 

559 # Gen3: use the visitSummary dataRef 

560 summary = dataRef.get() 

561 

562 summaryRow = summary.find(self.config.referenceCCD) 

563 if summaryRow is None: 

564 # Take the first available ccd if reference isn't available 

565 summaryRow = summary[0] 

566 

567 summaryDetector = summaryRow['id'] 

568 visitInfo = summaryRow.getVisitInfo() 

569 physicalFilter = summaryRow['physical_filter'] 

570 # Compute the median psf sigma if possible 

571 goodSigma, = np.where(summary['psfSigma'] > 0) 

572 if goodSigma.size > 2: 

573 psfSigma = np.median(summary['psfSigma'][goodSigma]) 

574 elif goodSigma > 0: 

575 psfSigma = np.mean(summary['psfSigma'][goodSigma]) 

576 else: 

577 psfSigma = 0.0 

578 

579 rec = visitCat[i] 

580 rec['visit'] = visit 

581 rec['physicalFilter'] = physicalFilter 

582 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it 

583 # much easier to get the wcs's necessary to recompute the pointing 

584 # ra/dec at the center of the camera. 

585 radec = visitInfo.getBoresightRaDec() 

586 rec['telra'] = radec.getRa().asDegrees() 

587 rec['teldec'] = radec.getDec().asDegrees() 

588 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees() 

589 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees() 

590 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD) 

591 rec['exptime'] = visitInfo.getExposureTime() 

592 # convert from Pa to millibar 

593 # Note that I don't know if this unit will need to be per-camera config 

594 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100 

595 # Flag to signify if this is a "deep" field. Not currently used 

596 rec['deepFlag'] = 0 

597 # Relative flat scaling (1.0 means no relative scaling) 

598 rec['scaling'][:] = 1.0 

599 # Median delta aperture, to be measured from stars 

600 rec['deltaAper'] = 0.0 

601 rec['psfSigma'] = psfSigma 

602 

603 if self.config.doModelErrorsWithBackground: 

604 foundBkg = False 

605 if isinstance(dataRef, dafPersist.ButlerDataRef): 

606 # Gen2-style dataRef 

607 det = dataRef.dataId[self.config.ccdDataRefName] 

608 if dataRef.datasetExists(datasetType='calexpBackground'): 

609 bgList = dataRef.get(datasetType='calexpBackground') 

610 foundBkg = True 

611 else: 

612 # Gen3-style dataRef 

613 try: 

614 # Use the same detector used from the summary. 

615 bkgRef = bkgDataRefDict[(visit, summaryDetector)] 

616 bgList = bkgRef.get() 

617 foundBkg = True 

618 except KeyError: 

619 pass 

620 

621 if foundBkg: 

622 bgStats = (bg[0].getStatsImage().getImage().array 

623 for bg in bgList) 

624 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats) 

625 else: 

626 self.log.warning('Sky background not found for visit %d / ccd %d' % 

627 (visit, det)) 

628 rec['skyBackground'] = -1.0 

629 else: 

630 rec['skyBackground'] = -1.0 

631 

632 rec['used'] = 1 

633 

634 def _makeSourceMapper(self, sourceSchema): 

635 """ 

636 Make a schema mapper for fgcm sources 

637 

638 Parameters 

639 ---------- 

640 sourceSchema: `afwTable.Schema` 

641 Default source schema from the butler 

642 

643 Returns 

644 ------- 

645 sourceMapper: `afwTable.schemaMapper` 

646 Mapper to the FGCM source schema 

647 """ 

648 

649 # create a mapper to the preferred output 

650 sourceMapper = afwTable.SchemaMapper(sourceSchema) 

651 

652 # map to ra/dec 

653 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

654 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

655 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x') 

656 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y') 

657 # Add the mapping if the field exists in the input catalog. 

658 # If the field does not exist, simply add it (set to False). 

659 # This field is not required for calibration, but is useful 

660 # to collate if available. 

661 try: 

662 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(), 

663 'psf_candidate') 

664 except LookupError: 

665 sourceMapper.editOutputSchema().addField( 

666 "psf_candidate", type='Flag', 

667 doc=("Flag set if the source was a candidate for PSF determination, " 

668 "as determined by the star selector.")) 

669 

670 # and add the fields we want 

671 sourceMapper.editOutputSchema().addField( 

672 "visit", type=np.int32, doc="Visit number") 

673 sourceMapper.editOutputSchema().addField( 

674 "ccd", type=np.int32, doc="CCD number") 

675 sourceMapper.editOutputSchema().addField( 

676 "instMag", type=np.float32, doc="Instrumental magnitude") 

677 sourceMapper.editOutputSchema().addField( 

678 "instMagErr", type=np.float32, doc="Instrumental magnitude error") 

679 sourceMapper.editOutputSchema().addField( 

680 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian") 

681 sourceMapper.editOutputSchema().addField( 

682 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset") 

683 

684 return sourceMapper 

685 

686 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None): 

687 """ 

688 Use FGCM code to match observations into unique stars. 

689 

690 Parameters 

691 ---------- 

692 visitCat: `afw.table.BaseCatalog` 

693 Catalog with visit data for fgcm 

694 obsCat: `afw.table.BaseCatalog` 

695 Full catalog of star observations for fgcm 

696 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or 

697 `lsst.daf.butler.DeferredDatasetHandle`, optional 

698 Data reference to fgcm look-up table (used if matching reference stars). 

699 

700 Returns 

701 ------- 

702 fgcmStarIdCat: `afw.table.BaseCatalog` 

703 Catalog of unique star identifiers and index keys 

704 fgcmStarIndicesCat: `afwTable.BaseCatalog` 

705 Catalog of unique star indices 

706 fgcmRefCat: `afw.table.BaseCatalog` 

707 Catalog of matched reference stars. 

708 Will be None if `config.doReferenceMatches` is False. 

709 """ 

710 # get filter names into a numpy array... 

711 # This is the type that is expected by the fgcm code 

712 visitFilterNames = np.zeros(len(visitCat), dtype='a30') 

713 for i in range(len(visitCat)): 

714 visitFilterNames[i] = visitCat[i]['physicalFilter'] 

715 

716 # match to put filterNames with observations 

717 visitIndex = np.searchsorted(visitCat['visit'], 

718 obsCat['visit']) 

719 

720 obsFilterNames = visitFilterNames[visitIndex] 

721 

722 if self.config.doReferenceMatches: 

723 # Get the reference filter names, using the LUT 

724 lutCat = lutDataRef.get() 

725 

726 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in 

727 zip(lutCat[0]['physicalFilters'].split(','), 

728 lutCat[0]['stdPhysicalFilters'].split(','))} 

729 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in 

730 zip(lutCat[0]['stdPhysicalFilters'].split(','), 

731 lutCat[0]['lambdaStdFilter'])} 

732 

733 del lutCat 

734 

735 referenceFilterNames = self._getReferenceFilterNames(visitCat, 

736 stdFilterDict, 

737 stdLambdaDict) 

738 self.log.info("Using the following reference filters: %s" % 

739 (', '.join(referenceFilterNames))) 

740 

741 else: 

742 # This should be an empty list 

743 referenceFilterNames = [] 

744 

745 # make the fgcm starConfig dict 

746 starConfig = {'logger': self.log, 

747 'filterToBand': self.config.physicalFilterMap, 

748 'requiredBands': self.config.requiredBands, 

749 'minPerBand': self.config.minPerBand, 

750 'matchRadius': self.config.matchRadius, 

751 'isolationRadius': self.config.isolationRadius, 

752 'matchNSide': self.config.matchNside, 

753 'coarseNSide': self.config.coarseNside, 

754 'densNSide': self.config.densityCutNside, 

755 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 

756 'randomSeed': self.config.randomSeed, 

757 'primaryBands': self.config.primaryBands, 

758 'referenceFilterNames': referenceFilterNames} 

759 

760 # initialize the FgcmMakeStars object 

761 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) 

762 

763 # make the primary stars 

764 # note that the ra/dec native Angle format is radians 

765 # We determine the conversion from the native units (typically 

766 # radians) to degrees for the first observation. This allows us 

767 # to treate ra/dec as numpy arrays rather than Angles, which would 

768 # be approximately 600x slower. 

769 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) 

770 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, 

771 obsCat['dec'] * conv, 

772 filterNameArray=obsFilterNames, 

773 bandSelected=False) 

774 

775 # and match all the stars 

776 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, 

777 obsCat['dec'] * conv, 

778 obsFilterNames) 

779 

780 if self.config.doReferenceMatches: 

781 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) 

782 

783 # now persist 

784 

785 objSchema = self._makeFgcmObjSchema() 

786 

787 # make catalog and records 

788 fgcmStarIdCat = afwTable.BaseCatalog(objSchema) 

789 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) 

790 for i in range(fgcmMakeStars.objIndexCat.size): 

791 fgcmStarIdCat.addNew() 

792 

793 # fill the catalog 

794 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] 

795 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] 

796 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] 

797 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex'] 

798 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] 

799 

800 obsSchema = self._makeFgcmObsSchema() 

801 

802 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) 

803 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) 

804 for i in range(fgcmMakeStars.obsIndexCat.size): 

805 fgcmStarIndicesCat.addNew() 

806 

807 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex'] 

808 

809 if self.config.doReferenceMatches: 

810 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) 

811 

812 fgcmRefCat = afwTable.BaseCatalog(refSchema) 

813 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) 

814 

815 for i in range(fgcmMakeStars.referenceCat.size): 

816 fgcmRefCat.addNew() 

817 

818 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] 

819 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] 

820 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr'] 

821 

822 md = PropertyList() 

823 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION) 

824 md.set("FILTERNAMES", referenceFilterNames) 

825 fgcmRefCat.setMetadata(md) 

826 

827 else: 

828 fgcmRefCat = None 

829 

830 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat 

831 

832 def _makeFgcmVisitSchema(self, nCcd): 

833 """ 

834 Make a schema for an fgcmVisitCatalog 

835 

836 Parameters 

837 ---------- 

838 nCcd: `int` 

839 Number of CCDs in the camera 

840 

841 Returns 

842 ------- 

843 schema: `afwTable.Schema` 

844 """ 

845 

846 schema = afwTable.Schema() 

847 schema.addField('visit', type=np.int32, doc="Visit number") 

848 schema.addField('physicalFilter', type=str, size=30, doc="Physical filter") 

849 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") 

850 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") 

851 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") 

852 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)") 

853 schema.addField('mjd', type=np.float64, doc="MJD of visit") 

854 schema.addField('exptime', type=np.float32, doc="Exposure time") 

855 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") 

856 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") 

857 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") 

858 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") 

859 # the following field is not used yet 

860 schema.addField('deepFlag', type=np.int32, doc="Deep observation") 

861 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", 

862 size=nCcd) 

863 schema.addField('used', type=np.int32, doc="This visit has been ingested.") 

864 schema.addField('sources_read', type='Flag', doc="This visit had sources read.") 

865 

866 return schema 

867 

868 def _makeFgcmObjSchema(self): 

869 """ 

870 Make a schema for the objIndexCat from fgcmMakeStars 

871 

872 Returns 

873 ------- 

874 schema: `afwTable.Schema` 

875 """ 

876 

877 objSchema = afwTable.Schema() 

878 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

879 # Will investigate making these angles... 

880 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)') 

881 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)') 

882 objSchema.addField('obsArrIndex', type=np.int32, 

883 doc='Index in obsIndexTable for first observation') 

884 objSchema.addField('nObs', type=np.int32, doc='Total number of observations') 

885 

886 return objSchema 

887 

888 def _makeFgcmObsSchema(self): 

889 """ 

890 Make a schema for the obsIndexCat from fgcmMakeStars 

891 

892 Returns 

893 ------- 

894 schema: `afwTable.Schema` 

895 """ 

896 

897 obsSchema = afwTable.Schema() 

898 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table') 

899 

900 return obsSchema 

901 

902 def _makeFgcmRefSchema(self, nReferenceBands): 

903 """ 

904 Make a schema for the referenceCat from fgcmMakeStars 

905 

906 Parameters 

907 ---------- 

908 nReferenceBands: `int` 

909 Number of reference bands 

910 

911 Returns 

912 ------- 

913 schema: `afwTable.Schema` 

914 """ 

915 

916 refSchema = afwTable.Schema() 

917 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

918 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)', 

919 size=nReferenceBands) 

920 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array', 

921 size=nReferenceBands) 

922 

923 return refSchema 

924 

925 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict): 

926 """ 

927 Get the reference filter names, in wavelength order, from the visitCat and 

928 information from the look-up-table. 

929 

930 Parameters 

931 ---------- 

932 visitCat: `afw.table.BaseCatalog` 

933 Catalog with visit data for FGCM 

934 stdFilterDict: `dict` 

935 Mapping of filterName to stdFilterName from LUT 

936 stdLambdaDict: `dict` 

937 Mapping of stdFilterName to stdLambda from LUT 

938 

939 Returns 

940 ------- 

941 referenceFilterNames: `list` 

942 Wavelength-ordered list of reference filter names 

943 """ 

944 

945 # Find the unique list of filter names in visitCat 

946 filterNames = np.unique(visitCat.asAstropy()['physicalFilter']) 

947 

948 # Find the unique list of "standard" filters 

949 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames} 

950 

951 # And sort these by wavelength 

952 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get) 

953 

954 return referenceFilterNames