Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for BuildStars using src tables or sourceTable_visit tables. 

22""" 

23 

24import os 

25import sys 

26import traceback 

27import abc 

28 

29import numpy as np 

30 

31import lsst.daf.persistence as dafPersist 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34import lsst.afw.table as afwTable 

35import lsst.geom as geom 

36from lsst.daf.base import PropertyList 

37from lsst.daf.base.dateTime import DateTime 

38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

39 

40from .utilities import computeApertureRadiusFromDataRef 

41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask 

42 

43import fgcm 

44 

45REFSTARS_FORMAT_VERSION = 1 

46 

47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask'] 

48 

49 

50class FgcmBuildStarsConfigBase(pexConfig.Config): 

51 """Base config for FgcmBuildStars tasks""" 

52 

53 instFluxField = pexConfig.Field( 

54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. " 

55 "The associated flag will be implicitly included in badFlags"), 

56 dtype=str, 

57 default='slot_CalibFlux_instFlux', 

58 ) 

59 minPerBand = pexConfig.Field( 

60 doc="Minimum observations per band", 

61 dtype=int, 

62 default=2, 

63 ) 

64 matchRadius = pexConfig.Field( 

65 doc="Match radius (arcseconds)", 

66 dtype=float, 

67 default=1.0, 

68 ) 

69 isolationRadius = pexConfig.Field( 

70 doc="Isolation radius (arcseconds)", 

71 dtype=float, 

72 default=2.0, 

73 ) 

74 densityCutNside = pexConfig.Field( 

75 doc="Density cut healpix nside", 

76 dtype=int, 

77 default=128, 

78 ) 

79 densityCutMaxPerPixel = pexConfig.Field( 

80 doc="Density cut number of stars per pixel", 

81 dtype=int, 

82 default=1000, 

83 ) 

84 matchNside = pexConfig.Field( 

85 doc="Healpix Nside for matching", 

86 dtype=int, 

87 default=4096, 

88 ) 

89 coarseNside = pexConfig.Field( 

90 doc="Healpix coarse Nside for partitioning matches", 

91 dtype=int, 

92 default=8, 

93 ) 

94 filterMap = pexConfig.DictField( 

95 doc="Mapping from 'filterName' to band.", 

96 keytype=str, 

97 itemtype=str, 

98 default={}, 

99 ) 

100 requiredBands = pexConfig.ListField( 

101 doc="Bands required for each star", 

102 dtype=str, 

103 default=(), 

104 ) 

105 primaryBands = pexConfig.ListField( 

106 doc=("Bands for 'primary' star matches. " 

107 "A star must be observed in one of these bands to be considered " 

108 "as a calibration star."), 

109 dtype=str, 

110 default=None 

111 ) 

112 visitDataRefName = pexConfig.Field( 

113 doc="dataRef name for the 'visit' field, usually 'visit'.", 

114 dtype=str, 

115 default="visit" 

116 ) 

117 ccdDataRefName = pexConfig.Field( 

118 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.", 

119 dtype=str, 

120 default="ccd" 

121 ) 

122 doApplyWcsJacobian = pexConfig.Field( 

123 doc="Apply the jacobian of the WCS to the star observations prior to fit?", 

124 dtype=bool, 

125 default=True 

126 ) 

127 doModelErrorsWithBackground = pexConfig.Field( 

128 doc="Model flux errors with background term?", 

129 dtype=bool, 

130 default=True 

131 ) 

132 psfCandidateName = pexConfig.Field( 

133 doc="Name of field with psf candidate flag for propagation", 

134 dtype=str, 

135 default="calib_psf_candidate" 

136 ) 

137 doSubtractLocalBackground = pexConfig.Field( 

138 doc=("Subtract the local background before performing calibration? " 

139 "This is only supported for circular aperture calibration fluxes."), 

140 dtype=bool, 

141 default=False 

142 ) 

143 localBackgroundFluxField = pexConfig.Field( 

144 doc="Full name of the local background instFlux field to use.", 

145 dtype=str, 

146 default='base_LocalBackground_instFlux' 

147 ) 

148 sourceSelector = sourceSelectorRegistry.makeField( 

149 doc="How to select sources", 

150 default="science" 

151 ) 

152 apertureInnerInstFluxField = pexConfig.Field( 

153 doc=("Full name of instFlux field that contains inner aperture " 

154 "flux for aperture correction proxy"), 

155 dtype=str, 

156 default='base_CircularApertureFlux_12_0_instFlux' 

157 ) 

158 apertureOuterInstFluxField = pexConfig.Field( 

159 doc=("Full name of instFlux field that contains outer aperture " 

160 "flux for aperture correction proxy"), 

161 dtype=str, 

162 default='base_CircularApertureFlux_17_0_instFlux' 

163 ) 

164 doReferenceMatches = pexConfig.Field( 

165 doc="Match reference catalog as additional constraint on calibration", 

166 dtype=bool, 

167 default=True, 

168 ) 

169 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField( 

170 target=FgcmLoadReferenceCatalogTask, 

171 doc="FGCM reference object loader", 

172 ) 

173 nVisitsPerCheckpoint = pexConfig.Field( 

174 doc="Number of visits read between checkpoints", 

175 dtype=int, 

176 default=500, 

177 ) 

178 

179 def setDefaults(self): 

180 sourceSelector = self.sourceSelector["science"] 

181 sourceSelector.setDefaults() 

182 

183 sourceSelector.doFlags = True 

184 sourceSelector.doUnresolved = True 

185 sourceSelector.doSignalToNoise = True 

186 sourceSelector.doIsolated = True 

187 

188 sourceSelector.signalToNoise.minimum = 10.0 

189 sourceSelector.signalToNoise.maximum = 1000.0 

190 

191 # FGCM operates on unresolved sources, and this setting is 

192 # appropriate for the current base_ClassificationExtendedness 

193 sourceSelector.unresolved.maximum = 0.5 

194 

195 

196class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner): 

197 """Subclass of TaskRunner for FgcmBuildStars tasks 

198 

199 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of 

200 arguments, one of which is the butler (for persistence and mapper data), 

201 and a list of dataRefs extracted from the command line. Note that FGCM 

202 runs on a large set of dataRefs, and not on single dataRef/tract/patch. 

203 This class transforms the process arguments generated by the ArgumentParser 

204 into the arguments expected by FgcmBuildStarsTask.run(). This runner does 

205 not use any parallelization. 

206 """ 

207 @staticmethod 

208 def getTargetList(parsedCmd): 

209 """ 

210 Return a list with one element: a tuple with the butler and 

211 list of dataRefs 

212 """ 

213 # we want to combine the butler with any (or no!) dataRefs 

214 return [(parsedCmd.butler, parsedCmd.id.refList)] 

215 

216 def __call__(self, args): 

217 """ 

218 Parameters 

219 ---------- 

220 args: `tuple` with (butler, dataRefList) 

221 

222 Returns 

223 ------- 

224 exitStatus: `list` with `lsst.pipe.base.Struct` 

225 exitStatus (0: success; 1: failure) 

226 """ 

227 butler, dataRefList = args 

228 

229 task = self.TaskClass(config=self.config, log=self.log) 

230 

231 exitStatus = 0 

232 if self.doRaise: 

233 task.runDataRef(butler, dataRefList) 

234 else: 

235 try: 

236 task.runDataRef(butler, dataRefList) 

237 except Exception as e: 

238 exitStatus = 1 

239 task.log.fatal("Failed: %s" % e) 

240 if not isinstance(e, pipeBase.TaskError): 

241 traceback.print_exc(file=sys.stderr) 

242 

243 task.writeMetadata(butler) 

244 

245 # The task does not return any results: 

246 return [pipeBase.Struct(exitStatus=exitStatus)] 

247 

248 def run(self, parsedCmd): 

249 """ 

250 Run the task, with no multiprocessing 

251 

252 Parameters 

253 ---------- 

254 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

255 """ 

256 

257 resultList = [] 

258 

259 if self.precall(parsedCmd): 

260 targetList = self.getTargetList(parsedCmd) 

261 resultList = self(targetList[0]) 

262 

263 return resultList 

264 

265 

266class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC): 

267 """ 

268 Base task to build stars for FGCM global calibration 

269 

270 Parameters 

271 ---------- 

272 butler : `lsst.daf.persistence.Butler` 

273 """ 

274 def __init__(self, butler=None, initInputs=None, **kwargs): 

275 super().__init__(**kwargs) 

276 

277 self.makeSubtask("sourceSelector") 

278 # Only log warning and fatal errors from the sourceSelector 

279 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN) 

280 

281 # no saving of metadata for now 

282 def _getMetadataName(self): 

283 return None 

284 

285 @pipeBase.timeMethod 

286 def runDataRef(self, butler, dataRefs): 

287 """ 

288 Cross-match and make star list for FGCM Input 

289 

290 Parameters 

291 ---------- 

292 butler: `lsst.daf.persistence.Butler` 

293 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

294 Source data references for the input visits. 

295 

296 Raises 

297 ------ 

298 RuntimeErrror: Raised if `config.doReferenceMatches` is set and 

299 an fgcmLookUpTable is not available, or if computeFluxApertureRadius() 

300 fails if the calibFlux is not a CircularAperture flux. 

301 """ 

302 datasetType = dataRefs[0].butlerSubset.datasetType 

303 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType) 

304 

305 if self.config.doReferenceMatches: 

306 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler) 

307 # Ensure that we have a LUT 

308 if not butler.datasetExists('fgcmLookUpTable'): 

309 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches") 

310 # Compute aperture radius if necessary. This is useful to do now before 

311 # any heavy lifting has happened (fail early). 

312 calibFluxApertureRadius = None 

313 if self.config.doSubtractLocalBackground: 

314 try: 

315 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

316 self.config.instFluxField) 

317 except RuntimeError as e: 

318 raise RuntimeError("Could not determine aperture radius from %s. " 

319 "Cannot use doSubtractLocalBackground." % 

320 (self.config.instFluxField)) from e 

321 

322 camera = butler.get('camera') 

323 groupedDataRefs = self._findAndGroupDataRefs(camera, dataRefs, butler=butler) 

324 

325 # Make the visit catalog if necessary 

326 # First check if the visit catalog is in the _current_ path 

327 # We cannot use Gen2 datasetExists() because that checks all parent 

328 # directories as well, which would make recovering from faults 

329 # and fgcmcal reruns impossible. 

330 visitCatDataRef = butler.dataRef('fgcmVisitCatalog') 

331 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0] 

332 if os.path.exists(filename): 

333 # This file exists and we should continue processing 

334 inVisitCat = visitCatDataRef.get() 

335 if len(inVisitCat) != len(groupedDataRefs): 

336 raise RuntimeError("Existing visitCatalog found, but has an inconsistent " 

337 "number of visits. Cannot continue.") 

338 else: 

339 inVisitCat = None 

340 

341 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs, 

342 visitCatDataRef=visitCatDataRef, 

343 inVisitCat=inVisitCat) 

344 

345 # Persist the visitCat as a checkpoint file. 

346 visitCatDataRef.put(visitCat) 

347 

348 starObsDataRef = butler.dataRef('fgcmStarObservations') 

349 filename = starObsDataRef.get('fgcmStarObservations_filename')[0] 

350 if os.path.exists(filename): 

351 inStarObsCat = starObsDataRef.get() 

352 else: 

353 inStarObsCat = None 

354 

355 rad = calibFluxApertureRadius 

356 sourceSchemaDataRef = butler.dataRef('src_schema') 

357 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs, 

358 visitCat, 

359 sourceSchemaDataRef, 

360 camera, 

361 calibFluxApertureRadius=rad, 

362 starObsDataRef=starObsDataRef, 

363 visitCatDataRef=visitCatDataRef, 

364 inStarObsCat=inStarObsCat) 

365 visitCatDataRef.put(visitCat) 

366 starObsDataRef.put(fgcmStarObservationCat) 

367 

368 # Always do the matching. 

369 if self.config.doReferenceMatches: 

370 lutDataRef = butler.dataRef('fgcmLookUpTable') 

371 else: 

372 lutDataRef = None 

373 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

374 fgcmStarObservationCat, 

375 lutDataRef=lutDataRef) 

376 

377 # Persist catalogs via the butler 

378 butler.put(fgcmStarIdCat, 'fgcmStarIds') 

379 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') 

380 if fgcmRefCat is not None: 

381 butler.put(fgcmRefCat, 'fgcmReferenceStars') 

382 

383 @abc.abstractmethod 

384 def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None): 

385 """ 

386 Find and group dataRefs (by visit). For Gen2 usage, set butler, and for 

387 Gen3, use calexpDataRefDict 

388 

389 Parameters 

390 ---------- 

391 camera : `lsst.afw.cameraGeom.Camera` 

392 Camera from the butler. 

393 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` or 

394 `lsst.daf.butler.DeferredDatasetHandle` 

395 Data references for the input visits. 

396 butler : `lsst.daf.persistence.Butler`, optional 

397 Gen2 butler when used as CommandLineTask 

398 calexpDataRefDict : `dict`, optional 

399 Dictionary of Gen3 deferred data refs for calexps 

400 

401 Returns 

402 ------- 

403 groupedDataRefs : `OrderedDict` [`int`, `list`] 

404 Dictionary with sorted visit keys, and `list`s of 

405 `lsst.daf.persistence.ButlerDataRef` or 

406 `lsst.daf.butler.DeferredDatasetHandle` 

407 

408 Raises 

409 ------ 

410 RuntimeError : Raised if neither or both of butler and dataRefDict are set. 

411 """ 

412 raise NotImplementedError("_findAndGroupDataRefs not implemented.") 

413 

414 @abc.abstractmethod 

415 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

416 sourceSchemaDataRef, 

417 camera, 

418 calibFluxApertureRadius=None, 

419 visitCatDataRef=None, 

420 starObsDataRef=None, 

421 inStarObsCat=None): 

422 """ 

423 Compile all good star observations from visits in visitCat. Checkpoint files 

424 will be stored if both visitCatDataRef and starObsDataRef are not None. 

425 

426 Parameters 

427 ---------- 

428 groupedDataRefs: `dict` of `list`s 

429 Lists of `~lsst.daf.persistence.ButlerDataRef` or 

430 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit. 

431 visitCat: `~afw.table.BaseCatalog` 

432 Catalog with visit data for FGCM 

433 sourceSchemaDataRef: `~lsst.daf.persistence.ButlerDataRef` or 

434 `~lsst.daf.butler.DeferredDatasetHandle` 

435 DataRef for the schema of the src catalogs. 

436 camera: `~lsst.afw.cameraGeom.Camera` 

437 calibFluxApertureRadius: `float`, optional 

438 Aperture radius for calibration flux. 

439 visitCatDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional 

440 Dataref to write visitCat for checkpoints 

441 starObsDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional 

442 Dataref to write the star observation catalog for checkpoints. 

443 inStarObsCat: `~afw.table.BaseCatalog` 

444 Input observation catalog. If this is incomplete, observations 

445 will be appended from when it was cut off. 

446 

447 Returns 

448 ------- 

449 fgcmStarObservations: `afw.table.BaseCatalog` 

450 Full catalog of good observations. 

451 

452 Raises 

453 ------ 

454 RuntimeError: Raised if doSubtractLocalBackground is True and 

455 calibFluxApertureRadius is not set. 

456 """ 

457 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.") 

458 

459 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, 

460 visitCatDataRef=None, inVisitCat=None): 

461 """ 

462 Make a visit catalog with all the keys from each visit 

463 

464 Parameters 

465 ---------- 

466 camera: `lsst.afw.cameraGeom.Camera` 

467 Camera from the butler 

468 groupedDataRefs: `dict` 

469 Dictionary with visit keys, and `list`s of 

470 `lsst.daf.persistence.ButlerDataRef` 

471 bkgDataRefDict: `dict`, optional 

472 Dictionary of gen3 dataRefHandles for background info. 

473 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

474 Dataref to write visitCat for checkpoints 

475 inVisitCat: `afw.table.BaseCatalog`, optional 

476 Input (possibly incomplete) visit catalog 

477 

478 Returns 

479 ------- 

480 visitCat: `afw.table.BaseCatalog` 

481 """ 

482 

483 self.log.info("Assembling visitCatalog from %d %ss" % 

484 (len(groupedDataRefs), self.config.visitDataRefName)) 

485 

486 nCcd = len(camera) 

487 

488 if inVisitCat is None: 

489 schema = self._makeFgcmVisitSchema(nCcd) 

490 

491 visitCat = afwTable.BaseCatalog(schema) 

492 visitCat.reserve(len(groupedDataRefs)) 

493 visitCat.resize(len(groupedDataRefs)) 

494 

495 visitCat['visit'] = list(groupedDataRefs.keys()) 

496 visitCat['used'] = 0 

497 visitCat['sources_read'] = False 

498 else: 

499 visitCat = inVisitCat 

500 

501 # No matter what, fill the catalog. This will check if it was 

502 # already read. 

503 self._fillVisitCatalog(visitCat, groupedDataRefs, 

504 bkgDataRefDict=bkgDataRefDict, 

505 visitCatDataRef=visitCatDataRef) 

506 

507 return visitCat 

508 

509 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, 

510 visitCatDataRef=None): 

511 """ 

512 Fill the visit catalog with visit metadata 

513 

514 Parameters 

515 ---------- 

516 visitCat: `afw.table.BaseCatalog` 

517 Catalog with schema from _makeFgcmVisitSchema() 

518 groupedDataRefs: `dict` 

519 Dictionary with visit keys, and `list`s of 

520 `lsst.daf.persistence.ButlerDataRef` 

521 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

522 Dataref to write visitCat for checkpoints 

523 bkgDataRefDict: `dict`, optional 

524 Dictionary of gen3 dataRefHandles for background info. FIXME 

525 """ 

526 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1)) 

527 

528 for i, visit in enumerate(groupedDataRefs): 

529 # We don't use the bypasses since we need the psf info which does 

530 # not have a bypass 

531 # TODO: When DM-15500 is implemented in the Gen3 Butler, this 

532 # can be fixed 

533 

534 # Do not read those that have already been read 

535 if visitCat['used'][i]: 

536 continue 

537 

538 if (i % self.config.nVisitsPerCheckpoint) == 0: 

539 self.log.info("Retrieving metadata for %s %d (%d/%d)" % 

540 (self.config.visitDataRefName, visit, i, len(groupedDataRefs))) 

541 # Save checkpoint if desired 

542 if visitCatDataRef is not None: 

543 visitCatDataRef.put(visitCat) 

544 

545 # Note that the reference ccd is first in the list (if available). 

546 

547 # The first dataRef in the group will be the reference ccd (if available) 

548 dataRef = groupedDataRefs[visit][0] 

549 if isinstance(dataRef, dafPersist.ButlerDataRef): 

550 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox) 

551 visitInfo = exp.getInfo().getVisitInfo() 

552 f = exp.getFilter() 

553 psf = exp.getPsf() 

554 else: 

555 visitInfo = dataRef.get(component='visitInfo') 

556 f = dataRef.get(component='filter') 

557 psf = dataRef.get(component='psf') 

558 

559 rec = visitCat[i] 

560 rec['visit'] = visit 

561 rec['filtername'] = f.getName() 

562 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it 

563 # much easier to get the wcs's necessary to recompute the pointing 

564 # ra/dec at the center of the camera. 

565 radec = visitInfo.getBoresightRaDec() 

566 rec['telra'] = radec.getRa().asDegrees() 

567 rec['teldec'] = radec.getDec().asDegrees() 

568 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees() 

569 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees() 

570 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD) 

571 rec['exptime'] = visitInfo.getExposureTime() 

572 # convert from Pa to millibar 

573 # Note that I don't know if this unit will need to be per-camera config 

574 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100 

575 # Flag to signify if this is a "deep" field. Not currently used 

576 rec['deepFlag'] = 0 

577 # Relative flat scaling (1.0 means no relative scaling) 

578 rec['scaling'][:] = 1.0 

579 # Median delta aperture, to be measured from stars 

580 rec['deltaAper'] = 0.0 

581 

582 rec['psfSigma'] = psf.computeShape().getDeterminantRadius() 

583 

584 if self.config.doModelErrorsWithBackground: 

585 foundBkg = False 

586 if isinstance(dataRef, dafPersist.ButlerDataRef): 

587 det = dataRef.dataId[self.config.ccdDataRefName] 

588 if dataRef.datasetExists(datasetType='calexpBackground'): 

589 bgList = dataRef.get(datasetType='calexpBackground') 

590 foundBkg = True 

591 else: 

592 det = dataRef.dataId.byName()['detector'] 

593 try: 

594 bkgRef = bkgDataRefDict[(visit, det)] 

595 bgList = bkgRef.get() 

596 foundBkg = True 

597 except KeyError: 

598 pass 

599 

600 if foundBkg: 

601 bgStats = (bg[0].getStatsImage().getImage().array 

602 for bg in bgList) 

603 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats) 

604 else: 

605 self.log.warn('Sky background not found for visit %d / ccd %d' % 

606 (visit, det)) 

607 rec['skyBackground'] = -1.0 

608 else: 

609 rec['skyBackground'] = -1.0 

610 

611 rec['used'] = 1 

612 

613 def _makeSourceMapper(self, sourceSchema): 

614 """ 

615 Make a schema mapper for fgcm sources 

616 

617 Parameters 

618 ---------- 

619 sourceSchema: `afwTable.Schema` 

620 Default source schema from the butler 

621 

622 Returns 

623 ------- 

624 sourceMapper: `afwTable.schemaMapper` 

625 Mapper to the FGCM source schema 

626 """ 

627 

628 # create a mapper to the preferred output 

629 sourceMapper = afwTable.SchemaMapper(sourceSchema) 

630 

631 # map to ra/dec 

632 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

633 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

634 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x') 

635 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y') 

636 # Add the mapping if the field exists in the input catalog. 

637 # If the field does not exist, simply add it (set to False). 

638 # This field is not required for calibration, but is useful 

639 # to collate if available. 

640 try: 

641 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(), 

642 'psf_candidate') 

643 except LookupError: 

644 sourceMapper.editOutputSchema().addField( 

645 "psf_candidate", type='Flag', 

646 doc=("Flag set if the source was a candidate for PSF determination, " 

647 "as determined by the star selector.")) 

648 

649 # and add the fields we want 

650 sourceMapper.editOutputSchema().addField( 

651 "visit", type=np.int32, doc="Visit number") 

652 sourceMapper.editOutputSchema().addField( 

653 "ccd", type=np.int32, doc="CCD number") 

654 sourceMapper.editOutputSchema().addField( 

655 "instMag", type=np.float32, doc="Instrumental magnitude") 

656 sourceMapper.editOutputSchema().addField( 

657 "instMagErr", type=np.float32, doc="Instrumental magnitude error") 

658 sourceMapper.editOutputSchema().addField( 

659 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian") 

660 sourceMapper.editOutputSchema().addField( 

661 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset") 

662 

663 return sourceMapper 

664 

665 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None): 

666 """ 

667 Use FGCM code to match observations into unique stars. 

668 

669 Parameters 

670 ---------- 

671 visitCat: `afw.table.BaseCatalog` 

672 Catalog with visit data for fgcm 

673 obsCat: `afw.table.BaseCatalog` 

674 Full catalog of star observations for fgcm 

675 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or 

676 `lsst.daf.butler.DeferredDatasetHandle`, optional 

677 Data reference to fgcm look-up table (used if matching reference stars). 

678 

679 Returns 

680 ------- 

681 fgcmStarIdCat: `afw.table.BaseCatalog` 

682 Catalog of unique star identifiers and index keys 

683 fgcmStarIndicesCat: `afwTable.BaseCatalog` 

684 Catalog of unique star indices 

685 fgcmRefCat: `afw.table.BaseCatalog` 

686 Catalog of matched reference stars. 

687 Will be None if `config.doReferenceMatches` is False. 

688 """ 

689 # get filter names into a numpy array... 

690 # This is the type that is expected by the fgcm code 

691 visitFilterNames = np.zeros(len(visitCat), dtype='a10') 

692 for i in range(len(visitCat)): 

693 visitFilterNames[i] = visitCat[i]['filtername'] 

694 

695 # match to put filterNames with observations 

696 visitIndex = np.searchsorted(visitCat['visit'], 

697 obsCat['visit']) 

698 

699 obsFilterNames = visitFilterNames[visitIndex] 

700 

701 if self.config.doReferenceMatches: 

702 # Get the reference filter names, using the LUT 

703 lutCat = lutDataRef.get() 

704 

705 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in 

706 zip(lutCat[0]['filterNames'].split(','), 

707 lutCat[0]['stdFilterNames'].split(','))} 

708 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in 

709 zip(lutCat[0]['stdFilterNames'].split(','), 

710 lutCat[0]['lambdaStdFilter'])} 

711 

712 del lutCat 

713 

714 referenceFilterNames = self._getReferenceFilterNames(visitCat, 

715 stdFilterDict, 

716 stdLambdaDict) 

717 self.log.info("Using the following reference filters: %s" % 

718 (', '.join(referenceFilterNames))) 

719 

720 else: 

721 # This should be an empty list 

722 referenceFilterNames = [] 

723 

724 # make the fgcm starConfig dict 

725 

726 starConfig = {'logger': self.log, 

727 'filterToBand': self.config.filterMap, 

728 'requiredBands': self.config.requiredBands, 

729 'minPerBand': self.config.minPerBand, 

730 'matchRadius': self.config.matchRadius, 

731 'isolationRadius': self.config.isolationRadius, 

732 'matchNSide': self.config.matchNside, 

733 'coarseNSide': self.config.coarseNside, 

734 'densNSide': self.config.densityCutNside, 

735 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 

736 'primaryBands': self.config.primaryBands, 

737 'referenceFilterNames': referenceFilterNames} 

738 

739 # initialize the FgcmMakeStars object 

740 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) 

741 

742 # make the primary stars 

743 # note that the ra/dec native Angle format is radians 

744 # We determine the conversion from the native units (typically 

745 # radians) to degrees for the first observation. This allows us 

746 # to treate ra/dec as numpy arrays rather than Angles, which would 

747 # be approximately 600x slower. 

748 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) 

749 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, 

750 obsCat['dec'] * conv, 

751 filterNameArray=obsFilterNames, 

752 bandSelected=False) 

753 

754 # and match all the stars 

755 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, 

756 obsCat['dec'] * conv, 

757 obsFilterNames) 

758 

759 if self.config.doReferenceMatches: 

760 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) 

761 

762 # now persist 

763 

764 objSchema = self._makeFgcmObjSchema() 

765 

766 # make catalog and records 

767 fgcmStarIdCat = afwTable.BaseCatalog(objSchema) 

768 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) 

769 for i in range(fgcmMakeStars.objIndexCat.size): 

770 fgcmStarIdCat.addNew() 

771 

772 # fill the catalog 

773 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] 

774 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] 

775 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] 

776 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex'] 

777 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] 

778 

779 obsSchema = self._makeFgcmObsSchema() 

780 

781 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) 

782 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) 

783 for i in range(fgcmMakeStars.obsIndexCat.size): 

784 fgcmStarIndicesCat.addNew() 

785 

786 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex'] 

787 

788 if self.config.doReferenceMatches: 

789 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) 

790 

791 fgcmRefCat = afwTable.BaseCatalog(refSchema) 

792 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) 

793 

794 for i in range(fgcmMakeStars.referenceCat.size): 

795 fgcmRefCat.addNew() 

796 

797 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] 

798 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] 

799 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr'] 

800 

801 md = PropertyList() 

802 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION) 

803 md.set("FILTERNAMES", referenceFilterNames) 

804 fgcmRefCat.setMetadata(md) 

805 

806 else: 

807 fgcmRefCat = None 

808 

809 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat 

810 

811 def _makeFgcmVisitSchema(self, nCcd): 

812 """ 

813 Make a schema for an fgcmVisitCatalog 

814 

815 Parameters 

816 ---------- 

817 nCcd: `int` 

818 Number of CCDs in the camera 

819 

820 Returns 

821 ------- 

822 schema: `afwTable.Schema` 

823 """ 

824 

825 schema = afwTable.Schema() 

826 schema.addField('visit', type=np.int32, doc="Visit number") 

827 # Note that the FGCM code currently handles filternames up to 2 characters long 

828 schema.addField('filtername', type=str, size=10, doc="Filter name") 

829 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") 

830 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") 

831 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") 

832 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)") 

833 schema.addField('mjd', type=np.float64, doc="MJD of visit") 

834 schema.addField('exptime', type=np.float32, doc="Exposure time") 

835 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") 

836 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") 

837 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") 

838 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") 

839 # the following field is not used yet 

840 schema.addField('deepFlag', type=np.int32, doc="Deep observation") 

841 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", 

842 size=nCcd) 

843 schema.addField('used', type=np.int32, doc="This visit has been ingested.") 

844 schema.addField('sources_read', type='Flag', doc="This visit had sources read.") 

845 

846 return schema 

847 

848 def _makeFgcmObjSchema(self): 

849 """ 

850 Make a schema for the objIndexCat from fgcmMakeStars 

851 

852 Returns 

853 ------- 

854 schema: `afwTable.Schema` 

855 """ 

856 

857 objSchema = afwTable.Schema() 

858 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

859 # Will investigate making these angles... 

860 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)') 

861 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)') 

862 objSchema.addField('obsArrIndex', type=np.int32, 

863 doc='Index in obsIndexTable for first observation') 

864 objSchema.addField('nObs', type=np.int32, doc='Total number of observations') 

865 

866 return objSchema 

867 

868 def _makeFgcmObsSchema(self): 

869 """ 

870 Make a schema for the obsIndexCat from fgcmMakeStars 

871 

872 Returns 

873 ------- 

874 schema: `afwTable.Schema` 

875 """ 

876 

877 obsSchema = afwTable.Schema() 

878 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table') 

879 

880 return obsSchema 

881 

882 def _makeFgcmRefSchema(self, nReferenceBands): 

883 """ 

884 Make a schema for the referenceCat from fgcmMakeStars 

885 

886 Parameters 

887 ---------- 

888 nReferenceBands: `int` 

889 Number of reference bands 

890 

891 Returns 

892 ------- 

893 schema: `afwTable.Schema` 

894 """ 

895 

896 refSchema = afwTable.Schema() 

897 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

898 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)', 

899 size=nReferenceBands) 

900 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array', 

901 size=nReferenceBands) 

902 

903 return refSchema 

904 

905 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict): 

906 """ 

907 Get the reference filter names, in wavelength order, from the visitCat and 

908 information from the look-up-table. 

909 

910 Parameters 

911 ---------- 

912 visitCat: `afw.table.BaseCatalog` 

913 Catalog with visit data for FGCM 

914 stdFilterDict: `dict` 

915 Mapping of filterName to stdFilterName from LUT 

916 stdLambdaDict: `dict` 

917 Mapping of stdFilterName to stdLambda from LUT 

918 

919 Returns 

920 ------- 

921 referenceFilterNames: `list` 

922 Wavelength-ordered list of reference filter names 

923 """ 

924 

925 # Find the unique list of filter names in visitCat 

926 filterNames = np.unique(visitCat.asAstropy()['filtername']) 

927 

928 # Find the unique list of "standard" filters 

929 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames} 

930 

931 # And sort these by wavelength 

932 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get) 

933 

934 return referenceFilterNames