Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for BuildStars using src tables or sourceTable_visit tables. 

22""" 

23 

24import os 

25import sys 

26import traceback 

27import abc 

28 

29import numpy as np 

30 

31import lsst.daf.persistence as dafPersist 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34import lsst.afw.table as afwTable 

35import lsst.geom as geom 

36from lsst.daf.base import PropertyList 

37from lsst.daf.base.dateTime import DateTime 

38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

39 

40from .utilities import computeApertureRadiusFromDataRef 

41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask 

42 

43import fgcm 

44 

45REFSTARS_FORMAT_VERSION = 1 

46 

47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask'] 

48 

49 

50class FgcmBuildStarsConfigBase(pexConfig.Config): 

51 """Base config for FgcmBuildStars tasks""" 

52 

53 instFluxField = pexConfig.Field( 

54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. " 

55 "The associated flag will be implicitly included in badFlags"), 

56 dtype=str, 

57 default='slot_CalibFlux_instFlux', 

58 ) 

59 minPerBand = pexConfig.Field( 

60 doc="Minimum observations per band", 

61 dtype=int, 

62 default=2, 

63 ) 

64 matchRadius = pexConfig.Field( 

65 doc="Match radius (arcseconds)", 

66 dtype=float, 

67 default=1.0, 

68 ) 

69 isolationRadius = pexConfig.Field( 

70 doc="Isolation radius (arcseconds)", 

71 dtype=float, 

72 default=2.0, 

73 ) 

74 densityCutNside = pexConfig.Field( 

75 doc="Density cut healpix nside", 

76 dtype=int, 

77 default=128, 

78 ) 

79 densityCutMaxPerPixel = pexConfig.Field( 

80 doc="Density cut number of stars per pixel", 

81 dtype=int, 

82 default=1000, 

83 ) 

84 matchNside = pexConfig.Field( 

85 doc="Healpix Nside for matching", 

86 dtype=int, 

87 default=4096, 

88 ) 

89 coarseNside = pexConfig.Field( 

90 doc="Healpix coarse Nside for partitioning matches", 

91 dtype=int, 

92 default=8, 

93 ) 

94 filterMap = pexConfig.DictField( 

95 doc="Mapping from 'filterName' to band.", 

96 keytype=str, 

97 itemtype=str, 

98 default={}, 

99 deprecated=("This field is no longer used, and has been deprecated by " 

100 "DM-28088. It will be removed after v22. Use " 

101 "physicalFilterMap instead.") 

102 ) 

103 # The following config will not be necessary after Gen2 retirement. 

104 # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band' 

105 # which is easiest to access in the config file. 

106 physicalFilterMap = pexConfig.DictField( 

107 doc="Mapping from 'physicalFilter' to band.", 

108 keytype=str, 

109 itemtype=str, 

110 default={}, 

111 ) 

112 requiredBands = pexConfig.ListField( 

113 doc="Bands required for each star", 

114 dtype=str, 

115 default=(), 

116 ) 

117 primaryBands = pexConfig.ListField( 

118 doc=("Bands for 'primary' star matches. " 

119 "A star must be observed in one of these bands to be considered " 

120 "as a calibration star."), 

121 dtype=str, 

122 default=None 

123 ) 

124 visitDataRefName = pexConfig.Field( 

125 doc="dataRef name for the 'visit' field, usually 'visit'.", 

126 dtype=str, 

127 default="visit" 

128 ) 

129 ccdDataRefName = pexConfig.Field( 

130 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.", 

131 dtype=str, 

132 default="ccd" 

133 ) 

134 doApplyWcsJacobian = pexConfig.Field( 

135 doc="Apply the jacobian of the WCS to the star observations prior to fit?", 

136 dtype=bool, 

137 default=True 

138 ) 

139 doModelErrorsWithBackground = pexConfig.Field( 

140 doc="Model flux errors with background term?", 

141 dtype=bool, 

142 default=True 

143 ) 

144 psfCandidateName = pexConfig.Field( 

145 doc="Name of field with psf candidate flag for propagation", 

146 dtype=str, 

147 default="calib_psf_candidate" 

148 ) 

149 doSubtractLocalBackground = pexConfig.Field( 

150 doc=("Subtract the local background before performing calibration? " 

151 "This is only supported for circular aperture calibration fluxes."), 

152 dtype=bool, 

153 default=False 

154 ) 

155 localBackgroundFluxField = pexConfig.Field( 

156 doc="Full name of the local background instFlux field to use.", 

157 dtype=str, 

158 default='base_LocalBackground_instFlux' 

159 ) 

160 sourceSelector = sourceSelectorRegistry.makeField( 

161 doc="How to select sources", 

162 default="science" 

163 ) 

164 apertureInnerInstFluxField = pexConfig.Field( 

165 doc=("Full name of instFlux field that contains inner aperture " 

166 "flux for aperture correction proxy"), 

167 dtype=str, 

168 default='base_CircularApertureFlux_12_0_instFlux' 

169 ) 

170 apertureOuterInstFluxField = pexConfig.Field( 

171 doc=("Full name of instFlux field that contains outer aperture " 

172 "flux for aperture correction proxy"), 

173 dtype=str, 

174 default='base_CircularApertureFlux_17_0_instFlux' 

175 ) 

176 doReferenceMatches = pexConfig.Field( 

177 doc="Match reference catalog as additional constraint on calibration", 

178 dtype=bool, 

179 default=True, 

180 ) 

181 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField( 

182 target=FgcmLoadReferenceCatalogTask, 

183 doc="FGCM reference object loader", 

184 ) 

185 nVisitsPerCheckpoint = pexConfig.Field( 

186 doc="Number of visits read between checkpoints", 

187 dtype=int, 

188 default=500, 

189 ) 

190 

191 def setDefaults(self): 

192 sourceSelector = self.sourceSelector["science"] 

193 sourceSelector.setDefaults() 

194 

195 sourceSelector.doFlags = True 

196 sourceSelector.doUnresolved = True 

197 sourceSelector.doSignalToNoise = True 

198 sourceSelector.doIsolated = True 

199 

200 sourceSelector.signalToNoise.minimum = 10.0 

201 sourceSelector.signalToNoise.maximum = 1000.0 

202 

203 # FGCM operates on unresolved sources, and this setting is 

204 # appropriate for the current base_ClassificationExtendedness 

205 sourceSelector.unresolved.maximum = 0.5 

206 

207 

208class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner): 

209 """Subclass of TaskRunner for FgcmBuildStars tasks 

210 

211 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of 

212 arguments, one of which is the butler (for persistence and mapper data), 

213 and a list of dataRefs extracted from the command line. Note that FGCM 

214 runs on a large set of dataRefs, and not on single dataRef/tract/patch. 

215 This class transforms the process arguments generated by the ArgumentParser 

216 into the arguments expected by FgcmBuildStarsTask.run(). This runner does 

217 not use any parallelization. 

218 """ 

219 @staticmethod 

220 def getTargetList(parsedCmd): 

221 """ 

222 Return a list with one element: a tuple with the butler and 

223 list of dataRefs 

224 """ 

225 # we want to combine the butler with any (or no!) dataRefs 

226 return [(parsedCmd.butler, parsedCmd.id.refList)] 

227 

228 def __call__(self, args): 

229 """ 

230 Parameters 

231 ---------- 

232 args: `tuple` with (butler, dataRefList) 

233 

234 Returns 

235 ------- 

236 exitStatus: `list` with `lsst.pipe.base.Struct` 

237 exitStatus (0: success; 1: failure) 

238 """ 

239 butler, dataRefList = args 

240 

241 task = self.TaskClass(config=self.config, log=self.log) 

242 

243 exitStatus = 0 

244 if self.doRaise: 

245 task.runDataRef(butler, dataRefList) 

246 else: 

247 try: 

248 task.runDataRef(butler, dataRefList) 

249 except Exception as e: 

250 exitStatus = 1 

251 task.log.fatal("Failed: %s" % e) 

252 if not isinstance(e, pipeBase.TaskError): 

253 traceback.print_exc(file=sys.stderr) 

254 

255 task.writeMetadata(butler) 

256 

257 # The task does not return any results: 

258 return [pipeBase.Struct(exitStatus=exitStatus)] 

259 

260 def run(self, parsedCmd): 

261 """ 

262 Run the task, with no multiprocessing 

263 

264 Parameters 

265 ---------- 

266 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

267 """ 

268 

269 resultList = [] 

270 

271 if self.precall(parsedCmd): 

272 targetList = self.getTargetList(parsedCmd) 

273 resultList = self(targetList[0]) 

274 

275 return resultList 

276 

277 

278class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC): 

279 """ 

280 Base task to build stars for FGCM global calibration 

281 

282 Parameters 

283 ---------- 

284 butler : `lsst.daf.persistence.Butler` 

285 """ 

286 def __init__(self, butler=None, initInputs=None, **kwargs): 

287 super().__init__(**kwargs) 

288 

289 self.makeSubtask("sourceSelector") 

290 # Only log warning and fatal errors from the sourceSelector 

291 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN) 

292 

293 # no saving of metadata for now 

294 def _getMetadataName(self): 

295 return None 

296 

297 @pipeBase.timeMethod 

298 def runDataRef(self, butler, dataRefs): 

299 """ 

300 Cross-match and make star list for FGCM Input 

301 

302 Parameters 

303 ---------- 

304 butler: `lsst.daf.persistence.Butler` 

305 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

306 Source data references for the input visits. 

307 

308 Raises 

309 ------ 

310 RuntimeErrror: Raised if `config.doReferenceMatches` is set and 

311 an fgcmLookUpTable is not available, or if computeFluxApertureRadius() 

312 fails if the calibFlux is not a CircularAperture flux. 

313 """ 

314 datasetType = dataRefs[0].butlerSubset.datasetType 

315 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType) 

316 

317 if self.config.doReferenceMatches: 

318 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler) 

319 # Ensure that we have a LUT 

320 if not butler.datasetExists('fgcmLookUpTable'): 

321 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches") 

322 # Compute aperture radius if necessary. This is useful to do now before 

323 # any heavy lifting has happened (fail early). 

324 calibFluxApertureRadius = None 

325 if self.config.doSubtractLocalBackground: 

326 try: 

327 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

328 self.config.instFluxField) 

329 except RuntimeError as e: 

330 raise RuntimeError("Could not determine aperture radius from %s. " 

331 "Cannot use doSubtractLocalBackground." % 

332 (self.config.instFluxField)) from e 

333 

334 camera = butler.get('camera') 

335 groupedDataRefs = self._findAndGroupDataRefs(camera, dataRefs, butler=butler) 

336 

337 # Make the visit catalog if necessary 

338 # First check if the visit catalog is in the _current_ path 

339 # We cannot use Gen2 datasetExists() because that checks all parent 

340 # directories as well, which would make recovering from faults 

341 # and fgcmcal reruns impossible. 

342 visitCatDataRef = butler.dataRef('fgcmVisitCatalog') 

343 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0] 

344 if os.path.exists(filename): 

345 # This file exists and we should continue processing 

346 inVisitCat = visitCatDataRef.get() 

347 if len(inVisitCat) != len(groupedDataRefs): 

348 raise RuntimeError("Existing visitCatalog found, but has an inconsistent " 

349 "number of visits. Cannot continue.") 

350 else: 

351 inVisitCat = None 

352 

353 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs, 

354 visitCatDataRef=visitCatDataRef, 

355 inVisitCat=inVisitCat) 

356 

357 # Persist the visitCat as a checkpoint file. 

358 visitCatDataRef.put(visitCat) 

359 

360 starObsDataRef = butler.dataRef('fgcmStarObservations') 

361 filename = starObsDataRef.get('fgcmStarObservations_filename')[0] 

362 if os.path.exists(filename): 

363 inStarObsCat = starObsDataRef.get() 

364 else: 

365 inStarObsCat = None 

366 

367 rad = calibFluxApertureRadius 

368 sourceSchemaDataRef = butler.dataRef('src_schema') 

369 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs, 

370 visitCat, 

371 sourceSchemaDataRef, 

372 camera, 

373 calibFluxApertureRadius=rad, 

374 starObsDataRef=starObsDataRef, 

375 visitCatDataRef=visitCatDataRef, 

376 inStarObsCat=inStarObsCat) 

377 visitCatDataRef.put(visitCat) 

378 starObsDataRef.put(fgcmStarObservationCat) 

379 

380 # Always do the matching. 

381 if self.config.doReferenceMatches: 

382 lutDataRef = butler.dataRef('fgcmLookUpTable') 

383 else: 

384 lutDataRef = None 

385 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

386 fgcmStarObservationCat, 

387 lutDataRef=lutDataRef) 

388 

389 # Persist catalogs via the butler 

390 butler.put(fgcmStarIdCat, 'fgcmStarIds') 

391 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') 

392 if fgcmRefCat is not None: 

393 butler.put(fgcmRefCat, 'fgcmReferenceStars') 

394 

395 @abc.abstractmethod 

396 def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None): 

397 """ 

398 Find and group dataRefs (by visit). For Gen2 usage, set butler, and for 

399 Gen3, use calexpDataRefDict 

400 

401 Parameters 

402 ---------- 

403 camera : `lsst.afw.cameraGeom.Camera` 

404 Camera from the butler. 

405 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` or 

406 `lsst.daf.butler.DeferredDatasetHandle` 

407 Data references for the input visits. 

408 butler : `lsst.daf.persistence.Butler`, optional 

409 Gen2 butler when used as CommandLineTask 

410 calexpDataRefDict : `dict`, optional 

411 Dictionary of Gen3 deferred data refs for calexps 

412 

413 Returns 

414 ------- 

415 groupedDataRefs : `OrderedDict` [`int`, `list`] 

416 Dictionary with sorted visit keys, and `list`s of 

417 `lsst.daf.persistence.ButlerDataRef` or 

418 `lsst.daf.butler.DeferredDatasetHandle` 

419 

420 Raises 

421 ------ 

422 RuntimeError : Raised if neither or both of butler and dataRefDict are set. 

423 """ 

424 raise NotImplementedError("_findAndGroupDataRefs not implemented.") 

425 

426 @abc.abstractmethod 

427 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

428 sourceSchemaDataRef, 

429 camera, 

430 calibFluxApertureRadius=None, 

431 visitCatDataRef=None, 

432 starObsDataRef=None, 

433 inStarObsCat=None): 

434 """ 

435 Compile all good star observations from visits in visitCat. Checkpoint files 

436 will be stored if both visitCatDataRef and starObsDataRef are not None. 

437 

438 Parameters 

439 ---------- 

440 groupedDataRefs: `dict` of `list`s 

441 Lists of `~lsst.daf.persistence.ButlerDataRef` or 

442 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit. 

443 visitCat: `~afw.table.BaseCatalog` 

444 Catalog with visit data for FGCM 

445 sourceSchemaDataRef: `~lsst.daf.persistence.ButlerDataRef` or 

446 `~lsst.daf.butler.DeferredDatasetHandle` 

447 DataRef for the schema of the src catalogs. 

448 camera: `~lsst.afw.cameraGeom.Camera` 

449 calibFluxApertureRadius: `float`, optional 

450 Aperture radius for calibration flux. 

451 visitCatDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional 

452 Dataref to write visitCat for checkpoints 

453 starObsDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional 

454 Dataref to write the star observation catalog for checkpoints. 

455 inStarObsCat: `~afw.table.BaseCatalog` 

456 Input observation catalog. If this is incomplete, observations 

457 will be appended from when it was cut off. 

458 

459 Returns 

460 ------- 

461 fgcmStarObservations: `afw.table.BaseCatalog` 

462 Full catalog of good observations. 

463 

464 Raises 

465 ------ 

466 RuntimeError: Raised if doSubtractLocalBackground is True and 

467 calibFluxApertureRadius is not set. 

468 """ 

469 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.") 

470 

471 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, 

472 visitCatDataRef=None, inVisitCat=None): 

473 """ 

474 Make a visit catalog with all the keys from each visit 

475 

476 Parameters 

477 ---------- 

478 camera: `lsst.afw.cameraGeom.Camera` 

479 Camera from the butler 

480 groupedDataRefs: `dict` 

481 Dictionary with visit keys, and `list`s of 

482 `lsst.daf.persistence.ButlerDataRef` 

483 bkgDataRefDict: `dict`, optional 

484 Dictionary of gen3 dataRefHandles for background info. 

485 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

486 Dataref to write visitCat for checkpoints 

487 inVisitCat: `afw.table.BaseCatalog`, optional 

488 Input (possibly incomplete) visit catalog 

489 

490 Returns 

491 ------- 

492 visitCat: `afw.table.BaseCatalog` 

493 """ 

494 

495 self.log.info("Assembling visitCatalog from %d %ss" % 

496 (len(groupedDataRefs), self.config.visitDataRefName)) 

497 

498 nCcd = len(camera) 

499 

500 if inVisitCat is None: 

501 schema = self._makeFgcmVisitSchema(nCcd) 

502 

503 visitCat = afwTable.BaseCatalog(schema) 

504 visitCat.reserve(len(groupedDataRefs)) 

505 visitCat.resize(len(groupedDataRefs)) 

506 

507 visitCat['visit'] = list(groupedDataRefs.keys()) 

508 visitCat['used'] = 0 

509 visitCat['sources_read'] = False 

510 else: 

511 visitCat = inVisitCat 

512 

513 # No matter what, fill the catalog. This will check if it was 

514 # already read. 

515 self._fillVisitCatalog(visitCat, groupedDataRefs, 

516 bkgDataRefDict=bkgDataRefDict, 

517 visitCatDataRef=visitCatDataRef) 

518 

519 return visitCat 

520 

521 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, 

522 visitCatDataRef=None): 

523 """ 

524 Fill the visit catalog with visit metadata 

525 

526 Parameters 

527 ---------- 

528 visitCat: `afw.table.BaseCatalog` 

529 Catalog with schema from _makeFgcmVisitSchema() 

530 groupedDataRefs: `dict` 

531 Dictionary with visit keys, and `list`s of 

532 `lsst.daf.persistence.ButlerDataRef` 

533 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

534 Dataref to write visitCat for checkpoints 

535 bkgDataRefDict: `dict`, optional 

536 Dictionary of gen3 dataRefHandles for background info. FIXME 

537 """ 

538 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1)) 

539 

540 for i, visit in enumerate(groupedDataRefs): 

541 # We don't use the bypasses since we need the psf info which does 

542 # not have a bypass 

543 # TODO: When DM-15500 is implemented in the Gen3 Butler, this 

544 # can be fixed 

545 

546 # Do not read those that have already been read 

547 if visitCat['used'][i]: 

548 continue 

549 

550 if (i % self.config.nVisitsPerCheckpoint) == 0: 

551 self.log.info("Retrieving metadata for %s %d (%d/%d)" % 

552 (self.config.visitDataRefName, visit, i, len(groupedDataRefs))) 

553 # Save checkpoint if desired 

554 if visitCatDataRef is not None: 

555 visitCatDataRef.put(visitCat) 

556 

557 # Note that the reference ccd is first in the list (if available). 

558 

559 # The first dataRef in the group will be the reference ccd (if available) 

560 dataRef = groupedDataRefs[visit][0] 

561 if isinstance(dataRef, dafPersist.ButlerDataRef): 

562 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox) 

563 visitInfo = exp.getInfo().getVisitInfo() 

564 label = dataRef.get(datasetType='calexp_filterLabel') 

565 physicalFilter = label.physicalLabel 

566 psf = exp.getPsf() 

567 else: 

568 visitInfo = dataRef.get(component='visitInfo') 

569 # TODO: When DM-28583 is fixed we can get the filterLabel 

570 # via dataRef.get(component='filterLabel') 

571 physicalFilter = dataRef.dataId['physical_filter'] 

572 psf = dataRef.get(component='psf') 

573 

574 rec = visitCat[i] 

575 rec['visit'] = visit 

576 rec['physicalFilter'] = physicalFilter 

577 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it 

578 # much easier to get the wcs's necessary to recompute the pointing 

579 # ra/dec at the center of the camera. 

580 radec = visitInfo.getBoresightRaDec() 

581 rec['telra'] = radec.getRa().asDegrees() 

582 rec['teldec'] = radec.getDec().asDegrees() 

583 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees() 

584 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees() 

585 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD) 

586 rec['exptime'] = visitInfo.getExposureTime() 

587 # convert from Pa to millibar 

588 # Note that I don't know if this unit will need to be per-camera config 

589 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100 

590 # Flag to signify if this is a "deep" field. Not currently used 

591 rec['deepFlag'] = 0 

592 # Relative flat scaling (1.0 means no relative scaling) 

593 rec['scaling'][:] = 1.0 

594 # Median delta aperture, to be measured from stars 

595 rec['deltaAper'] = 0.0 

596 

597 rec['psfSigma'] = psf.computeShape().getDeterminantRadius() 

598 

599 if self.config.doModelErrorsWithBackground: 

600 foundBkg = False 

601 if isinstance(dataRef, dafPersist.ButlerDataRef): 

602 det = dataRef.dataId[self.config.ccdDataRefName] 

603 if dataRef.datasetExists(datasetType='calexpBackground'): 

604 bgList = dataRef.get(datasetType='calexpBackground') 

605 foundBkg = True 

606 else: 

607 det = dataRef.dataId.byName()['detector'] 

608 try: 

609 bkgRef = bkgDataRefDict[(visit, det)] 

610 bgList = bkgRef.get() 

611 foundBkg = True 

612 except KeyError: 

613 pass 

614 

615 if foundBkg: 

616 bgStats = (bg[0].getStatsImage().getImage().array 

617 for bg in bgList) 

618 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats) 

619 else: 

620 self.log.warn('Sky background not found for visit %d / ccd %d' % 

621 (visit, det)) 

622 rec['skyBackground'] = -1.0 

623 else: 

624 rec['skyBackground'] = -1.0 

625 

626 rec['used'] = 1 

627 

628 def _makeSourceMapper(self, sourceSchema): 

629 """ 

630 Make a schema mapper for fgcm sources 

631 

632 Parameters 

633 ---------- 

634 sourceSchema: `afwTable.Schema` 

635 Default source schema from the butler 

636 

637 Returns 

638 ------- 

639 sourceMapper: `afwTable.schemaMapper` 

640 Mapper to the FGCM source schema 

641 """ 

642 

643 # create a mapper to the preferred output 

644 sourceMapper = afwTable.SchemaMapper(sourceSchema) 

645 

646 # map to ra/dec 

647 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

648 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

649 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x') 

650 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y') 

651 # Add the mapping if the field exists in the input catalog. 

652 # If the field does not exist, simply add it (set to False). 

653 # This field is not required for calibration, but is useful 

654 # to collate if available. 

655 try: 

656 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(), 

657 'psf_candidate') 

658 except LookupError: 

659 sourceMapper.editOutputSchema().addField( 

660 "psf_candidate", type='Flag', 

661 doc=("Flag set if the source was a candidate for PSF determination, " 

662 "as determined by the star selector.")) 

663 

664 # and add the fields we want 

665 sourceMapper.editOutputSchema().addField( 

666 "visit", type=np.int32, doc="Visit number") 

667 sourceMapper.editOutputSchema().addField( 

668 "ccd", type=np.int32, doc="CCD number") 

669 sourceMapper.editOutputSchema().addField( 

670 "instMag", type=np.float32, doc="Instrumental magnitude") 

671 sourceMapper.editOutputSchema().addField( 

672 "instMagErr", type=np.float32, doc="Instrumental magnitude error") 

673 sourceMapper.editOutputSchema().addField( 

674 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian") 

675 sourceMapper.editOutputSchema().addField( 

676 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset") 

677 

678 return sourceMapper 

679 

680 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None): 

681 """ 

682 Use FGCM code to match observations into unique stars. 

683 

684 Parameters 

685 ---------- 

686 visitCat: `afw.table.BaseCatalog` 

687 Catalog with visit data for fgcm 

688 obsCat: `afw.table.BaseCatalog` 

689 Full catalog of star observations for fgcm 

690 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or 

691 `lsst.daf.butler.DeferredDatasetHandle`, optional 

692 Data reference to fgcm look-up table (used if matching reference stars). 

693 

694 Returns 

695 ------- 

696 fgcmStarIdCat: `afw.table.BaseCatalog` 

697 Catalog of unique star identifiers and index keys 

698 fgcmStarIndicesCat: `afwTable.BaseCatalog` 

699 Catalog of unique star indices 

700 fgcmRefCat: `afw.table.BaseCatalog` 

701 Catalog of matched reference stars. 

702 Will be None if `config.doReferenceMatches` is False. 

703 """ 

704 # get filter names into a numpy array... 

705 # This is the type that is expected by the fgcm code 

706 visitFilterNames = np.zeros(len(visitCat), dtype='a30') 

707 for i in range(len(visitCat)): 

708 visitFilterNames[i] = visitCat[i]['physicalFilter'] 

709 

710 # match to put filterNames with observations 

711 visitIndex = np.searchsorted(visitCat['visit'], 

712 obsCat['visit']) 

713 

714 obsFilterNames = visitFilterNames[visitIndex] 

715 

716 if self.config.doReferenceMatches: 

717 # Get the reference filter names, using the LUT 

718 lutCat = lutDataRef.get() 

719 

720 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in 

721 zip(lutCat[0]['physicalFilters'].split(','), 

722 lutCat[0]['stdPhysicalFilters'].split(','))} 

723 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in 

724 zip(lutCat[0]['stdPhysicalFilters'].split(','), 

725 lutCat[0]['lambdaStdFilter'])} 

726 

727 del lutCat 

728 

729 referenceFilterNames = self._getReferenceFilterNames(visitCat, 

730 stdFilterDict, 

731 stdLambdaDict) 

732 self.log.info("Using the following reference filters: %s" % 

733 (', '.join(referenceFilterNames))) 

734 

735 else: 

736 # This should be an empty list 

737 referenceFilterNames = [] 

738 

739 # make the fgcm starConfig dict 

740 starConfig = {'logger': self.log, 

741 'filterToBand': self.config.physicalFilterMap, 

742 'requiredBands': self.config.requiredBands, 

743 'minPerBand': self.config.minPerBand, 

744 'matchRadius': self.config.matchRadius, 

745 'isolationRadius': self.config.isolationRadius, 

746 'matchNSide': self.config.matchNside, 

747 'coarseNSide': self.config.coarseNside, 

748 'densNSide': self.config.densityCutNside, 

749 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 

750 'primaryBands': self.config.primaryBands, 

751 'referenceFilterNames': referenceFilterNames} 

752 

753 # initialize the FgcmMakeStars object 

754 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) 

755 

756 # make the primary stars 

757 # note that the ra/dec native Angle format is radians 

758 # We determine the conversion from the native units (typically 

759 # radians) to degrees for the first observation. This allows us 

760 # to treate ra/dec as numpy arrays rather than Angles, which would 

761 # be approximately 600x slower. 

762 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) 

763 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, 

764 obsCat['dec'] * conv, 

765 filterNameArray=obsFilterNames, 

766 bandSelected=False) 

767 

768 # and match all the stars 

769 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, 

770 obsCat['dec'] * conv, 

771 obsFilterNames) 

772 

773 if self.config.doReferenceMatches: 

774 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) 

775 

776 # now persist 

777 

778 objSchema = self._makeFgcmObjSchema() 

779 

780 # make catalog and records 

781 fgcmStarIdCat = afwTable.BaseCatalog(objSchema) 

782 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) 

783 for i in range(fgcmMakeStars.objIndexCat.size): 

784 fgcmStarIdCat.addNew() 

785 

786 # fill the catalog 

787 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] 

788 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] 

789 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] 

790 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex'] 

791 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] 

792 

793 obsSchema = self._makeFgcmObsSchema() 

794 

795 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) 

796 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) 

797 for i in range(fgcmMakeStars.obsIndexCat.size): 

798 fgcmStarIndicesCat.addNew() 

799 

800 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex'] 

801 

802 if self.config.doReferenceMatches: 

803 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) 

804 

805 fgcmRefCat = afwTable.BaseCatalog(refSchema) 

806 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) 

807 

808 for i in range(fgcmMakeStars.referenceCat.size): 

809 fgcmRefCat.addNew() 

810 

811 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] 

812 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] 

813 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr'] 

814 

815 md = PropertyList() 

816 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION) 

817 md.set("FILTERNAMES", referenceFilterNames) 

818 fgcmRefCat.setMetadata(md) 

819 

820 else: 

821 fgcmRefCat = None 

822 

823 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat 

824 

825 def _makeFgcmVisitSchema(self, nCcd): 

826 """ 

827 Make a schema for an fgcmVisitCatalog 

828 

829 Parameters 

830 ---------- 

831 nCcd: `int` 

832 Number of CCDs in the camera 

833 

834 Returns 

835 ------- 

836 schema: `afwTable.Schema` 

837 """ 

838 

839 schema = afwTable.Schema() 

840 schema.addField('visit', type=np.int32, doc="Visit number") 

841 schema.addField('physicalFilter', type=str, size=30, doc="Physical filter") 

842 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") 

843 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") 

844 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") 

845 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)") 

846 schema.addField('mjd', type=np.float64, doc="MJD of visit") 

847 schema.addField('exptime', type=np.float32, doc="Exposure time") 

848 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") 

849 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") 

850 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") 

851 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") 

852 # the following field is not used yet 

853 schema.addField('deepFlag', type=np.int32, doc="Deep observation") 

854 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", 

855 size=nCcd) 

856 schema.addField('used', type=np.int32, doc="This visit has been ingested.") 

857 schema.addField('sources_read', type='Flag', doc="This visit had sources read.") 

858 

859 return schema 

860 

861 def _makeFgcmObjSchema(self): 

862 """ 

863 Make a schema for the objIndexCat from fgcmMakeStars 

864 

865 Returns 

866 ------- 

867 schema: `afwTable.Schema` 

868 """ 

869 

870 objSchema = afwTable.Schema() 

871 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

872 # Will investigate making these angles... 

873 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)') 

874 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)') 

875 objSchema.addField('obsArrIndex', type=np.int32, 

876 doc='Index in obsIndexTable for first observation') 

877 objSchema.addField('nObs', type=np.int32, doc='Total number of observations') 

878 

879 return objSchema 

880 

881 def _makeFgcmObsSchema(self): 

882 """ 

883 Make a schema for the obsIndexCat from fgcmMakeStars 

884 

885 Returns 

886 ------- 

887 schema: `afwTable.Schema` 

888 """ 

889 

890 obsSchema = afwTable.Schema() 

891 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table') 

892 

893 return obsSchema 

894 

895 def _makeFgcmRefSchema(self, nReferenceBands): 

896 """ 

897 Make a schema for the referenceCat from fgcmMakeStars 

898 

899 Parameters 

900 ---------- 

901 nReferenceBands: `int` 

902 Number of reference bands 

903 

904 Returns 

905 ------- 

906 schema: `afwTable.Schema` 

907 """ 

908 

909 refSchema = afwTable.Schema() 

910 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

911 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)', 

912 size=nReferenceBands) 

913 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array', 

914 size=nReferenceBands) 

915 

916 return refSchema 

917 

918 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict): 

919 """ 

920 Get the reference filter names, in wavelength order, from the visitCat and 

921 information from the look-up-table. 

922 

923 Parameters 

924 ---------- 

925 visitCat: `afw.table.BaseCatalog` 

926 Catalog with visit data for FGCM 

927 stdFilterDict: `dict` 

928 Mapping of filterName to stdFilterName from LUT 

929 stdLambdaDict: `dict` 

930 Mapping of stdFilterName to stdLambda from LUT 

931 

932 Returns 

933 ------- 

934 referenceFilterNames: `list` 

935 Wavelength-ordered list of reference filter names 

936 """ 

937 

938 # Find the unique list of filter names in visitCat 

939 filterNames = np.unique(visitCat.asAstropy()['physicalFilter']) 

940 

941 # Find the unique list of "standard" filters 

942 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames} 

943 

944 # And sort these by wavelength 

945 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get) 

946 

947 return referenceFilterNames