Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for BuildStars using src tables or sourceTable_visit tables. 

22""" 

23 

24import os 

25import sys 

26import traceback 

27import abc 

28 

29import numpy as np 

30 

31import lsst.daf.persistence as dafPersist 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34import lsst.afw.table as afwTable 

35import lsst.geom as geom 

36from lsst.daf.base import PropertyList 

37from lsst.daf.base.dateTime import DateTime 

38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

39 

40from .utilities import computeApertureRadiusFromDataRef 

41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask 

42 

43import fgcm 

44 

45REFSTARS_FORMAT_VERSION = 1 

46 

47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask'] 

48 

49 

50class FgcmBuildStarsConfigBase(pexConfig.Config): 

51 """Base config for FgcmBuildStars tasks""" 

52 

53 instFluxField = pexConfig.Field( 

54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. " 

55 "The associated flag will be implicitly included in badFlags"), 

56 dtype=str, 

57 default='slot_CalibFlux_instFlux', 

58 ) 

59 minPerBand = pexConfig.Field( 

60 doc="Minimum observations per band", 

61 dtype=int, 

62 default=2, 

63 ) 

64 matchRadius = pexConfig.Field( 

65 doc="Match radius (arcseconds)", 

66 dtype=float, 

67 default=1.0, 

68 ) 

69 isolationRadius = pexConfig.Field( 

70 doc="Isolation radius (arcseconds)", 

71 dtype=float, 

72 default=2.0, 

73 ) 

74 densityCutNside = pexConfig.Field( 

75 doc="Density cut healpix nside", 

76 dtype=int, 

77 default=128, 

78 ) 

79 densityCutMaxPerPixel = pexConfig.Field( 

80 doc="Density cut number of stars per pixel", 

81 dtype=int, 

82 default=1000, 

83 ) 

84 matchNside = pexConfig.Field( 

85 doc="Healpix Nside for matching", 

86 dtype=int, 

87 default=4096, 

88 ) 

89 coarseNside = pexConfig.Field( 

90 doc="Healpix coarse Nside for partitioning matches", 

91 dtype=int, 

92 default=8, 

93 ) 

94 filterMap = pexConfig.DictField( 

95 doc="Mapping from 'filterName' to band.", 

96 keytype=str, 

97 itemtype=str, 

98 default={}, 

99 deprecated=("This field is no longer used, and has been deprecated by " 

100 "DM-28088. It will be removed after v22. Use " 

101 "physicalFilterMap instead.") 

102 ) 

103 # The following config will not be necessary after Gen2 retirement. 

104 # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band' 

105 # which is easiest to access in the config file. 

106 physicalFilterMap = pexConfig.DictField( 

107 doc="Mapping from 'physicalFilter' to band.", 

108 keytype=str, 

109 itemtype=str, 

110 default={}, 

111 ) 

112 requiredBands = pexConfig.ListField( 

113 doc="Bands required for each star", 

114 dtype=str, 

115 default=(), 

116 ) 

117 primaryBands = pexConfig.ListField( 

118 doc=("Bands for 'primary' star matches. " 

119 "A star must be observed in one of these bands to be considered " 

120 "as a calibration star."), 

121 dtype=str, 

122 default=None 

123 ) 

124 visitDataRefName = pexConfig.Field( 

125 doc="dataRef name for the 'visit' field, usually 'visit'.", 

126 dtype=str, 

127 default="visit" 

128 ) 

129 ccdDataRefName = pexConfig.Field( 

130 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.", 

131 dtype=str, 

132 default="ccd" 

133 ) 

134 doApplyWcsJacobian = pexConfig.Field( 

135 doc="Apply the jacobian of the WCS to the star observations prior to fit?", 

136 dtype=bool, 

137 default=True 

138 ) 

139 doModelErrorsWithBackground = pexConfig.Field( 

140 doc="Model flux errors with background term?", 

141 dtype=bool, 

142 default=True 

143 ) 

144 psfCandidateName = pexConfig.Field( 

145 doc="Name of field with psf candidate flag for propagation", 

146 dtype=str, 

147 default="calib_psf_candidate" 

148 ) 

149 doSubtractLocalBackground = pexConfig.Field( 

150 doc=("Subtract the local background before performing calibration? " 

151 "This is only supported for circular aperture calibration fluxes."), 

152 dtype=bool, 

153 default=False 

154 ) 

155 localBackgroundFluxField = pexConfig.Field( 

156 doc="Full name of the local background instFlux field to use.", 

157 dtype=str, 

158 default='base_LocalBackground_instFlux' 

159 ) 

160 sourceSelector = sourceSelectorRegistry.makeField( 

161 doc="How to select sources", 

162 default="science" 

163 ) 

164 apertureInnerInstFluxField = pexConfig.Field( 

165 doc=("Full name of instFlux field that contains inner aperture " 

166 "flux for aperture correction proxy"), 

167 dtype=str, 

168 default='base_CircularApertureFlux_12_0_instFlux' 

169 ) 

170 apertureOuterInstFluxField = pexConfig.Field( 

171 doc=("Full name of instFlux field that contains outer aperture " 

172 "flux for aperture correction proxy"), 

173 dtype=str, 

174 default='base_CircularApertureFlux_17_0_instFlux' 

175 ) 

176 doReferenceMatches = pexConfig.Field( 

177 doc="Match reference catalog as additional constraint on calibration", 

178 dtype=bool, 

179 default=True, 

180 ) 

181 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField( 

182 target=FgcmLoadReferenceCatalogTask, 

183 doc="FGCM reference object loader", 

184 ) 

185 nVisitsPerCheckpoint = pexConfig.Field( 

186 doc="Number of visits read between checkpoints", 

187 dtype=int, 

188 default=500, 

189 ) 

190 

191 def setDefaults(self): 

192 sourceSelector = self.sourceSelector["science"] 

193 sourceSelector.setDefaults() 

194 

195 sourceSelector.doFlags = True 

196 sourceSelector.doUnresolved = True 

197 sourceSelector.doSignalToNoise = True 

198 sourceSelector.doIsolated = True 

199 

200 sourceSelector.signalToNoise.minimum = 10.0 

201 sourceSelector.signalToNoise.maximum = 1000.0 

202 

203 # FGCM operates on unresolved sources, and this setting is 

204 # appropriate for the current base_ClassificationExtendedness 

205 sourceSelector.unresolved.maximum = 0.5 

206 

207 

208class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner): 

209 """Subclass of TaskRunner for FgcmBuildStars tasks 

210 

211 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of 

212 arguments, one of which is the butler (for persistence and mapper data), 

213 and a list of dataRefs extracted from the command line. Note that FGCM 

214 runs on a large set of dataRefs, and not on single dataRef/tract/patch. 

215 This class transforms the process arguments generated by the ArgumentParser 

216 into the arguments expected by FgcmBuildStarsTask.run(). This runner does 

217 not use any parallelization. 

218 """ 

219 @staticmethod 

220 def getTargetList(parsedCmd): 

221 """ 

222 Return a list with one element: a tuple with the butler and 

223 list of dataRefs 

224 """ 

225 # we want to combine the butler with any (or no!) dataRefs 

226 return [(parsedCmd.butler, parsedCmd.id.refList)] 

227 

228 def __call__(self, args): 

229 """ 

230 Parameters 

231 ---------- 

232 args: `tuple` with (butler, dataRefList) 

233 

234 Returns 

235 ------- 

236 exitStatus: `list` with `lsst.pipe.base.Struct` 

237 exitStatus (0: success; 1: failure) 

238 """ 

239 butler, dataRefList = args 

240 

241 task = self.TaskClass(config=self.config, log=self.log) 

242 

243 exitStatus = 0 

244 if self.doRaise: 

245 task.runDataRef(butler, dataRefList) 

246 else: 

247 try: 

248 task.runDataRef(butler, dataRefList) 

249 except Exception as e: 

250 exitStatus = 1 

251 task.log.fatal("Failed: %s" % e) 

252 if not isinstance(e, pipeBase.TaskError): 

253 traceback.print_exc(file=sys.stderr) 

254 

255 task.writeMetadata(butler) 

256 

257 # The task does not return any results: 

258 return [pipeBase.Struct(exitStatus=exitStatus)] 

259 

260 def run(self, parsedCmd): 

261 """ 

262 Run the task, with no multiprocessing 

263 

264 Parameters 

265 ---------- 

266 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

267 """ 

268 

269 resultList = [] 

270 

271 if self.precall(parsedCmd): 

272 targetList = self.getTargetList(parsedCmd) 

273 resultList = self(targetList[0]) 

274 

275 return resultList 

276 

277 

278class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC): 

279 """ 

280 Base task to build stars for FGCM global calibration 

281 

282 Parameters 

283 ---------- 

284 butler : `lsst.daf.persistence.Butler` 

285 """ 

286 def __init__(self, butler=None, initInputs=None, **kwargs): 

287 super().__init__(**kwargs) 

288 

289 self.makeSubtask("sourceSelector") 

290 # Only log warning and fatal errors from the sourceSelector 

291 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN) 

292 

293 # no saving of metadata for now 

294 def _getMetadataName(self): 

295 return None 

296 

297 @pipeBase.timeMethod 

298 def runDataRef(self, butler, dataRefs): 

299 """ 

300 Cross-match and make star list for FGCM Input 

301 

302 Parameters 

303 ---------- 

304 butler: `lsst.daf.persistence.Butler` 

305 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

306 Source data references for the input visits. 

307 

308 Raises 

309 ------ 

310 RuntimeErrror: Raised if `config.doReferenceMatches` is set and 

311 an fgcmLookUpTable is not available, or if computeFluxApertureRadius() 

312 fails if the calibFlux is not a CircularAperture flux. 

313 """ 

314 datasetType = dataRefs[0].butlerSubset.datasetType 

315 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType) 

316 

317 if self.config.doReferenceMatches: 

318 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler) 

319 # Ensure that we have a LUT 

320 if not butler.datasetExists('fgcmLookUpTable'): 

321 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches") 

322 # Compute aperture radius if necessary. This is useful to do now before 

323 # any heavy lifting has happened (fail early). 

324 calibFluxApertureRadius = None 

325 if self.config.doSubtractLocalBackground: 

326 try: 

327 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

328 self.config.instFluxField) 

329 except RuntimeError as e: 

330 raise RuntimeError("Could not determine aperture radius from %s. " 

331 "Cannot use doSubtractLocalBackground." % 

332 (self.config.instFluxField)) from e 

333 

334 camera = butler.get('camera') 

335 groupedDataRefs = self._findAndGroupDataRefsGen2(butler, camera, dataRefs) 

336 

337 # Make the visit catalog if necessary 

338 # First check if the visit catalog is in the _current_ path 

339 # We cannot use Gen2 datasetExists() because that checks all parent 

340 # directories as well, which would make recovering from faults 

341 # and fgcmcal reruns impossible. 

342 visitCatDataRef = butler.dataRef('fgcmVisitCatalog') 

343 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0] 

344 if os.path.exists(filename): 

345 # This file exists and we should continue processing 

346 inVisitCat = visitCatDataRef.get() 

347 if len(inVisitCat) != len(groupedDataRefs): 

348 raise RuntimeError("Existing visitCatalog found, but has an inconsistent " 

349 "number of visits. Cannot continue.") 

350 else: 

351 inVisitCat = None 

352 

353 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs, 

354 visitCatDataRef=visitCatDataRef, 

355 inVisitCat=inVisitCat) 

356 

357 # Persist the visitCat as a checkpoint file. 

358 visitCatDataRef.put(visitCat) 

359 

360 starObsDataRef = butler.dataRef('fgcmStarObservations') 

361 filename = starObsDataRef.get('fgcmStarObservations_filename')[0] 

362 if os.path.exists(filename): 

363 inStarObsCat = starObsDataRef.get() 

364 else: 

365 inStarObsCat = None 

366 

367 rad = calibFluxApertureRadius 

368 sourceSchemaDataRef = butler.dataRef('src_schema') 

369 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs, 

370 visitCat, 

371 sourceSchemaDataRef, 

372 camera, 

373 calibFluxApertureRadius=rad, 

374 starObsDataRef=starObsDataRef, 

375 visitCatDataRef=visitCatDataRef, 

376 inStarObsCat=inStarObsCat) 

377 visitCatDataRef.put(visitCat) 

378 starObsDataRef.put(fgcmStarObservationCat) 

379 

380 # Always do the matching. 

381 if self.config.doReferenceMatches: 

382 lutDataRef = butler.dataRef('fgcmLookUpTable') 

383 else: 

384 lutDataRef = None 

385 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

386 fgcmStarObservationCat, 

387 lutDataRef=lutDataRef) 

388 

389 # Persist catalogs via the butler 

390 butler.put(fgcmStarIdCat, 'fgcmStarIds') 

391 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') 

392 if fgcmRefCat is not None: 

393 butler.put(fgcmRefCat, 'fgcmReferenceStars') 

394 

395 @abc.abstractmethod 

396 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs): 

397 """ 

398 Find and group dataRefs (by visit); Gen2 only. 

399 

400 Parameters 

401 ---------- 

402 butler : `lsst.daf.persistence.Butler` 

403 Gen2 butler. 

404 camera : `lsst.afw.cameraGeom.Camera` 

405 Camera from the butler. 

406 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

407 Data references for the input visits. 

408 

409 Returns 

410 ------- 

411 groupedDataRefs : `dict` [`int`, `list`] 

412 Dictionary with sorted visit keys, and `list`s of 

413 `lsst.daf.persistence.ButlerDataRef` 

414 """ 

415 raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.") 

416 

417 @abc.abstractmethod 

418 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

419 sourceSchemaDataRef, 

420 camera, 

421 calibFluxApertureRadius=None, 

422 visitCatDataRef=None, 

423 starObsDataRef=None, 

424 inStarObsCat=None): 

425 """ 

426 Compile all good star observations from visits in visitCat. Checkpoint files 

427 will be stored if both visitCatDataRef and starObsDataRef are not None. 

428 

429 Parameters 

430 ---------- 

431 groupedDataRefs: `dict` of `list`s 

432 Lists of `~lsst.daf.persistence.ButlerDataRef` or 

433 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit. 

434 visitCat: `~afw.table.BaseCatalog` 

435 Catalog with visit data for FGCM 

436 sourceSchemaDataRef: `~lsst.daf.persistence.ButlerDataRef` or 

437 `~lsst.daf.butler.DeferredDatasetHandle` 

438 DataRef for the schema of the src catalogs. 

439 camera: `~lsst.afw.cameraGeom.Camera` 

440 calibFluxApertureRadius: `float`, optional 

441 Aperture radius for calibration flux. 

442 visitCatDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional 

443 Dataref to write visitCat for checkpoints 

444 starObsDataRef: `~lsst.daf.persistence.ButlerDataRef`, optional 

445 Dataref to write the star observation catalog for checkpoints. 

446 inStarObsCat: `~afw.table.BaseCatalog` 

447 Input observation catalog. If this is incomplete, observations 

448 will be appended from when it was cut off. 

449 

450 Returns 

451 ------- 

452 fgcmStarObservations: `afw.table.BaseCatalog` 

453 Full catalog of good observations. 

454 

455 Raises 

456 ------ 

457 RuntimeError: Raised if doSubtractLocalBackground is True and 

458 calibFluxApertureRadius is not set. 

459 """ 

460 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.") 

461 

462 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, 

463 visitCatDataRef=None, inVisitCat=None): 

464 """ 

465 Make a visit catalog with all the keys from each visit 

466 

467 Parameters 

468 ---------- 

469 camera: `lsst.afw.cameraGeom.Camera` 

470 Camera from the butler 

471 groupedDataRefs: `dict` 

472 Dictionary with visit keys, and `list`s of 

473 `lsst.daf.persistence.ButlerDataRef` 

474 bkgDataRefDict: `dict`, optional 

475 Dictionary of gen3 dataRefHandles for background info. 

476 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

477 Dataref to write visitCat for checkpoints 

478 inVisitCat: `afw.table.BaseCatalog`, optional 

479 Input (possibly incomplete) visit catalog 

480 

481 Returns 

482 ------- 

483 visitCat: `afw.table.BaseCatalog` 

484 """ 

485 

486 self.log.info("Assembling visitCatalog from %d %ss" % 

487 (len(groupedDataRefs), self.config.visitDataRefName)) 

488 

489 nCcd = len(camera) 

490 

491 if inVisitCat is None: 

492 schema = self._makeFgcmVisitSchema(nCcd) 

493 

494 visitCat = afwTable.BaseCatalog(schema) 

495 visitCat.reserve(len(groupedDataRefs)) 

496 visitCat.resize(len(groupedDataRefs)) 

497 

498 visitCat['visit'] = list(groupedDataRefs.keys()) 

499 visitCat['used'] = 0 

500 visitCat['sources_read'] = False 

501 else: 

502 visitCat = inVisitCat 

503 

504 # No matter what, fill the catalog. This will check if it was 

505 # already read. 

506 self._fillVisitCatalog(visitCat, groupedDataRefs, 

507 bkgDataRefDict=bkgDataRefDict, 

508 visitCatDataRef=visitCatDataRef) 

509 

510 return visitCat 

511 

512 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, 

513 visitCatDataRef=None): 

514 """ 

515 Fill the visit catalog with visit metadata 

516 

517 Parameters 

518 ---------- 

519 visitCat : `afw.table.BaseCatalog` 

520 Visit catalog. See _makeFgcmVisitSchema() for schema definition. 

521 groupedDataRefs : `dict` 

522 Dictionary with visit keys, and `list`s of 

523 `lsst.daf.persistence.ButlerDataRef` or 

524 `lsst.daf.butler.DeferredDatasetHandle` 

525 visitCatDataRef : `lsst.daf.persistence.ButlerDataRef`, optional 

526 Dataref to write ``visitCat`` for checkpoints. Gen2 only. 

527 bkgDataRefDict : `dict`, optional 

528 Dictionary of Gen3 `lsst.daf.butler.DeferredDatasetHandle` 

529 for background info. 

530 """ 

531 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1)) 

532 

533 for i, visit in enumerate(groupedDataRefs): 

534 # We don't use the bypasses since we need the psf info which does 

535 # not have a bypass 

536 # TODO: When DM-15500 is implemented in the Gen3 Butler, this 

537 # can be fixed 

538 

539 # Do not read those that have already been read 

540 if visitCat['used'][i]: 

541 continue 

542 

543 if (i % self.config.nVisitsPerCheckpoint) == 0: 

544 self.log.info("Retrieving metadata for %s %d (%d/%d)" % 

545 (self.config.visitDataRefName, visit, i, len(groupedDataRefs))) 

546 # Save checkpoint if desired 

547 if visitCatDataRef is not None: 

548 visitCatDataRef.put(visitCat) 

549 

550 dataRef = groupedDataRefs[visit][0] 

551 if isinstance(dataRef, dafPersist.ButlerDataRef): 

552 # Gen2: calexp dataRef 

553 # The first dataRef in the group will be the reference ccd (if available) 

554 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox) 

555 visitInfo = exp.getInfo().getVisitInfo() 

556 label = dataRef.get(datasetType='calexp_filterLabel') 

557 physicalFilter = label.physicalLabel 

558 psf = exp.getPsf() 

559 psfSigma = psf.computeShape().getDeterminantRadius() 

560 else: 

561 # Gen3: use the visitSummary dataRef 

562 summary = dataRef.get() 

563 

564 summaryRow = summary.find(self.config.referenceCCD) 

565 if summaryRow is None: 

566 # Take the first available ccd if reference isn't available 

567 summaryRow = summary[0] 

568 

569 summaryDetector = summaryRow['id'] 

570 visitInfo = summaryRow.getVisitInfo() 

571 physicalFilter = summaryRow['physical_filter'] 

572 # Compute the median psf sigma if possible 

573 goodSigma, = np.where(summary['psfSigma'] > 0) 

574 if goodSigma.size > 2: 

575 psfSigma = np.median(summary['psfSigma'][goodSigma]) 

576 elif goodSigma > 0: 

577 psfSigma = np.mean(summary['psfSigma'][goodSigma]) 

578 else: 

579 psfSigma = 0.0 

580 

581 rec = visitCat[i] 

582 rec['visit'] = visit 

583 rec['physicalFilter'] = physicalFilter 

584 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it 

585 # much easier to get the wcs's necessary to recompute the pointing 

586 # ra/dec at the center of the camera. 

587 radec = visitInfo.getBoresightRaDec() 

588 rec['telra'] = radec.getRa().asDegrees() 

589 rec['teldec'] = radec.getDec().asDegrees() 

590 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees() 

591 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees() 

592 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD) 

593 rec['exptime'] = visitInfo.getExposureTime() 

594 # convert from Pa to millibar 

595 # Note that I don't know if this unit will need to be per-camera config 

596 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100 

597 # Flag to signify if this is a "deep" field. Not currently used 

598 rec['deepFlag'] = 0 

599 # Relative flat scaling (1.0 means no relative scaling) 

600 rec['scaling'][:] = 1.0 

601 # Median delta aperture, to be measured from stars 

602 rec['deltaAper'] = 0.0 

603 rec['psfSigma'] = psfSigma 

604 

605 if self.config.doModelErrorsWithBackground: 

606 foundBkg = False 

607 if isinstance(dataRef, dafPersist.ButlerDataRef): 

608 # Gen2-style dataRef 

609 det = dataRef.dataId[self.config.ccdDataRefName] 

610 if dataRef.datasetExists(datasetType='calexpBackground'): 

611 bgList = dataRef.get(datasetType='calexpBackground') 

612 foundBkg = True 

613 else: 

614 # Gen3-style dataRef 

615 try: 

616 # Use the same detector used from the summary. 

617 bkgRef = bkgDataRefDict[(visit, summaryDetector)] 

618 bgList = bkgRef.get() 

619 foundBkg = True 

620 except KeyError: 

621 pass 

622 

623 if foundBkg: 

624 bgStats = (bg[0].getStatsImage().getImage().array 

625 for bg in bgList) 

626 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats) 

627 else: 

628 self.log.warn('Sky background not found for visit %d / ccd %d' % 

629 (visit, det)) 

630 rec['skyBackground'] = -1.0 

631 else: 

632 rec['skyBackground'] = -1.0 

633 

634 rec['used'] = 1 

635 

636 def _makeSourceMapper(self, sourceSchema): 

637 """ 

638 Make a schema mapper for fgcm sources 

639 

640 Parameters 

641 ---------- 

642 sourceSchema: `afwTable.Schema` 

643 Default source schema from the butler 

644 

645 Returns 

646 ------- 

647 sourceMapper: `afwTable.schemaMapper` 

648 Mapper to the FGCM source schema 

649 """ 

650 

651 # create a mapper to the preferred output 

652 sourceMapper = afwTable.SchemaMapper(sourceSchema) 

653 

654 # map to ra/dec 

655 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

656 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

657 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x') 

658 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y') 

659 # Add the mapping if the field exists in the input catalog. 

660 # If the field does not exist, simply add it (set to False). 

661 # This field is not required for calibration, but is useful 

662 # to collate if available. 

663 try: 

664 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(), 

665 'psf_candidate') 

666 except LookupError: 

667 sourceMapper.editOutputSchema().addField( 

668 "psf_candidate", type='Flag', 

669 doc=("Flag set if the source was a candidate for PSF determination, " 

670 "as determined by the star selector.")) 

671 

672 # and add the fields we want 

673 sourceMapper.editOutputSchema().addField( 

674 "visit", type=np.int32, doc="Visit number") 

675 sourceMapper.editOutputSchema().addField( 

676 "ccd", type=np.int32, doc="CCD number") 

677 sourceMapper.editOutputSchema().addField( 

678 "instMag", type=np.float32, doc="Instrumental magnitude") 

679 sourceMapper.editOutputSchema().addField( 

680 "instMagErr", type=np.float32, doc="Instrumental magnitude error") 

681 sourceMapper.editOutputSchema().addField( 

682 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian") 

683 sourceMapper.editOutputSchema().addField( 

684 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset") 

685 

686 return sourceMapper 

687 

688 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None): 

689 """ 

690 Use FGCM code to match observations into unique stars. 

691 

692 Parameters 

693 ---------- 

694 visitCat: `afw.table.BaseCatalog` 

695 Catalog with visit data for fgcm 

696 obsCat: `afw.table.BaseCatalog` 

697 Full catalog of star observations for fgcm 

698 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or 

699 `lsst.daf.butler.DeferredDatasetHandle`, optional 

700 Data reference to fgcm look-up table (used if matching reference stars). 

701 

702 Returns 

703 ------- 

704 fgcmStarIdCat: `afw.table.BaseCatalog` 

705 Catalog of unique star identifiers and index keys 

706 fgcmStarIndicesCat: `afwTable.BaseCatalog` 

707 Catalog of unique star indices 

708 fgcmRefCat: `afw.table.BaseCatalog` 

709 Catalog of matched reference stars. 

710 Will be None if `config.doReferenceMatches` is False. 

711 """ 

712 # get filter names into a numpy array... 

713 # This is the type that is expected by the fgcm code 

714 visitFilterNames = np.zeros(len(visitCat), dtype='a30') 

715 for i in range(len(visitCat)): 

716 visitFilterNames[i] = visitCat[i]['physicalFilter'] 

717 

718 # match to put filterNames with observations 

719 visitIndex = np.searchsorted(visitCat['visit'], 

720 obsCat['visit']) 

721 

722 obsFilterNames = visitFilterNames[visitIndex] 

723 

724 if self.config.doReferenceMatches: 

725 # Get the reference filter names, using the LUT 

726 lutCat = lutDataRef.get() 

727 

728 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in 

729 zip(lutCat[0]['physicalFilters'].split(','), 

730 lutCat[0]['stdPhysicalFilters'].split(','))} 

731 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in 

732 zip(lutCat[0]['stdPhysicalFilters'].split(','), 

733 lutCat[0]['lambdaStdFilter'])} 

734 

735 del lutCat 

736 

737 referenceFilterNames = self._getReferenceFilterNames(visitCat, 

738 stdFilterDict, 

739 stdLambdaDict) 

740 self.log.info("Using the following reference filters: %s" % 

741 (', '.join(referenceFilterNames))) 

742 

743 else: 

744 # This should be an empty list 

745 referenceFilterNames = [] 

746 

747 # make the fgcm starConfig dict 

748 starConfig = {'logger': self.log, 

749 'filterToBand': self.config.physicalFilterMap, 

750 'requiredBands': self.config.requiredBands, 

751 'minPerBand': self.config.minPerBand, 

752 'matchRadius': self.config.matchRadius, 

753 'isolationRadius': self.config.isolationRadius, 

754 'matchNSide': self.config.matchNside, 

755 'coarseNSide': self.config.coarseNside, 

756 'densNSide': self.config.densityCutNside, 

757 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 

758 'primaryBands': self.config.primaryBands, 

759 'referenceFilterNames': referenceFilterNames} 

760 

761 # initialize the FgcmMakeStars object 

762 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) 

763 

764 # make the primary stars 

765 # note that the ra/dec native Angle format is radians 

766 # We determine the conversion from the native units (typically 

767 # radians) to degrees for the first observation. This allows us 

768 # to treate ra/dec as numpy arrays rather than Angles, which would 

769 # be approximately 600x slower. 

770 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) 

771 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, 

772 obsCat['dec'] * conv, 

773 filterNameArray=obsFilterNames, 

774 bandSelected=False) 

775 

776 # and match all the stars 

777 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, 

778 obsCat['dec'] * conv, 

779 obsFilterNames) 

780 

781 if self.config.doReferenceMatches: 

782 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) 

783 

784 # now persist 

785 

786 objSchema = self._makeFgcmObjSchema() 

787 

788 # make catalog and records 

789 fgcmStarIdCat = afwTable.BaseCatalog(objSchema) 

790 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) 

791 for i in range(fgcmMakeStars.objIndexCat.size): 

792 fgcmStarIdCat.addNew() 

793 

794 # fill the catalog 

795 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] 

796 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] 

797 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] 

798 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex'] 

799 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] 

800 

801 obsSchema = self._makeFgcmObsSchema() 

802 

803 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) 

804 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) 

805 for i in range(fgcmMakeStars.obsIndexCat.size): 

806 fgcmStarIndicesCat.addNew() 

807 

808 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex'] 

809 

810 if self.config.doReferenceMatches: 

811 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) 

812 

813 fgcmRefCat = afwTable.BaseCatalog(refSchema) 

814 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) 

815 

816 for i in range(fgcmMakeStars.referenceCat.size): 

817 fgcmRefCat.addNew() 

818 

819 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] 

820 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] 

821 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr'] 

822 

823 md = PropertyList() 

824 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION) 

825 md.set("FILTERNAMES", referenceFilterNames) 

826 fgcmRefCat.setMetadata(md) 

827 

828 else: 

829 fgcmRefCat = None 

830 

831 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat 

832 

833 def _makeFgcmVisitSchema(self, nCcd): 

834 """ 

835 Make a schema for an fgcmVisitCatalog 

836 

837 Parameters 

838 ---------- 

839 nCcd: `int` 

840 Number of CCDs in the camera 

841 

842 Returns 

843 ------- 

844 schema: `afwTable.Schema` 

845 """ 

846 

847 schema = afwTable.Schema() 

848 schema.addField('visit', type=np.int32, doc="Visit number") 

849 schema.addField('physicalFilter', type=str, size=30, doc="Physical filter") 

850 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") 

851 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") 

852 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") 

853 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)") 

854 schema.addField('mjd', type=np.float64, doc="MJD of visit") 

855 schema.addField('exptime', type=np.float32, doc="Exposure time") 

856 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") 

857 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") 

858 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") 

859 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") 

860 # the following field is not used yet 

861 schema.addField('deepFlag', type=np.int32, doc="Deep observation") 

862 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", 

863 size=nCcd) 

864 schema.addField('used', type=np.int32, doc="This visit has been ingested.") 

865 schema.addField('sources_read', type='Flag', doc="This visit had sources read.") 

866 

867 return schema 

868 

869 def _makeFgcmObjSchema(self): 

870 """ 

871 Make a schema for the objIndexCat from fgcmMakeStars 

872 

873 Returns 

874 ------- 

875 schema: `afwTable.Schema` 

876 """ 

877 

878 objSchema = afwTable.Schema() 

879 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

880 # Will investigate making these angles... 

881 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)') 

882 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)') 

883 objSchema.addField('obsArrIndex', type=np.int32, 

884 doc='Index in obsIndexTable for first observation') 

885 objSchema.addField('nObs', type=np.int32, doc='Total number of observations') 

886 

887 return objSchema 

888 

889 def _makeFgcmObsSchema(self): 

890 """ 

891 Make a schema for the obsIndexCat from fgcmMakeStars 

892 

893 Returns 

894 ------- 

895 schema: `afwTable.Schema` 

896 """ 

897 

898 obsSchema = afwTable.Schema() 

899 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table') 

900 

901 return obsSchema 

902 

903 def _makeFgcmRefSchema(self, nReferenceBands): 

904 """ 

905 Make a schema for the referenceCat from fgcmMakeStars 

906 

907 Parameters 

908 ---------- 

909 nReferenceBands: `int` 

910 Number of reference bands 

911 

912 Returns 

913 ------- 

914 schema: `afwTable.Schema` 

915 """ 

916 

917 refSchema = afwTable.Schema() 

918 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

919 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)', 

920 size=nReferenceBands) 

921 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array', 

922 size=nReferenceBands) 

923 

924 return refSchema 

925 

926 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict): 

927 """ 

928 Get the reference filter names, in wavelength order, from the visitCat and 

929 information from the look-up-table. 

930 

931 Parameters 

932 ---------- 

933 visitCat: `afw.table.BaseCatalog` 

934 Catalog with visit data for FGCM 

935 stdFilterDict: `dict` 

936 Mapping of filterName to stdFilterName from LUT 

937 stdLambdaDict: `dict` 

938 Mapping of stdFilterName to stdLambda from LUT 

939 

940 Returns 

941 ------- 

942 referenceFilterNames: `list` 

943 Wavelength-ordered list of reference filter names 

944 """ 

945 

946 # Find the unique list of filter names in visitCat 

947 filterNames = np.unique(visitCat.asAstropy()['physicalFilter']) 

948 

949 # Find the unique list of "standard" filters 

950 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames} 

951 

952 # And sort these by wavelength 

953 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get) 

954 

955 return referenceFilterNames