Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for BuildStars using src tables or sourceTable_visit tables. 

22""" 

23 

24import os 

25import sys 

26import traceback 

27import abc 

28 

29import numpy as np 

30 

31import lsst.daf.persistence as dafPersist 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34import lsst.afw.table as afwTable 

35import lsst.geom as geom 

36from lsst.daf.base import PropertyList 

37from lsst.daf.base.dateTime import DateTime 

38from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

39 

40from .utilities import computeApertureRadiusFromDataRef 

41from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask 

42 

43import fgcm 

44 

45REFSTARS_FORMAT_VERSION = 1 

46 

47__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask'] 

48 

49 

50class FgcmBuildStarsConfigBase(pexConfig.Config): 

51 """Base config for FgcmBuildStars tasks""" 

52 

53 instFluxField = pexConfig.Field( 

54 doc=("Faull name of the source instFlux field to use, including 'instFlux'. " 

55 "The associated flag will be implicitly included in badFlags"), 

56 dtype=str, 

57 default='slot_CalibFlux_instFlux', 

58 ) 

59 minPerBand = pexConfig.Field( 

60 doc="Minimum observations per band", 

61 dtype=int, 

62 default=2, 

63 ) 

64 matchRadius = pexConfig.Field( 

65 doc="Match radius (arcseconds)", 

66 dtype=float, 

67 default=1.0, 

68 ) 

69 isolationRadius = pexConfig.Field( 

70 doc="Isolation radius (arcseconds)", 

71 dtype=float, 

72 default=2.0, 

73 ) 

74 densityCutNside = pexConfig.Field( 

75 doc="Density cut healpix nside", 

76 dtype=int, 

77 default=128, 

78 ) 

79 densityCutMaxPerPixel = pexConfig.Field( 

80 doc="Density cut number of stars per pixel", 

81 dtype=int, 

82 default=1000, 

83 ) 

84 matchNside = pexConfig.Field( 

85 doc="Healpix Nside for matching", 

86 dtype=int, 

87 default=4096, 

88 ) 

89 coarseNside = pexConfig.Field( 

90 doc="Healpix coarse Nside for partitioning matches", 

91 dtype=int, 

92 default=8, 

93 ) 

94 # The following config will not be necessary after Gen2 retirement. 

95 # In the meantime, obs packages should set to 'filterDefinitions.filter_to_band' 

96 # which is easiest to access in the config file. 

97 physicalFilterMap = pexConfig.DictField( 

98 doc="Mapping from 'physicalFilter' to band.", 

99 keytype=str, 

100 itemtype=str, 

101 default={}, 

102 ) 

103 requiredBands = pexConfig.ListField( 

104 doc="Bands required for each star", 

105 dtype=str, 

106 default=(), 

107 ) 

108 primaryBands = pexConfig.ListField( 

109 doc=("Bands for 'primary' star matches. " 

110 "A star must be observed in one of these bands to be considered " 

111 "as a calibration star."), 

112 dtype=str, 

113 default=None 

114 ) 

115 visitDataRefName = pexConfig.Field( 

116 doc="dataRef name for the 'visit' field, usually 'visit'.", 

117 dtype=str, 

118 default="visit" 

119 ) 

120 ccdDataRefName = pexConfig.Field( 

121 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.", 

122 dtype=str, 

123 default="ccd" 

124 ) 

125 doApplyWcsJacobian = pexConfig.Field( 

126 doc="Apply the jacobian of the WCS to the star observations prior to fit?", 

127 dtype=bool, 

128 default=True 

129 ) 

130 doModelErrorsWithBackground = pexConfig.Field( 

131 doc="Model flux errors with background term?", 

132 dtype=bool, 

133 default=True 

134 ) 

135 psfCandidateName = pexConfig.Field( 

136 doc="Name of field with psf candidate flag for propagation", 

137 dtype=str, 

138 default="calib_psf_candidate" 

139 ) 

140 doSubtractLocalBackground = pexConfig.Field( 

141 doc=("Subtract the local background before performing calibration? " 

142 "This is only supported for circular aperture calibration fluxes."), 

143 dtype=bool, 

144 default=False 

145 ) 

146 localBackgroundFluxField = pexConfig.Field( 

147 doc="Full name of the local background instFlux field to use.", 

148 dtype=str, 

149 default='base_LocalBackground_instFlux' 

150 ) 

151 sourceSelector = sourceSelectorRegistry.makeField( 

152 doc="How to select sources", 

153 default="science" 

154 ) 

155 apertureInnerInstFluxField = pexConfig.Field( 

156 doc=("Full name of instFlux field that contains inner aperture " 

157 "flux for aperture correction proxy"), 

158 dtype=str, 

159 default='base_CircularApertureFlux_12_0_instFlux' 

160 ) 

161 apertureOuterInstFluxField = pexConfig.Field( 

162 doc=("Full name of instFlux field that contains outer aperture " 

163 "flux for aperture correction proxy"), 

164 dtype=str, 

165 default='base_CircularApertureFlux_17_0_instFlux' 

166 ) 

167 doReferenceMatches = pexConfig.Field( 

168 doc="Match reference catalog as additional constraint on calibration", 

169 dtype=bool, 

170 default=True, 

171 ) 

172 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField( 

173 target=FgcmLoadReferenceCatalogTask, 

174 doc="FGCM reference object loader", 

175 ) 

176 nVisitsPerCheckpoint = pexConfig.Field( 

177 doc="Number of visits read between checkpoints", 

178 dtype=int, 

179 default=500, 

180 ) 

181 

182 def setDefaults(self): 

183 sourceSelector = self.sourceSelector["science"] 

184 sourceSelector.setDefaults() 

185 

186 sourceSelector.doFlags = True 

187 sourceSelector.doUnresolved = True 

188 sourceSelector.doSignalToNoise = True 

189 sourceSelector.doIsolated = True 

190 

191 sourceSelector.signalToNoise.minimum = 10.0 

192 sourceSelector.signalToNoise.maximum = 1000.0 

193 

194 # FGCM operates on unresolved sources, and this setting is 

195 # appropriate for the current base_ClassificationExtendedness 

196 sourceSelector.unresolved.maximum = 0.5 

197 

198 

199class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner): 

200 """Subclass of TaskRunner for FgcmBuildStars tasks 

201 

202 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of 

203 arguments, one of which is the butler (for persistence and mapper data), 

204 and a list of dataRefs extracted from the command line. Note that FGCM 

205 runs on a large set of dataRefs, and not on single dataRef/tract/patch. 

206 This class transforms the process arguments generated by the ArgumentParser 

207 into the arguments expected by FgcmBuildStarsTask.run(). This runner does 

208 not use any parallelization. 

209 """ 

210 @staticmethod 

211 def getTargetList(parsedCmd): 

212 """ 

213 Return a list with one element: a tuple with the butler and 

214 list of dataRefs 

215 """ 

216 # we want to combine the butler with any (or no!) dataRefs 

217 return [(parsedCmd.butler, parsedCmd.id.refList)] 

218 

219 def __call__(self, args): 

220 """ 

221 Parameters 

222 ---------- 

223 args: `tuple` with (butler, dataRefList) 

224 

225 Returns 

226 ------- 

227 exitStatus: `list` with `lsst.pipe.base.Struct` 

228 exitStatus (0: success; 1: failure) 

229 """ 

230 butler, dataRefList = args 

231 

232 task = self.TaskClass(config=self.config, log=self.log) 

233 

234 exitStatus = 0 

235 if self.doRaise: 

236 task.runDataRef(butler, dataRefList) 

237 else: 

238 try: 

239 task.runDataRef(butler, dataRefList) 

240 except Exception as e: 

241 exitStatus = 1 

242 task.log.fatal("Failed: %s" % e) 

243 if not isinstance(e, pipeBase.TaskError): 

244 traceback.print_exc(file=sys.stderr) 

245 

246 task.writeMetadata(butler) 

247 

248 # The task does not return any results: 

249 return [pipeBase.Struct(exitStatus=exitStatus)] 

250 

251 def run(self, parsedCmd): 

252 """ 

253 Run the task, with no multiprocessing 

254 

255 Parameters 

256 ---------- 

257 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

258 """ 

259 

260 resultList = [] 

261 

262 if self.precall(parsedCmd): 

263 targetList = self.getTargetList(parsedCmd) 

264 resultList = self(targetList[0]) 

265 

266 return resultList 

267 

268 

269class FgcmBuildStarsBaseTask(pipeBase.PipelineTask, pipeBase.CmdLineTask, abc.ABC): 

270 """ 

271 Base task to build stars for FGCM global calibration 

272 

273 Parameters 

274 ---------- 

275 butler : `lsst.daf.persistence.Butler` 

276 """ 

277 def __init__(self, initInputs=None, butler=None, **kwargs): 

278 super().__init__(**kwargs) 

279 

280 self.makeSubtask("sourceSelector") 

281 # Only log warning and fatal errors from the sourceSelector 

282 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN) 

283 

284 # no saving of metadata for now 

285 def _getMetadataName(self): 

286 return None 

287 

288 @pipeBase.timeMethod 

289 def runDataRef(self, butler, dataRefs): 

290 """ 

291 Cross-match and make star list for FGCM Input 

292 

293 Parameters 

294 ---------- 

295 butler: `lsst.daf.persistence.Butler` 

296 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

297 Source data references for the input visits. 

298 

299 Raises 

300 ------ 

301 RuntimeErrror: Raised if `config.doReferenceMatches` is set and 

302 an fgcmLookUpTable is not available, or if computeFluxApertureRadius() 

303 fails if the calibFlux is not a CircularAperture flux. 

304 """ 

305 datasetType = dataRefs[0].butlerSubset.datasetType 

306 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType) 

307 

308 if self.config.doReferenceMatches: 

309 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler) 

310 # Ensure that we have a LUT 

311 if not butler.datasetExists('fgcmLookUpTable'): 

312 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches") 

313 # Compute aperture radius if necessary. This is useful to do now before 

314 # any heavy lifting has happened (fail early). 

315 calibFluxApertureRadius = None 

316 if self.config.doSubtractLocalBackground: 

317 try: 

318 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

319 self.config.instFluxField) 

320 except RuntimeError as e: 

321 raise RuntimeError("Could not determine aperture radius from %s. " 

322 "Cannot use doSubtractLocalBackground." % 

323 (self.config.instFluxField)) from e 

324 

325 camera = butler.get('camera') 

326 groupedDataRefs = self._findAndGroupDataRefsGen2(butler, camera, dataRefs) 

327 

328 # Make the visit catalog if necessary 

329 # First check if the visit catalog is in the _current_ path 

330 # We cannot use Gen2 datasetExists() because that checks all parent 

331 # directories as well, which would make recovering from faults 

332 # and fgcmcal reruns impossible. 

333 visitCatDataRef = butler.dataRef('fgcmVisitCatalog') 

334 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0] 

335 if os.path.exists(filename): 

336 # This file exists and we should continue processing 

337 inVisitCat = visitCatDataRef.get() 

338 if len(inVisitCat) != len(groupedDataRefs): 

339 raise RuntimeError("Existing visitCatalog found, but has an inconsistent " 

340 "number of visits. Cannot continue.") 

341 else: 

342 inVisitCat = None 

343 

344 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs, 

345 visitCatDataRef=visitCatDataRef, 

346 inVisitCat=inVisitCat) 

347 

348 # Persist the visitCat as a checkpoint file. 

349 visitCatDataRef.put(visitCat) 

350 

351 starObsDataRef = butler.dataRef('fgcmStarObservations') 

352 filename = starObsDataRef.get('fgcmStarObservations_filename')[0] 

353 if os.path.exists(filename): 

354 inStarObsCat = starObsDataRef.get() 

355 else: 

356 inStarObsCat = None 

357 

358 rad = calibFluxApertureRadius 

359 sourceSchemaDataRef = butler.dataRef('src_schema') 

360 sourceSchema = sourceSchemaDataRef.get('src_schema', immediate=True).schema 

361 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs, 

362 visitCat, 

363 sourceSchema, 

364 camera, 

365 calibFluxApertureRadius=rad, 

366 starObsDataRef=starObsDataRef, 

367 visitCatDataRef=visitCatDataRef, 

368 inStarObsCat=inStarObsCat) 

369 visitCatDataRef.put(visitCat) 

370 starObsDataRef.put(fgcmStarObservationCat) 

371 

372 # Always do the matching. 

373 if self.config.doReferenceMatches: 

374 lutDataRef = butler.dataRef('fgcmLookUpTable') 

375 else: 

376 lutDataRef = None 

377 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

378 fgcmStarObservationCat, 

379 lutDataRef=lutDataRef) 

380 

381 # Persist catalogs via the butler 

382 butler.put(fgcmStarIdCat, 'fgcmStarIds') 

383 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') 

384 if fgcmRefCat is not None: 

385 butler.put(fgcmRefCat, 'fgcmReferenceStars') 

386 

387 @abc.abstractmethod 

388 def _findAndGroupDataRefsGen2(self, butler, camera, dataRefs): 

389 """ 

390 Find and group dataRefs (by visit); Gen2 only. 

391 

392 Parameters 

393 ---------- 

394 butler : `lsst.daf.persistence.Butler` 

395 Gen2 butler. 

396 camera : `lsst.afw.cameraGeom.Camera` 

397 Camera from the butler. 

398 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

399 Data references for the input visits. 

400 

401 Returns 

402 ------- 

403 groupedDataRefs : `dict` [`int`, `list`] 

404 Dictionary with sorted visit keys, and `list`s of 

405 `lsst.daf.persistence.ButlerDataRef` 

406 """ 

407 raise NotImplementedError("_findAndGroupDataRefsGen2 not implemented.") 

408 

409 @abc.abstractmethod 

410 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

411 sourceSchema, 

412 camera, 

413 calibFluxApertureRadius=None, 

414 visitCatDataRef=None, 

415 starObsDataRef=None, 

416 inStarObsCat=None): 

417 """ 

418 Compile all good star observations from visits in visitCat. Checkpoint files 

419 will be stored if both visitCatDataRef and starObsDataRef are not None. 

420 

421 Parameters 

422 ---------- 

423 groupedDataRefs : `dict` of `list`s 

424 Lists of `~lsst.daf.persistence.ButlerDataRef` or 

425 `~lsst.daf.butler.DeferredDatasetHandle`, grouped by visit. 

426 visitCat : `~afw.table.BaseCatalog` 

427 Catalog with visit data for FGCM 

428 sourceSchema : `~lsst.afw.table.Schema` 

429 Schema for the input src catalogs. 

430 camera : `~lsst.afw.cameraGeom.Camera` 

431 calibFluxApertureRadius : `float`, optional 

432 Aperture radius for calibration flux. 

433 visitCatDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional 

434 Dataref to write visitCat for checkpoints 

435 starObsDataRef : `~lsst.daf.persistence.ButlerDataRef`, optional 

436 Dataref to write the star observation catalog for checkpoints. 

437 inStarObsCat : `~afw.table.BaseCatalog` 

438 Input observation catalog. If this is incomplete, observations 

439 will be appended from when it was cut off. 

440 

441 Returns 

442 ------- 

443 fgcmStarObservations : `afw.table.BaseCatalog` 

444 Full catalog of good observations. 

445 

446 Raises 

447 ------ 

448 RuntimeError: Raised if doSubtractLocalBackground is True and 

449 calibFluxApertureRadius is not set. 

450 """ 

451 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.") 

452 

453 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, bkgDataRefDict=None, 

454 visitCatDataRef=None, inVisitCat=None): 

455 """ 

456 Make a visit catalog with all the keys from each visit 

457 

458 Parameters 

459 ---------- 

460 camera: `lsst.afw.cameraGeom.Camera` 

461 Camera from the butler 

462 groupedDataRefs: `dict` 

463 Dictionary with visit keys, and `list`s of 

464 `lsst.daf.persistence.ButlerDataRef` 

465 bkgDataRefDict: `dict`, optional 

466 Dictionary of gen3 dataRefHandles for background info. 

467 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

468 Dataref to write visitCat for checkpoints 

469 inVisitCat: `afw.table.BaseCatalog`, optional 

470 Input (possibly incomplete) visit catalog 

471 

472 Returns 

473 ------- 

474 visitCat: `afw.table.BaseCatalog` 

475 """ 

476 

477 self.log.info("Assembling visitCatalog from %d %ss" % 

478 (len(groupedDataRefs), self.config.visitDataRefName)) 

479 

480 nCcd = len(camera) 

481 

482 if inVisitCat is None: 

483 schema = self._makeFgcmVisitSchema(nCcd) 

484 

485 visitCat = afwTable.BaseCatalog(schema) 

486 visitCat.reserve(len(groupedDataRefs)) 

487 visitCat.resize(len(groupedDataRefs)) 

488 

489 visitCat['visit'] = list(groupedDataRefs.keys()) 

490 visitCat['used'] = 0 

491 visitCat['sources_read'] = False 

492 else: 

493 visitCat = inVisitCat 

494 

495 # No matter what, fill the catalog. This will check if it was 

496 # already read. 

497 self._fillVisitCatalog(visitCat, groupedDataRefs, 

498 bkgDataRefDict=bkgDataRefDict, 

499 visitCatDataRef=visitCatDataRef) 

500 

501 return visitCat 

502 

503 def _fillVisitCatalog(self, visitCat, groupedDataRefs, bkgDataRefDict=None, 

504 visitCatDataRef=None): 

505 """ 

506 Fill the visit catalog with visit metadata 

507 

508 Parameters 

509 ---------- 

510 visitCat : `afw.table.BaseCatalog` 

511 Visit catalog. See _makeFgcmVisitSchema() for schema definition. 

512 groupedDataRefs : `dict` 

513 Dictionary with visit keys, and `list`s of 

514 `lsst.daf.persistence.ButlerDataRef` or 

515 `lsst.daf.butler.DeferredDatasetHandle` 

516 visitCatDataRef : `lsst.daf.persistence.ButlerDataRef`, optional 

517 Dataref to write ``visitCat`` for checkpoints. Gen2 only. 

518 bkgDataRefDict : `dict`, optional 

519 Dictionary of Gen3 `lsst.daf.butler.DeferredDatasetHandle` 

520 for background info. 

521 """ 

522 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1)) 

523 

524 for i, visit in enumerate(groupedDataRefs): 

525 # We don't use the bypasses since we need the psf info which does 

526 # not have a bypass 

527 # TODO: When DM-15500 is implemented in the Gen3 Butler, this 

528 # can be fixed 

529 

530 # Do not read those that have already been read 

531 if visitCat['used'][i]: 

532 continue 

533 

534 if (i % self.config.nVisitsPerCheckpoint) == 0: 

535 self.log.info("Retrieving metadata for %s %d (%d/%d)" % 

536 (self.config.visitDataRefName, visit, i, len(groupedDataRefs))) 

537 # Save checkpoint if desired 

538 if visitCatDataRef is not None: 

539 visitCatDataRef.put(visitCat) 

540 

541 dataRef = groupedDataRefs[visit][0] 

542 if isinstance(dataRef, dafPersist.ButlerDataRef): 

543 # Gen2: calexp dataRef 

544 # The first dataRef in the group will be the reference ccd (if available) 

545 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox) 

546 visitInfo = exp.getInfo().getVisitInfo() 

547 label = dataRef.get(datasetType='calexp_filterLabel') 

548 physicalFilter = label.physicalLabel 

549 psf = exp.getPsf() 

550 psfSigma = psf.computeShape().getDeterminantRadius() 

551 else: 

552 # Gen3: use the visitSummary dataRef 

553 summary = dataRef.get() 

554 

555 summaryRow = summary.find(self.config.referenceCCD) 

556 if summaryRow is None: 

557 # Take the first available ccd if reference isn't available 

558 summaryRow = summary[0] 

559 

560 summaryDetector = summaryRow['id'] 

561 visitInfo = summaryRow.getVisitInfo() 

562 physicalFilter = summaryRow['physical_filter'] 

563 # Compute the median psf sigma if possible 

564 goodSigma, = np.where(summary['psfSigma'] > 0) 

565 if goodSigma.size > 2: 

566 psfSigma = np.median(summary['psfSigma'][goodSigma]) 

567 elif goodSigma > 0: 

568 psfSigma = np.mean(summary['psfSigma'][goodSigma]) 

569 else: 

570 psfSigma = 0.0 

571 

572 rec = visitCat[i] 

573 rec['visit'] = visit 

574 rec['physicalFilter'] = physicalFilter 

575 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it 

576 # much easier to get the wcs's necessary to recompute the pointing 

577 # ra/dec at the center of the camera. 

578 radec = visitInfo.getBoresightRaDec() 

579 rec['telra'] = radec.getRa().asDegrees() 

580 rec['teldec'] = radec.getDec().asDegrees() 

581 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees() 

582 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees() 

583 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD) 

584 rec['exptime'] = visitInfo.getExposureTime() 

585 # convert from Pa to millibar 

586 # Note that I don't know if this unit will need to be per-camera config 

587 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100 

588 # Flag to signify if this is a "deep" field. Not currently used 

589 rec['deepFlag'] = 0 

590 # Relative flat scaling (1.0 means no relative scaling) 

591 rec['scaling'][:] = 1.0 

592 # Median delta aperture, to be measured from stars 

593 rec['deltaAper'] = 0.0 

594 rec['psfSigma'] = psfSigma 

595 

596 if self.config.doModelErrorsWithBackground: 

597 foundBkg = False 

598 if isinstance(dataRef, dafPersist.ButlerDataRef): 

599 # Gen2-style dataRef 

600 det = dataRef.dataId[self.config.ccdDataRefName] 

601 if dataRef.datasetExists(datasetType='calexpBackground'): 

602 bgList = dataRef.get(datasetType='calexpBackground') 

603 foundBkg = True 

604 else: 

605 # Gen3-style dataRef 

606 try: 

607 # Use the same detector used from the summary. 

608 bkgRef = bkgDataRefDict[(visit, summaryDetector)] 

609 bgList = bkgRef.get() 

610 foundBkg = True 

611 except KeyError: 

612 pass 

613 

614 if foundBkg: 

615 bgStats = (bg[0].getStatsImage().getImage().array 

616 for bg in bgList) 

617 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats) 

618 else: 

619 self.log.warn('Sky background not found for visit %d / ccd %d' % 

620 (visit, det)) 

621 rec['skyBackground'] = -1.0 

622 else: 

623 rec['skyBackground'] = -1.0 

624 

625 rec['used'] = 1 

626 

627 def _makeSourceMapper(self, sourceSchema): 

628 """ 

629 Make a schema mapper for fgcm sources 

630 

631 Parameters 

632 ---------- 

633 sourceSchema: `afwTable.Schema` 

634 Default source schema from the butler 

635 

636 Returns 

637 ------- 

638 sourceMapper: `afwTable.schemaMapper` 

639 Mapper to the FGCM source schema 

640 """ 

641 

642 # create a mapper to the preferred output 

643 sourceMapper = afwTable.SchemaMapper(sourceSchema) 

644 

645 # map to ra/dec 

646 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

647 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

648 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x') 

649 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y') 

650 # Add the mapping if the field exists in the input catalog. 

651 # If the field does not exist, simply add it (set to False). 

652 # This field is not required for calibration, but is useful 

653 # to collate if available. 

654 try: 

655 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(), 

656 'psf_candidate') 

657 except LookupError: 

658 sourceMapper.editOutputSchema().addField( 

659 "psf_candidate", type='Flag', 

660 doc=("Flag set if the source was a candidate for PSF determination, " 

661 "as determined by the star selector.")) 

662 

663 # and add the fields we want 

664 sourceMapper.editOutputSchema().addField( 

665 "visit", type=np.int32, doc="Visit number") 

666 sourceMapper.editOutputSchema().addField( 

667 "ccd", type=np.int32, doc="CCD number") 

668 sourceMapper.editOutputSchema().addField( 

669 "instMag", type=np.float32, doc="Instrumental magnitude") 

670 sourceMapper.editOutputSchema().addField( 

671 "instMagErr", type=np.float32, doc="Instrumental magnitude error") 

672 sourceMapper.editOutputSchema().addField( 

673 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian") 

674 sourceMapper.editOutputSchema().addField( 

675 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset") 

676 

677 return sourceMapper 

678 

679 def fgcmMatchStars(self, visitCat, obsCat, lutDataRef=None): 

680 """ 

681 Use FGCM code to match observations into unique stars. 

682 

683 Parameters 

684 ---------- 

685 visitCat: `afw.table.BaseCatalog` 

686 Catalog with visit data for fgcm 

687 obsCat: `afw.table.BaseCatalog` 

688 Full catalog of star observations for fgcm 

689 lutDataRef: `lsst.daf.persistence.ButlerDataRef` or 

690 `lsst.daf.butler.DeferredDatasetHandle`, optional 

691 Data reference to fgcm look-up table (used if matching reference stars). 

692 

693 Returns 

694 ------- 

695 fgcmStarIdCat: `afw.table.BaseCatalog` 

696 Catalog of unique star identifiers and index keys 

697 fgcmStarIndicesCat: `afwTable.BaseCatalog` 

698 Catalog of unique star indices 

699 fgcmRefCat: `afw.table.BaseCatalog` 

700 Catalog of matched reference stars. 

701 Will be None if `config.doReferenceMatches` is False. 

702 """ 

703 # get filter names into a numpy array... 

704 # This is the type that is expected by the fgcm code 

705 visitFilterNames = np.zeros(len(visitCat), dtype='a30') 

706 for i in range(len(visitCat)): 

707 visitFilterNames[i] = visitCat[i]['physicalFilter'] 

708 

709 # match to put filterNames with observations 

710 visitIndex = np.searchsorted(visitCat['visit'], 

711 obsCat['visit']) 

712 

713 obsFilterNames = visitFilterNames[visitIndex] 

714 

715 if self.config.doReferenceMatches: 

716 # Get the reference filter names, using the LUT 

717 lutCat = lutDataRef.get() 

718 

719 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in 

720 zip(lutCat[0]['physicalFilters'].split(','), 

721 lutCat[0]['stdPhysicalFilters'].split(','))} 

722 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in 

723 zip(lutCat[0]['stdPhysicalFilters'].split(','), 

724 lutCat[0]['lambdaStdFilter'])} 

725 

726 del lutCat 

727 

728 referenceFilterNames = self._getReferenceFilterNames(visitCat, 

729 stdFilterDict, 

730 stdLambdaDict) 

731 self.log.info("Using the following reference filters: %s" % 

732 (', '.join(referenceFilterNames))) 

733 

734 else: 

735 # This should be an empty list 

736 referenceFilterNames = [] 

737 

738 # make the fgcm starConfig dict 

739 starConfig = {'logger': self.log, 

740 'filterToBand': self.config.physicalFilterMap, 

741 'requiredBands': self.config.requiredBands, 

742 'minPerBand': self.config.minPerBand, 

743 'matchRadius': self.config.matchRadius, 

744 'isolationRadius': self.config.isolationRadius, 

745 'matchNSide': self.config.matchNside, 

746 'coarseNSide': self.config.coarseNside, 

747 'densNSide': self.config.densityCutNside, 

748 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 

749 'primaryBands': self.config.primaryBands, 

750 'referenceFilterNames': referenceFilterNames} 

751 

752 # initialize the FgcmMakeStars object 

753 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) 

754 

755 # make the primary stars 

756 # note that the ra/dec native Angle format is radians 

757 # We determine the conversion from the native units (typically 

758 # radians) to degrees for the first observation. This allows us 

759 # to treate ra/dec as numpy arrays rather than Angles, which would 

760 # be approximately 600x slower. 

761 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) 

762 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, 

763 obsCat['dec'] * conv, 

764 filterNameArray=obsFilterNames, 

765 bandSelected=False) 

766 

767 # and match all the stars 

768 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, 

769 obsCat['dec'] * conv, 

770 obsFilterNames) 

771 

772 if self.config.doReferenceMatches: 

773 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) 

774 

775 # now persist 

776 

777 objSchema = self._makeFgcmObjSchema() 

778 

779 # make catalog and records 

780 fgcmStarIdCat = afwTable.BaseCatalog(objSchema) 

781 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) 

782 for i in range(fgcmMakeStars.objIndexCat.size): 

783 fgcmStarIdCat.addNew() 

784 

785 # fill the catalog 

786 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] 

787 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] 

788 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] 

789 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex'] 

790 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] 

791 

792 obsSchema = self._makeFgcmObsSchema() 

793 

794 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) 

795 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) 

796 for i in range(fgcmMakeStars.obsIndexCat.size): 

797 fgcmStarIndicesCat.addNew() 

798 

799 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex'] 

800 

801 if self.config.doReferenceMatches: 

802 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) 

803 

804 fgcmRefCat = afwTable.BaseCatalog(refSchema) 

805 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) 

806 

807 for i in range(fgcmMakeStars.referenceCat.size): 

808 fgcmRefCat.addNew() 

809 

810 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] 

811 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] 

812 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr'] 

813 

814 md = PropertyList() 

815 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION) 

816 md.set("FILTERNAMES", referenceFilterNames) 

817 fgcmRefCat.setMetadata(md) 

818 

819 else: 

820 fgcmRefCat = None 

821 

822 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat 

823 

824 def _makeFgcmVisitSchema(self, nCcd): 

825 """ 

826 Make a schema for an fgcmVisitCatalog 

827 

828 Parameters 

829 ---------- 

830 nCcd: `int` 

831 Number of CCDs in the camera 

832 

833 Returns 

834 ------- 

835 schema: `afwTable.Schema` 

836 """ 

837 

838 schema = afwTable.Schema() 

839 schema.addField('visit', type=np.int32, doc="Visit number") 

840 schema.addField('physicalFilter', type=str, size=30, doc="Physical filter") 

841 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") 

842 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") 

843 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") 

844 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)") 

845 schema.addField('mjd', type=np.float64, doc="MJD of visit") 

846 schema.addField('exptime', type=np.float32, doc="Exposure time") 

847 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") 

848 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") 

849 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") 

850 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") 

851 # the following field is not used yet 

852 schema.addField('deepFlag', type=np.int32, doc="Deep observation") 

853 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", 

854 size=nCcd) 

855 schema.addField('used', type=np.int32, doc="This visit has been ingested.") 

856 schema.addField('sources_read', type='Flag', doc="This visit had sources read.") 

857 

858 return schema 

859 

860 def _makeFgcmObjSchema(self): 

861 """ 

862 Make a schema for the objIndexCat from fgcmMakeStars 

863 

864 Returns 

865 ------- 

866 schema: `afwTable.Schema` 

867 """ 

868 

869 objSchema = afwTable.Schema() 

870 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

871 # Will investigate making these angles... 

872 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)') 

873 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)') 

874 objSchema.addField('obsArrIndex', type=np.int32, 

875 doc='Index in obsIndexTable for first observation') 

876 objSchema.addField('nObs', type=np.int32, doc='Total number of observations') 

877 

878 return objSchema 

879 

880 def _makeFgcmObsSchema(self): 

881 """ 

882 Make a schema for the obsIndexCat from fgcmMakeStars 

883 

884 Returns 

885 ------- 

886 schema: `afwTable.Schema` 

887 """ 

888 

889 obsSchema = afwTable.Schema() 

890 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table') 

891 

892 return obsSchema 

893 

894 def _makeFgcmRefSchema(self, nReferenceBands): 

895 """ 

896 Make a schema for the referenceCat from fgcmMakeStars 

897 

898 Parameters 

899 ---------- 

900 nReferenceBands: `int` 

901 Number of reference bands 

902 

903 Returns 

904 ------- 

905 schema: `afwTable.Schema` 

906 """ 

907 

908 refSchema = afwTable.Schema() 

909 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

910 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)', 

911 size=nReferenceBands) 

912 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array', 

913 size=nReferenceBands) 

914 

915 return refSchema 

916 

917 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict): 

918 """ 

919 Get the reference filter names, in wavelength order, from the visitCat and 

920 information from the look-up-table. 

921 

922 Parameters 

923 ---------- 

924 visitCat: `afw.table.BaseCatalog` 

925 Catalog with visit data for FGCM 

926 stdFilterDict: `dict` 

927 Mapping of filterName to stdFilterName from LUT 

928 stdLambdaDict: `dict` 

929 Mapping of stdFilterName to stdLambda from LUT 

930 

931 Returns 

932 ------- 

933 referenceFilterNames: `list` 

934 Wavelength-ordered list of reference filter names 

935 """ 

936 

937 # Find the unique list of filter names in visitCat 

938 filterNames = np.unique(visitCat.asAstropy()['physicalFilter']) 

939 

940 # Find the unique list of "standard" filters 

941 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames} 

942 

943 # And sort these by wavelength 

944 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get) 

945 

946 return referenceFilterNames