Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of fgcmcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21"""Base class for BuildStars using src tables or sourceTable_visit tables. 

22""" 

23 

24import os 

25import sys 

26import traceback 

27import abc 

28 

29import numpy as np 

30 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33import lsst.afw.table as afwTable 

34import lsst.geom as geom 

35from lsst.daf.base import PropertyList 

36from lsst.daf.base.dateTime import DateTime 

37from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

38 

39from .utilities import computeApertureRadiusFromDataRef 

40from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask 

41 

42import fgcm 

43 

44REFSTARS_FORMAT_VERSION = 1 

45 

46__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask'] 

47 

48 

49class FgcmBuildStarsConfigBase(pexConfig.Config): 

50 """Base config for FgcmBuildStars tasks""" 

51 

52 instFluxField = pexConfig.Field( 

53 doc=("Faull name of the source instFlux field to use, including 'instFlux'. " 

54 "The associated flag will be implicitly included in badFlags"), 

55 dtype=str, 

56 default='slot_CalibFlux_instFlux', 

57 ) 

58 minPerBand = pexConfig.Field( 

59 doc="Minimum observations per band", 

60 dtype=int, 

61 default=2, 

62 ) 

63 matchRadius = pexConfig.Field( 

64 doc="Match radius (arcseconds)", 

65 dtype=float, 

66 default=1.0, 

67 ) 

68 isolationRadius = pexConfig.Field( 

69 doc="Isolation radius (arcseconds)", 

70 dtype=float, 

71 default=2.0, 

72 ) 

73 densityCutNside = pexConfig.Field( 

74 doc="Density cut healpix nside", 

75 dtype=int, 

76 default=128, 

77 ) 

78 densityCutMaxPerPixel = pexConfig.Field( 

79 doc="Density cut number of stars per pixel", 

80 dtype=int, 

81 default=1000, 

82 ) 

83 matchNside = pexConfig.Field( 

84 doc="Healpix Nside for matching", 

85 dtype=int, 

86 default=4096, 

87 ) 

88 coarseNside = pexConfig.Field( 

89 doc="Healpix coarse Nside for partitioning matches", 

90 dtype=int, 

91 default=8, 

92 ) 

93 filterMap = pexConfig.DictField( 

94 doc="Mapping from 'filterName' to band.", 

95 keytype=str, 

96 itemtype=str, 

97 default={}, 

98 ) 

99 requiredBands = pexConfig.ListField( 

100 doc="Bands required for each star", 

101 dtype=str, 

102 default=(), 

103 ) 

104 primaryBands = pexConfig.ListField( 

105 doc=("Bands for 'primary' star matches. " 

106 "A star must be observed in one of these bands to be considered " 

107 "as a calibration star."), 

108 dtype=str, 

109 default=None 

110 ) 

111 visitDataRefName = pexConfig.Field( 

112 doc="dataRef name for the 'visit' field, usually 'visit'.", 

113 dtype=str, 

114 default="visit" 

115 ) 

116 ccdDataRefName = pexConfig.Field( 

117 doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.", 

118 dtype=str, 

119 default="ccd" 

120 ) 

121 doApplyWcsJacobian = pexConfig.Field( 

122 doc="Apply the jacobian of the WCS to the star observations prior to fit?", 

123 dtype=bool, 

124 default=True 

125 ) 

126 psfCandidateName = pexConfig.Field( 

127 doc="Name of field with psf candidate flag for propagation", 

128 dtype=str, 

129 default="calib_psf_candidate" 

130 ) 

131 doSubtractLocalBackground = pexConfig.Field( 

132 doc=("Subtract the local background before performing calibration? " 

133 "This is only supported for circular aperture calibration fluxes."), 

134 dtype=bool, 

135 default=False 

136 ) 

137 localBackgroundFluxField = pexConfig.Field( 

138 doc="Full name of the local background instFlux field to use.", 

139 dtype=str, 

140 default='base_LocalBackground_instFlux' 

141 ) 

142 sourceSelector = sourceSelectorRegistry.makeField( 

143 doc="How to select sources", 

144 default="science" 

145 ) 

146 apertureInnerInstFluxField = pexConfig.Field( 

147 doc=("Full name of instFlux field that contains inner aperture " 

148 "flux for aperture correction proxy"), 

149 dtype=str, 

150 default='base_CircularApertureFlux_12_0_instFlux' 

151 ) 

152 apertureOuterInstFluxField = pexConfig.Field( 

153 doc=("Full name of instFlux field that contains outer aperture " 

154 "flux for aperture correction proxy"), 

155 dtype=str, 

156 default='base_CircularApertureFlux_17_0_instFlux' 

157 ) 

158 doReferenceMatches = pexConfig.Field( 

159 doc="Match reference catalog as additional constraint on calibration", 

160 dtype=bool, 

161 default=True, 

162 ) 

163 fgcmLoadReferenceCatalog = pexConfig.ConfigurableField( 

164 target=FgcmLoadReferenceCatalogTask, 

165 doc="FGCM reference object loader", 

166 ) 

167 nVisitsPerCheckpoint = pexConfig.Field( 

168 doc="Number of visits read between checkpoints", 

169 dtype=int, 

170 default=500, 

171 ) 

172 

173 def setDefaults(self): 

174 sourceSelector = self.sourceSelector["science"] 

175 sourceSelector.setDefaults() 

176 

177 sourceSelector.doFlags = True 

178 sourceSelector.doUnresolved = True 

179 sourceSelector.doSignalToNoise = True 

180 sourceSelector.doIsolated = True 

181 

182 sourceSelector.signalToNoise.minimum = 10.0 

183 sourceSelector.signalToNoise.maximum = 1000.0 

184 

185 # FGCM operates on unresolved sources, and this setting is 

186 # appropriate for the current base_ClassificationExtendedness 

187 sourceSelector.unresolved.maximum = 0.5 

188 

189 

190class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner): 

191 """Subclass of TaskRunner for FgcmBuildStars tasks 

192 

193 fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of 

194 arguments, one of which is the butler (for persistence and mapper data), 

195 and a list of dataRefs extracted from the command line. Note that FGCM 

196 runs on a large set of dataRefs, and not on single dataRef/tract/patch. 

197 This class transforms the process arguments generated by the ArgumentParser 

198 into the arguments expected by FgcmBuildStarsTask.run(). This runner does 

199 not use any parallelization. 

200 """ 

201 @staticmethod 

202 def getTargetList(parsedCmd): 

203 """ 

204 Return a list with one element: a tuple with the butler and 

205 list of dataRefs 

206 """ 

207 # we want to combine the butler with any (or no!) dataRefs 

208 return [(parsedCmd.butler, parsedCmd.id.refList)] 

209 

210 def __call__(self, args): 

211 """ 

212 Parameters 

213 ---------- 

214 args: `tuple` with (butler, dataRefList) 

215 

216 Returns 

217 ------- 

218 exitStatus: `list` with `lsst.pipe.base.Struct` 

219 exitStatus (0: success; 1: failure) 

220 """ 

221 butler, dataRefList = args 

222 

223 task = self.TaskClass(config=self.config, log=self.log) 

224 

225 exitStatus = 0 

226 if self.doRaise: 

227 task.runDataRef(butler, dataRefList) 

228 else: 

229 try: 

230 task.runDataRef(butler, dataRefList) 

231 except Exception as e: 

232 exitStatus = 1 

233 task.log.fatal("Failed: %s" % e) 

234 if not isinstance(e, pipeBase.TaskError): 

235 traceback.print_exc(file=sys.stderr) 

236 

237 task.writeMetadata(butler) 

238 

239 # The task does not return any results: 

240 return [pipeBase.Struct(exitStatus=exitStatus)] 

241 

242 def run(self, parsedCmd): 

243 """ 

244 Run the task, with no multiprocessing 

245 

246 Parameters 

247 ---------- 

248 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

249 """ 

250 

251 resultList = [] 

252 

253 if self.precall(parsedCmd): 

254 targetList = self.getTargetList(parsedCmd) 

255 resultList = self(targetList[0]) 

256 

257 return resultList 

258 

259 

260class FgcmBuildStarsBaseTask(pipeBase.CmdLineTask, abc.ABC): 

261 """ 

262 Base task to build stars for FGCM global calibration 

263 

264 Parameters 

265 ---------- 

266 butler : `lsst.daf.persistence.Butler` 

267 """ 

268 def __init__(self, butler=None, **kwargs): 

269 pipeBase.CmdLineTask.__init__(self, **kwargs) 

270 self.makeSubtask("sourceSelector") 

271 # Only log warning and fatal errors from the sourceSelector 

272 self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN) 

273 

274 # no saving of metadata for now 

275 def _getMetadataName(self): 

276 return None 

277 

278 @pipeBase.timeMethod 

279 def runDataRef(self, butler, dataRefs): 

280 """ 

281 Cross-match and make star list for FGCM Input 

282 

283 Parameters 

284 ---------- 

285 butler: `lsst.daf.persistence.Butler` 

286 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

287 Source data references for the input visits. 

288 

289 Raises 

290 ------ 

291 RuntimeErrror: Raised if `config.doReferenceMatches` is set and 

292 an fgcmLookUpTable is not available, or if computeFluxApertureRadius() 

293 fails if the calibFlux is not a CircularAperture flux. 

294 """ 

295 datasetType = dataRefs[0].butlerSubset.datasetType 

296 self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType) 

297 

298 if self.config.doReferenceMatches: 

299 # Ensure that we have a LUT 

300 if not butler.datasetExists('fgcmLookUpTable'): 

301 raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches") 

302 # Compute aperture radius if necessary. This is useful to do now before 

303 # any heavy lifting has happened (fail early). 

304 calibFluxApertureRadius = None 

305 if self.config.doSubtractLocalBackground: 

306 try: 

307 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

308 self.config.instFluxField) 

309 except RuntimeError as e: 

310 raise RuntimeError("Could not determine aperture radius from %s. " 

311 "Cannot use doSubtractLocalBackground." % 

312 (self.config.instFluxField)) from e 

313 

314 groupedDataRefs = self.findAndGroupDataRefs(butler, dataRefs) 

315 

316 camera = butler.get('camera') 

317 

318 # Make the visit catalog if necessary 

319 # First check if the visit catalog is in the _current_ path 

320 # We cannot use Gen2 datasetExists() because that checks all parent 

321 # directories as well, which would make recovering from faults 

322 # and fgcmcal reruns impossible. 

323 visitCatDataRef = butler.dataRef('fgcmVisitCatalog') 

324 filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0] 

325 if os.path.exists(filename): 

326 # This file exists and we should continue processing 

327 inVisitCat = visitCatDataRef.get() 

328 if len(inVisitCat) != len(groupedDataRefs): 

329 raise RuntimeError("Existing visitCatalog found, but has an inconsistent " 

330 "number of visits. Cannot continue.") 

331 else: 

332 inVisitCat = None 

333 

334 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs, 

335 visitCatDataRef=visitCatDataRef, 

336 inVisitCat=inVisitCat) 

337 

338 # Persist the visitCat as a checkpoint file. 

339 visitCatDataRef.put(visitCat) 

340 

341 starObsDataRef = butler.dataRef('fgcmStarObservations') 

342 filename = starObsDataRef.get('fgcmStarObservations_filename')[0] 

343 if os.path.exists(filename): 

344 inStarObsCat = starObsDataRef.get() 

345 else: 

346 inStarObsCat = None 

347 

348 rad = calibFluxApertureRadius 

349 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs, 

350 visitCat, 

351 calibFluxApertureRadius=rad, 

352 starObsDataRef=starObsDataRef, 

353 visitCatDataRef=visitCatDataRef, 

354 inStarObsCat=inStarObsCat) 

355 visitCatDataRef.put(visitCat) 

356 starObsDataRef.put(fgcmStarObservationCat) 

357 

358 # Always do the matching. 

359 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(butler, 

360 visitCat, 

361 fgcmStarObservationCat) 

362 

363 # Persist catalogs via the butler 

364 butler.put(fgcmStarIdCat, 'fgcmStarIds') 

365 butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') 

366 if fgcmRefCat is not None: 

367 butler.put(fgcmRefCat, 'fgcmReferenceStars') 

368 

369 @abc.abstractmethod 

370 def findAndGroupDataRefs(self, butler, dataRefs): 

371 """ 

372 Find and group dataRefs (by visit). 

373 

374 Parameters 

375 ---------- 

376 butler: `lsst.daf.persistence.Butler` 

377 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 

378 Data references for the input visits. 

379 

380 Returns 

381 ------- 

382 groupedDataRefs: `dict` [`int`, `list`] 

383 Dictionary with visit keys, and `list`s of `lsst.daf.persistence.ButlerDataRef` 

384 """ 

385 raise NotImplementedError("findAndGroupDataRefs not implemented.") 

386 

387 @abc.abstractmethod 

388 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

389 calibFluxApertureRadius=None, 

390 visitCatDataRef=None, 

391 starObsDataRef=None, 

392 inStarObsCat=None): 

393 """ 

394 Compile all good star observations from visits in visitCat. Checkpoint files 

395 will be stored if both visitCatDataRef and starObsDataRef are not None. 

396 

397 Parameters 

398 ---------- 

399 groupedDataRefs: `dict` of `list`s 

400 Lists of `lsst.daf.persistence.ButlerDataRef`, grouped by visit. 

401 visitCat: `afw.table.BaseCatalog` 

402 Catalog with visit data for FGCM 

403 calibFluxApertureRadius: `float`, optional 

404 Aperture radius for calibration flux. 

405 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

406 Dataref to write visitCat for checkpoints 

407 starObsDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

408 Dataref to write the star observation catalog for checkpoints. 

409 inStarObsCat: `afw.table.BaseCatalog` 

410 Input observation catalog. If this is incomplete, observations 

411 will be appended from when it was cut off. 

412 

413 Returns 

414 ------- 

415 fgcmStarObservations: `afw.table.BaseCatalog` 

416 Full catalog of good observations. 

417 

418 Raises 

419 ------ 

420 RuntimeError: Raised if doSubtractLocalBackground is True and 

421 calibFluxApertureRadius is not set. 

422 """ 

423 raise NotImplementedError("fgcmMakeAllStarObservations not implemented.") 

424 

425 def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, 

426 visitCatDataRef=None, inVisitCat=None): 

427 """ 

428 Make a visit catalog with all the keys from each visit 

429 

430 Parameters 

431 ---------- 

432 camera: `lsst.afw.cameraGeom.Camera` 

433 Camera from the butler 

434 groupedDataRefs: `dict` 

435 Dictionary with visit keys, and `list`s of 

436 `lsst.daf.persistence.ButlerDataRef` 

437 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

438 Dataref to write visitCat for checkpoints 

439 inVisitCat: `afw.table.BaseCatalog` 

440 Input (possibly incomplete) visit catalog 

441 

442 Returns 

443 ------- 

444 visitCat: `afw.table.BaseCatalog` 

445 """ 

446 

447 self.log.info("Assembling visitCatalog from %d %ss" % 

448 (len(groupedDataRefs), self.config.visitDataRefName)) 

449 

450 nCcd = len(camera) 

451 

452 if inVisitCat is None: 

453 schema = self._makeFgcmVisitSchema(nCcd) 

454 

455 visitCat = afwTable.BaseCatalog(schema) 

456 visitCat.reserve(len(groupedDataRefs)) 

457 

458 for i, visit in enumerate(sorted(groupedDataRefs)): 

459 rec = visitCat.addNew() 

460 rec['visit'] = visit 

461 rec['used'] = 0 

462 rec['sources_read'] = 0 

463 else: 

464 visitCat = inVisitCat 

465 

466 # No matter what, fill the catalog. This will check if it was 

467 # already read. 

468 self._fillVisitCatalog(visitCat, groupedDataRefs, 

469 visitCatDataRef=visitCatDataRef) 

470 

471 return visitCat 

472 

473 def _fillVisitCatalog(self, visitCat, groupedDataRefs, 

474 visitCatDataRef=None): 

475 """ 

476 Fill the visit catalog with visit metadata 

477 

478 Parameters 

479 ---------- 

480 visitCat: `afw.table.BaseCatalog` 

481 Catalog with schema from _makeFgcmVisitSchema() 

482 groupedDataRefs: `dict` 

483 Dictionary with visit keys, and `list`s of 

484 `lsst.daf.persistence.ButlerDataRef 

485 visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional 

486 Dataref to write visitCat for checkpoints 

487 """ 

488 

489 bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1)) 

490 

491 for i, visit in enumerate(sorted(groupedDataRefs)): 

492 # We don't use the bypasses since we need the psf info which does 

493 # not have a bypass 

494 # TODO: When DM-15500 is implemented in the Gen3 Butler, this 

495 # can be fixed 

496 

497 # Do not read those that have already been read 

498 if visitCat['used'][i]: 

499 continue 

500 

501 if (i % self.config.nVisitsPerCheckpoint) == 0: 

502 self.log.info("Retrieving metadata for %s %d (%d/%d)" % 

503 (self.config.visitDataRefName, visit, i, len(groupedDataRefs))) 

504 # Save checkpoint if desired 

505 if visitCatDataRef is not None: 

506 visitCatDataRef.put(visitCat) 

507 

508 # Note that the reference ccd is first in the list (if available). 

509 

510 # The first dataRef in the group will be the reference ccd (if available) 

511 dataRef = groupedDataRefs[visit][0] 

512 

513 exp = dataRef.get(datasetType='calexp_sub', bbox=bbox, 

514 flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) 

515 

516 visitInfo = exp.getInfo().getVisitInfo() 

517 f = exp.getFilter() 

518 psf = exp.getPsf() 

519 

520 rec = visitCat[i] 

521 rec['visit'] = visit 

522 rec['filtername'] = f.getName() 

523 # TODO DM-26991: when gen2 is removed, gen3 workflow will make it 

524 # much easier to get the wcs's necessary to recompute the pointing 

525 # ra/dec at the center of the camera. 

526 radec = visitInfo.getBoresightRaDec() 

527 rec['telra'] = radec.getRa().asDegrees() 

528 rec['teldec'] = radec.getDec().asDegrees() 

529 rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees() 

530 rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees() 

531 rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD) 

532 rec['exptime'] = visitInfo.getExposureTime() 

533 # convert from Pa to millibar 

534 # Note that I don't know if this unit will need to be per-camera config 

535 rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100 

536 # Flag to signify if this is a "deep" field. Not currently used 

537 rec['deepFlag'] = 0 

538 # Relative flat scaling (1.0 means no relative scaling) 

539 rec['scaling'][:] = 1.0 

540 # Median delta aperture, to be measured from stars 

541 rec['deltaAper'] = 0.0 

542 

543 rec['psfSigma'] = psf.computeShape().getDeterminantRadius() 

544 

545 if dataRef.datasetExists(datasetType='calexpBackground'): 

546 # Get background for reference CCD 

547 # This approximation is good enough for now 

548 bgStats = (bg[0].getStatsImage().getImage().array 

549 for bg in dataRef.get(datasetType='calexpBackground')) 

550 rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats) 

551 else: 

552 self.log.warn('Sky background not found for visit %d / ccd %d' % 

553 (visit, dataRef.dataId[self.config.ccdDataRefName])) 

554 rec['skyBackground'] = -1.0 

555 

556 rec['used'] = 1 

557 

558 def _makeSourceMapper(self, sourceSchema): 

559 """ 

560 Make a schema mapper for fgcm sources 

561 

562 Parameters 

563 ---------- 

564 sourceSchema: `afwTable.Schema` 

565 Default source schema from the butler 

566 

567 Returns 

568 ------- 

569 sourceMapper: `afwTable.schemaMapper` 

570 Mapper to the FGCM source schema 

571 """ 

572 

573 # create a mapper to the preferred output 

574 sourceMapper = afwTable.SchemaMapper(sourceSchema) 

575 

576 # map to ra/dec 

577 sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra') 

578 sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec') 

579 sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x') 

580 sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y') 

581 # Add the mapping if the field exists in the input catalog. 

582 # If the field does not exist, simply add it (set to False). 

583 # This field is not required for calibration, but is useful 

584 # to collate if available. 

585 try: 

586 sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(), 

587 'psf_candidate') 

588 except LookupError: 

589 sourceMapper.editOutputSchema().addField( 

590 "psf_candidate", type='Flag', 

591 doc=("Flag set if the source was a candidate for PSF determination, " 

592 "as determined by the star selector.")) 

593 

594 # and add the fields we want 

595 sourceMapper.editOutputSchema().addField( 

596 "visit", type=np.int32, doc="Visit number") 

597 sourceMapper.editOutputSchema().addField( 

598 "ccd", type=np.int32, doc="CCD number") 

599 sourceMapper.editOutputSchema().addField( 

600 "instMag", type=np.float32, doc="Instrumental magnitude") 

601 sourceMapper.editOutputSchema().addField( 

602 "instMagErr", type=np.float32, doc="Instrumental magnitude error") 

603 sourceMapper.editOutputSchema().addField( 

604 "jacobian", type=np.float32, doc="Relative pixel scale from wcs jacobian") 

605 sourceMapper.editOutputSchema().addField( 

606 "deltaMagBkg", type=np.float32, doc="Change in magnitude due to local background offset") 

607 

608 return sourceMapper 

609 

610 def fgcmMatchStars(self, butler, visitCat, obsCat): 

611 """ 

612 Use FGCM code to match observations into unique stars. 

613 

614 Parameters 

615 ---------- 

616 butler: `lsst.daf.persistence.Butler` 

617 visitCat: `afw.table.BaseCatalog` 

618 Catalog with visit data for fgcm 

619 obsCat: `afw.table.BaseCatalog` 

620 Full catalog of star observations for fgcm 

621 

622 Returns 

623 ------- 

624 fgcmStarIdCat: `afw.table.BaseCatalog` 

625 Catalog of unique star identifiers and index keys 

626 fgcmStarIndicesCat: `afwTable.BaseCatalog` 

627 Catalog of unique star indices 

628 fgcmRefCat: `afw.table.BaseCatalog` 

629 Catalog of matched reference stars. 

630 Will be None if `config.doReferenceMatches` is False. 

631 """ 

632 

633 if self.config.doReferenceMatches: 

634 # Make a subtask for reference loading 

635 self.makeSubtask("fgcmLoadReferenceCatalog", butler=butler) 

636 

637 # get filter names into a numpy array... 

638 # This is the type that is expected by the fgcm code 

639 visitFilterNames = np.zeros(len(visitCat), dtype='a10') 

640 for i in range(len(visitCat)): 

641 visitFilterNames[i] = visitCat[i]['filtername'] 

642 

643 # match to put filterNames with observations 

644 visitIndex = np.searchsorted(visitCat['visit'], 

645 obsCat['visit']) 

646 

647 obsFilterNames = visitFilterNames[visitIndex] 

648 

649 if self.config.doReferenceMatches: 

650 # Get the reference filter names, using the LUT 

651 lutCat = butler.get('fgcmLookUpTable') 

652 

653 stdFilterDict = {filterName: stdFilter for (filterName, stdFilter) in 

654 zip(lutCat[0]['filterNames'].split(','), 

655 lutCat[0]['stdFilterNames'].split(','))} 

656 stdLambdaDict = {stdFilter: stdLambda for (stdFilter, stdLambda) in 

657 zip(lutCat[0]['stdFilterNames'].split(','), 

658 lutCat[0]['lambdaStdFilter'])} 

659 

660 del lutCat 

661 

662 referenceFilterNames = self._getReferenceFilterNames(visitCat, 

663 stdFilterDict, 

664 stdLambdaDict) 

665 self.log.info("Using the following reference filters: %s" % 

666 (', '.join(referenceFilterNames))) 

667 

668 else: 

669 # This should be an empty list 

670 referenceFilterNames = [] 

671 

672 # make the fgcm starConfig dict 

673 

674 starConfig = {'logger': self.log, 

675 'filterToBand': self.config.filterMap, 

676 'requiredBands': self.config.requiredBands, 

677 'minPerBand': self.config.minPerBand, 

678 'matchRadius': self.config.matchRadius, 

679 'isolationRadius': self.config.isolationRadius, 

680 'matchNSide': self.config.matchNside, 

681 'coarseNSide': self.config.coarseNside, 

682 'densNSide': self.config.densityCutNside, 

683 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 

684 'primaryBands': self.config.primaryBands, 

685 'referenceFilterNames': referenceFilterNames} 

686 

687 # initialize the FgcmMakeStars object 

688 fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) 

689 

690 # make the primary stars 

691 # note that the ra/dec native Angle format is radians 

692 # We determine the conversion from the native units (typically 

693 # radians) to degrees for the first observation. This allows us 

694 # to treate ra/dec as numpy arrays rather than Angles, which would 

695 # be approximately 600x slower. 

696 conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) 

697 fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, 

698 obsCat['dec'] * conv, 

699 filterNameArray=obsFilterNames, 

700 bandSelected=False) 

701 

702 # and match all the stars 

703 fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, 

704 obsCat['dec'] * conv, 

705 obsFilterNames) 

706 

707 if self.config.doReferenceMatches: 

708 fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) 

709 

710 # now persist 

711 

712 objSchema = self._makeFgcmObjSchema() 

713 

714 # make catalog and records 

715 fgcmStarIdCat = afwTable.BaseCatalog(objSchema) 

716 fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) 

717 for i in range(fgcmMakeStars.objIndexCat.size): 

718 fgcmStarIdCat.addNew() 

719 

720 # fill the catalog 

721 fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] 

722 fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] 

723 fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] 

724 fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat['obsarrindex'] 

725 fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] 

726 

727 obsSchema = self._makeFgcmObsSchema() 

728 

729 fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) 

730 fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) 

731 for i in range(fgcmMakeStars.obsIndexCat.size): 

732 fgcmStarIndicesCat.addNew() 

733 

734 fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat['obsindex'] 

735 

736 if self.config.doReferenceMatches: 

737 refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) 

738 

739 fgcmRefCat = afwTable.BaseCatalog(refSchema) 

740 fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) 

741 

742 for i in range(fgcmMakeStars.referenceCat.size): 

743 fgcmRefCat.addNew() 

744 

745 fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] 

746 fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] 

747 fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat['refMagErr'] 

748 

749 md = PropertyList() 

750 md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION) 

751 md.set("FILTERNAMES", referenceFilterNames) 

752 fgcmRefCat.setMetadata(md) 

753 

754 else: 

755 fgcmRefCat = None 

756 

757 return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat 

758 

759 def _makeFgcmVisitSchema(self, nCcd): 

760 """ 

761 Make a schema for an fgcmVisitCatalog 

762 

763 Parameters 

764 ---------- 

765 nCcd: `int` 

766 Number of CCDs in the camera 

767 

768 Returns 

769 ------- 

770 schema: `afwTable.Schema` 

771 """ 

772 

773 schema = afwTable.Schema() 

774 schema.addField('visit', type=np.int32, doc="Visit number") 

775 # Note that the FGCM code currently handles filternames up to 2 characters long 

776 schema.addField('filtername', type=str, size=10, doc="Filter name") 

777 schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") 

778 schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") 

779 schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") 

780 schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)") 

781 schema.addField('mjd', type=np.float64, doc="MJD of visit") 

782 schema.addField('exptime', type=np.float32, doc="Exposure time") 

783 schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") 

784 schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") 

785 schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") 

786 schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") 

787 # the following field is not used yet 

788 schema.addField('deepFlag', type=np.int32, doc="Deep observation") 

789 schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", 

790 size=nCcd) 

791 schema.addField('used', type=np.int32, doc="This visit has been ingested.") 

792 schema.addField('sources_read', type='Flag', doc="This visit had sources read.") 

793 

794 return schema 

795 

796 def _makeFgcmObjSchema(self): 

797 """ 

798 Make a schema for the objIndexCat from fgcmMakeStars 

799 

800 Returns 

801 ------- 

802 schema: `afwTable.Schema` 

803 """ 

804 

805 objSchema = afwTable.Schema() 

806 objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

807 # Will investigate making these angles... 

808 objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)') 

809 objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)') 

810 objSchema.addField('obsArrIndex', type=np.int32, 

811 doc='Index in obsIndexTable for first observation') 

812 objSchema.addField('nObs', type=np.int32, doc='Total number of observations') 

813 

814 return objSchema 

815 

816 def _makeFgcmObsSchema(self): 

817 """ 

818 Make a schema for the obsIndexCat from fgcmMakeStars 

819 

820 Returns 

821 ------- 

822 schema: `afwTable.Schema` 

823 """ 

824 

825 obsSchema = afwTable.Schema() 

826 obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table') 

827 

828 return obsSchema 

829 

830 def _makeFgcmRefSchema(self, nReferenceBands): 

831 """ 

832 Make a schema for the referenceCat from fgcmMakeStars 

833 

834 Parameters 

835 ---------- 

836 nReferenceBands: `int` 

837 Number of reference bands 

838 

839 Returns 

840 ------- 

841 schema: `afwTable.Schema` 

842 """ 

843 

844 refSchema = afwTable.Schema() 

845 refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') 

846 refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)', 

847 size=nReferenceBands) 

848 refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array', 

849 size=nReferenceBands) 

850 

851 return refSchema 

852 

853 def _getReferenceFilterNames(self, visitCat, stdFilterDict, stdLambdaDict): 

854 """ 

855 Get the reference filter names, in wavelength order, from the visitCat and 

856 information from the look-up-table. 

857 

858 Parameters 

859 ---------- 

860 visitCat: `afw.table.BaseCatalog` 

861 Catalog with visit data for FGCM 

862 stdFilterDict: `dict` 

863 Mapping of filterName to stdFilterName from LUT 

864 stdLambdaDict: `dict` 

865 Mapping of stdFilterName to stdLambda from LUT 

866 

867 Returns 

868 ------- 

869 referenceFilterNames: `list` 

870 Wavelength-ordered list of reference filter names 

871 """ 

872 

873 # Find the unique list of filter names in visitCat 

874 filterNames = np.unique(visitCat.asAstropy()['filtername']) 

875 

876 # Find the unique list of "standard" filters 

877 stdFilterNames = {stdFilterDict[filterName] for filterName in filterNames} 

878 

879 # And sort these by wavelength 

880 referenceFilterNames = sorted(stdFilterNames, key=stdLambdaDict.get) 

881 

882 return referenceFilterNames