Coverage for python/lsst/jointcal/jointcal.py: 18%

543 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-01 04:12 -0700

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import dataclasses 

23import collections 

24import os 

25import logging 

26 

27import astropy.time 

28import numpy as np 

29import astropy.units as u 

30 

31import lsst.geom 

32import lsst.utils 

33import lsst.pex.config as pexConfig 

34import lsst.pipe.base as pipeBase 

35from lsst.afw.image import fluxErrFromABMagErr 

36import lsst.afw.cameraGeom 

37import lsst.afw.table 

38from lsst.pipe.base import Instrument 

39from lsst.pipe.tasks.colorterms import ColortermLibrary 

40from lsst.verify import Job, Measurement 

41 

42from lsst.meas.algorithms import (ReferenceObjectLoader, ReferenceSourceSelectorTask, 

43 LoadReferenceObjectsConfig) 

44from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

45 

46import lsst.jointcal 

47from lsst.jointcal import MinimizeResult 

48 

49__all__ = ["JointcalConfig", "JointcalTask"] 

50 

51Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

52Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

53 

54 

55# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

56def add_measurement(job, name, value): 

57 meas = Measurement(job.metrics[name], value) 

58 job.measurements.insert(meas) 

59 

60 

61def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

62 """Lookup function that asserts/hopes that a static calibration dataset 

63 exists in a particular collection, since this task can't provide a single 

64 date/time to use to search for one properly. 

65 

66 This is mostly useful for the ``camera`` dataset, in cases where the task's 

67 quantum dimensions do *not* include something temporal, like ``exposure`` 

68 or ``visit``. 

69 

70 Parameters 

71 ---------- 

72 datasetType : `lsst.daf.butler.DatasetType` 

73 Type of dataset being searched for. 

74 registry : `lsst.daf.butler.Registry` 

75 Data repository registry to search. 

76 quantumDataId : `lsst.daf.butler.DataCoordinate` 

77 Data ID of the quantum this camera should match. 

78 collections : `Iterable` [ `str` ] 

79 Collections that should be searched - but this lookup function works 

80 by ignoring this in favor of a more-or-less hard-coded value. 

81 

82 Returns 

83 ------- 

84 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

85 Iterator over dataset references; should have only one element. 

86 

87 Notes 

88 ----- 

89 This implementation duplicates one in fgcmcal, and is at least quite 

90 similar to another in cp_pipe. This duplicate has the most documentation. 

91 Fixing this is DM-29661. 

92 """ 

93 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

94 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

95 return registry.queryDatasets(datasetType, 

96 dataId=quantumDataId, 

97 collections=[unboundedCollection], 

98 findFirst=True) 

99 

100 

101def lookupVisitRefCats(datasetType, registry, quantumDataId, collections): 

102 """Lookup function that finds all refcats for all visits that overlap a 

103 tract, rather than just the refcats that directly overlap the tract. 

104 

105 Parameters 

106 ---------- 

107 datasetType : `lsst.daf.butler.DatasetType` 

108 Type of dataset being searched for. 

109 registry : `lsst.daf.butler.Registry` 

110 Data repository registry to search. 

111 quantumDataId : `lsst.daf.butler.DataCoordinate` 

112 Data ID of the quantum; expected to be something we can use as a 

113 constraint to query for overlapping visits. 

114 collections : `Iterable` [ `str` ] 

115 Collections to search. 

116 

117 Returns 

118 ------- 

119 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

120 Iterator over refcat references. 

121 """ 

122 refs = set() 

123 # Use .expanded() on the query methods below because we need data IDs with 

124 # regions, both in the outer loop over visits (queryDatasets will expand 

125 # any data ID we give it, but doing it up-front in bulk is much more 

126 # efficient) and in the data IDs of the DatasetRefs this function yields 

127 # (because the RefCatLoader relies on them to do some of its own 

128 # filtering). 

129 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()): 

130 refs.update( 

131 registry.queryDatasets( 

132 datasetType, 

133 collections=collections, 

134 dataId=visit_data_id, 

135 findFirst=True, 

136 ).expanded() 

137 ) 

138 yield from refs 

139 

140 

141class JointcalTaskConnections(pipeBase.PipelineTaskConnections, 

142 dimensions=("skymap", "tract", "instrument", "physical_filter")): 

143 """Middleware input/output connections for jointcal data.""" 

144 inputCamera = pipeBase.connectionTypes.PrerequisiteInput( 

145 doc="The camera instrument that took these observations.", 

146 name="camera", 

147 storageClass="Camera", 

148 dimensions=("instrument",), 

149 isCalibration=True, 

150 lookupFunction=lookupStaticCalibrations, 

151 ) 

152 inputSourceTableVisit = pipeBase.connectionTypes.Input( 

153 doc="Source table in parquet format, per visit", 

154 name="sourceTable_visit", 

155 storageClass="DataFrame", 

156 dimensions=("instrument", "visit"), 

157 deferLoad=True, 

158 multiple=True, 

159 ) 

160 inputVisitSummary = pipeBase.connectionTypes.Input( 

161 doc=("Per-visit consolidated exposure metadata built from calexps. " 

162 "These catalogs use detector id for the id and must be sorted for " 

163 "fast lookups of a detector."), 

164 name="visitSummary", 

165 storageClass="ExposureCatalog", 

166 dimensions=("instrument", "visit"), 

167 deferLoad=True, 

168 multiple=True, 

169 ) 

170 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

171 doc="The astrometry reference catalog to match to loaded input catalog sources.", 

172 name="gaia_dr2_20200414", 

173 storageClass="SimpleCatalog", 

174 dimensions=("skypix",), 

175 deferLoad=True, 

176 multiple=True, 

177 lookupFunction=lookupVisitRefCats, 

178 ) 

179 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

180 doc="The photometry reference catalog to match to loaded input catalog sources.", 

181 name="ps1_pv3_3pi_20170110", 

182 storageClass="SimpleCatalog", 

183 dimensions=("skypix",), 

184 deferLoad=True, 

185 multiple=True, 

186 lookupFunction=lookupVisitRefCats, 

187 ) 

188 

189 outputWcs = pipeBase.connectionTypes.Output( 

190 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model." 

191 " These catalogs only contain entries for detectors with an output, and use" 

192 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

193 name="jointcalSkyWcsCatalog", 

194 storageClass="ExposureCatalog", 

195 dimensions=("instrument", "visit", "skymap", "tract"), 

196 multiple=True 

197 ) 

198 outputPhotoCalib = pipeBase.connectionTypes.Output( 

199 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model." 

200 " These catalogs only contain entries for detectors with an output, and use" 

201 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

202 name="jointcalPhotoCalibCatalog", 

203 storageClass="ExposureCatalog", 

204 dimensions=("instrument", "visit", "skymap", "tract"), 

205 multiple=True 

206 ) 

207 

208 # measurements of metrics 

209 # The vars() trick used here allows us to set class attributes 

210 # programatically. Taken from: 

211 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class 

212 for name in ("astrometry", "photometry"): 

213 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output( 

214 doc=f"The number of cross-matched fittedStars for {name}", 

215 name=f"metricvalue_jointcal_{name}_matched_fittedStars", 

216 storageClass="MetricValue", 

217 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

218 ) 

219 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output( 

220 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.", 

221 name=f"metricvalue_jointcal_{name}_collected_refStars", 

222 storageClass="MetricValue", 

223 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

224 ) 

225 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output( 

226 doc=f"The number of {name} reference stars matched to fittedStars.", 

227 name=f"metricvalue_jointcal_{name}_prepared_refStars", 

228 storageClass="MetricValue", 

229 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

230 ) 

231 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output( 

232 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.", 

233 name=f"metricvalue_jointcal_{name}_prepared_fittedStars", 

234 storageClass="MetricValue", 

235 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

236 ) 

237 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output( 

238 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.", 

239 name=f"metricvalue_jointcal_{name}_prepared_ccdImages", 

240 storageClass="MetricValue", 

241 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

242 ) 

243 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output( 

244 doc=f"The final chi2 of the {name} fit.", 

245 name=f"metricvalue_jointcal_{name}_final_chi2", 

246 storageClass="MetricValue", 

247 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

248 ) 

249 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output( 

250 doc=f"The number of degrees of freedom of the fitted {name}.", 

251 name=f"metricvalue_jointcal_{name}_final_ndof", 

252 storageClass="MetricValue", 

253 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

254 ) 

255 

256 def __init__(self, *, config=None): 

257 super().__init__(config=config) 

258 # When we are only doing one of astrometry or photometry, we don't 

259 # need the reference catalog or produce the outputs for the other. 

260 # This informs the middleware of that when the QuantumGraph is 

261 # generated, so we don't block on getting something we won't need or 

262 # create an expectation that downstream tasks will be able to consume 

263 # something we won't produce. 

264 if not config.doAstrometry: 

265 self.prerequisiteInputs.remove("astrometryRefCat") 

266 self.outputs.remove("outputWcs") 

267 for key in list(self.outputs): 

268 if "metricvalue_jointcal_astrometry" in key: 

269 self.outputs.remove(key) 

270 if not config.doPhotometry: 

271 self.prerequisiteInputs.remove("photometryRefCat") 

272 self.outputs.remove("outputPhotoCalib") 

273 for key in list(self.outputs): 

274 if "metricvalue_jointcal_photometry" in key: 

275 self.outputs.remove(key) 

276 

277 

278class JointcalConfig(pipeBase.PipelineTaskConfig, 

279 pipelineConnections=JointcalTaskConnections): 

280 """Configuration for JointcalTask""" 

281 

282 doAstrometry = pexConfig.Field( 

283 doc="Fit astrometry and write the fitted result.", 

284 dtype=bool, 

285 default=True 

286 ) 

287 doPhotometry = pexConfig.Field( 

288 doc="Fit photometry and write the fitted result.", 

289 dtype=bool, 

290 default=True 

291 ) 

292 sourceFluxType = pexConfig.Field( 

293 dtype=str, 

294 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

295 default='apFlux_12_0' 

296 ) 

297 positionErrorPedestal = pexConfig.Field( 

298 doc="Systematic term to apply to the measured position error (pixels)", 

299 dtype=float, 

300 default=0.02, 

301 ) 

302 photometryErrorPedestal = pexConfig.Field( 

303 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

304 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

305 dtype=float, 

306 default=0.0, 

307 ) 

308 # TODO: DM-6885 matchCut should be an geom.Angle 

309 matchCut = pexConfig.Field( 

310 doc="Matching radius between fitted and reference stars (arcseconds)", 

311 dtype=float, 

312 default=3.0, 

313 ) 

314 minMeasurements = pexConfig.Field( 

315 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

316 dtype=int, 

317 default=2, 

318 ) 

319 minMeasuredStarsPerCcd = pexConfig.Field( 

320 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

321 dtype=int, 

322 default=100, 

323 ) 

324 minRefStarsPerCcd = pexConfig.Field( 

325 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

326 dtype=int, 

327 default=30, 

328 ) 

329 allowLineSearch = pexConfig.Field( 

330 doc="Allow a line search during minimization, if it is reasonable for the model" 

331 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

332 dtype=bool, 

333 default=False 

334 ) 

335 astrometrySimpleOrder = pexConfig.Field( 

336 doc="Polynomial order for fitting the simple astrometry model.", 

337 dtype=int, 

338 default=3, 

339 ) 

340 astrometryChipOrder = pexConfig.Field( 

341 doc="Order of the per-chip transform for the constrained astrometry model.", 

342 dtype=int, 

343 default=1, 

344 ) 

345 astrometryVisitOrder = pexConfig.Field( 

346 doc="Order of the per-visit transform for the constrained astrometry model.", 

347 dtype=int, 

348 default=5, 

349 ) 

350 useInputWcs = pexConfig.Field( 

351 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

352 dtype=bool, 

353 default=True, 

354 ) 

355 astrometryModel = pexConfig.ChoiceField( 

356 doc="Type of model to fit to astrometry", 

357 dtype=str, 

358 default="constrained", 

359 allowed={"simple": "One polynomial per ccd", 

360 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

361 ) 

362 photometryModel = pexConfig.ChoiceField( 

363 doc="Type of model to fit to photometry", 

364 dtype=str, 

365 default="constrainedMagnitude", 

366 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

367 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

368 " fitting in flux space.", 

369 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

370 " fitting in magnitude space.", 

371 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

372 " fitting in magnitude space.", 

373 } 

374 ) 

375 applyColorTerms = pexConfig.Field( 

376 doc="Apply photometric color terms to reference stars?" 

377 "Requires that colorterms be set to a ColortermLibrary", 

378 dtype=bool, 

379 default=False 

380 ) 

381 colorterms = pexConfig.ConfigField( 

382 doc="Library of photometric reference catalog name to color term dict.", 

383 dtype=ColortermLibrary, 

384 ) 

385 photometryVisitOrder = pexConfig.Field( 

386 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

387 dtype=int, 

388 default=7, 

389 ) 

390 photometryDoRankUpdate = pexConfig.Field( 

391 doc=("Do the rank update step during minimization. " 

392 "Skipping this can help deal with models that are too non-linear."), 

393 dtype=bool, 

394 default=True, 

395 ) 

396 astrometryDoRankUpdate = pexConfig.Field( 

397 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

398 "Skipping this can help deal with models that are too non-linear."), 

399 dtype=bool, 

400 default=True, 

401 ) 

402 outlierRejectSigma = pexConfig.Field( 

403 doc="How many sigma to reject outliers at during minimization.", 

404 dtype=float, 

405 default=5.0, 

406 ) 

407 astrometryOutlierRelativeTolerance = pexConfig.Field( 

408 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will " 

409 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set " 

410 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002" 

411 "as a balance between a shorter minimization runtime and achieving a final fitted model that is" 

412 "close to the solution found when removing all outliers."), 

413 dtype=float, 

414 default=0, 

415 ) 

416 maxPhotometrySteps = pexConfig.Field( 

417 doc="Maximum number of minimize iterations to take when fitting photometry.", 

418 dtype=int, 

419 default=20, 

420 ) 

421 maxAstrometrySteps = pexConfig.Field( 

422 doc="Maximum number of minimize iterations to take when fitting astrometry.", 

423 dtype=int, 

424 default=20, 

425 ) 

426 astrometryRefObjLoader = pexConfig.ConfigField( 

427 dtype=LoadReferenceObjectsConfig, 

428 doc="Reference object loader for astrometric fit", 

429 ) 

430 photometryRefObjLoader = pexConfig.ConfigField( 

431 dtype=LoadReferenceObjectsConfig, 

432 doc="Reference object loader for photometric fit", 

433 ) 

434 sourceSelector = sourceSelectorRegistry.makeField( 

435 doc="How to select sources for cross-matching", 

436 default="science" 

437 ) 

438 astrometryReferenceSelector = pexConfig.ConfigurableField( 

439 target=ReferenceSourceSelectorTask, 

440 doc="How to down-select the loaded astrometry reference catalog.", 

441 ) 

442 photometryReferenceSelector = pexConfig.ConfigurableField( 

443 target=ReferenceSourceSelectorTask, 

444 doc="How to down-select the loaded photometry reference catalog.", 

445 ) 

446 astrometryReferenceErr = pexConfig.Field( 

447 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

448 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

449 "If specified, overrides any existing `coord_*Err` values."), 

450 dtype=float, 

451 default=None, 

452 optional=True 

453 ) 

454 

455 # configs for outputting debug information 

456 writeInitMatrix = pexConfig.Field( 

457 dtype=bool, 

458 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

459 "Output files will be written to `config.debugOutputPath` and will " 

460 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. " 

461 "Note that these files are the dense versions of the matrix, and so may be very large."), 

462 default=False 

463 ) 

464 writeChi2FilesInitialFinal = pexConfig.Field( 

465 dtype=bool, 

466 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. " 

467 "Output files will be written to `config.debugOutputPath` and will " 

468 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."), 

469 default=False 

470 ) 

471 writeChi2FilesOuterLoop = pexConfig.Field( 

472 dtype=bool, 

473 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. " 

474 "Output files will be written to `config.debugOutputPath` and will " 

475 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."), 

476 default=False 

477 ) 

478 writeInitialModel = pexConfig.Field( 

479 dtype=bool, 

480 doc=("Write the pre-initialization model to text files, for debugging. " 

481 "Output files will be written to `config.debugOutputPath` and will be " 

482 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."), 

483 default=False 

484 ) 

485 debugOutputPath = pexConfig.Field( 

486 dtype=str, 

487 default=".", 

488 doc=("Path to write debug output files to. Used by " 

489 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

490 ) 

491 detailedProfile = pexConfig.Field( 

492 dtype=bool, 

493 default=False, 

494 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting" 

495 ) 

496 

497 def validate(self): 

498 super().validate() 

499 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

500 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

501 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

502 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

503 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

504 "applyColorTerms=True will be ignored.") 

505 logging.getLogger("lsst.jointcal").warning(msg) 

506 

507 def setDefaults(self): 

508 # Use only primary stars. 

509 self.sourceSelector["science"].doRequirePrimary = True 

510 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing. 

511 self.sourceSelector["science"].doUnresolved = True 

512 self.sourceSelector["science"].unresolved.name = "extendedness" 

513 # with dependable signal to noise ratio. 

514 self.sourceSelector["science"].doSignalToNoise = True 

515 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

516 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

517 self.sourceSelector["science"].signalToNoise.minimum = 10. 

518 # Base SNR selection on `sourceFluxType` because that is the flux that jointcal fits. 

519 self.sourceSelector["science"].signalToNoise.fluxField = f"{self.sourceFluxType}_instFlux" 

520 self.sourceSelector["science"].signalToNoise.errField = f"{self.sourceFluxType}_instFluxErr" 

521 # Do not trust blended sources" aperture fluxes which also depend on seeing. 

522 self.sourceSelector["science"].doIsolated = True 

523 self.sourceSelector["science"].isolated.parentName = "parentSourceId" 

524 self.sourceSelector["science"].isolated.nChildName = "deblend_nChild" 

525 # Do not trust either flux or centroid measurements with flags, 

526 # chosen from the usual QA flags for stars) 

527 self.sourceSelector["science"].doFlags = True 

528 badFlags = ["pixelFlags_edge", 

529 "pixelFlags_saturated", 

530 "pixelFlags_interpolatedCenter", 

531 "pixelFlags_interpolated", 

532 "pixelFlags_crCenter", 

533 "pixelFlags_bad", 

534 "hsmPsfMoments_flag", 

535 f"{self.sourceFluxType}_flag", 

536 ] 

537 self.sourceSelector["science"].flags.bad = badFlags 

538 self.sourceSelector["science"].doRequireFiniteRaDec = True 

539 self.sourceSelector["science"].requireFiniteRaDec.raColName = "ra" 

540 self.sourceSelector["science"].requireFiniteRaDec.decColName = "decl" 

541 

542 # Use Gaia-DR2 with proper motions for astrometry; phot_g_mean is the 

543 # primary Gaia band, but is not like any normal photometric band. 

544 self.astrometryRefObjLoader.requireProperMotion = True 

545 self.astrometryRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

546 

547 

548def writeModel(model, filename, log): 

549 """Write model to outfile.""" 

550 with open(filename, "w") as file: 

551 file.write(repr(model)) 

552 log.info("Wrote %s to file: %s", model, filename) 

553 

554 

555@dataclasses.dataclass 

556class JointcalInputData: 

557 """The input data jointcal needs for each detector/visit.""" 

558 visit: int 

559 """The visit identifier of this exposure.""" 

560 catalog: lsst.afw.table.SourceCatalog 

561 """The catalog derived from this exposure.""" 

562 visitInfo: lsst.afw.image.VisitInfo 

563 """The VisitInfo of this exposure.""" 

564 detector: lsst.afw.cameraGeom.Detector 

565 """The detector of this exposure.""" 

566 photoCalib: lsst.afw.image.PhotoCalib 

567 """The photometric calibration of this exposure.""" 

568 wcs: lsst.afw.geom.skyWcs 

569 """The WCS of this exposure.""" 

570 bbox: lsst.geom.Box2I 

571 """The bounding box of this exposure.""" 

572 filter: lsst.afw.image.FilterLabel 

573 """The filter of this exposure.""" 

574 

575 

576class JointcalTask(pipeBase.PipelineTask): 

577 """Astrometricly and photometricly calibrate across multiple visits of the 

578 same field. 

579 """ 

580 

581 ConfigClass = JointcalConfig 

582 _DefaultName = "jointcal" 

583 

584 def __init__(self, **kwargs): 

585 super().__init__(**kwargs) 

586 self.makeSubtask("sourceSelector") 

587 if self.config.doAstrometry: 

588 self.makeSubtask("astrometryReferenceSelector") 

589 else: 

590 self.astrometryRefObjLoader = None 

591 if self.config.doPhotometry: 

592 self.makeSubtask("photometryReferenceSelector") 

593 else: 

594 self.photometryRefObjLoader = None 

595 

596 # To hold various computed metrics for use by tests 

597 self.job = Job.load_metrics_package(subset='jointcal') 

598 

599 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

600 # We override runQuantum to set up the refObjLoaders and write the 

601 # outputs to the correct refs. 

602 inputs = butlerQC.get(inputRefs) 

603 # We want the tract number for writing debug files 

604 tract = butlerQC.quantum.dataId['tract'] 

605 if self.config.doAstrometry: 

606 self.astrometryRefObjLoader = ReferenceObjectLoader( 

607 dataIds=[ref.datasetRef.dataId 

608 for ref in inputRefs.astrometryRefCat], 

609 refCats=inputs.pop('astrometryRefCat'), 

610 config=self.config.astrometryRefObjLoader, 

611 name=self.config.connections.astrometryRefCat, 

612 log=self.log) 

613 if self.config.doPhotometry: 

614 self.photometryRefObjLoader = ReferenceObjectLoader( 

615 dataIds=[ref.datasetRef.dataId 

616 for ref in inputRefs.photometryRefCat], 

617 refCats=inputs.pop('photometryRefCat'), 

618 config=self.config.photometryRefObjLoader, 

619 name=self.config.connections.photometryRefCat, 

620 log=self.log) 

621 outputs = self.run(**inputs, tract=tract) 

622 self._put_metrics(butlerQC, outputs.job, outputRefs) 

623 if self.config.doAstrometry: 

624 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs, 

625 inputs['inputCamera'], "setWcs") 

626 if self.config.doPhotometry: 

627 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib, 

628 inputs['inputCamera'], "setPhotoCalib") 

629 

630 def _put_metrics(self, butlerQC, job, outputRefs): 

631 """Persist all measured metrics stored in a job. 

632 

633 Parameters 

634 ---------- 

635 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

636 A butler which is specialized to operate in the context of a 

637 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

638 job : `lsst.verify.job.Job` 

639 Measurements of metrics to persist. 

640 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

641 The DatasetRefs to persist the data to. 

642 """ 

643 for key in job.measurements.keys(): 

644 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', ''))) 

645 

646 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter): 

647 """Persist the output datasets to their appropriate datarefs. 

648 

649 Parameters 

650 ---------- 

651 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

652 A butler which is specialized to operate in the context of a 

653 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

654 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or 

655 `dict` [`tuple, `lsst.afw.image.PhotoCalib`] 

656 The fitted objects to persist. 

657 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

658 The DatasetRefs to persist the data to. 

659 camera : `lsst.afw.cameraGeom.Camera` 

660 The camera for this instrument, to get detector ids from. 

661 setter : `str` 

662 The method to call on the ExposureCatalog to set each output. 

663 """ 

664 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

665 schema.addField('visit', type='L', doc='Visit number') 

666 

667 def new_catalog(visit, size): 

668 """Return an catalog ready to be filled with appropriate output.""" 

669 catalog = lsst.afw.table.ExposureCatalog(schema) 

670 catalog.resize(size) 

671 catalog['visit'] = visit 

672 metadata = lsst.daf.base.PropertyList() 

673 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

674 metadata.add("COMMENT", "Only detectors with data have entries.") 

675 return catalog 

676 

677 # count how many detectors have output for each visit 

678 detectors_per_visit = collections.defaultdict(int) 

679 for key in outputs: 

680 # key is (visit, detector_id), and we only need visit here 

681 detectors_per_visit[key[0]] += 1 

682 

683 for ref in outputRefs: 

684 visit = ref.dataId['visit'] 

685 catalog = new_catalog(visit, detectors_per_visit[visit]) 

686 # Iterate over every detector and skip the ones we don't have output for. 

687 i = 0 

688 for detector in camera: 

689 detectorId = detector.getId() 

690 key = (ref.dataId['visit'], detectorId) 

691 if key not in outputs: 

692 # skip detectors we don't have output for 

693 self.log.debug("No %s output for detector %s in visit %s", 

694 setter[3:], detectorId, visit) 

695 continue 

696 

697 catalog[i].setId(detectorId) 

698 getattr(catalog[i], setter)(outputs[key]) 

699 i += 1 

700 

701 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups 

702 butlerQC.put(catalog, ref) 

703 self.log.info("Wrote %s detectors to %s", i, ref) 

704 

705 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None): 

706 # Docstring inherited. 

707 

708 # We take values out of the Parquet table, and put them in "flux_", 

709 # and the config.sourceFluxType field is used during that extraction, 

710 # so just use "flux" here. 

711 sourceFluxField = "flux" 

712 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

713 associations = lsst.jointcal.Associations() 

714 self.focalPlaneBBox = inputCamera.getFpBBox() 

715 oldWcsList, bands = self._load_data(inputSourceTableVisit, 

716 inputVisitSummary, 

717 associations, 

718 jointcalControl, 

719 inputCamera) 

720 

721 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands) 

722 

723 if self.config.doAstrometry: 

724 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

725 name="astrometry", 

726 refObjLoader=self.astrometryRefObjLoader, 

727 referenceSelector=self.astrometryReferenceSelector, 

728 fit_function=self._fit_astrometry, 

729 tract=tract, 

730 epoch=epoch) 

731 astrometry_output = self._make_output(associations.getCcdImageList(), 

732 astrometry.model, 

733 "makeSkyWcs") 

734 else: 

735 astrometry_output = None 

736 

737 if self.config.doPhotometry: 

738 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

739 name="photometry", 

740 refObjLoader=self.photometryRefObjLoader, 

741 referenceSelector=self.photometryReferenceSelector, 

742 fit_function=self._fit_photometry, 

743 tract=tract, 

744 epoch=epoch, 

745 reject_bad_fluxes=True) 

746 photometry_output = self._make_output(associations.getCcdImageList(), 

747 photometry.model, 

748 "toPhotoCalib") 

749 else: 

750 photometry_output = None 

751 

752 return pipeBase.Struct(outputWcs=astrometry_output, 

753 outputPhotoCalib=photometry_output, 

754 job=self.job, 

755 astrometryRefObjLoader=self.astrometryRefObjLoader, 

756 photometryRefObjLoader=self.photometryRefObjLoader) 

757 

758 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, 

759 jointcalControl, camera): 

760 """Read the data that jointcal needs to run. 

761 

762 Modifies ``associations`` in-place with the loaded data. 

763 

764 Parameters 

765 ---------- 

766 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

767 References to visit-level DataFrames to load the catalog data from. 

768 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

769 Visit-level exposure summary catalog with metadata. 

770 associations : `lsst.jointcal.Associations` 

771 Object to add the loaded data to by constructing new CcdImages. 

772 jointcalControl : `jointcal.JointcalControl` 

773 Control object for C++ associations management. 

774 camera : `lsst.afw.cameraGeom.Camera` 

775 Camera object for detector geometry. 

776 

777 Returns 

778 ------- 

779 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`] 

780 The original WCS of the input data, to aid in writing tests. 

781 bands : `list` [`str`] 

782 The filter bands of each input dataset. 

783 """ 

784 oldWcsList = [] 

785 filters = [] 

786 load_cat_profile_file = 'jointcal_load_data.prof' if self.config.detailedProfile else '' 

787 with lsst.utils.timer.profile(load_cat_profile_file): 

788 table = make_schema_table() # every detector catalog has the same layout 

789 # No guarantee that the input is in the same order of visits, so we have to map one of them. 

790 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)} 

791 detectorDict = {detector.getId(): detector for detector in camera} 

792 

793 columns = None 

794 

795 for visitSummaryRef in inputVisitSummary: 

796 visitSummary = visitSummaryRef.get() 

797 

798 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]] 

799 if columns is None: 

800 inColumns = dataRef.get(component='columns') 

801 columns, ixxColumns = get_sourceTable_visit_columns(inColumns, 

802 self.config, 

803 self.sourceSelector) 

804 visitCatalog = dataRef.get(parameters={'columns': columns}) 

805 

806 selected = self.sourceSelector.run(visitCatalog) 

807 if len(selected.sourceCat) == 0: 

808 self.log.warning("No sources selected in visit %s. Skipping...", 

809 visitSummary["visit"][0]) 

810 continue 

811 

812 # Build a CcdImage for each detector in this visit. 

813 detectors = {id: index for index, id in enumerate(visitSummary['id'])} 

814 for id, index in detectors.items(): 

815 catalog = extract_detector_catalog_from_visit_catalog(table, 

816 selected.sourceCat, 

817 id, 

818 ixxColumns, 

819 self.config.sourceFluxType, 

820 self.log) 

821 if catalog is None: 

822 continue 

823 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict) 

824 result = self._build_ccdImage(data, associations, jointcalControl) 

825 if result is not None: 

826 oldWcsList.append(result.wcs) 

827 # A visit has only one band, so we can just use the first. 

828 filters.append(data.filter) 

829 if len(filters) == 0: 

830 raise RuntimeError("No data to process: did source selector remove all sources?") 

831 filters = collections.Counter(filters) 

832 

833 return oldWcsList, filters 

834 

835 def _make_one_input_data(self, visitRecord, catalog, detectorDict): 

836 """Return a data structure for this detector+visit.""" 

837 return JointcalInputData(visit=visitRecord['visit'], 

838 catalog=catalog, 

839 visitInfo=visitRecord.getVisitInfo(), 

840 detector=detectorDict[visitRecord.getId()], 

841 photoCalib=visitRecord.getPhotoCalib(), 

842 wcs=visitRecord.getWcs(), 

843 bbox=visitRecord.getBBox(), 

844 # ExposureRecord doesn't have a FilterLabel yet, 

845 # so we have to make one. 

846 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'], 

847 physical=visitRecord['physical_filter'])) 

848 

849 def _build_ccdImage(self, data, associations, jointcalControl): 

850 """ 

851 Extract the necessary things from this catalog+metadata to add a new 

852 ccdImage. 

853 

854 Parameters 

855 ---------- 

856 data : `JointcalInputData` 

857 The loaded input data. 

858 associations : `lsst.jointcal.Associations` 

859 Object to add the info to, to construct a new CcdImage 

860 jointcalControl : `jointcal.JointcalControl` 

861 Control object for associations management 

862 

863 Returns 

864 ------ 

865 namedtuple or `None` 

866 ``wcs`` 

867 The TAN WCS of this image, read from the calexp 

868 (`lsst.afw.geom.SkyWcs`). 

869 ``key`` 

870 A key to identify this dataRef by its visit and ccd ids 

871 (`namedtuple`). 

872 `None` 

873 if there are no sources in the loaded catalog. 

874 """ 

875 if len(data.catalog) == 0: 

876 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId()) 

877 return None 

878 

879 associations.createCcdImage(data.catalog, 

880 data.wcs, 

881 data.visitInfo, 

882 data.bbox, 

883 data.filter.physicalLabel, 

884 data.photoCalib, 

885 data.detector, 

886 data.visit, 

887 data.detector.getId(), 

888 jointcalControl) 

889 

890 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key')) 

891 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

892 return Result(data.wcs, Key(data.visit, data.detector.getId())) 

893 

894 def _getDebugPath(self, filename): 

895 """Constructs a path to filename using the configured debug path. 

896 """ 

897 return os.path.join(self.config.debugOutputPath, filename) 

898 

899 def _prep_sky(self, associations, filters): 

900 """Prepare on-sky and other data that must be computed after data has 

901 been read. 

902 """ 

903 associations.computeCommonTangentPoint() 

904 

905 boundingCircle = associations.computeBoundingCircle() 

906 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

907 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

908 

909 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

910 

911 # Determine a default filter band associated with the catalog. See DM-9093 

912 defaultFilter = filters.most_common(1)[0][0] 

913 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel) 

914 

915 # compute and set the reference epoch of the observations, for proper motion corrections 

916 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

917 associations.setEpoch(epoch.jyear) 

918 

919 return boundingCircle, center, radius, defaultFilter, epoch 

920 

921 def _get_refcat_coordinate_error_override(self, refCat, name): 

922 """Check whether we should override the refcat coordinate errors, and 

923 return the overridden error if necessary. 

924 

925 Parameters 

926 ---------- 

927 refCat : `lsst.afw.table.SimpleCatalog` 

928 The reference catalog to check for a ``coord_raErr`` field. 

929 name : `str` 

930 Whether we are doing "astrometry" or "photometry". 

931 

932 Returns 

933 ------- 

934 refCoordErr : `float` 

935 The refcat coordinate error to use, or NaN if we are not overriding 

936 those fields. 

937 

938 Raises 

939 ------ 

940 lsst.pex.config.FieldValidationError 

941 Raised if the refcat does not contain coordinate errors and 

942 ``config.astrometryReferenceErr`` is not set. 

943 """ 

944 # This value doesn't matter for photometry, so just set something to 

945 # keep old refcats from causing problems. 

946 if name.lower() == "photometry": 

947 if 'coord_raErr' not in refCat.schema: 

948 return 100 

949 else: 

950 return float('nan') 

951 

952 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

953 msg = ("Reference catalog does not contain coordinate errors, " 

954 "and config.astrometryReferenceErr not supplied.") 

955 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

956 self.config, 

957 msg) 

958 

959 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

960 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

961 self.config.astrometryReferenceErr) 

962 

963 if self.config.astrometryReferenceErr is None: 

964 return float('nan') 

965 else: 

966 return self.config.astrometryReferenceErr 

967 

968 def _compute_proper_motion_epoch(self, ccdImageList): 

969 """Return the proper motion correction epoch of the provided images. 

970 

971 Parameters 

972 ---------- 

973 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

974 The images to compute the appropriate epoch for. 

975 

976 Returns 

977 ------- 

978 epoch : `astropy.time.Time` 

979 The date to use for proper motion corrections. 

980 """ 

981 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]), 

982 format="jyear", 

983 scale="tai") 

984 

985 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

986 tract="", match_cut=3.0, 

987 reject_bad_fluxes=False, *, 

988 name="", refObjLoader=None, referenceSelector=None, 

989 fit_function=None, epoch=None): 

990 """Load reference catalog, perform the fit, and return the result. 

991 

992 Parameters 

993 ---------- 

994 associations : `lsst.jointcal.Associations` 

995 The star/reference star associations to fit. 

996 defaultFilter : `lsst.afw.image.FilterLabel` 

997 filter to load from reference catalog. 

998 center : `lsst.geom.SpherePoint` 

999 ICRS center of field to load from reference catalog. 

1000 radius : `lsst.geom.Angle` 

1001 On-sky radius to load from reference catalog. 

1002 name : `str` 

1003 Name of thing being fit: "astrometry" or "photometry". 

1004 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

1005 Reference object loader to use to load a reference catalog. 

1006 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1007 Selector to use to pick objects from the loaded reference catalog. 

1008 fit_function : callable 

1009 Function to call to perform fit (takes Associations object). 

1010 tract : `str`, optional 

1011 Name of tract currently being fit. 

1012 match_cut : `float`, optional 

1013 Radius in arcseconds to find cross-catalog matches to during 

1014 associations.associateCatalogs. 

1015 reject_bad_fluxes : `bool`, optional 

1016 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

1017 epoch : `astropy.time.Time`, optional 

1018 Epoch to which to correct refcat proper motion and parallax, 

1019 or `None` to not apply such corrections. 

1020 

1021 Returns 

1022 ------- 

1023 result : `Photometry` or `Astrometry` 

1024 Result of `fit_function()` 

1025 """ 

1026 self.log.info("====== Now processing %s...", name) 

1027 # TODO: this should not print "trying to invert a singular transformation:" 

1028 # if it does that, something's not right about the WCS... 

1029 associations.associateCatalogs(match_cut) 

1030 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name, 

1031 associations.fittedStarListSize()) 

1032 

1033 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

1034 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

1035 center, radius, defaultFilter, 

1036 applyColorterms=applyColorterms, 

1037 epoch=epoch) 

1038 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

1039 

1040 associations.collectRefStars(refCat, 

1041 self.config.matchCut*lsst.geom.arcseconds, 

1042 fluxField, 

1043 refCoordinateErr=refCoordErr, 

1044 rejectBadFluxes=reject_bad_fluxes) 

1045 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name, 

1046 associations.refStarListSize()) 

1047 

1048 associations.prepareFittedStars(self.config.minMeasurements) 

1049 

1050 self._check_star_lists(associations, name) 

1051 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name, 

1052 associations.nFittedStarsWithAssociatedRefStar()) 

1053 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name, 

1054 associations.fittedStarListSize()) 

1055 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name, 

1056 associations.nCcdImagesValidForFit()) 

1057 

1058 fit_profile_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else '' 

1059 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel) 

1060 with lsst.utils.timer.profile(fit_profile_file): 

1061 result = fit_function(associations, dataName) 

1062 # TODO DM-12446: turn this into a "butler save" somehow. 

1063 # Save reference and measurement chi2 contributions for this data 

1064 if self.config.writeChi2FilesInitialFinal: 

1065 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

1066 result.fit.saveChi2Contributions(baseName+"{type}") 

1067 self.log.info("Wrote chi2 contributions files: %s", baseName) 

1068 

1069 return result 

1070 

1071 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, 

1072 applyColorterms=False, epoch=None): 

1073 """Load the necessary reference catalog sources, convert fluxes to 

1074 correct units, and apply color term corrections if requested. 

1075 

1076 Parameters 

1077 ---------- 

1078 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

1079 The reference catalog loader to use to get the data. 

1080 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1081 Source selector to apply to loaded reference catalog. 

1082 center : `lsst.geom.SpherePoint` 

1083 The center around which to load sources. 

1084 radius : `lsst.geom.Angle` 

1085 The radius around ``center`` to load sources in. 

1086 filterLabel : `lsst.afw.image.FilterLabel` 

1087 The camera filter to load fluxes for. 

1088 applyColorterms : `bool` 

1089 Apply colorterm corrections to the refcat for ``filterName``? 

1090 epoch : `astropy.time.Time`, optional 

1091 Epoch to which to correct refcat proper motion and parallax, 

1092 or `None` to not apply such corrections. 

1093 

1094 Returns 

1095 ------- 

1096 refCat : `lsst.afw.table.SimpleCatalog` 

1097 The loaded reference catalog. 

1098 fluxField : `str` 

1099 The name of the reference catalog flux field appropriate for ``filterName``. 

1100 """ 

1101 skyCircle = refObjLoader.loadSkyCircle(center, 

1102 radius, 

1103 filterLabel.bandLabel, 

1104 epoch=epoch) 

1105 

1106 selected = referenceSelector.run(skyCircle.refCat) 

1107 # Need memory contiguity to get reference filters as a vector. 

1108 if not selected.sourceCat.isContiguous(): 

1109 refCat = selected.sourceCat.copy(deep=True) 

1110 else: 

1111 refCat = selected.sourceCat 

1112 

1113 if applyColorterms: 

1114 refCatName = refObjLoader.name 

1115 self.log.info("Applying color terms for physical filter=%r reference catalog=%s", 

1116 filterLabel.physicalLabel, refCatName) 

1117 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel, 

1118 refCatName, 

1119 doRaise=True) 

1120 

1121 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

1122 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

1123 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

1124 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

1125 

1126 return refCat, skyCircle.fluxField 

1127 

1128 def _check_star_lists(self, associations, name): 

1129 # TODO: these should be len(blah), but we need this properly wrapped first. 

1130 if associations.nCcdImagesValidForFit() == 0: 

1131 raise RuntimeError('No images in the ccdImageList!') 

1132 if associations.fittedStarListSize() == 0: 

1133 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

1134 if associations.refStarListSize() == 0: 

1135 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

1136 

1137 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

1138 """Compute chi2, log it, validate the model, and return chi2. 

1139 

1140 Parameters 

1141 ---------- 

1142 associations : `lsst.jointcal.Associations` 

1143 The star/reference star associations to fit. 

1144 fit : `lsst.jointcal.FitterBase` 

1145 The fitter to use for minimization. 

1146 model : `lsst.jointcal.Model` 

1147 The model being fit. 

1148 chi2Label : `str` 

1149 Label to describe the chi2 (e.g. "Initialized", "Final"). 

1150 writeChi2Name : `str`, optional 

1151 Filename prefix to write the chi2 contributions to. 

1152 Do not supply an extension: an appropriate one will be added. 

1153 

1154 Returns 

1155 ------- 

1156 chi2: `lsst.jointcal.Chi2Accumulator` 

1157 The chi2 object for the current fitter and model. 

1158 

1159 Raises 

1160 ------ 

1161 FloatingPointError 

1162 Raised if chi2 is infinite or NaN. 

1163 ValueError 

1164 Raised if the model is not valid. 

1165 """ 

1166 if writeChi2Name is not None: 

1167 fullpath = self._getDebugPath(writeChi2Name) 

1168 fit.saveChi2Contributions(fullpath+"{type}") 

1169 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

1170 

1171 chi2 = fit.computeChi2() 

1172 self.log.info("%s %s", chi2Label, chi2) 

1173 self._check_stars(associations) 

1174 if not np.isfinite(chi2.chi2): 

1175 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

1176 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

1177 raise ValueError("Model is not valid: check log messages for warnings.") 

1178 return chi2 

1179 

1180 def _fit_photometry(self, associations, dataName=None): 

1181 """ 

1182 Fit the photometric data. 

1183 

1184 Parameters 

1185 ---------- 

1186 associations : `lsst.jointcal.Associations` 

1187 The star/reference star associations to fit. 

1188 dataName : `str` 

1189 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1190 identifying debugging files. 

1191 

1192 Returns 

1193 ------- 

1194 fit_result : `namedtuple` 

1195 fit : `lsst.jointcal.PhotometryFit` 

1196 The photometric fitter used to perform the fit. 

1197 model : `lsst.jointcal.PhotometryModel` 

1198 The photometric model that was fit. 

1199 """ 

1200 self.log.info("=== Starting photometric fitting...") 

1201 

1202 # TODO: should use pex.config.RegistryField here (see DM-9195) 

1203 if self.config.photometryModel == "constrainedFlux": 

1204 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

1205 self.focalPlaneBBox, 

1206 visitOrder=self.config.photometryVisitOrder, 

1207 errorPedestal=self.config.photometryErrorPedestal) 

1208 # potentially nonlinear problem, so we may need a line search to converge. 

1209 doLineSearch = self.config.allowLineSearch 

1210 elif self.config.photometryModel == "constrainedMagnitude": 

1211 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

1212 self.focalPlaneBBox, 

1213 visitOrder=self.config.photometryVisitOrder, 

1214 errorPedestal=self.config.photometryErrorPedestal) 

1215 # potentially nonlinear problem, so we may need a line search to converge. 

1216 doLineSearch = self.config.allowLineSearch 

1217 elif self.config.photometryModel == "simpleFlux": 

1218 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

1219 errorPedestal=self.config.photometryErrorPedestal) 

1220 doLineSearch = False # purely linear in model parameters, so no line search needed 

1221 elif self.config.photometryModel == "simpleMagnitude": 

1222 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

1223 errorPedestal=self.config.photometryErrorPedestal) 

1224 doLineSearch = False # purely linear in model parameters, so no line search needed 

1225 

1226 fit = lsst.jointcal.PhotometryFit(associations, model) 

1227 # TODO DM-12446: turn this into a "butler save" somehow. 

1228 # Save reference and measurement chi2 contributions for this data 

1229 if self.config.writeChi2FilesInitialFinal: 

1230 baseName = f"photometry_initial_chi2-{dataName}" 

1231 else: 

1232 baseName = None 

1233 if self.config.writeInitialModel: 

1234 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt") 

1235 writeModel(model, fullpath, self.log) 

1236 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

1237 

1238 def getChi2Name(whatToFit): 

1239 if self.config.writeChi2FilesOuterLoop: 

1240 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

1241 else: 

1242 return None 

1243 

1244 # The constrained model needs the visit transform fit first; the chip 

1245 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

1246 if self.config.writeInitMatrix: 

1247 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}") 

1248 else: 

1249 dumpMatrixFile = "" 

1250 if self.config.photometryModel.startswith("constrained"): 

1251 # no line search: should be purely (or nearly) linear, 

1252 # and we want a large step size to initialize with. 

1253 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

1254 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

1255 writeChi2Name=getChi2Name("ModelVisit")) 

1256 dumpMatrixFile = "" # so we don't redo the output on the next step 

1257 

1258 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

1259 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

1260 writeChi2Name=getChi2Name("Model")) 

1261 

1262 fit.minimize("Fluxes") # no line search: always purely linear. 

1263 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

1264 writeChi2Name=getChi2Name("Fluxes")) 

1265 

1266 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

1267 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

1268 writeChi2Name=getChi2Name("ModelFluxes")) 

1269 

1270 model.freezeErrorTransform() 

1271 self.log.debug("Photometry error scales are frozen.") 

1272 

1273 chi2 = self._iterate_fit(associations, 

1274 fit, 

1275 self.config.maxPhotometrySteps, 

1276 "photometry", 

1277 "Model Fluxes", 

1278 doRankUpdate=self.config.photometryDoRankUpdate, 

1279 doLineSearch=doLineSearch, 

1280 dataName=dataName) 

1281 

1282 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

1283 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

1284 return Photometry(fit, model) 

1285 

1286 def _fit_astrometry(self, associations, dataName=None): 

1287 """ 

1288 Fit the astrometric data. 

1289 

1290 Parameters 

1291 ---------- 

1292 associations : `lsst.jointcal.Associations` 

1293 The star/reference star associations to fit. 

1294 dataName : `str` 

1295 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1296 identifying debugging files. 

1297 

1298 Returns 

1299 ------- 

1300 fit_result : `namedtuple` 

1301 fit : `lsst.jointcal.AstrometryFit` 

1302 The astrometric fitter used to perform the fit. 

1303 model : `lsst.jointcal.AstrometryModel` 

1304 The astrometric model that was fit. 

1305 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

1306 The model for the sky to tangent plane projection that was used in the fit. 

1307 """ 

1308 

1309 self.log.info("=== Starting astrometric fitting...") 

1310 

1311 associations.deprojectFittedStars() 

1312 

1313 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1314 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1315 # them so carefully? 

1316 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1317 

1318 if self.config.astrometryModel == "constrained": 

1319 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1320 sky_to_tan_projection, 

1321 chipOrder=self.config.astrometryChipOrder, 

1322 visitOrder=self.config.astrometryVisitOrder) 

1323 elif self.config.astrometryModel == "simple": 

1324 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1325 sky_to_tan_projection, 

1326 self.config.useInputWcs, 

1327 nNotFit=0, 

1328 order=self.config.astrometrySimpleOrder) 

1329 

1330 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1331 # TODO DM-12446: turn this into a "butler save" somehow. 

1332 # Save reference and measurement chi2 contributions for this data 

1333 if self.config.writeChi2FilesInitialFinal: 

1334 baseName = f"astrometry_initial_chi2-{dataName}" 

1335 else: 

1336 baseName = None 

1337 if self.config.writeInitialModel: 

1338 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt") 

1339 writeModel(model, fullpath, self.log) 

1340 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1341 

1342 def getChi2Name(whatToFit): 

1343 if self.config.writeChi2FilesOuterLoop: 

1344 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1345 else: 

1346 return None 

1347 

1348 if self.config.writeInitMatrix: 

1349 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}") 

1350 else: 

1351 dumpMatrixFile = "" 

1352 # The constrained model needs the visit transform fit first; the chip 

1353 # transform is initialized from the detector's cameraGeom, so it's close. 

1354 if self.config.astrometryModel == "constrained": 

1355 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1356 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1357 writeChi2Name=getChi2Name("DistortionsVisit")) 

1358 dumpMatrixFile = "" # so we don't redo the output on the next step 

1359 

1360 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1361 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1362 writeChi2Name=getChi2Name("Distortions")) 

1363 

1364 fit.minimize("Positions") 

1365 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1366 writeChi2Name=getChi2Name("Positions")) 

1367 

1368 fit.minimize("Distortions Positions") 

1369 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1370 writeChi2Name=getChi2Name("DistortionsPositions")) 

1371 

1372 chi2 = self._iterate_fit(associations, 

1373 fit, 

1374 self.config.maxAstrometrySteps, 

1375 "astrometry", 

1376 "Distortions Positions", 

1377 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance, 

1378 doRankUpdate=self.config.astrometryDoRankUpdate, 

1379 dataName=dataName) 

1380 

1381 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1382 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1383 

1384 return Astrometry(fit, model, sky_to_tan_projection) 

1385 

1386 def _check_stars(self, associations): 

1387 """Count measured and reference stars per ccd and warn/log them.""" 

1388 for ccdImage in associations.getCcdImageList(): 

1389 nMeasuredStars, nRefStars = ccdImage.countStars() 

1390 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1391 ccdImage.getName(), nMeasuredStars, nRefStars) 

1392 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1393 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)", 

1394 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1395 if nRefStars < self.config.minRefStarsPerCcd: 

1396 self.log.warning("ccdImage %s has only %s RefStars (desired %s)", 

1397 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1398 

1399 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1400 dataName="", 

1401 sigmaRelativeTolerance=0, 

1402 doRankUpdate=True, 

1403 doLineSearch=False): 

1404 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1405 

1406 Parameters 

1407 ---------- 

1408 associations : `lsst.jointcal.Associations` 

1409 The star/reference star associations to fit. 

1410 fitter : `lsst.jointcal.FitterBase` 

1411 The fitter to use for minimization. 

1412 max_steps : `int` 

1413 Maximum number of steps to run outlier rejection before declaring 

1414 convergence failure. 

1415 name : {'photometry' or 'astrometry'} 

1416 What type of data are we fitting (for logs and debugging files). 

1417 whatToFit : `str` 

1418 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1419 dataName : `str`, optional 

1420 Descriptive name for this dataset (e.g. tract and filter), 

1421 for debugging. 

1422 sigmaRelativeTolerance : `float`, optional 

1423 Convergence tolerance for the fractional change in the chi2 cut 

1424 level for determining outliers. If set to zero, iterations will 

1425 continue until there are no outliers. 

1426 doRankUpdate : `bool`, optional 

1427 Do an Eigen rank update during minimization, or recompute the full 

1428 matrix and gradient? 

1429 doLineSearch : `bool`, optional 

1430 Do a line search for the optimum step during minimization? 

1431 

1432 Returns 

1433 ------- 

1434 chi2: `lsst.jointcal.Chi2Statistic` 

1435 The final chi2 after the fit converges, or is forced to end. 

1436 

1437 Raises 

1438 ------ 

1439 FloatingPointError 

1440 Raised if the fitter fails with a non-finite value. 

1441 RuntimeError 

1442 Raised if the fitter fails for some other reason; 

1443 log messages will provide further details. 

1444 """ 

1445 if self.config.writeInitMatrix: 

1446 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}") 

1447 else: 

1448 dumpMatrixFile = "" 

1449 oldChi2 = lsst.jointcal.Chi2Statistic() 

1450 oldChi2.chi2 = float("inf") 

1451 for i in range(max_steps): 

1452 if self.config.writeChi2FilesOuterLoop: 

1453 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1454 else: 

1455 writeChi2Name = None 

1456 result = fitter.minimize(whatToFit, 

1457 self.config.outlierRejectSigma, 

1458 sigmaRelativeTolerance=sigmaRelativeTolerance, 

1459 doRankUpdate=doRankUpdate, 

1460 doLineSearch=doLineSearch, 

1461 dumpMatrixFile=dumpMatrixFile) 

1462 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1463 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1464 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1465 

1466 if result == MinimizeResult.Converged: 

1467 if doRankUpdate: 

1468 self.log.debug("fit has converged - no more outliers - redo minimization " 

1469 "one more time in case we have lost accuracy in rank update.") 

1470 # Redo minimization one more time in case we have lost accuracy in rank update 

1471 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma, 

1472 sigmaRelativeTolerance=sigmaRelativeTolerance) 

1473 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1474 

1475 # log a message for a large final chi2, TODO: DM-15247 for something better 

1476 if chi2.chi2/chi2.ndof >= 4.0: 

1477 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1478 

1479 break 

1480 elif result == MinimizeResult.Chi2Increased: 

1481 self.log.warning("Still some outliers remaining but chi2 increased - retry") 

1482 # Check whether the increase was large enough to cause trouble. 

1483 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1484 if chi2Ratio > 1.5: 

1485 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1486 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1487 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1488 # appears that chi2 increases more than a factor of ~2 always 

1489 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1490 # Using 10 as the "failure" threshold gives some room between 

1491 # leaving a warning and bailing early. 

1492 if chi2Ratio > 10: 

1493 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1494 " Try setting one or more of the `writeChi2*` config fields and looking" 

1495 " at how individual star chi2-values evolve during the fit.") 

1496 raise RuntimeError(msg) 

1497 oldChi2 = chi2 

1498 elif result == MinimizeResult.NonFinite: 

1499 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1500 # TODO DM-12446: turn this into a "butler save" somehow. 

1501 fitter.saveChi2Contributions(filename+"{type}") 

1502 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1503 raise FloatingPointError(msg.format(filename)) 

1504 elif result == MinimizeResult.Failed: 

1505 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1506 else: 

1507 raise RuntimeError("Unxepected return code from minimize().") 

1508 else: 

1509 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1510 

1511 return chi2 

1512 

1513 def _make_output(self, ccdImageList, model, func): 

1514 """Return the internal jointcal models converted to the afw 

1515 structures that will be saved to disk. 

1516 

1517 Parameters 

1518 ---------- 

1519 ccdImageList : `lsst.jointcal.CcdImageList` 

1520 The list of CcdImages to get the output for. 

1521 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel` 

1522 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`. 

1523 func : `str` 

1524 The name of the function to call on ``model`` to get the converted 

1525 structure. Must accept an `lsst.jointcal.CcdImage`. 

1526 

1527 Returns 

1528 ------- 

1529 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or 

1530 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`] 

1531 The data to be saved, keyed on (visit, detector). 

1532 """ 

1533 output = {} 

1534 for ccdImage in ccdImageList: 

1535 ccd = ccdImage.ccdId 

1536 visit = ccdImage.visit 

1537 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd) 

1538 output[(visit, ccd)] = getattr(model, func)(ccdImage) 

1539 return output 

1540 

1541 

1542def make_schema_table(): 

1543 """Return an afw SourceTable to use as a base for creating the 

1544 SourceCatalog to insert values from the dataFrame into. 

1545 

1546 Returns 

1547 ------- 

1548 table : `lsst.afw.table.SourceTable` 

1549 Table with schema and slots to use to make SourceCatalogs. 

1550 """ 

1551 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

1552 schema.addField("centroid_x", "D") 

1553 schema.addField("centroid_y", "D") 

1554 schema.addField("centroid_xErr", "F") 

1555 schema.addField("centroid_yErr", "F") 

1556 schema.addField("shape_xx", "D") 

1557 schema.addField("shape_yy", "D") 

1558 schema.addField("shape_xy", "D") 

1559 schema.addField("flux_instFlux", "D") 

1560 schema.addField("flux_instFluxErr", "D") 

1561 table = lsst.afw.table.SourceTable.make(schema) 

1562 table.defineCentroid("centroid") 

1563 table.defineShape("shape") 

1564 return table 

1565 

1566 

1567def get_sourceTable_visit_columns(inColumns, config, sourceSelector): 

1568 """ 

1569 Get the sourceTable_visit columns to load from the catalogs. 

1570 

1571 Parameters 

1572 ---------- 

1573 inColumns : `list` 

1574 List of columns known to be available in the sourceTable_visit. 

1575 config : `JointcalConfig` 

1576 A filled-in config to to help define column names. 

1577 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask` 

1578 A configured source selector to define column names to load. 

1579 

1580 Returns 

1581 ------- 

1582 columns : `list` 

1583 List of columns to read from sourceTable_visit. 

1584 ixxColumns : `list` 

1585 Name of the ixx/iyy/ixy columns. 

1586 """ 

1587 columns = ['visit', 'detector', 

1588 'sourceId', 'x', 'xErr', 'y', 'yErr', 

1589 config.sourceFluxType + '_instFlux', config.sourceFluxType + '_instFluxErr'] 

1590 

1591 if 'ixx' in inColumns: 

1592 # New columns post-DM-31825 

1593 ixxColumns = ['ixx', 'iyy', 'ixy'] 

1594 else: 

1595 # Old columns pre-DM-31825 

1596 ixxColumns = ['Ixx', 'Iyy', 'Ixy'] 

1597 columns.extend(ixxColumns) 

1598 

1599 if sourceSelector.config.doFlags: 

1600 columns.extend(sourceSelector.config.flags.bad) 

1601 if sourceSelector.config.doUnresolved: 

1602 columns.append(sourceSelector.config.unresolved.name) 

1603 if sourceSelector.config.doIsolated: 

1604 columns.append(sourceSelector.config.isolated.parentName) 

1605 columns.append(sourceSelector.config.isolated.nChildName) 

1606 if sourceSelector.config.doRequireFiniteRaDec: 

1607 columns.append(sourceSelector.config.requireFiniteRaDec.raColName) 

1608 columns.append(sourceSelector.config.requireFiniteRaDec.decColName) 

1609 if sourceSelector.config.doRequirePrimary: 

1610 columns.append(sourceSelector.config.requirePrimary.primaryColName) 

1611 

1612 return columns, ixxColumns 

1613 

1614 

1615def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId, 

1616 ixxColumns, sourceFluxType, log): 

1617 """Return an afw SourceCatalog extracted from a visit-level dataframe, 

1618 limited to just one detector. 

1619 

1620 Parameters 

1621 ---------- 

1622 table : `lsst.afw.table.SourceTable` 

1623 Table factory to use to make the SourceCatalog that will be 

1624 populated with data from ``visitCatalog``. 

1625 visitCatalog : `pandas.DataFrame` 

1626 DataFrame to extract a detector catalog from. 

1627 detectorId : `int` 

1628 Numeric id of the detector to extract from ``visitCatalog``. 

1629 ixxColumns : `list` [`str`] 

1630 Names of the ixx/iyy/ixy columns in the catalog. 

1631 sourceFluxType : `str` 

1632 Name of the catalog field to load instFluxes from. 

1633 log : `logging.Logger` 

1634 Logging instance to log to. 

1635 

1636 Returns 

1637 ------- 

1638 catalog : `lsst.afw.table.SourceCatalog`, or `None` 

1639 Detector-level catalog extracted from ``visitCatalog``, or `None` 

1640 if there was no data to load. 

1641 """ 

1642 # map from dataFrame column to afw table column 

1643 mapping = {'x': 'centroid_x', 

1644 'y': 'centroid_y', 

1645 'xErr': 'centroid_xErr', 

1646 'yErr': 'centroid_yErr', 

1647 ixxColumns[0]: 'shape_xx', 

1648 ixxColumns[1]: 'shape_yy', 

1649 ixxColumns[2]: 'shape_xy', 

1650 f'{sourceFluxType}_instFlux': 'flux_instFlux', 

1651 f'{sourceFluxType}_instFluxErr': 'flux_instFluxErr', 

1652 } 

1653 

1654 catalog = lsst.afw.table.SourceCatalog(table) 

1655 matched = visitCatalog['detector'] == detectorId 

1656 n = sum(matched) 

1657 if n == 0: 

1658 return None 

1659 catalog.resize(sum(matched)) 

1660 view = visitCatalog.loc[matched] 

1661 catalog['id'] = view.index 

1662 for dfCol, afwCol in mapping.items(): 

1663 catalog[afwCol] = view[dfCol] 

1664 

1665 log.debug("%d sources selected in visit %d detector %d", 

1666 len(catalog), 

1667 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first 

1668 detectorId) 

1669 return catalog