Coverage for python/lsst/jointcal/jointcal.py: 18%

540 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-09 03:27 -0700

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import dataclasses 

23import collections 

24import os 

25import logging 

26 

27import astropy.time 

28import numpy as np 

29import astropy.units as u 

30 

31import lsst.geom 

32import lsst.utils 

33import lsst.pex.config as pexConfig 

34import lsst.pipe.base as pipeBase 

35from lsst.afw.image import fluxErrFromABMagErr 

36import lsst.afw.cameraGeom 

37import lsst.afw.table 

38from lsst.pipe.base import Instrument 

39from lsst.pipe.tasks.colorterms import ColortermLibrary 

40from lsst.verify import Job, Measurement 

41 

42from lsst.meas.algorithms import (ReferenceObjectLoader, ReferenceSourceSelectorTask, 

43 LoadReferenceObjectsConfig) 

44from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

45 

46import lsst.jointcal 

47from lsst.jointcal import MinimizeResult 

48 

49__all__ = ["JointcalConfig", "JointcalTask"] 

50 

51Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

52Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

53 

54 

55# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

56def add_measurement(job, name, value): 

57 meas = Measurement(job.metrics[name], value) 

58 job.measurements.insert(meas) 

59 

60 

61def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

62 """Lookup function that asserts/hopes that a static calibration dataset 

63 exists in a particular collection, since this task can't provide a single 

64 date/time to use to search for one properly. 

65 

66 This is mostly useful for the ``camera`` dataset, in cases where the task's 

67 quantum dimensions do *not* include something temporal, like ``exposure`` 

68 or ``visit``. 

69 

70 Parameters 

71 ---------- 

72 datasetType : `lsst.daf.butler.DatasetType` 

73 Type of dataset being searched for. 

74 registry : `lsst.daf.butler.Registry` 

75 Data repository registry to search. 

76 quantumDataId : `lsst.daf.butler.DataCoordinate` 

77 Data ID of the quantum this camera should match. 

78 collections : `Iterable` [ `str` ] 

79 Collections that should be searched - but this lookup function works 

80 by ignoring this in favor of a more-or-less hard-coded value. 

81 

82 Returns 

83 ------- 

84 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

85 Iterator over dataset references; should have only one element. 

86 

87 Notes 

88 ----- 

89 This implementation duplicates one in fgcmcal, and is at least quite 

90 similar to another in cp_pipe. This duplicate has the most documentation. 

91 Fixing this is DM-29661. 

92 """ 

93 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

94 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

95 return registry.queryDatasets(datasetType, 

96 dataId=quantumDataId, 

97 collections=[unboundedCollection], 

98 findFirst=True) 

99 

100 

101def lookupVisitRefCats(datasetType, registry, quantumDataId, collections): 

102 """Lookup function that finds all refcats for all visits that overlap a 

103 tract, rather than just the refcats that directly overlap the tract. 

104 

105 Parameters 

106 ---------- 

107 datasetType : `lsst.daf.butler.DatasetType` 

108 Type of dataset being searched for. 

109 registry : `lsst.daf.butler.Registry` 

110 Data repository registry to search. 

111 quantumDataId : `lsst.daf.butler.DataCoordinate` 

112 Data ID of the quantum; expected to be something we can use as a 

113 constraint to query for overlapping visits. 

114 collections : `Iterable` [ `str` ] 

115 Collections to search. 

116 

117 Returns 

118 ------- 

119 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

120 Iterator over refcat references. 

121 """ 

122 refs = set() 

123 # Use .expanded() on the query methods below because we need data IDs with 

124 # regions, both in the outer loop over visits (queryDatasets will expand 

125 # any data ID we give it, but doing it up-front in bulk is much more 

126 # efficient) and in the data IDs of the DatasetRefs this function yields 

127 # (because the RefCatLoader relies on them to do some of its own 

128 # filtering). 

129 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()): 

130 refs.update( 

131 registry.queryDatasets( 

132 datasetType, 

133 collections=collections, 

134 dataId=visit_data_id, 

135 findFirst=True, 

136 ).expanded() 

137 ) 

138 yield from refs 

139 

140 

141class JointcalTaskConnections(pipeBase.PipelineTaskConnections, 

142 dimensions=("skymap", "tract", "instrument", "physical_filter")): 

143 """Middleware input/output connections for jointcal data.""" 

144 inputCamera = pipeBase.connectionTypes.PrerequisiteInput( 

145 doc="The camera instrument that took these observations.", 

146 name="camera", 

147 storageClass="Camera", 

148 dimensions=("instrument",), 

149 isCalibration=True, 

150 lookupFunction=lookupStaticCalibrations, 

151 ) 

152 inputSourceTableVisit = pipeBase.connectionTypes.Input( 

153 doc="Source table in parquet format, per visit", 

154 name="sourceTable_visit", 

155 storageClass="DataFrame", 

156 dimensions=("instrument", "visit"), 

157 deferLoad=True, 

158 multiple=True, 

159 ) 

160 inputVisitSummary = pipeBase.connectionTypes.Input( 

161 doc=("Per-visit consolidated exposure metadata built from calexps. " 

162 "These catalogs use detector id for the id and must be sorted for " 

163 "fast lookups of a detector."), 

164 name="visitSummary", 

165 storageClass="ExposureCatalog", 

166 dimensions=("instrument", "visit"), 

167 deferLoad=True, 

168 multiple=True, 

169 ) 

170 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

171 doc="The astrometry reference catalog to match to loaded input catalog sources.", 

172 name="gaia_dr2_20200414", 

173 storageClass="SimpleCatalog", 

174 dimensions=("skypix",), 

175 deferLoad=True, 

176 multiple=True, 

177 lookupFunction=lookupVisitRefCats, 

178 ) 

179 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

180 doc="The photometry reference catalog to match to loaded input catalog sources.", 

181 name="ps1_pv3_3pi_20170110", 

182 storageClass="SimpleCatalog", 

183 dimensions=("skypix",), 

184 deferLoad=True, 

185 multiple=True, 

186 lookupFunction=lookupVisitRefCats, 

187 ) 

188 

189 outputWcs = pipeBase.connectionTypes.Output( 

190 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model." 

191 " These catalogs only contain entries for detectors with an output, and use" 

192 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

193 name="jointcalSkyWcsCatalog", 

194 storageClass="ExposureCatalog", 

195 dimensions=("instrument", "visit", "skymap", "tract"), 

196 multiple=True 

197 ) 

198 outputPhotoCalib = pipeBase.connectionTypes.Output( 

199 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model." 

200 " These catalogs only contain entries for detectors with an output, and use" 

201 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

202 name="jointcalPhotoCalibCatalog", 

203 storageClass="ExposureCatalog", 

204 dimensions=("instrument", "visit", "skymap", "tract"), 

205 multiple=True 

206 ) 

207 

208 # measurements of metrics 

209 # The vars() trick used here allows us to set class attributes 

210 # programatically. Taken from: 

211 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class 

212 for name in ("astrometry", "photometry"): 

213 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output( 

214 doc=f"The number of cross-matched fittedStars for {name}", 

215 name=f"metricvalue_jointcal_{name}_matched_fittedStars", 

216 storageClass="MetricValue", 

217 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

218 ) 

219 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output( 

220 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.", 

221 name=f"metricvalue_jointcal_{name}_collected_refStars", 

222 storageClass="MetricValue", 

223 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

224 ) 

225 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output( 

226 doc=f"The number of {name} reference stars matched to fittedStars.", 

227 name=f"metricvalue_jointcal_{name}_prepared_refStars", 

228 storageClass="MetricValue", 

229 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

230 ) 

231 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output( 

232 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.", 

233 name=f"metricvalue_jointcal_{name}_prepared_fittedStars", 

234 storageClass="MetricValue", 

235 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

236 ) 

237 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output( 

238 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.", 

239 name=f"metricvalue_jointcal_{name}_prepared_ccdImages", 

240 storageClass="MetricValue", 

241 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

242 ) 

243 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output( 

244 doc=f"The final chi2 of the {name} fit.", 

245 name=f"metricvalue_jointcal_{name}_final_chi2", 

246 storageClass="MetricValue", 

247 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

248 ) 

249 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output( 

250 doc=f"The number of degrees of freedom of the fitted {name}.", 

251 name=f"metricvalue_jointcal_{name}_final_ndof", 

252 storageClass="MetricValue", 

253 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

254 ) 

255 

256 def __init__(self, *, config=None): 

257 super().__init__(config=config) 

258 # When we are only doing one of astrometry or photometry, we don't 

259 # need the reference catalog or produce the outputs for the other. 

260 # This informs the middleware of that when the QuantumGraph is 

261 # generated, so we don't block on getting something we won't need or 

262 # create an expectation that downstream tasks will be able to consume 

263 # something we won't produce. 

264 if not config.doAstrometry: 

265 self.prerequisiteInputs.remove("astrometryRefCat") 

266 self.outputs.remove("outputWcs") 

267 for key in list(self.outputs): 

268 if "metricvalue_jointcal_astrometry" in key: 

269 self.outputs.remove(key) 

270 if not config.doPhotometry: 

271 self.prerequisiteInputs.remove("photometryRefCat") 

272 self.outputs.remove("outputPhotoCalib") 

273 for key in list(self.outputs): 

274 if "metricvalue_jointcal_photometry" in key: 

275 self.outputs.remove(key) 

276 

277 

278class JointcalConfig(pipeBase.PipelineTaskConfig, 

279 pipelineConnections=JointcalTaskConnections): 

280 """Configuration for JointcalTask""" 

281 

282 doAstrometry = pexConfig.Field( 

283 doc="Fit astrometry and write the fitted result.", 

284 dtype=bool, 

285 default=True 

286 ) 

287 doPhotometry = pexConfig.Field( 

288 doc="Fit photometry and write the fitted result.", 

289 dtype=bool, 

290 default=True 

291 ) 

292 sourceFluxType = pexConfig.Field( 

293 dtype=str, 

294 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

295 default='apFlux_12_0' 

296 ) 

297 positionErrorPedestal = pexConfig.Field( 

298 doc="Systematic term to apply to the measured position error (pixels)", 

299 dtype=float, 

300 default=0.02, 

301 ) 

302 photometryErrorPedestal = pexConfig.Field( 

303 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

304 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

305 dtype=float, 

306 default=0.0, 

307 ) 

308 # TODO: DM-6885 matchCut should be an geom.Angle 

309 matchCut = pexConfig.Field( 

310 doc="Matching radius between fitted and reference stars (arcseconds)", 

311 dtype=float, 

312 default=3.0, 

313 ) 

314 minMeasurements = pexConfig.Field( 

315 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

316 dtype=int, 

317 default=2, 

318 ) 

319 minMeasuredStarsPerCcd = pexConfig.Field( 

320 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

321 dtype=int, 

322 default=100, 

323 ) 

324 minRefStarsPerCcd = pexConfig.Field( 

325 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

326 dtype=int, 

327 default=30, 

328 ) 

329 allowLineSearch = pexConfig.Field( 

330 doc="Allow a line search during minimization, if it is reasonable for the model" 

331 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

332 dtype=bool, 

333 default=False 

334 ) 

335 astrometrySimpleOrder = pexConfig.Field( 

336 doc="Polynomial order for fitting the simple astrometry model.", 

337 dtype=int, 

338 default=3, 

339 ) 

340 astrometryChipOrder = pexConfig.Field( 

341 doc="Order of the per-chip transform for the constrained astrometry model.", 

342 dtype=int, 

343 default=1, 

344 ) 

345 astrometryVisitOrder = pexConfig.Field( 

346 doc="Order of the per-visit transform for the constrained astrometry model.", 

347 dtype=int, 

348 default=5, 

349 ) 

350 useInputWcs = pexConfig.Field( 

351 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

352 dtype=bool, 

353 default=True, 

354 ) 

355 astrometryModel = pexConfig.ChoiceField( 

356 doc="Type of model to fit to astrometry", 

357 dtype=str, 

358 default="constrained", 

359 allowed={"simple": "One polynomial per ccd", 

360 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

361 ) 

362 photometryModel = pexConfig.ChoiceField( 

363 doc="Type of model to fit to photometry", 

364 dtype=str, 

365 default="constrainedMagnitude", 

366 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

367 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

368 " fitting in flux space.", 

369 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

370 " fitting in magnitude space.", 

371 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

372 " fitting in magnitude space.", 

373 } 

374 ) 

375 applyColorTerms = pexConfig.Field( 

376 doc="Apply photometric color terms to reference stars?" 

377 "Requires that colorterms be set to a ColortermLibrary", 

378 dtype=bool, 

379 default=False 

380 ) 

381 colorterms = pexConfig.ConfigField( 

382 doc="Library of photometric reference catalog name to color term dict.", 

383 dtype=ColortermLibrary, 

384 ) 

385 photometryVisitOrder = pexConfig.Field( 

386 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

387 dtype=int, 

388 default=7, 

389 ) 

390 photometryDoRankUpdate = pexConfig.Field( 

391 doc=("Do the rank update step during minimization. " 

392 "Skipping this can help deal with models that are too non-linear."), 

393 dtype=bool, 

394 default=True, 

395 ) 

396 astrometryDoRankUpdate = pexConfig.Field( 

397 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

398 "Skipping this can help deal with models that are too non-linear."), 

399 dtype=bool, 

400 default=True, 

401 ) 

402 outlierRejectSigma = pexConfig.Field( 

403 doc="How many sigma to reject outliers at during minimization.", 

404 dtype=float, 

405 default=5.0, 

406 ) 

407 astrometryOutlierRelativeTolerance = pexConfig.Field( 

408 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will " 

409 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set " 

410 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002" 

411 "as a balance between a shorter minimization runtime and achieving a final fitted model that is" 

412 "close to the solution found when removing all outliers."), 

413 dtype=float, 

414 default=0, 

415 ) 

416 maxPhotometrySteps = pexConfig.Field( 

417 doc="Maximum number of minimize iterations to take when fitting photometry.", 

418 dtype=int, 

419 default=20, 

420 ) 

421 maxAstrometrySteps = pexConfig.Field( 

422 doc="Maximum number of minimize iterations to take when fitting astrometry.", 

423 dtype=int, 

424 default=20, 

425 ) 

426 astrometryRefObjLoader = pexConfig.ConfigField( 

427 dtype=LoadReferenceObjectsConfig, 

428 doc="Reference object loader for astrometric fit", 

429 ) 

430 photometryRefObjLoader = pexConfig.ConfigField( 

431 dtype=LoadReferenceObjectsConfig, 

432 doc="Reference object loader for photometric fit", 

433 ) 

434 sourceSelector = sourceSelectorRegistry.makeField( 

435 doc="How to select sources for cross-matching", 

436 default="science" 

437 ) 

438 astrometryReferenceSelector = pexConfig.ConfigurableField( 

439 target=ReferenceSourceSelectorTask, 

440 doc="How to down-select the loaded astrometry reference catalog.", 

441 ) 

442 photometryReferenceSelector = pexConfig.ConfigurableField( 

443 target=ReferenceSourceSelectorTask, 

444 doc="How to down-select the loaded photometry reference catalog.", 

445 ) 

446 astrometryReferenceErr = pexConfig.Field( 

447 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

448 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

449 "If specified, overrides any existing `coord_*Err` values."), 

450 dtype=float, 

451 default=None, 

452 optional=True 

453 ) 

454 

455 # configs for outputting debug information 

456 writeInitMatrix = pexConfig.Field( 

457 dtype=bool, 

458 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

459 "Output files will be written to `config.debugOutputPath` and will " 

460 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. " 

461 "Note that these files are the dense versions of the matrix, and so may be very large."), 

462 default=False 

463 ) 

464 writeChi2FilesInitialFinal = pexConfig.Field( 

465 dtype=bool, 

466 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. " 

467 "Output files will be written to `config.debugOutputPath` and will " 

468 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."), 

469 default=False 

470 ) 

471 writeChi2FilesOuterLoop = pexConfig.Field( 

472 dtype=bool, 

473 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. " 

474 "Output files will be written to `config.debugOutputPath` and will " 

475 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."), 

476 default=False 

477 ) 

478 writeInitialModel = pexConfig.Field( 

479 dtype=bool, 

480 doc=("Write the pre-initialization model to text files, for debugging. " 

481 "Output files will be written to `config.debugOutputPath` and will be " 

482 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."), 

483 default=False 

484 ) 

485 debugOutputPath = pexConfig.Field( 

486 dtype=str, 

487 default=".", 

488 doc=("Path to write debug output files to. Used by " 

489 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

490 ) 

491 detailedProfile = pexConfig.Field( 

492 dtype=bool, 

493 default=False, 

494 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting" 

495 ) 

496 

497 def validate(self): 

498 super().validate() 

499 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

500 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

501 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

502 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

503 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

504 "applyColorTerms=True will be ignored.") 

505 logging.getLogger("lsst.jointcal").warning(msg) 

506 

507 def setDefaults(self): 

508 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing. 

509 self.sourceSelector["science"].doUnresolved = True 

510 self.sourceSelector["science"].unresolved.name = "extendedness" 

511 # with dependable signal to noise ratio. 

512 self.sourceSelector["science"].doSignalToNoise = True 

513 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

514 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

515 self.sourceSelector["science"].signalToNoise.minimum = 10. 

516 # Base SNR selection on `sourceFluxType` because that is the flux that jointcal fits. 

517 self.sourceSelector["science"].signalToNoise.fluxField = f"{self.sourceFluxType}_instFlux" 

518 self.sourceSelector["science"].signalToNoise.errField = f"{self.sourceFluxType}_instFluxErr" 

519 # Do not trust blended sources" aperture fluxes which also depend on seeing. 

520 self.sourceSelector["science"].doIsolated = True 

521 self.sourceSelector["science"].isolated.parentName = "parentSourceId" 

522 self.sourceSelector["science"].isolated.nChildName = "deblend_nChild" 

523 # Do not trust either flux or centroid measurements with flags, 

524 # chosen from the usual QA flags for stars) 

525 self.sourceSelector["science"].doFlags = True 

526 badFlags = ["pixelFlags_edge", 

527 "pixelFlags_saturated", 

528 "pixelFlags_interpolatedCenter", 

529 "pixelFlags_interpolated", 

530 "pixelFlags_crCenter", 

531 "pixelFlags_bad", 

532 "hsmPsfMoments_flag", 

533 f"{self.sourceFluxType}_flag", 

534 ] 

535 self.sourceSelector["science"].flags.bad = badFlags 

536 self.sourceSelector["science"].doRequireFiniteRaDec = True 

537 self.sourceSelector["science"].requireFiniteRaDec.raColName = "ra" 

538 self.sourceSelector["science"].requireFiniteRaDec.decColName = "decl" 

539 

540 # Use Gaia-DR2 with proper motions for astrometry; phot_g_mean is the 

541 # primary Gaia band, but is not like any normal photometric band. 

542 self.astrometryRefObjLoader.requireProperMotion = True 

543 self.astrometryRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

544 

545 

546def writeModel(model, filename, log): 

547 """Write model to outfile.""" 

548 with open(filename, "w") as file: 

549 file.write(repr(model)) 

550 log.info("Wrote %s to file: %s", model, filename) 

551 

552 

553@dataclasses.dataclass 

554class JointcalInputData: 

555 """The input data jointcal needs for each detector/visit.""" 

556 visit: int 

557 """The visit identifier of this exposure.""" 

558 catalog: lsst.afw.table.SourceCatalog 

559 """The catalog derived from this exposure.""" 

560 visitInfo: lsst.afw.image.VisitInfo 

561 """The VisitInfo of this exposure.""" 

562 detector: lsst.afw.cameraGeom.Detector 

563 """The detector of this exposure.""" 

564 photoCalib: lsst.afw.image.PhotoCalib 

565 """The photometric calibration of this exposure.""" 

566 wcs: lsst.afw.geom.skyWcs 

567 """The WCS of this exposure.""" 

568 bbox: lsst.geom.Box2I 

569 """The bounding box of this exposure.""" 

570 filter: lsst.afw.image.FilterLabel 

571 """The filter of this exposure.""" 

572 

573 

574class JointcalTask(pipeBase.PipelineTask): 

575 """Astrometricly and photometricly calibrate across multiple visits of the 

576 same field. 

577 """ 

578 

579 ConfigClass = JointcalConfig 

580 _DefaultName = "jointcal" 

581 

582 def __init__(self, **kwargs): 

583 super().__init__(**kwargs) 

584 self.makeSubtask("sourceSelector") 

585 if self.config.doAstrometry: 

586 self.makeSubtask("astrometryReferenceSelector") 

587 else: 

588 self.astrometryRefObjLoader = None 

589 if self.config.doPhotometry: 

590 self.makeSubtask("photometryReferenceSelector") 

591 else: 

592 self.photometryRefObjLoader = None 

593 

594 # To hold various computed metrics for use by tests 

595 self.job = Job.load_metrics_package(subset='jointcal') 

596 

597 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

598 # We override runQuantum to set up the refObjLoaders and write the 

599 # outputs to the correct refs. 

600 inputs = butlerQC.get(inputRefs) 

601 # We want the tract number for writing debug files 

602 tract = butlerQC.quantum.dataId['tract'] 

603 if self.config.doAstrometry: 

604 self.astrometryRefObjLoader = ReferenceObjectLoader( 

605 dataIds=[ref.datasetRef.dataId 

606 for ref in inputRefs.astrometryRefCat], 

607 refCats=inputs.pop('astrometryRefCat'), 

608 config=self.config.astrometryRefObjLoader, 

609 name=self.config.connections.astrometryRefCat, 

610 log=self.log) 

611 if self.config.doPhotometry: 

612 self.photometryRefObjLoader = ReferenceObjectLoader( 

613 dataIds=[ref.datasetRef.dataId 

614 for ref in inputRefs.photometryRefCat], 

615 refCats=inputs.pop('photometryRefCat'), 

616 config=self.config.photometryRefObjLoader, 

617 name=self.config.connections.photometryRefCat, 

618 log=self.log) 

619 outputs = self.run(**inputs, tract=tract) 

620 self._put_metrics(butlerQC, outputs.job, outputRefs) 

621 if self.config.doAstrometry: 

622 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs, 

623 inputs['inputCamera'], "setWcs") 

624 if self.config.doPhotometry: 

625 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib, 

626 inputs['inputCamera'], "setPhotoCalib") 

627 

628 def _put_metrics(self, butlerQC, job, outputRefs): 

629 """Persist all measured metrics stored in a job. 

630 

631 Parameters 

632 ---------- 

633 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

634 A butler which is specialized to operate in the context of a 

635 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

636 job : `lsst.verify.job.Job` 

637 Measurements of metrics to persist. 

638 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

639 The DatasetRefs to persist the data to. 

640 """ 

641 for key in job.measurements.keys(): 

642 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', ''))) 

643 

644 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter): 

645 """Persist the output datasets to their appropriate datarefs. 

646 

647 Parameters 

648 ---------- 

649 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

650 A butler which is specialized to operate in the context of a 

651 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

652 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or 

653 `dict` [`tuple, `lsst.afw.image.PhotoCalib`] 

654 The fitted objects to persist. 

655 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

656 The DatasetRefs to persist the data to. 

657 camera : `lsst.afw.cameraGeom.Camera` 

658 The camera for this instrument, to get detector ids from. 

659 setter : `str` 

660 The method to call on the ExposureCatalog to set each output. 

661 """ 

662 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

663 schema.addField('visit', type='L', doc='Visit number') 

664 

665 def new_catalog(visit, size): 

666 """Return an catalog ready to be filled with appropriate output.""" 

667 catalog = lsst.afw.table.ExposureCatalog(schema) 

668 catalog.resize(size) 

669 catalog['visit'] = visit 

670 metadata = lsst.daf.base.PropertyList() 

671 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

672 metadata.add("COMMENT", "Only detectors with data have entries.") 

673 return catalog 

674 

675 # count how many detectors have output for each visit 

676 detectors_per_visit = collections.defaultdict(int) 

677 for key in outputs: 

678 # key is (visit, detector_id), and we only need visit here 

679 detectors_per_visit[key[0]] += 1 

680 

681 for ref in outputRefs: 

682 visit = ref.dataId['visit'] 

683 catalog = new_catalog(visit, detectors_per_visit[visit]) 

684 # Iterate over every detector and skip the ones we don't have output for. 

685 i = 0 

686 for detector in camera: 

687 detectorId = detector.getId() 

688 key = (ref.dataId['visit'], detectorId) 

689 if key not in outputs: 

690 # skip detectors we don't have output for 

691 self.log.debug("No %s output for detector %s in visit %s", 

692 setter[3:], detectorId, visit) 

693 continue 

694 

695 catalog[i].setId(detectorId) 

696 getattr(catalog[i], setter)(outputs[key]) 

697 i += 1 

698 

699 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups 

700 butlerQC.put(catalog, ref) 

701 self.log.info("Wrote %s detectors to %s", i, ref) 

702 

703 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None): 

704 # Docstring inherited. 

705 

706 # We take values out of the Parquet table, and put them in "flux_", 

707 # and the config.sourceFluxType field is used during that extraction, 

708 # so just use "flux" here. 

709 sourceFluxField = "flux" 

710 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

711 associations = lsst.jointcal.Associations() 

712 self.focalPlaneBBox = inputCamera.getFpBBox() 

713 oldWcsList, bands = self._load_data(inputSourceTableVisit, 

714 inputVisitSummary, 

715 associations, 

716 jointcalControl, 

717 inputCamera) 

718 

719 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands) 

720 

721 if self.config.doAstrometry: 

722 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

723 name="astrometry", 

724 refObjLoader=self.astrometryRefObjLoader, 

725 referenceSelector=self.astrometryReferenceSelector, 

726 fit_function=self._fit_astrometry, 

727 tract=tract, 

728 epoch=epoch) 

729 astrometry_output = self._make_output(associations.getCcdImageList(), 

730 astrometry.model, 

731 "makeSkyWcs") 

732 else: 

733 astrometry_output = None 

734 

735 if self.config.doPhotometry: 

736 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

737 name="photometry", 

738 refObjLoader=self.photometryRefObjLoader, 

739 referenceSelector=self.photometryReferenceSelector, 

740 fit_function=self._fit_photometry, 

741 tract=tract, 

742 epoch=epoch, 

743 reject_bad_fluxes=True) 

744 photometry_output = self._make_output(associations.getCcdImageList(), 

745 photometry.model, 

746 "toPhotoCalib") 

747 else: 

748 photometry_output = None 

749 

750 return pipeBase.Struct(outputWcs=astrometry_output, 

751 outputPhotoCalib=photometry_output, 

752 job=self.job, 

753 astrometryRefObjLoader=self.astrometryRefObjLoader, 

754 photometryRefObjLoader=self.photometryRefObjLoader) 

755 

756 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, 

757 jointcalControl, camera): 

758 """Read the data that jointcal needs to run. 

759 

760 Modifies ``associations`` in-place with the loaded data. 

761 

762 Parameters 

763 ---------- 

764 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

765 References to visit-level DataFrames to load the catalog data from. 

766 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

767 Visit-level exposure summary catalog with metadata. 

768 associations : `lsst.jointcal.Associations` 

769 Object to add the loaded data to by constructing new CcdImages. 

770 jointcalControl : `jointcal.JointcalControl` 

771 Control object for C++ associations management. 

772 camera : `lsst.afw.cameraGeom.Camera` 

773 Camera object for detector geometry. 

774 

775 Returns 

776 ------- 

777 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`] 

778 The original WCS of the input data, to aid in writing tests. 

779 bands : `list` [`str`] 

780 The filter bands of each input dataset. 

781 """ 

782 oldWcsList = [] 

783 filters = [] 

784 load_cat_profile_file = 'jointcal_load_data.prof' if self.config.detailedProfile else '' 

785 with lsst.utils.timer.profile(load_cat_profile_file): 

786 table = make_schema_table() # every detector catalog has the same layout 

787 # No guarantee that the input is in the same order of visits, so we have to map one of them. 

788 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)} 

789 detectorDict = {detector.getId(): detector for detector in camera} 

790 

791 columns = None 

792 

793 for visitSummaryRef in inputVisitSummary: 

794 visitSummary = visitSummaryRef.get() 

795 

796 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]] 

797 if columns is None: 

798 inColumns = dataRef.get(component='columns') 

799 columns, ixxColumns = get_sourceTable_visit_columns(inColumns, 

800 self.config, 

801 self.sourceSelector) 

802 visitCatalog = dataRef.get(parameters={'columns': columns}) 

803 

804 selected = self.sourceSelector.run(visitCatalog) 

805 if len(selected.sourceCat) == 0: 

806 self.log.warning("No sources selected in visit %s. Skipping...", 

807 visitSummary["visit"][0]) 

808 continue 

809 

810 # Build a CcdImage for each detector in this visit. 

811 detectors = {id: index for index, id in enumerate(visitSummary['id'])} 

812 for id, index in detectors.items(): 

813 catalog = extract_detector_catalog_from_visit_catalog(table, 

814 selected.sourceCat, 

815 id, 

816 ixxColumns, 

817 self.config.sourceFluxType, 

818 self.log) 

819 if catalog is None: 

820 continue 

821 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict) 

822 result = self._build_ccdImage(data, associations, jointcalControl) 

823 if result is not None: 

824 oldWcsList.append(result.wcs) 

825 # A visit has only one band, so we can just use the first. 

826 filters.append(data.filter) 

827 if len(filters) == 0: 

828 raise RuntimeError("No data to process: did source selector remove all sources?") 

829 filters = collections.Counter(filters) 

830 

831 return oldWcsList, filters 

832 

833 def _make_one_input_data(self, visitRecord, catalog, detectorDict): 

834 """Return a data structure for this detector+visit.""" 

835 return JointcalInputData(visit=visitRecord['visit'], 

836 catalog=catalog, 

837 visitInfo=visitRecord.getVisitInfo(), 

838 detector=detectorDict[visitRecord.getId()], 

839 photoCalib=visitRecord.getPhotoCalib(), 

840 wcs=visitRecord.getWcs(), 

841 bbox=visitRecord.getBBox(), 

842 # ExposureRecord doesn't have a FilterLabel yet, 

843 # so we have to make one. 

844 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'], 

845 physical=visitRecord['physical_filter'])) 

846 

847 def _build_ccdImage(self, data, associations, jointcalControl): 

848 """ 

849 Extract the necessary things from this catalog+metadata to add a new 

850 ccdImage. 

851 

852 Parameters 

853 ---------- 

854 data : `JointcalInputData` 

855 The loaded input data. 

856 associations : `lsst.jointcal.Associations` 

857 Object to add the info to, to construct a new CcdImage 

858 jointcalControl : `jointcal.JointcalControl` 

859 Control object for associations management 

860 

861 Returns 

862 ------ 

863 namedtuple or `None` 

864 ``wcs`` 

865 The TAN WCS of this image, read from the calexp 

866 (`lsst.afw.geom.SkyWcs`). 

867 ``key`` 

868 A key to identify this dataRef by its visit and ccd ids 

869 (`namedtuple`). 

870 `None` 

871 if there are no sources in the loaded catalog. 

872 """ 

873 if len(data.catalog) == 0: 

874 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId()) 

875 return None 

876 

877 associations.createCcdImage(data.catalog, 

878 data.wcs, 

879 data.visitInfo, 

880 data.bbox, 

881 data.filter.physicalLabel, 

882 data.photoCalib, 

883 data.detector, 

884 data.visit, 

885 data.detector.getId(), 

886 jointcalControl) 

887 

888 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key')) 

889 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

890 return Result(data.wcs, Key(data.visit, data.detector.getId())) 

891 

892 def _getDebugPath(self, filename): 

893 """Constructs a path to filename using the configured debug path. 

894 """ 

895 return os.path.join(self.config.debugOutputPath, filename) 

896 

897 def _prep_sky(self, associations, filters): 

898 """Prepare on-sky and other data that must be computed after data has 

899 been read. 

900 """ 

901 associations.computeCommonTangentPoint() 

902 

903 boundingCircle = associations.computeBoundingCircle() 

904 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

905 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

906 

907 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

908 

909 # Determine a default filter band associated with the catalog. See DM-9093 

910 defaultFilter = filters.most_common(1)[0][0] 

911 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel) 

912 

913 # compute and set the reference epoch of the observations, for proper motion corrections 

914 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

915 associations.setEpoch(epoch.jyear) 

916 

917 return boundingCircle, center, radius, defaultFilter, epoch 

918 

919 def _get_refcat_coordinate_error_override(self, refCat, name): 

920 """Check whether we should override the refcat coordinate errors, and 

921 return the overridden error if necessary. 

922 

923 Parameters 

924 ---------- 

925 refCat : `lsst.afw.table.SimpleCatalog` 

926 The reference catalog to check for a ``coord_raErr`` field. 

927 name : `str` 

928 Whether we are doing "astrometry" or "photometry". 

929 

930 Returns 

931 ------- 

932 refCoordErr : `float` 

933 The refcat coordinate error to use, or NaN if we are not overriding 

934 those fields. 

935 

936 Raises 

937 ------ 

938 lsst.pex.config.FieldValidationError 

939 Raised if the refcat does not contain coordinate errors and 

940 ``config.astrometryReferenceErr`` is not set. 

941 """ 

942 # This value doesn't matter for photometry, so just set something to 

943 # keep old refcats from causing problems. 

944 if name.lower() == "photometry": 

945 if 'coord_raErr' not in refCat.schema: 

946 return 100 

947 else: 

948 return float('nan') 

949 

950 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

951 msg = ("Reference catalog does not contain coordinate errors, " 

952 "and config.astrometryReferenceErr not supplied.") 

953 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

954 self.config, 

955 msg) 

956 

957 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

958 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

959 self.config.astrometryReferenceErr) 

960 

961 if self.config.astrometryReferenceErr is None: 

962 return float('nan') 

963 else: 

964 return self.config.astrometryReferenceErr 

965 

966 def _compute_proper_motion_epoch(self, ccdImageList): 

967 """Return the proper motion correction epoch of the provided images. 

968 

969 Parameters 

970 ---------- 

971 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

972 The images to compute the appropriate epoch for. 

973 

974 Returns 

975 ------- 

976 epoch : `astropy.time.Time` 

977 The date to use for proper motion corrections. 

978 """ 

979 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]), 

980 format="jyear", 

981 scale="tai") 

982 

983 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

984 tract="", match_cut=3.0, 

985 reject_bad_fluxes=False, *, 

986 name="", refObjLoader=None, referenceSelector=None, 

987 fit_function=None, epoch=None): 

988 """Load reference catalog, perform the fit, and return the result. 

989 

990 Parameters 

991 ---------- 

992 associations : `lsst.jointcal.Associations` 

993 The star/reference star associations to fit. 

994 defaultFilter : `lsst.afw.image.FilterLabel` 

995 filter to load from reference catalog. 

996 center : `lsst.geom.SpherePoint` 

997 ICRS center of field to load from reference catalog. 

998 radius : `lsst.geom.Angle` 

999 On-sky radius to load from reference catalog. 

1000 name : `str` 

1001 Name of thing being fit: "astrometry" or "photometry". 

1002 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

1003 Reference object loader to use to load a reference catalog. 

1004 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1005 Selector to use to pick objects from the loaded reference catalog. 

1006 fit_function : callable 

1007 Function to call to perform fit (takes Associations object). 

1008 tract : `str`, optional 

1009 Name of tract currently being fit. 

1010 match_cut : `float`, optional 

1011 Radius in arcseconds to find cross-catalog matches to during 

1012 associations.associateCatalogs. 

1013 reject_bad_fluxes : `bool`, optional 

1014 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

1015 epoch : `astropy.time.Time`, optional 

1016 Epoch to which to correct refcat proper motion and parallax, 

1017 or `None` to not apply such corrections. 

1018 

1019 Returns 

1020 ------- 

1021 result : `Photometry` or `Astrometry` 

1022 Result of `fit_function()` 

1023 """ 

1024 self.log.info("====== Now processing %s...", name) 

1025 # TODO: this should not print "trying to invert a singular transformation:" 

1026 # if it does that, something's not right about the WCS... 

1027 associations.associateCatalogs(match_cut) 

1028 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name, 

1029 associations.fittedStarListSize()) 

1030 

1031 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

1032 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

1033 center, radius, defaultFilter, 

1034 applyColorterms=applyColorterms, 

1035 epoch=epoch) 

1036 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

1037 

1038 associations.collectRefStars(refCat, 

1039 self.config.matchCut*lsst.geom.arcseconds, 

1040 fluxField, 

1041 refCoordinateErr=refCoordErr, 

1042 rejectBadFluxes=reject_bad_fluxes) 

1043 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name, 

1044 associations.refStarListSize()) 

1045 

1046 associations.prepareFittedStars(self.config.minMeasurements) 

1047 

1048 self._check_star_lists(associations, name) 

1049 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name, 

1050 associations.nFittedStarsWithAssociatedRefStar()) 

1051 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name, 

1052 associations.fittedStarListSize()) 

1053 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name, 

1054 associations.nCcdImagesValidForFit()) 

1055 

1056 fit_profile_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else '' 

1057 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel) 

1058 with lsst.utils.timer.profile(fit_profile_file): 

1059 result = fit_function(associations, dataName) 

1060 # TODO DM-12446: turn this into a "butler save" somehow. 

1061 # Save reference and measurement chi2 contributions for this data 

1062 if self.config.writeChi2FilesInitialFinal: 

1063 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

1064 result.fit.saveChi2Contributions(baseName+"{type}") 

1065 self.log.info("Wrote chi2 contributions files: %s", baseName) 

1066 

1067 return result 

1068 

1069 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, 

1070 applyColorterms=False, epoch=None): 

1071 """Load the necessary reference catalog sources, convert fluxes to 

1072 correct units, and apply color term corrections if requested. 

1073 

1074 Parameters 

1075 ---------- 

1076 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

1077 The reference catalog loader to use to get the data. 

1078 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1079 Source selector to apply to loaded reference catalog. 

1080 center : `lsst.geom.SpherePoint` 

1081 The center around which to load sources. 

1082 radius : `lsst.geom.Angle` 

1083 The radius around ``center`` to load sources in. 

1084 filterLabel : `lsst.afw.image.FilterLabel` 

1085 The camera filter to load fluxes for. 

1086 applyColorterms : `bool` 

1087 Apply colorterm corrections to the refcat for ``filterName``? 

1088 epoch : `astropy.time.Time`, optional 

1089 Epoch to which to correct refcat proper motion and parallax, 

1090 or `None` to not apply such corrections. 

1091 

1092 Returns 

1093 ------- 

1094 refCat : `lsst.afw.table.SimpleCatalog` 

1095 The loaded reference catalog. 

1096 fluxField : `str` 

1097 The name of the reference catalog flux field appropriate for ``filterName``. 

1098 """ 

1099 skyCircle = refObjLoader.loadSkyCircle(center, 

1100 radius, 

1101 filterLabel.bandLabel, 

1102 epoch=epoch) 

1103 

1104 selected = referenceSelector.run(skyCircle.refCat) 

1105 # Need memory contiguity to get reference filters as a vector. 

1106 if not selected.sourceCat.isContiguous(): 

1107 refCat = selected.sourceCat.copy(deep=True) 

1108 else: 

1109 refCat = selected.sourceCat 

1110 

1111 if applyColorterms: 

1112 refCatName = refObjLoader.name 

1113 self.log.info("Applying color terms for physical filter=%r reference catalog=%s", 

1114 filterLabel.physicalLabel, refCatName) 

1115 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel, 

1116 refCatName, 

1117 doRaise=True) 

1118 

1119 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

1120 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

1121 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

1122 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

1123 

1124 return refCat, skyCircle.fluxField 

1125 

1126 def _check_star_lists(self, associations, name): 

1127 # TODO: these should be len(blah), but we need this properly wrapped first. 

1128 if associations.nCcdImagesValidForFit() == 0: 

1129 raise RuntimeError('No images in the ccdImageList!') 

1130 if associations.fittedStarListSize() == 0: 

1131 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

1132 if associations.refStarListSize() == 0: 

1133 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

1134 

1135 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

1136 """Compute chi2, log it, validate the model, and return chi2. 

1137 

1138 Parameters 

1139 ---------- 

1140 associations : `lsst.jointcal.Associations` 

1141 The star/reference star associations to fit. 

1142 fit : `lsst.jointcal.FitterBase` 

1143 The fitter to use for minimization. 

1144 model : `lsst.jointcal.Model` 

1145 The model being fit. 

1146 chi2Label : `str` 

1147 Label to describe the chi2 (e.g. "Initialized", "Final"). 

1148 writeChi2Name : `str`, optional 

1149 Filename prefix to write the chi2 contributions to. 

1150 Do not supply an extension: an appropriate one will be added. 

1151 

1152 Returns 

1153 ------- 

1154 chi2: `lsst.jointcal.Chi2Accumulator` 

1155 The chi2 object for the current fitter and model. 

1156 

1157 Raises 

1158 ------ 

1159 FloatingPointError 

1160 Raised if chi2 is infinite or NaN. 

1161 ValueError 

1162 Raised if the model is not valid. 

1163 """ 

1164 if writeChi2Name is not None: 

1165 fullpath = self._getDebugPath(writeChi2Name) 

1166 fit.saveChi2Contributions(fullpath+"{type}") 

1167 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

1168 

1169 chi2 = fit.computeChi2() 

1170 self.log.info("%s %s", chi2Label, chi2) 

1171 self._check_stars(associations) 

1172 if not np.isfinite(chi2.chi2): 

1173 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

1174 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

1175 raise ValueError("Model is not valid: check log messages for warnings.") 

1176 return chi2 

1177 

1178 def _fit_photometry(self, associations, dataName=None): 

1179 """ 

1180 Fit the photometric data. 

1181 

1182 Parameters 

1183 ---------- 

1184 associations : `lsst.jointcal.Associations` 

1185 The star/reference star associations to fit. 

1186 dataName : `str` 

1187 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1188 identifying debugging files. 

1189 

1190 Returns 

1191 ------- 

1192 fit_result : `namedtuple` 

1193 fit : `lsst.jointcal.PhotometryFit` 

1194 The photometric fitter used to perform the fit. 

1195 model : `lsst.jointcal.PhotometryModel` 

1196 The photometric model that was fit. 

1197 """ 

1198 self.log.info("=== Starting photometric fitting...") 

1199 

1200 # TODO: should use pex.config.RegistryField here (see DM-9195) 

1201 if self.config.photometryModel == "constrainedFlux": 

1202 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

1203 self.focalPlaneBBox, 

1204 visitOrder=self.config.photometryVisitOrder, 

1205 errorPedestal=self.config.photometryErrorPedestal) 

1206 # potentially nonlinear problem, so we may need a line search to converge. 

1207 doLineSearch = self.config.allowLineSearch 

1208 elif self.config.photometryModel == "constrainedMagnitude": 

1209 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

1210 self.focalPlaneBBox, 

1211 visitOrder=self.config.photometryVisitOrder, 

1212 errorPedestal=self.config.photometryErrorPedestal) 

1213 # potentially nonlinear problem, so we may need a line search to converge. 

1214 doLineSearch = self.config.allowLineSearch 

1215 elif self.config.photometryModel == "simpleFlux": 

1216 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

1217 errorPedestal=self.config.photometryErrorPedestal) 

1218 doLineSearch = False # purely linear in model parameters, so no line search needed 

1219 elif self.config.photometryModel == "simpleMagnitude": 

1220 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

1221 errorPedestal=self.config.photometryErrorPedestal) 

1222 doLineSearch = False # purely linear in model parameters, so no line search needed 

1223 

1224 fit = lsst.jointcal.PhotometryFit(associations, model) 

1225 # TODO DM-12446: turn this into a "butler save" somehow. 

1226 # Save reference and measurement chi2 contributions for this data 

1227 if self.config.writeChi2FilesInitialFinal: 

1228 baseName = f"photometry_initial_chi2-{dataName}" 

1229 else: 

1230 baseName = None 

1231 if self.config.writeInitialModel: 

1232 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt") 

1233 writeModel(model, fullpath, self.log) 

1234 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

1235 

1236 def getChi2Name(whatToFit): 

1237 if self.config.writeChi2FilesOuterLoop: 

1238 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

1239 else: 

1240 return None 

1241 

1242 # The constrained model needs the visit transform fit first; the chip 

1243 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

1244 if self.config.writeInitMatrix: 

1245 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}") 

1246 else: 

1247 dumpMatrixFile = "" 

1248 if self.config.photometryModel.startswith("constrained"): 

1249 # no line search: should be purely (or nearly) linear, 

1250 # and we want a large step size to initialize with. 

1251 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

1252 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

1253 writeChi2Name=getChi2Name("ModelVisit")) 

1254 dumpMatrixFile = "" # so we don't redo the output on the next step 

1255 

1256 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

1257 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

1258 writeChi2Name=getChi2Name("Model")) 

1259 

1260 fit.minimize("Fluxes") # no line search: always purely linear. 

1261 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

1262 writeChi2Name=getChi2Name("Fluxes")) 

1263 

1264 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

1265 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

1266 writeChi2Name=getChi2Name("ModelFluxes")) 

1267 

1268 model.freezeErrorTransform() 

1269 self.log.debug("Photometry error scales are frozen.") 

1270 

1271 chi2 = self._iterate_fit(associations, 

1272 fit, 

1273 self.config.maxPhotometrySteps, 

1274 "photometry", 

1275 "Model Fluxes", 

1276 doRankUpdate=self.config.photometryDoRankUpdate, 

1277 doLineSearch=doLineSearch, 

1278 dataName=dataName) 

1279 

1280 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

1281 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

1282 return Photometry(fit, model) 

1283 

1284 def _fit_astrometry(self, associations, dataName=None): 

1285 """ 

1286 Fit the astrometric data. 

1287 

1288 Parameters 

1289 ---------- 

1290 associations : `lsst.jointcal.Associations` 

1291 The star/reference star associations to fit. 

1292 dataName : `str` 

1293 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1294 identifying debugging files. 

1295 

1296 Returns 

1297 ------- 

1298 fit_result : `namedtuple` 

1299 fit : `lsst.jointcal.AstrometryFit` 

1300 The astrometric fitter used to perform the fit. 

1301 model : `lsst.jointcal.AstrometryModel` 

1302 The astrometric model that was fit. 

1303 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

1304 The model for the sky to tangent plane projection that was used in the fit. 

1305 """ 

1306 

1307 self.log.info("=== Starting astrometric fitting...") 

1308 

1309 associations.deprojectFittedStars() 

1310 

1311 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1312 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1313 # them so carefully? 

1314 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1315 

1316 if self.config.astrometryModel == "constrained": 

1317 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1318 sky_to_tan_projection, 

1319 chipOrder=self.config.astrometryChipOrder, 

1320 visitOrder=self.config.astrometryVisitOrder) 

1321 elif self.config.astrometryModel == "simple": 

1322 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1323 sky_to_tan_projection, 

1324 self.config.useInputWcs, 

1325 nNotFit=0, 

1326 order=self.config.astrometrySimpleOrder) 

1327 

1328 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1329 # TODO DM-12446: turn this into a "butler save" somehow. 

1330 # Save reference and measurement chi2 contributions for this data 

1331 if self.config.writeChi2FilesInitialFinal: 

1332 baseName = f"astrometry_initial_chi2-{dataName}" 

1333 else: 

1334 baseName = None 

1335 if self.config.writeInitialModel: 

1336 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt") 

1337 writeModel(model, fullpath, self.log) 

1338 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1339 

1340 def getChi2Name(whatToFit): 

1341 if self.config.writeChi2FilesOuterLoop: 

1342 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1343 else: 

1344 return None 

1345 

1346 if self.config.writeInitMatrix: 

1347 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}") 

1348 else: 

1349 dumpMatrixFile = "" 

1350 # The constrained model needs the visit transform fit first; the chip 

1351 # transform is initialized from the detector's cameraGeom, so it's close. 

1352 if self.config.astrometryModel == "constrained": 

1353 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1354 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1355 writeChi2Name=getChi2Name("DistortionsVisit")) 

1356 dumpMatrixFile = "" # so we don't redo the output on the next step 

1357 

1358 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1359 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1360 writeChi2Name=getChi2Name("Distortions")) 

1361 

1362 fit.minimize("Positions") 

1363 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1364 writeChi2Name=getChi2Name("Positions")) 

1365 

1366 fit.minimize("Distortions Positions") 

1367 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1368 writeChi2Name=getChi2Name("DistortionsPositions")) 

1369 

1370 chi2 = self._iterate_fit(associations, 

1371 fit, 

1372 self.config.maxAstrometrySteps, 

1373 "astrometry", 

1374 "Distortions Positions", 

1375 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance, 

1376 doRankUpdate=self.config.astrometryDoRankUpdate, 

1377 dataName=dataName) 

1378 

1379 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1380 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1381 

1382 return Astrometry(fit, model, sky_to_tan_projection) 

1383 

1384 def _check_stars(self, associations): 

1385 """Count measured and reference stars per ccd and warn/log them.""" 

1386 for ccdImage in associations.getCcdImageList(): 

1387 nMeasuredStars, nRefStars = ccdImage.countStars() 

1388 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1389 ccdImage.getName(), nMeasuredStars, nRefStars) 

1390 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1391 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)", 

1392 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1393 if nRefStars < self.config.minRefStarsPerCcd: 

1394 self.log.warning("ccdImage %s has only %s RefStars (desired %s)", 

1395 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1396 

1397 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1398 dataName="", 

1399 sigmaRelativeTolerance=0, 

1400 doRankUpdate=True, 

1401 doLineSearch=False): 

1402 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1403 

1404 Parameters 

1405 ---------- 

1406 associations : `lsst.jointcal.Associations` 

1407 The star/reference star associations to fit. 

1408 fitter : `lsst.jointcal.FitterBase` 

1409 The fitter to use for minimization. 

1410 max_steps : `int` 

1411 Maximum number of steps to run outlier rejection before declaring 

1412 convergence failure. 

1413 name : {'photometry' or 'astrometry'} 

1414 What type of data are we fitting (for logs and debugging files). 

1415 whatToFit : `str` 

1416 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1417 dataName : `str`, optional 

1418 Descriptive name for this dataset (e.g. tract and filter), 

1419 for debugging. 

1420 sigmaRelativeTolerance : `float`, optional 

1421 Convergence tolerance for the fractional change in the chi2 cut 

1422 level for determining outliers. If set to zero, iterations will 

1423 continue until there are no outliers. 

1424 doRankUpdate : `bool`, optional 

1425 Do an Eigen rank update during minimization, or recompute the full 

1426 matrix and gradient? 

1427 doLineSearch : `bool`, optional 

1428 Do a line search for the optimum step during minimization? 

1429 

1430 Returns 

1431 ------- 

1432 chi2: `lsst.jointcal.Chi2Statistic` 

1433 The final chi2 after the fit converges, or is forced to end. 

1434 

1435 Raises 

1436 ------ 

1437 FloatingPointError 

1438 Raised if the fitter fails with a non-finite value. 

1439 RuntimeError 

1440 Raised if the fitter fails for some other reason; 

1441 log messages will provide further details. 

1442 """ 

1443 if self.config.writeInitMatrix: 

1444 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}") 

1445 else: 

1446 dumpMatrixFile = "" 

1447 oldChi2 = lsst.jointcal.Chi2Statistic() 

1448 oldChi2.chi2 = float("inf") 

1449 for i in range(max_steps): 

1450 if self.config.writeChi2FilesOuterLoop: 

1451 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1452 else: 

1453 writeChi2Name = None 

1454 result = fitter.minimize(whatToFit, 

1455 self.config.outlierRejectSigma, 

1456 sigmaRelativeTolerance=sigmaRelativeTolerance, 

1457 doRankUpdate=doRankUpdate, 

1458 doLineSearch=doLineSearch, 

1459 dumpMatrixFile=dumpMatrixFile) 

1460 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1461 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1462 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1463 

1464 if result == MinimizeResult.Converged: 

1465 if doRankUpdate: 

1466 self.log.debug("fit has converged - no more outliers - redo minimization " 

1467 "one more time in case we have lost accuracy in rank update.") 

1468 # Redo minimization one more time in case we have lost accuracy in rank update 

1469 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma, 

1470 sigmaRelativeTolerance=sigmaRelativeTolerance) 

1471 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1472 

1473 # log a message for a large final chi2, TODO: DM-15247 for something better 

1474 if chi2.chi2/chi2.ndof >= 4.0: 

1475 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1476 

1477 break 

1478 elif result == MinimizeResult.Chi2Increased: 

1479 self.log.warning("Still some outliers remaining but chi2 increased - retry") 

1480 # Check whether the increase was large enough to cause trouble. 

1481 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1482 if chi2Ratio > 1.5: 

1483 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1484 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1485 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1486 # appears that chi2 increases more than a factor of ~2 always 

1487 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1488 # Using 10 as the "failure" threshold gives some room between 

1489 # leaving a warning and bailing early. 

1490 if chi2Ratio > 10: 

1491 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1492 " Try setting one or more of the `writeChi2*` config fields and looking" 

1493 " at how individual star chi2-values evolve during the fit.") 

1494 raise RuntimeError(msg) 

1495 oldChi2 = chi2 

1496 elif result == MinimizeResult.NonFinite: 

1497 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1498 # TODO DM-12446: turn this into a "butler save" somehow. 

1499 fitter.saveChi2Contributions(filename+"{type}") 

1500 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1501 raise FloatingPointError(msg.format(filename)) 

1502 elif result == MinimizeResult.Failed: 

1503 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1504 else: 

1505 raise RuntimeError("Unxepected return code from minimize().") 

1506 else: 

1507 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1508 

1509 return chi2 

1510 

1511 def _make_output(self, ccdImageList, model, func): 

1512 """Return the internal jointcal models converted to the afw 

1513 structures that will be saved to disk. 

1514 

1515 Parameters 

1516 ---------- 

1517 ccdImageList : `lsst.jointcal.CcdImageList` 

1518 The list of CcdImages to get the output for. 

1519 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel` 

1520 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`. 

1521 func : `str` 

1522 The name of the function to call on ``model`` to get the converted 

1523 structure. Must accept an `lsst.jointcal.CcdImage`. 

1524 

1525 Returns 

1526 ------- 

1527 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or 

1528 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`] 

1529 The data to be saved, keyed on (visit, detector). 

1530 """ 

1531 output = {} 

1532 for ccdImage in ccdImageList: 

1533 ccd = ccdImage.ccdId 

1534 visit = ccdImage.visit 

1535 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd) 

1536 output[(visit, ccd)] = getattr(model, func)(ccdImage) 

1537 return output 

1538 

1539 

1540def make_schema_table(): 

1541 """Return an afw SourceTable to use as a base for creating the 

1542 SourceCatalog to insert values from the dataFrame into. 

1543 

1544 Returns 

1545 ------- 

1546 table : `lsst.afw.table.SourceTable` 

1547 Table with schema and slots to use to make SourceCatalogs. 

1548 """ 

1549 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

1550 schema.addField("centroid_x", "D") 

1551 schema.addField("centroid_y", "D") 

1552 schema.addField("centroid_xErr", "F") 

1553 schema.addField("centroid_yErr", "F") 

1554 schema.addField("shape_xx", "D") 

1555 schema.addField("shape_yy", "D") 

1556 schema.addField("shape_xy", "D") 

1557 schema.addField("flux_instFlux", "D") 

1558 schema.addField("flux_instFluxErr", "D") 

1559 table = lsst.afw.table.SourceTable.make(schema) 

1560 table.defineCentroid("centroid") 

1561 table.defineShape("shape") 

1562 return table 

1563 

1564 

1565def get_sourceTable_visit_columns(inColumns, config, sourceSelector): 

1566 """ 

1567 Get the sourceTable_visit columns to load from the catalogs. 

1568 

1569 Parameters 

1570 ---------- 

1571 inColumns : `list` 

1572 List of columns known to be available in the sourceTable_visit. 

1573 config : `JointcalConfig` 

1574 A filled-in config to to help define column names. 

1575 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask` 

1576 A configured source selector to define column names to load. 

1577 

1578 Returns 

1579 ------- 

1580 columns : `list` 

1581 List of columns to read from sourceTable_visit. 

1582 ixxColumns : `list` 

1583 Name of the ixx/iyy/ixy columns. 

1584 """ 

1585 columns = ['visit', 'detector', 

1586 'sourceId', 'x', 'xErr', 'y', 'yErr', 

1587 config.sourceFluxType + '_instFlux', config.sourceFluxType + '_instFluxErr'] 

1588 

1589 if 'ixx' in inColumns: 

1590 # New columns post-DM-31825 

1591 ixxColumns = ['ixx', 'iyy', 'ixy'] 

1592 else: 

1593 # Old columns pre-DM-31825 

1594 ixxColumns = ['Ixx', 'Iyy', 'Ixy'] 

1595 columns.extend(ixxColumns) 

1596 

1597 if sourceSelector.config.doFlags: 

1598 columns.extend(sourceSelector.config.flags.bad) 

1599 if sourceSelector.config.doUnresolved: 

1600 columns.append(sourceSelector.config.unresolved.name) 

1601 if sourceSelector.config.doIsolated: 

1602 columns.append(sourceSelector.config.isolated.parentName) 

1603 columns.append(sourceSelector.config.isolated.nChildName) 

1604 if sourceSelector.config.doRequireFiniteRaDec: 

1605 columns.append(sourceSelector.config.requireFiniteRaDec.raColName) 

1606 columns.append(sourceSelector.config.requireFiniteRaDec.decColName) 

1607 

1608 return columns, ixxColumns 

1609 

1610 

1611def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId, 

1612 ixxColumns, sourceFluxType, log): 

1613 """Return an afw SourceCatalog extracted from a visit-level dataframe, 

1614 limited to just one detector. 

1615 

1616 Parameters 

1617 ---------- 

1618 table : `lsst.afw.table.SourceTable` 

1619 Table factory to use to make the SourceCatalog that will be 

1620 populated with data from ``visitCatalog``. 

1621 visitCatalog : `pandas.DataFrame` 

1622 DataFrame to extract a detector catalog from. 

1623 detectorId : `int` 

1624 Numeric id of the detector to extract from ``visitCatalog``. 

1625 ixxColumns : `list` [`str`] 

1626 Names of the ixx/iyy/ixy columns in the catalog. 

1627 sourceFluxType : `str` 

1628 Name of the catalog field to load instFluxes from. 

1629 log : `logging.Logger` 

1630 Logging instance to log to. 

1631 

1632 Returns 

1633 ------- 

1634 catalog : `lsst.afw.table.SourceCatalog`, or `None` 

1635 Detector-level catalog extracted from ``visitCatalog``, or `None` 

1636 if there was no data to load. 

1637 """ 

1638 # map from dataFrame column to afw table column 

1639 mapping = {'x': 'centroid_x', 

1640 'y': 'centroid_y', 

1641 'xErr': 'centroid_xErr', 

1642 'yErr': 'centroid_yErr', 

1643 ixxColumns[0]: 'shape_xx', 

1644 ixxColumns[1]: 'shape_yy', 

1645 ixxColumns[2]: 'shape_xy', 

1646 f'{sourceFluxType}_instFlux': 'flux_instFlux', 

1647 f'{sourceFluxType}_instFluxErr': 'flux_instFluxErr', 

1648 } 

1649 

1650 catalog = lsst.afw.table.SourceCatalog(table) 

1651 matched = visitCatalog['detector'] == detectorId 

1652 n = sum(matched) 

1653 if n == 0: 

1654 return None 

1655 catalog.resize(sum(matched)) 

1656 view = visitCatalog.loc[matched] 

1657 catalog['id'] = view.index 

1658 for dfCol, afwCol in mapping.items(): 

1659 catalog[afwCol] = view[dfCol] 

1660 

1661 log.debug("%d sources selected in visit %d detector %d", 

1662 len(catalog), 

1663 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first 

1664 detectorId) 

1665 return catalog