Coverage for python/lsst/jointcal/jointcal.py: 19%

534 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-13 11:09 +0000

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import dataclasses 

23import collections 

24import os 

25import logging 

26 

27import astropy.time 

28import numpy as np 

29import astropy.units as u 

30 

31import lsst.geom 

32import lsst.utils 

33import lsst.pex.config as pexConfig 

34import lsst.pipe.base as pipeBase 

35from lsst.afw.image import fluxErrFromABMagErr 

36import lsst.afw.cameraGeom 

37import lsst.afw.table 

38from lsst.pipe.base import Instrument 

39from lsst.pipe.tasks.colorterms import ColortermLibrary 

40from lsst.verify import Job, Measurement 

41 

42from lsst.meas.algorithms import (ReferenceObjectLoader, ReferenceSourceSelectorTask, 

43 LoadIndexedReferenceObjectsConfig) 

44from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

45 

46import lsst.jointcal 

47from lsst.jointcal import MinimizeResult 

48 

49__all__ = ["JointcalConfig", "JointcalTask"] 

50 

51Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

52Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

53 

54 

55# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

56def add_measurement(job, name, value): 

57 meas = Measurement(job.metrics[name], value) 

58 job.measurements.insert(meas) 

59 

60 

61def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

62 """Lookup function that asserts/hopes that a static calibration dataset 

63 exists in a particular collection, since this task can't provide a single 

64 date/time to use to search for one properly. 

65 

66 This is mostly useful for the ``camera`` dataset, in cases where the task's 

67 quantum dimensions do *not* include something temporal, like ``exposure`` 

68 or ``visit``. 

69 

70 Parameters 

71 ---------- 

72 datasetType : `lsst.daf.butler.DatasetType` 

73 Type of dataset being searched for. 

74 registry : `lsst.daf.butler.Registry` 

75 Data repository registry to search. 

76 quantumDataId : `lsst.daf.butler.DataCoordinate` 

77 Data ID of the quantum this camera should match. 

78 collections : `Iterable` [ `str` ] 

79 Collections that should be searched - but this lookup function works 

80 by ignoring this in favor of a more-or-less hard-coded value. 

81 

82 Returns 

83 ------- 

84 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

85 Iterator over dataset references; should have only one element. 

86 

87 Notes 

88 ----- 

89 This implementation duplicates one in fgcmcal, and is at least quite 

90 similar to another in cp_pipe. This duplicate has the most documentation. 

91 Fixing this is DM-29661. 

92 """ 

93 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

94 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

95 return registry.queryDatasets(datasetType, 

96 dataId=quantumDataId, 

97 collections=[unboundedCollection], 

98 findFirst=True) 

99 

100 

101def lookupVisitRefCats(datasetType, registry, quantumDataId, collections): 

102 """Lookup function that finds all refcats for all visits that overlap a 

103 tract, rather than just the refcats that directly overlap the tract. 

104 

105 Parameters 

106 ---------- 

107 datasetType : `lsst.daf.butler.DatasetType` 

108 Type of dataset being searched for. 

109 registry : `lsst.daf.butler.Registry` 

110 Data repository registry to search. 

111 quantumDataId : `lsst.daf.butler.DataCoordinate` 

112 Data ID of the quantum; expected to be something we can use as a 

113 constraint to query for overlapping visits. 

114 collections : `Iterable` [ `str` ] 

115 Collections to search. 

116 

117 Returns 

118 ------- 

119 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

120 Iterator over refcat references. 

121 """ 

122 refs = set() 

123 # Use .expanded() on the query methods below because we need data IDs with 

124 # regions, both in the outer loop over visits (queryDatasets will expand 

125 # any data ID we give it, but doing it up-front in bulk is much more 

126 # efficient) and in the data IDs of the DatasetRefs this function yields 

127 # (because the RefCatLoader relies on them to do some of its own 

128 # filtering). 

129 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()): 

130 refs.update( 

131 registry.queryDatasets( 

132 datasetType, 

133 collections=collections, 

134 dataId=visit_data_id, 

135 findFirst=True, 

136 ).expanded() 

137 ) 

138 yield from refs 

139 

140 

141class JointcalTaskConnections(pipeBase.PipelineTaskConnections, 

142 dimensions=("skymap", "tract", "instrument", "physical_filter")): 

143 """Middleware input/output connections for jointcal data.""" 

144 inputCamera = pipeBase.connectionTypes.PrerequisiteInput( 

145 doc="The camera instrument that took these observations.", 

146 name="camera", 

147 storageClass="Camera", 

148 dimensions=("instrument",), 

149 isCalibration=True, 

150 lookupFunction=lookupStaticCalibrations, 

151 ) 

152 inputSourceTableVisit = pipeBase.connectionTypes.Input( 

153 doc="Source table in parquet format, per visit", 

154 name="sourceTable_visit", 

155 storageClass="DataFrame", 

156 dimensions=("instrument", "visit"), 

157 deferLoad=True, 

158 multiple=True, 

159 ) 

160 inputVisitSummary = pipeBase.connectionTypes.Input( 

161 doc=("Per-visit consolidated exposure metadata built from calexps. " 

162 "These catalogs use detector id for the id and must be sorted for " 

163 "fast lookups of a detector."), 

164 name="visitSummary", 

165 storageClass="ExposureCatalog", 

166 dimensions=("instrument", "visit"), 

167 deferLoad=True, 

168 multiple=True, 

169 ) 

170 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

171 doc="The astrometry reference catalog to match to loaded input catalog sources.", 

172 name="gaia_dr2_20200414", 

173 storageClass="SimpleCatalog", 

174 dimensions=("skypix",), 

175 deferLoad=True, 

176 multiple=True, 

177 lookupFunction=lookupVisitRefCats, 

178 ) 

179 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

180 doc="The photometry reference catalog to match to loaded input catalog sources.", 

181 name="ps1_pv3_3pi_20170110", 

182 storageClass="SimpleCatalog", 

183 dimensions=("skypix",), 

184 deferLoad=True, 

185 multiple=True, 

186 lookupFunction=lookupVisitRefCats, 

187 ) 

188 

189 outputWcs = pipeBase.connectionTypes.Output( 

190 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model." 

191 " These catalogs only contain entries for detectors with an output, and use" 

192 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

193 name="jointcalSkyWcsCatalog", 

194 storageClass="ExposureCatalog", 

195 dimensions=("instrument", "visit", "skymap", "tract"), 

196 multiple=True 

197 ) 

198 outputPhotoCalib = pipeBase.connectionTypes.Output( 

199 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model." 

200 " These catalogs only contain entries for detectors with an output, and use" 

201 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

202 name="jointcalPhotoCalibCatalog", 

203 storageClass="ExposureCatalog", 

204 dimensions=("instrument", "visit", "skymap", "tract"), 

205 multiple=True 

206 ) 

207 

208 # measurements of metrics 

209 # The vars() trick used here allows us to set class attributes 

210 # programatically. Taken from: 

211 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class 

212 for name in ("astrometry", "photometry"): 

213 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output( 

214 doc=f"The number of cross-matched fittedStars for {name}", 

215 name=f"metricvalue_jointcal_{name}_matched_fittedStars", 

216 storageClass="MetricValue", 

217 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

218 ) 

219 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output( 

220 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.", 

221 name=f"metricvalue_jointcal_{name}_collected_refStars", 

222 storageClass="MetricValue", 

223 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

224 ) 

225 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output( 

226 doc=f"The number of {name} reference stars matched to fittedStars.", 

227 name=f"metricvalue_jointcal_{name}_prepared_refStars", 

228 storageClass="MetricValue", 

229 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

230 ) 

231 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output( 

232 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.", 

233 name=f"metricvalue_jointcal_{name}_prepared_fittedStars", 

234 storageClass="MetricValue", 

235 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

236 ) 

237 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output( 

238 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.", 

239 name=f"metricvalue_jointcal_{name}_prepared_ccdImages", 

240 storageClass="MetricValue", 

241 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

242 ) 

243 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output( 

244 doc=f"The final chi2 of the {name} fit.", 

245 name=f"metricvalue_jointcal_{name}_final_chi2", 

246 storageClass="MetricValue", 

247 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

248 ) 

249 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output( 

250 doc=f"The number of degrees of freedom of the fitted {name}.", 

251 name=f"metricvalue_jointcal_{name}_final_ndof", 

252 storageClass="MetricValue", 

253 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

254 ) 

255 

256 def __init__(self, *, config=None): 

257 super().__init__(config=config) 

258 # When we are only doing one of astrometry or photometry, we don't 

259 # need the reference catalog or produce the outputs for the other. 

260 # This informs the middleware of that when the QuantumGraph is 

261 # generated, so we don't block on getting something we won't need or 

262 # create an expectation that downstream tasks will be able to consume 

263 # something we won't produce. 

264 if not config.doAstrometry: 

265 self.prerequisiteInputs.remove("astrometryRefCat") 

266 self.outputs.remove("outputWcs") 

267 for key in list(self.outputs): 

268 if "metricvalue_jointcal_astrometry" in key: 

269 self.outputs.remove(key) 

270 if not config.doPhotometry: 

271 self.prerequisiteInputs.remove("photometryRefCat") 

272 self.outputs.remove("outputPhotoCalib") 

273 for key in list(self.outputs): 

274 if "metricvalue_jointcal_photometry" in key: 

275 self.outputs.remove(key) 

276 

277 

278class JointcalConfig(pipeBase.PipelineTaskConfig, 

279 pipelineConnections=JointcalTaskConnections): 

280 """Configuration for JointcalTask""" 

281 

282 doAstrometry = pexConfig.Field( 

283 doc="Fit astrometry and write the fitted result.", 

284 dtype=bool, 

285 default=True 

286 ) 

287 doPhotometry = pexConfig.Field( 

288 doc="Fit photometry and write the fitted result.", 

289 dtype=bool, 

290 default=True 

291 ) 

292 sourceFluxType = pexConfig.Field( 

293 dtype=str, 

294 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

295 default='apFlux_12_0' 

296 ) 

297 positionErrorPedestal = pexConfig.Field( 

298 doc="Systematic term to apply to the measured position error (pixels)", 

299 dtype=float, 

300 default=0.02, 

301 ) 

302 photometryErrorPedestal = pexConfig.Field( 

303 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

304 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

305 dtype=float, 

306 default=0.0, 

307 ) 

308 # TODO: DM-6885 matchCut should be an geom.Angle 

309 matchCut = pexConfig.Field( 

310 doc="Matching radius between fitted and reference stars (arcseconds)", 

311 dtype=float, 

312 default=3.0, 

313 ) 

314 minMeasurements = pexConfig.Field( 

315 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

316 dtype=int, 

317 default=2, 

318 ) 

319 minMeasuredStarsPerCcd = pexConfig.Field( 

320 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

321 dtype=int, 

322 default=100, 

323 ) 

324 minRefStarsPerCcd = pexConfig.Field( 

325 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

326 dtype=int, 

327 default=30, 

328 ) 

329 allowLineSearch = pexConfig.Field( 

330 doc="Allow a line search during minimization, if it is reasonable for the model" 

331 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

332 dtype=bool, 

333 default=False 

334 ) 

335 astrometrySimpleOrder = pexConfig.Field( 

336 doc="Polynomial order for fitting the simple astrometry model.", 

337 dtype=int, 

338 default=3, 

339 ) 

340 astrometryChipOrder = pexConfig.Field( 

341 doc="Order of the per-chip transform for the constrained astrometry model.", 

342 dtype=int, 

343 default=1, 

344 ) 

345 astrometryVisitOrder = pexConfig.Field( 

346 doc="Order of the per-visit transform for the constrained astrometry model.", 

347 dtype=int, 

348 default=5, 

349 ) 

350 useInputWcs = pexConfig.Field( 

351 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

352 dtype=bool, 

353 default=True, 

354 ) 

355 astrometryModel = pexConfig.ChoiceField( 

356 doc="Type of model to fit to astrometry", 

357 dtype=str, 

358 default="constrained", 

359 allowed={"simple": "One polynomial per ccd", 

360 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

361 ) 

362 photometryModel = pexConfig.ChoiceField( 

363 doc="Type of model to fit to photometry", 

364 dtype=str, 

365 default="constrainedMagnitude", 

366 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

367 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

368 " fitting in flux space.", 

369 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

370 " fitting in magnitude space.", 

371 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

372 " fitting in magnitude space.", 

373 } 

374 ) 

375 applyColorTerms = pexConfig.Field( 

376 doc="Apply photometric color terms to reference stars?" 

377 "Requires that colorterms be set to a ColortermLibrary", 

378 dtype=bool, 

379 default=False 

380 ) 

381 colorterms = pexConfig.ConfigField( 

382 doc="Library of photometric reference catalog name to color term dict.", 

383 dtype=ColortermLibrary, 

384 ) 

385 photometryVisitOrder = pexConfig.Field( 

386 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

387 dtype=int, 

388 default=7, 

389 ) 

390 photometryDoRankUpdate = pexConfig.Field( 

391 doc=("Do the rank update step during minimization. " 

392 "Skipping this can help deal with models that are too non-linear."), 

393 dtype=bool, 

394 default=True, 

395 ) 

396 astrometryDoRankUpdate = pexConfig.Field( 

397 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

398 "Skipping this can help deal with models that are too non-linear."), 

399 dtype=bool, 

400 default=True, 

401 ) 

402 outlierRejectSigma = pexConfig.Field( 

403 doc="How many sigma to reject outliers at during minimization.", 

404 dtype=float, 

405 default=5.0, 

406 ) 

407 astrometryOutlierRelativeTolerance = pexConfig.Field( 

408 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will " 

409 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set " 

410 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002" 

411 "as a balance between a shorter minimization runtime and achieving a final fitted model that is" 

412 "close to the solution found when removing all outliers."), 

413 dtype=float, 

414 default=0, 

415 ) 

416 maxPhotometrySteps = pexConfig.Field( 

417 doc="Maximum number of minimize iterations to take when fitting photometry.", 

418 dtype=int, 

419 default=20, 

420 ) 

421 maxAstrometrySteps = pexConfig.Field( 

422 doc="Maximum number of minimize iterations to take when fitting astrometry.", 

423 dtype=int, 

424 default=20, 

425 ) 

426 astrometryRefObjLoader = pexConfig.ConfigField( 

427 dtype=LoadIndexedReferenceObjectsConfig, 

428 doc="Reference object loader for astrometric fit", 

429 ) 

430 photometryRefObjLoader = pexConfig.ConfigField( 

431 dtype=LoadIndexedReferenceObjectsConfig, 

432 doc="Reference object loader for photometric fit", 

433 ) 

434 sourceSelector = sourceSelectorRegistry.makeField( 

435 doc="How to select sources for cross-matching", 

436 default="science" 

437 ) 

438 astrometryReferenceSelector = pexConfig.ConfigurableField( 

439 target=ReferenceSourceSelectorTask, 

440 doc="How to down-select the loaded astrometry reference catalog.", 

441 ) 

442 photometryReferenceSelector = pexConfig.ConfigurableField( 

443 target=ReferenceSourceSelectorTask, 

444 doc="How to down-select the loaded photometry reference catalog.", 

445 ) 

446 astrometryReferenceErr = pexConfig.Field( 

447 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

448 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

449 "If specified, overrides any existing `coord_*Err` values."), 

450 dtype=float, 

451 default=None, 

452 optional=True 

453 ) 

454 

455 # configs for outputting debug information 

456 writeInitMatrix = pexConfig.Field( 

457 dtype=bool, 

458 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

459 "Output files will be written to `config.debugOutputPath` and will " 

460 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. " 

461 "Note that these files are the dense versions of the matrix, and so may be very large."), 

462 default=False 

463 ) 

464 writeChi2FilesInitialFinal = pexConfig.Field( 

465 dtype=bool, 

466 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. " 

467 "Output files will be written to `config.debugOutputPath` and will " 

468 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."), 

469 default=False 

470 ) 

471 writeChi2FilesOuterLoop = pexConfig.Field( 

472 dtype=bool, 

473 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. " 

474 "Output files will be written to `config.debugOutputPath` and will " 

475 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."), 

476 default=False 

477 ) 

478 writeInitialModel = pexConfig.Field( 

479 dtype=bool, 

480 doc=("Write the pre-initialization model to text files, for debugging. " 

481 "Output files will be written to `config.debugOutputPath` and will be " 

482 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."), 

483 default=False 

484 ) 

485 debugOutputPath = pexConfig.Field( 

486 dtype=str, 

487 default=".", 

488 doc=("Path to write debug output files to. Used by " 

489 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

490 ) 

491 detailedProfile = pexConfig.Field( 

492 dtype=bool, 

493 default=False, 

494 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting" 

495 ) 

496 

497 def validate(self): 

498 super().validate() 

499 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

500 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

501 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

502 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

503 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

504 "applyColorTerms=True will be ignored.") 

505 logging.getLogger("lsst.jointcal").warning(msg) 

506 

507 def setDefaults(self): 

508 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing. 

509 self.sourceSelector["science"].doUnresolved = True 

510 self.sourceSelector["science"].unresolved.name = "extendedness" 

511 # with dependable signal to noise ratio. 

512 self.sourceSelector["science"].doSignalToNoise = True 

513 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

514 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

515 self.sourceSelector["science"].signalToNoise.minimum = 10. 

516 # Base SNR selection on `sourceFluxType` because that is the flux that jointcal fits. 

517 self.sourceSelector["science"].signalToNoise.fluxField = f"{self.sourceFluxType}_instFlux" 

518 self.sourceSelector["science"].signalToNoise.errField = f"{self.sourceFluxType}_instFluxErr" 

519 # Do not trust blended sources" aperture fluxes which also depend on seeing. 

520 self.sourceSelector["science"].doIsolated = True 

521 self.sourceSelector["science"].isolated.parentName = "parentSourceId" 

522 self.sourceSelector["science"].isolated.nChildName = "deblend_nChild" 

523 # Do not trust either flux or centroid measurements with flags, 

524 # chosen from the usual QA flags for stars) 

525 self.sourceSelector["science"].doFlags = True 

526 badFlags = ["pixelFlags_edge", 

527 "pixelFlags_saturated", 

528 "pixelFlags_interpolatedCenter", 

529 "pixelFlags_interpolated", 

530 "pixelFlags_crCenter", 

531 "pixelFlags_bad", 

532 "hsmPsfMoments_flag", 

533 f"{self.sourceFluxType}_flag", 

534 ] 

535 self.sourceSelector["science"].flags.bad = badFlags 

536 

537 # Use Gaia-DR2 with proper motions for astrometry; phot_g_mean is the 

538 # primary Gaia band, but is not like any normal photometric band. 

539 self.astrometryRefObjLoader.requireProperMotion = True 

540 self.astrometryRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

541 

542 

543def writeModel(model, filename, log): 

544 """Write model to outfile.""" 

545 with open(filename, "w") as file: 

546 file.write(repr(model)) 

547 log.info("Wrote %s to file: %s", model, filename) 

548 

549 

550@dataclasses.dataclass 

551class JointcalInputData: 

552 """The input data jointcal needs for each detector/visit.""" 

553 visit: int 

554 """The visit identifier of this exposure.""" 

555 catalog: lsst.afw.table.SourceCatalog 

556 """The catalog derived from this exposure.""" 

557 visitInfo: lsst.afw.image.VisitInfo 

558 """The VisitInfo of this exposure.""" 

559 detector: lsst.afw.cameraGeom.Detector 

560 """The detector of this exposure.""" 

561 photoCalib: lsst.afw.image.PhotoCalib 

562 """The photometric calibration of this exposure.""" 

563 wcs: lsst.afw.geom.skyWcs 

564 """The WCS of this exposure.""" 

565 bbox: lsst.geom.Box2I 

566 """The bounding box of this exposure.""" 

567 filter: lsst.afw.image.FilterLabel 

568 """The filter of this exposure.""" 

569 

570 

571class JointcalTask(pipeBase.PipelineTask): 

572 """Astrometricly and photometricly calibrate across multiple visits of the 

573 same field. 

574 """ 

575 

576 ConfigClass = JointcalConfig 

577 _DefaultName = "jointcal" 

578 

579 def __init__(self, **kwargs): 

580 super().__init__(**kwargs) 

581 self.makeSubtask("sourceSelector") 

582 if self.config.doAstrometry: 

583 self.makeSubtask("astrometryReferenceSelector") 

584 else: 

585 self.astrometryRefObjLoader = None 

586 if self.config.doPhotometry: 

587 self.makeSubtask("photometryReferenceSelector") 

588 else: 

589 self.photometryRefObjLoader = None 

590 

591 # To hold various computed metrics for use by tests 

592 self.job = Job.load_metrics_package(subset='jointcal') 

593 

594 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

595 # We override runQuantum to set up the refObjLoaders and write the 

596 # outputs to the correct refs. 

597 inputs = butlerQC.get(inputRefs) 

598 # We want the tract number for writing debug files 

599 tract = butlerQC.quantum.dataId['tract'] 

600 if self.config.doAstrometry: 

601 self.astrometryRefObjLoader = ReferenceObjectLoader( 

602 dataIds=[ref.datasetRef.dataId 

603 for ref in inputRefs.astrometryRefCat], 

604 refCats=inputs.pop('astrometryRefCat'), 

605 config=self.config.astrometryRefObjLoader, 

606 name=self.config.connections.astrometryRefCat, 

607 log=self.log) 

608 if self.config.doPhotometry: 

609 self.photometryRefObjLoader = ReferenceObjectLoader( 

610 dataIds=[ref.datasetRef.dataId 

611 for ref in inputRefs.photometryRefCat], 

612 refCats=inputs.pop('photometryRefCat'), 

613 config=self.config.photometryRefObjLoader, 

614 name=self.config.connections.photometryRefCat, 

615 log=self.log) 

616 outputs = self.run(**inputs, tract=tract) 

617 self._put_metrics(butlerQC, outputs.job, outputRefs) 

618 if self.config.doAstrometry: 

619 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs, 

620 inputs['inputCamera'], "setWcs") 

621 if self.config.doPhotometry: 

622 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib, 

623 inputs['inputCamera'], "setPhotoCalib") 

624 

625 def _put_metrics(self, butlerQC, job, outputRefs): 

626 """Persist all measured metrics stored in a job. 

627 

628 Parameters 

629 ---------- 

630 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

631 A butler which is specialized to operate in the context of a 

632 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

633 job : `lsst.verify.job.Job` 

634 Measurements of metrics to persist. 

635 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

636 The DatasetRefs to persist the data to. 

637 """ 

638 for key in job.measurements.keys(): 

639 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', ''))) 

640 

641 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter): 

642 """Persist the output datasets to their appropriate datarefs. 

643 

644 Parameters 

645 ---------- 

646 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

647 A butler which is specialized to operate in the context of a 

648 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

649 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or 

650 `dict` [`tuple, `lsst.afw.image.PhotoCalib`] 

651 The fitted objects to persist. 

652 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

653 The DatasetRefs to persist the data to. 

654 camera : `lsst.afw.cameraGeom.Camera` 

655 The camera for this instrument, to get detector ids from. 

656 setter : `str` 

657 The method to call on the ExposureCatalog to set each output. 

658 """ 

659 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

660 schema.addField('visit', type='L', doc='Visit number') 

661 

662 def new_catalog(visit, size): 

663 """Return an catalog ready to be filled with appropriate output.""" 

664 catalog = lsst.afw.table.ExposureCatalog(schema) 

665 catalog.resize(size) 

666 catalog['visit'] = visit 

667 metadata = lsst.daf.base.PropertyList() 

668 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

669 metadata.add("COMMENT", "Only detectors with data have entries.") 

670 return catalog 

671 

672 # count how many detectors have output for each visit 

673 detectors_per_visit = collections.defaultdict(int) 

674 for key in outputs: 

675 # key is (visit, detector_id), and we only need visit here 

676 detectors_per_visit[key[0]] += 1 

677 

678 for ref in outputRefs: 

679 visit = ref.dataId['visit'] 

680 catalog = new_catalog(visit, detectors_per_visit[visit]) 

681 # Iterate over every detector and skip the ones we don't have output for. 

682 i = 0 

683 for detector in camera: 

684 detectorId = detector.getId() 

685 key = (ref.dataId['visit'], detectorId) 

686 if key not in outputs: 

687 # skip detectors we don't have output for 

688 self.log.debug("No %s output for detector %s in visit %s", 

689 setter[3:], detectorId, visit) 

690 continue 

691 

692 catalog[i].setId(detectorId) 

693 getattr(catalog[i], setter)(outputs[key]) 

694 i += 1 

695 

696 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups 

697 butlerQC.put(catalog, ref) 

698 self.log.info("Wrote %s detectors to %s", i, ref) 

699 

700 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None): 

701 # Docstring inherited. 

702 

703 # We take values out of the Parquet table, and put them in "flux_", 

704 # and the config.sourceFluxType field is used during that extraction, 

705 # so just use "flux" here. 

706 sourceFluxField = "flux" 

707 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

708 associations = lsst.jointcal.Associations() 

709 self.focalPlaneBBox = inputCamera.getFpBBox() 

710 oldWcsList, bands = self._load_data(inputSourceTableVisit, 

711 inputVisitSummary, 

712 associations, 

713 jointcalControl, 

714 inputCamera) 

715 

716 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands) 

717 

718 if self.config.doAstrometry: 

719 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

720 name="astrometry", 

721 refObjLoader=self.astrometryRefObjLoader, 

722 referenceSelector=self.astrometryReferenceSelector, 

723 fit_function=self._fit_astrometry, 

724 tract=tract, 

725 epoch=epoch) 

726 astrometry_output = self._make_output(associations.getCcdImageList(), 

727 astrometry.model, 

728 "makeSkyWcs") 

729 else: 

730 astrometry_output = None 

731 

732 if self.config.doPhotometry: 

733 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

734 name="photometry", 

735 refObjLoader=self.photometryRefObjLoader, 

736 referenceSelector=self.photometryReferenceSelector, 

737 fit_function=self._fit_photometry, 

738 tract=tract, 

739 epoch=epoch, 

740 reject_bad_fluxes=True) 

741 photometry_output = self._make_output(associations.getCcdImageList(), 

742 photometry.model, 

743 "toPhotoCalib") 

744 else: 

745 photometry_output = None 

746 

747 return pipeBase.Struct(outputWcs=astrometry_output, 

748 outputPhotoCalib=photometry_output, 

749 job=self.job, 

750 astrometryRefObjLoader=self.astrometryRefObjLoader, 

751 photometryRefObjLoader=self.photometryRefObjLoader) 

752 

753 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, 

754 jointcalControl, camera): 

755 """Read the data that jointcal needs to run. 

756 

757 Modifies ``associations`` in-place with the loaded data. 

758 

759 Parameters 

760 ---------- 

761 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

762 References to visit-level DataFrames to load the catalog data from. 

763 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

764 Visit-level exposure summary catalog with metadata. 

765 associations : `lsst.jointcal.Associations` 

766 Object to add the loaded data to by constructing new CcdImages. 

767 jointcalControl : `jointcal.JointcalControl` 

768 Control object for C++ associations management. 

769 camera : `lsst.afw.cameraGeom.Camera` 

770 Camera object for detector geometry. 

771 

772 Returns 

773 ------- 

774 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`] 

775 The original WCS of the input data, to aid in writing tests. 

776 bands : `list` [`str`] 

777 The filter bands of each input dataset. 

778 """ 

779 oldWcsList = [] 

780 filters = [] 

781 load_cat_profile_file = 'jointcal_load_data.prof' if self.config.detailedProfile else '' 

782 with lsst.utils.timer.profile(load_cat_profile_file): 

783 table = make_schema_table() # every detector catalog has the same layout 

784 # No guarantee that the input is in the same order of visits, so we have to map one of them. 

785 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)} 

786 detectorDict = {detector.getId(): detector for detector in camera} 

787 

788 columns = None 

789 

790 for visitSummaryRef in inputVisitSummary: 

791 visitSummary = visitSummaryRef.get() 

792 

793 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]] 

794 if columns is None: 

795 inColumns = dataRef.get(component='columns') 

796 columns, detColumn, ixxColumns = get_sourceTable_visit_columns(inColumns, 

797 self.config, 

798 self.sourceSelector) 

799 visitCatalog = dataRef.get(parameters={'columns': columns}) 

800 

801 selected = self.sourceSelector.run(visitCatalog) 

802 

803 # Build a CcdImage for each detector in this visit. 

804 detectors = {id: index for index, id in enumerate(visitSummary['id'])} 

805 for id, index in detectors.items(): 

806 catalog = extract_detector_catalog_from_visit_catalog(table, 

807 selected.sourceCat, 

808 id, 

809 detColumn, 

810 ixxColumns, 

811 self.config.sourceFluxType, 

812 self.log) 

813 if catalog is None: 

814 continue 

815 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict) 

816 result = self._build_ccdImage(data, associations, jointcalControl) 

817 if result is not None: 

818 oldWcsList.append(result.wcs) 

819 # A visit has only one band, so we can just use the first. 

820 filters.append(data.filter) 

821 if len(filters) == 0: 

822 raise RuntimeError("No data to process: did source selector remove all sources?") 

823 filters = collections.Counter(filters) 

824 

825 return oldWcsList, filters 

826 

827 def _make_one_input_data(self, visitRecord, catalog, detectorDict): 

828 """Return a data structure for this detector+visit.""" 

829 return JointcalInputData(visit=visitRecord['visit'], 

830 catalog=catalog, 

831 visitInfo=visitRecord.getVisitInfo(), 

832 detector=detectorDict[visitRecord.getId()], 

833 photoCalib=visitRecord.getPhotoCalib(), 

834 wcs=visitRecord.getWcs(), 

835 bbox=visitRecord.getBBox(), 

836 # ExposureRecord doesn't have a FilterLabel yet, 

837 # so we have to make one. 

838 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'], 

839 physical=visitRecord['physical_filter'])) 

840 

841 def _build_ccdImage(self, data, associations, jointcalControl): 

842 """ 

843 Extract the necessary things from this catalog+metadata to add a new 

844 ccdImage. 

845 

846 Parameters 

847 ---------- 

848 data : `JointcalInputData` 

849 The loaded input data. 

850 associations : `lsst.jointcal.Associations` 

851 Object to add the info to, to construct a new CcdImage 

852 jointcalControl : `jointcal.JointcalControl` 

853 Control object for associations management 

854 

855 Returns 

856 ------ 

857 namedtuple or `None` 

858 ``wcs`` 

859 The TAN WCS of this image, read from the calexp 

860 (`lsst.afw.geom.SkyWcs`). 

861 ``key`` 

862 A key to identify this dataRef by its visit and ccd ids 

863 (`namedtuple`). 

864 `None` 

865 if there are no sources in the loaded catalog. 

866 """ 

867 if len(data.catalog) == 0: 

868 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId()) 

869 return None 

870 

871 associations.createCcdImage(data.catalog, 

872 data.wcs, 

873 data.visitInfo, 

874 data.bbox, 

875 data.filter.physicalLabel, 

876 data.photoCalib, 

877 data.detector, 

878 data.visit, 

879 data.detector.getId(), 

880 jointcalControl) 

881 

882 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key')) 

883 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

884 return Result(data.wcs, Key(data.visit, data.detector.getId())) 

885 

886 def _getDebugPath(self, filename): 

887 """Constructs a path to filename using the configured debug path. 

888 """ 

889 return os.path.join(self.config.debugOutputPath, filename) 

890 

891 def _prep_sky(self, associations, filters): 

892 """Prepare on-sky and other data that must be computed after data has 

893 been read. 

894 """ 

895 associations.computeCommonTangentPoint() 

896 

897 boundingCircle = associations.computeBoundingCircle() 

898 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

899 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

900 

901 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

902 

903 # Determine a default filter band associated with the catalog. See DM-9093 

904 defaultFilter = filters.most_common(1)[0][0] 

905 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel) 

906 

907 # compute and set the reference epoch of the observations, for proper motion corrections 

908 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

909 associations.setEpoch(epoch.jyear) 

910 

911 return boundingCircle, center, radius, defaultFilter, epoch 

912 

913 def _get_refcat_coordinate_error_override(self, refCat, name): 

914 """Check whether we should override the refcat coordinate errors, and 

915 return the overridden error if necessary. 

916 

917 Parameters 

918 ---------- 

919 refCat : `lsst.afw.table.SimpleCatalog` 

920 The reference catalog to check for a ``coord_raErr`` field. 

921 name : `str` 

922 Whether we are doing "astrometry" or "photometry". 

923 

924 Returns 

925 ------- 

926 refCoordErr : `float` 

927 The refcat coordinate error to use, or NaN if we are not overriding 

928 those fields. 

929 

930 Raises 

931 ------ 

932 lsst.pex.config.FieldValidationError 

933 Raised if the refcat does not contain coordinate errors and 

934 ``config.astrometryReferenceErr`` is not set. 

935 """ 

936 # This value doesn't matter for photometry, so just set something to 

937 # keep old refcats from causing problems. 

938 if name.lower() == "photometry": 

939 if 'coord_raErr' not in refCat.schema: 

940 return 100 

941 else: 

942 return float('nan') 

943 

944 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

945 msg = ("Reference catalog does not contain coordinate errors, " 

946 "and config.astrometryReferenceErr not supplied.") 

947 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

948 self.config, 

949 msg) 

950 

951 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

952 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

953 self.config.astrometryReferenceErr) 

954 

955 if self.config.astrometryReferenceErr is None: 

956 return float('nan') 

957 else: 

958 return self.config.astrometryReferenceErr 

959 

960 def _compute_proper_motion_epoch(self, ccdImageList): 

961 """Return the proper motion correction epoch of the provided images. 

962 

963 Parameters 

964 ---------- 

965 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

966 The images to compute the appropriate epoch for. 

967 

968 Returns 

969 ------- 

970 epoch : `astropy.time.Time` 

971 The date to use for proper motion corrections. 

972 """ 

973 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]), 

974 format="jyear", 

975 scale="tai") 

976 

977 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

978 tract="", match_cut=3.0, 

979 reject_bad_fluxes=False, *, 

980 name="", refObjLoader=None, referenceSelector=None, 

981 fit_function=None, epoch=None): 

982 """Load reference catalog, perform the fit, and return the result. 

983 

984 Parameters 

985 ---------- 

986 associations : `lsst.jointcal.Associations` 

987 The star/reference star associations to fit. 

988 defaultFilter : `lsst.afw.image.FilterLabel` 

989 filter to load from reference catalog. 

990 center : `lsst.geom.SpherePoint` 

991 ICRS center of field to load from reference catalog. 

992 radius : `lsst.geom.Angle` 

993 On-sky radius to load from reference catalog. 

994 name : `str` 

995 Name of thing being fit: "astrometry" or "photometry". 

996 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

997 Reference object loader to use to load a reference catalog. 

998 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

999 Selector to use to pick objects from the loaded reference catalog. 

1000 fit_function : callable 

1001 Function to call to perform fit (takes Associations object). 

1002 tract : `str`, optional 

1003 Name of tract currently being fit. 

1004 match_cut : `float`, optional 

1005 Radius in arcseconds to find cross-catalog matches to during 

1006 associations.associateCatalogs. 

1007 reject_bad_fluxes : `bool`, optional 

1008 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

1009 epoch : `astropy.time.Time`, optional 

1010 Epoch to which to correct refcat proper motion and parallax, 

1011 or `None` to not apply such corrections. 

1012 

1013 Returns 

1014 ------- 

1015 result : `Photometry` or `Astrometry` 

1016 Result of `fit_function()` 

1017 """ 

1018 self.log.info("====== Now processing %s...", name) 

1019 # TODO: this should not print "trying to invert a singular transformation:" 

1020 # if it does that, something's not right about the WCS... 

1021 associations.associateCatalogs(match_cut) 

1022 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name, 

1023 associations.fittedStarListSize()) 

1024 

1025 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

1026 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

1027 center, radius, defaultFilter, 

1028 applyColorterms=applyColorterms, 

1029 epoch=epoch) 

1030 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

1031 

1032 associations.collectRefStars(refCat, 

1033 self.config.matchCut*lsst.geom.arcseconds, 

1034 fluxField, 

1035 refCoordinateErr=refCoordErr, 

1036 rejectBadFluxes=reject_bad_fluxes) 

1037 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name, 

1038 associations.refStarListSize()) 

1039 

1040 associations.prepareFittedStars(self.config.minMeasurements) 

1041 

1042 self._check_star_lists(associations, name) 

1043 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name, 

1044 associations.nFittedStarsWithAssociatedRefStar()) 

1045 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name, 

1046 associations.fittedStarListSize()) 

1047 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name, 

1048 associations.nCcdImagesValidForFit()) 

1049 

1050 fit_profile_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else '' 

1051 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel) 

1052 with lsst.utils.timer.profile(fit_profile_file): 

1053 result = fit_function(associations, dataName) 

1054 # TODO DM-12446: turn this into a "butler save" somehow. 

1055 # Save reference and measurement chi2 contributions for this data 

1056 if self.config.writeChi2FilesInitialFinal: 

1057 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

1058 result.fit.saveChi2Contributions(baseName+"{type}") 

1059 self.log.info("Wrote chi2 contributions files: %s", baseName) 

1060 

1061 return result 

1062 

1063 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, 

1064 applyColorterms=False, epoch=None): 

1065 """Load the necessary reference catalog sources, convert fluxes to 

1066 correct units, and apply color term corrections if requested. 

1067 

1068 Parameters 

1069 ---------- 

1070 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

1071 The reference catalog loader to use to get the data. 

1072 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1073 Source selector to apply to loaded reference catalog. 

1074 center : `lsst.geom.SpherePoint` 

1075 The center around which to load sources. 

1076 radius : `lsst.geom.Angle` 

1077 The radius around ``center`` to load sources in. 

1078 filterLabel : `lsst.afw.image.FilterLabel` 

1079 The camera filter to load fluxes for. 

1080 applyColorterms : `bool` 

1081 Apply colorterm corrections to the refcat for ``filterName``? 

1082 epoch : `astropy.time.Time`, optional 

1083 Epoch to which to correct refcat proper motion and parallax, 

1084 or `None` to not apply such corrections. 

1085 

1086 Returns 

1087 ------- 

1088 refCat : `lsst.afw.table.SimpleCatalog` 

1089 The loaded reference catalog. 

1090 fluxField : `str` 

1091 The name of the reference catalog flux field appropriate for ``filterName``. 

1092 """ 

1093 skyCircle = refObjLoader.loadSkyCircle(center, 

1094 radius, 

1095 filterLabel.bandLabel, 

1096 epoch=epoch) 

1097 

1098 selected = referenceSelector.run(skyCircle.refCat) 

1099 # Need memory contiguity to get reference filters as a vector. 

1100 if not selected.sourceCat.isContiguous(): 

1101 refCat = selected.sourceCat.copy(deep=True) 

1102 else: 

1103 refCat = selected.sourceCat 

1104 

1105 if applyColorterms: 

1106 refCatName = refObjLoader.name 

1107 self.log.info("Applying color terms for physical filter=%r reference catalog=%s", 

1108 filterLabel.physicalLabel, refCatName) 

1109 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel, 

1110 refCatName, 

1111 doRaise=True) 

1112 

1113 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

1114 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

1115 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

1116 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

1117 

1118 return refCat, skyCircle.fluxField 

1119 

1120 def _check_star_lists(self, associations, name): 

1121 # TODO: these should be len(blah), but we need this properly wrapped first. 

1122 if associations.nCcdImagesValidForFit() == 0: 

1123 raise RuntimeError('No images in the ccdImageList!') 

1124 if associations.fittedStarListSize() == 0: 

1125 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

1126 if associations.refStarListSize() == 0: 

1127 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

1128 

1129 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

1130 """Compute chi2, log it, validate the model, and return chi2. 

1131 

1132 Parameters 

1133 ---------- 

1134 associations : `lsst.jointcal.Associations` 

1135 The star/reference star associations to fit. 

1136 fit : `lsst.jointcal.FitterBase` 

1137 The fitter to use for minimization. 

1138 model : `lsst.jointcal.Model` 

1139 The model being fit. 

1140 chi2Label : `str` 

1141 Label to describe the chi2 (e.g. "Initialized", "Final"). 

1142 writeChi2Name : `str`, optional 

1143 Filename prefix to write the chi2 contributions to. 

1144 Do not supply an extension: an appropriate one will be added. 

1145 

1146 Returns 

1147 ------- 

1148 chi2: `lsst.jointcal.Chi2Accumulator` 

1149 The chi2 object for the current fitter and model. 

1150 

1151 Raises 

1152 ------ 

1153 FloatingPointError 

1154 Raised if chi2 is infinite or NaN. 

1155 ValueError 

1156 Raised if the model is not valid. 

1157 """ 

1158 if writeChi2Name is not None: 

1159 fullpath = self._getDebugPath(writeChi2Name) 

1160 fit.saveChi2Contributions(fullpath+"{type}") 

1161 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

1162 

1163 chi2 = fit.computeChi2() 

1164 self.log.info("%s %s", chi2Label, chi2) 

1165 self._check_stars(associations) 

1166 if not np.isfinite(chi2.chi2): 

1167 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

1168 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

1169 raise ValueError("Model is not valid: check log messages for warnings.") 

1170 return chi2 

1171 

1172 def _fit_photometry(self, associations, dataName=None): 

1173 """ 

1174 Fit the photometric data. 

1175 

1176 Parameters 

1177 ---------- 

1178 associations : `lsst.jointcal.Associations` 

1179 The star/reference star associations to fit. 

1180 dataName : `str` 

1181 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1182 identifying debugging files. 

1183 

1184 Returns 

1185 ------- 

1186 fit_result : `namedtuple` 

1187 fit : `lsst.jointcal.PhotometryFit` 

1188 The photometric fitter used to perform the fit. 

1189 model : `lsst.jointcal.PhotometryModel` 

1190 The photometric model that was fit. 

1191 """ 

1192 self.log.info("=== Starting photometric fitting...") 

1193 

1194 # TODO: should use pex.config.RegistryField here (see DM-9195) 

1195 if self.config.photometryModel == "constrainedFlux": 

1196 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

1197 self.focalPlaneBBox, 

1198 visitOrder=self.config.photometryVisitOrder, 

1199 errorPedestal=self.config.photometryErrorPedestal) 

1200 # potentially nonlinear problem, so we may need a line search to converge. 

1201 doLineSearch = self.config.allowLineSearch 

1202 elif self.config.photometryModel == "constrainedMagnitude": 

1203 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

1204 self.focalPlaneBBox, 

1205 visitOrder=self.config.photometryVisitOrder, 

1206 errorPedestal=self.config.photometryErrorPedestal) 

1207 # potentially nonlinear problem, so we may need a line search to converge. 

1208 doLineSearch = self.config.allowLineSearch 

1209 elif self.config.photometryModel == "simpleFlux": 

1210 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

1211 errorPedestal=self.config.photometryErrorPedestal) 

1212 doLineSearch = False # purely linear in model parameters, so no line search needed 

1213 elif self.config.photometryModel == "simpleMagnitude": 

1214 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

1215 errorPedestal=self.config.photometryErrorPedestal) 

1216 doLineSearch = False # purely linear in model parameters, so no line search needed 

1217 

1218 fit = lsst.jointcal.PhotometryFit(associations, model) 

1219 # TODO DM-12446: turn this into a "butler save" somehow. 

1220 # Save reference and measurement chi2 contributions for this data 

1221 if self.config.writeChi2FilesInitialFinal: 

1222 baseName = f"photometry_initial_chi2-{dataName}" 

1223 else: 

1224 baseName = None 

1225 if self.config.writeInitialModel: 

1226 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt") 

1227 writeModel(model, fullpath, self.log) 

1228 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

1229 

1230 def getChi2Name(whatToFit): 

1231 if self.config.writeChi2FilesOuterLoop: 

1232 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

1233 else: 

1234 return None 

1235 

1236 # The constrained model needs the visit transform fit first; the chip 

1237 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

1238 if self.config.writeInitMatrix: 

1239 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}") 

1240 else: 

1241 dumpMatrixFile = "" 

1242 if self.config.photometryModel.startswith("constrained"): 

1243 # no line search: should be purely (or nearly) linear, 

1244 # and we want a large step size to initialize with. 

1245 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

1246 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

1247 writeChi2Name=getChi2Name("ModelVisit")) 

1248 dumpMatrixFile = "" # so we don't redo the output on the next step 

1249 

1250 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

1251 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

1252 writeChi2Name=getChi2Name("Model")) 

1253 

1254 fit.minimize("Fluxes") # no line search: always purely linear. 

1255 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

1256 writeChi2Name=getChi2Name("Fluxes")) 

1257 

1258 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

1259 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

1260 writeChi2Name=getChi2Name("ModelFluxes")) 

1261 

1262 model.freezeErrorTransform() 

1263 self.log.debug("Photometry error scales are frozen.") 

1264 

1265 chi2 = self._iterate_fit(associations, 

1266 fit, 

1267 self.config.maxPhotometrySteps, 

1268 "photometry", 

1269 "Model Fluxes", 

1270 doRankUpdate=self.config.photometryDoRankUpdate, 

1271 doLineSearch=doLineSearch, 

1272 dataName=dataName) 

1273 

1274 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

1275 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

1276 return Photometry(fit, model) 

1277 

1278 def _fit_astrometry(self, associations, dataName=None): 

1279 """ 

1280 Fit the astrometric data. 

1281 

1282 Parameters 

1283 ---------- 

1284 associations : `lsst.jointcal.Associations` 

1285 The star/reference star associations to fit. 

1286 dataName : `str` 

1287 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1288 identifying debugging files. 

1289 

1290 Returns 

1291 ------- 

1292 fit_result : `namedtuple` 

1293 fit : `lsst.jointcal.AstrometryFit` 

1294 The astrometric fitter used to perform the fit. 

1295 model : `lsst.jointcal.AstrometryModel` 

1296 The astrometric model that was fit. 

1297 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

1298 The model for the sky to tangent plane projection that was used in the fit. 

1299 """ 

1300 

1301 self.log.info("=== Starting astrometric fitting...") 

1302 

1303 associations.deprojectFittedStars() 

1304 

1305 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1306 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1307 # them so carefully? 

1308 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1309 

1310 if self.config.astrometryModel == "constrained": 

1311 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1312 sky_to_tan_projection, 

1313 chipOrder=self.config.astrometryChipOrder, 

1314 visitOrder=self.config.astrometryVisitOrder) 

1315 elif self.config.astrometryModel == "simple": 

1316 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1317 sky_to_tan_projection, 

1318 self.config.useInputWcs, 

1319 nNotFit=0, 

1320 order=self.config.astrometrySimpleOrder) 

1321 

1322 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1323 # TODO DM-12446: turn this into a "butler save" somehow. 

1324 # Save reference and measurement chi2 contributions for this data 

1325 if self.config.writeChi2FilesInitialFinal: 

1326 baseName = f"astrometry_initial_chi2-{dataName}" 

1327 else: 

1328 baseName = None 

1329 if self.config.writeInitialModel: 

1330 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt") 

1331 writeModel(model, fullpath, self.log) 

1332 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1333 

1334 def getChi2Name(whatToFit): 

1335 if self.config.writeChi2FilesOuterLoop: 

1336 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1337 else: 

1338 return None 

1339 

1340 if self.config.writeInitMatrix: 

1341 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}") 

1342 else: 

1343 dumpMatrixFile = "" 

1344 # The constrained model needs the visit transform fit first; the chip 

1345 # transform is initialized from the detector's cameraGeom, so it's close. 

1346 if self.config.astrometryModel == "constrained": 

1347 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1348 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1349 writeChi2Name=getChi2Name("DistortionsVisit")) 

1350 dumpMatrixFile = "" # so we don't redo the output on the next step 

1351 

1352 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1353 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1354 writeChi2Name=getChi2Name("Distortions")) 

1355 

1356 fit.minimize("Positions") 

1357 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1358 writeChi2Name=getChi2Name("Positions")) 

1359 

1360 fit.minimize("Distortions Positions") 

1361 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1362 writeChi2Name=getChi2Name("DistortionsPositions")) 

1363 

1364 chi2 = self._iterate_fit(associations, 

1365 fit, 

1366 self.config.maxAstrometrySteps, 

1367 "astrometry", 

1368 "Distortions Positions", 

1369 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance, 

1370 doRankUpdate=self.config.astrometryDoRankUpdate, 

1371 dataName=dataName) 

1372 

1373 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1374 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1375 

1376 return Astrometry(fit, model, sky_to_tan_projection) 

1377 

1378 def _check_stars(self, associations): 

1379 """Count measured and reference stars per ccd and warn/log them.""" 

1380 for ccdImage in associations.getCcdImageList(): 

1381 nMeasuredStars, nRefStars = ccdImage.countStars() 

1382 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1383 ccdImage.getName(), nMeasuredStars, nRefStars) 

1384 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1385 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)", 

1386 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1387 if nRefStars < self.config.minRefStarsPerCcd: 

1388 self.log.warning("ccdImage %s has only %s RefStars (desired %s)", 

1389 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1390 

1391 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1392 dataName="", 

1393 sigmaRelativeTolerance=0, 

1394 doRankUpdate=True, 

1395 doLineSearch=False): 

1396 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1397 

1398 Parameters 

1399 ---------- 

1400 associations : `lsst.jointcal.Associations` 

1401 The star/reference star associations to fit. 

1402 fitter : `lsst.jointcal.FitterBase` 

1403 The fitter to use for minimization. 

1404 max_steps : `int` 

1405 Maximum number of steps to run outlier rejection before declaring 

1406 convergence failure. 

1407 name : {'photometry' or 'astrometry'} 

1408 What type of data are we fitting (for logs and debugging files). 

1409 whatToFit : `str` 

1410 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1411 dataName : `str`, optional 

1412 Descriptive name for this dataset (e.g. tract and filter), 

1413 for debugging. 

1414 sigmaRelativeTolerance : `float`, optional 

1415 Convergence tolerance for the fractional change in the chi2 cut 

1416 level for determining outliers. If set to zero, iterations will 

1417 continue until there are no outliers. 

1418 doRankUpdate : `bool`, optional 

1419 Do an Eigen rank update during minimization, or recompute the full 

1420 matrix and gradient? 

1421 doLineSearch : `bool`, optional 

1422 Do a line search for the optimum step during minimization? 

1423 

1424 Returns 

1425 ------- 

1426 chi2: `lsst.jointcal.Chi2Statistic` 

1427 The final chi2 after the fit converges, or is forced to end. 

1428 

1429 Raises 

1430 ------ 

1431 FloatingPointError 

1432 Raised if the fitter fails with a non-finite value. 

1433 RuntimeError 

1434 Raised if the fitter fails for some other reason; 

1435 log messages will provide further details. 

1436 """ 

1437 if self.config.writeInitMatrix: 

1438 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}") 

1439 else: 

1440 dumpMatrixFile = "" 

1441 oldChi2 = lsst.jointcal.Chi2Statistic() 

1442 oldChi2.chi2 = float("inf") 

1443 for i in range(max_steps): 

1444 if self.config.writeChi2FilesOuterLoop: 

1445 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1446 else: 

1447 writeChi2Name = None 

1448 result = fitter.minimize(whatToFit, 

1449 self.config.outlierRejectSigma, 

1450 sigmaRelativeTolerance=sigmaRelativeTolerance, 

1451 doRankUpdate=doRankUpdate, 

1452 doLineSearch=doLineSearch, 

1453 dumpMatrixFile=dumpMatrixFile) 

1454 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1455 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1456 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1457 

1458 if result == MinimizeResult.Converged: 

1459 if doRankUpdate: 

1460 self.log.debug("fit has converged - no more outliers - redo minimization " 

1461 "one more time in case we have lost accuracy in rank update.") 

1462 # Redo minimization one more time in case we have lost accuracy in rank update 

1463 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma, 

1464 sigmaRelativeTolerance=sigmaRelativeTolerance) 

1465 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1466 

1467 # log a message for a large final chi2, TODO: DM-15247 for something better 

1468 if chi2.chi2/chi2.ndof >= 4.0: 

1469 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1470 

1471 break 

1472 elif result == MinimizeResult.Chi2Increased: 

1473 self.log.warning("Still some outliers remaining but chi2 increased - retry") 

1474 # Check whether the increase was large enough to cause trouble. 

1475 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1476 if chi2Ratio > 1.5: 

1477 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1478 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1479 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1480 # appears that chi2 increases more than a factor of ~2 always 

1481 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1482 # Using 10 as the "failure" threshold gives some room between 

1483 # leaving a warning and bailing early. 

1484 if chi2Ratio > 10: 

1485 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1486 " Try setting one or more of the `writeChi2*` config fields and looking" 

1487 " at how individual star chi2-values evolve during the fit.") 

1488 raise RuntimeError(msg) 

1489 oldChi2 = chi2 

1490 elif result == MinimizeResult.NonFinite: 

1491 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1492 # TODO DM-12446: turn this into a "butler save" somehow. 

1493 fitter.saveChi2Contributions(filename+"{type}") 

1494 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1495 raise FloatingPointError(msg.format(filename)) 

1496 elif result == MinimizeResult.Failed: 

1497 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1498 else: 

1499 raise RuntimeError("Unxepected return code from minimize().") 

1500 else: 

1501 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1502 

1503 return chi2 

1504 

1505 def _make_output(self, ccdImageList, model, func): 

1506 """Return the internal jointcal models converted to the afw 

1507 structures that will be saved to disk. 

1508 

1509 Parameters 

1510 ---------- 

1511 ccdImageList : `lsst.jointcal.CcdImageList` 

1512 The list of CcdImages to get the output for. 

1513 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel` 

1514 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`. 

1515 func : `str` 

1516 The name of the function to call on ``model`` to get the converted 

1517 structure. Must accept an `lsst.jointcal.CcdImage`. 

1518 

1519 Returns 

1520 ------- 

1521 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or 

1522 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`] 

1523 The data to be saved, keyed on (visit, detector). 

1524 """ 

1525 output = {} 

1526 for ccdImage in ccdImageList: 

1527 ccd = ccdImage.ccdId 

1528 visit = ccdImage.visit 

1529 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd) 

1530 output[(visit, ccd)] = getattr(model, func)(ccdImage) 

1531 return output 

1532 

1533 

1534def make_schema_table(): 

1535 """Return an afw SourceTable to use as a base for creating the 

1536 SourceCatalog to insert values from the dataFrame into. 

1537 

1538 Returns 

1539 ------- 

1540 table : `lsst.afw.table.SourceTable` 

1541 Table with schema and slots to use to make SourceCatalogs. 

1542 """ 

1543 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

1544 schema.addField("centroid_x", "D") 

1545 schema.addField("centroid_y", "D") 

1546 schema.addField("centroid_xErr", "F") 

1547 schema.addField("centroid_yErr", "F") 

1548 schema.addField("shape_xx", "D") 

1549 schema.addField("shape_yy", "D") 

1550 schema.addField("shape_xy", "D") 

1551 schema.addField("flux_instFlux", "D") 

1552 schema.addField("flux_instFluxErr", "D") 

1553 table = lsst.afw.table.SourceTable.make(schema) 

1554 table.defineCentroid("centroid") 

1555 table.defineShape("shape") 

1556 return table 

1557 

1558 

1559def get_sourceTable_visit_columns(inColumns, config, sourceSelector): 

1560 """ 

1561 Get the sourceTable_visit columns to load from the catalogs. 

1562 

1563 Parameters 

1564 ---------- 

1565 inColumns : `list` 

1566 List of columns known to be available in the sourceTable_visit. 

1567 config : `JointcalConfig` 

1568 A filled-in config to to help define column names. 

1569 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask` 

1570 A configured source selector to define column names to load. 

1571 

1572 Returns 

1573 ------- 

1574 columns : `list` 

1575 List of columns to read from sourceTable_visit. 

1576 detectorColumn : `str` 

1577 Name of the detector column. 

1578 ixxColumns : `list` 

1579 Name of the ixx/iyy/ixy columns. 

1580 """ 

1581 if 'detector' in inColumns: 

1582 # Default name for Gen3. 

1583 detectorColumn = 'detector' 

1584 else: 

1585 # Default name for Gen2 conversions (still used in tests, CI, and older catalogs) 

1586 detectorColumn = 'ccd' 

1587 

1588 columns = ['visit', detectorColumn, 

1589 'sourceId', 'x', 'xErr', 'y', 'yErr', 

1590 config.sourceFluxType + '_instFlux', config.sourceFluxType + '_instFluxErr'] 

1591 

1592 if 'ixx' in inColumns: 

1593 # New columns post-DM-31825 

1594 ixxColumns = ['ixx', 'iyy', 'ixy'] 

1595 else: 

1596 # Old columns pre-DM-31825 

1597 ixxColumns = ['Ixx', 'Iyy', 'Ixy'] 

1598 columns.extend(ixxColumns) 

1599 

1600 if sourceSelector.config.doFlags: 

1601 columns.extend(sourceSelector.config.flags.bad) 

1602 if sourceSelector.config.doUnresolved: 

1603 columns.append(sourceSelector.config.unresolved.name) 

1604 if sourceSelector.config.doIsolated: 

1605 columns.append(sourceSelector.config.isolated.parentName) 

1606 columns.append(sourceSelector.config.isolated.nChildName) 

1607 

1608 return columns, detectorColumn, ixxColumns 

1609 

1610 

1611def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId, 

1612 detectorColumn, ixxColumns, sourceFluxType, log): 

1613 """Return an afw SourceCatalog extracted from a visit-level dataframe, 

1614 limited to just one detector. 

1615 

1616 Parameters 

1617 ---------- 

1618 table : `lsst.afw.table.SourceTable` 

1619 Table factory to use to make the SourceCatalog that will be 

1620 populated with data from ``visitCatalog``. 

1621 visitCatalog : `pandas.DataFrame` 

1622 DataFrame to extract a detector catalog from. 

1623 detectorId : `int` 

1624 Numeric id of the detector to extract from ``visitCatalog``. 

1625 detectorColumn : `str` 

1626 Name of the detector column in the catalog. 

1627 ixxColumns : `list` [`str`] 

1628 Names of the ixx/iyy/ixy columns in the catalog. 

1629 sourceFluxType : `str` 

1630 Name of the catalog field to load instFluxes from. 

1631 log : `logging.Logger` 

1632 Logging instance to log to. 

1633 

1634 Returns 

1635 ------- 

1636 catalog : `lsst.afw.table.SourceCatalog`, or `None` 

1637 Detector-level catalog extracted from ``visitCatalog``, or `None` 

1638 if there was no data to load. 

1639 """ 

1640 # map from dataFrame column to afw table column 

1641 mapping = {'x': 'centroid_x', 

1642 'y': 'centroid_y', 

1643 'xErr': 'centroid_xErr', 

1644 'yErr': 'centroid_yErr', 

1645 ixxColumns[0]: 'shape_xx', 

1646 ixxColumns[1]: 'shape_yy', 

1647 ixxColumns[2]: 'shape_xy', 

1648 f'{sourceFluxType}_instFlux': 'flux_instFlux', 

1649 f'{sourceFluxType}_instFluxErr': 'flux_instFluxErr', 

1650 } 

1651 

1652 catalog = lsst.afw.table.SourceCatalog(table) 

1653 matched = visitCatalog[detectorColumn] == detectorId 

1654 n = sum(matched) 

1655 if n == 0: 

1656 return None 

1657 catalog.resize(sum(matched)) 

1658 view = visitCatalog.loc[matched] 

1659 catalog['id'] = view.index 

1660 for dfCol, afwCol in mapping.items(): 

1661 catalog[afwCol] = view[dfCol] 

1662 

1663 log.debug("%d sources selected in visit %d detector %d", 

1664 len(catalog), 

1665 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first 

1666 detectorId) 

1667 return catalog