Coverage for python/lsst/jointcal/jointcal.py: 19%

534 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-19 13:25 -0700

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import dataclasses 

23import collections 

24import os 

25import logging 

26 

27import astropy.time 

28import numpy as np 

29import astropy.units as u 

30 

31import lsst.geom 

32import lsst.utils 

33import lsst.pex.config as pexConfig 

34import lsst.pipe.base as pipeBase 

35from lsst.afw.image import fluxErrFromABMagErr 

36import lsst.afw.cameraGeom 

37import lsst.afw.table 

38from lsst.pipe.base import Instrument 

39from lsst.pipe.tasks.colorterms import ColortermLibrary 

40from lsst.verify import Job, Measurement 

41 

42from lsst.meas.algorithms import (ReferenceObjectLoader, ReferenceSourceSelectorTask, 

43 LoadIndexedReferenceObjectsConfig) 

44from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

45 

46import lsst.jointcal 

47from lsst.jointcal import MinimizeResult 

48 

49__all__ = ["JointcalConfig", "JointcalTask"] 

50 

51Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

52Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

53 

54 

55# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

56def add_measurement(job, name, value): 

57 meas = Measurement(job.metrics[name], value) 

58 job.measurements.insert(meas) 

59 

60 

61def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

62 """Lookup function that asserts/hopes that a static calibration dataset 

63 exists in a particular collection, since this task can't provide a single 

64 date/time to use to search for one properly. 

65 

66 This is mostly useful for the ``camera`` dataset, in cases where the task's 

67 quantum dimensions do *not* include something temporal, like ``exposure`` 

68 or ``visit``. 

69 

70 Parameters 

71 ---------- 

72 datasetType : `lsst.daf.butler.DatasetType` 

73 Type of dataset being searched for. 

74 registry : `lsst.daf.butler.Registry` 

75 Data repository registry to search. 

76 quantumDataId : `lsst.daf.butler.DataCoordinate` 

77 Data ID of the quantum this camera should match. 

78 collections : `Iterable` [ `str` ] 

79 Collections that should be searched - but this lookup function works 

80 by ignoring this in favor of a more-or-less hard-coded value. 

81 

82 Returns 

83 ------- 

84 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

85 Iterator over dataset references; should have only one element. 

86 

87 Notes 

88 ----- 

89 This implementation duplicates one in fgcmcal, and is at least quite 

90 similar to another in cp_pipe. This duplicate has the most documentation. 

91 Fixing this is DM-29661. 

92 """ 

93 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

94 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

95 return registry.queryDatasets(datasetType, 

96 dataId=quantumDataId, 

97 collections=[unboundedCollection], 

98 findFirst=True) 

99 

100 

101def lookupVisitRefCats(datasetType, registry, quantumDataId, collections): 

102 """Lookup function that finds all refcats for all visits that overlap a 

103 tract, rather than just the refcats that directly overlap the tract. 

104 

105 Parameters 

106 ---------- 

107 datasetType : `lsst.daf.butler.DatasetType` 

108 Type of dataset being searched for. 

109 registry : `lsst.daf.butler.Registry` 

110 Data repository registry to search. 

111 quantumDataId : `lsst.daf.butler.DataCoordinate` 

112 Data ID of the quantum; expected to be something we can use as a 

113 constraint to query for overlapping visits. 

114 collections : `Iterable` [ `str` ] 

115 Collections to search. 

116 

117 Returns 

118 ------- 

119 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

120 Iterator over refcat references. 

121 """ 

122 refs = set() 

123 # Use .expanded() on the query methods below because we need data IDs with 

124 # regions, both in the outer loop over visits (queryDatasets will expand 

125 # any data ID we give it, but doing it up-front in bulk is much more 

126 # efficient) and in the data IDs of the DatasetRefs this function yields 

127 # (because the RefCatLoader relies on them to do some of its own 

128 # filtering). 

129 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()): 

130 refs.update( 

131 registry.queryDatasets( 

132 datasetType, 

133 collections=collections, 

134 dataId=visit_data_id, 

135 findFirst=True, 

136 ).expanded() 

137 ) 

138 yield from refs 

139 

140 

141class JointcalTaskConnections(pipeBase.PipelineTaskConnections, 

142 dimensions=("skymap", "tract", "instrument", "physical_filter")): 

143 """Middleware input/output connections for jointcal data.""" 

144 inputCamera = pipeBase.connectionTypes.PrerequisiteInput( 

145 doc="The camera instrument that took these observations.", 

146 name="camera", 

147 storageClass="Camera", 

148 dimensions=("instrument",), 

149 isCalibration=True, 

150 lookupFunction=lookupStaticCalibrations, 

151 ) 

152 inputSourceTableVisit = pipeBase.connectionTypes.Input( 

153 doc="Source table in parquet format, per visit", 

154 name="sourceTable_visit", 

155 storageClass="DataFrame", 

156 dimensions=("instrument", "visit"), 

157 deferLoad=True, 

158 multiple=True, 

159 ) 

160 inputVisitSummary = pipeBase.connectionTypes.Input( 

161 doc=("Per-visit consolidated exposure metadata built from calexps. " 

162 "These catalogs use detector id for the id and must be sorted for " 

163 "fast lookups of a detector."), 

164 name="visitSummary", 

165 storageClass="ExposureCatalog", 

166 dimensions=("instrument", "visit"), 

167 deferLoad=True, 

168 multiple=True, 

169 ) 

170 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

171 doc="The astrometry reference catalog to match to loaded input catalog sources.", 

172 name="gaia_dr2_20200414", 

173 storageClass="SimpleCatalog", 

174 dimensions=("skypix",), 

175 deferLoad=True, 

176 multiple=True, 

177 lookupFunction=lookupVisitRefCats, 

178 ) 

179 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

180 doc="The photometry reference catalog to match to loaded input catalog sources.", 

181 name="ps1_pv3_3pi_20170110", 

182 storageClass="SimpleCatalog", 

183 dimensions=("skypix",), 

184 deferLoad=True, 

185 multiple=True, 

186 lookupFunction=lookupVisitRefCats, 

187 ) 

188 

189 outputWcs = pipeBase.connectionTypes.Output( 

190 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model." 

191 " These catalogs only contain entries for detectors with an output, and use" 

192 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

193 name="jointcalSkyWcsCatalog", 

194 storageClass="ExposureCatalog", 

195 dimensions=("instrument", "visit", "skymap", "tract"), 

196 multiple=True 

197 ) 

198 outputPhotoCalib = pipeBase.connectionTypes.Output( 

199 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model." 

200 " These catalogs only contain entries for detectors with an output, and use" 

201 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

202 name="jointcalPhotoCalibCatalog", 

203 storageClass="ExposureCatalog", 

204 dimensions=("instrument", "visit", "skymap", "tract"), 

205 multiple=True 

206 ) 

207 

208 # measurements of metrics 

209 # The vars() trick used here allows us to set class attributes 

210 # programatically. Taken from: 

211 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class 

212 for name in ("astrometry", "photometry"): 

213 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output( 

214 doc=f"The number of cross-matched fittedStars for {name}", 

215 name=f"metricvalue_jointcal_{name}_matched_fittedStars", 

216 storageClass="MetricValue", 

217 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

218 ) 

219 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output( 

220 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.", 

221 name=f"metricvalue_jointcal_{name}_collected_refStars", 

222 storageClass="MetricValue", 

223 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

224 ) 

225 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output( 

226 doc=f"The number of {name} reference stars matched to fittedStars.", 

227 name=f"metricvalue_jointcal_{name}_prepared_refStars", 

228 storageClass="MetricValue", 

229 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

230 ) 

231 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output( 

232 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.", 

233 name=f"metricvalue_jointcal_{name}_prepared_fittedStars", 

234 storageClass="MetricValue", 

235 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

236 ) 

237 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output( 

238 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.", 

239 name=f"metricvalue_jointcal_{name}_prepared_ccdImages", 

240 storageClass="MetricValue", 

241 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

242 ) 

243 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output( 

244 doc=f"The final chi2 of the {name} fit.", 

245 name=f"metricvalue_jointcal_{name}_final_chi2", 

246 storageClass="MetricValue", 

247 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

248 ) 

249 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output( 

250 doc=f"The number of degrees of freedom of the fitted {name}.", 

251 name=f"metricvalue_jointcal_{name}_final_ndof", 

252 storageClass="MetricValue", 

253 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

254 ) 

255 

256 def __init__(self, *, config=None): 

257 super().__init__(config=config) 

258 # When we are only doing one of astrometry or photometry, we don't 

259 # need the reference catalog or produce the outputs for the other. 

260 # This informs the middleware of that when the QuantumGraph is 

261 # generated, so we don't block on getting something we won't need or 

262 # create an expectation that downstream tasks will be able to consume 

263 # something we won't produce. 

264 if not config.doAstrometry: 

265 self.prerequisiteInputs.remove("astrometryRefCat") 

266 self.outputs.remove("outputWcs") 

267 for key in list(self.outputs): 

268 if "metricvalue_jointcal_astrometry" in key: 

269 self.outputs.remove(key) 

270 if not config.doPhotometry: 

271 self.prerequisiteInputs.remove("photometryRefCat") 

272 self.outputs.remove("outputPhotoCalib") 

273 for key in list(self.outputs): 

274 if "metricvalue_jointcal_photometry" in key: 

275 self.outputs.remove(key) 

276 

277 

278class JointcalConfig(pipeBase.PipelineTaskConfig, 

279 pipelineConnections=JointcalTaskConnections): 

280 """Configuration for JointcalTask""" 

281 

282 doAstrometry = pexConfig.Field( 

283 doc="Fit astrometry and write the fitted result.", 

284 dtype=bool, 

285 default=True 

286 ) 

287 doPhotometry = pexConfig.Field( 

288 doc="Fit photometry and write the fitted result.", 

289 dtype=bool, 

290 default=True 

291 ) 

292 sourceFluxType = pexConfig.Field( 

293 dtype=str, 

294 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

295 default='apFlux_12_0' 

296 ) 

297 positionErrorPedestal = pexConfig.Field( 

298 doc="Systematic term to apply to the measured position error (pixels)", 

299 dtype=float, 

300 default=0.02, 

301 ) 

302 photometryErrorPedestal = pexConfig.Field( 

303 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

304 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

305 dtype=float, 

306 default=0.0, 

307 ) 

308 # TODO: DM-6885 matchCut should be an geom.Angle 

309 matchCut = pexConfig.Field( 

310 doc="Matching radius between fitted and reference stars (arcseconds)", 

311 dtype=float, 

312 default=3.0, 

313 ) 

314 minMeasurements = pexConfig.Field( 

315 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

316 dtype=int, 

317 default=2, 

318 ) 

319 minMeasuredStarsPerCcd = pexConfig.Field( 

320 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

321 dtype=int, 

322 default=100, 

323 ) 

324 minRefStarsPerCcd = pexConfig.Field( 

325 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

326 dtype=int, 

327 default=30, 

328 ) 

329 allowLineSearch = pexConfig.Field( 

330 doc="Allow a line search during minimization, if it is reasonable for the model" 

331 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

332 dtype=bool, 

333 default=False 

334 ) 

335 astrometrySimpleOrder = pexConfig.Field( 

336 doc="Polynomial order for fitting the simple astrometry model.", 

337 dtype=int, 

338 default=3, 

339 ) 

340 astrometryChipOrder = pexConfig.Field( 

341 doc="Order of the per-chip transform for the constrained astrometry model.", 

342 dtype=int, 

343 default=1, 

344 ) 

345 astrometryVisitOrder = pexConfig.Field( 

346 doc="Order of the per-visit transform for the constrained astrometry model.", 

347 dtype=int, 

348 default=5, 

349 ) 

350 useInputWcs = pexConfig.Field( 

351 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

352 dtype=bool, 

353 default=True, 

354 ) 

355 astrometryModel = pexConfig.ChoiceField( 

356 doc="Type of model to fit to astrometry", 

357 dtype=str, 

358 default="constrained", 

359 allowed={"simple": "One polynomial per ccd", 

360 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

361 ) 

362 photometryModel = pexConfig.ChoiceField( 

363 doc="Type of model to fit to photometry", 

364 dtype=str, 

365 default="constrainedMagnitude", 

366 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

367 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

368 " fitting in flux space.", 

369 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

370 " fitting in magnitude space.", 

371 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

372 " fitting in magnitude space.", 

373 } 

374 ) 

375 applyColorTerms = pexConfig.Field( 

376 doc="Apply photometric color terms to reference stars?" 

377 "Requires that colorterms be set to a ColortermLibrary", 

378 dtype=bool, 

379 default=False 

380 ) 

381 colorterms = pexConfig.ConfigField( 

382 doc="Library of photometric reference catalog name to color term dict.", 

383 dtype=ColortermLibrary, 

384 ) 

385 photometryVisitOrder = pexConfig.Field( 

386 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

387 dtype=int, 

388 default=7, 

389 ) 

390 photometryDoRankUpdate = pexConfig.Field( 

391 doc=("Do the rank update step during minimization. " 

392 "Skipping this can help deal with models that are too non-linear."), 

393 dtype=bool, 

394 default=True, 

395 ) 

396 astrometryDoRankUpdate = pexConfig.Field( 

397 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

398 "Skipping this can help deal with models that are too non-linear."), 

399 dtype=bool, 

400 default=True, 

401 ) 

402 outlierRejectSigma = pexConfig.Field( 

403 doc="How many sigma to reject outliers at during minimization.", 

404 dtype=float, 

405 default=5.0, 

406 ) 

407 astrometryOutlierRelativeTolerance = pexConfig.Field( 

408 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will " 

409 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set " 

410 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002" 

411 "as a balance between a shorter minimization runtime and achieving a final fitted model that is" 

412 "close to the solution found when removing all outliers."), 

413 dtype=float, 

414 default=0, 

415 ) 

416 maxPhotometrySteps = pexConfig.Field( 

417 doc="Maximum number of minimize iterations to take when fitting photometry.", 

418 dtype=int, 

419 default=20, 

420 ) 

421 maxAstrometrySteps = pexConfig.Field( 

422 doc="Maximum number of minimize iterations to take when fitting astrometry.", 

423 dtype=int, 

424 default=20, 

425 ) 

426 astrometryRefObjLoader = pexConfig.ConfigField( 

427 dtype=LoadIndexedReferenceObjectsConfig, 

428 doc="Reference object loader for astrometric fit", 

429 ) 

430 photometryRefObjLoader = pexConfig.ConfigField( 

431 dtype=LoadIndexedReferenceObjectsConfig, 

432 doc="Reference object loader for photometric fit", 

433 ) 

434 sourceSelector = sourceSelectorRegistry.makeField( 

435 doc="How to select sources for cross-matching", 

436 default="science" 

437 ) 

438 astrometryReferenceSelector = pexConfig.ConfigurableField( 

439 target=ReferenceSourceSelectorTask, 

440 doc="How to down-select the loaded astrometry reference catalog.", 

441 ) 

442 photometryReferenceSelector = pexConfig.ConfigurableField( 

443 target=ReferenceSourceSelectorTask, 

444 doc="How to down-select the loaded photometry reference catalog.", 

445 ) 

446 astrometryReferenceErr = pexConfig.Field( 

447 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

448 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

449 "If specified, overrides any existing `coord_*Err` values."), 

450 dtype=float, 

451 default=None, 

452 optional=True 

453 ) 

454 

455 # configs for outputting debug information 

456 writeInitMatrix = pexConfig.Field( 

457 dtype=bool, 

458 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

459 "Output files will be written to `config.debugOutputPath` and will " 

460 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. " 

461 "Note that these files are the dense versions of the matrix, and so may be very large."), 

462 default=False 

463 ) 

464 writeChi2FilesInitialFinal = pexConfig.Field( 

465 dtype=bool, 

466 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. " 

467 "Output files will be written to `config.debugOutputPath` and will " 

468 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."), 

469 default=False 

470 ) 

471 writeChi2FilesOuterLoop = pexConfig.Field( 

472 dtype=bool, 

473 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. " 

474 "Output files will be written to `config.debugOutputPath` and will " 

475 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."), 

476 default=False 

477 ) 

478 writeInitialModel = pexConfig.Field( 

479 dtype=bool, 

480 doc=("Write the pre-initialization model to text files, for debugging. " 

481 "Output files will be written to `config.debugOutputPath` and will be " 

482 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."), 

483 default=False 

484 ) 

485 debugOutputPath = pexConfig.Field( 

486 dtype=str, 

487 default=".", 

488 doc=("Path to write debug output files to. Used by " 

489 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

490 ) 

491 detailedProfile = pexConfig.Field( 

492 dtype=bool, 

493 default=False, 

494 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting" 

495 ) 

496 

497 def validate(self): 

498 super().validate() 

499 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

500 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

501 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

502 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

503 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

504 "applyColorTerms=True will be ignored.") 

505 logging.getLogger("lsst.jointcal").warning(msg) 

506 

507 def setDefaults(self): 

508 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing. 

509 self.sourceSelector["science"].doUnresolved = True 

510 self.sourceSelector["science"].unresolved.name = "extendedness" 

511 # with dependable signal to noise ratio. 

512 self.sourceSelector["science"].doSignalToNoise = True 

513 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

514 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

515 self.sourceSelector["science"].signalToNoise.minimum = 10. 

516 # Base SNR selection on `sourceFluxType` because that is the flux that jointcal fits. 

517 self.sourceSelector["science"].signalToNoise.fluxField = f"{self.sourceFluxType}_instFlux" 

518 self.sourceSelector["science"].signalToNoise.errField = f"{self.sourceFluxType}_instFluxErr" 

519 # Do not trust blended sources" aperture fluxes which also depend on seeing. 

520 self.sourceSelector["science"].doIsolated = True 

521 self.sourceSelector["science"].isolated.parentName = "parentSourceId" 

522 self.sourceSelector["science"].isolated.nChildName = "deblend_nChild" 

523 # Do not trust either flux or centroid measurements with flags, 

524 # chosen from the usual QA flags for stars) 

525 self.sourceSelector["science"].doFlags = True 

526 badFlags = ["pixelFlags_edge", 

527 "pixelFlags_saturated", 

528 "pixelFlags_interpolatedCenter", 

529 "pixelFlags_interpolated", 

530 "pixelFlags_crCenter", 

531 "pixelFlags_bad", 

532 "hsmPsfMoments_flag", 

533 f"{self.sourceFluxType}_flag", 

534 ] 

535 self.sourceSelector["science"].flags.bad = badFlags 

536 

537 # Use Gaia-DR2 with proper motions for astrometry; phot_g_mean is the 

538 # primary Gaia band, but is not like any normal photometric band. 

539 self.astrometryRefObjLoader.requireProperMotion = True 

540 self.astrometryRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

541 

542 

543def writeModel(model, filename, log): 

544 """Write model to outfile.""" 

545 with open(filename, "w") as file: 

546 file.write(repr(model)) 

547 log.info("Wrote %s to file: %s", model, filename) 

548 

549 

550@dataclasses.dataclass 

551class JointcalInputData: 

552 """The input data jointcal needs for each detector/visit.""" 

553 visit: int 

554 """The visit identifier of this exposure.""" 

555 catalog: lsst.afw.table.SourceCatalog 

556 """The catalog derived from this exposure.""" 

557 visitInfo: lsst.afw.image.VisitInfo 

558 """The VisitInfo of this exposure.""" 

559 detector: lsst.afw.cameraGeom.Detector 

560 """The detector of this exposure.""" 

561 photoCalib: lsst.afw.image.PhotoCalib 

562 """The photometric calibration of this exposure.""" 

563 wcs: lsst.afw.geom.skyWcs 

564 """The WCS of this exposure.""" 

565 bbox: lsst.geom.Box2I 

566 """The bounding box of this exposure.""" 

567 filter: lsst.afw.image.FilterLabel 

568 """The filter of this exposure.""" 

569 

570 

571class JointcalTask(pipeBase.PipelineTask): 

572 """Astrometricly and photometricly calibrate across multiple visits of the 

573 same field. 

574 """ 

575 

576 ConfigClass = JointcalConfig 

577 _DefaultName = "jointcal" 

578 

579 def __init__(self, **kwargs): 

580 super().__init__(**kwargs) 

581 self.makeSubtask("sourceSelector") 

582 if self.config.doAstrometry: 

583 self.makeSubtask("astrometryReferenceSelector") 

584 else: 

585 self.astrometryRefObjLoader = None 

586 if self.config.doPhotometry: 

587 self.makeSubtask("photometryReferenceSelector") 

588 else: 

589 self.photometryRefObjLoader = None 

590 

591 # To hold various computed metrics for use by tests 

592 self.job = Job.load_metrics_package(subset='jointcal') 

593 

594 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

595 # We override runQuantum to set up the refObjLoaders and write the 

596 # outputs to the correct refs. 

597 inputs = butlerQC.get(inputRefs) 

598 # We want the tract number for writing debug files 

599 tract = butlerQC.quantum.dataId['tract'] 

600 if self.config.doAstrometry: 

601 self.astrometryRefObjLoader = ReferenceObjectLoader( 

602 dataIds=[ref.datasetRef.dataId 

603 for ref in inputRefs.astrometryRefCat], 

604 refCats=inputs.pop('astrometryRefCat'), 

605 config=self.config.astrometryRefObjLoader, 

606 log=self.log) 

607 if self.config.doPhotometry: 

608 self.photometryRefObjLoader = ReferenceObjectLoader( 

609 dataIds=[ref.datasetRef.dataId 

610 for ref in inputRefs.photometryRefCat], 

611 refCats=inputs.pop('photometryRefCat'), 

612 config=self.config.photometryRefObjLoader, 

613 log=self.log) 

614 outputs = self.run(**inputs, tract=tract) 

615 self._put_metrics(butlerQC, outputs.job, outputRefs) 

616 if self.config.doAstrometry: 

617 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs, 

618 inputs['inputCamera'], "setWcs") 

619 if self.config.doPhotometry: 

620 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib, 

621 inputs['inputCamera'], "setPhotoCalib") 

622 

623 def _put_metrics(self, butlerQC, job, outputRefs): 

624 """Persist all measured metrics stored in a job. 

625 

626 Parameters 

627 ---------- 

628 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

629 A butler which is specialized to operate in the context of a 

630 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

631 job : `lsst.verify.job.Job` 

632 Measurements of metrics to persist. 

633 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

634 The DatasetRefs to persist the data to. 

635 """ 

636 for key in job.measurements.keys(): 

637 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', ''))) 

638 

639 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter): 

640 """Persist the output datasets to their appropriate datarefs. 

641 

642 Parameters 

643 ---------- 

644 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

645 A butler which is specialized to operate in the context of a 

646 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

647 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or 

648 `dict` [`tuple, `lsst.afw.image.PhotoCalib`] 

649 The fitted objects to persist. 

650 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

651 The DatasetRefs to persist the data to. 

652 camera : `lsst.afw.cameraGeom.Camera` 

653 The camera for this instrument, to get detector ids from. 

654 setter : `str` 

655 The method to call on the ExposureCatalog to set each output. 

656 """ 

657 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

658 schema.addField('visit', type='L', doc='Visit number') 

659 

660 def new_catalog(visit, size): 

661 """Return an catalog ready to be filled with appropriate output.""" 

662 catalog = lsst.afw.table.ExposureCatalog(schema) 

663 catalog.resize(size) 

664 catalog['visit'] = visit 

665 metadata = lsst.daf.base.PropertyList() 

666 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

667 metadata.add("COMMENT", "Only detectors with data have entries.") 

668 return catalog 

669 

670 # count how many detectors have output for each visit 

671 detectors_per_visit = collections.defaultdict(int) 

672 for key in outputs: 

673 # key is (visit, detector_id), and we only need visit here 

674 detectors_per_visit[key[0]] += 1 

675 

676 for ref in outputRefs: 

677 visit = ref.dataId['visit'] 

678 catalog = new_catalog(visit, detectors_per_visit[visit]) 

679 # Iterate over every detector and skip the ones we don't have output for. 

680 i = 0 

681 for detector in camera: 

682 detectorId = detector.getId() 

683 key = (ref.dataId['visit'], detectorId) 

684 if key not in outputs: 

685 # skip detectors we don't have output for 

686 self.log.debug("No %s output for detector %s in visit %s", 

687 setter[3:], detectorId, visit) 

688 continue 

689 

690 catalog[i].setId(detectorId) 

691 getattr(catalog[i], setter)(outputs[key]) 

692 i += 1 

693 

694 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups 

695 butlerQC.put(catalog, ref) 

696 self.log.info("Wrote %s detectors to %s", i, ref) 

697 

698 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None): 

699 # Docstring inherited. 

700 

701 # We take values out of the Parquet table, and put them in "flux_", 

702 # and the config.sourceFluxType field is used during that extraction, 

703 # so just use "flux" here. 

704 sourceFluxField = "flux" 

705 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

706 associations = lsst.jointcal.Associations() 

707 self.focalPlaneBBox = inputCamera.getFpBBox() 

708 oldWcsList, bands = self._load_data(inputSourceTableVisit, 

709 inputVisitSummary, 

710 associations, 

711 jointcalControl, 

712 inputCamera) 

713 

714 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands) 

715 

716 if self.config.doAstrometry: 

717 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

718 name="astrometry", 

719 refObjLoader=self.astrometryRefObjLoader, 

720 referenceSelector=self.astrometryReferenceSelector, 

721 fit_function=self._fit_astrometry, 

722 tract=tract, 

723 epoch=epoch) 

724 astrometry_output = self._make_output(associations.getCcdImageList(), 

725 astrometry.model, 

726 "makeSkyWcs") 

727 else: 

728 astrometry_output = None 

729 

730 if self.config.doPhotometry: 

731 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

732 name="photometry", 

733 refObjLoader=self.photometryRefObjLoader, 

734 referenceSelector=self.photometryReferenceSelector, 

735 fit_function=self._fit_photometry, 

736 tract=tract, 

737 epoch=epoch, 

738 reject_bad_fluxes=True) 

739 photometry_output = self._make_output(associations.getCcdImageList(), 

740 photometry.model, 

741 "toPhotoCalib") 

742 else: 

743 photometry_output = None 

744 

745 return pipeBase.Struct(outputWcs=astrometry_output, 

746 outputPhotoCalib=photometry_output, 

747 job=self.job, 

748 astrometryRefObjLoader=self.astrometryRefObjLoader, 

749 photometryRefObjLoader=self.photometryRefObjLoader) 

750 

751 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, 

752 jointcalControl, camera): 

753 """Read the data that jointcal needs to run. 

754 

755 Modifies ``associations`` in-place with the loaded data. 

756 

757 Parameters 

758 ---------- 

759 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

760 References to visit-level DataFrames to load the catalog data from. 

761 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

762 Visit-level exposure summary catalog with metadata. 

763 associations : `lsst.jointcal.Associations` 

764 Object to add the loaded data to by constructing new CcdImages. 

765 jointcalControl : `jointcal.JointcalControl` 

766 Control object for C++ associations management. 

767 camera : `lsst.afw.cameraGeom.Camera` 

768 Camera object for detector geometry. 

769 

770 Returns 

771 ------- 

772 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`] 

773 The original WCS of the input data, to aid in writing tests. 

774 bands : `list` [`str`] 

775 The filter bands of each input dataset. 

776 """ 

777 oldWcsList = [] 

778 filters = [] 

779 load_cat_profile_file = 'jointcal_load_data.prof' if self.config.detailedProfile else '' 

780 with lsst.utils.timer.profile(load_cat_profile_file): 

781 table = make_schema_table() # every detector catalog has the same layout 

782 # No guarantee that the input is in the same order of visits, so we have to map one of them. 

783 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)} 

784 detectorDict = {detector.getId(): detector for detector in camera} 

785 

786 columns = None 

787 

788 for visitSummaryRef in inputVisitSummary: 

789 visitSummary = visitSummaryRef.get() 

790 

791 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]] 

792 if columns is None: 

793 inColumns = dataRef.get(component='columns') 

794 columns, detColumn, ixxColumns = get_sourceTable_visit_columns(inColumns, 

795 self.config, 

796 self.sourceSelector) 

797 visitCatalog = dataRef.get(parameters={'columns': columns}) 

798 

799 selected = self.sourceSelector.run(visitCatalog) 

800 

801 # Build a CcdImage for each detector in this visit. 

802 detectors = {id: index for index, id in enumerate(visitSummary['id'])} 

803 for id, index in detectors.items(): 

804 catalog = extract_detector_catalog_from_visit_catalog(table, 

805 selected.sourceCat, 

806 id, 

807 detColumn, 

808 ixxColumns, 

809 self.config.sourceFluxType, 

810 self.log) 

811 if catalog is None: 

812 continue 

813 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict) 

814 result = self._build_ccdImage(data, associations, jointcalControl) 

815 if result is not None: 

816 oldWcsList.append(result.wcs) 

817 # A visit has only one band, so we can just use the first. 

818 filters.append(data.filter) 

819 if len(filters) == 0: 

820 raise RuntimeError("No data to process: did source selector remove all sources?") 

821 filters = collections.Counter(filters) 

822 

823 return oldWcsList, filters 

824 

825 def _make_one_input_data(self, visitRecord, catalog, detectorDict): 

826 """Return a data structure for this detector+visit.""" 

827 return JointcalInputData(visit=visitRecord['visit'], 

828 catalog=catalog, 

829 visitInfo=visitRecord.getVisitInfo(), 

830 detector=detectorDict[visitRecord.getId()], 

831 photoCalib=visitRecord.getPhotoCalib(), 

832 wcs=visitRecord.getWcs(), 

833 bbox=visitRecord.getBBox(), 

834 # ExposureRecord doesn't have a FilterLabel yet, 

835 # so we have to make one. 

836 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'], 

837 physical=visitRecord['physical_filter'])) 

838 

839 def _build_ccdImage(self, data, associations, jointcalControl): 

840 """ 

841 Extract the necessary things from this catalog+metadata to add a new 

842 ccdImage. 

843 

844 Parameters 

845 ---------- 

846 data : `JointcalInputData` 

847 The loaded input data. 

848 associations : `lsst.jointcal.Associations` 

849 Object to add the info to, to construct a new CcdImage 

850 jointcalControl : `jointcal.JointcalControl` 

851 Control object for associations management 

852 

853 Returns 

854 ------ 

855 namedtuple or `None` 

856 ``wcs`` 

857 The TAN WCS of this image, read from the calexp 

858 (`lsst.afw.geom.SkyWcs`). 

859 ``key`` 

860 A key to identify this dataRef by its visit and ccd ids 

861 (`namedtuple`). 

862 `None` 

863 if there are no sources in the loaded catalog. 

864 """ 

865 if len(data.catalog) == 0: 

866 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId()) 

867 return None 

868 

869 associations.createCcdImage(data.catalog, 

870 data.wcs, 

871 data.visitInfo, 

872 data.bbox, 

873 data.filter.physicalLabel, 

874 data.photoCalib, 

875 data.detector, 

876 data.visit, 

877 data.detector.getId(), 

878 jointcalControl) 

879 

880 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key')) 

881 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

882 return Result(data.wcs, Key(data.visit, data.detector.getId())) 

883 

884 def _getDebugPath(self, filename): 

885 """Constructs a path to filename using the configured debug path. 

886 """ 

887 return os.path.join(self.config.debugOutputPath, filename) 

888 

889 def _prep_sky(self, associations, filters): 

890 """Prepare on-sky and other data that must be computed after data has 

891 been read. 

892 """ 

893 associations.computeCommonTangentPoint() 

894 

895 boundingCircle = associations.computeBoundingCircle() 

896 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

897 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

898 

899 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

900 

901 # Determine a default filter band associated with the catalog. See DM-9093 

902 defaultFilter = filters.most_common(1)[0][0] 

903 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel) 

904 

905 # compute and set the reference epoch of the observations, for proper motion corrections 

906 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

907 associations.setEpoch(epoch.jyear) 

908 

909 return boundingCircle, center, radius, defaultFilter, epoch 

910 

911 def _get_refcat_coordinate_error_override(self, refCat, name): 

912 """Check whether we should override the refcat coordinate errors, and 

913 return the overridden error if necessary. 

914 

915 Parameters 

916 ---------- 

917 refCat : `lsst.afw.table.SimpleCatalog` 

918 The reference catalog to check for a ``coord_raErr`` field. 

919 name : `str` 

920 Whether we are doing "astrometry" or "photometry". 

921 

922 Returns 

923 ------- 

924 refCoordErr : `float` 

925 The refcat coordinate error to use, or NaN if we are not overriding 

926 those fields. 

927 

928 Raises 

929 ------ 

930 lsst.pex.config.FieldValidationError 

931 Raised if the refcat does not contain coordinate errors and 

932 ``config.astrometryReferenceErr`` is not set. 

933 """ 

934 # This value doesn't matter for photometry, so just set something to 

935 # keep old refcats from causing problems. 

936 if name.lower() == "photometry": 

937 if 'coord_raErr' not in refCat.schema: 

938 return 100 

939 else: 

940 return float('nan') 

941 

942 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

943 msg = ("Reference catalog does not contain coordinate errors, " 

944 "and config.astrometryReferenceErr not supplied.") 

945 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

946 self.config, 

947 msg) 

948 

949 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

950 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

951 self.config.astrometryReferenceErr) 

952 

953 if self.config.astrometryReferenceErr is None: 

954 return float('nan') 

955 else: 

956 return self.config.astrometryReferenceErr 

957 

958 def _compute_proper_motion_epoch(self, ccdImageList): 

959 """Return the proper motion correction epoch of the provided images. 

960 

961 Parameters 

962 ---------- 

963 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

964 The images to compute the appropriate epoch for. 

965 

966 Returns 

967 ------- 

968 epoch : `astropy.time.Time` 

969 The date to use for proper motion corrections. 

970 """ 

971 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]), 

972 format="jyear", 

973 scale="tai") 

974 

975 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

976 tract="", match_cut=3.0, 

977 reject_bad_fluxes=False, *, 

978 name="", refObjLoader=None, referenceSelector=None, 

979 fit_function=None, epoch=None): 

980 """Load reference catalog, perform the fit, and return the result. 

981 

982 Parameters 

983 ---------- 

984 associations : `lsst.jointcal.Associations` 

985 The star/reference star associations to fit. 

986 defaultFilter : `lsst.afw.image.FilterLabel` 

987 filter to load from reference catalog. 

988 center : `lsst.geom.SpherePoint` 

989 ICRS center of field to load from reference catalog. 

990 radius : `lsst.geom.Angle` 

991 On-sky radius to load from reference catalog. 

992 name : `str` 

993 Name of thing being fit: "astrometry" or "photometry". 

994 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

995 Reference object loader to use to load a reference catalog. 

996 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

997 Selector to use to pick objects from the loaded reference catalog. 

998 fit_function : callable 

999 Function to call to perform fit (takes Associations object). 

1000 tract : `str`, optional 

1001 Name of tract currently being fit. 

1002 match_cut : `float`, optional 

1003 Radius in arcseconds to find cross-catalog matches to during 

1004 associations.associateCatalogs. 

1005 reject_bad_fluxes : `bool`, optional 

1006 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

1007 epoch : `astropy.time.Time`, optional 

1008 Epoch to which to correct refcat proper motion and parallax, 

1009 or `None` to not apply such corrections. 

1010 

1011 Returns 

1012 ------- 

1013 result : `Photometry` or `Astrometry` 

1014 Result of `fit_function()` 

1015 """ 

1016 self.log.info("====== Now processing %s...", name) 

1017 # TODO: this should not print "trying to invert a singular transformation:" 

1018 # if it does that, something's not right about the WCS... 

1019 associations.associateCatalogs(match_cut) 

1020 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name, 

1021 associations.fittedStarListSize()) 

1022 

1023 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

1024 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

1025 center, radius, defaultFilter, 

1026 applyColorterms=applyColorterms, 

1027 epoch=epoch) 

1028 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

1029 

1030 associations.collectRefStars(refCat, 

1031 self.config.matchCut*lsst.geom.arcseconds, 

1032 fluxField, 

1033 refCoordinateErr=refCoordErr, 

1034 rejectBadFluxes=reject_bad_fluxes) 

1035 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name, 

1036 associations.refStarListSize()) 

1037 

1038 associations.prepareFittedStars(self.config.minMeasurements) 

1039 

1040 self._check_star_lists(associations, name) 

1041 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name, 

1042 associations.nFittedStarsWithAssociatedRefStar()) 

1043 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name, 

1044 associations.fittedStarListSize()) 

1045 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name, 

1046 associations.nCcdImagesValidForFit()) 

1047 

1048 fit_profile_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else '' 

1049 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel) 

1050 with lsst.utils.timer.profile(fit_profile_file): 

1051 result = fit_function(associations, dataName) 

1052 # TODO DM-12446: turn this into a "butler save" somehow. 

1053 # Save reference and measurement chi2 contributions for this data 

1054 if self.config.writeChi2FilesInitialFinal: 

1055 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

1056 result.fit.saveChi2Contributions(baseName+"{type}") 

1057 self.log.info("Wrote chi2 contributions files: %s", baseName) 

1058 

1059 return result 

1060 

1061 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, 

1062 applyColorterms=False, epoch=None): 

1063 """Load the necessary reference catalog sources, convert fluxes to 

1064 correct units, and apply color term corrections if requested. 

1065 

1066 Parameters 

1067 ---------- 

1068 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

1069 The reference catalog loader to use to get the data. 

1070 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1071 Source selector to apply to loaded reference catalog. 

1072 center : `lsst.geom.SpherePoint` 

1073 The center around which to load sources. 

1074 radius : `lsst.geom.Angle` 

1075 The radius around ``center`` to load sources in. 

1076 filterLabel : `lsst.afw.image.FilterLabel` 

1077 The camera filter to load fluxes for. 

1078 applyColorterms : `bool` 

1079 Apply colorterm corrections to the refcat for ``filterName``? 

1080 epoch : `astropy.time.Time`, optional 

1081 Epoch to which to correct refcat proper motion and parallax, 

1082 or `None` to not apply such corrections. 

1083 

1084 Returns 

1085 ------- 

1086 refCat : `lsst.afw.table.SimpleCatalog` 

1087 The loaded reference catalog. 

1088 fluxField : `str` 

1089 The name of the reference catalog flux field appropriate for ``filterName``. 

1090 """ 

1091 skyCircle = refObjLoader.loadSkyCircle(center, 

1092 radius, 

1093 filterLabel.bandLabel, 

1094 epoch=epoch) 

1095 

1096 selected = referenceSelector.run(skyCircle.refCat) 

1097 # Need memory contiguity to get reference filters as a vector. 

1098 if not selected.sourceCat.isContiguous(): 

1099 refCat = selected.sourceCat.copy(deep=True) 

1100 else: 

1101 refCat = selected.sourceCat 

1102 

1103 if applyColorterms: 

1104 refCatName = self.config.connections.photometryRefCat 

1105 self.log.info("Applying color terms for physical filter=%r reference catalog=%s", 

1106 filterLabel.physicalLabel, refCatName) 

1107 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel, 

1108 refCatName, 

1109 doRaise=True) 

1110 

1111 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

1112 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

1113 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

1114 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

1115 

1116 return refCat, skyCircle.fluxField 

1117 

1118 def _check_star_lists(self, associations, name): 

1119 # TODO: these should be len(blah), but we need this properly wrapped first. 

1120 if associations.nCcdImagesValidForFit() == 0: 

1121 raise RuntimeError('No images in the ccdImageList!') 

1122 if associations.fittedStarListSize() == 0: 

1123 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

1124 if associations.refStarListSize() == 0: 

1125 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

1126 

1127 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

1128 """Compute chi2, log it, validate the model, and return chi2. 

1129 

1130 Parameters 

1131 ---------- 

1132 associations : `lsst.jointcal.Associations` 

1133 The star/reference star associations to fit. 

1134 fit : `lsst.jointcal.FitterBase` 

1135 The fitter to use for minimization. 

1136 model : `lsst.jointcal.Model` 

1137 The model being fit. 

1138 chi2Label : `str` 

1139 Label to describe the chi2 (e.g. "Initialized", "Final"). 

1140 writeChi2Name : `str`, optional 

1141 Filename prefix to write the chi2 contributions to. 

1142 Do not supply an extension: an appropriate one will be added. 

1143 

1144 Returns 

1145 ------- 

1146 chi2: `lsst.jointcal.Chi2Accumulator` 

1147 The chi2 object for the current fitter and model. 

1148 

1149 Raises 

1150 ------ 

1151 FloatingPointError 

1152 Raised if chi2 is infinite or NaN. 

1153 ValueError 

1154 Raised if the model is not valid. 

1155 """ 

1156 if writeChi2Name is not None: 

1157 fullpath = self._getDebugPath(writeChi2Name) 

1158 fit.saveChi2Contributions(fullpath+"{type}") 

1159 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

1160 

1161 chi2 = fit.computeChi2() 

1162 self.log.info("%s %s", chi2Label, chi2) 

1163 self._check_stars(associations) 

1164 if not np.isfinite(chi2.chi2): 

1165 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

1166 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

1167 raise ValueError("Model is not valid: check log messages for warnings.") 

1168 return chi2 

1169 

1170 def _fit_photometry(self, associations, dataName=None): 

1171 """ 

1172 Fit the photometric data. 

1173 

1174 Parameters 

1175 ---------- 

1176 associations : `lsst.jointcal.Associations` 

1177 The star/reference star associations to fit. 

1178 dataName : `str` 

1179 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1180 identifying debugging files. 

1181 

1182 Returns 

1183 ------- 

1184 fit_result : `namedtuple` 

1185 fit : `lsst.jointcal.PhotometryFit` 

1186 The photometric fitter used to perform the fit. 

1187 model : `lsst.jointcal.PhotometryModel` 

1188 The photometric model that was fit. 

1189 """ 

1190 self.log.info("=== Starting photometric fitting...") 

1191 

1192 # TODO: should use pex.config.RegistryField here (see DM-9195) 

1193 if self.config.photometryModel == "constrainedFlux": 

1194 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

1195 self.focalPlaneBBox, 

1196 visitOrder=self.config.photometryVisitOrder, 

1197 errorPedestal=self.config.photometryErrorPedestal) 

1198 # potentially nonlinear problem, so we may need a line search to converge. 

1199 doLineSearch = self.config.allowLineSearch 

1200 elif self.config.photometryModel == "constrainedMagnitude": 

1201 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

1202 self.focalPlaneBBox, 

1203 visitOrder=self.config.photometryVisitOrder, 

1204 errorPedestal=self.config.photometryErrorPedestal) 

1205 # potentially nonlinear problem, so we may need a line search to converge. 

1206 doLineSearch = self.config.allowLineSearch 

1207 elif self.config.photometryModel == "simpleFlux": 

1208 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

1209 errorPedestal=self.config.photometryErrorPedestal) 

1210 doLineSearch = False # purely linear in model parameters, so no line search needed 

1211 elif self.config.photometryModel == "simpleMagnitude": 

1212 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

1213 errorPedestal=self.config.photometryErrorPedestal) 

1214 doLineSearch = False # purely linear in model parameters, so no line search needed 

1215 

1216 fit = lsst.jointcal.PhotometryFit(associations, model) 

1217 # TODO DM-12446: turn this into a "butler save" somehow. 

1218 # Save reference and measurement chi2 contributions for this data 

1219 if self.config.writeChi2FilesInitialFinal: 

1220 baseName = f"photometry_initial_chi2-{dataName}" 

1221 else: 

1222 baseName = None 

1223 if self.config.writeInitialModel: 

1224 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt") 

1225 writeModel(model, fullpath, self.log) 

1226 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

1227 

1228 def getChi2Name(whatToFit): 

1229 if self.config.writeChi2FilesOuterLoop: 

1230 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

1231 else: 

1232 return None 

1233 

1234 # The constrained model needs the visit transform fit first; the chip 

1235 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

1236 if self.config.writeInitMatrix: 

1237 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}") 

1238 else: 

1239 dumpMatrixFile = "" 

1240 if self.config.photometryModel.startswith("constrained"): 

1241 # no line search: should be purely (or nearly) linear, 

1242 # and we want a large step size to initialize with. 

1243 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

1244 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

1245 writeChi2Name=getChi2Name("ModelVisit")) 

1246 dumpMatrixFile = "" # so we don't redo the output on the next step 

1247 

1248 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

1249 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

1250 writeChi2Name=getChi2Name("Model")) 

1251 

1252 fit.minimize("Fluxes") # no line search: always purely linear. 

1253 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

1254 writeChi2Name=getChi2Name("Fluxes")) 

1255 

1256 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

1257 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

1258 writeChi2Name=getChi2Name("ModelFluxes")) 

1259 

1260 model.freezeErrorTransform() 

1261 self.log.debug("Photometry error scales are frozen.") 

1262 

1263 chi2 = self._iterate_fit(associations, 

1264 fit, 

1265 self.config.maxPhotometrySteps, 

1266 "photometry", 

1267 "Model Fluxes", 

1268 doRankUpdate=self.config.photometryDoRankUpdate, 

1269 doLineSearch=doLineSearch, 

1270 dataName=dataName) 

1271 

1272 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

1273 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

1274 return Photometry(fit, model) 

1275 

1276 def _fit_astrometry(self, associations, dataName=None): 

1277 """ 

1278 Fit the astrometric data. 

1279 

1280 Parameters 

1281 ---------- 

1282 associations : `lsst.jointcal.Associations` 

1283 The star/reference star associations to fit. 

1284 dataName : `str` 

1285 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1286 identifying debugging files. 

1287 

1288 Returns 

1289 ------- 

1290 fit_result : `namedtuple` 

1291 fit : `lsst.jointcal.AstrometryFit` 

1292 The astrometric fitter used to perform the fit. 

1293 model : `lsst.jointcal.AstrometryModel` 

1294 The astrometric model that was fit. 

1295 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

1296 The model for the sky to tangent plane projection that was used in the fit. 

1297 """ 

1298 

1299 self.log.info("=== Starting astrometric fitting...") 

1300 

1301 associations.deprojectFittedStars() 

1302 

1303 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1304 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1305 # them so carefully? 

1306 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1307 

1308 if self.config.astrometryModel == "constrained": 

1309 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1310 sky_to_tan_projection, 

1311 chipOrder=self.config.astrometryChipOrder, 

1312 visitOrder=self.config.astrometryVisitOrder) 

1313 elif self.config.astrometryModel == "simple": 

1314 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1315 sky_to_tan_projection, 

1316 self.config.useInputWcs, 

1317 nNotFit=0, 

1318 order=self.config.astrometrySimpleOrder) 

1319 

1320 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1321 # TODO DM-12446: turn this into a "butler save" somehow. 

1322 # Save reference and measurement chi2 contributions for this data 

1323 if self.config.writeChi2FilesInitialFinal: 

1324 baseName = f"astrometry_initial_chi2-{dataName}" 

1325 else: 

1326 baseName = None 

1327 if self.config.writeInitialModel: 

1328 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt") 

1329 writeModel(model, fullpath, self.log) 

1330 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1331 

1332 def getChi2Name(whatToFit): 

1333 if self.config.writeChi2FilesOuterLoop: 

1334 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1335 else: 

1336 return None 

1337 

1338 if self.config.writeInitMatrix: 

1339 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}") 

1340 else: 

1341 dumpMatrixFile = "" 

1342 # The constrained model needs the visit transform fit first; the chip 

1343 # transform is initialized from the detector's cameraGeom, so it's close. 

1344 if self.config.astrometryModel == "constrained": 

1345 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1346 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1347 writeChi2Name=getChi2Name("DistortionsVisit")) 

1348 dumpMatrixFile = "" # so we don't redo the output on the next step 

1349 

1350 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1351 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1352 writeChi2Name=getChi2Name("Distortions")) 

1353 

1354 fit.minimize("Positions") 

1355 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1356 writeChi2Name=getChi2Name("Positions")) 

1357 

1358 fit.minimize("Distortions Positions") 

1359 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1360 writeChi2Name=getChi2Name("DistortionsPositions")) 

1361 

1362 chi2 = self._iterate_fit(associations, 

1363 fit, 

1364 self.config.maxAstrometrySteps, 

1365 "astrometry", 

1366 "Distortions Positions", 

1367 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance, 

1368 doRankUpdate=self.config.astrometryDoRankUpdate, 

1369 dataName=dataName) 

1370 

1371 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1372 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1373 

1374 return Astrometry(fit, model, sky_to_tan_projection) 

1375 

1376 def _check_stars(self, associations): 

1377 """Count measured and reference stars per ccd and warn/log them.""" 

1378 for ccdImage in associations.getCcdImageList(): 

1379 nMeasuredStars, nRefStars = ccdImage.countStars() 

1380 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1381 ccdImage.getName(), nMeasuredStars, nRefStars) 

1382 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1383 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)", 

1384 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1385 if nRefStars < self.config.minRefStarsPerCcd: 

1386 self.log.warning("ccdImage %s has only %s RefStars (desired %s)", 

1387 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1388 

1389 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1390 dataName="", 

1391 sigmaRelativeTolerance=0, 

1392 doRankUpdate=True, 

1393 doLineSearch=False): 

1394 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1395 

1396 Parameters 

1397 ---------- 

1398 associations : `lsst.jointcal.Associations` 

1399 The star/reference star associations to fit. 

1400 fitter : `lsst.jointcal.FitterBase` 

1401 The fitter to use for minimization. 

1402 max_steps : `int` 

1403 Maximum number of steps to run outlier rejection before declaring 

1404 convergence failure. 

1405 name : {'photometry' or 'astrometry'} 

1406 What type of data are we fitting (for logs and debugging files). 

1407 whatToFit : `str` 

1408 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1409 dataName : `str`, optional 

1410 Descriptive name for this dataset (e.g. tract and filter), 

1411 for debugging. 

1412 sigmaRelativeTolerance : `float`, optional 

1413 Convergence tolerance for the fractional change in the chi2 cut 

1414 level for determining outliers. If set to zero, iterations will 

1415 continue until there are no outliers. 

1416 doRankUpdate : `bool`, optional 

1417 Do an Eigen rank update during minimization, or recompute the full 

1418 matrix and gradient? 

1419 doLineSearch : `bool`, optional 

1420 Do a line search for the optimum step during minimization? 

1421 

1422 Returns 

1423 ------- 

1424 chi2: `lsst.jointcal.Chi2Statistic` 

1425 The final chi2 after the fit converges, or is forced to end. 

1426 

1427 Raises 

1428 ------ 

1429 FloatingPointError 

1430 Raised if the fitter fails with a non-finite value. 

1431 RuntimeError 

1432 Raised if the fitter fails for some other reason; 

1433 log messages will provide further details. 

1434 """ 

1435 if self.config.writeInitMatrix: 

1436 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}") 

1437 else: 

1438 dumpMatrixFile = "" 

1439 oldChi2 = lsst.jointcal.Chi2Statistic() 

1440 oldChi2.chi2 = float("inf") 

1441 for i in range(max_steps): 

1442 if self.config.writeChi2FilesOuterLoop: 

1443 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1444 else: 

1445 writeChi2Name = None 

1446 result = fitter.minimize(whatToFit, 

1447 self.config.outlierRejectSigma, 

1448 sigmaRelativeTolerance=sigmaRelativeTolerance, 

1449 doRankUpdate=doRankUpdate, 

1450 doLineSearch=doLineSearch, 

1451 dumpMatrixFile=dumpMatrixFile) 

1452 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1453 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1454 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1455 

1456 if result == MinimizeResult.Converged: 

1457 if doRankUpdate: 

1458 self.log.debug("fit has converged - no more outliers - redo minimization " 

1459 "one more time in case we have lost accuracy in rank update.") 

1460 # Redo minimization one more time in case we have lost accuracy in rank update 

1461 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma, 

1462 sigmaRelativeTolerance=sigmaRelativeTolerance) 

1463 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1464 

1465 # log a message for a large final chi2, TODO: DM-15247 for something better 

1466 if chi2.chi2/chi2.ndof >= 4.0: 

1467 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1468 

1469 break 

1470 elif result == MinimizeResult.Chi2Increased: 

1471 self.log.warning("Still some outliers remaining but chi2 increased - retry") 

1472 # Check whether the increase was large enough to cause trouble. 

1473 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1474 if chi2Ratio > 1.5: 

1475 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1476 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1477 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1478 # appears that chi2 increases more than a factor of ~2 always 

1479 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1480 # Using 10 as the "failure" threshold gives some room between 

1481 # leaving a warning and bailing early. 

1482 if chi2Ratio > 10: 

1483 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1484 " Try setting one or more of the `writeChi2*` config fields and looking" 

1485 " at how individual star chi2-values evolve during the fit.") 

1486 raise RuntimeError(msg) 

1487 oldChi2 = chi2 

1488 elif result == MinimizeResult.NonFinite: 

1489 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1490 # TODO DM-12446: turn this into a "butler save" somehow. 

1491 fitter.saveChi2Contributions(filename+"{type}") 

1492 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1493 raise FloatingPointError(msg.format(filename)) 

1494 elif result == MinimizeResult.Failed: 

1495 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1496 else: 

1497 raise RuntimeError("Unxepected return code from minimize().") 

1498 else: 

1499 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1500 

1501 return chi2 

1502 

1503 def _make_output(self, ccdImageList, model, func): 

1504 """Return the internal jointcal models converted to the afw 

1505 structures that will be saved to disk. 

1506 

1507 Parameters 

1508 ---------- 

1509 ccdImageList : `lsst.jointcal.CcdImageList` 

1510 The list of CcdImages to get the output for. 

1511 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel` 

1512 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`. 

1513 func : `str` 

1514 The name of the function to call on ``model`` to get the converted 

1515 structure. Must accept an `lsst.jointcal.CcdImage`. 

1516 

1517 Returns 

1518 ------- 

1519 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or 

1520 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`] 

1521 The data to be saved, keyed on (visit, detector). 

1522 """ 

1523 output = {} 

1524 for ccdImage in ccdImageList: 

1525 ccd = ccdImage.ccdId 

1526 visit = ccdImage.visit 

1527 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd) 

1528 output[(visit, ccd)] = getattr(model, func)(ccdImage) 

1529 return output 

1530 

1531 

1532def make_schema_table(): 

1533 """Return an afw SourceTable to use as a base for creating the 

1534 SourceCatalog to insert values from the dataFrame into. 

1535 

1536 Returns 

1537 ------- 

1538 table : `lsst.afw.table.SourceTable` 

1539 Table with schema and slots to use to make SourceCatalogs. 

1540 """ 

1541 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

1542 schema.addField("centroid_x", "D") 

1543 schema.addField("centroid_y", "D") 

1544 schema.addField("centroid_xErr", "F") 

1545 schema.addField("centroid_yErr", "F") 

1546 schema.addField("shape_xx", "D") 

1547 schema.addField("shape_yy", "D") 

1548 schema.addField("shape_xy", "D") 

1549 schema.addField("flux_instFlux", "D") 

1550 schema.addField("flux_instFluxErr", "D") 

1551 table = lsst.afw.table.SourceTable.make(schema) 

1552 table.defineCentroid("centroid") 

1553 table.defineShape("shape") 

1554 return table 

1555 

1556 

1557def get_sourceTable_visit_columns(inColumns, config, sourceSelector): 

1558 """ 

1559 Get the sourceTable_visit columns to load from the catalogs. 

1560 

1561 Parameters 

1562 ---------- 

1563 inColumns : `list` 

1564 List of columns known to be available in the sourceTable_visit. 

1565 config : `JointcalConfig` 

1566 A filled-in config to to help define column names. 

1567 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask` 

1568 A configured source selector to define column names to load. 

1569 

1570 Returns 

1571 ------- 

1572 columns : `list` 

1573 List of columns to read from sourceTable_visit. 

1574 detectorColumn : `str` 

1575 Name of the detector column. 

1576 ixxColumns : `list` 

1577 Name of the ixx/iyy/ixy columns. 

1578 """ 

1579 if 'detector' in inColumns: 

1580 # Default name for Gen3. 

1581 detectorColumn = 'detector' 

1582 else: 

1583 # Default name for Gen2 conversions (still used in tests, CI, and older catalogs) 

1584 detectorColumn = 'ccd' 

1585 

1586 columns = ['visit', detectorColumn, 

1587 'sourceId', 'x', 'xErr', 'y', 'yErr', 

1588 config.sourceFluxType + '_instFlux', config.sourceFluxType + '_instFluxErr'] 

1589 

1590 if 'ixx' in inColumns: 

1591 # New columns post-DM-31825 

1592 ixxColumns = ['ixx', 'iyy', 'ixy'] 

1593 else: 

1594 # Old columns pre-DM-31825 

1595 ixxColumns = ['Ixx', 'Iyy', 'Ixy'] 

1596 columns.extend(ixxColumns) 

1597 

1598 if sourceSelector.config.doFlags: 

1599 columns.extend(sourceSelector.config.flags.bad) 

1600 if sourceSelector.config.doUnresolved: 

1601 columns.append(sourceSelector.config.unresolved.name) 

1602 if sourceSelector.config.doIsolated: 

1603 columns.append(sourceSelector.config.isolated.parentName) 

1604 columns.append(sourceSelector.config.isolated.nChildName) 

1605 

1606 return columns, detectorColumn, ixxColumns 

1607 

1608 

1609def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId, 

1610 detectorColumn, ixxColumns, sourceFluxType, log): 

1611 """Return an afw SourceCatalog extracted from a visit-level dataframe, 

1612 limited to just one detector. 

1613 

1614 Parameters 

1615 ---------- 

1616 table : `lsst.afw.table.SourceTable` 

1617 Table factory to use to make the SourceCatalog that will be 

1618 populated with data from ``visitCatalog``. 

1619 visitCatalog : `pandas.DataFrame` 

1620 DataFrame to extract a detector catalog from. 

1621 detectorId : `int` 

1622 Numeric id of the detector to extract from ``visitCatalog``. 

1623 detectorColumn : `str` 

1624 Name of the detector column in the catalog. 

1625 ixxColumns : `list` [`str`] 

1626 Names of the ixx/iyy/ixy columns in the catalog. 

1627 sourceFluxType : `str` 

1628 Name of the catalog field to load instFluxes from. 

1629 log : `logging.Logger` 

1630 Logging instance to log to. 

1631 

1632 Returns 

1633 ------- 

1634 catalog : `lsst.afw.table.SourceCatalog`, or `None` 

1635 Detector-level catalog extracted from ``visitCatalog``, or `None` 

1636 if there was no data to load. 

1637 """ 

1638 # map from dataFrame column to afw table column 

1639 mapping = {'x': 'centroid_x', 

1640 'y': 'centroid_y', 

1641 'xErr': 'centroid_xErr', 

1642 'yErr': 'centroid_yErr', 

1643 ixxColumns[0]: 'shape_xx', 

1644 ixxColumns[1]: 'shape_yy', 

1645 ixxColumns[2]: 'shape_xy', 

1646 f'{sourceFluxType}_instFlux': 'flux_instFlux', 

1647 f'{sourceFluxType}_instFluxErr': 'flux_instFluxErr', 

1648 } 

1649 

1650 catalog = lsst.afw.table.SourceCatalog(table) 

1651 matched = visitCatalog[detectorColumn] == detectorId 

1652 n = sum(matched) 

1653 if n == 0: 

1654 return None 

1655 catalog.resize(sum(matched)) 

1656 view = visitCatalog.loc[matched] 

1657 catalog['id'] = view.index 

1658 for dfCol, afwCol in mapping.items(): 

1659 catalog[afwCol] = view[dfCol] 

1660 

1661 log.debug("%d sources selected in visit %d detector %d", 

1662 len(catalog), 

1663 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first 

1664 detectorId) 

1665 return catalog