Coverage for python/lsst/jointcal/jointcal.py: 19%

557 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-08-06 02:27 -0700

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import dataclasses 

23import collections 

24import os 

25 

26import astropy.time 

27import numpy as np 

28import astropy.units as u 

29 

30import lsst.geom 

31import lsst.utils 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34from lsst.afw.image import fluxErrFromABMagErr 

35import lsst.pex.exceptions as pexExceptions 

36import lsst.afw.cameraGeom 

37import lsst.afw.table 

38import lsst.log 

39from lsst.pipe.base import Instrument 

40from lsst.pipe.tasks.colorterms import ColortermLibrary 

41from lsst.verify import Job, Measurement 

42 

43from lsst.meas.algorithms import (ReferenceObjectLoader, ReferenceSourceSelectorTask, 

44 LoadIndexedReferenceObjectsConfig) 

45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

46 

47import lsst.jointcal 

48from lsst.jointcal import MinimizeResult 

49 

50__all__ = ["JointcalConfig", "JointcalTask"] 

51 

52Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

53Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

54 

55 

56# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

57def add_measurement(job, name, value): 

58 meas = Measurement(job.metrics[name], value) 

59 job.measurements.insert(meas) 

60 

61 

62def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

63 """Lookup function that asserts/hopes that a static calibration dataset 

64 exists in a particular collection, since this task can't provide a single 

65 date/time to use to search for one properly. 

66 

67 This is mostly useful for the ``camera`` dataset, in cases where the task's 

68 quantum dimensions do *not* include something temporal, like ``exposure`` 

69 or ``visit``. 

70 

71 Parameters 

72 ---------- 

73 datasetType : `lsst.daf.butler.DatasetType` 

74 Type of dataset being searched for. 

75 registry : `lsst.daf.butler.Registry` 

76 Data repository registry to search. 

77 quantumDataId : `lsst.daf.butler.DataCoordinate` 

78 Data ID of the quantum this camera should match. 

79 collections : `Iterable` [ `str` ] 

80 Collections that should be searched - but this lookup function works 

81 by ignoring this in favor of a more-or-less hard-coded value. 

82 

83 Returns 

84 ------- 

85 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

86 Iterator over dataset references; should have only one element. 

87 

88 Notes 

89 ----- 

90 This implementation duplicates one in fgcmcal, and is at least quite 

91 similar to another in cp_pipe. This duplicate has the most documentation. 

92 Fixing this is DM-29661. 

93 """ 

94 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

95 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

96 return registry.queryDatasets(datasetType, 

97 dataId=quantumDataId, 

98 collections=[unboundedCollection], 

99 findFirst=True) 

100 

101 

102def lookupVisitRefCats(datasetType, registry, quantumDataId, collections): 

103 """Lookup function that finds all refcats for all visits that overlap a 

104 tract, rather than just the refcats that directly overlap the tract. 

105 

106 Parameters 

107 ---------- 

108 datasetType : `lsst.daf.butler.DatasetType` 

109 Type of dataset being searched for. 

110 registry : `lsst.daf.butler.Registry` 

111 Data repository registry to search. 

112 quantumDataId : `lsst.daf.butler.DataCoordinate` 

113 Data ID of the quantum; expected to be something we can use as a 

114 constraint to query for overlapping visits. 

115 collections : `Iterable` [ `str` ] 

116 Collections to search. 

117 

118 Returns 

119 ------- 

120 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

121 Iterator over refcat references. 

122 """ 

123 refs = set() 

124 # Use .expanded() on the query methods below because we need data IDs with 

125 # regions, both in the outer loop over visits (queryDatasets will expand 

126 # any data ID we give it, but doing it up-front in bulk is much more 

127 # efficient) and in the data IDs of the DatasetRefs this function yields 

128 # (because the RefCatLoader relies on them to do some of its own 

129 # filtering). 

130 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()): 

131 refs.update( 

132 registry.queryDatasets( 

133 datasetType, 

134 collections=collections, 

135 dataId=visit_data_id, 

136 findFirst=True, 

137 ).expanded() 

138 ) 

139 yield from refs 

140 

141 

142class JointcalTaskConnections(pipeBase.PipelineTaskConnections, 

143 dimensions=("skymap", "tract", "instrument", "physical_filter")): 

144 """Middleware input/output connections for jointcal data.""" 

145 inputCamera = pipeBase.connectionTypes.PrerequisiteInput( 

146 doc="The camera instrument that took these observations.", 

147 name="camera", 

148 storageClass="Camera", 

149 dimensions=("instrument",), 

150 isCalibration=True, 

151 lookupFunction=lookupStaticCalibrations, 

152 ) 

153 inputSourceTableVisit = pipeBase.connectionTypes.Input( 

154 doc="Source table in parquet format, per visit", 

155 name="sourceTable_visit", 

156 storageClass="DataFrame", 

157 dimensions=("instrument", "visit"), 

158 deferLoad=True, 

159 multiple=True, 

160 ) 

161 inputVisitSummary = pipeBase.connectionTypes.Input( 

162 doc=("Per-visit consolidated exposure metadata built from calexps. " 

163 "These catalogs use detector id for the id and must be sorted for " 

164 "fast lookups of a detector."), 

165 name="visitSummary", 

166 storageClass="ExposureCatalog", 

167 dimensions=("instrument", "visit"), 

168 deferLoad=True, 

169 multiple=True, 

170 ) 

171 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

172 doc="The astrometry reference catalog to match to loaded input catalog sources.", 

173 name="gaia_dr2_20200414", 

174 storageClass="SimpleCatalog", 

175 dimensions=("skypix",), 

176 deferLoad=True, 

177 multiple=True, 

178 lookupFunction=lookupVisitRefCats, 

179 ) 

180 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

181 doc="The photometry reference catalog to match to loaded input catalog sources.", 

182 name="ps1_pv3_3pi_20170110", 

183 storageClass="SimpleCatalog", 

184 dimensions=("skypix",), 

185 deferLoad=True, 

186 multiple=True, 

187 lookupFunction=lookupVisitRefCats, 

188 ) 

189 

190 outputWcs = pipeBase.connectionTypes.Output( 

191 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model." 

192 " These catalogs only contain entries for detectors with an output, and use" 

193 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

194 name="jointcalSkyWcsCatalog", 

195 storageClass="ExposureCatalog", 

196 dimensions=("instrument", "visit", "skymap", "tract"), 

197 multiple=True 

198 ) 

199 outputPhotoCalib = pipeBase.connectionTypes.Output( 

200 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model." 

201 " These catalogs only contain entries for detectors with an output, and use" 

202 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

203 name="jointcalPhotoCalibCatalog", 

204 storageClass="ExposureCatalog", 

205 dimensions=("instrument", "visit", "skymap", "tract"), 

206 multiple=True 

207 ) 

208 

209 # measurements of metrics 

210 # The vars() trick used here allows us to set class attributes 

211 # programatically. Taken from: 

212 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class 

213 for name in ("astrometry", "photometry"): 

214 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output( 

215 doc=f"The number of cross-matched fittedStars for {name}", 

216 name=f"metricvalue_jointcal_{name}_matched_fittedStars", 

217 storageClass="MetricValue", 

218 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

219 ) 

220 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output( 

221 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.", 

222 name=f"metricvalue_jointcal_{name}_collected_refStars", 

223 storageClass="MetricValue", 

224 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

225 ) 

226 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output( 

227 doc=f"The number of {name} reference stars matched to fittedStars.", 

228 name=f"metricvalue_jointcal_{name}_prepared_refStars", 

229 storageClass="MetricValue", 

230 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

231 ) 

232 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output( 

233 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.", 

234 name=f"metricvalue_jointcal_{name}_prepared_fittedStars", 

235 storageClass="MetricValue", 

236 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

237 ) 

238 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output( 

239 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.", 

240 name=f"metricvalue_jointcal_{name}_prepared_ccdImages", 

241 storageClass="MetricValue", 

242 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

243 ) 

244 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output( 

245 doc=f"The final chi2 of the {name} fit.", 

246 name=f"metricvalue_jointcal_{name}_final_chi2", 

247 storageClass="MetricValue", 

248 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

249 ) 

250 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output( 

251 doc=f"The number of degrees of freedom of the fitted {name}.", 

252 name=f"metricvalue_jointcal_{name}_final_ndof", 

253 storageClass="MetricValue", 

254 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

255 ) 

256 

257 def __init__(self, *, config=None): 

258 super().__init__(config=config) 

259 # When we are only doing one of astrometry or photometry, we don't 

260 # need the reference catalog or produce the outputs for the other. 

261 # This informs the middleware of that when the QuantumGraph is 

262 # generated, so we don't block on getting something we won't need or 

263 # create an expectation that downstream tasks will be able to consume 

264 # something we won't produce. 

265 if not config.doAstrometry: 

266 self.prerequisiteInputs.remove("astrometryRefCat") 

267 self.outputs.remove("outputWcs") 

268 for key in list(self.outputs): 

269 if "metricvalue_jointcal_astrometry" in key: 

270 self.outputs.remove(key) 

271 if not config.doPhotometry: 

272 self.prerequisiteInputs.remove("photometryRefCat") 

273 self.outputs.remove("outputPhotoCalib") 

274 for key in list(self.outputs): 

275 if "metricvalue_jointcal_photometry" in key: 

276 self.outputs.remove(key) 

277 

278 

279class JointcalConfig(pipeBase.PipelineTaskConfig, 

280 pipelineConnections=JointcalTaskConnections): 

281 """Configuration for JointcalTask""" 

282 

283 doAstrometry = pexConfig.Field( 

284 doc="Fit astrometry and write the fitted result.", 

285 dtype=bool, 

286 default=True 

287 ) 

288 doPhotometry = pexConfig.Field( 

289 doc="Fit photometry and write the fitted result.", 

290 dtype=bool, 

291 default=True 

292 ) 

293 sourceFluxType = pexConfig.Field( 

294 dtype=str, 

295 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

296 default='apFlux_12_0' 

297 ) 

298 positionErrorPedestal = pexConfig.Field( 

299 doc="Systematic term to apply to the measured position error (pixels)", 

300 dtype=float, 

301 default=0.02, 

302 ) 

303 photometryErrorPedestal = pexConfig.Field( 

304 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

305 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

306 dtype=float, 

307 default=0.0, 

308 ) 

309 # TODO: DM-6885 matchCut should be an geom.Angle 

310 matchCut = pexConfig.Field( 

311 doc="Matching radius between fitted and reference stars (arcseconds)", 

312 dtype=float, 

313 default=3.0, 

314 ) 

315 minMeasurements = pexConfig.Field( 

316 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

317 dtype=int, 

318 default=2, 

319 ) 

320 minMeasuredStarsPerCcd = pexConfig.Field( 

321 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

322 dtype=int, 

323 default=100, 

324 ) 

325 minRefStarsPerCcd = pexConfig.Field( 

326 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

327 dtype=int, 

328 default=30, 

329 ) 

330 allowLineSearch = pexConfig.Field( 

331 doc="Allow a line search during minimization, if it is reasonable for the model" 

332 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

333 dtype=bool, 

334 default=False 

335 ) 

336 astrometrySimpleOrder = pexConfig.Field( 

337 doc="Polynomial order for fitting the simple astrometry model.", 

338 dtype=int, 

339 default=3, 

340 ) 

341 astrometryChipOrder = pexConfig.Field( 

342 doc="Order of the per-chip transform for the constrained astrometry model.", 

343 dtype=int, 

344 default=1, 

345 ) 

346 astrometryVisitOrder = pexConfig.Field( 

347 doc="Order of the per-visit transform for the constrained astrometry model.", 

348 dtype=int, 

349 default=5, 

350 ) 

351 useInputWcs = pexConfig.Field( 

352 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

353 dtype=bool, 

354 default=True, 

355 ) 

356 astrometryModel = pexConfig.ChoiceField( 

357 doc="Type of model to fit to astrometry", 

358 dtype=str, 

359 default="constrained", 

360 allowed={"simple": "One polynomial per ccd", 

361 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

362 ) 

363 photometryModel = pexConfig.ChoiceField( 

364 doc="Type of model to fit to photometry", 

365 dtype=str, 

366 default="constrainedMagnitude", 

367 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

368 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

369 " fitting in flux space.", 

370 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

371 " fitting in magnitude space.", 

372 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

373 " fitting in magnitude space.", 

374 } 

375 ) 

376 applyColorTerms = pexConfig.Field( 

377 doc="Apply photometric color terms to reference stars?" 

378 "Requires that colorterms be set to a ColortermLibrary", 

379 dtype=bool, 

380 default=False 

381 ) 

382 colorterms = pexConfig.ConfigField( 

383 doc="Library of photometric reference catalog name to color term dict.", 

384 dtype=ColortermLibrary, 

385 ) 

386 photometryVisitOrder = pexConfig.Field( 

387 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

388 dtype=int, 

389 default=7, 

390 ) 

391 photometryDoRankUpdate = pexConfig.Field( 

392 doc=("Do the rank update step during minimization. " 

393 "Skipping this can help deal with models that are too non-linear."), 

394 dtype=bool, 

395 default=True, 

396 ) 

397 astrometryDoRankUpdate = pexConfig.Field( 

398 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

399 "Skipping this can help deal with models that are too non-linear."), 

400 dtype=bool, 

401 default=True, 

402 ) 

403 outlierRejectSigma = pexConfig.Field( 

404 doc="How many sigma to reject outliers at during minimization.", 

405 dtype=float, 

406 default=5.0, 

407 ) 

408 astrometryOutlierRelativeTolerance = pexConfig.Field( 

409 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will " 

410 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set " 

411 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002" 

412 "as a balance between a shorter minimization runtime and achieving a final fitted model that is" 

413 "close to the solution found when removing all outliers."), 

414 dtype=float, 

415 default=0, 

416 ) 

417 maxPhotometrySteps = pexConfig.Field( 

418 doc="Maximum number of minimize iterations to take when fitting photometry.", 

419 dtype=int, 

420 default=20, 

421 ) 

422 maxAstrometrySteps = pexConfig.Field( 

423 doc="Maximum number of minimize iterations to take when fitting astrometry.", 

424 dtype=int, 

425 default=20, 

426 ) 

427 astrometryRefObjLoader = pexConfig.ConfigField( 

428 dtype=LoadIndexedReferenceObjectsConfig, 

429 doc="Reference object loader for astrometric fit", 

430 ) 

431 photometryRefObjLoader = pexConfig.ConfigField( 

432 dtype=LoadIndexedReferenceObjectsConfig, 

433 doc="Reference object loader for photometric fit", 

434 ) 

435 sourceSelector = sourceSelectorRegistry.makeField( 

436 doc="How to select sources for cross-matching", 

437 default="science" 

438 ) 

439 astrometryReferenceSelector = pexConfig.ConfigurableField( 

440 target=ReferenceSourceSelectorTask, 

441 doc="How to down-select the loaded astrometry reference catalog.", 

442 ) 

443 photometryReferenceSelector = pexConfig.ConfigurableField( 

444 target=ReferenceSourceSelectorTask, 

445 doc="How to down-select the loaded photometry reference catalog.", 

446 ) 

447 astrometryReferenceErr = pexConfig.Field( 

448 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

449 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

450 "If specified, overrides any existing `coord_*Err` values."), 

451 dtype=float, 

452 default=None, 

453 optional=True 

454 ) 

455 

456 # configs for outputting debug information 

457 writeInitMatrix = pexConfig.Field( 

458 dtype=bool, 

459 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

460 "Output files will be written to `config.debugOutputPath` and will " 

461 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. " 

462 "Note that these files are the dense versions of the matrix, and so may be very large."), 

463 default=False 

464 ) 

465 writeChi2FilesInitialFinal = pexConfig.Field( 

466 dtype=bool, 

467 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. " 

468 "Output files will be written to `config.debugOutputPath` and will " 

469 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."), 

470 default=False 

471 ) 

472 writeChi2FilesOuterLoop = pexConfig.Field( 

473 dtype=bool, 

474 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. " 

475 "Output files will be written to `config.debugOutputPath` and will " 

476 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."), 

477 default=False 

478 ) 

479 writeInitialModel = pexConfig.Field( 

480 dtype=bool, 

481 doc=("Write the pre-initialization model to text files, for debugging. " 

482 "Output files will be written to `config.debugOutputPath` and will be " 

483 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."), 

484 default=False 

485 ) 

486 debugOutputPath = pexConfig.Field( 

487 dtype=str, 

488 default=".", 

489 doc=("Path to write debug output files to. Used by " 

490 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

491 ) 

492 detailedProfile = pexConfig.Field( 

493 dtype=bool, 

494 default=False, 

495 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting" 

496 ) 

497 

498 def validate(self): 

499 super().validate() 

500 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

501 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

502 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

503 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

504 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

505 "applyColorTerms=True will be ignored.") 

506 lsst.log.warning(msg) 

507 

508 def setDefaults(self): 

509 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing. 

510 self.sourceSelector["science"].doUnresolved = True 

511 self.sourceSelector["science"].unresolved.name = "extendedness" 

512 # with dependable signal to noise ratio. 

513 self.sourceSelector["science"].doSignalToNoise = True 

514 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

515 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

516 self.sourceSelector["science"].signalToNoise.minimum = 10. 

517 # Base SNR selection on `sourceFluxType` because that is the flux that jointcal fits. 

518 self.sourceSelector["science"].signalToNoise.fluxField = f"{self.sourceFluxType}_instFlux" 

519 self.sourceSelector["science"].signalToNoise.errField = f"{self.sourceFluxType}_instFluxErr" 

520 # Do not trust blended sources" aperture fluxes which also depend on seeing. 

521 self.sourceSelector["science"].doIsolated = True 

522 self.sourceSelector["science"].isolated.parentName = "parentSourceId" 

523 self.sourceSelector["science"].isolated.nChildName = "deblend_nChild" 

524 # Do not trust either flux or centroid measurements with flags, 

525 # chosen from the usual QA flags for stars) 

526 self.sourceSelector["science"].doFlags = True 

527 badFlags = ["pixelFlags_edge", 

528 "pixelFlags_saturated", 

529 "pixelFlags_interpolatedCenter", 

530 "pixelFlags_interpolated", 

531 "pixelFlags_crCenter", 

532 "pixelFlags_bad", 

533 "hsmPsfMoments_flag", 

534 f"{self.sourceFluxType}_flag", 

535 ] 

536 self.sourceSelector["science"].flags.bad = badFlags 

537 

538 # Use Gaia-DR2 with proper motions for astrometry; phot_g_mean is the 

539 # primary Gaia band, but is not like any normal photometric band. 

540 self.astrometryRefObjLoader.requireProperMotion = True 

541 self.astrometryRefObjLoader.anyFilterMapsToThis = "phot_g_mean" 

542 

543 

544def writeModel(model, filename, log): 

545 """Write model to outfile.""" 

546 with open(filename, "w") as file: 

547 file.write(repr(model)) 

548 log.info("Wrote %s to file: %s", model, filename) 

549 

550 

551@dataclasses.dataclass 

552class JointcalInputData: 

553 """The input data jointcal needs for each detector/visit.""" 

554 visit: int 

555 """The visit identifier of this exposure.""" 

556 catalog: lsst.afw.table.SourceCatalog 

557 """The catalog derived from this exposure.""" 

558 visitInfo: lsst.afw.image.VisitInfo 

559 """The VisitInfo of this exposure.""" 

560 detector: lsst.afw.cameraGeom.Detector 

561 """The detector of this exposure.""" 

562 photoCalib: lsst.afw.image.PhotoCalib 

563 """The photometric calibration of this exposure.""" 

564 wcs: lsst.afw.geom.skyWcs 

565 """The WCS of this exposure.""" 

566 bbox: lsst.geom.Box2I 

567 """The bounding box of this exposure.""" 

568 filter: lsst.afw.image.FilterLabel 

569 """The filter of this exposure.""" 

570 

571 

572class JointcalTask(pipeBase.PipelineTask): 

573 """Astrometricly and photometricly calibrate across multiple visits of the 

574 same field. 

575 """ 

576 

577 ConfigClass = JointcalConfig 

578 _DefaultName = "jointcal" 

579 

580 def __init__(self, **kwargs): 

581 super().__init__(**kwargs) 

582 self.makeSubtask("sourceSelector") 

583 if self.config.doAstrometry: 

584 self.makeSubtask("astrometryReferenceSelector") 

585 else: 

586 self.astrometryRefObjLoader = None 

587 if self.config.doPhotometry: 

588 self.makeSubtask("photometryReferenceSelector") 

589 else: 

590 self.photometryRefObjLoader = None 

591 

592 # To hold various computed metrics for use by tests 

593 self.job = Job.load_metrics_package(subset='jointcal') 

594 

595 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

596 # We override runQuantum to set up the refObjLoaders and write the 

597 # outputs to the correct refs. 

598 inputs = butlerQC.get(inputRefs) 

599 # We want the tract number for writing debug files 

600 tract = butlerQC.quantum.dataId['tract'] 

601 if self.config.doAstrometry: 

602 self.astrometryRefObjLoader = ReferenceObjectLoader( 

603 dataIds=[ref.datasetRef.dataId 

604 for ref in inputRefs.astrometryRefCat], 

605 refCats=inputs.pop('astrometryRefCat'), 

606 config=self.config.astrometryRefObjLoader, 

607 log=self.log) 

608 if self.config.doPhotometry: 

609 self.photometryRefObjLoader = ReferenceObjectLoader( 

610 dataIds=[ref.datasetRef.dataId 

611 for ref in inputRefs.photometryRefCat], 

612 refCats=inputs.pop('photometryRefCat'), 

613 config=self.config.photometryRefObjLoader, 

614 log=self.log) 

615 outputs = self.run(**inputs, tract=tract) 

616 self._put_metrics(butlerQC, outputs.job, outputRefs) 

617 if self.config.doAstrometry: 

618 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs, 

619 inputs['inputCamera'], "setWcs") 

620 if self.config.doPhotometry: 

621 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib, 

622 inputs['inputCamera'], "setPhotoCalib") 

623 

624 def _put_metrics(self, butlerQC, job, outputRefs): 

625 """Persist all measured metrics stored in a job. 

626 

627 Parameters 

628 ---------- 

629 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

630 A butler which is specialized to operate in the context of a 

631 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

632 job : `lsst.verify.job.Job` 

633 Measurements of metrics to persist. 

634 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

635 The DatasetRefs to persist the data to. 

636 """ 

637 for key in job.measurements.keys(): 

638 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', ''))) 

639 

640 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter): 

641 """Persist the output datasets to their appropriate datarefs. 

642 

643 Parameters 

644 ---------- 

645 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

646 A butler which is specialized to operate in the context of a 

647 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

648 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or 

649 `dict` [`tuple, `lsst.afw.image.PhotoCalib`] 

650 The fitted objects to persist. 

651 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

652 The DatasetRefs to persist the data to. 

653 camera : `lsst.afw.cameraGeom.Camera` 

654 The camera for this instrument, to get detector ids from. 

655 setter : `str` 

656 The method to call on the ExposureCatalog to set each output. 

657 """ 

658 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

659 schema.addField('visit', type='L', doc='Visit number') 

660 

661 def new_catalog(visit, size): 

662 """Return an catalog ready to be filled with appropriate output.""" 

663 catalog = lsst.afw.table.ExposureCatalog(schema) 

664 catalog.resize(size) 

665 catalog['visit'] = visit 

666 metadata = lsst.daf.base.PropertyList() 

667 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

668 metadata.add("COMMENT", "Only detectors with data have entries.") 

669 return catalog 

670 

671 # count how many detectors have output for each visit 

672 detectors_per_visit = collections.defaultdict(int) 

673 for key in outputs: 

674 # key is (visit, detector_id), and we only need visit here 

675 detectors_per_visit[key[0]] += 1 

676 

677 for ref in outputRefs: 

678 visit = ref.dataId['visit'] 

679 catalog = new_catalog(visit, detectors_per_visit[visit]) 

680 # Iterate over every detector and skip the ones we don't have output for. 

681 i = 0 

682 for detector in camera: 

683 detectorId = detector.getId() 

684 key = (ref.dataId['visit'], detectorId) 

685 if key not in outputs: 

686 # skip detectors we don't have output for 

687 self.log.debug("No %s output for detector %s in visit %s", 

688 setter[3:], detectorId, visit) 

689 continue 

690 

691 catalog[i].setId(detectorId) 

692 getattr(catalog[i], setter)(outputs[key]) 

693 i += 1 

694 

695 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups 

696 butlerQC.put(catalog, ref) 

697 self.log.info("Wrote %s detectors to %s", i, ref) 

698 

699 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None): 

700 # Docstring inherited. 

701 

702 # We take values out of the Parquet table, and put them in "flux_", 

703 # and the config.sourceFluxType field is used during that extraction, 

704 # so just use "flux" here. 

705 sourceFluxField = "flux" 

706 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

707 associations = lsst.jointcal.Associations() 

708 self.focalPlaneBBox = inputCamera.getFpBBox() 

709 oldWcsList, bands = self._load_data(inputSourceTableVisit, 

710 inputVisitSummary, 

711 associations, 

712 jointcalControl, 

713 inputCamera) 

714 

715 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands) 

716 

717 if self.config.doAstrometry: 

718 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

719 name="astrometry", 

720 refObjLoader=self.astrometryRefObjLoader, 

721 referenceSelector=self.astrometryReferenceSelector, 

722 fit_function=self._fit_astrometry, 

723 tract=tract, 

724 epoch=epoch) 

725 astrometry_output = self._make_output(associations.getCcdImageList(), 

726 astrometry.model, 

727 "makeSkyWcs") 

728 else: 

729 astrometry_output = None 

730 

731 if self.config.doPhotometry: 

732 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

733 name="photometry", 

734 refObjLoader=self.photometryRefObjLoader, 

735 referenceSelector=self.photometryReferenceSelector, 

736 fit_function=self._fit_photometry, 

737 tract=tract, 

738 epoch=epoch, 

739 reject_bad_fluxes=True) 

740 photometry_output = self._make_output(associations.getCcdImageList(), 

741 photometry.model, 

742 "toPhotoCalib") 

743 else: 

744 photometry_output = None 

745 

746 return pipeBase.Struct(outputWcs=astrometry_output, 

747 outputPhotoCalib=photometry_output, 

748 job=self.job, 

749 astrometryRefObjLoader=self.astrometryRefObjLoader, 

750 photometryRefObjLoader=self.photometryRefObjLoader) 

751 

752 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, 

753 jointcalControl, camera): 

754 """Read the data that jointcal needs to run. 

755 

756 Modifies ``associations`` in-place with the loaded data. 

757 

758 Parameters 

759 ---------- 

760 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

761 References to visit-level DataFrames to load the catalog data from. 

762 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

763 Visit-level exposure summary catalog with metadata. 

764 associations : `lsst.jointcal.Associations` 

765 Object to add the loaded data to by constructing new CcdImages. 

766 jointcalControl : `jointcal.JointcalControl` 

767 Control object for C++ associations management. 

768 camera : `lsst.afw.cameraGeom.Camera` 

769 Camera object for detector geometry. 

770 

771 Returns 

772 ------- 

773 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`] 

774 The original WCS of the input data, to aid in writing tests. 

775 bands : `list` [`str`] 

776 The filter bands of each input dataset. 

777 """ 

778 oldWcsList = [] 

779 filters = [] 

780 load_cat_profile_file = 'jointcal_load_data.prof' if self.config.detailedProfile else '' 

781 with pipeBase.cmdLineTask.profile(load_cat_profile_file): 

782 table = make_schema_table() # every detector catalog has the same layout 

783 # No guarantee that the input is in the same order of visits, so we have to map one of them. 

784 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)} 

785 detectorDict = {detector.getId(): detector for detector in camera} 

786 

787 columns = None 

788 

789 for visitSummaryRef in inputVisitSummary: 

790 visitSummary = visitSummaryRef.get() 

791 

792 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]] 

793 if columns is None: 

794 inColumns = dataRef.get(component='columns') 

795 columns, detColumn, ixxColumns = get_sourceTable_visit_columns(inColumns, 

796 self.config, 

797 self.sourceSelector) 

798 visitCatalog = dataRef.get(parameters={'columns': columns}) 

799 

800 selected = self.sourceSelector.run(visitCatalog) 

801 

802 # Build a CcdImage for each detector in this visit. 

803 detectors = {id: index for index, id in enumerate(visitSummary['id'])} 

804 for id, index in detectors.items(): 

805 catalog = extract_detector_catalog_from_visit_catalog(table, 

806 selected.sourceCat, 

807 id, 

808 detColumn, 

809 ixxColumns, 

810 self.config.sourceFluxType, 

811 self.log) 

812 if catalog is None: 

813 continue 

814 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict) 

815 result = self._build_ccdImage(data, associations, jointcalControl) 

816 if result is not None: 

817 oldWcsList.append(result.wcs) 

818 # A visit has only one band, so we can just use the first. 

819 filters.append(data.filter) 

820 if len(filters) == 0: 

821 raise RuntimeError("No data to process: did source selector remove all sources?") 

822 filters = collections.Counter(filters) 

823 

824 return oldWcsList, filters 

825 

826 def _make_one_input_data(self, visitRecord, catalog, detectorDict): 

827 """Return a data structure for this detector+visit.""" 

828 return JointcalInputData(visit=visitRecord['visit'], 

829 catalog=catalog, 

830 visitInfo=visitRecord.getVisitInfo(), 

831 detector=detectorDict[visitRecord.getId()], 

832 photoCalib=visitRecord.getPhotoCalib(), 

833 wcs=visitRecord.getWcs(), 

834 bbox=visitRecord.getBBox(), 

835 # ExposureRecord doesn't have a FilterLabel yet, 

836 # so we have to make one. 

837 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'], 

838 physical=visitRecord['physical_filter'])) 

839 

840 # We don't currently need to persist the metadata. 

841 # If we do in the future, we will have to add appropriate dataset templates 

842 # to each obs package (the metadata template should look like `jointcal_wcs`). 

843 def _getMetadataName(self): 

844 return None 

845 

846 def _build_ccdImage(self, data, associations, jointcalControl): 

847 """ 

848 Extract the necessary things from this catalog+metadata to add a new 

849 ccdImage. 

850 

851 Parameters 

852 ---------- 

853 data : `JointcalInputData` 

854 The loaded input data. 

855 associations : `lsst.jointcal.Associations` 

856 Object to add the info to, to construct a new CcdImage 

857 jointcalControl : `jointcal.JointcalControl` 

858 Control object for associations management 

859 

860 Returns 

861 ------ 

862 namedtuple or `None` 

863 ``wcs`` 

864 The TAN WCS of this image, read from the calexp 

865 (`lsst.afw.geom.SkyWcs`). 

866 ``key`` 

867 A key to identify this dataRef by its visit and ccd ids 

868 (`namedtuple`). 

869 `None` 

870 if there are no sources in the loaded catalog. 

871 """ 

872 if len(data.catalog) == 0: 

873 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId()) 

874 return None 

875 

876 associations.createCcdImage(data.catalog, 

877 data.wcs, 

878 data.visitInfo, 

879 data.bbox, 

880 data.filter.physicalLabel, 

881 data.photoCalib, 

882 data.detector, 

883 data.visit, 

884 data.detector.getId(), 

885 jointcalControl) 

886 

887 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key')) 

888 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

889 return Result(data.wcs, Key(data.visit, data.detector.getId())) 

890 

891 def _getDebugPath(self, filename): 

892 """Constructs a path to filename using the configured debug path. 

893 """ 

894 return os.path.join(self.config.debugOutputPath, filename) 

895 

896 def _prep_sky(self, associations, filters): 

897 """Prepare on-sky and other data that must be computed after data has 

898 been read. 

899 """ 

900 associations.computeCommonTangentPoint() 

901 

902 boundingCircle = associations.computeBoundingCircle() 

903 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

904 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

905 

906 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

907 

908 # Determine a default filter band associated with the catalog. See DM-9093 

909 defaultFilter = filters.most_common(1)[0][0] 

910 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel) 

911 

912 # compute and set the reference epoch of the observations, for proper motion corrections 

913 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

914 associations.setEpoch(epoch.jyear) 

915 

916 return boundingCircle, center, radius, defaultFilter, epoch 

917 

918 def _get_refcat_coordinate_error_override(self, refCat, name): 

919 """Check whether we should override the refcat coordinate errors, and 

920 return the overridden error if necessary. 

921 

922 Parameters 

923 ---------- 

924 refCat : `lsst.afw.table.SimpleCatalog` 

925 The reference catalog to check for a ``coord_raErr`` field. 

926 name : `str` 

927 Whether we are doing "astrometry" or "photometry". 

928 

929 Returns 

930 ------- 

931 refCoordErr : `float` 

932 The refcat coordinate error to use, or NaN if we are not overriding 

933 those fields. 

934 

935 Raises 

936 ------ 

937 lsst.pex.config.FieldValidationError 

938 Raised if the refcat does not contain coordinate errors and 

939 ``config.astrometryReferenceErr`` is not set. 

940 """ 

941 # This value doesn't matter for photometry, so just set something to 

942 # keep old refcats from causing problems. 

943 if name.lower() == "photometry": 

944 if 'coord_raErr' not in refCat.schema: 

945 return 100 

946 else: 

947 return float('nan') 

948 

949 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

950 msg = ("Reference catalog does not contain coordinate errors, " 

951 "and config.astrometryReferenceErr not supplied.") 

952 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

953 self.config, 

954 msg) 

955 

956 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

957 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

958 self.config.astrometryReferenceErr) 

959 

960 if self.config.astrometryReferenceErr is None: 

961 return float('nan') 

962 else: 

963 return self.config.astrometryReferenceErr 

964 

965 def _compute_proper_motion_epoch(self, ccdImageList): 

966 """Return the proper motion correction epoch of the provided images. 

967 

968 Parameters 

969 ---------- 

970 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

971 The images to compute the appropriate epoch for. 

972 

973 Returns 

974 ------- 

975 epoch : `astropy.time.Time` 

976 The date to use for proper motion corrections. 

977 """ 

978 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]), 

979 format="jyear", 

980 scale="tai") 

981 

982 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

983 tract="", match_cut=3.0, 

984 reject_bad_fluxes=False, *, 

985 name="", refObjLoader=None, referenceSelector=None, 

986 fit_function=None, epoch=None): 

987 """Load reference catalog, perform the fit, and return the result. 

988 

989 Parameters 

990 ---------- 

991 associations : `lsst.jointcal.Associations` 

992 The star/reference star associations to fit. 

993 defaultFilter : `lsst.afw.image.FilterLabel` 

994 filter to load from reference catalog. 

995 center : `lsst.geom.SpherePoint` 

996 ICRS center of field to load from reference catalog. 

997 radius : `lsst.geom.Angle` 

998 On-sky radius to load from reference catalog. 

999 name : `str` 

1000 Name of thing being fit: "astrometry" or "photometry". 

1001 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

1002 Reference object loader to use to load a reference catalog. 

1003 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1004 Selector to use to pick objects from the loaded reference catalog. 

1005 fit_function : callable 

1006 Function to call to perform fit (takes Associations object). 

1007 tract : `str`, optional 

1008 Name of tract currently being fit. 

1009 match_cut : `float`, optional 

1010 Radius in arcseconds to find cross-catalog matches to during 

1011 associations.associateCatalogs. 

1012 reject_bad_fluxes : `bool`, optional 

1013 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

1014 epoch : `astropy.time.Time`, optional 

1015 Epoch to which to correct refcat proper motion and parallax, 

1016 or `None` to not apply such corrections. 

1017 

1018 Returns 

1019 ------- 

1020 result : `Photometry` or `Astrometry` 

1021 Result of `fit_function()` 

1022 """ 

1023 self.log.info("====== Now processing %s...", name) 

1024 # TODO: this should not print "trying to invert a singular transformation:" 

1025 # if it does that, something's not right about the WCS... 

1026 associations.associateCatalogs(match_cut) 

1027 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name, 

1028 associations.fittedStarListSize()) 

1029 

1030 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

1031 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

1032 center, radius, defaultFilter, 

1033 applyColorterms=applyColorterms, 

1034 epoch=epoch) 

1035 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

1036 

1037 associations.collectRefStars(refCat, 

1038 self.config.matchCut*lsst.geom.arcseconds, 

1039 fluxField, 

1040 refCoordinateErr=refCoordErr, 

1041 rejectBadFluxes=reject_bad_fluxes) 

1042 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name, 

1043 associations.refStarListSize()) 

1044 

1045 associations.prepareFittedStars(self.config.minMeasurements) 

1046 

1047 self._check_star_lists(associations, name) 

1048 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name, 

1049 associations.nFittedStarsWithAssociatedRefStar()) 

1050 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name, 

1051 associations.fittedStarListSize()) 

1052 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name, 

1053 associations.nCcdImagesValidForFit()) 

1054 

1055 fit_profile_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else '' 

1056 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel) 

1057 with pipeBase.cmdLineTask.profile(fit_profile_file): 

1058 result = fit_function(associations, dataName) 

1059 # TODO DM-12446: turn this into a "butler save" somehow. 

1060 # Save reference and measurement chi2 contributions for this data 

1061 if self.config.writeChi2FilesInitialFinal: 

1062 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

1063 result.fit.saveChi2Contributions(baseName+"{type}") 

1064 self.log.info("Wrote chi2 contributions files: %s", baseName) 

1065 

1066 return result 

1067 

1068 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, 

1069 applyColorterms=False, epoch=None): 

1070 """Load the necessary reference catalog sources, convert fluxes to 

1071 correct units, and apply color term corrections if requested. 

1072 

1073 Parameters 

1074 ---------- 

1075 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader` 

1076 The reference catalog loader to use to get the data. 

1077 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1078 Source selector to apply to loaded reference catalog. 

1079 center : `lsst.geom.SpherePoint` 

1080 The center around which to load sources. 

1081 radius : `lsst.geom.Angle` 

1082 The radius around ``center`` to load sources in. 

1083 filterLabel : `lsst.afw.image.FilterLabel` 

1084 The camera filter to load fluxes for. 

1085 applyColorterms : `bool` 

1086 Apply colorterm corrections to the refcat for ``filterName``? 

1087 epoch : `astropy.time.Time`, optional 

1088 Epoch to which to correct refcat proper motion and parallax, 

1089 or `None` to not apply such corrections. 

1090 

1091 Returns 

1092 ------- 

1093 refCat : `lsst.afw.table.SimpleCatalog` 

1094 The loaded reference catalog. 

1095 fluxField : `str` 

1096 The name of the reference catalog flux field appropriate for ``filterName``. 

1097 """ 

1098 skyCircle = refObjLoader.loadSkyCircle(center, 

1099 radius, 

1100 filterLabel.bandLabel, 

1101 epoch=epoch) 

1102 

1103 selected = referenceSelector.run(skyCircle.refCat) 

1104 # Need memory contiguity to get reference filters as a vector. 

1105 if not selected.sourceCat.isContiguous(): 

1106 refCat = selected.sourceCat.copy(deep=True) 

1107 else: 

1108 refCat = selected.sourceCat 

1109 

1110 if applyColorterms: 

1111 refCatName = self.config.connections.photometryRefCat 

1112 self.log.info("Applying color terms for physical filter=%r reference catalog=%s", 

1113 filterLabel.physicalLabel, refCatName) 

1114 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel, 

1115 refCatName, 

1116 doRaise=True) 

1117 

1118 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

1119 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

1120 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

1121 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

1122 

1123 return refCat, skyCircle.fluxField 

1124 

1125 def _check_star_lists(self, associations, name): 

1126 # TODO: these should be len(blah), but we need this properly wrapped first. 

1127 if associations.nCcdImagesValidForFit() == 0: 

1128 raise RuntimeError('No images in the ccdImageList!') 

1129 if associations.fittedStarListSize() == 0: 

1130 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

1131 if associations.refStarListSize() == 0: 

1132 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

1133 

1134 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

1135 """Compute chi2, log it, validate the model, and return chi2. 

1136 

1137 Parameters 

1138 ---------- 

1139 associations : `lsst.jointcal.Associations` 

1140 The star/reference star associations to fit. 

1141 fit : `lsst.jointcal.FitterBase` 

1142 The fitter to use for minimization. 

1143 model : `lsst.jointcal.Model` 

1144 The model being fit. 

1145 chi2Label : `str` 

1146 Label to describe the chi2 (e.g. "Initialized", "Final"). 

1147 writeChi2Name : `str`, optional 

1148 Filename prefix to write the chi2 contributions to. 

1149 Do not supply an extension: an appropriate one will be added. 

1150 

1151 Returns 

1152 ------- 

1153 chi2: `lsst.jointcal.Chi2Accumulator` 

1154 The chi2 object for the current fitter and model. 

1155 

1156 Raises 

1157 ------ 

1158 FloatingPointError 

1159 Raised if chi2 is infinite or NaN. 

1160 ValueError 

1161 Raised if the model is not valid. 

1162 """ 

1163 if writeChi2Name is not None: 

1164 fullpath = self._getDebugPath(writeChi2Name) 

1165 fit.saveChi2Contributions(fullpath+"{type}") 

1166 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

1167 

1168 chi2 = fit.computeChi2() 

1169 self.log.info("%s %s", chi2Label, chi2) 

1170 self._check_stars(associations) 

1171 if not np.isfinite(chi2.chi2): 

1172 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

1173 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

1174 raise ValueError("Model is not valid: check log messages for warnings.") 

1175 return chi2 

1176 

1177 def _fit_photometry(self, associations, dataName=None): 

1178 """ 

1179 Fit the photometric data. 

1180 

1181 Parameters 

1182 ---------- 

1183 associations : `lsst.jointcal.Associations` 

1184 The star/reference star associations to fit. 

1185 dataName : `str` 

1186 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1187 identifying debugging files. 

1188 

1189 Returns 

1190 ------- 

1191 fit_result : `namedtuple` 

1192 fit : `lsst.jointcal.PhotometryFit` 

1193 The photometric fitter used to perform the fit. 

1194 model : `lsst.jointcal.PhotometryModel` 

1195 The photometric model that was fit. 

1196 """ 

1197 self.log.info("=== Starting photometric fitting...") 

1198 

1199 # TODO: should use pex.config.RegistryField here (see DM-9195) 

1200 if self.config.photometryModel == "constrainedFlux": 

1201 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

1202 self.focalPlaneBBox, 

1203 visitOrder=self.config.photometryVisitOrder, 

1204 errorPedestal=self.config.photometryErrorPedestal) 

1205 # potentially nonlinear problem, so we may need a line search to converge. 

1206 doLineSearch = self.config.allowLineSearch 

1207 elif self.config.photometryModel == "constrainedMagnitude": 

1208 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

1209 self.focalPlaneBBox, 

1210 visitOrder=self.config.photometryVisitOrder, 

1211 errorPedestal=self.config.photometryErrorPedestal) 

1212 # potentially nonlinear problem, so we may need a line search to converge. 

1213 doLineSearch = self.config.allowLineSearch 

1214 elif self.config.photometryModel == "simpleFlux": 

1215 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

1216 errorPedestal=self.config.photometryErrorPedestal) 

1217 doLineSearch = False # purely linear in model parameters, so no line search needed 

1218 elif self.config.photometryModel == "simpleMagnitude": 

1219 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

1220 errorPedestal=self.config.photometryErrorPedestal) 

1221 doLineSearch = False # purely linear in model parameters, so no line search needed 

1222 

1223 fit = lsst.jointcal.PhotometryFit(associations, model) 

1224 # TODO DM-12446: turn this into a "butler save" somehow. 

1225 # Save reference and measurement chi2 contributions for this data 

1226 if self.config.writeChi2FilesInitialFinal: 

1227 baseName = f"photometry_initial_chi2-{dataName}" 

1228 else: 

1229 baseName = None 

1230 if self.config.writeInitialModel: 

1231 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt") 

1232 writeModel(model, fullpath, self.log) 

1233 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

1234 

1235 def getChi2Name(whatToFit): 

1236 if self.config.writeChi2FilesOuterLoop: 

1237 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

1238 else: 

1239 return None 

1240 

1241 # The constrained model needs the visit transform fit first; the chip 

1242 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

1243 if self.config.writeInitMatrix: 

1244 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}") 

1245 else: 

1246 dumpMatrixFile = "" 

1247 if self.config.photometryModel.startswith("constrained"): 

1248 # no line search: should be purely (or nearly) linear, 

1249 # and we want a large step size to initialize with. 

1250 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

1251 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

1252 writeChi2Name=getChi2Name("ModelVisit")) 

1253 dumpMatrixFile = "" # so we don't redo the output on the next step 

1254 

1255 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

1256 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

1257 writeChi2Name=getChi2Name("Model")) 

1258 

1259 fit.minimize("Fluxes") # no line search: always purely linear. 

1260 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

1261 writeChi2Name=getChi2Name("Fluxes")) 

1262 

1263 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

1264 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

1265 writeChi2Name=getChi2Name("ModelFluxes")) 

1266 

1267 model.freezeErrorTransform() 

1268 self.log.debug("Photometry error scales are frozen.") 

1269 

1270 chi2 = self._iterate_fit(associations, 

1271 fit, 

1272 self.config.maxPhotometrySteps, 

1273 "photometry", 

1274 "Model Fluxes", 

1275 doRankUpdate=self.config.photometryDoRankUpdate, 

1276 doLineSearch=doLineSearch, 

1277 dataName=dataName) 

1278 

1279 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

1280 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

1281 return Photometry(fit, model) 

1282 

1283 def _fit_astrometry(self, associations, dataName=None): 

1284 """ 

1285 Fit the astrometric data. 

1286 

1287 Parameters 

1288 ---------- 

1289 associations : `lsst.jointcal.Associations` 

1290 The star/reference star associations to fit. 

1291 dataName : `str` 

1292 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1293 identifying debugging files. 

1294 

1295 Returns 

1296 ------- 

1297 fit_result : `namedtuple` 

1298 fit : `lsst.jointcal.AstrometryFit` 

1299 The astrometric fitter used to perform the fit. 

1300 model : `lsst.jointcal.AstrometryModel` 

1301 The astrometric model that was fit. 

1302 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

1303 The model for the sky to tangent plane projection that was used in the fit. 

1304 """ 

1305 

1306 self.log.info("=== Starting astrometric fitting...") 

1307 

1308 associations.deprojectFittedStars() 

1309 

1310 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1311 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1312 # them so carefully? 

1313 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1314 

1315 if self.config.astrometryModel == "constrained": 

1316 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1317 sky_to_tan_projection, 

1318 chipOrder=self.config.astrometryChipOrder, 

1319 visitOrder=self.config.astrometryVisitOrder) 

1320 elif self.config.astrometryModel == "simple": 

1321 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1322 sky_to_tan_projection, 

1323 self.config.useInputWcs, 

1324 nNotFit=0, 

1325 order=self.config.astrometrySimpleOrder) 

1326 

1327 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1328 # TODO DM-12446: turn this into a "butler save" somehow. 

1329 # Save reference and measurement chi2 contributions for this data 

1330 if self.config.writeChi2FilesInitialFinal: 

1331 baseName = f"astrometry_initial_chi2-{dataName}" 

1332 else: 

1333 baseName = None 

1334 if self.config.writeInitialModel: 

1335 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt") 

1336 writeModel(model, fullpath, self.log) 

1337 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1338 

1339 def getChi2Name(whatToFit): 

1340 if self.config.writeChi2FilesOuterLoop: 

1341 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1342 else: 

1343 return None 

1344 

1345 if self.config.writeInitMatrix: 

1346 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}") 

1347 else: 

1348 dumpMatrixFile = "" 

1349 # The constrained model needs the visit transform fit first; the chip 

1350 # transform is initialized from the detector's cameraGeom, so it's close. 

1351 if self.config.astrometryModel == "constrained": 

1352 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1353 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1354 writeChi2Name=getChi2Name("DistortionsVisit")) 

1355 dumpMatrixFile = "" # so we don't redo the output on the next step 

1356 

1357 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1358 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1359 writeChi2Name=getChi2Name("Distortions")) 

1360 

1361 fit.minimize("Positions") 

1362 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1363 writeChi2Name=getChi2Name("Positions")) 

1364 

1365 fit.minimize("Distortions Positions") 

1366 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1367 writeChi2Name=getChi2Name("DistortionsPositions")) 

1368 

1369 chi2 = self._iterate_fit(associations, 

1370 fit, 

1371 self.config.maxAstrometrySteps, 

1372 "astrometry", 

1373 "Distortions Positions", 

1374 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance, 

1375 doRankUpdate=self.config.astrometryDoRankUpdate, 

1376 dataName=dataName) 

1377 

1378 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1379 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1380 

1381 return Astrometry(fit, model, sky_to_tan_projection) 

1382 

1383 def _check_stars(self, associations): 

1384 """Count measured and reference stars per ccd and warn/log them.""" 

1385 for ccdImage in associations.getCcdImageList(): 

1386 nMeasuredStars, nRefStars = ccdImage.countStars() 

1387 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1388 ccdImage.getName(), nMeasuredStars, nRefStars) 

1389 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1390 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)", 

1391 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1392 if nRefStars < self.config.minRefStarsPerCcd: 

1393 self.log.warning("ccdImage %s has only %s RefStars (desired %s)", 

1394 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1395 

1396 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1397 dataName="", 

1398 sigmaRelativeTolerance=0, 

1399 doRankUpdate=True, 

1400 doLineSearch=False): 

1401 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1402 

1403 Parameters 

1404 ---------- 

1405 associations : `lsst.jointcal.Associations` 

1406 The star/reference star associations to fit. 

1407 fitter : `lsst.jointcal.FitterBase` 

1408 The fitter to use for minimization. 

1409 max_steps : `int` 

1410 Maximum number of steps to run outlier rejection before declaring 

1411 convergence failure. 

1412 name : {'photometry' or 'astrometry'} 

1413 What type of data are we fitting (for logs and debugging files). 

1414 whatToFit : `str` 

1415 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1416 dataName : `str`, optional 

1417 Descriptive name for this dataset (e.g. tract and filter), 

1418 for debugging. 

1419 sigmaRelativeTolerance : `float`, optional 

1420 Convergence tolerance for the fractional change in the chi2 cut 

1421 level for determining outliers. If set to zero, iterations will 

1422 continue until there are no outliers. 

1423 doRankUpdate : `bool`, optional 

1424 Do an Eigen rank update during minimization, or recompute the full 

1425 matrix and gradient? 

1426 doLineSearch : `bool`, optional 

1427 Do a line search for the optimum step during minimization? 

1428 

1429 Returns 

1430 ------- 

1431 chi2: `lsst.jointcal.Chi2Statistic` 

1432 The final chi2 after the fit converges, or is forced to end. 

1433 

1434 Raises 

1435 ------ 

1436 FloatingPointError 

1437 Raised if the fitter fails with a non-finite value. 

1438 RuntimeError 

1439 Raised if the fitter fails for some other reason; 

1440 log messages will provide further details. 

1441 """ 

1442 if self.config.writeInitMatrix: 

1443 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}") 

1444 else: 

1445 dumpMatrixFile = "" 

1446 oldChi2 = lsst.jointcal.Chi2Statistic() 

1447 oldChi2.chi2 = float("inf") 

1448 for i in range(max_steps): 

1449 if self.config.writeChi2FilesOuterLoop: 

1450 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1451 else: 

1452 writeChi2Name = None 

1453 result = fitter.minimize(whatToFit, 

1454 self.config.outlierRejectSigma, 

1455 sigmaRelativeTolerance=sigmaRelativeTolerance, 

1456 doRankUpdate=doRankUpdate, 

1457 doLineSearch=doLineSearch, 

1458 dumpMatrixFile=dumpMatrixFile) 

1459 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1460 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1461 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1462 

1463 if result == MinimizeResult.Converged: 

1464 if doRankUpdate: 

1465 self.log.debug("fit has converged - no more outliers - redo minimization " 

1466 "one more time in case we have lost accuracy in rank update.") 

1467 # Redo minimization one more time in case we have lost accuracy in rank update 

1468 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma, 

1469 sigmaRelativeTolerance=sigmaRelativeTolerance) 

1470 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1471 

1472 # log a message for a large final chi2, TODO: DM-15247 for something better 

1473 if chi2.chi2/chi2.ndof >= 4.0: 

1474 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1475 

1476 break 

1477 elif result == MinimizeResult.Chi2Increased: 

1478 self.log.warning("Still some outliers remaining but chi2 increased - retry") 

1479 # Check whether the increase was large enough to cause trouble. 

1480 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1481 if chi2Ratio > 1.5: 

1482 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1483 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1484 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1485 # appears that chi2 increases more than a factor of ~2 always 

1486 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1487 # Using 10 as the "failure" threshold gives some room between 

1488 # leaving a warning and bailing early. 

1489 if chi2Ratio > 10: 

1490 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1491 " Try setting one or more of the `writeChi2*` config fields and looking" 

1492 " at how individual star chi2-values evolve during the fit.") 

1493 raise RuntimeError(msg) 

1494 oldChi2 = chi2 

1495 elif result == MinimizeResult.NonFinite: 

1496 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1497 # TODO DM-12446: turn this into a "butler save" somehow. 

1498 fitter.saveChi2Contributions(filename+"{type}") 

1499 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1500 raise FloatingPointError(msg.format(filename)) 

1501 elif result == MinimizeResult.Failed: 

1502 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1503 else: 

1504 raise RuntimeError("Unxepected return code from minimize().") 

1505 else: 

1506 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1507 

1508 return chi2 

1509 

1510 def _make_output(self, ccdImageList, model, func): 

1511 """Return the internal jointcal models converted to the afw 

1512 structures that will be saved to disk. 

1513 

1514 Parameters 

1515 ---------- 

1516 ccdImageList : `lsst.jointcal.CcdImageList` 

1517 The list of CcdImages to get the output for. 

1518 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel` 

1519 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`. 

1520 func : `str` 

1521 The name of the function to call on ``model`` to get the converted 

1522 structure. Must accept an `lsst.jointcal.CcdImage`. 

1523 

1524 Returns 

1525 ------- 

1526 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or 

1527 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`] 

1528 The data to be saved, keyed on (visit, detector). 

1529 """ 

1530 output = {} 

1531 for ccdImage in ccdImageList: 

1532 ccd = ccdImage.ccdId 

1533 visit = ccdImage.visit 

1534 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd) 

1535 output[(visit, ccd)] = getattr(model, func)(ccdImage) 

1536 return output 

1537 

1538 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef): 

1539 """ 

1540 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 

1541 

1542 Parameters 

1543 ---------- 

1544 associations : `lsst.jointcal.Associations` 

1545 The star/reference star associations to fit. 

1546 model : `lsst.jointcal.AstrometryModel` 

1547 The astrometric model that was fit. 

1548 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1549 Dict of ccdImage identifiers to dataRefs that were fit. 

1550 """ 

1551 ccdImageList = associations.getCcdImageList() 

1552 output = self._make_output(ccdImageList, model, "makeSkyWcs") 

1553 for key, skyWcs in output.items(): 

1554 dataRef = visit_ccd_to_dataRef[key] 

1555 try: 

1556 dataRef.put(skyWcs, 'jointcal_wcs') 

1557 except pexExceptions.Exception as e: 

1558 self.log.fatal('Failed to write updated Wcs: %s', str(e)) 

1559 raise e 

1560 

1561 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef): 

1562 """ 

1563 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 

1564 

1565 Parameters 

1566 ---------- 

1567 associations : `lsst.jointcal.Associations` 

1568 The star/reference star associations to fit. 

1569 model : `lsst.jointcal.PhotometryModel` 

1570 The photoometric model that was fit. 

1571 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1572 Dict of ccdImage identifiers to dataRefs that were fit. 

1573 """ 

1574 

1575 ccdImageList = associations.getCcdImageList() 

1576 output = self._make_output(ccdImageList, model, "toPhotoCalib") 

1577 for key, photoCalib in output.items(): 

1578 dataRef = visit_ccd_to_dataRef[key] 

1579 try: 

1580 dataRef.put(photoCalib, 'jointcal_photoCalib') 

1581 except pexExceptions.Exception as e: 

1582 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e)) 

1583 raise e 

1584 

1585 

1586def make_schema_table(): 

1587 """Return an afw SourceTable to use as a base for creating the 

1588 SourceCatalog to insert values from the dataFrame into. 

1589 

1590 Returns 

1591 ------- 

1592 table : `lsst.afw.table.SourceTable` 

1593 Table with schema and slots to use to make SourceCatalogs. 

1594 """ 

1595 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

1596 schema.addField("centroid_x", "D") 

1597 schema.addField("centroid_y", "D") 

1598 schema.addField("centroid_xErr", "F") 

1599 schema.addField("centroid_yErr", "F") 

1600 schema.addField("shape_xx", "D") 

1601 schema.addField("shape_yy", "D") 

1602 schema.addField("shape_xy", "D") 

1603 schema.addField("flux_instFlux", "D") 

1604 schema.addField("flux_instFluxErr", "D") 

1605 table = lsst.afw.table.SourceTable.make(schema) 

1606 table.defineCentroid("centroid") 

1607 table.defineShape("shape") 

1608 return table 

1609 

1610 

1611def get_sourceTable_visit_columns(inColumns, config, sourceSelector): 

1612 """ 

1613 Get the sourceTable_visit columns to load from the catalogs. 

1614 

1615 Parameters 

1616 ---------- 

1617 inColumns : `list` 

1618 List of columns known to be available in the sourceTable_visit. 

1619 config : `JointcalConfig` 

1620 A filled-in config to to help define column names. 

1621 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask` 

1622 A configured source selector to define column names to load. 

1623 

1624 Returns 

1625 ------- 

1626 columns : `list` 

1627 List of columns to read from sourceTable_visit. 

1628 detectorColumn : `str` 

1629 Name of the detector column. 

1630 ixxColumns : `list` 

1631 Name of the ixx/iyy/ixy columns. 

1632 """ 

1633 if 'detector' in inColumns: 

1634 # Default name for Gen3. 

1635 detectorColumn = 'detector' 

1636 else: 

1637 # Default name for Gen2 conversions (still used in tests, CI, and older catalogs) 

1638 detectorColumn = 'ccd' 

1639 

1640 columns = ['visit', detectorColumn, 

1641 'sourceId', 'x', 'xErr', 'y', 'yErr', 

1642 config.sourceFluxType + '_instFlux', config.sourceFluxType + '_instFluxErr'] 

1643 

1644 if 'ixx' in inColumns: 

1645 # New columns post-DM-31825 

1646 ixxColumns = ['ixx', 'iyy', 'ixy'] 

1647 else: 

1648 # Old columns pre-DM-31825 

1649 ixxColumns = ['Ixx', 'Iyy', 'Ixy'] 

1650 columns.extend(ixxColumns) 

1651 

1652 if sourceSelector.config.doFlags: 

1653 columns.extend(sourceSelector.config.flags.bad) 

1654 if sourceSelector.config.doUnresolved: 

1655 columns.append(sourceSelector.config.unresolved.name) 

1656 if sourceSelector.config.doIsolated: 

1657 columns.append(sourceSelector.config.isolated.parentName) 

1658 columns.append(sourceSelector.config.isolated.nChildName) 

1659 

1660 return columns, detectorColumn, ixxColumns 

1661 

1662 

1663def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId, 

1664 detectorColumn, ixxColumns, sourceFluxType, log): 

1665 """Return an afw SourceCatalog extracted from a visit-level dataframe, 

1666 limited to just one detector. 

1667 

1668 Parameters 

1669 ---------- 

1670 table : `lsst.afw.table.SourceTable` 

1671 Table factory to use to make the SourceCatalog that will be 

1672 populated with data from ``visitCatalog``. 

1673 visitCatalog : `pandas.DataFrame` 

1674 DataFrame to extract a detector catalog from. 

1675 detectorId : `int` 

1676 Numeric id of the detector to extract from ``visitCatalog``. 

1677 detectorColumn : `str` 

1678 Name of the detector column in the catalog. 

1679 ixxColumns : `list` [`str`] 

1680 Names of the ixx/iyy/ixy columns in the catalog. 

1681 sourceFluxType : `str` 

1682 Name of the catalog field to load instFluxes from. 

1683 log : `lsst.log.Log` 

1684 Logging instance to log to. 

1685 

1686 Returns 

1687 ------- 

1688 catalog : `lsst.afw.table.SourceCatalog`, or `None` 

1689 Detector-level catalog extracted from ``visitCatalog``, or `None` 

1690 if there was no data to load. 

1691 """ 

1692 # map from dataFrame column to afw table column 

1693 mapping = {'x': 'centroid_x', 

1694 'y': 'centroid_y', 

1695 'xErr': 'centroid_xErr', 

1696 'yErr': 'centroid_yErr', 

1697 ixxColumns[0]: 'shape_xx', 

1698 ixxColumns[1]: 'shape_yy', 

1699 ixxColumns[2]: 'shape_xy', 

1700 f'{sourceFluxType}_instFlux': 'flux_instFlux', 

1701 f'{sourceFluxType}_instFluxErr': 'flux_instFluxErr', 

1702 } 

1703 

1704 catalog = lsst.afw.table.SourceCatalog(table) 

1705 matched = visitCatalog[detectorColumn] == detectorId 

1706 n = sum(matched) 

1707 if n == 0: 

1708 return None 

1709 catalog.resize(sum(matched)) 

1710 view = visitCatalog.loc[matched] 

1711 catalog['id'] = view.index 

1712 for dfCol, afwCol in mapping.items(): 

1713 catalog[afwCol] = view[dfCol] 

1714 

1715 log.debug("%d sources selected in visit %d detector %d", 

1716 len(catalog), 

1717 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first 

1718 detectorId) 

1719 return catalog