Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import collections 

23import os 

24 

25import astropy.time 

26import numpy as np 

27import astropy.units as u 

28 

29import lsst.geom 

30import lsst.utils 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33from lsst.afw.image import fluxErrFromABMagErr 

34import lsst.pex.exceptions as pexExceptions 

35import lsst.afw.table 

36import lsst.log 

37import lsst.meas.algorithms 

38from lsst.pipe.tasks.colorterms import ColortermLibrary 

39from lsst.verify import Job, Measurement 

40 

41from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask 

42from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

43 

44from .dataIds import PerTractCcdDataIdContainer 

45 

46import lsst.jointcal 

47from lsst.jointcal import MinimizeResult 

48 

49__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"] 

50 

51Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

52Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

53 

54 

55# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

56def add_measurement(job, name, value): 

57 meas = Measurement(job.metrics[name], value) 

58 job.measurements.insert(meas) 

59 

60 

61class JointcalRunner(pipeBase.ButlerInitializedTaskRunner): 

62 """Subclass of TaskRunner for jointcalTask 

63 

64 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 

65 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 

66 single dataRef, are are called repeatedly). This class transforms the processed 

67 arguments generated by the ArgumentParser into the arguments expected by 

68 Jointcal.runDataRef(). 

69 

70 See pipeBase.TaskRunner for more information. 

71 """ 

72 

73 @staticmethod 

74 def getTargetList(parsedCmd, **kwargs): 

75 """ 

76 Return a list of tuples per tract, each containing (dataRefs, kwargs). 

77 

78 Jointcal operates on lists of dataRefs simultaneously. 

79 """ 

80 kwargs['profile_jointcal'] = parsedCmd.profile_jointcal 

81 kwargs['butler'] = parsedCmd.butler 

82 

83 # organize data IDs by tract 

84 refListDict = {} 

85 for ref in parsedCmd.id.refList: 

86 refListDict.setdefault(ref.dataId["tract"], []).append(ref) 

87 # we call runDataRef() once with each tract 

88 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())] 

89 return result 

90 

91 def __call__(self, args): 

92 """ 

93 Parameters 

94 ---------- 

95 args 

96 Arguments for Task.runDataRef() 

97 

98 Returns 

99 ------- 

100 pipe.base.Struct 

101 if self.doReturnResults is False: 

102 

103 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

104 

105 if self.doReturnResults is True: 

106 

107 - ``result``: the result of calling jointcal.runDataRef() 

108 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

109 """ 

110 exitStatus = 0 # exit status for shell 

111 

112 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef. 

113 dataRefList, kwargs = args 

114 butler = kwargs.pop('butler') 

115 task = self.TaskClass(config=self.config, log=self.log, butler=butler) 

116 result = None 

117 try: 

118 result = task.runDataRef(dataRefList, **kwargs) 

119 exitStatus = result.exitStatus 

120 job_path = butler.get('verify_job_filename') 

121 result.job.write(job_path[0]) 

122 except Exception as e: # catch everything, sort it out later. 

123 if self.doRaise: 

124 raise e 

125 else: 

126 exitStatus = 1 

127 eName = type(e).__name__ 

128 tract = dataRefList[0].dataId['tract'] 

129 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e) 

130 

131 # Put the butler back into kwargs for the other Tasks. 

132 kwargs['butler'] = butler 

133 if self.doReturnResults: 

134 return pipeBase.Struct(result=result, exitStatus=exitStatus) 

135 else: 

136 return pipeBase.Struct(exitStatus=exitStatus) 

137 

138 

139class JointcalConfig(pexConfig.Config): 

140 """Configuration for JointcalTask""" 

141 

142 doAstrometry = pexConfig.Field( 

143 doc="Fit astrometry and write the fitted result.", 

144 dtype=bool, 

145 default=True 

146 ) 

147 doPhotometry = pexConfig.Field( 

148 doc="Fit photometry and write the fitted result.", 

149 dtype=bool, 

150 default=True 

151 ) 

152 coaddName = pexConfig.Field( 

153 doc="Type of coadd, typically deep or goodSeeing", 

154 dtype=str, 

155 default="deep" 

156 ) 

157 positionErrorPedestal = pexConfig.Field( 

158 doc="Systematic term to apply to the measured position error (pixels)", 

159 dtype=float, 

160 default=0.02, 

161 ) 

162 photometryErrorPedestal = pexConfig.Field( 

163 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

164 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

165 dtype=float, 

166 default=0.0, 

167 ) 

168 # TODO: DM-6885 matchCut should be an geom.Angle 

169 matchCut = pexConfig.Field( 

170 doc="Matching radius between fitted and reference stars (arcseconds)", 

171 dtype=float, 

172 default=3.0, 

173 ) 

174 minMeasurements = pexConfig.Field( 

175 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

176 dtype=int, 

177 default=2, 

178 ) 

179 minMeasuredStarsPerCcd = pexConfig.Field( 

180 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

181 dtype=int, 

182 default=100, 

183 ) 

184 minRefStarsPerCcd = pexConfig.Field( 

185 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

186 dtype=int, 

187 default=30, 

188 ) 

189 allowLineSearch = pexConfig.Field( 

190 doc="Allow a line search during minimization, if it is reasonable for the model" 

191 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

192 dtype=bool, 

193 default=False 

194 ) 

195 astrometrySimpleOrder = pexConfig.Field( 

196 doc="Polynomial order for fitting the simple astrometry model.", 

197 dtype=int, 

198 default=3, 

199 ) 

200 astrometryChipOrder = pexConfig.Field( 

201 doc="Order of the per-chip transform for the constrained astrometry model.", 

202 dtype=int, 

203 default=1, 

204 ) 

205 astrometryVisitOrder = pexConfig.Field( 

206 doc="Order of the per-visit transform for the constrained astrometry model.", 

207 dtype=int, 

208 default=5, 

209 ) 

210 useInputWcs = pexConfig.Field( 

211 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

212 dtype=bool, 

213 default=True, 

214 ) 

215 astrometryModel = pexConfig.ChoiceField( 

216 doc="Type of model to fit to astrometry", 

217 dtype=str, 

218 default="constrained", 

219 allowed={"simple": "One polynomial per ccd", 

220 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

221 ) 

222 photometryModel = pexConfig.ChoiceField( 

223 doc="Type of model to fit to photometry", 

224 dtype=str, 

225 default="constrainedMagnitude", 

226 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

227 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

228 " fitting in flux space.", 

229 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

230 " fitting in magnitude space.", 

231 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

232 " fitting in magnitude space.", 

233 } 

234 ) 

235 applyColorTerms = pexConfig.Field( 

236 doc="Apply photometric color terms to reference stars?" 

237 "Requires that colorterms be set to a ColortermLibrary", 

238 dtype=bool, 

239 default=False 

240 ) 

241 colorterms = pexConfig.ConfigField( 

242 doc="Library of photometric reference catalog name to color term dict.", 

243 dtype=ColortermLibrary, 

244 ) 

245 photometryVisitOrder = pexConfig.Field( 

246 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

247 dtype=int, 

248 default=7, 

249 ) 

250 photometryDoRankUpdate = pexConfig.Field( 

251 doc=("Do the rank update step during minimization. " 

252 "Skipping this can help deal with models that are too non-linear."), 

253 dtype=bool, 

254 default=True, 

255 ) 

256 astrometryDoRankUpdate = pexConfig.Field( 

257 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

258 "Skipping this can help deal with models that are too non-linear."), 

259 dtype=bool, 

260 default=True, 

261 ) 

262 outlierRejectSigma = pexConfig.Field( 

263 doc="How many sigma to reject outliers at during minimization.", 

264 dtype=float, 

265 default=5.0, 

266 ) 

267 maxPhotometrySteps = pexConfig.Field( 

268 doc="Maximum number of minimize iterations to take when fitting photometry.", 

269 dtype=int, 

270 default=20, 

271 ) 

272 maxAstrometrySteps = pexConfig.Field( 

273 doc="Maximum number of minimize iterations to take when fitting photometry.", 

274 dtype=int, 

275 default=20, 

276 ) 

277 astrometryRefObjLoader = pexConfig.ConfigurableField( 

278 target=LoadIndexedReferenceObjectsTask, 

279 doc="Reference object loader for astrometric fit", 

280 ) 

281 photometryRefObjLoader = pexConfig.ConfigurableField( 

282 target=LoadIndexedReferenceObjectsTask, 

283 doc="Reference object loader for photometric fit", 

284 ) 

285 sourceSelector = sourceSelectorRegistry.makeField( 

286 doc="How to select sources for cross-matching", 

287 default="astrometry" 

288 ) 

289 astrometryReferenceSelector = pexConfig.ConfigurableField( 

290 target=ReferenceSourceSelectorTask, 

291 doc="How to down-select the loaded astrometry reference catalog.", 

292 ) 

293 photometryReferenceSelector = pexConfig.ConfigurableField( 

294 target=ReferenceSourceSelectorTask, 

295 doc="How to down-select the loaded photometry reference catalog.", 

296 ) 

297 astrometryReferenceErr = pexConfig.Field( 

298 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

299 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

300 "If specified, overrides any existing `coord_*Err` values."), 

301 dtype=float, 

302 default=None, 

303 optional=True 

304 ) 

305 writeInitMatrix = pexConfig.Field( 

306 dtype=bool, 

307 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

308 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory. " 

309 "Note that these files are the dense versions of the matrix, and so may be very large."), 

310 default=False 

311 ) 

312 writeChi2FilesInitialFinal = pexConfig.Field( 

313 dtype=bool, 

314 doc="Write .csv files containing the contributions to chi2 for the initialization and final fit.", 

315 default=False 

316 ) 

317 writeChi2FilesOuterLoop = pexConfig.Field( 

318 dtype=bool, 

319 doc="Write .csv files containing the contributions to chi2 for the outer fit loop.", 

320 default=False 

321 ) 

322 writeInitialModel = pexConfig.Field( 

323 dtype=bool, 

324 doc=("Write the pre-initialization model to text files, for debugging." 

325 " Output is written to `initial[Astro|Photo]metryModel.txt` in the current working directory."), 

326 default=False 

327 ) 

328 debugOutputPath = pexConfig.Field( 

329 dtype=str, 

330 default=".", 

331 doc=("Path to write debug output files to. Used by " 

332 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

333 ) 

334 sourceFluxType = pexConfig.Field( 

335 dtype=str, 

336 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

337 default='Calib' 

338 ) 

339 

340 def validate(self): 

341 super().validate() 

342 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

343 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

344 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

345 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

346 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

347 "applyColorTerms=True will be ignored.") 

348 lsst.log.warn(msg) 

349 

350 def setDefaults(self): 

351 # Use science source selector which can filter on extendedness, SNR, and whether blended 

352 self.sourceSelector.name = 'science' 

353 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing 

354 self.sourceSelector['science'].doUnresolved = True 

355 # with dependable signal to noise ratio. 

356 self.sourceSelector['science'].doSignalToNoise = True 

357 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

358 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

359 self.sourceSelector['science'].signalToNoise.minimum = 10. 

360 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive 

361 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux" 

362 self.sourceSelector['science'].signalToNoise.fluxField = fluxField 

363 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err" 

364 # Do not trust blended sources' aperture fluxes which also depend on seeing 

365 self.sourceSelector['science'].doIsolated = True 

366 # Do not trust either flux or centroid measurements with flags, 

367 # chosen from the usual QA flags for stars) 

368 self.sourceSelector['science'].doFlags = True 

369 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 

370 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag', 

371 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter'] 

372 self.sourceSelector['science'].flags.bad = badFlags 

373 

374 # Default to Gaia-DR2 (with proper motions) for astrometry and 

375 # PS1-DR1 for photometry, with a reasonable initial filterMap. 

376 self.astrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414" 

377 self.astrometryRefObjLoader.requireProperMotion = True 

378 self.astrometryRefObjLoader.filterMap = {'u': 'phot_g_mean', 

379 'g': 'phot_g_mean', 

380 'r': 'phot_g_mean', 

381 'i': 'phot_g_mean', 

382 'z': 'phot_g_mean', 

383 'y': 'phot_g_mean'} 

384 self.photometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110" 

385 

386 

387def writeModel(model, filename, log): 

388 """Write model to outfile.""" 

389 with open(filename, "w") as file: 

390 file.write(repr(model)) 

391 log.info("Wrote %s to file: %s", model, filename) 

392 

393 

394class JointcalTask(pipeBase.CmdLineTask): 

395 """Jointly astrometrically and photometrically calibrate a group of images.""" 

396 

397 ConfigClass = JointcalConfig 

398 RunnerClass = JointcalRunner 

399 _DefaultName = "jointcal" 

400 

401 def __init__(self, butler=None, profile_jointcal=False, **kwargs): 

402 """ 

403 Instantiate a JointcalTask. 

404 

405 Parameters 

406 ---------- 

407 butler : `lsst.daf.persistence.Butler` 

408 The butler is passed to the refObjLoader constructor in case it is 

409 needed. Ignored if the refObjLoader argument provides a loader directly. 

410 Used to initialize the astrometry and photometry refObjLoaders. 

411 profile_jointcal : `bool` 

412 Set to True to profile different stages of this jointcal run. 

413 """ 

414 pipeBase.CmdLineTask.__init__(self, **kwargs) 

415 self.profile_jointcal = profile_jointcal 

416 self.makeSubtask("sourceSelector") 

417 if self.config.doAstrometry: 

418 self.makeSubtask('astrometryRefObjLoader', butler=butler) 

419 self.makeSubtask("astrometryReferenceSelector") 

420 else: 

421 self.astrometryRefObjLoader = None 

422 if self.config.doPhotometry: 

423 self.makeSubtask('photometryRefObjLoader', butler=butler) 

424 self.makeSubtask("photometryReferenceSelector") 

425 else: 

426 self.photometryRefObjLoader = None 

427 

428 # To hold various computed metrics for use by tests 

429 self.job = Job.load_metrics_package(subset='jointcal') 

430 

431 # We don't currently need to persist the metadata. 

432 # If we do in the future, we will have to add appropriate dataset templates 

433 # to each obs package (the metadata template should look like `jointcal_wcs`). 

434 def _getMetadataName(self): 

435 return None 

436 

437 @classmethod 

438 def _makeArgumentParser(cls): 

439 """Create an argument parser""" 

440 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

441 parser.add_argument("--profile_jointcal", default=False, action="store_true", 

442 help="Profile steps of jointcal separately.") 

443 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9", 

444 ContainerClass=PerTractCcdDataIdContainer) 

445 return parser 

446 

447 def _build_ccdImage(self, dataRef, associations, jointcalControl): 

448 """ 

449 Extract the necessary things from this dataRef to add a new ccdImage. 

450 

451 Parameters 

452 ---------- 

453 dataRef : `lsst.daf.persistence.ButlerDataRef` 

454 DataRef to extract info from. 

455 associations : `lsst.jointcal.Associations` 

456 Object to add the info to, to construct a new CcdImage 

457 jointcalControl : `jointcal.JointcalControl` 

458 Control object for associations management 

459 

460 Returns 

461 ------ 

462 namedtuple 

463 ``wcs`` 

464 The TAN WCS of this image, read from the calexp 

465 (`lsst.afw.geom.SkyWcs`). 

466 ``key`` 

467 A key to identify this dataRef by its visit and ccd ids 

468 (`namedtuple`). 

469 ``filter`` 

470 This calexp's filter (`str`). 

471 """ 

472 if "visit" in dataRef.dataId.keys(): 

473 visit = dataRef.dataId["visit"] 

474 else: 

475 visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0] 

476 

477 src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True) 

478 

479 visitInfo = dataRef.get('calexp_visitInfo') 

480 detector = dataRef.get('calexp_detector') 

481 ccdId = detector.getId() 

482 photoCalib = dataRef.get('calexp_photoCalib') 

483 tanWcs = dataRef.get('calexp_wcs') 

484 bbox = dataRef.get('calexp_bbox') 

485 filt = dataRef.get('calexp_filter') 

486 filterName = filt.getName() 

487 

488 goodSrc = self.sourceSelector.run(src) 

489 

490 if len(goodSrc.sourceCat) == 0: 

491 self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId) 

492 else: 

493 self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId) 

494 associations.createCcdImage(goodSrc.sourceCat, 

495 tanWcs, 

496 visitInfo, 

497 bbox, 

498 filterName, 

499 photoCalib, 

500 detector, 

501 visit, 

502 ccdId, 

503 jointcalControl) 

504 

505 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter')) 

506 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

507 return Result(tanWcs, Key(visit, ccdId), filterName) 

508 

509 def _getDebugPath(self, filename): 

510 """Constructs a path to filename using the configured debug path. 

511 """ 

512 return os.path.join(self.config.debugOutputPath, filename) 

513 

514 @pipeBase.timeMethod 

515 def runDataRef(self, dataRefs, profile_jointcal=False): 

516 """ 

517 Jointly calibrate the astrometry and photometry across a set of images. 

518 

519 Parameters 

520 ---------- 

521 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

522 List of data references to the exposures to be fit. 

523 profile_jointcal : `bool` 

524 Profile the individual steps of jointcal. 

525 

526 Returns 

527 ------- 

528 result : `lsst.pipe.base.Struct` 

529 Struct of metadata from the fit, containing: 

530 

531 ``dataRefs`` 

532 The provided data references that were fit (with updated WCSs) 

533 ``oldWcsList`` 

534 The original WCS from each dataRef 

535 ``metrics`` 

536 Dictionary of internally-computed metrics for testing/validation. 

537 """ 

538 if len(dataRefs) == 0: 

539 raise ValueError('Need a non-empty list of data references!') 

540 

541 exitStatus = 0 # exit status for shell 

542 

543 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,) 

544 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

545 associations = lsst.jointcal.Associations() 

546 

547 visit_ccd_to_dataRef = {} 

548 oldWcsList = [] 

549 filters = [] 

550 load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else '' 

551 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

552 # We need the bounding-box of the focal plane for photometry visit models. 

553 # NOTE: we only need to read it once, because its the same for all exposures of a camera. 

554 camera = dataRefs[0].get('camera', immediate=True) 

555 self.focalPlaneBBox = camera.getFpBBox() 

556 for ref in dataRefs: 

557 result = self._build_ccdImage(ref, associations, jointcalControl) 

558 oldWcsList.append(result.wcs) 

559 visit_ccd_to_dataRef[result.key] = ref 

560 filters.append(result.filter) 

561 filters = collections.Counter(filters) 

562 

563 associations.computeCommonTangentPoint() 

564 

565 boundingCircle = associations.computeBoundingCircle() 

566 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

567 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

568 

569 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

570 

571 # Determine a default filter associated with the catalog. See DM-9093 

572 defaultFilter = filters.most_common(1)[0][0] 

573 self.log.debug("Using %s band for reference flux", defaultFilter) 

574 

575 # TODO: need a better way to get the tract. 

576 tract = dataRefs[0].dataId['tract'] 

577 

578 if self.config.doAstrometry: 

579 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

580 name="astrometry", 

581 refObjLoader=self.astrometryRefObjLoader, 

582 referenceSelector=self.astrometryReferenceSelector, 

583 fit_function=self._fit_astrometry, 

584 profile_jointcal=profile_jointcal, 

585 tract=tract) 

586 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef) 

587 else: 

588 astrometry = Astrometry(None, None, None) 

589 

590 if self.config.doPhotometry: 

591 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

592 name="photometry", 

593 refObjLoader=self.photometryRefObjLoader, 

594 referenceSelector=self.photometryReferenceSelector, 

595 fit_function=self._fit_photometry, 

596 profile_jointcal=profile_jointcal, 

597 tract=tract, 

598 filters=filters, 

599 reject_bad_fluxes=True) 

600 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef) 

601 else: 

602 photometry = Photometry(None, None) 

603 

604 return pipeBase.Struct(dataRefs=dataRefs, 

605 oldWcsList=oldWcsList, 

606 job=self.job, 

607 astrometryRefObjLoader=self.astrometryRefObjLoader, 

608 photometryRefObjLoader=self.photometryRefObjLoader, 

609 defaultFilter=defaultFilter, 

610 exitStatus=exitStatus) 

611 

612 def _get_refcat_coordinate_error_override(self, refCat, name): 

613 """Check whether we should override the refcat coordinate errors, and 

614 return the overridden error if necessary. 

615 

616 Parameters 

617 ---------- 

618 refCat : `lsst.afw.table.SimpleCatalog` 

619 The reference catalog to check for a ``coord_raErr`` field. 

620 name : `str` 

621 Whether we are doing "astrometry" or "photometry". 

622 

623 Returns 

624 ------- 

625 refCoordErr : `float` 

626 The refcat coordinate error to use, or NaN if we are not overriding 

627 those fields. 

628 

629 Raises 

630 ------ 

631 lsst.pex.config.FieldValidationError 

632 Raised if the refcat does not contain coordinate errors and 

633 ``config.astrometryReferenceErr`` is not set. 

634 """ 

635 # This value doesn't matter for photometry, so just set something to 

636 # keep old refcats from causing problems. 

637 if name.lower() == "photometry": 

638 if 'coord_raErr' not in refCat.schema: 

639 return 100 

640 else: 

641 return float('nan') 

642 

643 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

644 msg = ("Reference catalog does not contain coordinate errors, " 

645 "and config.astrometryReferenceErr not supplied.") 

646 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

647 self.config, 

648 msg) 

649 

650 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

651 self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

652 self.config.astrometryReferenceErr) 

653 

654 if self.config.astrometryReferenceErr is None: 

655 return float('nan') 

656 else: 

657 return self.config.astrometryReferenceErr 

658 

659 def _compute_proper_motion_epoch(self, ccdImageList): 

660 """Return the proper motion correction epoch of the provided images. 

661 

662 Parameters 

663 ---------- 

664 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

665 The images to compute the appropriate epoch for. 

666 

667 Returns 

668 ------- 

669 epoch : `astropy.time.Time` 

670 The date to use for proper motion corrections. 

671 """ 

672 mjds = [ccdImage.getMjd() for ccdImage in ccdImageList] 

673 return astropy.time.Time(np.mean(mjds), format='mjd', scale="tai") 

674 

675 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

676 filters=[], 

677 tract="", profile_jointcal=False, match_cut=3.0, 

678 reject_bad_fluxes=False, *, 

679 name="", refObjLoader=None, referenceSelector=None, 

680 fit_function=None): 

681 """Load reference catalog, perform the fit, and return the result. 

682 

683 Parameters 

684 ---------- 

685 associations : `lsst.jointcal.Associations` 

686 The star/reference star associations to fit. 

687 defaultFilter : `str` 

688 filter to load from reference catalog. 

689 center : `lsst.geom.SpherePoint` 

690 ICRS center of field to load from reference catalog. 

691 radius : `lsst.geom.Angle` 

692 On-sky radius to load from reference catalog. 

693 name : `str` 

694 Name of thing being fit: "astrometry" or "photometry". 

695 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

696 Reference object loader to use to load a reference catalog. 

697 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

698 Selector to use to pick objects from the loaded reference catalog. 

699 fit_function : callable 

700 Function to call to perform fit (takes Associations object). 

701 filters : `list` [`str`], optional 

702 List of filters to load from the reference catalog. 

703 tract : `str`, optional 

704 Name of tract currently being fit. 

705 profile_jointcal : `bool`, optional 

706 Separately profile the fitting step. 

707 match_cut : `float`, optional 

708 Radius in arcseconds to find cross-catalog matches to during 

709 associations.associateCatalogs. 

710 reject_bad_fluxes : `bool`, optional 

711 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

712 

713 Returns 

714 ------- 

715 result : `Photometry` or `Astrometry` 

716 Result of `fit_function()` 

717 """ 

718 self.log.info("====== Now processing %s...", name) 

719 # TODO: this should not print "trying to invert a singular transformation:" 

720 # if it does that, something's not right about the WCS... 

721 associations.associateCatalogs(match_cut) 

722 add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name, 

723 associations.fittedStarListSize()) 

724 

725 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

726 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

727 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

728 center, radius, defaultFilter, 

729 applyColorterms=applyColorterms, 

730 epoch=epoch) 

731 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

732 

733 associations.collectRefStars(refCat, 

734 self.config.matchCut*lsst.geom.arcseconds, 

735 fluxField, 

736 refCoordinateErr=refCoordErr, 

737 rejectBadFluxes=reject_bad_fluxes) 

738 add_measurement(self.job, 'jointcal.collected_%s_refStars' % name, 

739 associations.refStarListSize()) 

740 

741 associations.prepareFittedStars(self.config.minMeasurements) 

742 

743 self._check_star_lists(associations, name) 

744 add_measurement(self.job, 'jointcal.selected_%s_refStars' % name, 

745 associations.nFittedStarsWithAssociatedRefStar()) 

746 add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name, 

747 associations.fittedStarListSize()) 

748 add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name, 

749 associations.nCcdImagesValidForFit()) 

750 

751 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else '' 

752 dataName = "{}_{}".format(tract, defaultFilter) 

753 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

754 result = fit_function(associations, dataName) 

755 # TODO DM-12446: turn this into a "butler save" somehow. 

756 # Save reference and measurement chi2 contributions for this data 

757 if self.config.writeChi2FilesInitialFinal: 

758 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

759 result.fit.saveChi2Contributions(baseName+"{type}") 

760 self.log.info("Wrote chi2 contributions files: %s", baseName) 

761 

762 return result 

763 

764 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, 

765 applyColorterms=False, epoch=None): 

766 """Load the necessary reference catalog sources, convert fluxes to 

767 correct units, and apply color term corrections if requested. 

768 

769 Parameters 

770 ---------- 

771 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

772 The reference catalog loader to use to get the data. 

773 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

774 Source selector to apply to loaded reference catalog. 

775 center : `lsst.geom.SpherePoint` 

776 The center around which to load sources. 

777 radius : `lsst.geom.Angle` 

778 The radius around ``center`` to load sources in. 

779 filterName : `str` 

780 The name of the camera filter to load fluxes for. 

781 applyColorterms : `bool` 

782 Apply colorterm corrections to the refcat for ``filterName``? 

783 epoch : `astropy.time.Time`, optional 

784 Epoch to which to correct refcat proper motion and parallax, 

785 or `None` to not apply such corrections. 

786 

787 Returns 

788 ------- 

789 refCat : `lsst.afw.table.SimpleCatalog` 

790 The loaded reference catalog. 

791 fluxField : `str` 

792 The name of the reference catalog flux field appropriate for ``filterName``. 

793 """ 

794 skyCircle = refObjLoader.loadSkyCircle(center, 

795 radius, 

796 filterName, 

797 epoch=epoch) 

798 

799 selected = referenceSelector.run(skyCircle.refCat) 

800 # Need memory contiguity to get reference filters as a vector. 

801 if not selected.sourceCat.isContiguous(): 

802 refCat = selected.sourceCat.copy(deep=True) 

803 else: 

804 refCat = selected.sourceCat 

805 

806 if applyColorterms: 

807 refCatName = refObjLoader.ref_dataset_name 

808 self.log.info("Applying color terms for filterName=%r reference catalog=%s", 

809 filterName, refCatName) 

810 colorterm = self.config.colorterms.getColorterm( 

811 filterName=filterName, photoCatName=refCatName, doRaise=True) 

812 

813 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName) 

814 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

815 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

816 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

817 

818 return refCat, skyCircle.fluxField 

819 

820 def _check_star_lists(self, associations, name): 

821 # TODO: these should be len(blah), but we need this properly wrapped first. 

822 if associations.nCcdImagesValidForFit() == 0: 

823 raise RuntimeError('No images in the ccdImageList!') 

824 if associations.fittedStarListSize() == 0: 

825 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

826 if associations.refStarListSize() == 0: 

827 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

828 

829 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

830 """Compute chi2, log it, validate the model, and return chi2. 

831 

832 Parameters 

833 ---------- 

834 associations : `lsst.jointcal.Associations` 

835 The star/reference star associations to fit. 

836 fit : `lsst.jointcal.FitterBase` 

837 The fitter to use for minimization. 

838 model : `lsst.jointcal.Model` 

839 The model being fit. 

840 chi2Label : `str` 

841 Label to describe the chi2 (e.g. "Initialized", "Final"). 

842 writeChi2Name : `str`, optional 

843 Filename prefix to write the chi2 contributions to. 

844 Do not supply an extension: an appropriate one will be added. 

845 

846 Returns 

847 ------- 

848 chi2: `lsst.jointcal.Chi2Accumulator` 

849 The chi2 object for the current fitter and model. 

850 

851 Raises 

852 ------ 

853 FloatingPointError 

854 Raised if chi2 is infinite or NaN. 

855 ValueError 

856 Raised if the model is not valid. 

857 """ 

858 if writeChi2Name is not None: 

859 fullpath = self._getDebugPath(writeChi2Name) 

860 fit.saveChi2Contributions(fullpath+"{type}") 

861 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

862 

863 chi2 = fit.computeChi2() 

864 self.log.info("%s %s", chi2Label, chi2) 

865 self._check_stars(associations) 

866 if not np.isfinite(chi2.chi2): 

867 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

868 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

869 raise ValueError("Model is not valid: check log messages for warnings.") 

870 return chi2 

871 

872 def _fit_photometry(self, associations, dataName=None): 

873 """ 

874 Fit the photometric data. 

875 

876 Parameters 

877 ---------- 

878 associations : `lsst.jointcal.Associations` 

879 The star/reference star associations to fit. 

880 dataName : `str` 

881 Name of the data being processed (e.g. "1234_HSC-Y"), for 

882 identifying debugging files. 

883 

884 Returns 

885 ------- 

886 fit_result : `namedtuple` 

887 fit : `lsst.jointcal.PhotometryFit` 

888 The photometric fitter used to perform the fit. 

889 model : `lsst.jointcal.PhotometryModel` 

890 The photometric model that was fit. 

891 """ 

892 self.log.info("=== Starting photometric fitting...") 

893 

894 # TODO: should use pex.config.RegistryField here (see DM-9195) 

895 if self.config.photometryModel == "constrainedFlux": 

896 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

897 self.focalPlaneBBox, 

898 visitOrder=self.config.photometryVisitOrder, 

899 errorPedestal=self.config.photometryErrorPedestal) 

900 # potentially nonlinear problem, so we may need a line search to converge. 

901 doLineSearch = self.config.allowLineSearch 

902 elif self.config.photometryModel == "constrainedMagnitude": 

903 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

904 self.focalPlaneBBox, 

905 visitOrder=self.config.photometryVisitOrder, 

906 errorPedestal=self.config.photometryErrorPedestal) 

907 # potentially nonlinear problem, so we may need a line search to converge. 

908 doLineSearch = self.config.allowLineSearch 

909 elif self.config.photometryModel == "simpleFlux": 

910 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

911 errorPedestal=self.config.photometryErrorPedestal) 

912 doLineSearch = False # purely linear in model parameters, so no line search needed 

913 elif self.config.photometryModel == "simpleMagnitude": 

914 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

915 errorPedestal=self.config.photometryErrorPedestal) 

916 doLineSearch = False # purely linear in model parameters, so no line search needed 

917 

918 fit = lsst.jointcal.PhotometryFit(associations, model) 

919 # TODO DM-12446: turn this into a "butler save" somehow. 

920 # Save reference and measurement chi2 contributions for this data 

921 if self.config.writeChi2FilesInitialFinal: 

922 baseName = f"photometry_initial_chi2-{dataName}" 

923 else: 

924 baseName = None 

925 if self.config.writeInitialModel: 

926 fullpath = self._getDebugPath("initialPhotometryModel.txt") 

927 writeModel(model, fullpath, self.log) 

928 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

929 

930 def getChi2Name(whatToFit): 

931 if self.config.writeChi2FilesOuterLoop: 

932 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

933 else: 

934 return None 

935 

936 # The constrained model needs the visit transform fit first; the chip 

937 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

938 dumpMatrixFile = self._getDebugPath("photometry_preinit") if self.config.writeInitMatrix else "" 

939 if self.config.photometryModel.startswith("constrained"): 

940 # no line search: should be purely (or nearly) linear, 

941 # and we want a large step size to initialize with. 

942 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

943 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

944 writeChi2Name=getChi2Name("ModelVisit")) 

945 dumpMatrixFile = "" # so we don't redo the output on the next step 

946 

947 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

948 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

949 writeChi2Name=getChi2Name("Model")) 

950 

951 fit.minimize("Fluxes") # no line search: always purely linear. 

952 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

953 writeChi2Name=getChi2Name("Fluxes")) 

954 

955 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

956 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

957 writeChi2Name=getChi2Name("ModelFluxes")) 

958 

959 model.freezeErrorTransform() 

960 self.log.debug("Photometry error scales are frozen.") 

961 

962 chi2 = self._iterate_fit(associations, 

963 fit, 

964 self.config.maxPhotometrySteps, 

965 "photometry", 

966 "Model Fluxes", 

967 doRankUpdate=self.config.photometryDoRankUpdate, 

968 doLineSearch=doLineSearch, 

969 dataName=dataName) 

970 

971 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

972 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

973 return Photometry(fit, model) 

974 

975 def _fit_astrometry(self, associations, dataName=None): 

976 """ 

977 Fit the astrometric data. 

978 

979 Parameters 

980 ---------- 

981 associations : `lsst.jointcal.Associations` 

982 The star/reference star associations to fit. 

983 dataName : `str` 

984 Name of the data being processed (e.g. "1234_HSC-Y"), for 

985 identifying debugging files. 

986 

987 Returns 

988 ------- 

989 fit_result : `namedtuple` 

990 fit : `lsst.jointcal.AstrometryFit` 

991 The astrometric fitter used to perform the fit. 

992 model : `lsst.jointcal.AstrometryModel` 

993 The astrometric model that was fit. 

994 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

995 The model for the sky to tangent plane projection that was used in the fit. 

996 """ 

997 

998 self.log.info("=== Starting astrometric fitting...") 

999 

1000 associations.deprojectFittedStars() 

1001 

1002 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1003 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1004 # them so carefully? 

1005 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1006 

1007 if self.config.astrometryModel == "constrained": 

1008 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1009 sky_to_tan_projection, 

1010 chipOrder=self.config.astrometryChipOrder, 

1011 visitOrder=self.config.astrometryVisitOrder) 

1012 elif self.config.astrometryModel == "simple": 

1013 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1014 sky_to_tan_projection, 

1015 self.config.useInputWcs, 

1016 nNotFit=0, 

1017 order=self.config.astrometrySimpleOrder) 

1018 

1019 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1020 # TODO DM-12446: turn this into a "butler save" somehow. 

1021 # Save reference and measurement chi2 contributions for this data 

1022 if self.config.writeChi2FilesInitialFinal: 

1023 baseName = f"astrometry_initial_chi2-{dataName}" 

1024 else: 

1025 baseName = None 

1026 if self.config.writeInitialModel: 

1027 fullpath = self._getDebugPath("initialAstrometryModel.txt") 

1028 writeModel(model, fullpath, self.log) 

1029 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1030 

1031 def getChi2Name(whatToFit): 

1032 if self.config.writeChi2FilesOuterLoop: 

1033 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1034 else: 

1035 return None 

1036 

1037 dumpMatrixFile = self._getDebugPath("astrometry_preinit") if self.config.writeInitMatrix else "" 

1038 # The constrained model needs the visit transform fit first; the chip 

1039 # transform is initialized from the detector's cameraGeom, so it's close. 

1040 if self.config.astrometryModel == "constrained": 

1041 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1042 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1043 writeChi2Name=getChi2Name("DistortionsVisit")) 

1044 dumpMatrixFile = "" # so we don't redo the output on the next step 

1045 

1046 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1047 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1048 writeChi2Name=getChi2Name("Distortions")) 

1049 

1050 fit.minimize("Positions") 

1051 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1052 writeChi2Name=getChi2Name("Positions")) 

1053 

1054 fit.minimize("Distortions Positions") 

1055 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1056 writeChi2Name=getChi2Name("DistortionsPositions")) 

1057 

1058 chi2 = self._iterate_fit(associations, 

1059 fit, 

1060 self.config.maxAstrometrySteps, 

1061 "astrometry", 

1062 "Distortions Positions", 

1063 doRankUpdate=self.config.astrometryDoRankUpdate, 

1064 dataName=dataName) 

1065 

1066 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1067 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1068 

1069 return Astrometry(fit, model, sky_to_tan_projection) 

1070 

1071 def _check_stars(self, associations): 

1072 """Count measured and reference stars per ccd and warn/log them.""" 

1073 for ccdImage in associations.getCcdImageList(): 

1074 nMeasuredStars, nRefStars = ccdImage.countStars() 

1075 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1076 ccdImage.getName(), nMeasuredStars, nRefStars) 

1077 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1078 self.log.warn("ccdImage %s has only %s measuredStars (desired %s)", 

1079 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1080 if nRefStars < self.config.minRefStarsPerCcd: 

1081 self.log.warn("ccdImage %s has only %s RefStars (desired %s)", 

1082 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1083 

1084 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1085 dataName="", 

1086 doRankUpdate=True, 

1087 doLineSearch=False): 

1088 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1089 

1090 Parameters 

1091 ---------- 

1092 associations : `lsst.jointcal.Associations` 

1093 The star/reference star associations to fit. 

1094 fitter : `lsst.jointcal.FitterBase` 

1095 The fitter to use for minimization. 

1096 max_steps : `int` 

1097 Maximum number of steps to run outlier rejection before declaring 

1098 convergence failure. 

1099 name : {'photometry' or 'astrometry'} 

1100 What type of data are we fitting (for logs and debugging files). 

1101 whatToFit : `str` 

1102 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1103 dataName : `str`, optional 

1104 Descriptive name for this dataset (e.g. tract and filter), 

1105 for debugging. 

1106 doRankUpdate : `bool`, optional 

1107 Do an Eigen rank update during minimization, or recompute the full 

1108 matrix and gradient? 

1109 doLineSearch : `bool`, optional 

1110 Do a line search for the optimum step during minimization? 

1111 

1112 Returns 

1113 ------- 

1114 chi2: `lsst.jointcal.Chi2Statistic` 

1115 The final chi2 after the fit converges, or is forced to end. 

1116 

1117 Raises 

1118 ------ 

1119 FloatingPointError 

1120 Raised if the fitter fails with a non-finite value. 

1121 RuntimeError 

1122 Raised if the fitter fails for some other reason; 

1123 log messages will provide further details. 

1124 """ 

1125 dumpMatrixFile = self._getDebugPath(f"{name}_postinit") if self.config.writeInitMatrix else "" 

1126 oldChi2 = lsst.jointcal.Chi2Statistic() 

1127 oldChi2.chi2 = float("inf") 

1128 for i in range(max_steps): 

1129 if self.config.writeChi2FilesOuterLoop: 

1130 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1131 else: 

1132 writeChi2Name = None 

1133 result = fitter.minimize(whatToFit, 

1134 self.config.outlierRejectSigma, 

1135 doRankUpdate=doRankUpdate, 

1136 doLineSearch=doLineSearch, 

1137 dumpMatrixFile=dumpMatrixFile) 

1138 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1139 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1140 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1141 

1142 if result == MinimizeResult.Converged: 

1143 if doRankUpdate: 

1144 self.log.debug("fit has converged - no more outliers - redo minimization " 

1145 "one more time in case we have lost accuracy in rank update.") 

1146 # Redo minimization one more time in case we have lost accuracy in rank update 

1147 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma) 

1148 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1149 

1150 # log a message for a large final chi2, TODO: DM-15247 for something better 

1151 if chi2.chi2/chi2.ndof >= 4.0: 

1152 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1153 

1154 break 

1155 elif result == MinimizeResult.Chi2Increased: 

1156 self.log.warn("Still some outliers remaining but chi2 increased - retry") 

1157 # Check whether the increase was large enough to cause trouble. 

1158 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1159 if chi2Ratio > 1.5: 

1160 self.log.warn('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1161 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1162 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1163 # appears that chi2 increases more than a factor of ~2 always 

1164 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1165 # Using 10 as the "failure" threshold gives some room between 

1166 # leaving a warning and bailing early. 

1167 if chi2Ratio > 10: 

1168 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1169 " Try setting one or more of the `writeChi2*` config fields and looking" 

1170 " at how individual star chi2-values evolve during the fit.") 

1171 raise RuntimeError(msg) 

1172 oldChi2 = chi2 

1173 elif result == MinimizeResult.NonFinite: 

1174 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1175 # TODO DM-12446: turn this into a "butler save" somehow. 

1176 fitter.saveChi2Contributions(filename+"{type}") 

1177 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1178 raise FloatingPointError(msg.format(filename)) 

1179 elif result == MinimizeResult.Failed: 

1180 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1181 else: 

1182 raise RuntimeError("Unxepected return code from minimize().") 

1183 else: 

1184 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1185 

1186 return chi2 

1187 

1188 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef): 

1189 """ 

1190 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 

1191 

1192 Parameters 

1193 ---------- 

1194 associations : `lsst.jointcal.Associations` 

1195 The star/reference star associations to fit. 

1196 model : `lsst.jointcal.AstrometryModel` 

1197 The astrometric model that was fit. 

1198 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1199 Dict of ccdImage identifiers to dataRefs that were fit. 

1200 """ 

1201 

1202 ccdImageList = associations.getCcdImageList() 

1203 for ccdImage in ccdImageList: 

1204 # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair? 

1205 ccd = ccdImage.ccdId 

1206 visit = ccdImage.visit 

1207 dataRef = visit_ccd_to_dataRef[(visit, ccd)] 

1208 self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd) 

1209 skyWcs = model.makeSkyWcs(ccdImage) 

1210 try: 

1211 dataRef.put(skyWcs, 'jointcal_wcs') 

1212 except pexExceptions.Exception as e: 

1213 self.log.fatal('Failed to write updated Wcs: %s', str(e)) 

1214 raise e 

1215 

1216 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef): 

1217 """ 

1218 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 

1219 

1220 Parameters 

1221 ---------- 

1222 associations : `lsst.jointcal.Associations` 

1223 The star/reference star associations to fit. 

1224 model : `lsst.jointcal.PhotometryModel` 

1225 The photoometric model that was fit. 

1226 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1227 Dict of ccdImage identifiers to dataRefs that were fit. 

1228 """ 

1229 

1230 ccdImageList = associations.getCcdImageList() 

1231 for ccdImage in ccdImageList: 

1232 # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair? 

1233 ccd = ccdImage.ccdId 

1234 visit = ccdImage.visit 

1235 dataRef = visit_ccd_to_dataRef[(visit, ccd)] 

1236 self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd) 

1237 photoCalib = model.toPhotoCalib(ccdImage) 

1238 try: 

1239 dataRef.put(photoCalib, 'jointcal_photoCalib') 

1240 except pexExceptions.Exception as e: 

1241 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e)) 

1242 raise e