Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import collections 

23import os 

24 

25import numpy as np 

26import astropy.units as u 

27 

28import lsst.geom 

29import lsst.utils 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32from lsst.afw.image import fluxErrFromABMagErr 

33import lsst.pex.exceptions as pexExceptions 

34import lsst.afw.table 

35import lsst.log 

36import lsst.meas.algorithms 

37from lsst.pipe.tasks.colorterms import ColortermLibrary 

38from lsst.verify import Job, Measurement 

39 

40from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask 

41from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

42 

43from .dataIds import PerTractCcdDataIdContainer 

44 

45import lsst.jointcal 

46from lsst.jointcal import MinimizeResult 

47 

48__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"] 

49 

50Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

51Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

52 

53 

54# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

55def add_measurement(job, name, value): 

56 meas = Measurement(job.metrics[name], value) 

57 job.measurements.insert(meas) 

58 

59 

60class JointcalRunner(pipeBase.ButlerInitializedTaskRunner): 

61 """Subclass of TaskRunner for jointcalTask 

62 

63 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 

64 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 

65 single dataRef, are are called repeatedly). This class transforms the processed 

66 arguments generated by the ArgumentParser into the arguments expected by 

67 Jointcal.runDataRef(). 

68 

69 See pipeBase.TaskRunner for more information. 

70 """ 

71 

72 @staticmethod 

73 def getTargetList(parsedCmd, **kwargs): 

74 """ 

75 Return a list of tuples per tract, each containing (dataRefs, kwargs). 

76 

77 Jointcal operates on lists of dataRefs simultaneously. 

78 """ 

79 kwargs['profile_jointcal'] = parsedCmd.profile_jointcal 

80 kwargs['butler'] = parsedCmd.butler 

81 

82 # organize data IDs by tract 

83 refListDict = {} 

84 for ref in parsedCmd.id.refList: 

85 refListDict.setdefault(ref.dataId["tract"], []).append(ref) 

86 # we call runDataRef() once with each tract 

87 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())] 

88 return result 

89 

90 def __call__(self, args): 

91 """ 

92 Parameters 

93 ---------- 

94 args 

95 Arguments for Task.runDataRef() 

96 

97 Returns 

98 ------- 

99 pipe.base.Struct 

100 if self.doReturnResults is False: 

101 

102 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

103 

104 if self.doReturnResults is True: 

105 

106 - ``result``: the result of calling jointcal.runDataRef() 

107 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

108 """ 

109 exitStatus = 0 # exit status for shell 

110 

111 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef. 

112 dataRefList, kwargs = args 

113 butler = kwargs.pop('butler') 

114 task = self.TaskClass(config=self.config, log=self.log, butler=butler) 

115 result = None 

116 try: 

117 result = task.runDataRef(dataRefList, **kwargs) 

118 exitStatus = result.exitStatus 

119 job_path = butler.get('verify_job_filename') 

120 result.job.write(job_path[0]) 

121 except Exception as e: # catch everything, sort it out later. 

122 if self.doRaise: 

123 raise e 

124 else: 

125 exitStatus = 1 

126 eName = type(e).__name__ 

127 tract = dataRefList[0].dataId['tract'] 

128 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e) 

129 

130 # Put the butler back into kwargs for the other Tasks. 

131 kwargs['butler'] = butler 

132 if self.doReturnResults: 

133 return pipeBase.Struct(result=result, exitStatus=exitStatus) 

134 else: 

135 return pipeBase.Struct(exitStatus=exitStatus) 

136 

137 

138class JointcalConfig(pexConfig.Config): 

139 """Configuration for JointcalTask""" 

140 

141 doAstrometry = pexConfig.Field( 

142 doc="Fit astrometry and write the fitted result.", 

143 dtype=bool, 

144 default=True 

145 ) 

146 doPhotometry = pexConfig.Field( 

147 doc="Fit photometry and write the fitted result.", 

148 dtype=bool, 

149 default=True 

150 ) 

151 coaddName = pexConfig.Field( 

152 doc="Type of coadd, typically deep or goodSeeing", 

153 dtype=str, 

154 default="deep" 

155 ) 

156 positionErrorPedestal = pexConfig.Field( 

157 doc="Systematic term to apply to the measured position error (pixels)", 

158 dtype=float, 

159 default=0.02, 

160 ) 

161 photometryErrorPedestal = pexConfig.Field( 

162 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

163 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

164 dtype=float, 

165 default=0.0, 

166 ) 

167 # TODO: DM-6885 matchCut should be an geom.Angle 

168 matchCut = pexConfig.Field( 

169 doc="Matching radius between fitted and reference stars (arcseconds)", 

170 dtype=float, 

171 default=3.0, 

172 ) 

173 minMeasurements = pexConfig.Field( 

174 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

175 dtype=int, 

176 default=2, 

177 ) 

178 minMeasuredStarsPerCcd = pexConfig.Field( 

179 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

180 dtype=int, 

181 default=100, 

182 ) 

183 minRefStarsPerCcd = pexConfig.Field( 

184 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

185 dtype=int, 

186 default=30, 

187 ) 

188 allowLineSearch = pexConfig.Field( 

189 doc="Allow a line search during minimization, if it is reasonable for the model" 

190 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

191 dtype=bool, 

192 default=False 

193 ) 

194 astrometrySimpleOrder = pexConfig.Field( 

195 doc="Polynomial order for fitting the simple astrometry model.", 

196 dtype=int, 

197 default=3, 

198 ) 

199 astrometryChipOrder = pexConfig.Field( 

200 doc="Order of the per-chip transform for the constrained astrometry model.", 

201 dtype=int, 

202 default=1, 

203 ) 

204 astrometryVisitOrder = pexConfig.Field( 

205 doc="Order of the per-visit transform for the constrained astrometry model.", 

206 dtype=int, 

207 default=5, 

208 ) 

209 useInputWcs = pexConfig.Field( 

210 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

211 dtype=bool, 

212 default=True, 

213 ) 

214 astrometryModel = pexConfig.ChoiceField( 

215 doc="Type of model to fit to astrometry", 

216 dtype=str, 

217 default="constrained", 

218 allowed={"simple": "One polynomial per ccd", 

219 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

220 ) 

221 photometryModel = pexConfig.ChoiceField( 

222 doc="Type of model to fit to photometry", 

223 dtype=str, 

224 default="constrainedMagnitude", 

225 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

226 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

227 " fitting in flux space.", 

228 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

229 " fitting in magnitude space.", 

230 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

231 " fitting in magnitude space.", 

232 } 

233 ) 

234 applyColorTerms = pexConfig.Field( 

235 doc="Apply photometric color terms to reference stars?" 

236 "Requires that colorterms be set to a ColortermLibrary", 

237 dtype=bool, 

238 default=False 

239 ) 

240 colorterms = pexConfig.ConfigField( 

241 doc="Library of photometric reference catalog name to color term dict.", 

242 dtype=ColortermLibrary, 

243 ) 

244 photometryVisitOrder = pexConfig.Field( 

245 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

246 dtype=int, 

247 default=7, 

248 ) 

249 photometryDoRankUpdate = pexConfig.Field( 

250 doc=("Do the rank update step during minimization. " 

251 "Skipping this can help deal with models that are too non-linear."), 

252 dtype=bool, 

253 default=True, 

254 ) 

255 astrometryDoRankUpdate = pexConfig.Field( 

256 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

257 "Skipping this can help deal with models that are too non-linear."), 

258 dtype=bool, 

259 default=True, 

260 ) 

261 outlierRejectSigma = pexConfig.Field( 

262 doc="How many sigma to reject outliers at during minimization.", 

263 dtype=float, 

264 default=5.0, 

265 ) 

266 maxPhotometrySteps = pexConfig.Field( 

267 doc="Maximum number of minimize iterations to take when fitting photometry.", 

268 dtype=int, 

269 default=20, 

270 ) 

271 maxAstrometrySteps = pexConfig.Field( 

272 doc="Maximum number of minimize iterations to take when fitting photometry.", 

273 dtype=int, 

274 default=20, 

275 ) 

276 astrometryRefObjLoader = pexConfig.ConfigurableField( 

277 target=LoadIndexedReferenceObjectsTask, 

278 doc="Reference object loader for astrometric fit", 

279 ) 

280 photometryRefObjLoader = pexConfig.ConfigurableField( 

281 target=LoadIndexedReferenceObjectsTask, 

282 doc="Reference object loader for photometric fit", 

283 ) 

284 sourceSelector = sourceSelectorRegistry.makeField( 

285 doc="How to select sources for cross-matching", 

286 default="astrometry" 

287 ) 

288 astrometryReferenceSelector = pexConfig.ConfigurableField( 

289 target=ReferenceSourceSelectorTask, 

290 doc="How to down-select the loaded astrometry reference catalog.", 

291 ) 

292 photometryReferenceSelector = pexConfig.ConfigurableField( 

293 target=ReferenceSourceSelectorTask, 

294 doc="How to down-select the loaded photometry reference catalog.", 

295 ) 

296 astrometryReferenceErr = pexConfig.Field( 

297 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

298 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

299 "If specified, overrides any existing `coord_*Err` values."), 

300 dtype=float, 

301 default=None, 

302 optional=True 

303 ) 

304 writeInitMatrix = pexConfig.Field( 

305 dtype=bool, 

306 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

307 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory. " 

308 "Note that these files are the dense versions of the matrix, and so may be very large."), 

309 default=False 

310 ) 

311 writeChi2FilesInitialFinal = pexConfig.Field( 

312 dtype=bool, 

313 doc="Write .csv files containing the contributions to chi2 for the initialization and final fit.", 

314 default=False 

315 ) 

316 writeChi2FilesOuterLoop = pexConfig.Field( 

317 dtype=bool, 

318 doc="Write .csv files containing the contributions to chi2 for the outer fit loop.", 

319 default=False 

320 ) 

321 writeInitialModel = pexConfig.Field( 

322 dtype=bool, 

323 doc=("Write the pre-initialization model to text files, for debugging." 

324 " Output is written to `initial[Astro|Photo]metryModel.txt` in the current working directory."), 

325 default=False 

326 ) 

327 debugOutputPath = pexConfig.Field( 

328 dtype=str, 

329 default=".", 

330 doc=("Path to write debug output files to. Used by " 

331 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

332 ) 

333 sourceFluxType = pexConfig.Field( 

334 dtype=str, 

335 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

336 default='Calib' 

337 ) 

338 

339 def validate(self): 

340 super().validate() 

341 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

342 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

343 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

344 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

345 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

346 "applyColorTerms=True will be ignored.") 

347 lsst.log.warn(msg) 

348 

349 def setDefaults(self): 

350 # Use science source selector which can filter on extendedness, SNR, and whether blended 

351 self.sourceSelector.name = 'science' 

352 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing 

353 self.sourceSelector['science'].doUnresolved = True 

354 # with dependable signal to noise ratio. 

355 self.sourceSelector['science'].doSignalToNoise = True 

356 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

357 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

358 self.sourceSelector['science'].signalToNoise.minimum = 10. 

359 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive 

360 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux" 

361 self.sourceSelector['science'].signalToNoise.fluxField = fluxField 

362 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err" 

363 # Do not trust blended sources' aperture fluxes which also depend on seeing 

364 self.sourceSelector['science'].doIsolated = True 

365 # Do not trust either flux or centroid measurements with flags, 

366 # chosen from the usual QA flags for stars) 

367 self.sourceSelector['science'].doFlags = True 

368 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 

369 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag', 

370 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter'] 

371 self.sourceSelector['science'].flags.bad = badFlags 

372 

373 

374def writeModel(model, filename, log): 

375 """Write model to outfile.""" 

376 with open(filename, "w") as file: 

377 file.write(repr(model)) 

378 log.info("Wrote %s to file: %s", model, filename) 

379 

380 

381class JointcalTask(pipeBase.CmdLineTask): 

382 """Jointly astrometrically and photometrically calibrate a group of images.""" 

383 

384 ConfigClass = JointcalConfig 

385 RunnerClass = JointcalRunner 

386 _DefaultName = "jointcal" 

387 

388 def __init__(self, butler=None, profile_jointcal=False, **kwargs): 

389 """ 

390 Instantiate a JointcalTask. 

391 

392 Parameters 

393 ---------- 

394 butler : `lsst.daf.persistence.Butler` 

395 The butler is passed to the refObjLoader constructor in case it is 

396 needed. Ignored if the refObjLoader argument provides a loader directly. 

397 Used to initialize the astrometry and photometry refObjLoaders. 

398 profile_jointcal : `bool` 

399 Set to True to profile different stages of this jointcal run. 

400 """ 

401 pipeBase.CmdLineTask.__init__(self, **kwargs) 

402 self.profile_jointcal = profile_jointcal 

403 self.makeSubtask("sourceSelector") 

404 if self.config.doAstrometry: 

405 self.makeSubtask('astrometryRefObjLoader', butler=butler) 

406 self.makeSubtask("astrometryReferenceSelector") 

407 else: 

408 self.astrometryRefObjLoader = None 

409 if self.config.doPhotometry: 

410 self.makeSubtask('photometryRefObjLoader', butler=butler) 

411 self.makeSubtask("photometryReferenceSelector") 

412 else: 

413 self.photometryRefObjLoader = None 

414 

415 # To hold various computed metrics for use by tests 

416 self.job = Job.load_metrics_package(subset='jointcal') 

417 

418 # We don't currently need to persist the metadata. 

419 # If we do in the future, we will have to add appropriate dataset templates 

420 # to each obs package (the metadata template should look like `jointcal_wcs`). 

421 def _getMetadataName(self): 

422 return None 

423 

424 @classmethod 

425 def _makeArgumentParser(cls): 

426 """Create an argument parser""" 

427 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

428 parser.add_argument("--profile_jointcal", default=False, action="store_true", 

429 help="Profile steps of jointcal separately.") 

430 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9", 

431 ContainerClass=PerTractCcdDataIdContainer) 

432 return parser 

433 

434 def _build_ccdImage(self, dataRef, associations, jointcalControl): 

435 """ 

436 Extract the necessary things from this dataRef to add a new ccdImage. 

437 

438 Parameters 

439 ---------- 

440 dataRef : `lsst.daf.persistence.ButlerDataRef` 

441 DataRef to extract info from. 

442 associations : `lsst.jointcal.Associations` 

443 Object to add the info to, to construct a new CcdImage 

444 jointcalControl : `jointcal.JointcalControl` 

445 Control object for associations management 

446 

447 Returns 

448 ------ 

449 namedtuple 

450 ``wcs`` 

451 The TAN WCS of this image, read from the calexp 

452 (`lsst.afw.geom.SkyWcs`). 

453 ``key`` 

454 A key to identify this dataRef by its visit and ccd ids 

455 (`namedtuple`). 

456 ``filter`` 

457 This calexp's filter (`str`). 

458 """ 

459 if "visit" in dataRef.dataId.keys(): 

460 visit = dataRef.dataId["visit"] 

461 else: 

462 visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0] 

463 

464 src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True) 

465 

466 visitInfo = dataRef.get('calexp_visitInfo') 

467 detector = dataRef.get('calexp_detector') 

468 ccdId = detector.getId() 

469 photoCalib = dataRef.get('calexp_photoCalib') 

470 tanWcs = dataRef.get('calexp_wcs') 

471 bbox = dataRef.get('calexp_bbox') 

472 filt = dataRef.get('calexp_filter') 

473 filterName = filt.getName() 

474 

475 goodSrc = self.sourceSelector.run(src) 

476 

477 if len(goodSrc.sourceCat) == 0: 

478 self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId) 

479 else: 

480 self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId) 

481 associations.createCcdImage(goodSrc.sourceCat, 

482 tanWcs, 

483 visitInfo, 

484 bbox, 

485 filterName, 

486 photoCalib, 

487 detector, 

488 visit, 

489 ccdId, 

490 jointcalControl) 

491 

492 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter')) 

493 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

494 return Result(tanWcs, Key(visit, ccdId), filterName) 

495 

496 def _getDebugPath(self, filename): 

497 """Constructs a path to filename using the configured debug path. 

498 """ 

499 return os.path.join(self.config.debugOutputPath, filename) 

500 

501 @pipeBase.timeMethod 

502 def runDataRef(self, dataRefs, profile_jointcal=False): 

503 """ 

504 Jointly calibrate the astrometry and photometry across a set of images. 

505 

506 Parameters 

507 ---------- 

508 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

509 List of data references to the exposures to be fit. 

510 profile_jointcal : `bool` 

511 Profile the individual steps of jointcal. 

512 

513 Returns 

514 ------- 

515 result : `lsst.pipe.base.Struct` 

516 Struct of metadata from the fit, containing: 

517 

518 ``dataRefs`` 

519 The provided data references that were fit (with updated WCSs) 

520 ``oldWcsList`` 

521 The original WCS from each dataRef 

522 ``metrics`` 

523 Dictionary of internally-computed metrics for testing/validation. 

524 """ 

525 if len(dataRefs) == 0: 

526 raise ValueError('Need a non-empty list of data references!') 

527 

528 exitStatus = 0 # exit status for shell 

529 

530 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,) 

531 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

532 associations = lsst.jointcal.Associations() 

533 

534 visit_ccd_to_dataRef = {} 

535 oldWcsList = [] 

536 filters = [] 

537 load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else '' 

538 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

539 # We need the bounding-box of the focal plane for photometry visit models. 

540 # NOTE: we only need to read it once, because its the same for all exposures of a camera. 

541 camera = dataRefs[0].get('camera', immediate=True) 

542 self.focalPlaneBBox = camera.getFpBBox() 

543 for ref in dataRefs: 

544 result = self._build_ccdImage(ref, associations, jointcalControl) 

545 oldWcsList.append(result.wcs) 

546 visit_ccd_to_dataRef[result.key] = ref 

547 filters.append(result.filter) 

548 filters = collections.Counter(filters) 

549 

550 associations.computeCommonTangentPoint() 

551 

552 boundingCircle = associations.computeBoundingCircle() 

553 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

554 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

555 

556 # Determine a default filter associated with the catalog. See DM-9093 

557 defaultFilter = filters.most_common(1)[0][0] 

558 self.log.debug("Using %s band for reference flux", defaultFilter) 

559 

560 # TODO: need a better way to get the tract. 

561 tract = dataRefs[0].dataId['tract'] 

562 

563 if self.config.doAstrometry: 

564 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

565 name="astrometry", 

566 refObjLoader=self.astrometryRefObjLoader, 

567 referenceSelector=self.astrometryReferenceSelector, 

568 fit_function=self._fit_astrometry, 

569 profile_jointcal=profile_jointcal, 

570 tract=tract) 

571 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef) 

572 else: 

573 astrometry = Astrometry(None, None, None) 

574 

575 if self.config.doPhotometry: 

576 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

577 name="photometry", 

578 refObjLoader=self.photometryRefObjLoader, 

579 referenceSelector=self.photometryReferenceSelector, 

580 fit_function=self._fit_photometry, 

581 profile_jointcal=profile_jointcal, 

582 tract=tract, 

583 filters=filters, 

584 reject_bad_fluxes=True) 

585 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef) 

586 else: 

587 photometry = Photometry(None, None) 

588 

589 return pipeBase.Struct(dataRefs=dataRefs, 

590 oldWcsList=oldWcsList, 

591 job=self.job, 

592 astrometryRefObjLoader=self.astrometryRefObjLoader, 

593 photometryRefObjLoader=self.photometryRefObjLoader, 

594 defaultFilter=defaultFilter, 

595 exitStatus=exitStatus) 

596 

597 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

598 filters=[], 

599 tract="", profile_jointcal=False, match_cut=3.0, 

600 reject_bad_fluxes=False, *, 

601 name="", refObjLoader=None, referenceSelector=None, 

602 fit_function=None): 

603 """Load reference catalog, perform the fit, and return the result. 

604 

605 Parameters 

606 ---------- 

607 associations : `lsst.jointcal.Associations` 

608 The star/reference star associations to fit. 

609 defaultFilter : `str` 

610 filter to load from reference catalog. 

611 center : `lsst.geom.SpherePoint` 

612 ICRS center of field to load from reference catalog. 

613 radius : `lsst.geom.Angle` 

614 On-sky radius to load from reference catalog. 

615 name : `str` 

616 Name of thing being fit: "astrometry" or "photometry". 

617 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

618 Reference object loader to use to load a reference catalog. 

619 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

620 Selector to use to pick objects from the loaded reference catalog. 

621 fit_function : callable 

622 Function to call to perform fit (takes Associations object). 

623 filters : `list` [`str`], optional 

624 List of filters to load from the reference catalog. 

625 tract : `str`, optional 

626 Name of tract currently being fit. 

627 profile_jointcal : `bool`, optional 

628 Separately profile the fitting step. 

629 match_cut : `float`, optional 

630 Radius in arcseconds to find cross-catalog matches to during 

631 associations.associateCatalogs. 

632 reject_bad_fluxes : `bool`, optional 

633 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

634 

635 Returns 

636 ------- 

637 result : `Photometry` or `Astrometry` 

638 Result of `fit_function()` 

639 """ 

640 self.log.info("====== Now processing %s...", name) 

641 # TODO: this should not print "trying to invert a singular transformation:" 

642 # if it does that, something's not right about the WCS... 

643 associations.associateCatalogs(match_cut) 

644 add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name, 

645 associations.fittedStarListSize()) 

646 

647 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

648 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

649 center, radius, defaultFilter, 

650 applyColorterms=applyColorterms) 

651 

652 if self.config.astrometryReferenceErr is None: 

653 refCoordErr = float('nan') 

654 else: 

655 refCoordErr = self.config.astrometryReferenceErr 

656 

657 associations.collectRefStars(refCat, 

658 self.config.matchCut*lsst.geom.arcseconds, 

659 fluxField, 

660 refCoordinateErr=refCoordErr, 

661 rejectBadFluxes=reject_bad_fluxes) 

662 add_measurement(self.job, 'jointcal.collected_%s_refStars' % name, 

663 associations.refStarListSize()) 

664 

665 associations.prepareFittedStars(self.config.minMeasurements) 

666 

667 self._check_star_lists(associations, name) 

668 add_measurement(self.job, 'jointcal.selected_%s_refStars' % name, 

669 associations.nFittedStarsWithAssociatedRefStar()) 

670 add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name, 

671 associations.fittedStarListSize()) 

672 add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name, 

673 associations.nCcdImagesValidForFit()) 

674 

675 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else '' 

676 dataName = "{}_{}".format(tract, defaultFilter) 

677 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

678 result = fit_function(associations, dataName) 

679 # TODO DM-12446: turn this into a "butler save" somehow. 

680 # Save reference and measurement chi2 contributions for this data 

681 if self.config.writeChi2FilesInitialFinal: 

682 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

683 result.fit.saveChi2Contributions(baseName+"{type}") 

684 self.log.info("Wrote chi2 contributions files: %s", baseName) 

685 

686 return result 

687 

688 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, 

689 applyColorterms=False): 

690 """Load the necessary reference catalog sources, convert fluxes to 

691 correct units, and apply color term corrections if requested. 

692 

693 Parameters 

694 ---------- 

695 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

696 The reference catalog loader to use to get the data. 

697 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

698 Source selector to apply to loaded reference catalog. 

699 center : `lsst.geom.SpherePoint` 

700 The center around which to load sources. 

701 radius : `lsst.geom.Angle` 

702 The radius around ``center`` to load sources in. 

703 filterName : `str` 

704 The name of the camera filter to load fluxes for. 

705 applyColorterms : `bool` 

706 Apply colorterm corrections to the refcat for ``filterName``? 

707 

708 Returns 

709 ------- 

710 refCat : `lsst.afw.table.SimpleCatalog` 

711 The loaded reference catalog. 

712 fluxField : `str` 

713 The name of the reference catalog flux field appropriate for ``filterName``. 

714 """ 

715 skyCircle = refObjLoader.loadSkyCircle(center, 

716 radius, 

717 filterName) 

718 

719 selected = referenceSelector.run(skyCircle.refCat) 

720 # Need memory contiguity to get reference filters as a vector. 

721 if not selected.sourceCat.isContiguous(): 

722 refCat = selected.sourceCat.copy(deep=True) 

723 else: 

724 refCat = selected.sourceCat 

725 

726 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

727 msg = ("Reference catalog does not contain coordinate errors, " 

728 "and config.astrometryReferenceErr not supplied.") 

729 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

730 self.config, 

731 msg) 

732 

733 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

734 self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

735 self.config.astrometryReferenceErr) 

736 

737 if applyColorterms: 

738 try: 

739 refCatName = refObjLoader.ref_dataset_name 

740 except AttributeError: 

741 # NOTE: we need this try:except: block in place until we've completely removed a.net support. 

742 raise RuntimeError("Cannot perform colorterm corrections with a.net refcats.") 

743 self.log.info("Applying color terms for filterName=%r reference catalog=%s", 

744 filterName, refCatName) 

745 colorterm = self.config.colorterms.getColorterm( 

746 filterName=filterName, photoCatName=refCatName, doRaise=True) 

747 

748 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName) 

749 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

750 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

751 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

752 

753 return refCat, skyCircle.fluxField 

754 

755 def _check_star_lists(self, associations, name): 

756 # TODO: these should be len(blah), but we need this properly wrapped first. 

757 if associations.nCcdImagesValidForFit() == 0: 

758 raise RuntimeError('No images in the ccdImageList!') 

759 if associations.fittedStarListSize() == 0: 

760 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

761 if associations.refStarListSize() == 0: 

762 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

763 

764 def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model", 

765 writeChi2Name=None): 

766 """Compute chi2, log it, validate the model, and return chi2. 

767 

768 Parameters 

769 ---------- 

770 associations : `lsst.jointcal.Associations` 

771 The star/reference star associations to fit. 

772 fit : `lsst.jointcal.FitterBase` 

773 The fitter to use for minimization. 

774 model : `lsst.jointcal.Model` 

775 The model being fit. 

776 chi2Label : str, optional 

777 Label to describe the chi2 (e.g. "Initialized", "Final"). 

778 writeChi2Name : `str`, optional 

779 Filename prefix to write the chi2 contributions to. 

780 Do not supply an extension: an appropriate one will be added. 

781 

782 Returns 

783 ------- 

784 chi2: `lsst.jointcal.Chi2Accumulator` 

785 The chi2 object for the current fitter and model. 

786 

787 Raises 

788 ------ 

789 FloatingPointError 

790 Raised if chi2 is infinite or NaN. 

791 ValueError 

792 Raised if the model is not valid. 

793 """ 

794 if writeChi2Name is not None: 

795 fullpath = self._getDebugPath(writeChi2Name) 

796 fit.saveChi2Contributions(fullpath+"{type}") 

797 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

798 

799 chi2 = fit.computeChi2() 

800 self.log.info("%s %s", chi2Label, chi2) 

801 self._check_stars(associations) 

802 if not np.isfinite(chi2.chi2): 

803 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

804 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

805 raise ValueError("Model is not valid: check log messages for warnings.") 

806 return chi2 

807 

808 def _fit_photometry(self, associations, dataName=None): 

809 """ 

810 Fit the photometric data. 

811 

812 Parameters 

813 ---------- 

814 associations : `lsst.jointcal.Associations` 

815 The star/reference star associations to fit. 

816 dataName : `str` 

817 Name of the data being processed (e.g. "1234_HSC-Y"), for 

818 identifying debugging files. 

819 

820 Returns 

821 ------- 

822 fit_result : `namedtuple` 

823 fit : `lsst.jointcal.PhotometryFit` 

824 The photometric fitter used to perform the fit. 

825 model : `lsst.jointcal.PhotometryModel` 

826 The photometric model that was fit. 

827 """ 

828 self.log.info("=== Starting photometric fitting...") 

829 

830 # TODO: should use pex.config.RegistryField here (see DM-9195) 

831 if self.config.photometryModel == "constrainedFlux": 

832 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

833 self.focalPlaneBBox, 

834 visitOrder=self.config.photometryVisitOrder, 

835 errorPedestal=self.config.photometryErrorPedestal) 

836 # potentially nonlinear problem, so we may need a line search to converge. 

837 doLineSearch = self.config.allowLineSearch 

838 elif self.config.photometryModel == "constrainedMagnitude": 

839 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

840 self.focalPlaneBBox, 

841 visitOrder=self.config.photometryVisitOrder, 

842 errorPedestal=self.config.photometryErrorPedestal) 

843 # potentially nonlinear problem, so we may need a line search to converge. 

844 doLineSearch = self.config.allowLineSearch 

845 elif self.config.photometryModel == "simpleFlux": 

846 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

847 errorPedestal=self.config.photometryErrorPedestal) 

848 doLineSearch = False # purely linear in model parameters, so no line search needed 

849 elif self.config.photometryModel == "simpleMagnitude": 

850 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

851 errorPedestal=self.config.photometryErrorPedestal) 

852 doLineSearch = False # purely linear in model parameters, so no line search needed 

853 

854 fit = lsst.jointcal.PhotometryFit(associations, model) 

855 # TODO DM-12446: turn this into a "butler save" somehow. 

856 # Save reference and measurement chi2 contributions for this data 

857 if self.config.writeChi2FilesInitialFinal: 

858 baseName = f"photometry_initial_chi2-{dataName}" 

859 else: 

860 baseName = None 

861 if self.config.writeInitialModel: 

862 fullpath = self._getDebugPath("initialPhotometryModel.txt") 

863 writeModel(model, fullpath, self.log) 

864 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

865 

866 def getChi2Name(whatToFit): 

867 if self.config.writeChi2FilesOuterLoop: 

868 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

869 else: 

870 return None 

871 

872 # The constrained model needs the visit transform fit first; the chip 

873 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

874 dumpMatrixFile = self._getDebugPath("photometry_preinit") if self.config.writeInitMatrix else "" 

875 if self.config.photometryModel.startswith("constrained"): 

876 # no line search: should be purely (or nearly) linear, 

877 # and we want a large step size to initialize with. 

878 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

879 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("ModelVisit")) 

880 dumpMatrixFile = "" # so we don't redo the output on the next step 

881 

882 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

883 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Model")) 

884 

885 fit.minimize("Fluxes") # no line search: always purely linear. 

886 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Fluxes")) 

887 

888 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

889 self._logChi2AndValidate(associations, fit, model, "Fit prepared", 

890 writeChi2Name=getChi2Name("ModelFluxes")) 

891 

892 model.freezeErrorTransform() 

893 self.log.debug("Photometry error scales are frozen.") 

894 

895 chi2 = self._iterate_fit(associations, 

896 fit, 

897 self.config.maxPhotometrySteps, 

898 "photometry", 

899 "Model Fluxes", 

900 doRankUpdate=self.config.photometryDoRankUpdate, 

901 doLineSearch=doLineSearch, 

902 dataName=dataName) 

903 

904 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

905 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

906 return Photometry(fit, model) 

907 

908 def _fit_astrometry(self, associations, dataName=None): 

909 """ 

910 Fit the astrometric data. 

911 

912 Parameters 

913 ---------- 

914 associations : `lsst.jointcal.Associations` 

915 The star/reference star associations to fit. 

916 dataName : `str` 

917 Name of the data being processed (e.g. "1234_HSC-Y"), for 

918 identifying debugging files. 

919 

920 Returns 

921 ------- 

922 fit_result : `namedtuple` 

923 fit : `lsst.jointcal.AstrometryFit` 

924 The astrometric fitter used to perform the fit. 

925 model : `lsst.jointcal.AstrometryModel` 

926 The astrometric model that was fit. 

927 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

928 The model for the sky to tangent plane projection that was used in the fit. 

929 """ 

930 

931 self.log.info("=== Starting astrometric fitting...") 

932 

933 associations.deprojectFittedStars() 

934 

935 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

936 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

937 # them so carefully? 

938 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

939 

940 if self.config.astrometryModel == "constrained": 

941 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

942 sky_to_tan_projection, 

943 chipOrder=self.config.astrometryChipOrder, 

944 visitOrder=self.config.astrometryVisitOrder) 

945 elif self.config.astrometryModel == "simple": 

946 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

947 sky_to_tan_projection, 

948 self.config.useInputWcs, 

949 nNotFit=0, 

950 order=self.config.astrometrySimpleOrder) 

951 

952 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

953 # TODO DM-12446: turn this into a "butler save" somehow. 

954 # Save reference and measurement chi2 contributions for this data 

955 if self.config.writeChi2FilesInitialFinal: 

956 baseName = f"astrometry_initial_chi2-{dataName}" 

957 else: 

958 baseName = None 

959 if self.config.writeInitialModel: 

960 fullpath = self._getDebugPath("initialAstrometryModel.txt") 

961 writeModel(model, fullpath, self.log) 

962 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

963 

964 def getChi2Name(whatToFit): 

965 if self.config.writeChi2FilesOuterLoop: 

966 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

967 else: 

968 return None 

969 

970 dumpMatrixFile = self._getDebugPath("astrometry_preinit") if self.config.writeInitMatrix else "" 

971 # The constrained model needs the visit transform fit first; the chip 

972 # transform is initialized from the detector's cameraGeom, so it's close. 

973 if self.config.astrometryModel == "constrained": 

974 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

975 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("DistortionsVisit")) 

976 dumpMatrixFile = "" # so we don't redo the output on the next step 

977 

978 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

979 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Distortions")) 

980 

981 fit.minimize("Positions") 

982 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Positions")) 

983 

984 fit.minimize("Distortions Positions") 

985 self._logChi2AndValidate(associations, fit, model, "Fit prepared", 

986 writeChi2Name=getChi2Name("DistortionsPositions")) 

987 

988 chi2 = self._iterate_fit(associations, 

989 fit, 

990 self.config.maxAstrometrySteps, 

991 "astrometry", 

992 "Distortions Positions", 

993 doRankUpdate=self.config.astrometryDoRankUpdate, 

994 dataName=dataName) 

995 

996 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

997 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

998 

999 return Astrometry(fit, model, sky_to_tan_projection) 

1000 

1001 def _check_stars(self, associations): 

1002 """Count measured and reference stars per ccd and warn/log them.""" 

1003 for ccdImage in associations.getCcdImageList(): 

1004 nMeasuredStars, nRefStars = ccdImage.countStars() 

1005 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1006 ccdImage.getName(), nMeasuredStars, nRefStars) 

1007 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1008 self.log.warn("ccdImage %s has only %s measuredStars (desired %s)", 

1009 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1010 if nRefStars < self.config.minRefStarsPerCcd: 

1011 self.log.warn("ccdImage %s has only %s RefStars (desired %s)", 

1012 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1013 

1014 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1015 dataName="", 

1016 doRankUpdate=True, 

1017 doLineSearch=False): 

1018 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1019 

1020 Parameters 

1021 ---------- 

1022 associations : `lsst.jointcal.Associations` 

1023 The star/reference star associations to fit. 

1024 fitter : `lsst.jointcal.FitterBase` 

1025 The fitter to use for minimization. 

1026 max_steps : `int` 

1027 Maximum number of steps to run outlier rejection before declaring 

1028 convergence failure. 

1029 name : {'photometry' or 'astrometry'} 

1030 What type of data are we fitting (for logs and debugging files). 

1031 whatToFit : `str` 

1032 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1033 dataName : `str`, optional 

1034 Descriptive name for this dataset (e.g. tract and filter), 

1035 for debugging. 

1036 doRankUpdate : `bool`, optional 

1037 Do an Eigen rank update during minimization, or recompute the full 

1038 matrix and gradient? 

1039 doLineSearch : `bool`, optional 

1040 Do a line search for the optimum step during minimization? 

1041 

1042 Returns 

1043 ------- 

1044 chi2: `lsst.jointcal.Chi2Statistic` 

1045 The final chi2 after the fit converges, or is forced to end. 

1046 

1047 Raises 

1048 ------ 

1049 FloatingPointError 

1050 Raised if the fitter fails with a non-finite value. 

1051 RuntimeError 

1052 Raised if the fitter fails for some other reason; 

1053 log messages will provide further details. 

1054 """ 

1055 dumpMatrixFile = self._getDebugPath(f"{name}_postinit") if self.config.writeInitMatrix else "" 

1056 for i in range(max_steps): 

1057 if self.config.writeChi2FilesOuterLoop: 

1058 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1059 else: 

1060 writeChi2Name = None 

1061 result = fitter.minimize(whatToFit, 

1062 self.config.outlierRejectSigma, 

1063 doRankUpdate=doRankUpdate, 

1064 doLineSearch=doLineSearch, 

1065 dumpMatrixFile=dumpMatrixFile) 

1066 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1067 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1068 writeChi2Name=writeChi2Name) 

1069 

1070 if result == MinimizeResult.Converged: 

1071 if doRankUpdate: 

1072 self.log.debug("fit has converged - no more outliers - redo minimization " 

1073 "one more time in case we have lost accuracy in rank update.") 

1074 # Redo minimization one more time in case we have lost accuracy in rank update 

1075 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma) 

1076 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1077 

1078 # log a message for a large final chi2, TODO: DM-15247 for something better 

1079 if chi2.chi2/chi2.ndof >= 4.0: 

1080 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1081 

1082 break 

1083 elif result == MinimizeResult.Chi2Increased: 

1084 self.log.warn("still some outliers but chi2 increases - retry") 

1085 elif result == MinimizeResult.NonFinite: 

1086 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1087 # TODO DM-12446: turn this into a "butler save" somehow. 

1088 fitter.saveChi2Contributions(filename+"{type}") 

1089 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1090 raise FloatingPointError(msg.format(filename)) 

1091 elif result == MinimizeResult.Failed: 

1092 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1093 else: 

1094 raise RuntimeError("Unxepected return code from minimize().") 

1095 else: 

1096 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1097 

1098 return chi2 

1099 

1100 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef): 

1101 """ 

1102 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 

1103 

1104 Parameters 

1105 ---------- 

1106 associations : `lsst.jointcal.Associations` 

1107 The star/reference star associations to fit. 

1108 model : `lsst.jointcal.AstrometryModel` 

1109 The astrometric model that was fit. 

1110 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1111 Dict of ccdImage identifiers to dataRefs that were fit. 

1112 """ 

1113 

1114 ccdImageList = associations.getCcdImageList() 

1115 for ccdImage in ccdImageList: 

1116 # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair? 

1117 ccd = ccdImage.ccdId 

1118 visit = ccdImage.visit 

1119 dataRef = visit_ccd_to_dataRef[(visit, ccd)] 

1120 self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd) 

1121 skyWcs = model.makeSkyWcs(ccdImage) 

1122 try: 

1123 dataRef.put(skyWcs, 'jointcal_wcs') 

1124 except pexExceptions.Exception as e: 

1125 self.log.fatal('Failed to write updated Wcs: %s', str(e)) 

1126 raise e 

1127 

1128 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef): 

1129 """ 

1130 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 

1131 

1132 Parameters 

1133 ---------- 

1134 associations : `lsst.jointcal.Associations` 

1135 The star/reference star associations to fit. 

1136 model : `lsst.jointcal.PhotometryModel` 

1137 The photoometric model that was fit. 

1138 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1139 Dict of ccdImage identifiers to dataRefs that were fit. 

1140 """ 

1141 

1142 ccdImageList = associations.getCcdImageList() 

1143 for ccdImage in ccdImageList: 

1144 # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair? 

1145 ccd = ccdImage.ccdId 

1146 visit = ccdImage.visit 

1147 dataRef = visit_ccd_to_dataRef[(visit, ccd)] 

1148 self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd) 

1149 photoCalib = model.toPhotoCalib(ccdImage) 

1150 try: 

1151 dataRef.put(photoCalib, 'jointcal_photoCalib') 

1152 except pexExceptions.Exception as e: 

1153 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e)) 

1154 raise e