Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import collections 

23import os 

24 

25import numpy as np 

26import astropy.units as u 

27 

28import lsst.geom 

29import lsst.utils 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32from lsst.afw.image import fluxErrFromABMagErr 

33import lsst.pex.exceptions as pexExceptions 

34import lsst.afw.table 

35import lsst.log 

36import lsst.meas.algorithms 

37from lsst.pipe.tasks.colorterms import ColortermLibrary 

38from lsst.verify import Job, Measurement 

39 

40from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask 

41from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

42 

43from .dataIds import PerTractCcdDataIdContainer 

44 

45import lsst.jointcal 

46from lsst.jointcal import MinimizeResult 

47 

48__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"] 

49 

50Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

51Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

52 

53 

54# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

55def add_measurement(job, name, value): 

56 meas = Measurement(job.metrics[name], value) 

57 job.measurements.insert(meas) 

58 

59 

60class JointcalRunner(pipeBase.ButlerInitializedTaskRunner): 

61 """Subclass of TaskRunner for jointcalTask 

62 

63 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 

64 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 

65 single dataRef, are are called repeatedly). This class transforms the processed 

66 arguments generated by the ArgumentParser into the arguments expected by 

67 Jointcal.runDataRef(). 

68 

69 See pipeBase.TaskRunner for more information. 

70 """ 

71 

72 @staticmethod 

73 def getTargetList(parsedCmd, **kwargs): 

74 """ 

75 Return a list of tuples per tract, each containing (dataRefs, kwargs). 

76 

77 Jointcal operates on lists of dataRefs simultaneously. 

78 """ 

79 kwargs['profile_jointcal'] = parsedCmd.profile_jointcal 

80 kwargs['butler'] = parsedCmd.butler 

81 

82 # organize data IDs by tract 

83 refListDict = {} 

84 for ref in parsedCmd.id.refList: 

85 refListDict.setdefault(ref.dataId["tract"], []).append(ref) 

86 # we call runDataRef() once with each tract 

87 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())] 

88 return result 

89 

90 def __call__(self, args): 

91 """ 

92 Parameters 

93 ---------- 

94 args 

95 Arguments for Task.runDataRef() 

96 

97 Returns 

98 ------- 

99 pipe.base.Struct 

100 if self.doReturnResults is False: 

101 

102 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

103 

104 if self.doReturnResults is True: 

105 

106 - ``result``: the result of calling jointcal.runDataRef() 

107 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

108 """ 

109 exitStatus = 0 # exit status for shell 

110 

111 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef. 

112 dataRefList, kwargs = args 

113 butler = kwargs.pop('butler') 

114 task = self.TaskClass(config=self.config, log=self.log, butler=butler) 

115 result = None 

116 try: 

117 result = task.runDataRef(dataRefList, **kwargs) 

118 exitStatus = result.exitStatus 

119 job_path = butler.get('verify_job_filename') 

120 result.job.write(job_path[0]) 

121 except Exception as e: # catch everything, sort it out later. 

122 if self.doRaise: 

123 raise e 

124 else: 

125 exitStatus = 1 

126 eName = type(e).__name__ 

127 tract = dataRefList[0].dataId['tract'] 

128 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e) 

129 

130 # Put the butler back into kwargs for the other Tasks. 

131 kwargs['butler'] = butler 

132 if self.doReturnResults: 

133 return pipeBase.Struct(result=result, exitStatus=exitStatus) 

134 else: 

135 return pipeBase.Struct(exitStatus=exitStatus) 

136 

137 

138class JointcalConfig(pexConfig.Config): 

139 """Configuration for JointcalTask""" 

140 

141 doAstrometry = pexConfig.Field( 

142 doc="Fit astrometry and write the fitted result.", 

143 dtype=bool, 

144 default=True 

145 ) 

146 doPhotometry = pexConfig.Field( 

147 doc="Fit photometry and write the fitted result.", 

148 dtype=bool, 

149 default=True 

150 ) 

151 coaddName = pexConfig.Field( 

152 doc="Type of coadd, typically deep or goodSeeing", 

153 dtype=str, 

154 default="deep" 

155 ) 

156 positionErrorPedestal = pexConfig.Field( 

157 doc="Systematic term to apply to the measured position error (pixels)", 

158 dtype=float, 

159 default=0.02, 

160 ) 

161 photometryErrorPedestal = pexConfig.Field( 

162 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

163 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

164 dtype=float, 

165 default=0.0, 

166 ) 

167 # TODO: DM-6885 matchCut should be an geom.Angle 

168 matchCut = pexConfig.Field( 

169 doc="Matching radius between fitted and reference stars (arcseconds)", 

170 dtype=float, 

171 default=3.0, 

172 ) 

173 minMeasurements = pexConfig.Field( 

174 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

175 dtype=int, 

176 default=2, 

177 ) 

178 minMeasuredStarsPerCcd = pexConfig.Field( 

179 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

180 dtype=int, 

181 default=100, 

182 ) 

183 minRefStarsPerCcd = pexConfig.Field( 

184 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

185 dtype=int, 

186 default=30, 

187 ) 

188 allowLineSearch = pexConfig.Field( 

189 doc="Allow a line search during minimization, if it is reasonable for the model" 

190 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

191 dtype=bool, 

192 default=False 

193 ) 

194 astrometrySimpleOrder = pexConfig.Field( 

195 doc="Polynomial order for fitting the simple astrometry model.", 

196 dtype=int, 

197 default=3, 

198 ) 

199 astrometryChipOrder = pexConfig.Field( 

200 doc="Order of the per-chip transform for the constrained astrometry model.", 

201 dtype=int, 

202 default=1, 

203 ) 

204 astrometryVisitOrder = pexConfig.Field( 

205 doc="Order of the per-visit transform for the constrained astrometry model.", 

206 dtype=int, 

207 default=5, 

208 ) 

209 useInputWcs = pexConfig.Field( 

210 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

211 dtype=bool, 

212 default=True, 

213 ) 

214 astrometryModel = pexConfig.ChoiceField( 

215 doc="Type of model to fit to astrometry", 

216 dtype=str, 

217 default="constrained", 

218 allowed={"simple": "One polynomial per ccd", 

219 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

220 ) 

221 photometryModel = pexConfig.ChoiceField( 

222 doc="Type of model to fit to photometry", 

223 dtype=str, 

224 default="constrainedMagnitude", 

225 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

226 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

227 " fitting in flux space.", 

228 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

229 " fitting in magnitude space.", 

230 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

231 " fitting in magnitude space.", 

232 } 

233 ) 

234 applyColorTerms = pexConfig.Field( 

235 doc="Apply photometric color terms to reference stars?" 

236 "Requires that colorterms be set to a ColortermLibrary", 

237 dtype=bool, 

238 default=False 

239 ) 

240 colorterms = pexConfig.ConfigField( 

241 doc="Library of photometric reference catalog name to color term dict.", 

242 dtype=ColortermLibrary, 

243 ) 

244 photometryVisitOrder = pexConfig.Field( 

245 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

246 dtype=int, 

247 default=7, 

248 ) 

249 photometryDoRankUpdate = pexConfig.Field( 

250 doc=("Do the rank update step during minimization. " 

251 "Skipping this can help deal with models that are too non-linear."), 

252 dtype=bool, 

253 default=True, 

254 ) 

255 astrometryDoRankUpdate = pexConfig.Field( 

256 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

257 "Skipping this can help deal with models that are too non-linear."), 

258 dtype=bool, 

259 default=True, 

260 ) 

261 outlierRejectSigma = pexConfig.Field( 

262 doc="How many sigma to reject outliers at during minimization.", 

263 dtype=float, 

264 default=5.0, 

265 ) 

266 maxPhotometrySteps = pexConfig.Field( 

267 doc="Maximum number of minimize iterations to take when fitting photometry.", 

268 dtype=int, 

269 default=20, 

270 ) 

271 maxAstrometrySteps = pexConfig.Field( 

272 doc="Maximum number of minimize iterations to take when fitting photometry.", 

273 dtype=int, 

274 default=20, 

275 ) 

276 astrometryRefObjLoader = pexConfig.ConfigurableField( 

277 target=LoadIndexedReferenceObjectsTask, 

278 doc="Reference object loader for astrometric fit", 

279 ) 

280 photometryRefObjLoader = pexConfig.ConfigurableField( 

281 target=LoadIndexedReferenceObjectsTask, 

282 doc="Reference object loader for photometric fit", 

283 ) 

284 sourceSelector = sourceSelectorRegistry.makeField( 

285 doc="How to select sources for cross-matching", 

286 default="astrometry" 

287 ) 

288 astrometryReferenceSelector = pexConfig.ConfigurableField( 

289 target=ReferenceSourceSelectorTask, 

290 doc="How to down-select the loaded astrometry reference catalog.", 

291 ) 

292 photometryReferenceSelector = pexConfig.ConfigurableField( 

293 target=ReferenceSourceSelectorTask, 

294 doc="How to down-select the loaded photometry reference catalog.", 

295 ) 

296 astrometryReferenceErr = pexConfig.Field( 

297 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

298 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

299 "If specified, overrides any existing `coord_*Err` values."), 

300 dtype=float, 

301 default=None, 

302 optional=True 

303 ) 

304 writeInitMatrix = pexConfig.Field( 

305 dtype=bool, 

306 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

307 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory. " 

308 "Note that these files are the dense versions of the matrix, and so may be very large."), 

309 default=False 

310 ) 

311 writeChi2FilesInitialFinal = pexConfig.Field( 

312 dtype=bool, 

313 doc="Write .csv files containing the contributions to chi2 for the initialization and final fit.", 

314 default=False 

315 ) 

316 writeChi2FilesOuterLoop = pexConfig.Field( 

317 dtype=bool, 

318 doc="Write .csv files containing the contributions to chi2 for the outer fit loop.", 

319 default=False 

320 ) 

321 writeInitialModel = pexConfig.Field( 

322 dtype=bool, 

323 doc=("Write the pre-initialization model to text files, for debugging." 

324 " Output is written to `initial[Astro|Photo]metryModel.txt` in the current working directory."), 

325 default=False 

326 ) 

327 debugOutputPath = pexConfig.Field( 

328 dtype=str, 

329 default=".", 

330 doc=("Path to write debug output files to. Used by " 

331 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

332 ) 

333 sourceFluxType = pexConfig.Field( 

334 dtype=str, 

335 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

336 default='Calib' 

337 ) 

338 

339 def validate(self): 

340 super().validate() 

341 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

342 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

343 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

344 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

345 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

346 "applyColorTerms=True will be ignored.") 

347 lsst.log.warn(msg) 

348 

349 def setDefaults(self): 

350 # Use science source selector which can filter on extendedness, SNR, and whether blended 

351 self.sourceSelector.name = 'science' 

352 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing 

353 self.sourceSelector['science'].doUnresolved = True 

354 # with dependable signal to noise ratio. 

355 self.sourceSelector['science'].doSignalToNoise = True 

356 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

357 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

358 self.sourceSelector['science'].signalToNoise.minimum = 10. 

359 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive 

360 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux" 

361 self.sourceSelector['science'].signalToNoise.fluxField = fluxField 

362 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err" 

363 # Do not trust blended sources' aperture fluxes which also depend on seeing 

364 self.sourceSelector['science'].doIsolated = True 

365 # Do not trust either flux or centroid measurements with flags, 

366 # chosen from the usual QA flags for stars) 

367 self.sourceSelector['science'].doFlags = True 

368 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 

369 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag', 

370 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter'] 

371 self.sourceSelector['science'].flags.bad = badFlags 

372 

373 

374def writeModel(model, filename, log): 

375 """Write model to outfile.""" 

376 with open(filename, "w") as file: 

377 file.write(repr(model)) 

378 log.info("Wrote %s to file: %s", model, filename) 

379 

380 

381class JointcalTask(pipeBase.CmdLineTask): 

382 """Jointly astrometrically and photometrically calibrate a group of images.""" 

383 

384 ConfigClass = JointcalConfig 

385 RunnerClass = JointcalRunner 

386 _DefaultName = "jointcal" 

387 

388 def __init__(self, butler=None, profile_jointcal=False, **kwargs): 

389 """ 

390 Instantiate a JointcalTask. 

391 

392 Parameters 

393 ---------- 

394 butler : `lsst.daf.persistence.Butler` 

395 The butler is passed to the refObjLoader constructor in case it is 

396 needed. Ignored if the refObjLoader argument provides a loader directly. 

397 Used to initialize the astrometry and photometry refObjLoaders. 

398 profile_jointcal : `bool` 

399 Set to True to profile different stages of this jointcal run. 

400 """ 

401 pipeBase.CmdLineTask.__init__(self, **kwargs) 

402 self.profile_jointcal = profile_jointcal 

403 self.makeSubtask("sourceSelector") 

404 if self.config.doAstrometry: 

405 self.makeSubtask('astrometryRefObjLoader', butler=butler) 

406 self.makeSubtask("astrometryReferenceSelector") 

407 else: 

408 self.astrometryRefObjLoader = None 

409 if self.config.doPhotometry: 

410 self.makeSubtask('photometryRefObjLoader', butler=butler) 

411 self.makeSubtask("photometryReferenceSelector") 

412 else: 

413 self.photometryRefObjLoader = None 

414 

415 # To hold various computed metrics for use by tests 

416 self.job = Job.load_metrics_package(subset='jointcal') 

417 

418 # We don't currently need to persist the metadata. 

419 # If we do in the future, we will have to add appropriate dataset templates 

420 # to each obs package (the metadata template should look like `jointcal_wcs`). 

421 def _getMetadataName(self): 

422 return None 

423 

424 @classmethod 

425 def _makeArgumentParser(cls): 

426 """Create an argument parser""" 

427 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

428 parser.add_argument("--profile_jointcal", default=False, action="store_true", 

429 help="Profile steps of jointcal separately.") 

430 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9", 

431 ContainerClass=PerTractCcdDataIdContainer) 

432 return parser 

433 

434 def _build_ccdImage(self, dataRef, associations, jointcalControl): 

435 """ 

436 Extract the necessary things from this dataRef to add a new ccdImage. 

437 

438 Parameters 

439 ---------- 

440 dataRef : `lsst.daf.persistence.ButlerDataRef` 

441 DataRef to extract info from. 

442 associations : `lsst.jointcal.Associations` 

443 Object to add the info to, to construct a new CcdImage 

444 jointcalControl : `jointcal.JointcalControl` 

445 Control object for associations management 

446 

447 Returns 

448 ------ 

449 namedtuple 

450 ``wcs`` 

451 The TAN WCS of this image, read from the calexp 

452 (`lsst.afw.geom.SkyWcs`). 

453 ``key`` 

454 A key to identify this dataRef by its visit and ccd ids 

455 (`namedtuple`). 

456 ``filter`` 

457 This calexp's filter (`str`). 

458 """ 

459 if "visit" in dataRef.dataId.keys(): 

460 visit = dataRef.dataId["visit"] 

461 else: 

462 visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0] 

463 

464 src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True) 

465 

466 visitInfo = dataRef.get('calexp_visitInfo') 

467 detector = dataRef.get('calexp_detector') 

468 ccdId = detector.getId() 

469 photoCalib = dataRef.get('calexp_photoCalib') 

470 tanWcs = dataRef.get('calexp_wcs') 

471 bbox = dataRef.get('calexp_bbox') 

472 filt = dataRef.get('calexp_filter') 

473 filterName = filt.getName() 

474 

475 goodSrc = self.sourceSelector.run(src) 

476 

477 if len(goodSrc.sourceCat) == 0: 

478 self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId) 

479 else: 

480 self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId) 

481 associations.createCcdImage(goodSrc.sourceCat, 

482 tanWcs, 

483 visitInfo, 

484 bbox, 

485 filterName, 

486 photoCalib, 

487 detector, 

488 visit, 

489 ccdId, 

490 jointcalControl) 

491 

492 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter')) 

493 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

494 return Result(tanWcs, Key(visit, ccdId), filterName) 

495 

496 def _getDebugPath(self, filename): 

497 """Constructs a path to filename using the configured debug path. 

498 """ 

499 return os.path.join(self.config.debugOutputPath, filename) 

500 

501 @pipeBase.timeMethod 

502 def runDataRef(self, dataRefs, profile_jointcal=False): 

503 """ 

504 Jointly calibrate the astrometry and photometry across a set of images. 

505 

506 Parameters 

507 ---------- 

508 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

509 List of data references to the exposures to be fit. 

510 profile_jointcal : `bool` 

511 Profile the individual steps of jointcal. 

512 

513 Returns 

514 ------- 

515 result : `lsst.pipe.base.Struct` 

516 Struct of metadata from the fit, containing: 

517 

518 ``dataRefs`` 

519 The provided data references that were fit (with updated WCSs) 

520 ``oldWcsList`` 

521 The original WCS from each dataRef 

522 ``metrics`` 

523 Dictionary of internally-computed metrics for testing/validation. 

524 """ 

525 if len(dataRefs) == 0: 

526 raise ValueError('Need a non-empty list of data references!') 

527 

528 exitStatus = 0 # exit status for shell 

529 

530 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,) 

531 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

532 associations = lsst.jointcal.Associations() 

533 

534 visit_ccd_to_dataRef = {} 

535 oldWcsList = [] 

536 filters = [] 

537 load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else '' 

538 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

539 # We need the bounding-box of the focal plane for photometry visit models. 

540 # NOTE: we only need to read it once, because its the same for all exposures of a camera. 

541 camera = dataRefs[0].get('camera', immediate=True) 

542 self.focalPlaneBBox = camera.getFpBBox() 

543 for ref in dataRefs: 

544 result = self._build_ccdImage(ref, associations, jointcalControl) 

545 oldWcsList.append(result.wcs) 

546 visit_ccd_to_dataRef[result.key] = ref 

547 filters.append(result.filter) 

548 filters = collections.Counter(filters) 

549 

550 associations.computeCommonTangentPoint() 

551 

552 boundingCircle = associations.computeBoundingCircle() 

553 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

554 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

555 

556 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

557 

558 # Determine a default filter associated with the catalog. See DM-9093 

559 defaultFilter = filters.most_common(1)[0][0] 

560 self.log.debug("Using %s band for reference flux", defaultFilter) 

561 

562 # TODO: need a better way to get the tract. 

563 tract = dataRefs[0].dataId['tract'] 

564 

565 if self.config.doAstrometry: 

566 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

567 name="astrometry", 

568 refObjLoader=self.astrometryRefObjLoader, 

569 referenceSelector=self.astrometryReferenceSelector, 

570 fit_function=self._fit_astrometry, 

571 profile_jointcal=profile_jointcal, 

572 tract=tract) 

573 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef) 

574 else: 

575 astrometry = Astrometry(None, None, None) 

576 

577 if self.config.doPhotometry: 

578 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

579 name="photometry", 

580 refObjLoader=self.photometryRefObjLoader, 

581 referenceSelector=self.photometryReferenceSelector, 

582 fit_function=self._fit_photometry, 

583 profile_jointcal=profile_jointcal, 

584 tract=tract, 

585 filters=filters, 

586 reject_bad_fluxes=True) 

587 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef) 

588 else: 

589 photometry = Photometry(None, None) 

590 

591 return pipeBase.Struct(dataRefs=dataRefs, 

592 oldWcsList=oldWcsList, 

593 job=self.job, 

594 astrometryRefObjLoader=self.astrometryRefObjLoader, 

595 photometryRefObjLoader=self.photometryRefObjLoader, 

596 defaultFilter=defaultFilter, 

597 exitStatus=exitStatus) 

598 

599 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

600 filters=[], 

601 tract="", profile_jointcal=False, match_cut=3.0, 

602 reject_bad_fluxes=False, *, 

603 name="", refObjLoader=None, referenceSelector=None, 

604 fit_function=None): 

605 """Load reference catalog, perform the fit, and return the result. 

606 

607 Parameters 

608 ---------- 

609 associations : `lsst.jointcal.Associations` 

610 The star/reference star associations to fit. 

611 defaultFilter : `str` 

612 filter to load from reference catalog. 

613 center : `lsst.geom.SpherePoint` 

614 ICRS center of field to load from reference catalog. 

615 radius : `lsst.geom.Angle` 

616 On-sky radius to load from reference catalog. 

617 name : `str` 

618 Name of thing being fit: "astrometry" or "photometry". 

619 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

620 Reference object loader to use to load a reference catalog. 

621 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

622 Selector to use to pick objects from the loaded reference catalog. 

623 fit_function : callable 

624 Function to call to perform fit (takes Associations object). 

625 filters : `list` [`str`], optional 

626 List of filters to load from the reference catalog. 

627 tract : `str`, optional 

628 Name of tract currently being fit. 

629 profile_jointcal : `bool`, optional 

630 Separately profile the fitting step. 

631 match_cut : `float`, optional 

632 Radius in arcseconds to find cross-catalog matches to during 

633 associations.associateCatalogs. 

634 reject_bad_fluxes : `bool`, optional 

635 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

636 

637 Returns 

638 ------- 

639 result : `Photometry` or `Astrometry` 

640 Result of `fit_function()` 

641 """ 

642 self.log.info("====== Now processing %s...", name) 

643 # TODO: this should not print "trying to invert a singular transformation:" 

644 # if it does that, something's not right about the WCS... 

645 associations.associateCatalogs(match_cut) 

646 add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name, 

647 associations.fittedStarListSize()) 

648 

649 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

650 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

651 center, radius, defaultFilter, 

652 applyColorterms=applyColorterms) 

653 

654 if self.config.astrometryReferenceErr is None: 

655 refCoordErr = float('nan') 

656 else: 

657 refCoordErr = self.config.astrometryReferenceErr 

658 

659 associations.collectRefStars(refCat, 

660 self.config.matchCut*lsst.geom.arcseconds, 

661 fluxField, 

662 refCoordinateErr=refCoordErr, 

663 rejectBadFluxes=reject_bad_fluxes) 

664 add_measurement(self.job, 'jointcal.collected_%s_refStars' % name, 

665 associations.refStarListSize()) 

666 

667 associations.prepareFittedStars(self.config.minMeasurements) 

668 

669 self._check_star_lists(associations, name) 

670 add_measurement(self.job, 'jointcal.selected_%s_refStars' % name, 

671 associations.nFittedStarsWithAssociatedRefStar()) 

672 add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name, 

673 associations.fittedStarListSize()) 

674 add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name, 

675 associations.nCcdImagesValidForFit()) 

676 

677 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else '' 

678 dataName = "{}_{}".format(tract, defaultFilter) 

679 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

680 result = fit_function(associations, dataName) 

681 # TODO DM-12446: turn this into a "butler save" somehow. 

682 # Save reference and measurement chi2 contributions for this data 

683 if self.config.writeChi2FilesInitialFinal: 

684 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

685 result.fit.saveChi2Contributions(baseName+"{type}") 

686 self.log.info("Wrote chi2 contributions files: %s", baseName) 

687 

688 return result 

689 

690 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, 

691 applyColorterms=False): 

692 """Load the necessary reference catalog sources, convert fluxes to 

693 correct units, and apply color term corrections if requested. 

694 

695 Parameters 

696 ---------- 

697 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

698 The reference catalog loader to use to get the data. 

699 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

700 Source selector to apply to loaded reference catalog. 

701 center : `lsst.geom.SpherePoint` 

702 The center around which to load sources. 

703 radius : `lsst.geom.Angle` 

704 The radius around ``center`` to load sources in. 

705 filterName : `str` 

706 The name of the camera filter to load fluxes for. 

707 applyColorterms : `bool` 

708 Apply colorterm corrections to the refcat for ``filterName``? 

709 

710 Returns 

711 ------- 

712 refCat : `lsst.afw.table.SimpleCatalog` 

713 The loaded reference catalog. 

714 fluxField : `str` 

715 The name of the reference catalog flux field appropriate for ``filterName``. 

716 """ 

717 skyCircle = refObjLoader.loadSkyCircle(center, 

718 radius, 

719 filterName) 

720 

721 selected = referenceSelector.run(skyCircle.refCat) 

722 # Need memory contiguity to get reference filters as a vector. 

723 if not selected.sourceCat.isContiguous(): 

724 refCat = selected.sourceCat.copy(deep=True) 

725 else: 

726 refCat = selected.sourceCat 

727 

728 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

729 msg = ("Reference catalog does not contain coordinate errors, " 

730 "and config.astrometryReferenceErr not supplied.") 

731 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

732 self.config, 

733 msg) 

734 

735 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

736 self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

737 self.config.astrometryReferenceErr) 

738 

739 if applyColorterms: 

740 try: 

741 refCatName = refObjLoader.ref_dataset_name 

742 except AttributeError: 

743 # NOTE: we need this try:except: block in place until we've completely removed a.net support. 

744 raise RuntimeError("Cannot perform colorterm corrections with a.net refcats.") 

745 self.log.info("Applying color terms for filterName=%r reference catalog=%s", 

746 filterName, refCatName) 

747 colorterm = self.config.colorterms.getColorterm( 

748 filterName=filterName, photoCatName=refCatName, doRaise=True) 

749 

750 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName) 

751 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

752 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

753 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

754 

755 return refCat, skyCircle.fluxField 

756 

757 def _check_star_lists(self, associations, name): 

758 # TODO: these should be len(blah), but we need this properly wrapped first. 

759 if associations.nCcdImagesValidForFit() == 0: 

760 raise RuntimeError('No images in the ccdImageList!') 

761 if associations.fittedStarListSize() == 0: 

762 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

763 if associations.refStarListSize() == 0: 

764 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

765 

766 def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model", 

767 writeChi2Name=None): 

768 """Compute chi2, log it, validate the model, and return chi2. 

769 

770 Parameters 

771 ---------- 

772 associations : `lsst.jointcal.Associations` 

773 The star/reference star associations to fit. 

774 fit : `lsst.jointcal.FitterBase` 

775 The fitter to use for minimization. 

776 model : `lsst.jointcal.Model` 

777 The model being fit. 

778 chi2Label : str, optional 

779 Label to describe the chi2 (e.g. "Initialized", "Final"). 

780 writeChi2Name : `str`, optional 

781 Filename prefix to write the chi2 contributions to. 

782 Do not supply an extension: an appropriate one will be added. 

783 

784 Returns 

785 ------- 

786 chi2: `lsst.jointcal.Chi2Accumulator` 

787 The chi2 object for the current fitter and model. 

788 

789 Raises 

790 ------ 

791 FloatingPointError 

792 Raised if chi2 is infinite or NaN. 

793 ValueError 

794 Raised if the model is not valid. 

795 """ 

796 if writeChi2Name is not None: 

797 fullpath = self._getDebugPath(writeChi2Name) 

798 fit.saveChi2Contributions(fullpath+"{type}") 

799 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

800 

801 chi2 = fit.computeChi2() 

802 self.log.info("%s %s", chi2Label, chi2) 

803 self._check_stars(associations) 

804 if not np.isfinite(chi2.chi2): 

805 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

806 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

807 raise ValueError("Model is not valid: check log messages for warnings.") 

808 return chi2 

809 

810 def _fit_photometry(self, associations, dataName=None): 

811 """ 

812 Fit the photometric data. 

813 

814 Parameters 

815 ---------- 

816 associations : `lsst.jointcal.Associations` 

817 The star/reference star associations to fit. 

818 dataName : `str` 

819 Name of the data being processed (e.g. "1234_HSC-Y"), for 

820 identifying debugging files. 

821 

822 Returns 

823 ------- 

824 fit_result : `namedtuple` 

825 fit : `lsst.jointcal.PhotometryFit` 

826 The photometric fitter used to perform the fit. 

827 model : `lsst.jointcal.PhotometryModel` 

828 The photometric model that was fit. 

829 """ 

830 self.log.info("=== Starting photometric fitting...") 

831 

832 # TODO: should use pex.config.RegistryField here (see DM-9195) 

833 if self.config.photometryModel == "constrainedFlux": 

834 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

835 self.focalPlaneBBox, 

836 visitOrder=self.config.photometryVisitOrder, 

837 errorPedestal=self.config.photometryErrorPedestal) 

838 # potentially nonlinear problem, so we may need a line search to converge. 

839 doLineSearch = self.config.allowLineSearch 

840 elif self.config.photometryModel == "constrainedMagnitude": 

841 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

842 self.focalPlaneBBox, 

843 visitOrder=self.config.photometryVisitOrder, 

844 errorPedestal=self.config.photometryErrorPedestal) 

845 # potentially nonlinear problem, so we may need a line search to converge. 

846 doLineSearch = self.config.allowLineSearch 

847 elif self.config.photometryModel == "simpleFlux": 

848 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

849 errorPedestal=self.config.photometryErrorPedestal) 

850 doLineSearch = False # purely linear in model parameters, so no line search needed 

851 elif self.config.photometryModel == "simpleMagnitude": 

852 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

853 errorPedestal=self.config.photometryErrorPedestal) 

854 doLineSearch = False # purely linear in model parameters, so no line search needed 

855 

856 fit = lsst.jointcal.PhotometryFit(associations, model) 

857 # TODO DM-12446: turn this into a "butler save" somehow. 

858 # Save reference and measurement chi2 contributions for this data 

859 if self.config.writeChi2FilesInitialFinal: 

860 baseName = f"photometry_initial_chi2-{dataName}" 

861 else: 

862 baseName = None 

863 if self.config.writeInitialModel: 

864 fullpath = self._getDebugPath("initialPhotometryModel.txt") 

865 writeModel(model, fullpath, self.log) 

866 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

867 

868 def getChi2Name(whatToFit): 

869 if self.config.writeChi2FilesOuterLoop: 

870 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

871 else: 

872 return None 

873 

874 # The constrained model needs the visit transform fit first; the chip 

875 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

876 dumpMatrixFile = self._getDebugPath("photometry_preinit") if self.config.writeInitMatrix else "" 

877 if self.config.photometryModel.startswith("constrained"): 

878 # no line search: should be purely (or nearly) linear, 

879 # and we want a large step size to initialize with. 

880 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

881 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("ModelVisit")) 

882 dumpMatrixFile = "" # so we don't redo the output on the next step 

883 

884 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

885 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Model")) 

886 

887 fit.minimize("Fluxes") # no line search: always purely linear. 

888 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Fluxes")) 

889 

890 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

891 self._logChi2AndValidate(associations, fit, model, "Fit prepared", 

892 writeChi2Name=getChi2Name("ModelFluxes")) 

893 

894 model.freezeErrorTransform() 

895 self.log.debug("Photometry error scales are frozen.") 

896 

897 chi2 = self._iterate_fit(associations, 

898 fit, 

899 self.config.maxPhotometrySteps, 

900 "photometry", 

901 "Model Fluxes", 

902 doRankUpdate=self.config.photometryDoRankUpdate, 

903 doLineSearch=doLineSearch, 

904 dataName=dataName) 

905 

906 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

907 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

908 return Photometry(fit, model) 

909 

910 def _fit_astrometry(self, associations, dataName=None): 

911 """ 

912 Fit the astrometric data. 

913 

914 Parameters 

915 ---------- 

916 associations : `lsst.jointcal.Associations` 

917 The star/reference star associations to fit. 

918 dataName : `str` 

919 Name of the data being processed (e.g. "1234_HSC-Y"), for 

920 identifying debugging files. 

921 

922 Returns 

923 ------- 

924 fit_result : `namedtuple` 

925 fit : `lsst.jointcal.AstrometryFit` 

926 The astrometric fitter used to perform the fit. 

927 model : `lsst.jointcal.AstrometryModel` 

928 The astrometric model that was fit. 

929 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

930 The model for the sky to tangent plane projection that was used in the fit. 

931 """ 

932 

933 self.log.info("=== Starting astrometric fitting...") 

934 

935 associations.deprojectFittedStars() 

936 

937 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

938 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

939 # them so carefully? 

940 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

941 

942 if self.config.astrometryModel == "constrained": 

943 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

944 sky_to_tan_projection, 

945 chipOrder=self.config.astrometryChipOrder, 

946 visitOrder=self.config.astrometryVisitOrder) 

947 elif self.config.astrometryModel == "simple": 

948 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

949 sky_to_tan_projection, 

950 self.config.useInputWcs, 

951 nNotFit=0, 

952 order=self.config.astrometrySimpleOrder) 

953 

954 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

955 # TODO DM-12446: turn this into a "butler save" somehow. 

956 # Save reference and measurement chi2 contributions for this data 

957 if self.config.writeChi2FilesInitialFinal: 

958 baseName = f"astrometry_initial_chi2-{dataName}" 

959 else: 

960 baseName = None 

961 if self.config.writeInitialModel: 

962 fullpath = self._getDebugPath("initialAstrometryModel.txt") 

963 writeModel(model, fullpath, self.log) 

964 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

965 

966 def getChi2Name(whatToFit): 

967 if self.config.writeChi2FilesOuterLoop: 

968 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

969 else: 

970 return None 

971 

972 dumpMatrixFile = self._getDebugPath("astrometry_preinit") if self.config.writeInitMatrix else "" 

973 # The constrained model needs the visit transform fit first; the chip 

974 # transform is initialized from the detector's cameraGeom, so it's close. 

975 if self.config.astrometryModel == "constrained": 

976 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

977 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("DistortionsVisit")) 

978 dumpMatrixFile = "" # so we don't redo the output on the next step 

979 

980 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

981 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Distortions")) 

982 

983 fit.minimize("Positions") 

984 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Positions")) 

985 

986 fit.minimize("Distortions Positions") 

987 self._logChi2AndValidate(associations, fit, model, "Fit prepared", 

988 writeChi2Name=getChi2Name("DistortionsPositions")) 

989 

990 chi2 = self._iterate_fit(associations, 

991 fit, 

992 self.config.maxAstrometrySteps, 

993 "astrometry", 

994 "Distortions Positions", 

995 doRankUpdate=self.config.astrometryDoRankUpdate, 

996 dataName=dataName) 

997 

998 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

999 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1000 

1001 return Astrometry(fit, model, sky_to_tan_projection) 

1002 

1003 def _check_stars(self, associations): 

1004 """Count measured and reference stars per ccd and warn/log them.""" 

1005 for ccdImage in associations.getCcdImageList(): 

1006 nMeasuredStars, nRefStars = ccdImage.countStars() 

1007 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1008 ccdImage.getName(), nMeasuredStars, nRefStars) 

1009 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1010 self.log.warn("ccdImage %s has only %s measuredStars (desired %s)", 

1011 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1012 if nRefStars < self.config.minRefStarsPerCcd: 

1013 self.log.warn("ccdImage %s has only %s RefStars (desired %s)", 

1014 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1015 

1016 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1017 dataName="", 

1018 doRankUpdate=True, 

1019 doLineSearch=False): 

1020 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1021 

1022 Parameters 

1023 ---------- 

1024 associations : `lsst.jointcal.Associations` 

1025 The star/reference star associations to fit. 

1026 fitter : `lsst.jointcal.FitterBase` 

1027 The fitter to use for minimization. 

1028 max_steps : `int` 

1029 Maximum number of steps to run outlier rejection before declaring 

1030 convergence failure. 

1031 name : {'photometry' or 'astrometry'} 

1032 What type of data are we fitting (for logs and debugging files). 

1033 whatToFit : `str` 

1034 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1035 dataName : `str`, optional 

1036 Descriptive name for this dataset (e.g. tract and filter), 

1037 for debugging. 

1038 doRankUpdate : `bool`, optional 

1039 Do an Eigen rank update during minimization, or recompute the full 

1040 matrix and gradient? 

1041 doLineSearch : `bool`, optional 

1042 Do a line search for the optimum step during minimization? 

1043 

1044 Returns 

1045 ------- 

1046 chi2: `lsst.jointcal.Chi2Statistic` 

1047 The final chi2 after the fit converges, or is forced to end. 

1048 

1049 Raises 

1050 ------ 

1051 FloatingPointError 

1052 Raised if the fitter fails with a non-finite value. 

1053 RuntimeError 

1054 Raised if the fitter fails for some other reason; 

1055 log messages will provide further details. 

1056 """ 

1057 dumpMatrixFile = self._getDebugPath(f"{name}_postinit") if self.config.writeInitMatrix else "" 

1058 for i in range(max_steps): 

1059 if self.config.writeChi2FilesOuterLoop: 

1060 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1061 else: 

1062 writeChi2Name = None 

1063 result = fitter.minimize(whatToFit, 

1064 self.config.outlierRejectSigma, 

1065 doRankUpdate=doRankUpdate, 

1066 doLineSearch=doLineSearch, 

1067 dumpMatrixFile=dumpMatrixFile) 

1068 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1069 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1070 writeChi2Name=writeChi2Name) 

1071 

1072 if result == MinimizeResult.Converged: 

1073 if doRankUpdate: 

1074 self.log.debug("fit has converged - no more outliers - redo minimization " 

1075 "one more time in case we have lost accuracy in rank update.") 

1076 # Redo minimization one more time in case we have lost accuracy in rank update 

1077 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma) 

1078 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1079 

1080 # log a message for a large final chi2, TODO: DM-15247 for something better 

1081 if chi2.chi2/chi2.ndof >= 4.0: 

1082 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1083 

1084 break 

1085 elif result == MinimizeResult.Chi2Increased: 

1086 self.log.warn("still some outliers but chi2 increases - retry") 

1087 elif result == MinimizeResult.NonFinite: 

1088 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1089 # TODO DM-12446: turn this into a "butler save" somehow. 

1090 fitter.saveChi2Contributions(filename+"{type}") 

1091 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1092 raise FloatingPointError(msg.format(filename)) 

1093 elif result == MinimizeResult.Failed: 

1094 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1095 else: 

1096 raise RuntimeError("Unxepected return code from minimize().") 

1097 else: 

1098 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1099 

1100 return chi2 

1101 

1102 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef): 

1103 """ 

1104 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 

1105 

1106 Parameters 

1107 ---------- 

1108 associations : `lsst.jointcal.Associations` 

1109 The star/reference star associations to fit. 

1110 model : `lsst.jointcal.AstrometryModel` 

1111 The astrometric model that was fit. 

1112 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1113 Dict of ccdImage identifiers to dataRefs that were fit. 

1114 """ 

1115 

1116 ccdImageList = associations.getCcdImageList() 

1117 for ccdImage in ccdImageList: 

1118 # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair? 

1119 ccd = ccdImage.ccdId 

1120 visit = ccdImage.visit 

1121 dataRef = visit_ccd_to_dataRef[(visit, ccd)] 

1122 self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd) 

1123 skyWcs = model.makeSkyWcs(ccdImage) 

1124 try: 

1125 dataRef.put(skyWcs, 'jointcal_wcs') 

1126 except pexExceptions.Exception as e: 

1127 self.log.fatal('Failed to write updated Wcs: %s', str(e)) 

1128 raise e 

1129 

1130 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef): 

1131 """ 

1132 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 

1133 

1134 Parameters 

1135 ---------- 

1136 associations : `lsst.jointcal.Associations` 

1137 The star/reference star associations to fit. 

1138 model : `lsst.jointcal.PhotometryModel` 

1139 The photoometric model that was fit. 

1140 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1141 Dict of ccdImage identifiers to dataRefs that were fit. 

1142 """ 

1143 

1144 ccdImageList = associations.getCcdImageList() 

1145 for ccdImage in ccdImageList: 

1146 # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair? 

1147 ccd = ccdImage.ccdId 

1148 visit = ccdImage.visit 

1149 dataRef = visit_ccd_to_dataRef[(visit, ccd)] 

1150 self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd) 

1151 photoCalib = model.toPhotoCalib(ccdImage) 

1152 try: 

1153 dataRef.put(photoCalib, 'jointcal_photoCalib') 

1154 except pexExceptions.Exception as e: 

1155 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e)) 

1156 raise e