Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import dataclasses 

23import collections 

24import os 

25 

26import astropy.time 

27import numpy as np 

28import astropy.units as u 

29 

30import lsst.geom 

31import lsst.utils 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34from lsst.afw.image import fluxErrFromABMagErr 

35import lsst.pex.exceptions as pexExceptions 

36import lsst.afw.cameraGeom 

37import lsst.afw.table 

38import lsst.log 

39from lsst.obs.base import Instrument 

40from lsst.pipe.tasks.colorterms import ColortermLibrary 

41from lsst.verify import Job, Measurement 

42 

43from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask, 

44 ReferenceObjectLoader) 

45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

46 

47from .dataIds import PerTractCcdDataIdContainer 

48 

49import lsst.jointcal 

50from lsst.jointcal import MinimizeResult 

51 

52__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"] 

53 

54Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

55Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

56 

57 

58# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

59def add_measurement(job, name, value): 

60 meas = Measurement(job.metrics[name], value) 

61 job.measurements.insert(meas) 

62 

63 

64class JointcalRunner(pipeBase.ButlerInitializedTaskRunner): 

65 """Subclass of TaskRunner for jointcalTask (gen2) 

66 

67 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 

68 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 

69 single dataRef, are are called repeatedly). This class transforms the processed 

70 arguments generated by the ArgumentParser into the arguments expected by 

71 Jointcal.runDataRef(). 

72 

73 See pipeBase.TaskRunner for more information. 

74 """ 

75 

76 @staticmethod 

77 def getTargetList(parsedCmd, **kwargs): 

78 """ 

79 Return a list of tuples per tract, each containing (dataRefs, kwargs). 

80 

81 Jointcal operates on lists of dataRefs simultaneously. 

82 """ 

83 kwargs['butler'] = parsedCmd.butler 

84 

85 # organize data IDs by tract 

86 refListDict = {} 

87 for ref in parsedCmd.id.refList: 

88 refListDict.setdefault(ref.dataId["tract"], []).append(ref) 

89 # we call runDataRef() once with each tract 

90 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())] 

91 return result 

92 

93 def __call__(self, args): 

94 """ 

95 Parameters 

96 ---------- 

97 args 

98 Arguments for Task.runDataRef() 

99 

100 Returns 

101 ------- 

102 pipe.base.Struct 

103 if self.doReturnResults is False: 

104 

105 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

106 

107 if self.doReturnResults is True: 

108 

109 - ``result``: the result of calling jointcal.runDataRef() 

110 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

111 """ 

112 exitStatus = 0 # exit status for shell 

113 

114 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef. 

115 dataRefList, kwargs = args 

116 butler = kwargs.pop('butler') 

117 task = self.TaskClass(config=self.config, log=self.log, butler=butler) 

118 result = None 

119 try: 

120 result = task.runDataRef(dataRefList, **kwargs) 

121 exitStatus = result.exitStatus 

122 job_path = butler.get('verify_job_filename') 

123 result.job.write(job_path[0]) 

124 except Exception as e: # catch everything, sort it out later. 

125 if self.doRaise: 

126 raise e 

127 else: 

128 exitStatus = 1 

129 eName = type(e).__name__ 

130 tract = dataRefList[0].dataId['tract'] 

131 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e) 

132 

133 # Put the butler back into kwargs for the other Tasks. 

134 kwargs['butler'] = butler 

135 if self.doReturnResults: 

136 return pipeBase.Struct(result=result, exitStatus=exitStatus) 

137 else: 

138 return pipeBase.Struct(exitStatus=exitStatus) 

139 

140 

141def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

142 """Lookup function that asserts/hopes that a static calibration dataset 

143 exists in a particular collection, since this task can't provide a single 

144 date/time to use to search for one properly. 

145 

146 This is mostly useful for the ``camera`` dataset, in cases where the task's 

147 quantum dimensions do *not* include something temporal, like ``exposure`` 

148 or ``visit``. 

149 

150 Parameters 

151 ---------- 

152 datasetType : `lsst.daf.butler.DatasetType` 

153 Type of dataset being searched for. 

154 registry : `lsst.daf.butler.Registry` 

155 Data repository registry to search. 

156 quantumDataId : `lsst.daf.butler.DataCoordinate` 

157 Data ID of the quantum this camera should match. 

158 collections : `Iterable` [ `str` ] 

159 Collections that should be searched - but this lookup function works 

160 by ignoring this in favor of a more-or-less hard-coded value. 

161 

162 Returns 

163 ------- 

164 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

165 Iterator over dataset references; should have only one element. 

166 

167 Notes 

168 ----- 

169 This implementation duplicates one in fgcmcal, and is at least quite 

170 similar to another in cp_pipe. This duplicate has the most documentation. 

171 Fixing this is DM-29661. 

172 """ 

173 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

174 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

175 return registry.queryDatasets(datasetType, 

176 dataId=quantumDataId, 

177 collections=[unboundedCollection], 

178 findFirst=True) 

179 

180 

181def lookupVisitRefCats(datasetType, registry, quantumDataId, collections): 

182 """Lookup function that finds all refcats for all visits that overlap a 

183 tract, rather than just the refcats that directly overlap the tract. 

184 

185 Parameters 

186 ---------- 

187 datasetType : `lsst.daf.butler.DatasetType` 

188 Type of dataset being searched for. 

189 registry : `lsst.daf.butler.Registry` 

190 Data repository registry to search. 

191 quantumDataId : `lsst.daf.butler.DataCoordinate` 

192 Data ID of the quantum; expected to be something we can use as a 

193 constraint to query for overlapping visits. 

194 collections : `Iterable` [ `str` ] 

195 Collections to search. 

196 

197 Returns 

198 ------- 

199 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

200 Iterator over refcat references. 

201 """ 

202 refs = set() 

203 # Use .expanded() on the query methods below because we need data IDs with 

204 # regions, both in the outer loop over visits (queryDatasets will expand 

205 # any data ID we give it, but doing it up-front in bulk is much more 

206 # efficient) and in the data IDs of the DatasetRefs this function yields 

207 # (because the RefCatLoader relies on them to do some of its own 

208 # filtering). 

209 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()): 

210 refs.update( 

211 registry.queryDatasets( 

212 datasetType, 

213 collections=collections, 

214 dataId=visit_data_id, 

215 findFirst=True, 

216 ).expanded() 

217 ) 

218 yield from refs 

219 

220 

221class JointcalTaskConnections(pipeBase.PipelineTaskConnections, 

222 dimensions=("skymap", "tract", "instrument", "physical_filter")): 

223 """Middleware input/output connections for jointcal data.""" 

224 inputCamera = pipeBase.connectionTypes.PrerequisiteInput( 

225 doc="The camera instrument that took these observations.", 

226 name="camera", 

227 storageClass="Camera", 

228 dimensions=("instrument",), 

229 isCalibration=True, 

230 lookupFunction=lookupStaticCalibrations, 

231 ) 

232 inputSourceTableVisit = pipeBase.connectionTypes.Input( 

233 doc="Source table in parquet format, per visit", 

234 name="sourceTable_visit", 

235 storageClass="DataFrame", 

236 dimensions=("instrument", "visit"), 

237 deferLoad=True, 

238 multiple=True, 

239 ) 

240 inputVisitSummary = pipeBase.connectionTypes.Input( 

241 doc=("Per-visit consolidated exposure metadata built from calexps. " 

242 "These catalogs use detector id for the id and must be sorted for " 

243 "fast lookups of a detector."), 

244 name="visitSummary", 

245 storageClass="ExposureCatalog", 

246 dimensions=("instrument", "visit"), 

247 deferLoad=True, 

248 multiple=True, 

249 ) 

250 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

251 doc="The astrometry reference catalog to match to loaded input catalog sources.", 

252 name="gaia_dr2_20200414", 

253 storageClass="SimpleCatalog", 

254 dimensions=("skypix",), 

255 deferLoad=True, 

256 multiple=True, 

257 lookupFunction=lookupVisitRefCats, 

258 ) 

259 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

260 doc="The photometry reference catalog to match to loaded input catalog sources.", 

261 name="ps1_pv3_3pi_20170110", 

262 storageClass="SimpleCatalog", 

263 dimensions=("skypix",), 

264 deferLoad=True, 

265 multiple=True, 

266 lookupFunction=lookupVisitRefCats, 

267 ) 

268 

269 outputWcs = pipeBase.connectionTypes.Output( 

270 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model." 

271 " These catalogs only contain entries for detectors with an output, and use" 

272 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

273 name="jointcalSkyWcsCatalog", 

274 storageClass="ExposureCatalog", 

275 dimensions=("instrument", "visit", "skymap", "tract"), 

276 multiple=True 

277 ) 

278 outputPhotoCalib = pipeBase.connectionTypes.Output( 

279 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model." 

280 " These catalogs only contain entries for detectors with an output, and use" 

281 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

282 name="jointcalPhotoCalibCatalog", 

283 storageClass="ExposureCatalog", 

284 dimensions=("instrument", "visit", "skymap", "tract"), 

285 multiple=True 

286 ) 

287 

288 def __init__(self, *, config=None): 

289 super().__init__(config=config) 

290 # When we are only doing one of astrometry or photometry, we don't 

291 # need the reference catalog or produce the outputs for the other. 

292 # This informs the middleware of that when the QuantumGraph is 

293 # generated, so we don't block on getting something we won't need or 

294 # create an expectation that downstream tasks will be able to consume 

295 # something we won't produce. 

296 if not config.doAstrometry: 

297 self.prerequisiteInputs.remove("astrometryRefCat") 

298 self.outputs.remove("outputWcs") 

299 if not config.doPhotometry: 

300 self.prerequisiteInputs.remove("photometryRefCat") 

301 self.outputs.remove("outputPhotoCalib") 

302 

303 

304class JointcalConfig(pipeBase.PipelineTaskConfig, 

305 pipelineConnections=JointcalTaskConnections): 

306 """Configuration for JointcalTask""" 

307 

308 doAstrometry = pexConfig.Field( 

309 doc="Fit astrometry and write the fitted result.", 

310 dtype=bool, 

311 default=True 

312 ) 

313 doPhotometry = pexConfig.Field( 

314 doc="Fit photometry and write the fitted result.", 

315 dtype=bool, 

316 default=True 

317 ) 

318 coaddName = pexConfig.Field( 

319 doc="Type of coadd, typically deep or goodSeeing", 

320 dtype=str, 

321 default="deep" 

322 ) 

323 # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal. 

324 sourceFluxType = pexConfig.Field( 

325 dtype=str, 

326 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

327 default='Calib' 

328 ) 

329 positionErrorPedestal = pexConfig.Field( 

330 doc="Systematic term to apply to the measured position error (pixels)", 

331 dtype=float, 

332 default=0.02, 

333 ) 

334 photometryErrorPedestal = pexConfig.Field( 

335 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

336 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

337 dtype=float, 

338 default=0.0, 

339 ) 

340 # TODO: DM-6885 matchCut should be an geom.Angle 

341 matchCut = pexConfig.Field( 

342 doc="Matching radius between fitted and reference stars (arcseconds)", 

343 dtype=float, 

344 default=3.0, 

345 ) 

346 minMeasurements = pexConfig.Field( 

347 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

348 dtype=int, 

349 default=2, 

350 ) 

351 minMeasuredStarsPerCcd = pexConfig.Field( 

352 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

353 dtype=int, 

354 default=100, 

355 ) 

356 minRefStarsPerCcd = pexConfig.Field( 

357 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

358 dtype=int, 

359 default=30, 

360 ) 

361 allowLineSearch = pexConfig.Field( 

362 doc="Allow a line search during minimization, if it is reasonable for the model" 

363 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

364 dtype=bool, 

365 default=False 

366 ) 

367 astrometrySimpleOrder = pexConfig.Field( 

368 doc="Polynomial order for fitting the simple astrometry model.", 

369 dtype=int, 

370 default=3, 

371 ) 

372 astrometryChipOrder = pexConfig.Field( 

373 doc="Order of the per-chip transform for the constrained astrometry model.", 

374 dtype=int, 

375 default=1, 

376 ) 

377 astrometryVisitOrder = pexConfig.Field( 

378 doc="Order of the per-visit transform for the constrained astrometry model.", 

379 dtype=int, 

380 default=5, 

381 ) 

382 useInputWcs = pexConfig.Field( 

383 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

384 dtype=bool, 

385 default=True, 

386 ) 

387 astrometryModel = pexConfig.ChoiceField( 

388 doc="Type of model to fit to astrometry", 

389 dtype=str, 

390 default="constrained", 

391 allowed={"simple": "One polynomial per ccd", 

392 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

393 ) 

394 photometryModel = pexConfig.ChoiceField( 

395 doc="Type of model to fit to photometry", 

396 dtype=str, 

397 default="constrainedMagnitude", 

398 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

399 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

400 " fitting in flux space.", 

401 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

402 " fitting in magnitude space.", 

403 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

404 " fitting in magnitude space.", 

405 } 

406 ) 

407 applyColorTerms = pexConfig.Field( 

408 doc="Apply photometric color terms to reference stars?" 

409 "Requires that colorterms be set to a ColortermLibrary", 

410 dtype=bool, 

411 default=False 

412 ) 

413 colorterms = pexConfig.ConfigField( 

414 doc="Library of photometric reference catalog name to color term dict.", 

415 dtype=ColortermLibrary, 

416 ) 

417 photometryVisitOrder = pexConfig.Field( 

418 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

419 dtype=int, 

420 default=7, 

421 ) 

422 photometryDoRankUpdate = pexConfig.Field( 

423 doc=("Do the rank update step during minimization. " 

424 "Skipping this can help deal with models that are too non-linear."), 

425 dtype=bool, 

426 default=True, 

427 ) 

428 astrometryDoRankUpdate = pexConfig.Field( 

429 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

430 "Skipping this can help deal with models that are too non-linear."), 

431 dtype=bool, 

432 default=True, 

433 ) 

434 outlierRejectSigma = pexConfig.Field( 

435 doc="How many sigma to reject outliers at during minimization.", 

436 dtype=float, 

437 default=5.0, 

438 ) 

439 maxPhotometrySteps = pexConfig.Field( 

440 doc="Maximum number of minimize iterations to take when fitting photometry.", 

441 dtype=int, 

442 default=20, 

443 ) 

444 maxAstrometrySteps = pexConfig.Field( 

445 doc="Maximum number of minimize iterations to take when fitting photometry.", 

446 dtype=int, 

447 default=20, 

448 ) 

449 astrometryRefObjLoader = pexConfig.ConfigurableField( 

450 target=LoadIndexedReferenceObjectsTask, 

451 doc="Reference object loader for astrometric fit", 

452 ) 

453 photometryRefObjLoader = pexConfig.ConfigurableField( 

454 target=LoadIndexedReferenceObjectsTask, 

455 doc="Reference object loader for photometric fit", 

456 ) 

457 sourceSelector = sourceSelectorRegistry.makeField( 

458 doc="How to select sources for cross-matching", 

459 default="astrometry" 

460 ) 

461 astrometryReferenceSelector = pexConfig.ConfigurableField( 

462 target=ReferenceSourceSelectorTask, 

463 doc="How to down-select the loaded astrometry reference catalog.", 

464 ) 

465 photometryReferenceSelector = pexConfig.ConfigurableField( 

466 target=ReferenceSourceSelectorTask, 

467 doc="How to down-select the loaded photometry reference catalog.", 

468 ) 

469 astrometryReferenceErr = pexConfig.Field( 

470 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

471 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

472 "If specified, overrides any existing `coord_*Err` values."), 

473 dtype=float, 

474 default=None, 

475 optional=True 

476 ) 

477 

478 # configs for outputting debug information 

479 writeInitMatrix = pexConfig.Field( 

480 dtype=bool, 

481 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

482 "Output files will be written to `config.debugOutputPath` and will " 

483 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. " 

484 "Note that these files are the dense versions of the matrix, and so may be very large."), 

485 default=False 

486 ) 

487 writeChi2FilesInitialFinal = pexConfig.Field( 

488 dtype=bool, 

489 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. " 

490 "Output files will be written to `config.debugOutputPath` and will " 

491 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER1."), 

492 default=False 

493 ) 

494 writeChi2FilesOuterLoop = pexConfig.Field( 

495 dtype=bool, 

496 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. " 

497 "Output files will be written to `config.debugOutputPath` and will " 

498 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."), 

499 default=False 

500 ) 

501 writeInitialModel = pexConfig.Field( 

502 dtype=bool, 

503 doc=("Write the pre-initialization model to text files, for debugging. " 

504 "Output files will be written to `config.debugOutputPath` and will be " 

505 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."), 

506 default=False 

507 ) 

508 debugOutputPath = pexConfig.Field( 

509 dtype=str, 

510 default=".", 

511 doc=("Path to write debug output files to. Used by " 

512 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

513 ) 

514 detailedProfile = pexConfig.Field( 

515 dtype=bool, 

516 default=False, 

517 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting" 

518 ) 

519 

520 def validate(self): 

521 super().validate() 

522 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

523 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

524 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

525 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

526 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

527 "applyColorTerms=True will be ignored.") 

528 lsst.log.warn(msg) 

529 

530 def setDefaults(self): 

531 # Use science source selector which can filter on extendedness, SNR, and whether blended 

532 self.sourceSelector.name = 'science' 

533 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing 

534 self.sourceSelector['science'].doUnresolved = True 

535 # with dependable signal to noise ratio. 

536 self.sourceSelector['science'].doSignalToNoise = True 

537 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

538 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

539 self.sourceSelector['science'].signalToNoise.minimum = 10. 

540 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive 

541 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux" 

542 self.sourceSelector['science'].signalToNoise.fluxField = fluxField 

543 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err" 

544 # Do not trust blended sources' aperture fluxes which also depend on seeing 

545 self.sourceSelector['science'].doIsolated = True 

546 # Do not trust either flux or centroid measurements with flags, 

547 # chosen from the usual QA flags for stars) 

548 self.sourceSelector['science'].doFlags = True 

549 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 

550 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag', 

551 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter'] 

552 self.sourceSelector['science'].flags.bad = badFlags 

553 

554 # Default to Gaia-DR2 (with proper motions) for astrometry and 

555 # PS1-DR1 for photometry, with a reasonable initial filterMap. 

556 self.astrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414" 

557 self.astrometryRefObjLoader.requireProperMotion = True 

558 self.astrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean' 

559 self.photometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110" 

560 

561 

562def writeModel(model, filename, log): 

563 """Write model to outfile.""" 

564 with open(filename, "w") as file: 

565 file.write(repr(model)) 

566 log.info("Wrote %s to file: %s", model, filename) 

567 

568 

569@dataclasses.dataclass 

570class JointcalInputData: 

571 """The input data jointcal needs for each detector/visit.""" 

572 visit: int 

573 """The visit identifier of this exposure.""" 

574 catalog: lsst.afw.table.SourceCatalog 

575 """The catalog derived from this exposure.""" 

576 visitInfo: lsst.afw.image.VisitInfo 

577 """The VisitInfo of this exposure.""" 

578 detector: lsst.afw.cameraGeom.Detector 

579 """The detector of this exposure.""" 

580 photoCalib: lsst.afw.image.PhotoCalib 

581 """The photometric calibration of this exposure.""" 

582 wcs: lsst.afw.geom.skyWcs 

583 """The WCS of this exposure.""" 

584 bbox: lsst.geom.Box2I 

585 """The bounding box of this exposure.""" 

586 filter: lsst.afw.image.FilterLabel 

587 """The filter of this exposure.""" 

588 

589 

590class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

591 """Astrometricly and photometricly calibrate across multiple visits of the 

592 same field. 

593 

594 Parameters 

595 ---------- 

596 butler : `lsst.daf.persistence.Butler` 

597 The butler is passed to the refObjLoader constructor in case it is 

598 needed. Ignored if the refObjLoader argument provides a loader directly. 

599 Used to initialize the astrometry and photometry refObjLoaders. 

600 initInputs : `dict`, optional 

601 Dictionary used to initialize PipelineTasks (empty for jointcal). 

602 """ 

603 

604 ConfigClass = JointcalConfig 

605 RunnerClass = JointcalRunner 

606 _DefaultName = "jointcal" 

607 

608 def __init__(self, butler=None, initInputs=None, **kwargs): 

609 super().__init__(**kwargs) 

610 self.makeSubtask("sourceSelector") 

611 if self.config.doAstrometry: 

612 if initInputs is None: 

613 # gen3 middleware does refcat things internally (and will not have a butler here) 

614 self.makeSubtask('astrometryRefObjLoader', butler=butler) 

615 self.makeSubtask("astrometryReferenceSelector") 

616 else: 

617 self.astrometryRefObjLoader = None 

618 if self.config.doPhotometry: 

619 if initInputs is None: 

620 # gen3 middleware does refcat things internally (and will not have a butler here) 

621 self.makeSubtask('photometryRefObjLoader', butler=butler) 

622 self.makeSubtask("photometryReferenceSelector") 

623 else: 

624 self.photometryRefObjLoader = None 

625 

626 # To hold various computed metrics for use by tests 

627 self.job = Job.load_metrics_package(subset='jointcal') 

628 

629 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

630 # We override runQuantum to set up the refObjLoaders and write the 

631 # outputs to the correct refs. 

632 inputs = butlerQC.get(inputRefs) 

633 # We want the tract number for writing debug files 

634 tract = butlerQC.quantum.dataId['tract'] 

635 if self.config.doAstrometry: 

636 self.astrometryRefObjLoader = ReferenceObjectLoader( 

637 dataIds=[ref.datasetRef.dataId 

638 for ref in inputRefs.astrometryRefCat], 

639 refCats=inputs.pop('astrometryRefCat'), 

640 config=self.config.astrometryRefObjLoader, 

641 log=self.log) 

642 if self.config.doPhotometry: 

643 self.photometryRefObjLoader = ReferenceObjectLoader( 

644 dataIds=[ref.datasetRef.dataId 

645 for ref in inputRefs.photometryRefCat], 

646 refCats=inputs.pop('photometryRefCat'), 

647 config=self.config.photometryRefObjLoader, 

648 log=self.log) 

649 outputs = self.run(**inputs, tract=tract) 

650 if self.config.doAstrometry: 

651 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs, 

652 inputs['inputCamera'], "setWcs") 

653 if self.config.doPhotometry: 

654 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib, 

655 inputs['inputCamera'], "setPhotoCalib") 

656 

657 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter): 

658 """Persist the output datasets to their appropriate datarefs. 

659 

660 Parameters 

661 ---------- 

662 butlerQC : `ButlerQuantumContext` 

663 A butler which is specialized to operate in the context of a 

664 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

665 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or 

666 `dict` [`tuple, `lsst.afw.image.PhotoCalib`] 

667 The fitted objects to persist. 

668 outputRefs : `list` [`OutputQuantizedConnection`] 

669 The DatasetRefs to persist the data to. 

670 camera : `lsst.afw.cameraGeom.Camera` 

671 The camera for this instrument, to get detector ids from. 

672 setter : `str` 

673 The method to call on the ExposureCatalog to set each output. 

674 """ 

675 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

676 schema.addField('visit', type='I', doc='Visit number') 

677 

678 def new_catalog(visit, size): 

679 """Return an catalog ready to be filled with appropriate output.""" 

680 catalog = lsst.afw.table.ExposureCatalog(schema) 

681 catalog.resize(size) 

682 catalog['visit'] = visit 

683 metadata = lsst.daf.base.PropertyList() 

684 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

685 metadata.add("COMMENT", "Only detectors with data have entries.") 

686 return catalog 

687 

688 # count how many detectors have output for each visit 

689 detectors_per_visit = collections.defaultdict(int) 

690 for key in outputs: 

691 # key is (visit, detector_id), and we only need visit here 

692 detectors_per_visit[key[0]] += 1 

693 

694 for ref in outputRefs: 

695 visit = ref.dataId['visit'] 

696 catalog = new_catalog(visit, detectors_per_visit[visit]) 

697 # Iterate over every detector and skip the ones we don't have output for. 

698 i = 0 

699 for detector in camera: 

700 detectorId = detector.getId() 

701 key = (ref.dataId['visit'], detectorId) 

702 if key not in outputs: 

703 # skip detectors we don't have output for 

704 self.log.debug("No %s output for detector %s in visit %s", 

705 setter[3:], detectorId, visit) 

706 continue 

707 

708 catalog[i].setId(detectorId) 

709 getattr(catalog[i], setter)(outputs[key]) 

710 i += 1 

711 

712 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups 

713 butlerQC.put(catalog, ref) 

714 self.log.info("Wrote %s detectors to %s", i, ref) 

715 

716 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None): 

717 # Docstring inherited. 

718 

719 # We take values out of the Parquet table, and put them in "flux_", 

720 # and the config.sourceFluxType field is used during that extraction, 

721 # so just use "flux" here. 

722 sourceFluxField = "flux" 

723 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

724 associations = lsst.jointcal.Associations() 

725 self.focalPlaneBBox = inputCamera.getFpBBox() 

726 oldWcsList, bands = self._load_data(inputSourceTableVisit, 

727 inputVisitSummary, 

728 associations, 

729 jointcalControl, 

730 inputCamera) 

731 

732 boundingCircle, center, radius, defaultFilter = self._prep_sky(associations, bands) 

733 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

734 

735 if self.config.doAstrometry: 

736 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

737 name="astrometry", 

738 refObjLoader=self.astrometryRefObjLoader, 

739 referenceSelector=self.astrometryReferenceSelector, 

740 fit_function=self._fit_astrometry, 

741 tract=tract, 

742 epoch=epoch) 

743 astrometry_output = self._make_output(associations.getCcdImageList(), 

744 astrometry.model, 

745 "makeSkyWcs") 

746 else: 

747 astrometry_output = None 

748 

749 if self.config.doPhotometry: 

750 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

751 name="photometry", 

752 refObjLoader=self.photometryRefObjLoader, 

753 referenceSelector=self.photometryReferenceSelector, 

754 fit_function=self._fit_photometry, 

755 tract=tract, 

756 epoch=epoch, 

757 reject_bad_fluxes=True) 

758 photometry_output = self._make_output(associations.getCcdImageList(), 

759 photometry.model, 

760 "toPhotoCalib") 

761 else: 

762 photometry_output = None 

763 

764 return pipeBase.Struct(outputWcs=astrometry_output, 

765 outputPhotoCalib=photometry_output, 

766 job=self.job, 

767 astrometryRefObjLoader=self.astrometryRefObjLoader, 

768 photometryRefObjLoader=self.photometryRefObjLoader) 

769 

770 def _make_schema_table(self): 

771 """Return an afw SourceTable to use as a base for creating the 

772 SourceCatalog to insert values from the dataFrame into. 

773 

774 Returns 

775 ------- 

776 table : `lsst.afw.table.SourceTable` 

777 Table with schema and slots to use to make SourceCatalogs. 

778 """ 

779 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

780 schema.addField("centroid_x", "D") 

781 schema.addField("centroid_y", "D") 

782 schema.addField("centroid_xErr", "F") 

783 schema.addField("centroid_yErr", "F") 

784 schema.addField("shape_xx", "D") 

785 schema.addField("shape_yy", "D") 

786 schema.addField("shape_xy", "D") 

787 schema.addField("flux_instFlux", "D") 

788 schema.addField("flux_instFluxErr", "D") 

789 table = lsst.afw.table.SourceTable.make(schema) 

790 table.defineCentroid("centroid") 

791 table.defineShape("shape") 

792 return table 

793 

794 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId): 

795 """Return an afw SourceCatalog extracted from a visit-level dataframe, 

796 limited to just one detector. 

797 

798 Parameters 

799 ---------- 

800 table : `lsst.afw.table.SourceTable` 

801 Table factory to use to make the SourceCatalog that will be 

802 populated with data from ``visitCatalog``. 

803 visitCatalog : `pandas.DataFrame` 

804 DataFrame to extract a detector catalog from. 

805 detectorId : `int` 

806 Numeric id of the detector to extract from ``visitCatalog``. 

807 

808 Returns 

809 ------- 

810 catalog : `lsst.afw.table.SourceCatalog` 

811 Detector-level catalog extracted from ``visitCatalog``. 

812 """ 

813 # map from dataFrame column to afw table column 

814 mapping = {'sourceId': 'id', 

815 'x': 'centroid_x', 

816 'y': 'centroid_y', 

817 'xErr': 'centroid_xErr', 

818 'yErr': 'centroid_yErr', 

819 'Ixx': 'shape_xx', 

820 'Iyy': 'shape_yy', 

821 'Ixy': 'shape_xy', 

822 f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux', 

823 f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr', 

824 } 

825 # If the DataFrame we're reading was generated by a task running with 

826 # Gen2 middleware, the column for the detector will be "ccd" for at 

827 # least HSC (who knows what it might be in general!); that would be 

828 # true even if the data repo is later converted to Gen3, because that 

829 # doesn't touch the files themselves. In Gen3, the column for the 

830 # detector will always be "detector". 

831 detector_column = "detector" if "detector" in visitCatalog.columns else "ccd" 

832 catalog = lsst.afw.table.SourceCatalog(table) 

833 matched = visitCatalog[detector_column] == detectorId 

834 catalog.resize(sum(matched)) 

835 view = visitCatalog.loc[matched] 

836 for dfCol, afwCol in mapping.items(): 

837 catalog[afwCol] = view[dfCol] 

838 

839 self.log.debug("%d sources selected in visit %d detector %d", 

840 len(catalog), 

841 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first 

842 detectorId) 

843 return catalog 

844 

845 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, 

846 jointcalControl, camera): 

847 """Read the data that jointcal needs to run. (Gen3 version) 

848 

849 Modifies ``associations`` in-place with the loaded data. 

850 

851 Parameters 

852 ---------- 

853 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

854 References to visit-level DataFrames to load the catalog data from. 

855 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

856 Visit-level exposure summary catalog with metadata. 

857 associations : `lsst.jointcal.Associations` 

858 Object to add the loaded data to by constructing new CcdImages. 

859 jointcalControl : `jointcal.JointcalControl` 

860 Control object for C++ associations management. 

861 camera : `lsst.afw.cameraGeom.Camera` 

862 Camera object for detector geometry. 

863 

864 Returns 

865 ------- 

866 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`] 

867 The original WCS of the input data, to aid in writing tests. 

868 bands : `list` [`str`] 

869 The filter bands of each input dataset. 

870 """ 

871 oldWcsList = [] 

872 filters = [] 

873 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else '' 

874 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

875 table = self._make_schema_table() # every detector catalog has the same layout 

876 # No guarantee that the input is in the same order of visits, so we have to map one of them. 

877 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)} 

878 detectorDict = {detector.getId(): detector for detector in camera} 

879 

880 for visitSummaryRef in inputVisitSummary: 

881 visitSummary = visitSummaryRef.get() 

882 visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]].get() 

883 selected = self.sourceSelector.run(visitCatalog) 

884 

885 # Build a CcdImage for each detector in this visit. 

886 detectors = {id: index for index, id in enumerate(visitSummary['id'])} 

887 for id, index in detectors.items(): 

888 catalog = self._extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id) 

889 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict) 

890 result = self._build_ccdImage(data, associations, jointcalControl) 

891 if result is not None: 

892 oldWcsList.append(result.wcs) 

893 # A visit has only one band, so we can just use the first. 

894 filters.append(data.filter) 

895 if len(filters) == 0: 

896 raise RuntimeError("No data to process: did source selector remove all sources?") 

897 filters = collections.Counter(filters) 

898 

899 return oldWcsList, filters 

900 

901 def _make_one_input_data(self, visitRecord, catalog, detectorDict): 

902 """Return a data structure for this detector+visit.""" 

903 return JointcalInputData(visit=visitRecord['visit'], 

904 catalog=catalog, 

905 visitInfo=visitRecord.getVisitInfo(), 

906 detector=detectorDict[visitRecord.getId()], 

907 photoCalib=visitRecord.getPhotoCalib(), 

908 wcs=visitRecord.getWcs(), 

909 bbox=visitRecord.getBBox(), 

910 # ExposureRecord doesn't have a FilterLabel yet, 

911 # so we have to make one. 

912 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'], 

913 physical=visitRecord['physical_filter'])) 

914 

915 # We don't currently need to persist the metadata. 

916 # If we do in the future, we will have to add appropriate dataset templates 

917 # to each obs package (the metadata template should look like `jointcal_wcs`). 

918 def _getMetadataName(self): 

919 return None 

920 

921 @classmethod 

922 def _makeArgumentParser(cls): 

923 """Create an argument parser""" 

924 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

925 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9", 

926 ContainerClass=PerTractCcdDataIdContainer) 

927 return parser 

928 

929 def _build_ccdImage(self, data, associations, jointcalControl): 

930 """ 

931 Extract the necessary things from this catalog+metadata to add a new 

932 ccdImage. 

933 

934 Parameters 

935 ---------- 

936 data : `JointcalInputData` 

937 The loaded input data. 

938 associations : `lsst.jointcal.Associations` 

939 Object to add the info to, to construct a new CcdImage 

940 jointcalControl : `jointcal.JointcalControl` 

941 Control object for associations management 

942 

943 Returns 

944 ------ 

945 namedtuple or `None` 

946 ``wcs`` 

947 The TAN WCS of this image, read from the calexp 

948 (`lsst.afw.geom.SkyWcs`). 

949 ``key`` 

950 A key to identify this dataRef by its visit and ccd ids 

951 (`namedtuple`). 

952 `None` 

953 if there are no sources in the loaded catalog. 

954 """ 

955 if len(data.catalog) == 0: 

956 self.log.warn("No sources selected in visit %s ccd %s", data.visit, data.detector.getId()) 

957 return None 

958 

959 associations.createCcdImage(data.catalog, 

960 data.wcs, 

961 data.visitInfo, 

962 data.bbox, 

963 data.filter.physicalLabel, 

964 data.photoCalib, 

965 data.detector, 

966 data.visit, 

967 data.detector.getId(), 

968 jointcalControl) 

969 

970 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key')) 

971 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

972 return Result(data.wcs, Key(data.visit, data.detector.getId())) 

973 

974 def _readDataId(self, butler, dataId): 

975 """Read all of the data for one dataId from the butler. (gen2 version)""" 

976 # Not all instruments have `visit` in their dataIds. 

977 if "visit" in dataId.keys(): 

978 visit = dataId["visit"] 

979 else: 

980 visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0] 

981 detector = butler.get('calexp_detector', dataId=dataId) 

982 

983 catalog = butler.get('src', 

984 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, 

985 dataId=dataId) 

986 goodSrc = self.sourceSelector.run(catalog) 

987 self.log.debug("%d sources selected in visit %d detector %d", 

988 len(goodSrc.sourceCat), 

989 visit, 

990 detector.getId()) 

991 return JointcalInputData(visit=visit, 

992 catalog=goodSrc.sourceCat, 

993 visitInfo=butler.get('calexp_visitInfo', dataId=dataId), 

994 detector=detector, 

995 photoCalib=butler.get('calexp_photoCalib', dataId=dataId), 

996 wcs=butler.get('calexp_wcs', dataId=dataId), 

997 bbox=butler.get('calexp_bbox', dataId=dataId), 

998 filter=butler.get('calexp_filterLabel', dataId=dataId)) 

999 

1000 def loadData(self, dataRefs, associations, jointcalControl): 

1001 """Read the data that jointcal needs to run. (Gen2 version)""" 

1002 visit_ccd_to_dataRef = {} 

1003 oldWcsList = [] 

1004 filters = [] 

1005 load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else '' 

1006 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

1007 # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models 

1008 camera = dataRefs[0].get('camera', immediate=True) 

1009 self.focalPlaneBBox = camera.getFpBBox() 

1010 for dataRef in dataRefs: 

1011 data = self._readDataId(dataRef.getButler(), dataRef.dataId) 

1012 result = self._build_ccdImage(data, associations, jointcalControl) 

1013 if result is None: 

1014 continue 

1015 oldWcsList.append(result.wcs) 

1016 visit_ccd_to_dataRef[result.key] = dataRef 

1017 filters.append(data.filter) 

1018 if len(filters) == 0: 

1019 raise RuntimeError("No data to process: did source selector remove all sources?") 

1020 filters = collections.Counter(filters) 

1021 

1022 return oldWcsList, filters, visit_ccd_to_dataRef 

1023 

1024 def _getDebugPath(self, filename): 

1025 """Constructs a path to filename using the configured debug path. 

1026 """ 

1027 return os.path.join(self.config.debugOutputPath, filename) 

1028 

1029 def _prep_sky(self, associations, filters): 

1030 """Prepare on-sky and other data that must be computed after data has 

1031 been read. 

1032 """ 

1033 associations.computeCommonTangentPoint() 

1034 

1035 boundingCircle = associations.computeBoundingCircle() 

1036 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

1037 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

1038 

1039 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

1040 

1041 # Determine a default filter band associated with the catalog. See DM-9093 

1042 defaultFilter = filters.most_common(1)[0][0] 

1043 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel) 

1044 

1045 return boundingCircle, center, radius, defaultFilter 

1046 

1047 @pipeBase.timeMethod 

1048 def runDataRef(self, dataRefs): 

1049 """ 

1050 Jointly calibrate the astrometry and photometry across a set of images. 

1051 

1052 NOTE: this is for gen2 middleware only. 

1053 

1054 Parameters 

1055 ---------- 

1056 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

1057 List of data references to the exposures to be fit. 

1058 

1059 Returns 

1060 ------- 

1061 result : `lsst.pipe.base.Struct` 

1062 Struct of metadata from the fit, containing: 

1063 

1064 ``dataRefs`` 

1065 The provided data references that were fit (with updated WCSs) 

1066 ``oldWcsList`` 

1067 The original WCS from each dataRef 

1068 ``metrics`` 

1069 Dictionary of internally-computed metrics for testing/validation. 

1070 """ 

1071 if len(dataRefs) == 0: 

1072 raise ValueError('Need a non-empty list of data references!') 

1073 

1074 exitStatus = 0 # exit status for shell 

1075 

1076 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,) 

1077 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

1078 associations = lsst.jointcal.Associations() 

1079 

1080 oldWcsList, filters, visit_ccd_to_dataRef = self.loadData(dataRefs, 

1081 associations, 

1082 jointcalControl) 

1083 

1084 boundingCircle, center, radius, defaultFilter = self._prep_sky(associations, filters) 

1085 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

1086 

1087 tract = dataRefs[0].dataId['tract'] 

1088 

1089 if self.config.doAstrometry: 

1090 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

1091 name="astrometry", 

1092 refObjLoader=self.astrometryRefObjLoader, 

1093 referenceSelector=self.astrometryReferenceSelector, 

1094 fit_function=self._fit_astrometry, 

1095 tract=tract, 

1096 epoch=epoch) 

1097 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef) 

1098 else: 

1099 astrometry = Astrometry(None, None, None) 

1100 

1101 if self.config.doPhotometry: 

1102 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

1103 name="photometry", 

1104 refObjLoader=self.photometryRefObjLoader, 

1105 referenceSelector=self.photometryReferenceSelector, 

1106 fit_function=self._fit_photometry, 

1107 tract=tract, 

1108 epoch=epoch, 

1109 reject_bad_fluxes=True) 

1110 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef) 

1111 else: 

1112 photometry = Photometry(None, None) 

1113 

1114 return pipeBase.Struct(dataRefs=dataRefs, 

1115 oldWcsList=oldWcsList, 

1116 job=self.job, 

1117 astrometryRefObjLoader=self.astrometryRefObjLoader, 

1118 photometryRefObjLoader=self.photometryRefObjLoader, 

1119 defaultFilter=defaultFilter, 

1120 epoch=epoch, 

1121 exitStatus=exitStatus) 

1122 

1123 def _get_refcat_coordinate_error_override(self, refCat, name): 

1124 """Check whether we should override the refcat coordinate errors, and 

1125 return the overridden error if necessary. 

1126 

1127 Parameters 

1128 ---------- 

1129 refCat : `lsst.afw.table.SimpleCatalog` 

1130 The reference catalog to check for a ``coord_raErr`` field. 

1131 name : `str` 

1132 Whether we are doing "astrometry" or "photometry". 

1133 

1134 Returns 

1135 ------- 

1136 refCoordErr : `float` 

1137 The refcat coordinate error to use, or NaN if we are not overriding 

1138 those fields. 

1139 

1140 Raises 

1141 ------ 

1142 lsst.pex.config.FieldValidationError 

1143 Raised if the refcat does not contain coordinate errors and 

1144 ``config.astrometryReferenceErr`` is not set. 

1145 """ 

1146 # This value doesn't matter for photometry, so just set something to 

1147 # keep old refcats from causing problems. 

1148 if name.lower() == "photometry": 

1149 if 'coord_raErr' not in refCat.schema: 

1150 return 100 

1151 else: 

1152 return float('nan') 

1153 

1154 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

1155 msg = ("Reference catalog does not contain coordinate errors, " 

1156 "and config.astrometryReferenceErr not supplied.") 

1157 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

1158 self.config, 

1159 msg) 

1160 

1161 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

1162 self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

1163 self.config.astrometryReferenceErr) 

1164 

1165 if self.config.astrometryReferenceErr is None: 

1166 return float('nan') 

1167 else: 

1168 return self.config.astrometryReferenceErr 

1169 

1170 def _compute_proper_motion_epoch(self, ccdImageList): 

1171 """Return the proper motion correction epoch of the provided images. 

1172 

1173 Parameters 

1174 ---------- 

1175 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

1176 The images to compute the appropriate epoch for. 

1177 

1178 Returns 

1179 ------- 

1180 epoch : `astropy.time.Time` 

1181 The date to use for proper motion corrections. 

1182 """ 

1183 mjds = [ccdImage.getMjd() for ccdImage in ccdImageList] 

1184 return astropy.time.Time(np.mean(mjds), format='mjd', scale="tai") 

1185 

1186 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

1187 tract="", match_cut=3.0, 

1188 reject_bad_fluxes=False, *, 

1189 name="", refObjLoader=None, referenceSelector=None, 

1190 fit_function=None, epoch=None): 

1191 """Load reference catalog, perform the fit, and return the result. 

1192 

1193 Parameters 

1194 ---------- 

1195 associations : `lsst.jointcal.Associations` 

1196 The star/reference star associations to fit. 

1197 defaultFilter : `lsst.afw.image.FilterLabel` 

1198 filter to load from reference catalog. 

1199 center : `lsst.geom.SpherePoint` 

1200 ICRS center of field to load from reference catalog. 

1201 radius : `lsst.geom.Angle` 

1202 On-sky radius to load from reference catalog. 

1203 name : `str` 

1204 Name of thing being fit: "astrometry" or "photometry". 

1205 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

1206 Reference object loader to use to load a reference catalog. 

1207 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1208 Selector to use to pick objects from the loaded reference catalog. 

1209 fit_function : callable 

1210 Function to call to perform fit (takes Associations object). 

1211 tract : `str`, optional 

1212 Name of tract currently being fit. 

1213 match_cut : `float`, optional 

1214 Radius in arcseconds to find cross-catalog matches to during 

1215 associations.associateCatalogs. 

1216 reject_bad_fluxes : `bool`, optional 

1217 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

1218 epoch : `astropy.time.Time`, optional 

1219 Epoch to which to correct refcat proper motion and parallax, 

1220 or `None` to not apply such corrections. 

1221 

1222 Returns 

1223 ------- 

1224 result : `Photometry` or `Astrometry` 

1225 Result of `fit_function()` 

1226 """ 

1227 self.log.info("====== Now processing %s...", name) 

1228 # TODO: this should not print "trying to invert a singular transformation:" 

1229 # if it does that, something's not right about the WCS... 

1230 associations.associateCatalogs(match_cut) 

1231 add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name, 

1232 associations.fittedStarListSize()) 

1233 

1234 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

1235 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

1236 center, radius, defaultFilter, 

1237 applyColorterms=applyColorterms, 

1238 epoch=epoch) 

1239 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

1240 

1241 associations.collectRefStars(refCat, 

1242 self.config.matchCut*lsst.geom.arcseconds, 

1243 fluxField, 

1244 refCoordinateErr=refCoordErr, 

1245 rejectBadFluxes=reject_bad_fluxes) 

1246 add_measurement(self.job, 'jointcal.collected_%s_refStars' % name, 

1247 associations.refStarListSize()) 

1248 

1249 associations.prepareFittedStars(self.config.minMeasurements) 

1250 

1251 self._check_star_lists(associations, name) 

1252 add_measurement(self.job, 'jointcal.selected_%s_refStars' % name, 

1253 associations.nFittedStarsWithAssociatedRefStar()) 

1254 add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name, 

1255 associations.fittedStarListSize()) 

1256 add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name, 

1257 associations.nCcdImagesValidForFit()) 

1258 

1259 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else '' 

1260 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel) 

1261 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

1262 result = fit_function(associations, dataName) 

1263 # TODO DM-12446: turn this into a "butler save" somehow. 

1264 # Save reference and measurement chi2 contributions for this data 

1265 if self.config.writeChi2FilesInitialFinal: 

1266 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

1267 result.fit.saveChi2Contributions(baseName+"{type}") 

1268 self.log.info("Wrote chi2 contributions files: %s", baseName) 

1269 

1270 return result 

1271 

1272 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, 

1273 applyColorterms=False, epoch=None): 

1274 """Load the necessary reference catalog sources, convert fluxes to 

1275 correct units, and apply color term corrections if requested. 

1276 

1277 Parameters 

1278 ---------- 

1279 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

1280 The reference catalog loader to use to get the data. 

1281 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1282 Source selector to apply to loaded reference catalog. 

1283 center : `lsst.geom.SpherePoint` 

1284 The center around which to load sources. 

1285 radius : `lsst.geom.Angle` 

1286 The radius around ``center`` to load sources in. 

1287 filterLabel : `lsst.afw.image.FilterLabel` 

1288 The camera filter to load fluxes for. 

1289 applyColorterms : `bool` 

1290 Apply colorterm corrections to the refcat for ``filterName``? 

1291 epoch : `astropy.time.Time`, optional 

1292 Epoch to which to correct refcat proper motion and parallax, 

1293 or `None` to not apply such corrections. 

1294 

1295 Returns 

1296 ------- 

1297 refCat : `lsst.afw.table.SimpleCatalog` 

1298 The loaded reference catalog. 

1299 fluxField : `str` 

1300 The name of the reference catalog flux field appropriate for ``filterName``. 

1301 """ 

1302 skyCircle = refObjLoader.loadSkyCircle(center, 

1303 radius, 

1304 filterLabel.bandLabel, 

1305 epoch=epoch) 

1306 

1307 selected = referenceSelector.run(skyCircle.refCat) 

1308 # Need memory contiguity to get reference filters as a vector. 

1309 if not selected.sourceCat.isContiguous(): 

1310 refCat = selected.sourceCat.copy(deep=True) 

1311 else: 

1312 refCat = selected.sourceCat 

1313 

1314 if applyColorterms: 

1315 refCatName = refObjLoader.ref_dataset_name 

1316 self.log.info("Applying color terms for physical filter=%r reference catalog=%s", 

1317 filterLabel.physicalLabel, refCatName) 

1318 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel, 

1319 refCatName, 

1320 doRaise=True) 

1321 

1322 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

1323 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

1324 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

1325 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

1326 

1327 return refCat, skyCircle.fluxField 

1328 

1329 def _check_star_lists(self, associations, name): 

1330 # TODO: these should be len(blah), but we need this properly wrapped first. 

1331 if associations.nCcdImagesValidForFit() == 0: 

1332 raise RuntimeError('No images in the ccdImageList!') 

1333 if associations.fittedStarListSize() == 0: 

1334 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

1335 if associations.refStarListSize() == 0: 

1336 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

1337 

1338 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

1339 """Compute chi2, log it, validate the model, and return chi2. 

1340 

1341 Parameters 

1342 ---------- 

1343 associations : `lsst.jointcal.Associations` 

1344 The star/reference star associations to fit. 

1345 fit : `lsst.jointcal.FitterBase` 

1346 The fitter to use for minimization. 

1347 model : `lsst.jointcal.Model` 

1348 The model being fit. 

1349 chi2Label : `str` 

1350 Label to describe the chi2 (e.g. "Initialized", "Final"). 

1351 writeChi2Name : `str`, optional 

1352 Filename prefix to write the chi2 contributions to. 

1353 Do not supply an extension: an appropriate one will be added. 

1354 

1355 Returns 

1356 ------- 

1357 chi2: `lsst.jointcal.Chi2Accumulator` 

1358 The chi2 object for the current fitter and model. 

1359 

1360 Raises 

1361 ------ 

1362 FloatingPointError 

1363 Raised if chi2 is infinite or NaN. 

1364 ValueError 

1365 Raised if the model is not valid. 

1366 """ 

1367 if writeChi2Name is not None: 

1368 fullpath = self._getDebugPath(writeChi2Name) 

1369 fit.saveChi2Contributions(fullpath+"{type}") 

1370 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

1371 

1372 chi2 = fit.computeChi2() 

1373 self.log.info("%s %s", chi2Label, chi2) 

1374 self._check_stars(associations) 

1375 if not np.isfinite(chi2.chi2): 

1376 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

1377 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

1378 raise ValueError("Model is not valid: check log messages for warnings.") 

1379 return chi2 

1380 

1381 def _fit_photometry(self, associations, dataName=None): 

1382 """ 

1383 Fit the photometric data. 

1384 

1385 Parameters 

1386 ---------- 

1387 associations : `lsst.jointcal.Associations` 

1388 The star/reference star associations to fit. 

1389 dataName : `str` 

1390 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1391 identifying debugging files. 

1392 

1393 Returns 

1394 ------- 

1395 fit_result : `namedtuple` 

1396 fit : `lsst.jointcal.PhotometryFit` 

1397 The photometric fitter used to perform the fit. 

1398 model : `lsst.jointcal.PhotometryModel` 

1399 The photometric model that was fit. 

1400 """ 

1401 self.log.info("=== Starting photometric fitting...") 

1402 

1403 # TODO: should use pex.config.RegistryField here (see DM-9195) 

1404 if self.config.photometryModel == "constrainedFlux": 

1405 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

1406 self.focalPlaneBBox, 

1407 visitOrder=self.config.photometryVisitOrder, 

1408 errorPedestal=self.config.photometryErrorPedestal) 

1409 # potentially nonlinear problem, so we may need a line search to converge. 

1410 doLineSearch = self.config.allowLineSearch 

1411 elif self.config.photometryModel == "constrainedMagnitude": 

1412 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

1413 self.focalPlaneBBox, 

1414 visitOrder=self.config.photometryVisitOrder, 

1415 errorPedestal=self.config.photometryErrorPedestal) 

1416 # potentially nonlinear problem, so we may need a line search to converge. 

1417 doLineSearch = self.config.allowLineSearch 

1418 elif self.config.photometryModel == "simpleFlux": 

1419 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

1420 errorPedestal=self.config.photometryErrorPedestal) 

1421 doLineSearch = False # purely linear in model parameters, so no line search needed 

1422 elif self.config.photometryModel == "simpleMagnitude": 

1423 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

1424 errorPedestal=self.config.photometryErrorPedestal) 

1425 doLineSearch = False # purely linear in model parameters, so no line search needed 

1426 

1427 fit = lsst.jointcal.PhotometryFit(associations, model) 

1428 # TODO DM-12446: turn this into a "butler save" somehow. 

1429 # Save reference and measurement chi2 contributions for this data 

1430 if self.config.writeChi2FilesInitialFinal: 

1431 baseName = f"photometry_initial_chi2-{dataName}" 

1432 else: 

1433 baseName = None 

1434 if self.config.writeInitialModel: 

1435 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt") 

1436 writeModel(model, fullpath, self.log) 

1437 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

1438 

1439 def getChi2Name(whatToFit): 

1440 if self.config.writeChi2FilesOuterLoop: 

1441 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

1442 else: 

1443 return None 

1444 

1445 # The constrained model needs the visit transform fit first; the chip 

1446 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

1447 if self.config.writeInitMatrix: 

1448 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}") 

1449 else: 

1450 dumpMatrixFile = "" 

1451 if self.config.photometryModel.startswith("constrained"): 

1452 # no line search: should be purely (or nearly) linear, 

1453 # and we want a large step size to initialize with. 

1454 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

1455 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

1456 writeChi2Name=getChi2Name("ModelVisit")) 

1457 dumpMatrixFile = "" # so we don't redo the output on the next step 

1458 

1459 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

1460 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

1461 writeChi2Name=getChi2Name("Model")) 

1462 

1463 fit.minimize("Fluxes") # no line search: always purely linear. 

1464 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

1465 writeChi2Name=getChi2Name("Fluxes")) 

1466 

1467 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

1468 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

1469 writeChi2Name=getChi2Name("ModelFluxes")) 

1470 

1471 model.freezeErrorTransform() 

1472 self.log.debug("Photometry error scales are frozen.") 

1473 

1474 chi2 = self._iterate_fit(associations, 

1475 fit, 

1476 self.config.maxPhotometrySteps, 

1477 "photometry", 

1478 "Model Fluxes", 

1479 doRankUpdate=self.config.photometryDoRankUpdate, 

1480 doLineSearch=doLineSearch, 

1481 dataName=dataName) 

1482 

1483 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

1484 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

1485 return Photometry(fit, model) 

1486 

1487 def _fit_astrometry(self, associations, dataName=None): 

1488 """ 

1489 Fit the astrometric data. 

1490 

1491 Parameters 

1492 ---------- 

1493 associations : `lsst.jointcal.Associations` 

1494 The star/reference star associations to fit. 

1495 dataName : `str` 

1496 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1497 identifying debugging files. 

1498 

1499 Returns 

1500 ------- 

1501 fit_result : `namedtuple` 

1502 fit : `lsst.jointcal.AstrometryFit` 

1503 The astrometric fitter used to perform the fit. 

1504 model : `lsst.jointcal.AstrometryModel` 

1505 The astrometric model that was fit. 

1506 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

1507 The model for the sky to tangent plane projection that was used in the fit. 

1508 """ 

1509 

1510 self.log.info("=== Starting astrometric fitting...") 

1511 

1512 associations.deprojectFittedStars() 

1513 

1514 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1515 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1516 # them so carefully? 

1517 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1518 

1519 if self.config.astrometryModel == "constrained": 

1520 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1521 sky_to_tan_projection, 

1522 chipOrder=self.config.astrometryChipOrder, 

1523 visitOrder=self.config.astrometryVisitOrder) 

1524 elif self.config.astrometryModel == "simple": 

1525 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1526 sky_to_tan_projection, 

1527 self.config.useInputWcs, 

1528 nNotFit=0, 

1529 order=self.config.astrometrySimpleOrder) 

1530 

1531 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1532 # TODO DM-12446: turn this into a "butler save" somehow. 

1533 # Save reference and measurement chi2 contributions for this data 

1534 if self.config.writeChi2FilesInitialFinal: 

1535 baseName = f"astrometry_initial_chi2-{dataName}" 

1536 else: 

1537 baseName = None 

1538 if self.config.writeInitialModel: 

1539 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt") 

1540 writeModel(model, fullpath, self.log) 

1541 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1542 

1543 def getChi2Name(whatToFit): 

1544 if self.config.writeChi2FilesOuterLoop: 

1545 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1546 else: 

1547 return None 

1548 

1549 if self.config.writeInitMatrix: 

1550 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}") 

1551 else: 

1552 dumpMatrixFile = "" 

1553 # The constrained model needs the visit transform fit first; the chip 

1554 # transform is initialized from the detector's cameraGeom, so it's close. 

1555 if self.config.astrometryModel == "constrained": 

1556 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1557 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1558 writeChi2Name=getChi2Name("DistortionsVisit")) 

1559 dumpMatrixFile = "" # so we don't redo the output on the next step 

1560 

1561 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1562 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1563 writeChi2Name=getChi2Name("Distortions")) 

1564 

1565 fit.minimize("Positions") 

1566 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1567 writeChi2Name=getChi2Name("Positions")) 

1568 

1569 fit.minimize("Distortions Positions") 

1570 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1571 writeChi2Name=getChi2Name("DistortionsPositions")) 

1572 

1573 chi2 = self._iterate_fit(associations, 

1574 fit, 

1575 self.config.maxAstrometrySteps, 

1576 "astrometry", 

1577 "Distortions Positions", 

1578 doRankUpdate=self.config.astrometryDoRankUpdate, 

1579 dataName=dataName) 

1580 

1581 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1582 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1583 

1584 return Astrometry(fit, model, sky_to_tan_projection) 

1585 

1586 def _check_stars(self, associations): 

1587 """Count measured and reference stars per ccd and warn/log them.""" 

1588 for ccdImage in associations.getCcdImageList(): 

1589 nMeasuredStars, nRefStars = ccdImage.countStars() 

1590 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1591 ccdImage.getName(), nMeasuredStars, nRefStars) 

1592 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1593 self.log.warn("ccdImage %s has only %s measuredStars (desired %s)", 

1594 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1595 if nRefStars < self.config.minRefStarsPerCcd: 

1596 self.log.warn("ccdImage %s has only %s RefStars (desired %s)", 

1597 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1598 

1599 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1600 dataName="", 

1601 doRankUpdate=True, 

1602 doLineSearch=False): 

1603 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1604 

1605 Parameters 

1606 ---------- 

1607 associations : `lsst.jointcal.Associations` 

1608 The star/reference star associations to fit. 

1609 fitter : `lsst.jointcal.FitterBase` 

1610 The fitter to use for minimization. 

1611 max_steps : `int` 

1612 Maximum number of steps to run outlier rejection before declaring 

1613 convergence failure. 

1614 name : {'photometry' or 'astrometry'} 

1615 What type of data are we fitting (for logs and debugging files). 

1616 whatToFit : `str` 

1617 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1618 dataName : `str`, optional 

1619 Descriptive name for this dataset (e.g. tract and filter), 

1620 for debugging. 

1621 doRankUpdate : `bool`, optional 

1622 Do an Eigen rank update during minimization, or recompute the full 

1623 matrix and gradient? 

1624 doLineSearch : `bool`, optional 

1625 Do a line search for the optimum step during minimization? 

1626 

1627 Returns 

1628 ------- 

1629 chi2: `lsst.jointcal.Chi2Statistic` 

1630 The final chi2 after the fit converges, or is forced to end. 

1631 

1632 Raises 

1633 ------ 

1634 FloatingPointError 

1635 Raised if the fitter fails with a non-finite value. 

1636 RuntimeError 

1637 Raised if the fitter fails for some other reason; 

1638 log messages will provide further details. 

1639 """ 

1640 if self.config.writeInitMatrix: 

1641 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}") 

1642 else: 

1643 dumpMatrixFile = "" 

1644 oldChi2 = lsst.jointcal.Chi2Statistic() 

1645 oldChi2.chi2 = float("inf") 

1646 for i in range(max_steps): 

1647 if self.config.writeChi2FilesOuterLoop: 

1648 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1649 else: 

1650 writeChi2Name = None 

1651 result = fitter.minimize(whatToFit, 

1652 self.config.outlierRejectSigma, 

1653 doRankUpdate=doRankUpdate, 

1654 doLineSearch=doLineSearch, 

1655 dumpMatrixFile=dumpMatrixFile) 

1656 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1657 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1658 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1659 

1660 if result == MinimizeResult.Converged: 

1661 if doRankUpdate: 

1662 self.log.debug("fit has converged - no more outliers - redo minimization " 

1663 "one more time in case we have lost accuracy in rank update.") 

1664 # Redo minimization one more time in case we have lost accuracy in rank update 

1665 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma) 

1666 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1667 

1668 # log a message for a large final chi2, TODO: DM-15247 for something better 

1669 if chi2.chi2/chi2.ndof >= 4.0: 

1670 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1671 

1672 break 

1673 elif result == MinimizeResult.Chi2Increased: 

1674 self.log.warn("Still some outliers remaining but chi2 increased - retry") 

1675 # Check whether the increase was large enough to cause trouble. 

1676 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1677 if chi2Ratio > 1.5: 

1678 self.log.warn('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1679 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1680 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1681 # appears that chi2 increases more than a factor of ~2 always 

1682 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1683 # Using 10 as the "failure" threshold gives some room between 

1684 # leaving a warning and bailing early. 

1685 if chi2Ratio > 10: 

1686 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1687 " Try setting one or more of the `writeChi2*` config fields and looking" 

1688 " at how individual star chi2-values evolve during the fit.") 

1689 raise RuntimeError(msg) 

1690 oldChi2 = chi2 

1691 elif result == MinimizeResult.NonFinite: 

1692 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1693 # TODO DM-12446: turn this into a "butler save" somehow. 

1694 fitter.saveChi2Contributions(filename+"{type}") 

1695 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1696 raise FloatingPointError(msg.format(filename)) 

1697 elif result == MinimizeResult.Failed: 

1698 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1699 else: 

1700 raise RuntimeError("Unxepected return code from minimize().") 

1701 else: 

1702 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1703 

1704 return chi2 

1705 

1706 def _make_output(self, ccdImageList, model, func): 

1707 """Return the internal jointcal models converted to the afw 

1708 structures that will be saved to disk. 

1709 

1710 Parameters 

1711 ---------- 

1712 ccdImageList : `lsst.jointcal.CcdImageList` 

1713 The list of CcdImages to get the output for. 

1714 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel` 

1715 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`. 

1716 func : `str` 

1717 The name of the function to call on ``model`` to get the converted 

1718 structure. Must accept an `lsst.jointcal.CcdImage`. 

1719 

1720 Returns 

1721 ------- 

1722 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or 

1723 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`] 

1724 The data to be saved, keyed on (visit, detector). 

1725 """ 

1726 output = {} 

1727 for ccdImage in ccdImageList: 

1728 ccd = ccdImage.ccdId 

1729 visit = ccdImage.visit 

1730 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd) 

1731 output[(visit, ccd)] = getattr(model, func)(ccdImage) 

1732 return output 

1733 

1734 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef): 

1735 """ 

1736 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 

1737 

1738 Parameters 

1739 ---------- 

1740 associations : `lsst.jointcal.Associations` 

1741 The star/reference star associations to fit. 

1742 model : `lsst.jointcal.AstrometryModel` 

1743 The astrometric model that was fit. 

1744 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1745 Dict of ccdImage identifiers to dataRefs that were fit. 

1746 """ 

1747 ccdImageList = associations.getCcdImageList() 

1748 output = self._make_output(ccdImageList, model, "makeSkyWcs") 

1749 for key, skyWcs in output.items(): 

1750 dataRef = visit_ccd_to_dataRef[key] 

1751 try: 

1752 dataRef.put(skyWcs, 'jointcal_wcs') 

1753 except pexExceptions.Exception as e: 

1754 self.log.fatal('Failed to write updated Wcs: %s', str(e)) 

1755 raise e 

1756 

1757 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef): 

1758 """ 

1759 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 

1760 

1761 Parameters 

1762 ---------- 

1763 associations : `lsst.jointcal.Associations` 

1764 The star/reference star associations to fit. 

1765 model : `lsst.jointcal.PhotometryModel` 

1766 The photoometric model that was fit. 

1767 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1768 Dict of ccdImage identifiers to dataRefs that were fit. 

1769 """ 

1770 

1771 ccdImageList = associations.getCcdImageList() 

1772 output = self._make_output(ccdImageList, model, "toPhotoCalib") 

1773 for key, photoCalib in output.items(): 

1774 dataRef = visit_ccd_to_dataRef[key] 

1775 try: 

1776 dataRef.put(photoCalib, 'jointcal_photoCalib') 

1777 except pexExceptions.Exception as e: 

1778 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e)) 

1779 raise e