Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import dataclasses 

23import collections 

24import os 

25 

26import astropy.time 

27import numpy as np 

28import astropy.units as u 

29 

30import lsst.geom 

31import lsst.utils 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34from lsst.afw.image import fluxErrFromABMagErr 

35import lsst.pex.exceptions as pexExceptions 

36import lsst.afw.cameraGeom 

37import lsst.afw.table 

38import lsst.log 

39from lsst.obs.base import Instrument 

40from lsst.pipe.tasks.colorterms import ColortermLibrary 

41from lsst.verify import Job, Measurement 

42 

43from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask, 

44 ReferenceObjectLoader) 

45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

46 

47from .dataIds import PerTractCcdDataIdContainer 

48 

49import lsst.jointcal 

50from lsst.jointcal import MinimizeResult 

51 

52__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"] 

53 

54Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

55Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

56 

57 

58# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

59def add_measurement(job, name, value): 

60 meas = Measurement(job.metrics[name], value) 

61 job.measurements.insert(meas) 

62 

63 

64class JointcalRunner(pipeBase.ButlerInitializedTaskRunner): 

65 """Subclass of TaskRunner for jointcalTask (gen2) 

66 

67 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 

68 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 

69 single dataRef, are are called repeatedly). This class transforms the processed 

70 arguments generated by the ArgumentParser into the arguments expected by 

71 Jointcal.runDataRef(). 

72 

73 See pipeBase.TaskRunner for more information. 

74 """ 

75 

76 @staticmethod 

77 def getTargetList(parsedCmd, **kwargs): 

78 """ 

79 Return a list of tuples per tract, each containing (dataRefs, kwargs). 

80 

81 Jointcal operates on lists of dataRefs simultaneously. 

82 """ 

83 kwargs['butler'] = parsedCmd.butler 

84 

85 # organize data IDs by tract 

86 refListDict = {} 

87 for ref in parsedCmd.id.refList: 

88 refListDict.setdefault(ref.dataId["tract"], []).append(ref) 

89 # we call runDataRef() once with each tract 

90 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())] 

91 return result 

92 

93 def __call__(self, args): 

94 """ 

95 Parameters 

96 ---------- 

97 args 

98 Arguments for Task.runDataRef() 

99 

100 Returns 

101 ------- 

102 pipe.base.Struct 

103 if self.doReturnResults is False: 

104 

105 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

106 

107 if self.doReturnResults is True: 

108 

109 - ``result``: the result of calling jointcal.runDataRef() 

110 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

111 """ 

112 exitStatus = 0 # exit status for shell 

113 

114 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef. 

115 dataRefList, kwargs = args 

116 butler = kwargs.pop('butler') 

117 task = self.TaskClass(config=self.config, log=self.log, butler=butler) 

118 result = None 

119 try: 

120 result = task.runDataRef(dataRefList, **kwargs) 

121 exitStatus = result.exitStatus 

122 job_path = butler.get('verify_job_filename') 

123 result.job.write(job_path[0]) 

124 except Exception as e: # catch everything, sort it out later. 

125 if self.doRaise: 

126 raise e 

127 else: 

128 exitStatus = 1 

129 eName = type(e).__name__ 

130 tract = dataRefList[0].dataId['tract'] 

131 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e) 

132 

133 # Put the butler back into kwargs for the other Tasks. 

134 kwargs['butler'] = butler 

135 if self.doReturnResults: 

136 return pipeBase.Struct(result=result, exitStatus=exitStatus) 

137 else: 

138 return pipeBase.Struct(exitStatus=exitStatus) 

139 

140 

141def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

142 """Lookup function that asserts/hopes that a static calibration dataset 

143 exists in a particular collection, since this task can't provide a single 

144 date/time to use to search for one properly. 

145 

146 This is mostly useful for the ``camera`` dataset, in cases where the task's 

147 quantum dimensions do *not* include something temporal, like ``exposure`` 

148 or ``visit``. 

149 

150 Parameters 

151 ---------- 

152 datasetType : `lsst.daf.butler.DatasetType` 

153 Type of dataset being searched for. 

154 registry : `lsst.daf.butler.Registry` 

155 Data repository registry to search. 

156 quantumDataId : `lsst.daf.butler.DataCoordinate` 

157 Data ID of the quantum this camera should match. 

158 collections : `Iterable` [ `str` ] 

159 Collections that should be searched - but this lookup function works 

160 by ignoring this in favor of a more-or-less hard-coded value. 

161 

162 Returns 

163 ------- 

164 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

165 Iterator over dataset references; should have only one element. 

166 

167 Notes 

168 ----- 

169 This implementation duplicates one in fgcmcal, and is at least quite 

170 similar to another in cp_pipe. This duplicate has the most documentation. 

171 Fixing this is DM-29661. 

172 """ 

173 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

174 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

175 return registry.queryDatasets(datasetType, 

176 dataId=quantumDataId, 

177 collections=[unboundedCollection], 

178 findFirst=True) 

179 

180 

181def lookupVisitRefCats(datasetType, registry, quantumDataId, collections): 

182 """Lookup function that finds all refcats for all visits that overlap a 

183 tract, rather than just the refcats that directly overlap the tract. 

184 

185 Parameters 

186 ---------- 

187 datasetType : `lsst.daf.butler.DatasetType` 

188 Type of dataset being searched for. 

189 registry : `lsst.daf.butler.Registry` 

190 Data repository registry to search. 

191 quantumDataId : `lsst.daf.butler.DataCoordinate` 

192 Data ID of the quantum; expected to be something we can use as a 

193 constraint to query for overlapping visits. 

194 collections : `Iterable` [ `str` ] 

195 Collections to search. 

196 

197 Returns 

198 ------- 

199 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

200 Iterator over refcat references. 

201 """ 

202 refs = set() 

203 # Use .expanded() on the query methods below because we need data IDs with 

204 # regions, both in the outer loop over visits (queryDatasets will expand 

205 # any data ID we give it, but doing it up-front in bulk is much more 

206 # efficient) and in the data IDs of the DatasetRefs this function yields 

207 # (because the RefCatLoader relies on them to do some of its own 

208 # filtering). 

209 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()): 

210 refs.update( 

211 registry.queryDatasets( 

212 datasetType, 

213 collections=collections, 

214 dataId=visit_data_id, 

215 findFirst=True, 

216 ).expanded() 

217 ) 

218 yield from refs 

219 

220 

221class JointcalTaskConnections(pipeBase.PipelineTaskConnections, 

222 dimensions=("skymap", "tract", "instrument", "physical_filter")): 

223 """Middleware input/output connections for jointcal data.""" 

224 inputCamera = pipeBase.connectionTypes.PrerequisiteInput( 

225 doc="The camera instrument that took these observations.", 

226 name="camera", 

227 storageClass="Camera", 

228 dimensions=("instrument",), 

229 isCalibration=True, 

230 lookupFunction=lookupStaticCalibrations, 

231 ) 

232 inputSourceTableVisit = pipeBase.connectionTypes.Input( 

233 doc="Source table in parquet format, per visit", 

234 name="sourceTable_visit", 

235 storageClass="DataFrame", 

236 dimensions=("instrument", "visit"), 

237 deferLoad=True, 

238 multiple=True, 

239 ) 

240 inputVisitSummary = pipeBase.connectionTypes.Input( 

241 doc=("Per-visit consolidated exposure metadata built from calexps. " 

242 "These catalogs use detector id for the id and must be sorted for " 

243 "fast lookups of a detector."), 

244 name="visitSummary", 

245 storageClass="ExposureCatalog", 

246 dimensions=("instrument", "visit"), 

247 deferLoad=True, 

248 multiple=True, 

249 ) 

250 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

251 doc="The astrometry reference catalog to match to loaded input catalog sources.", 

252 name="gaia_dr2_20200414", 

253 storageClass="SimpleCatalog", 

254 dimensions=("skypix",), 

255 deferLoad=True, 

256 multiple=True, 

257 lookupFunction=lookupVisitRefCats, 

258 ) 

259 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

260 doc="The photometry reference catalog to match to loaded input catalog sources.", 

261 name="ps1_pv3_3pi_20170110", 

262 storageClass="SimpleCatalog", 

263 dimensions=("skypix",), 

264 deferLoad=True, 

265 multiple=True, 

266 lookupFunction=lookupVisitRefCats, 

267 ) 

268 

269 outputWcs = pipeBase.connectionTypes.Output( 

270 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model." 

271 " These catalogs only contain entries for detectors with an output, and use" 

272 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

273 name="jointcalSkyWcsCatalog", 

274 storageClass="ExposureCatalog", 

275 dimensions=("instrument", "visit", "skymap", "tract"), 

276 multiple=True 

277 ) 

278 outputPhotoCalib = pipeBase.connectionTypes.Output( 

279 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model." 

280 " These catalogs only contain entries for detectors with an output, and use" 

281 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

282 name="jointcalPhotoCalibCatalog", 

283 storageClass="ExposureCatalog", 

284 dimensions=("instrument", "visit", "skymap", "tract"), 

285 multiple=True 

286 ) 

287 

288 def __init__(self, *, config=None): 

289 super().__init__(config=config) 

290 # When we are only doing one of astrometry or photometry, we don't 

291 # need the reference catalog or produce the outputs for the other. 

292 # This informs the middleware of that when the QuantumGraph is 

293 # generated, so we don't block on getting something we won't need or 

294 # create an expectation that downstream tasks will be able to consume 

295 # something we won't produce. 

296 if not config.doAstrometry: 

297 self.prerequisiteInputs.remove("astrometryRefCat") 

298 self.outputs.remove("outputWcs") 

299 if not config.doPhotometry: 

300 self.prerequisiteInputs.remove("photometryRefCat") 

301 self.outputs.remove("outputPhotoCalib") 

302 

303 

304class JointcalConfig(pipeBase.PipelineTaskConfig, 

305 pipelineConnections=JointcalTaskConnections): 

306 """Configuration for JointcalTask""" 

307 

308 doAstrometry = pexConfig.Field( 

309 doc="Fit astrometry and write the fitted result.", 

310 dtype=bool, 

311 default=True 

312 ) 

313 doPhotometry = pexConfig.Field( 

314 doc="Fit photometry and write the fitted result.", 

315 dtype=bool, 

316 default=True 

317 ) 

318 coaddName = pexConfig.Field( 

319 doc="Type of coadd, typically deep or goodSeeing", 

320 dtype=str, 

321 default="deep" 

322 ) 

323 # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal. 

324 sourceFluxType = pexConfig.Field( 

325 dtype=str, 

326 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

327 default='Calib' 

328 ) 

329 positionErrorPedestal = pexConfig.Field( 

330 doc="Systematic term to apply to the measured position error (pixels)", 

331 dtype=float, 

332 default=0.02, 

333 ) 

334 photometryErrorPedestal = pexConfig.Field( 

335 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

336 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

337 dtype=float, 

338 default=0.0, 

339 ) 

340 # TODO: DM-6885 matchCut should be an geom.Angle 

341 matchCut = pexConfig.Field( 

342 doc="Matching radius between fitted and reference stars (arcseconds)", 

343 dtype=float, 

344 default=3.0, 

345 ) 

346 minMeasurements = pexConfig.Field( 

347 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

348 dtype=int, 

349 default=2, 

350 ) 

351 minMeasuredStarsPerCcd = pexConfig.Field( 

352 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

353 dtype=int, 

354 default=100, 

355 ) 

356 minRefStarsPerCcd = pexConfig.Field( 

357 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

358 dtype=int, 

359 default=30, 

360 ) 

361 allowLineSearch = pexConfig.Field( 

362 doc="Allow a line search during minimization, if it is reasonable for the model" 

363 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

364 dtype=bool, 

365 default=False 

366 ) 

367 astrometrySimpleOrder = pexConfig.Field( 

368 doc="Polynomial order for fitting the simple astrometry model.", 

369 dtype=int, 

370 default=3, 

371 ) 

372 astrometryChipOrder = pexConfig.Field( 

373 doc="Order of the per-chip transform for the constrained astrometry model.", 

374 dtype=int, 

375 default=1, 

376 ) 

377 astrometryVisitOrder = pexConfig.Field( 

378 doc="Order of the per-visit transform for the constrained astrometry model.", 

379 dtype=int, 

380 default=5, 

381 ) 

382 useInputWcs = pexConfig.Field( 

383 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

384 dtype=bool, 

385 default=True, 

386 ) 

387 astrometryModel = pexConfig.ChoiceField( 

388 doc="Type of model to fit to astrometry", 

389 dtype=str, 

390 default="constrained", 

391 allowed={"simple": "One polynomial per ccd", 

392 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

393 ) 

394 photometryModel = pexConfig.ChoiceField( 

395 doc="Type of model to fit to photometry", 

396 dtype=str, 

397 default="constrainedMagnitude", 

398 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

399 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

400 " fitting in flux space.", 

401 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

402 " fitting in magnitude space.", 

403 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

404 " fitting in magnitude space.", 

405 } 

406 ) 

407 applyColorTerms = pexConfig.Field( 

408 doc="Apply photometric color terms to reference stars?" 

409 "Requires that colorterms be set to a ColortermLibrary", 

410 dtype=bool, 

411 default=False 

412 ) 

413 colorterms = pexConfig.ConfigField( 

414 doc="Library of photometric reference catalog name to color term dict.", 

415 dtype=ColortermLibrary, 

416 ) 

417 photometryVisitOrder = pexConfig.Field( 

418 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

419 dtype=int, 

420 default=7, 

421 ) 

422 photometryDoRankUpdate = pexConfig.Field( 

423 doc=("Do the rank update step during minimization. " 

424 "Skipping this can help deal with models that are too non-linear."), 

425 dtype=bool, 

426 default=True, 

427 ) 

428 astrometryDoRankUpdate = pexConfig.Field( 

429 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

430 "Skipping this can help deal with models that are too non-linear."), 

431 dtype=bool, 

432 default=True, 

433 ) 

434 outlierRejectSigma = pexConfig.Field( 

435 doc="How many sigma to reject outliers at during minimization.", 

436 dtype=float, 

437 default=5.0, 

438 ) 

439 astrometryOutlierRelativeTolerance = pexConfig.Field( 

440 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will " 

441 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set " 

442 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002" 

443 "as a balance between a shorter minimization runtime and achieving a final fitted model that is" 

444 "close to the solution found when removing all outliers."), 

445 dtype=float, 

446 default=0, 

447 ) 

448 maxPhotometrySteps = pexConfig.Field( 

449 doc="Maximum number of minimize iterations to take when fitting photometry.", 

450 dtype=int, 

451 default=20, 

452 ) 

453 maxAstrometrySteps = pexConfig.Field( 

454 doc="Maximum number of minimize iterations to take when fitting astrometry.", 

455 dtype=int, 

456 default=20, 

457 ) 

458 astrometryRefObjLoader = pexConfig.ConfigurableField( 

459 target=LoadIndexedReferenceObjectsTask, 

460 doc="Reference object loader for astrometric fit", 

461 ) 

462 photometryRefObjLoader = pexConfig.ConfigurableField( 

463 target=LoadIndexedReferenceObjectsTask, 

464 doc="Reference object loader for photometric fit", 

465 ) 

466 sourceSelector = sourceSelectorRegistry.makeField( 

467 doc="How to select sources for cross-matching", 

468 default="astrometry" 

469 ) 

470 astrometryReferenceSelector = pexConfig.ConfigurableField( 

471 target=ReferenceSourceSelectorTask, 

472 doc="How to down-select the loaded astrometry reference catalog.", 

473 ) 

474 photometryReferenceSelector = pexConfig.ConfigurableField( 

475 target=ReferenceSourceSelectorTask, 

476 doc="How to down-select the loaded photometry reference catalog.", 

477 ) 

478 astrometryReferenceErr = pexConfig.Field( 

479 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

480 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

481 "If specified, overrides any existing `coord_*Err` values."), 

482 dtype=float, 

483 default=None, 

484 optional=True 

485 ) 

486 

487 # configs for outputting debug information 

488 writeInitMatrix = pexConfig.Field( 

489 dtype=bool, 

490 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

491 "Output files will be written to `config.debugOutputPath` and will " 

492 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. " 

493 "Note that these files are the dense versions of the matrix, and so may be very large."), 

494 default=False 

495 ) 

496 writeChi2FilesInitialFinal = pexConfig.Field( 

497 dtype=bool, 

498 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. " 

499 "Output files will be written to `config.debugOutputPath` and will " 

500 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."), 

501 default=False 

502 ) 

503 writeChi2FilesOuterLoop = pexConfig.Field( 

504 dtype=bool, 

505 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. " 

506 "Output files will be written to `config.debugOutputPath` and will " 

507 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."), 

508 default=False 

509 ) 

510 writeInitialModel = pexConfig.Field( 

511 dtype=bool, 

512 doc=("Write the pre-initialization model to text files, for debugging. " 

513 "Output files will be written to `config.debugOutputPath` and will be " 

514 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."), 

515 default=False 

516 ) 

517 debugOutputPath = pexConfig.Field( 

518 dtype=str, 

519 default=".", 

520 doc=("Path to write debug output files to. Used by " 

521 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

522 ) 

523 detailedProfile = pexConfig.Field( 

524 dtype=bool, 

525 default=False, 

526 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting" 

527 ) 

528 

529 def validate(self): 

530 super().validate() 

531 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

532 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

533 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

534 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

535 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

536 "applyColorTerms=True will be ignored.") 

537 lsst.log.warn(msg) 

538 

539 def setDefaults(self): 

540 # Use science source selector which can filter on extendedness, SNR, and whether blended 

541 self.sourceSelector.name = 'science' 

542 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing 

543 self.sourceSelector['science'].doUnresolved = True 

544 # with dependable signal to noise ratio. 

545 self.sourceSelector['science'].doSignalToNoise = True 

546 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

547 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

548 self.sourceSelector['science'].signalToNoise.minimum = 10. 

549 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive 

550 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux" 

551 self.sourceSelector['science'].signalToNoise.fluxField = fluxField 

552 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err" 

553 # Do not trust blended sources' aperture fluxes which also depend on seeing 

554 self.sourceSelector['science'].doIsolated = True 

555 # Do not trust either flux or centroid measurements with flags, 

556 # chosen from the usual QA flags for stars) 

557 self.sourceSelector['science'].doFlags = True 

558 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 

559 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag', 

560 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter'] 

561 self.sourceSelector['science'].flags.bad = badFlags 

562 

563 # Default to Gaia-DR2 (with proper motions) for astrometry and 

564 # PS1-DR1 for photometry, with a reasonable initial filterMap. 

565 self.astrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414" 

566 self.astrometryRefObjLoader.requireProperMotion = True 

567 self.astrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean' 

568 self.photometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110" 

569 

570 

571def writeModel(model, filename, log): 

572 """Write model to outfile.""" 

573 with open(filename, "w") as file: 

574 file.write(repr(model)) 

575 log.info("Wrote %s to file: %s", model, filename) 

576 

577 

578@dataclasses.dataclass 

579class JointcalInputData: 

580 """The input data jointcal needs for each detector/visit.""" 

581 visit: int 

582 """The visit identifier of this exposure.""" 

583 catalog: lsst.afw.table.SourceCatalog 

584 """The catalog derived from this exposure.""" 

585 visitInfo: lsst.afw.image.VisitInfo 

586 """The VisitInfo of this exposure.""" 

587 detector: lsst.afw.cameraGeom.Detector 

588 """The detector of this exposure.""" 

589 photoCalib: lsst.afw.image.PhotoCalib 

590 """The photometric calibration of this exposure.""" 

591 wcs: lsst.afw.geom.skyWcs 

592 """The WCS of this exposure.""" 

593 bbox: lsst.geom.Box2I 

594 """The bounding box of this exposure.""" 

595 filter: lsst.afw.image.FilterLabel 

596 """The filter of this exposure.""" 

597 

598 

599class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

600 """Astrometricly and photometricly calibrate across multiple visits of the 

601 same field. 

602 

603 Parameters 

604 ---------- 

605 butler : `lsst.daf.persistence.Butler` 

606 The butler is passed to the refObjLoader constructor in case it is 

607 needed. Ignored if the refObjLoader argument provides a loader directly. 

608 Used to initialize the astrometry and photometry refObjLoaders. 

609 initInputs : `dict`, optional 

610 Dictionary used to initialize PipelineTasks (empty for jointcal). 

611 """ 

612 

613 ConfigClass = JointcalConfig 

614 RunnerClass = JointcalRunner 

615 _DefaultName = "jointcal" 

616 

617 def __init__(self, butler=None, initInputs=None, **kwargs): 

618 super().__init__(**kwargs) 

619 self.makeSubtask("sourceSelector") 

620 if self.config.doAstrometry: 

621 if initInputs is None: 

622 # gen3 middleware does refcat things internally (and will not have a butler here) 

623 self.makeSubtask('astrometryRefObjLoader', butler=butler) 

624 self.makeSubtask("astrometryReferenceSelector") 

625 else: 

626 self.astrometryRefObjLoader = None 

627 if self.config.doPhotometry: 

628 if initInputs is None: 

629 # gen3 middleware does refcat things internally (and will not have a butler here) 

630 self.makeSubtask('photometryRefObjLoader', butler=butler) 

631 self.makeSubtask("photometryReferenceSelector") 

632 else: 

633 self.photometryRefObjLoader = None 

634 

635 # To hold various computed metrics for use by tests 

636 self.job = Job.load_metrics_package(subset='jointcal') 

637 

638 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

639 # We override runQuantum to set up the refObjLoaders and write the 

640 # outputs to the correct refs. 

641 inputs = butlerQC.get(inputRefs) 

642 # We want the tract number for writing debug files 

643 tract = butlerQC.quantum.dataId['tract'] 

644 if self.config.doAstrometry: 

645 self.astrometryRefObjLoader = ReferenceObjectLoader( 

646 dataIds=[ref.datasetRef.dataId 

647 for ref in inputRefs.astrometryRefCat], 

648 refCats=inputs.pop('astrometryRefCat'), 

649 config=self.config.astrometryRefObjLoader, 

650 log=self.log) 

651 if self.config.doPhotometry: 

652 self.photometryRefObjLoader = ReferenceObjectLoader( 

653 dataIds=[ref.datasetRef.dataId 

654 for ref in inputRefs.photometryRefCat], 

655 refCats=inputs.pop('photometryRefCat'), 

656 config=self.config.photometryRefObjLoader, 

657 log=self.log) 

658 outputs = self.run(**inputs, tract=tract) 

659 if self.config.doAstrometry: 

660 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs, 

661 inputs['inputCamera'], "setWcs") 

662 if self.config.doPhotometry: 

663 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib, 

664 inputs['inputCamera'], "setPhotoCalib") 

665 

666 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter): 

667 """Persist the output datasets to their appropriate datarefs. 

668 

669 Parameters 

670 ---------- 

671 butlerQC : `ButlerQuantumContext` 

672 A butler which is specialized to operate in the context of a 

673 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

674 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or 

675 `dict` [`tuple, `lsst.afw.image.PhotoCalib`] 

676 The fitted objects to persist. 

677 outputRefs : `list` [`OutputQuantizedConnection`] 

678 The DatasetRefs to persist the data to. 

679 camera : `lsst.afw.cameraGeom.Camera` 

680 The camera for this instrument, to get detector ids from. 

681 setter : `str` 

682 The method to call on the ExposureCatalog to set each output. 

683 """ 

684 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

685 schema.addField('visit', type='I', doc='Visit number') 

686 

687 def new_catalog(visit, size): 

688 """Return an catalog ready to be filled with appropriate output.""" 

689 catalog = lsst.afw.table.ExposureCatalog(schema) 

690 catalog.resize(size) 

691 catalog['visit'] = visit 

692 metadata = lsst.daf.base.PropertyList() 

693 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

694 metadata.add("COMMENT", "Only detectors with data have entries.") 

695 return catalog 

696 

697 # count how many detectors have output for each visit 

698 detectors_per_visit = collections.defaultdict(int) 

699 for key in outputs: 

700 # key is (visit, detector_id), and we only need visit here 

701 detectors_per_visit[key[0]] += 1 

702 

703 for ref in outputRefs: 

704 visit = ref.dataId['visit'] 

705 catalog = new_catalog(visit, detectors_per_visit[visit]) 

706 # Iterate over every detector and skip the ones we don't have output for. 

707 i = 0 

708 for detector in camera: 

709 detectorId = detector.getId() 

710 key = (ref.dataId['visit'], detectorId) 

711 if key not in outputs: 

712 # skip detectors we don't have output for 

713 self.log.debug("No %s output for detector %s in visit %s", 

714 setter[3:], detectorId, visit) 

715 continue 

716 

717 catalog[i].setId(detectorId) 

718 getattr(catalog[i], setter)(outputs[key]) 

719 i += 1 

720 

721 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups 

722 butlerQC.put(catalog, ref) 

723 self.log.info("Wrote %s detectors to %s", i, ref) 

724 

725 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None): 

726 # Docstring inherited. 

727 

728 # We take values out of the Parquet table, and put them in "flux_", 

729 # and the config.sourceFluxType field is used during that extraction, 

730 # so just use "flux" here. 

731 sourceFluxField = "flux" 

732 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

733 associations = lsst.jointcal.Associations() 

734 self.focalPlaneBBox = inputCamera.getFpBBox() 

735 oldWcsList, bands = self._load_data(inputSourceTableVisit, 

736 inputVisitSummary, 

737 associations, 

738 jointcalControl, 

739 inputCamera) 

740 

741 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands) 

742 

743 if self.config.doAstrometry: 

744 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

745 name="astrometry", 

746 refObjLoader=self.astrometryRefObjLoader, 

747 referenceSelector=self.astrometryReferenceSelector, 

748 fit_function=self._fit_astrometry, 

749 tract=tract, 

750 epoch=epoch) 

751 astrometry_output = self._make_output(associations.getCcdImageList(), 

752 astrometry.model, 

753 "makeSkyWcs") 

754 else: 

755 astrometry_output = None 

756 

757 if self.config.doPhotometry: 

758 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

759 name="photometry", 

760 refObjLoader=self.photometryRefObjLoader, 

761 referenceSelector=self.photometryReferenceSelector, 

762 fit_function=self._fit_photometry, 

763 tract=tract, 

764 epoch=epoch, 

765 reject_bad_fluxes=True) 

766 photometry_output = self._make_output(associations.getCcdImageList(), 

767 photometry.model, 

768 "toPhotoCalib") 

769 else: 

770 photometry_output = None 

771 

772 return pipeBase.Struct(outputWcs=astrometry_output, 

773 outputPhotoCalib=photometry_output, 

774 job=self.job, 

775 astrometryRefObjLoader=self.astrometryRefObjLoader, 

776 photometryRefObjLoader=self.photometryRefObjLoader) 

777 

778 def _make_schema_table(self): 

779 """Return an afw SourceTable to use as a base for creating the 

780 SourceCatalog to insert values from the dataFrame into. 

781 

782 Returns 

783 ------- 

784 table : `lsst.afw.table.SourceTable` 

785 Table with schema and slots to use to make SourceCatalogs. 

786 """ 

787 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

788 schema.addField("centroid_x", "D") 

789 schema.addField("centroid_y", "D") 

790 schema.addField("centroid_xErr", "F") 

791 schema.addField("centroid_yErr", "F") 

792 schema.addField("shape_xx", "D") 

793 schema.addField("shape_yy", "D") 

794 schema.addField("shape_xy", "D") 

795 schema.addField("flux_instFlux", "D") 

796 schema.addField("flux_instFluxErr", "D") 

797 table = lsst.afw.table.SourceTable.make(schema) 

798 table.defineCentroid("centroid") 

799 table.defineShape("shape") 

800 return table 

801 

802 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId): 

803 """Return an afw SourceCatalog extracted from a visit-level dataframe, 

804 limited to just one detector. 

805 

806 Parameters 

807 ---------- 

808 table : `lsst.afw.table.SourceTable` 

809 Table factory to use to make the SourceCatalog that will be 

810 populated with data from ``visitCatalog``. 

811 visitCatalog : `pandas.DataFrame` 

812 DataFrame to extract a detector catalog from. 

813 detectorId : `int` 

814 Numeric id of the detector to extract from ``visitCatalog``. 

815 

816 Returns 

817 ------- 

818 catalog : `lsst.afw.table.SourceCatalog` 

819 Detector-level catalog extracted from ``visitCatalog``. 

820 """ 

821 # map from dataFrame column to afw table column 

822 mapping = {'sourceId': 'id', 

823 'x': 'centroid_x', 

824 'y': 'centroid_y', 

825 'xErr': 'centroid_xErr', 

826 'yErr': 'centroid_yErr', 

827 'Ixx': 'shape_xx', 

828 'Iyy': 'shape_yy', 

829 'Ixy': 'shape_xy', 

830 f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux', 

831 f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr', 

832 } 

833 # If the DataFrame we're reading was generated by a task running with 

834 # Gen2 middleware, the column for the detector will be "ccd" for at 

835 # least HSC (who knows what it might be in general!); that would be 

836 # true even if the data repo is later converted to Gen3, because that 

837 # doesn't touch the files themselves. In Gen3, the column for the 

838 # detector will always be "detector". 

839 detector_column = "detector" if "detector" in visitCatalog.columns else "ccd" 

840 catalog = lsst.afw.table.SourceCatalog(table) 

841 matched = visitCatalog[detector_column] == detectorId 

842 catalog.resize(sum(matched)) 

843 view = visitCatalog.loc[matched] 

844 for dfCol, afwCol in mapping.items(): 

845 catalog[afwCol] = view[dfCol] 

846 

847 self.log.debug("%d sources selected in visit %d detector %d", 

848 len(catalog), 

849 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first 

850 detectorId) 

851 return catalog 

852 

853 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, 

854 jointcalControl, camera): 

855 """Read the data that jointcal needs to run. (Gen3 version) 

856 

857 Modifies ``associations`` in-place with the loaded data. 

858 

859 Parameters 

860 ---------- 

861 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

862 References to visit-level DataFrames to load the catalog data from. 

863 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

864 Visit-level exposure summary catalog with metadata. 

865 associations : `lsst.jointcal.Associations` 

866 Object to add the loaded data to by constructing new CcdImages. 

867 jointcalControl : `jointcal.JointcalControl` 

868 Control object for C++ associations management. 

869 camera : `lsst.afw.cameraGeom.Camera` 

870 Camera object for detector geometry. 

871 

872 Returns 

873 ------- 

874 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`] 

875 The original WCS of the input data, to aid in writing tests. 

876 bands : `list` [`str`] 

877 The filter bands of each input dataset. 

878 """ 

879 oldWcsList = [] 

880 filters = [] 

881 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else '' 

882 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

883 table = self._make_schema_table() # every detector catalog has the same layout 

884 # No guarantee that the input is in the same order of visits, so we have to map one of them. 

885 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)} 

886 detectorDict = {detector.getId(): detector for detector in camera} 

887 

888 for visitSummaryRef in inputVisitSummary: 

889 visitSummary = visitSummaryRef.get() 

890 visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]].get() 

891 selected = self.sourceSelector.run(visitCatalog) 

892 

893 # Build a CcdImage for each detector in this visit. 

894 detectors = {id: index for index, id in enumerate(visitSummary['id'])} 

895 for id, index in detectors.items(): 

896 catalog = self._extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id) 

897 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict) 

898 result = self._build_ccdImage(data, associations, jointcalControl) 

899 if result is not None: 

900 oldWcsList.append(result.wcs) 

901 # A visit has only one band, so we can just use the first. 

902 filters.append(data.filter) 

903 if len(filters) == 0: 

904 raise RuntimeError("No data to process: did source selector remove all sources?") 

905 filters = collections.Counter(filters) 

906 

907 return oldWcsList, filters 

908 

909 def _make_one_input_data(self, visitRecord, catalog, detectorDict): 

910 """Return a data structure for this detector+visit.""" 

911 return JointcalInputData(visit=visitRecord['visit'], 

912 catalog=catalog, 

913 visitInfo=visitRecord.getVisitInfo(), 

914 detector=detectorDict[visitRecord.getId()], 

915 photoCalib=visitRecord.getPhotoCalib(), 

916 wcs=visitRecord.getWcs(), 

917 bbox=visitRecord.getBBox(), 

918 # ExposureRecord doesn't have a FilterLabel yet, 

919 # so we have to make one. 

920 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'], 

921 physical=visitRecord['physical_filter'])) 

922 

923 # We don't currently need to persist the metadata. 

924 # If we do in the future, we will have to add appropriate dataset templates 

925 # to each obs package (the metadata template should look like `jointcal_wcs`). 

926 def _getMetadataName(self): 

927 return None 

928 

929 @classmethod 

930 def _makeArgumentParser(cls): 

931 """Create an argument parser""" 

932 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

933 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9", 

934 ContainerClass=PerTractCcdDataIdContainer) 

935 return parser 

936 

937 def _build_ccdImage(self, data, associations, jointcalControl): 

938 """ 

939 Extract the necessary things from this catalog+metadata to add a new 

940 ccdImage. 

941 

942 Parameters 

943 ---------- 

944 data : `JointcalInputData` 

945 The loaded input data. 

946 associations : `lsst.jointcal.Associations` 

947 Object to add the info to, to construct a new CcdImage 

948 jointcalControl : `jointcal.JointcalControl` 

949 Control object for associations management 

950 

951 Returns 

952 ------ 

953 namedtuple or `None` 

954 ``wcs`` 

955 The TAN WCS of this image, read from the calexp 

956 (`lsst.afw.geom.SkyWcs`). 

957 ``key`` 

958 A key to identify this dataRef by its visit and ccd ids 

959 (`namedtuple`). 

960 `None` 

961 if there are no sources in the loaded catalog. 

962 """ 

963 if len(data.catalog) == 0: 

964 self.log.warn("No sources selected in visit %s ccd %s", data.visit, data.detector.getId()) 

965 return None 

966 

967 associations.createCcdImage(data.catalog, 

968 data.wcs, 

969 data.visitInfo, 

970 data.bbox, 

971 data.filter.physicalLabel, 

972 data.photoCalib, 

973 data.detector, 

974 data.visit, 

975 data.detector.getId(), 

976 jointcalControl) 

977 

978 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key')) 

979 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

980 return Result(data.wcs, Key(data.visit, data.detector.getId())) 

981 

982 def _readDataId(self, butler, dataId): 

983 """Read all of the data for one dataId from the butler. (gen2 version)""" 

984 # Not all instruments have `visit` in their dataIds. 

985 if "visit" in dataId.keys(): 

986 visit = dataId["visit"] 

987 else: 

988 visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0] 

989 detector = butler.get('calexp_detector', dataId=dataId) 

990 

991 catalog = butler.get('src', 

992 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, 

993 dataId=dataId) 

994 goodSrc = self.sourceSelector.run(catalog) 

995 self.log.debug("%d sources selected in visit %d detector %d", 

996 len(goodSrc.sourceCat), 

997 visit, 

998 detector.getId()) 

999 return JointcalInputData(visit=visit, 

1000 catalog=goodSrc.sourceCat, 

1001 visitInfo=butler.get('calexp_visitInfo', dataId=dataId), 

1002 detector=detector, 

1003 photoCalib=butler.get('calexp_photoCalib', dataId=dataId), 

1004 wcs=butler.get('calexp_wcs', dataId=dataId), 

1005 bbox=butler.get('calexp_bbox', dataId=dataId), 

1006 filter=butler.get('calexp_filterLabel', dataId=dataId)) 

1007 

1008 def loadData(self, dataRefs, associations, jointcalControl): 

1009 """Read the data that jointcal needs to run. (Gen2 version)""" 

1010 visit_ccd_to_dataRef = {} 

1011 oldWcsList = [] 

1012 filters = [] 

1013 load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else '' 

1014 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

1015 # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models 

1016 camera = dataRefs[0].get('camera', immediate=True) 

1017 self.focalPlaneBBox = camera.getFpBBox() 

1018 for dataRef in dataRefs: 

1019 data = self._readDataId(dataRef.getButler(), dataRef.dataId) 

1020 result = self._build_ccdImage(data, associations, jointcalControl) 

1021 if result is None: 

1022 continue 

1023 oldWcsList.append(result.wcs) 

1024 visit_ccd_to_dataRef[result.key] = dataRef 

1025 filters.append(data.filter) 

1026 if len(filters) == 0: 

1027 raise RuntimeError("No data to process: did source selector remove all sources?") 

1028 filters = collections.Counter(filters) 

1029 

1030 return oldWcsList, filters, visit_ccd_to_dataRef 

1031 

1032 def _getDebugPath(self, filename): 

1033 """Constructs a path to filename using the configured debug path. 

1034 """ 

1035 return os.path.join(self.config.debugOutputPath, filename) 

1036 

1037 def _prep_sky(self, associations, filters): 

1038 """Prepare on-sky and other data that must be computed after data has 

1039 been read. 

1040 """ 

1041 associations.computeCommonTangentPoint() 

1042 

1043 boundingCircle = associations.computeBoundingCircle() 

1044 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

1045 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

1046 

1047 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

1048 

1049 # Determine a default filter band associated with the catalog. See DM-9093 

1050 defaultFilter = filters.most_common(1)[0][0] 

1051 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel) 

1052 

1053 # compute and set the reference epoch of the observations, for proper motion corrections 

1054 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

1055 associations.setEpoch(epoch.jyear) 

1056 

1057 return boundingCircle, center, radius, defaultFilter, epoch 

1058 

1059 @pipeBase.timeMethod 

1060 def runDataRef(self, dataRefs): 

1061 """ 

1062 Jointly calibrate the astrometry and photometry across a set of images. 

1063 

1064 NOTE: this is for gen2 middleware only. 

1065 

1066 Parameters 

1067 ---------- 

1068 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

1069 List of data references to the exposures to be fit. 

1070 

1071 Returns 

1072 ------- 

1073 result : `lsst.pipe.base.Struct` 

1074 Struct of metadata from the fit, containing: 

1075 

1076 ``dataRefs`` 

1077 The provided data references that were fit (with updated WCSs) 

1078 ``oldWcsList`` 

1079 The original WCS from each dataRef 

1080 ``metrics`` 

1081 Dictionary of internally-computed metrics for testing/validation. 

1082 """ 

1083 if len(dataRefs) == 0: 

1084 raise ValueError('Need a non-empty list of data references!') 

1085 

1086 exitStatus = 0 # exit status for shell 

1087 

1088 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,) 

1089 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

1090 associations = lsst.jointcal.Associations() 

1091 

1092 oldWcsList, filters, visit_ccd_to_dataRef = self.loadData(dataRefs, 

1093 associations, 

1094 jointcalControl) 

1095 

1096 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, filters) 

1097 

1098 tract = dataRefs[0].dataId['tract'] 

1099 

1100 if self.config.doAstrometry: 

1101 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

1102 name="astrometry", 

1103 refObjLoader=self.astrometryRefObjLoader, 

1104 referenceSelector=self.astrometryReferenceSelector, 

1105 fit_function=self._fit_astrometry, 

1106 tract=tract, 

1107 epoch=epoch) 

1108 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef) 

1109 else: 

1110 astrometry = Astrometry(None, None, None) 

1111 

1112 if self.config.doPhotometry: 

1113 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

1114 name="photometry", 

1115 refObjLoader=self.photometryRefObjLoader, 

1116 referenceSelector=self.photometryReferenceSelector, 

1117 fit_function=self._fit_photometry, 

1118 tract=tract, 

1119 epoch=epoch, 

1120 reject_bad_fluxes=True) 

1121 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef) 

1122 else: 

1123 photometry = Photometry(None, None) 

1124 

1125 return pipeBase.Struct(dataRefs=dataRefs, 

1126 oldWcsList=oldWcsList, 

1127 job=self.job, 

1128 astrometryRefObjLoader=self.astrometryRefObjLoader, 

1129 photometryRefObjLoader=self.photometryRefObjLoader, 

1130 defaultFilter=defaultFilter, 

1131 epoch=epoch, 

1132 exitStatus=exitStatus) 

1133 

1134 def _get_refcat_coordinate_error_override(self, refCat, name): 

1135 """Check whether we should override the refcat coordinate errors, and 

1136 return the overridden error if necessary. 

1137 

1138 Parameters 

1139 ---------- 

1140 refCat : `lsst.afw.table.SimpleCatalog` 

1141 The reference catalog to check for a ``coord_raErr`` field. 

1142 name : `str` 

1143 Whether we are doing "astrometry" or "photometry". 

1144 

1145 Returns 

1146 ------- 

1147 refCoordErr : `float` 

1148 The refcat coordinate error to use, or NaN if we are not overriding 

1149 those fields. 

1150 

1151 Raises 

1152 ------ 

1153 lsst.pex.config.FieldValidationError 

1154 Raised if the refcat does not contain coordinate errors and 

1155 ``config.astrometryReferenceErr`` is not set. 

1156 """ 

1157 # This value doesn't matter for photometry, so just set something to 

1158 # keep old refcats from causing problems. 

1159 if name.lower() == "photometry": 

1160 if 'coord_raErr' not in refCat.schema: 

1161 return 100 

1162 else: 

1163 return float('nan') 

1164 

1165 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

1166 msg = ("Reference catalog does not contain coordinate errors, " 

1167 "and config.astrometryReferenceErr not supplied.") 

1168 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

1169 self.config, 

1170 msg) 

1171 

1172 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

1173 self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

1174 self.config.astrometryReferenceErr) 

1175 

1176 if self.config.astrometryReferenceErr is None: 

1177 return float('nan') 

1178 else: 

1179 return self.config.astrometryReferenceErr 

1180 

1181 def _compute_proper_motion_epoch(self, ccdImageList): 

1182 """Return the proper motion correction epoch of the provided images. 

1183 

1184 Parameters 

1185 ---------- 

1186 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

1187 The images to compute the appropriate epoch for. 

1188 

1189 Returns 

1190 ------- 

1191 epoch : `astropy.time.Time` 

1192 The date to use for proper motion corrections. 

1193 """ 

1194 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]), 

1195 format="jyear", 

1196 scale="tai") 

1197 

1198 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

1199 tract="", match_cut=3.0, 

1200 reject_bad_fluxes=False, *, 

1201 name="", refObjLoader=None, referenceSelector=None, 

1202 fit_function=None, epoch=None): 

1203 """Load reference catalog, perform the fit, and return the result. 

1204 

1205 Parameters 

1206 ---------- 

1207 associations : `lsst.jointcal.Associations` 

1208 The star/reference star associations to fit. 

1209 defaultFilter : `lsst.afw.image.FilterLabel` 

1210 filter to load from reference catalog. 

1211 center : `lsst.geom.SpherePoint` 

1212 ICRS center of field to load from reference catalog. 

1213 radius : `lsst.geom.Angle` 

1214 On-sky radius to load from reference catalog. 

1215 name : `str` 

1216 Name of thing being fit: "astrometry" or "photometry". 

1217 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

1218 Reference object loader to use to load a reference catalog. 

1219 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1220 Selector to use to pick objects from the loaded reference catalog. 

1221 fit_function : callable 

1222 Function to call to perform fit (takes Associations object). 

1223 tract : `str`, optional 

1224 Name of tract currently being fit. 

1225 match_cut : `float`, optional 

1226 Radius in arcseconds to find cross-catalog matches to during 

1227 associations.associateCatalogs. 

1228 reject_bad_fluxes : `bool`, optional 

1229 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

1230 epoch : `astropy.time.Time`, optional 

1231 Epoch to which to correct refcat proper motion and parallax, 

1232 or `None` to not apply such corrections. 

1233 

1234 Returns 

1235 ------- 

1236 result : `Photometry` or `Astrometry` 

1237 Result of `fit_function()` 

1238 """ 

1239 self.log.info("====== Now processing %s...", name) 

1240 # TODO: this should not print "trying to invert a singular transformation:" 

1241 # if it does that, something's not right about the WCS... 

1242 associations.associateCatalogs(match_cut) 

1243 add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name, 

1244 associations.fittedStarListSize()) 

1245 

1246 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

1247 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

1248 center, radius, defaultFilter, 

1249 applyColorterms=applyColorterms, 

1250 epoch=epoch) 

1251 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

1252 

1253 associations.collectRefStars(refCat, 

1254 self.config.matchCut*lsst.geom.arcseconds, 

1255 fluxField, 

1256 refCoordinateErr=refCoordErr, 

1257 rejectBadFluxes=reject_bad_fluxes) 

1258 add_measurement(self.job, 'jointcal.collected_%s_refStars' % name, 

1259 associations.refStarListSize()) 

1260 

1261 associations.prepareFittedStars(self.config.minMeasurements) 

1262 

1263 self._check_star_lists(associations, name) 

1264 add_measurement(self.job, 'jointcal.selected_%s_refStars' % name, 

1265 associations.nFittedStarsWithAssociatedRefStar()) 

1266 add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name, 

1267 associations.fittedStarListSize()) 

1268 add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name, 

1269 associations.nCcdImagesValidForFit()) 

1270 

1271 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else '' 

1272 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel) 

1273 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

1274 result = fit_function(associations, dataName) 

1275 # TODO DM-12446: turn this into a "butler save" somehow. 

1276 # Save reference and measurement chi2 contributions for this data 

1277 if self.config.writeChi2FilesInitialFinal: 

1278 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

1279 result.fit.saveChi2Contributions(baseName+"{type}") 

1280 self.log.info("Wrote chi2 contributions files: %s", baseName) 

1281 

1282 return result 

1283 

1284 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, 

1285 applyColorterms=False, epoch=None): 

1286 """Load the necessary reference catalog sources, convert fluxes to 

1287 correct units, and apply color term corrections if requested. 

1288 

1289 Parameters 

1290 ---------- 

1291 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

1292 The reference catalog loader to use to get the data. 

1293 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1294 Source selector to apply to loaded reference catalog. 

1295 center : `lsst.geom.SpherePoint` 

1296 The center around which to load sources. 

1297 radius : `lsst.geom.Angle` 

1298 The radius around ``center`` to load sources in. 

1299 filterLabel : `lsst.afw.image.FilterLabel` 

1300 The camera filter to load fluxes for. 

1301 applyColorterms : `bool` 

1302 Apply colorterm corrections to the refcat for ``filterName``? 

1303 epoch : `astropy.time.Time`, optional 

1304 Epoch to which to correct refcat proper motion and parallax, 

1305 or `None` to not apply such corrections. 

1306 

1307 Returns 

1308 ------- 

1309 refCat : `lsst.afw.table.SimpleCatalog` 

1310 The loaded reference catalog. 

1311 fluxField : `str` 

1312 The name of the reference catalog flux field appropriate for ``filterName``. 

1313 """ 

1314 skyCircle = refObjLoader.loadSkyCircle(center, 

1315 radius, 

1316 filterLabel.bandLabel, 

1317 epoch=epoch) 

1318 

1319 selected = referenceSelector.run(skyCircle.refCat) 

1320 # Need memory contiguity to get reference filters as a vector. 

1321 if not selected.sourceCat.isContiguous(): 

1322 refCat = selected.sourceCat.copy(deep=True) 

1323 else: 

1324 refCat = selected.sourceCat 

1325 

1326 if applyColorterms: 

1327 refCatName = refObjLoader.ref_dataset_name 

1328 self.log.info("Applying color terms for physical filter=%r reference catalog=%s", 

1329 filterLabel.physicalLabel, refCatName) 

1330 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel, 

1331 refCatName, 

1332 doRaise=True) 

1333 

1334 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

1335 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

1336 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

1337 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

1338 

1339 return refCat, skyCircle.fluxField 

1340 

1341 def _check_star_lists(self, associations, name): 

1342 # TODO: these should be len(blah), but we need this properly wrapped first. 

1343 if associations.nCcdImagesValidForFit() == 0: 

1344 raise RuntimeError('No images in the ccdImageList!') 

1345 if associations.fittedStarListSize() == 0: 

1346 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

1347 if associations.refStarListSize() == 0: 

1348 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

1349 

1350 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

1351 """Compute chi2, log it, validate the model, and return chi2. 

1352 

1353 Parameters 

1354 ---------- 

1355 associations : `lsst.jointcal.Associations` 

1356 The star/reference star associations to fit. 

1357 fit : `lsst.jointcal.FitterBase` 

1358 The fitter to use for minimization. 

1359 model : `lsst.jointcal.Model` 

1360 The model being fit. 

1361 chi2Label : `str` 

1362 Label to describe the chi2 (e.g. "Initialized", "Final"). 

1363 writeChi2Name : `str`, optional 

1364 Filename prefix to write the chi2 contributions to. 

1365 Do not supply an extension: an appropriate one will be added. 

1366 

1367 Returns 

1368 ------- 

1369 chi2: `lsst.jointcal.Chi2Accumulator` 

1370 The chi2 object for the current fitter and model. 

1371 

1372 Raises 

1373 ------ 

1374 FloatingPointError 

1375 Raised if chi2 is infinite or NaN. 

1376 ValueError 

1377 Raised if the model is not valid. 

1378 """ 

1379 if writeChi2Name is not None: 

1380 fullpath = self._getDebugPath(writeChi2Name) 

1381 fit.saveChi2Contributions(fullpath+"{type}") 

1382 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

1383 

1384 chi2 = fit.computeChi2() 

1385 self.log.info("%s %s", chi2Label, chi2) 

1386 self._check_stars(associations) 

1387 if not np.isfinite(chi2.chi2): 

1388 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

1389 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

1390 raise ValueError("Model is not valid: check log messages for warnings.") 

1391 return chi2 

1392 

1393 def _fit_photometry(self, associations, dataName=None): 

1394 """ 

1395 Fit the photometric data. 

1396 

1397 Parameters 

1398 ---------- 

1399 associations : `lsst.jointcal.Associations` 

1400 The star/reference star associations to fit. 

1401 dataName : `str` 

1402 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1403 identifying debugging files. 

1404 

1405 Returns 

1406 ------- 

1407 fit_result : `namedtuple` 

1408 fit : `lsst.jointcal.PhotometryFit` 

1409 The photometric fitter used to perform the fit. 

1410 model : `lsst.jointcal.PhotometryModel` 

1411 The photometric model that was fit. 

1412 """ 

1413 self.log.info("=== Starting photometric fitting...") 

1414 

1415 # TODO: should use pex.config.RegistryField here (see DM-9195) 

1416 if self.config.photometryModel == "constrainedFlux": 

1417 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

1418 self.focalPlaneBBox, 

1419 visitOrder=self.config.photometryVisitOrder, 

1420 errorPedestal=self.config.photometryErrorPedestal) 

1421 # potentially nonlinear problem, so we may need a line search to converge. 

1422 doLineSearch = self.config.allowLineSearch 

1423 elif self.config.photometryModel == "constrainedMagnitude": 

1424 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

1425 self.focalPlaneBBox, 

1426 visitOrder=self.config.photometryVisitOrder, 

1427 errorPedestal=self.config.photometryErrorPedestal) 

1428 # potentially nonlinear problem, so we may need a line search to converge. 

1429 doLineSearch = self.config.allowLineSearch 

1430 elif self.config.photometryModel == "simpleFlux": 

1431 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

1432 errorPedestal=self.config.photometryErrorPedestal) 

1433 doLineSearch = False # purely linear in model parameters, so no line search needed 

1434 elif self.config.photometryModel == "simpleMagnitude": 

1435 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

1436 errorPedestal=self.config.photometryErrorPedestal) 

1437 doLineSearch = False # purely linear in model parameters, so no line search needed 

1438 

1439 fit = lsst.jointcal.PhotometryFit(associations, model) 

1440 # TODO DM-12446: turn this into a "butler save" somehow. 

1441 # Save reference and measurement chi2 contributions for this data 

1442 if self.config.writeChi2FilesInitialFinal: 

1443 baseName = f"photometry_initial_chi2-{dataName}" 

1444 else: 

1445 baseName = None 

1446 if self.config.writeInitialModel: 

1447 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt") 

1448 writeModel(model, fullpath, self.log) 

1449 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

1450 

1451 def getChi2Name(whatToFit): 

1452 if self.config.writeChi2FilesOuterLoop: 

1453 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

1454 else: 

1455 return None 

1456 

1457 # The constrained model needs the visit transform fit first; the chip 

1458 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

1459 if self.config.writeInitMatrix: 

1460 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}") 

1461 else: 

1462 dumpMatrixFile = "" 

1463 if self.config.photometryModel.startswith("constrained"): 

1464 # no line search: should be purely (or nearly) linear, 

1465 # and we want a large step size to initialize with. 

1466 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

1467 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

1468 writeChi2Name=getChi2Name("ModelVisit")) 

1469 dumpMatrixFile = "" # so we don't redo the output on the next step 

1470 

1471 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

1472 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

1473 writeChi2Name=getChi2Name("Model")) 

1474 

1475 fit.minimize("Fluxes") # no line search: always purely linear. 

1476 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

1477 writeChi2Name=getChi2Name("Fluxes")) 

1478 

1479 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

1480 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

1481 writeChi2Name=getChi2Name("ModelFluxes")) 

1482 

1483 model.freezeErrorTransform() 

1484 self.log.debug("Photometry error scales are frozen.") 

1485 

1486 chi2 = self._iterate_fit(associations, 

1487 fit, 

1488 self.config.maxPhotometrySteps, 

1489 "photometry", 

1490 "Model Fluxes", 

1491 doRankUpdate=self.config.photometryDoRankUpdate, 

1492 doLineSearch=doLineSearch, 

1493 dataName=dataName) 

1494 

1495 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

1496 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

1497 return Photometry(fit, model) 

1498 

1499 def _fit_astrometry(self, associations, dataName=None): 

1500 """ 

1501 Fit the astrometric data. 

1502 

1503 Parameters 

1504 ---------- 

1505 associations : `lsst.jointcal.Associations` 

1506 The star/reference star associations to fit. 

1507 dataName : `str` 

1508 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1509 identifying debugging files. 

1510 

1511 Returns 

1512 ------- 

1513 fit_result : `namedtuple` 

1514 fit : `lsst.jointcal.AstrometryFit` 

1515 The astrometric fitter used to perform the fit. 

1516 model : `lsst.jointcal.AstrometryModel` 

1517 The astrometric model that was fit. 

1518 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

1519 The model for the sky to tangent plane projection that was used in the fit. 

1520 """ 

1521 

1522 self.log.info("=== Starting astrometric fitting...") 

1523 

1524 associations.deprojectFittedStars() 

1525 

1526 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1527 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1528 # them so carefully? 

1529 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1530 

1531 if self.config.astrometryModel == "constrained": 

1532 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1533 sky_to_tan_projection, 

1534 chipOrder=self.config.astrometryChipOrder, 

1535 visitOrder=self.config.astrometryVisitOrder) 

1536 elif self.config.astrometryModel == "simple": 

1537 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1538 sky_to_tan_projection, 

1539 self.config.useInputWcs, 

1540 nNotFit=0, 

1541 order=self.config.astrometrySimpleOrder) 

1542 

1543 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1544 # TODO DM-12446: turn this into a "butler save" somehow. 

1545 # Save reference and measurement chi2 contributions for this data 

1546 if self.config.writeChi2FilesInitialFinal: 

1547 baseName = f"astrometry_initial_chi2-{dataName}" 

1548 else: 

1549 baseName = None 

1550 if self.config.writeInitialModel: 

1551 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt") 

1552 writeModel(model, fullpath, self.log) 

1553 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1554 

1555 def getChi2Name(whatToFit): 

1556 if self.config.writeChi2FilesOuterLoop: 

1557 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1558 else: 

1559 return None 

1560 

1561 if self.config.writeInitMatrix: 

1562 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}") 

1563 else: 

1564 dumpMatrixFile = "" 

1565 # The constrained model needs the visit transform fit first; the chip 

1566 # transform is initialized from the detector's cameraGeom, so it's close. 

1567 if self.config.astrometryModel == "constrained": 

1568 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1569 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1570 writeChi2Name=getChi2Name("DistortionsVisit")) 

1571 dumpMatrixFile = "" # so we don't redo the output on the next step 

1572 

1573 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1574 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1575 writeChi2Name=getChi2Name("Distortions")) 

1576 

1577 fit.minimize("Positions") 

1578 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1579 writeChi2Name=getChi2Name("Positions")) 

1580 

1581 fit.minimize("Distortions Positions") 

1582 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1583 writeChi2Name=getChi2Name("DistortionsPositions")) 

1584 

1585 chi2 = self._iterate_fit(associations, 

1586 fit, 

1587 self.config.maxAstrometrySteps, 

1588 "astrometry", 

1589 "Distortions Positions", 

1590 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance, 

1591 doRankUpdate=self.config.astrometryDoRankUpdate, 

1592 dataName=dataName) 

1593 

1594 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1595 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1596 

1597 return Astrometry(fit, model, sky_to_tan_projection) 

1598 

1599 def _check_stars(self, associations): 

1600 """Count measured and reference stars per ccd and warn/log them.""" 

1601 for ccdImage in associations.getCcdImageList(): 

1602 nMeasuredStars, nRefStars = ccdImage.countStars() 

1603 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1604 ccdImage.getName(), nMeasuredStars, nRefStars) 

1605 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1606 self.log.warn("ccdImage %s has only %s measuredStars (desired %s)", 

1607 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1608 if nRefStars < self.config.minRefStarsPerCcd: 

1609 self.log.warn("ccdImage %s has only %s RefStars (desired %s)", 

1610 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1611 

1612 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1613 dataName="", 

1614 sigmaRelativeTolerance=0, 

1615 doRankUpdate=True, 

1616 doLineSearch=False): 

1617 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1618 

1619 Parameters 

1620 ---------- 

1621 associations : `lsst.jointcal.Associations` 

1622 The star/reference star associations to fit. 

1623 fitter : `lsst.jointcal.FitterBase` 

1624 The fitter to use for minimization. 

1625 max_steps : `int` 

1626 Maximum number of steps to run outlier rejection before declaring 

1627 convergence failure. 

1628 name : {'photometry' or 'astrometry'} 

1629 What type of data are we fitting (for logs and debugging files). 

1630 whatToFit : `str` 

1631 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1632 dataName : `str`, optional 

1633 Descriptive name for this dataset (e.g. tract and filter), 

1634 for debugging. 

1635 sigmaRelativeTolerance : `float`, optional 

1636 Convergence tolerance for the fractional change in the chi2 cut 

1637 level for determining outliers. If set to zero, iterations will 

1638 continue until there are no outliers. 

1639 doRankUpdate : `bool`, optional 

1640 Do an Eigen rank update during minimization, or recompute the full 

1641 matrix and gradient? 

1642 doLineSearch : `bool`, optional 

1643 Do a line search for the optimum step during minimization? 

1644 

1645 Returns 

1646 ------- 

1647 chi2: `lsst.jointcal.Chi2Statistic` 

1648 The final chi2 after the fit converges, or is forced to end. 

1649 

1650 Raises 

1651 ------ 

1652 FloatingPointError 

1653 Raised if the fitter fails with a non-finite value. 

1654 RuntimeError 

1655 Raised if the fitter fails for some other reason; 

1656 log messages will provide further details. 

1657 """ 

1658 if self.config.writeInitMatrix: 

1659 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}") 

1660 else: 

1661 dumpMatrixFile = "" 

1662 oldChi2 = lsst.jointcal.Chi2Statistic() 

1663 oldChi2.chi2 = float("inf") 

1664 for i in range(max_steps): 

1665 if self.config.writeChi2FilesOuterLoop: 

1666 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1667 else: 

1668 writeChi2Name = None 

1669 result = fitter.minimize(whatToFit, 

1670 self.config.outlierRejectSigma, 

1671 sigmaRelativeTolerance=sigmaRelativeTolerance, 

1672 doRankUpdate=doRankUpdate, 

1673 doLineSearch=doLineSearch, 

1674 dumpMatrixFile=dumpMatrixFile) 

1675 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1676 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1677 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1678 

1679 if result == MinimizeResult.Converged: 

1680 if doRankUpdate: 

1681 self.log.debug("fit has converged - no more outliers - redo minimization " 

1682 "one more time in case we have lost accuracy in rank update.") 

1683 # Redo minimization one more time in case we have lost accuracy in rank update 

1684 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma, 

1685 sigmaRelativeTolerance=sigmaRelativeTolerance) 

1686 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1687 

1688 # log a message for a large final chi2, TODO: DM-15247 for something better 

1689 if chi2.chi2/chi2.ndof >= 4.0: 

1690 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1691 

1692 break 

1693 elif result == MinimizeResult.Chi2Increased: 

1694 self.log.warn("Still some outliers remaining but chi2 increased - retry") 

1695 # Check whether the increase was large enough to cause trouble. 

1696 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1697 if chi2Ratio > 1.5: 

1698 self.log.warn('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1699 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1700 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1701 # appears that chi2 increases more than a factor of ~2 always 

1702 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1703 # Using 10 as the "failure" threshold gives some room between 

1704 # leaving a warning and bailing early. 

1705 if chi2Ratio > 10: 

1706 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1707 " Try setting one or more of the `writeChi2*` config fields and looking" 

1708 " at how individual star chi2-values evolve during the fit.") 

1709 raise RuntimeError(msg) 

1710 oldChi2 = chi2 

1711 elif result == MinimizeResult.NonFinite: 

1712 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1713 # TODO DM-12446: turn this into a "butler save" somehow. 

1714 fitter.saveChi2Contributions(filename+"{type}") 

1715 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1716 raise FloatingPointError(msg.format(filename)) 

1717 elif result == MinimizeResult.Failed: 

1718 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1719 else: 

1720 raise RuntimeError("Unxepected return code from minimize().") 

1721 else: 

1722 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1723 

1724 return chi2 

1725 

1726 def _make_output(self, ccdImageList, model, func): 

1727 """Return the internal jointcal models converted to the afw 

1728 structures that will be saved to disk. 

1729 

1730 Parameters 

1731 ---------- 

1732 ccdImageList : `lsst.jointcal.CcdImageList` 

1733 The list of CcdImages to get the output for. 

1734 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel` 

1735 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`. 

1736 func : `str` 

1737 The name of the function to call on ``model`` to get the converted 

1738 structure. Must accept an `lsst.jointcal.CcdImage`. 

1739 

1740 Returns 

1741 ------- 

1742 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or 

1743 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`] 

1744 The data to be saved, keyed on (visit, detector). 

1745 """ 

1746 output = {} 

1747 for ccdImage in ccdImageList: 

1748 ccd = ccdImage.ccdId 

1749 visit = ccdImage.visit 

1750 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd) 

1751 output[(visit, ccd)] = getattr(model, func)(ccdImage) 

1752 return output 

1753 

1754 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef): 

1755 """ 

1756 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 

1757 

1758 Parameters 

1759 ---------- 

1760 associations : `lsst.jointcal.Associations` 

1761 The star/reference star associations to fit. 

1762 model : `lsst.jointcal.AstrometryModel` 

1763 The astrometric model that was fit. 

1764 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1765 Dict of ccdImage identifiers to dataRefs that were fit. 

1766 """ 

1767 ccdImageList = associations.getCcdImageList() 

1768 output = self._make_output(ccdImageList, model, "makeSkyWcs") 

1769 for key, skyWcs in output.items(): 

1770 dataRef = visit_ccd_to_dataRef[key] 

1771 try: 

1772 dataRef.put(skyWcs, 'jointcal_wcs') 

1773 except pexExceptions.Exception as e: 

1774 self.log.fatal('Failed to write updated Wcs: %s', str(e)) 

1775 raise e 

1776 

1777 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef): 

1778 """ 

1779 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 

1780 

1781 Parameters 

1782 ---------- 

1783 associations : `lsst.jointcal.Associations` 

1784 The star/reference star associations to fit. 

1785 model : `lsst.jointcal.PhotometryModel` 

1786 The photoometric model that was fit. 

1787 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1788 Dict of ccdImage identifiers to dataRefs that were fit. 

1789 """ 

1790 

1791 ccdImageList = associations.getCcdImageList() 

1792 output = self._make_output(ccdImageList, model, "toPhotoCalib") 

1793 for key, photoCalib in output.items(): 

1794 dataRef = visit_ccd_to_dataRef[key] 

1795 try: 

1796 dataRef.put(photoCalib, 'jointcal_photoCalib') 

1797 except pexExceptions.Exception as e: 

1798 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e)) 

1799 raise e