Coverage for python/lsst/jointcal/jointcal.py: 17%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

639 statements  

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import dataclasses 

23import collections 

24import os 

25 

26import astropy.time 

27import numpy as np 

28import astropy.units as u 

29 

30import lsst.geom 

31import lsst.utils 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34from lsst.afw.image import fluxErrFromABMagErr 

35import lsst.pex.exceptions as pexExceptions 

36import lsst.afw.cameraGeom 

37import lsst.afw.table 

38import lsst.log 

39from lsst.obs.base import Instrument 

40from lsst.pipe.tasks.colorterms import ColortermLibrary 

41from lsst.verify import Job, Measurement 

42 

43from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask, 

44 ReferenceObjectLoader) 

45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

46from lsst.utils.timer import timeMethod 

47 

48from .dataIds import PerTractCcdDataIdContainer 

49 

50import lsst.jointcal 

51from lsst.jointcal import MinimizeResult 

52 

53__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"] 

54 

55Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

56Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

57 

58 

59# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

60def add_measurement(job, name, value): 

61 meas = Measurement(job.metrics[name], value) 

62 job.measurements.insert(meas) 

63 

64 

65class JointcalRunner(pipeBase.ButlerInitializedTaskRunner): 

66 """Subclass of TaskRunner for jointcalTask (gen2) 

67 

68 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 

69 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 

70 single dataRef, are are called repeatedly). This class transforms the processed 

71 arguments generated by the ArgumentParser into the arguments expected by 

72 Jointcal.runDataRef(). 

73 

74 See pipeBase.TaskRunner for more information. 

75 """ 

76 

77 @staticmethod 

78 def getTargetList(parsedCmd, **kwargs): 

79 """ 

80 Return a list of tuples per tract, each containing (dataRefs, kwargs). 

81 

82 Jointcal operates on lists of dataRefs simultaneously. 

83 """ 

84 kwargs['butler'] = parsedCmd.butler 

85 

86 # organize data IDs by tract 

87 refListDict = {} 

88 for ref in parsedCmd.id.refList: 

89 refListDict.setdefault(ref.dataId["tract"], []).append(ref) 

90 # we call runDataRef() once with each tract 

91 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())] 

92 return result 

93 

94 def __call__(self, args): 

95 """ 

96 Parameters 

97 ---------- 

98 args 

99 Arguments for Task.runDataRef() 

100 

101 Returns 

102 ------- 

103 pipe.base.Struct 

104 if self.doReturnResults is False: 

105 

106 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

107 

108 if self.doReturnResults is True: 

109 

110 - ``result``: the result of calling jointcal.runDataRef() 

111 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

112 """ 

113 exitStatus = 0 # exit status for shell 

114 

115 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef. 

116 dataRefList, kwargs = args 

117 butler = kwargs.pop('butler') 

118 task = self.TaskClass(config=self.config, log=self.log, butler=butler) 

119 result = None 

120 try: 

121 result = task.runDataRef(dataRefList, **kwargs) 

122 exitStatus = result.exitStatus 

123 job_path = butler.get('verify_job_filename') 

124 result.job.write(job_path[0]) 

125 except Exception as e: # catch everything, sort it out later. 

126 if self.doRaise: 

127 raise e 

128 else: 

129 exitStatus = 1 

130 eName = type(e).__name__ 

131 tract = dataRefList[0].dataId['tract'] 

132 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e) 

133 

134 # Put the butler back into kwargs for the other Tasks. 

135 kwargs['butler'] = butler 

136 if self.doReturnResults: 

137 return pipeBase.Struct(result=result, exitStatus=exitStatus) 

138 else: 

139 return pipeBase.Struct(exitStatus=exitStatus) 

140 

141 

142def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

143 """Lookup function that asserts/hopes that a static calibration dataset 

144 exists in a particular collection, since this task can't provide a single 

145 date/time to use to search for one properly. 

146 

147 This is mostly useful for the ``camera`` dataset, in cases where the task's 

148 quantum dimensions do *not* include something temporal, like ``exposure`` 

149 or ``visit``. 

150 

151 Parameters 

152 ---------- 

153 datasetType : `lsst.daf.butler.DatasetType` 

154 Type of dataset being searched for. 

155 registry : `lsst.daf.butler.Registry` 

156 Data repository registry to search. 

157 quantumDataId : `lsst.daf.butler.DataCoordinate` 

158 Data ID of the quantum this camera should match. 

159 collections : `Iterable` [ `str` ] 

160 Collections that should be searched - but this lookup function works 

161 by ignoring this in favor of a more-or-less hard-coded value. 

162 

163 Returns 

164 ------- 

165 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

166 Iterator over dataset references; should have only one element. 

167 

168 Notes 

169 ----- 

170 This implementation duplicates one in fgcmcal, and is at least quite 

171 similar to another in cp_pipe. This duplicate has the most documentation. 

172 Fixing this is DM-29661. 

173 """ 

174 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

175 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

176 return registry.queryDatasets(datasetType, 

177 dataId=quantumDataId, 

178 collections=[unboundedCollection], 

179 findFirst=True) 

180 

181 

182def lookupVisitRefCats(datasetType, registry, quantumDataId, collections): 

183 """Lookup function that finds all refcats for all visits that overlap a 

184 tract, rather than just the refcats that directly overlap the tract. 

185 

186 Parameters 

187 ---------- 

188 datasetType : `lsst.daf.butler.DatasetType` 

189 Type of dataset being searched for. 

190 registry : `lsst.daf.butler.Registry` 

191 Data repository registry to search. 

192 quantumDataId : `lsst.daf.butler.DataCoordinate` 

193 Data ID of the quantum; expected to be something we can use as a 

194 constraint to query for overlapping visits. 

195 collections : `Iterable` [ `str` ] 

196 Collections to search. 

197 

198 Returns 

199 ------- 

200 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

201 Iterator over refcat references. 

202 """ 

203 refs = set() 

204 # Use .expanded() on the query methods below because we need data IDs with 

205 # regions, both in the outer loop over visits (queryDatasets will expand 

206 # any data ID we give it, but doing it up-front in bulk is much more 

207 # efficient) and in the data IDs of the DatasetRefs this function yields 

208 # (because the RefCatLoader relies on them to do some of its own 

209 # filtering). 

210 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()): 

211 refs.update( 

212 registry.queryDatasets( 

213 datasetType, 

214 collections=collections, 

215 dataId=visit_data_id, 

216 findFirst=True, 

217 ).expanded() 

218 ) 

219 yield from refs 

220 

221 

222class JointcalTaskConnections(pipeBase.PipelineTaskConnections, 

223 dimensions=("skymap", "tract", "instrument", "physical_filter")): 

224 """Middleware input/output connections for jointcal data.""" 

225 inputCamera = pipeBase.connectionTypes.PrerequisiteInput( 

226 doc="The camera instrument that took these observations.", 

227 name="camera", 

228 storageClass="Camera", 

229 dimensions=("instrument",), 

230 isCalibration=True, 

231 lookupFunction=lookupStaticCalibrations, 

232 ) 

233 inputSourceTableVisit = pipeBase.connectionTypes.Input( 

234 doc="Source table in parquet format, per visit", 

235 name="sourceTable_visit", 

236 storageClass="DataFrame", 

237 dimensions=("instrument", "visit"), 

238 deferLoad=True, 

239 multiple=True, 

240 ) 

241 inputVisitSummary = pipeBase.connectionTypes.Input( 

242 doc=("Per-visit consolidated exposure metadata built from calexps. " 

243 "These catalogs use detector id for the id and must be sorted for " 

244 "fast lookups of a detector."), 

245 name="visitSummary", 

246 storageClass="ExposureCatalog", 

247 dimensions=("instrument", "visit"), 

248 deferLoad=True, 

249 multiple=True, 

250 ) 

251 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

252 doc="The astrometry reference catalog to match to loaded input catalog sources.", 

253 name="gaia_dr2_20200414", 

254 storageClass="SimpleCatalog", 

255 dimensions=("skypix",), 

256 deferLoad=True, 

257 multiple=True, 

258 lookupFunction=lookupVisitRefCats, 

259 ) 

260 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

261 doc="The photometry reference catalog to match to loaded input catalog sources.", 

262 name="ps1_pv3_3pi_20170110", 

263 storageClass="SimpleCatalog", 

264 dimensions=("skypix",), 

265 deferLoad=True, 

266 multiple=True, 

267 lookupFunction=lookupVisitRefCats, 

268 ) 

269 

270 outputWcs = pipeBase.connectionTypes.Output( 

271 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model." 

272 " These catalogs only contain entries for detectors with an output, and use" 

273 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

274 name="jointcalSkyWcsCatalog", 

275 storageClass="ExposureCatalog", 

276 dimensions=("instrument", "visit", "skymap", "tract"), 

277 multiple=True 

278 ) 

279 outputPhotoCalib = pipeBase.connectionTypes.Output( 

280 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model." 

281 " These catalogs only contain entries for detectors with an output, and use" 

282 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

283 name="jointcalPhotoCalibCatalog", 

284 storageClass="ExposureCatalog", 

285 dimensions=("instrument", "visit", "skymap", "tract"), 

286 multiple=True 

287 ) 

288 

289 # measurements of metrics 

290 # The vars() trick used here allows us to set class attributes 

291 # programatically. Taken from: 

292 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class 

293 for name in ("astrometry", "photometry"): 

294 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output( 

295 doc=f"The number of cross-matched fittedStars for {name}", 

296 name=f"metricvalue_jointcal_{name}_matched_fittedStars", 

297 storageClass="MetricValue", 

298 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

299 ) 

300 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output( 

301 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.", 

302 name=f"metricvalue_jointcal_{name}_collected_refStars", 

303 storageClass="MetricValue", 

304 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

305 ) 

306 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output( 

307 doc=f"The number of {name} reference stars matched to fittedStars.", 

308 name=f"metricvalue_jointcal_{name}_prepared_refStars", 

309 storageClass="MetricValue", 

310 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

311 ) 

312 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output( 

313 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.", 

314 name=f"metricvalue_jointcal_{name}_prepared_fittedStars", 

315 storageClass="MetricValue", 

316 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

317 ) 

318 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output( 

319 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.", 

320 name=f"metricvalue_jointcal_{name}_prepared_ccdImages", 

321 storageClass="MetricValue", 

322 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

323 ) 

324 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output( 

325 doc=f"The final chi2 of the {name} fit.", 

326 name=f"metricvalue_jointcal_{name}_final_chi2", 

327 storageClass="MetricValue", 

328 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

329 ) 

330 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output( 

331 doc=f"The number of degrees of freedom of the fitted {name}.", 

332 name=f"metricvalue_jointcal_{name}_final_ndof", 

333 storageClass="MetricValue", 

334 dimensions=("skymap", "tract", "instrument", "physical_filter"), 

335 ) 

336 

337 def __init__(self, *, config=None): 

338 super().__init__(config=config) 

339 # When we are only doing one of astrometry or photometry, we don't 

340 # need the reference catalog or produce the outputs for the other. 

341 # This informs the middleware of that when the QuantumGraph is 

342 # generated, so we don't block on getting something we won't need or 

343 # create an expectation that downstream tasks will be able to consume 

344 # something we won't produce. 

345 if not config.doAstrometry: 

346 self.prerequisiteInputs.remove("astrometryRefCat") 

347 self.outputs.remove("outputWcs") 

348 for key in list(self.outputs): 

349 if "metricvalue_jointcal_astrometry" in key: 

350 self.outputs.remove(key) 

351 if not config.doPhotometry: 

352 self.prerequisiteInputs.remove("photometryRefCat") 

353 self.outputs.remove("outputPhotoCalib") 

354 for key in list(self.outputs): 

355 if "metricvalue_jointcal_photometry" in key: 

356 self.outputs.remove(key) 

357 

358 

359class JointcalConfig(pipeBase.PipelineTaskConfig, 

360 pipelineConnections=JointcalTaskConnections): 

361 """Configuration for JointcalTask""" 

362 

363 doAstrometry = pexConfig.Field( 

364 doc="Fit astrometry and write the fitted result.", 

365 dtype=bool, 

366 default=True 

367 ) 

368 doPhotometry = pexConfig.Field( 

369 doc="Fit photometry and write the fitted result.", 

370 dtype=bool, 

371 default=True 

372 ) 

373 coaddName = pexConfig.Field( 

374 doc="Type of coadd, typically deep or goodSeeing", 

375 dtype=str, 

376 default="deep" 

377 ) 

378 # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal. 

379 sourceFluxType = pexConfig.Field( 

380 dtype=str, 

381 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

382 default='Calib' 

383 ) 

384 positionErrorPedestal = pexConfig.Field( 

385 doc="Systematic term to apply to the measured position error (pixels)", 

386 dtype=float, 

387 default=0.02, 

388 ) 

389 photometryErrorPedestal = pexConfig.Field( 

390 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

391 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

392 dtype=float, 

393 default=0.0, 

394 ) 

395 # TODO: DM-6885 matchCut should be an geom.Angle 

396 matchCut = pexConfig.Field( 

397 doc="Matching radius between fitted and reference stars (arcseconds)", 

398 dtype=float, 

399 default=3.0, 

400 ) 

401 minMeasurements = pexConfig.Field( 

402 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

403 dtype=int, 

404 default=2, 

405 ) 

406 minMeasuredStarsPerCcd = pexConfig.Field( 

407 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

408 dtype=int, 

409 default=100, 

410 ) 

411 minRefStarsPerCcd = pexConfig.Field( 

412 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

413 dtype=int, 

414 default=30, 

415 ) 

416 allowLineSearch = pexConfig.Field( 

417 doc="Allow a line search during minimization, if it is reasonable for the model" 

418 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

419 dtype=bool, 

420 default=False 

421 ) 

422 astrometrySimpleOrder = pexConfig.Field( 

423 doc="Polynomial order for fitting the simple astrometry model.", 

424 dtype=int, 

425 default=3, 

426 ) 

427 astrometryChipOrder = pexConfig.Field( 

428 doc="Order of the per-chip transform for the constrained astrometry model.", 

429 dtype=int, 

430 default=1, 

431 ) 

432 astrometryVisitOrder = pexConfig.Field( 

433 doc="Order of the per-visit transform for the constrained astrometry model.", 

434 dtype=int, 

435 default=5, 

436 ) 

437 useInputWcs = pexConfig.Field( 

438 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

439 dtype=bool, 

440 default=True, 

441 ) 

442 astrometryModel = pexConfig.ChoiceField( 

443 doc="Type of model to fit to astrometry", 

444 dtype=str, 

445 default="constrained", 

446 allowed={"simple": "One polynomial per ccd", 

447 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

448 ) 

449 photometryModel = pexConfig.ChoiceField( 

450 doc="Type of model to fit to photometry", 

451 dtype=str, 

452 default="constrainedMagnitude", 

453 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

454 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

455 " fitting in flux space.", 

456 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

457 " fitting in magnitude space.", 

458 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

459 " fitting in magnitude space.", 

460 } 

461 ) 

462 applyColorTerms = pexConfig.Field( 

463 doc="Apply photometric color terms to reference stars?" 

464 "Requires that colorterms be set to a ColortermLibrary", 

465 dtype=bool, 

466 default=False 

467 ) 

468 colorterms = pexConfig.ConfigField( 

469 doc="Library of photometric reference catalog name to color term dict.", 

470 dtype=ColortermLibrary, 

471 ) 

472 photometryVisitOrder = pexConfig.Field( 

473 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

474 dtype=int, 

475 default=7, 

476 ) 

477 photometryDoRankUpdate = pexConfig.Field( 

478 doc=("Do the rank update step during minimization. " 

479 "Skipping this can help deal with models that are too non-linear."), 

480 dtype=bool, 

481 default=True, 

482 ) 

483 astrometryDoRankUpdate = pexConfig.Field( 

484 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

485 "Skipping this can help deal with models that are too non-linear."), 

486 dtype=bool, 

487 default=True, 

488 ) 

489 outlierRejectSigma = pexConfig.Field( 

490 doc="How many sigma to reject outliers at during minimization.", 

491 dtype=float, 

492 default=5.0, 

493 ) 

494 astrometryOutlierRelativeTolerance = pexConfig.Field( 

495 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will " 

496 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set " 

497 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002" 

498 "as a balance between a shorter minimization runtime and achieving a final fitted model that is" 

499 "close to the solution found when removing all outliers."), 

500 dtype=float, 

501 default=0, 

502 ) 

503 maxPhotometrySteps = pexConfig.Field( 

504 doc="Maximum number of minimize iterations to take when fitting photometry.", 

505 dtype=int, 

506 default=20, 

507 ) 

508 maxAstrometrySteps = pexConfig.Field( 

509 doc="Maximum number of minimize iterations to take when fitting astrometry.", 

510 dtype=int, 

511 default=20, 

512 ) 

513 astrometryRefObjLoader = pexConfig.ConfigurableField( 

514 target=LoadIndexedReferenceObjectsTask, 

515 doc="Reference object loader for astrometric fit", 

516 ) 

517 photometryRefObjLoader = pexConfig.ConfigurableField( 

518 target=LoadIndexedReferenceObjectsTask, 

519 doc="Reference object loader for photometric fit", 

520 ) 

521 sourceSelector = sourceSelectorRegistry.makeField( 

522 doc="How to select sources for cross-matching", 

523 default="astrometry" 

524 ) 

525 astrometryReferenceSelector = pexConfig.ConfigurableField( 

526 target=ReferenceSourceSelectorTask, 

527 doc="How to down-select the loaded astrometry reference catalog.", 

528 ) 

529 photometryReferenceSelector = pexConfig.ConfigurableField( 

530 target=ReferenceSourceSelectorTask, 

531 doc="How to down-select the loaded photometry reference catalog.", 

532 ) 

533 astrometryReferenceErr = pexConfig.Field( 

534 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

535 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

536 "If specified, overrides any existing `coord_*Err` values."), 

537 dtype=float, 

538 default=None, 

539 optional=True 

540 ) 

541 

542 # configs for outputting debug information 

543 writeInitMatrix = pexConfig.Field( 

544 dtype=bool, 

545 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

546 "Output files will be written to `config.debugOutputPath` and will " 

547 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. " 

548 "Note that these files are the dense versions of the matrix, and so may be very large."), 

549 default=False 

550 ) 

551 writeChi2FilesInitialFinal = pexConfig.Field( 

552 dtype=bool, 

553 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. " 

554 "Output files will be written to `config.debugOutputPath` and will " 

555 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."), 

556 default=False 

557 ) 

558 writeChi2FilesOuterLoop = pexConfig.Field( 

559 dtype=bool, 

560 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. " 

561 "Output files will be written to `config.debugOutputPath` and will " 

562 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."), 

563 default=False 

564 ) 

565 writeInitialModel = pexConfig.Field( 

566 dtype=bool, 

567 doc=("Write the pre-initialization model to text files, for debugging. " 

568 "Output files will be written to `config.debugOutputPath` and will be " 

569 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."), 

570 default=False 

571 ) 

572 debugOutputPath = pexConfig.Field( 

573 dtype=str, 

574 default=".", 

575 doc=("Path to write debug output files to. Used by " 

576 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

577 ) 

578 detailedProfile = pexConfig.Field( 

579 dtype=bool, 

580 default=False, 

581 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting" 

582 ) 

583 

584 def validate(self): 

585 super().validate() 

586 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

587 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

588 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

589 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

590 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

591 "applyColorTerms=True will be ignored.") 

592 lsst.log.warning(msg) 

593 

594 def setDefaults(self): 

595 # Use science source selector which can filter on extendedness, SNR, and whether blended 

596 self.sourceSelector.name = 'science' 

597 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing 

598 self.sourceSelector['science'].doUnresolved = True 

599 # with dependable signal to noise ratio. 

600 self.sourceSelector['science'].doSignalToNoise = True 

601 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

602 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

603 self.sourceSelector['science'].signalToNoise.minimum = 10. 

604 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive 

605 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux" 

606 self.sourceSelector['science'].signalToNoise.fluxField = fluxField 

607 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err" 

608 # Do not trust blended sources' aperture fluxes which also depend on seeing 

609 self.sourceSelector['science'].doIsolated = True 

610 # Do not trust either flux or centroid measurements with flags, 

611 # chosen from the usual QA flags for stars) 

612 self.sourceSelector['science'].doFlags = True 

613 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 

614 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag', 

615 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter'] 

616 self.sourceSelector['science'].flags.bad = badFlags 

617 

618 # Default to Gaia-DR2 (with proper motions) for astrometry and 

619 # PS1-DR1 for photometry, with a reasonable initial filterMap. 

620 self.astrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414" 

621 self.astrometryRefObjLoader.requireProperMotion = True 

622 self.astrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean' 

623 self.photometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110" 

624 

625 

626def writeModel(model, filename, log): 

627 """Write model to outfile.""" 

628 with open(filename, "w") as file: 

629 file.write(repr(model)) 

630 log.info("Wrote %s to file: %s", model, filename) 

631 

632 

633@dataclasses.dataclass 

634class JointcalInputData: 

635 """The input data jointcal needs for each detector/visit.""" 

636 visit: int 

637 """The visit identifier of this exposure.""" 

638 catalog: lsst.afw.table.SourceCatalog 

639 """The catalog derived from this exposure.""" 

640 visitInfo: lsst.afw.image.VisitInfo 

641 """The VisitInfo of this exposure.""" 

642 detector: lsst.afw.cameraGeom.Detector 

643 """The detector of this exposure.""" 

644 photoCalib: lsst.afw.image.PhotoCalib 

645 """The photometric calibration of this exposure.""" 

646 wcs: lsst.afw.geom.skyWcs 

647 """The WCS of this exposure.""" 

648 bbox: lsst.geom.Box2I 

649 """The bounding box of this exposure.""" 

650 filter: lsst.afw.image.FilterLabel 

651 """The filter of this exposure.""" 

652 

653 

654class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

655 """Astrometricly and photometricly calibrate across multiple visits of the 

656 same field. 

657 

658 Parameters 

659 ---------- 

660 butler : `lsst.daf.persistence.Butler` 

661 The butler is passed to the refObjLoader constructor in case it is 

662 needed. Ignored if the refObjLoader argument provides a loader directly. 

663 Used to initialize the astrometry and photometry refObjLoaders. 

664 initInputs : `dict`, optional 

665 Dictionary used to initialize PipelineTasks (empty for jointcal). 

666 """ 

667 

668 ConfigClass = JointcalConfig 

669 RunnerClass = JointcalRunner 

670 _DefaultName = "jointcal" 

671 

672 def __init__(self, butler=None, initInputs=None, **kwargs): 

673 super().__init__(**kwargs) 

674 self.makeSubtask("sourceSelector") 

675 if self.config.doAstrometry: 

676 if initInputs is None: 

677 # gen3 middleware does refcat things internally (and will not have a butler here) 

678 self.makeSubtask('astrometryRefObjLoader', butler=butler) 

679 self.makeSubtask("astrometryReferenceSelector") 

680 else: 

681 self.astrometryRefObjLoader = None 

682 if self.config.doPhotometry: 

683 if initInputs is None: 

684 # gen3 middleware does refcat things internally (and will not have a butler here) 

685 self.makeSubtask('photometryRefObjLoader', butler=butler) 

686 self.makeSubtask("photometryReferenceSelector") 

687 else: 

688 self.photometryRefObjLoader = None 

689 

690 # To hold various computed metrics for use by tests 

691 self.job = Job.load_metrics_package(subset='jointcal') 

692 

693 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

694 # We override runQuantum to set up the refObjLoaders and write the 

695 # outputs to the correct refs. 

696 inputs = butlerQC.get(inputRefs) 

697 # We want the tract number for writing debug files 

698 tract = butlerQC.quantum.dataId['tract'] 

699 if self.config.doAstrometry: 

700 self.astrometryRefObjLoader = ReferenceObjectLoader( 

701 dataIds=[ref.datasetRef.dataId 

702 for ref in inputRefs.astrometryRefCat], 

703 refCats=inputs.pop('astrometryRefCat'), 

704 config=self.config.astrometryRefObjLoader, 

705 log=self.log) 

706 if self.config.doPhotometry: 

707 self.photometryRefObjLoader = ReferenceObjectLoader( 

708 dataIds=[ref.datasetRef.dataId 

709 for ref in inputRefs.photometryRefCat], 

710 refCats=inputs.pop('photometryRefCat'), 

711 config=self.config.photometryRefObjLoader, 

712 log=self.log) 

713 outputs = self.run(**inputs, tract=tract) 

714 self._put_metrics(butlerQC, outputs.job, outputRefs) 

715 if self.config.doAstrometry: 

716 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs, 

717 inputs['inputCamera'], "setWcs") 

718 if self.config.doPhotometry: 

719 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib, 

720 inputs['inputCamera'], "setPhotoCalib") 

721 

722 def _put_metrics(self, butlerQC, job, outputRefs): 

723 """Persist all measured metrics stored in a job. 

724 

725 Parameters 

726 ---------- 

727 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

728 A butler which is specialized to operate in the context of a 

729 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

730 job : `lsst.verify.job.Job` 

731 Measurements of metrics to persist. 

732 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

733 The DatasetRefs to persist the data to. 

734 """ 

735 for key in job.measurements.keys(): 

736 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', ''))) 

737 

738 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter): 

739 """Persist the output datasets to their appropriate datarefs. 

740 

741 Parameters 

742 ---------- 

743 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

744 A butler which is specialized to operate in the context of a 

745 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

746 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or 

747 `dict` [`tuple, `lsst.afw.image.PhotoCalib`] 

748 The fitted objects to persist. 

749 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

750 The DatasetRefs to persist the data to. 

751 camera : `lsst.afw.cameraGeom.Camera` 

752 The camera for this instrument, to get detector ids from. 

753 setter : `str` 

754 The method to call on the ExposureCatalog to set each output. 

755 """ 

756 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

757 schema.addField('visit', type='I', doc='Visit number') 

758 

759 def new_catalog(visit, size): 

760 """Return an catalog ready to be filled with appropriate output.""" 

761 catalog = lsst.afw.table.ExposureCatalog(schema) 

762 catalog.resize(size) 

763 catalog['visit'] = visit 

764 metadata = lsst.daf.base.PropertyList() 

765 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

766 metadata.add("COMMENT", "Only detectors with data have entries.") 

767 return catalog 

768 

769 # count how many detectors have output for each visit 

770 detectors_per_visit = collections.defaultdict(int) 

771 for key in outputs: 

772 # key is (visit, detector_id), and we only need visit here 

773 detectors_per_visit[key[0]] += 1 

774 

775 for ref in outputRefs: 

776 visit = ref.dataId['visit'] 

777 catalog = new_catalog(visit, detectors_per_visit[visit]) 

778 # Iterate over every detector and skip the ones we don't have output for. 

779 i = 0 

780 for detector in camera: 

781 detectorId = detector.getId() 

782 key = (ref.dataId['visit'], detectorId) 

783 if key not in outputs: 

784 # skip detectors we don't have output for 

785 self.log.debug("No %s output for detector %s in visit %s", 

786 setter[3:], detectorId, visit) 

787 continue 

788 

789 catalog[i].setId(detectorId) 

790 getattr(catalog[i], setter)(outputs[key]) 

791 i += 1 

792 

793 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups 

794 butlerQC.put(catalog, ref) 

795 self.log.info("Wrote %s detectors to %s", i, ref) 

796 

797 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None): 

798 # Docstring inherited. 

799 

800 # We take values out of the Parquet table, and put them in "flux_", 

801 # and the config.sourceFluxType field is used during that extraction, 

802 # so just use "flux" here. 

803 sourceFluxField = "flux" 

804 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

805 associations = lsst.jointcal.Associations() 

806 self.focalPlaneBBox = inputCamera.getFpBBox() 

807 oldWcsList, bands = self._load_data(inputSourceTableVisit, 

808 inputVisitSummary, 

809 associations, 

810 jointcalControl, 

811 inputCamera) 

812 

813 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands) 

814 

815 if self.config.doAstrometry: 

816 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

817 name="astrometry", 

818 refObjLoader=self.astrometryRefObjLoader, 

819 referenceSelector=self.astrometryReferenceSelector, 

820 fit_function=self._fit_astrometry, 

821 tract=tract, 

822 epoch=epoch) 

823 astrometry_output = self._make_output(associations.getCcdImageList(), 

824 astrometry.model, 

825 "makeSkyWcs") 

826 else: 

827 astrometry_output = None 

828 

829 if self.config.doPhotometry: 

830 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

831 name="photometry", 

832 refObjLoader=self.photometryRefObjLoader, 

833 referenceSelector=self.photometryReferenceSelector, 

834 fit_function=self._fit_photometry, 

835 tract=tract, 

836 epoch=epoch, 

837 reject_bad_fluxes=True) 

838 photometry_output = self._make_output(associations.getCcdImageList(), 

839 photometry.model, 

840 "toPhotoCalib") 

841 else: 

842 photometry_output = None 

843 

844 return pipeBase.Struct(outputWcs=astrometry_output, 

845 outputPhotoCalib=photometry_output, 

846 job=self.job, 

847 astrometryRefObjLoader=self.astrometryRefObjLoader, 

848 photometryRefObjLoader=self.photometryRefObjLoader) 

849 

850 def _make_schema_table(self): 

851 """Return an afw SourceTable to use as a base for creating the 

852 SourceCatalog to insert values from the dataFrame into. 

853 

854 Returns 

855 ------- 

856 table : `lsst.afw.table.SourceTable` 

857 Table with schema and slots to use to make SourceCatalogs. 

858 """ 

859 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

860 schema.addField("centroid_x", "D") 

861 schema.addField("centroid_y", "D") 

862 schema.addField("centroid_xErr", "F") 

863 schema.addField("centroid_yErr", "F") 

864 schema.addField("shape_xx", "D") 

865 schema.addField("shape_yy", "D") 

866 schema.addField("shape_xy", "D") 

867 schema.addField("flux_instFlux", "D") 

868 schema.addField("flux_instFluxErr", "D") 

869 table = lsst.afw.table.SourceTable.make(schema) 

870 table.defineCentroid("centroid") 

871 table.defineShape("shape") 

872 return table 

873 

874 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId, 

875 detectorColumn, ixxColumns): 

876 """Return an afw SourceCatalog extracted from a visit-level dataframe, 

877 limited to just one detector. 

878 

879 Parameters 

880 ---------- 

881 table : `lsst.afw.table.SourceTable` 

882 Table factory to use to make the SourceCatalog that will be 

883 populated with data from ``visitCatalog``. 

884 visitCatalog : `pandas.DataFrame` 

885 DataFrame to extract a detector catalog from. 

886 detectorId : `int` 

887 Numeric id of the detector to extract from ``visitCatalog``. 

888 detectorColumn : `str` 

889 Name of the detector column in the catalog. 

890 ixxColumns : `list` [`str`] 

891 Names of the ixx/iyy/ixy columns in the catalog. 

892 

893 Returns 

894 ------- 

895 catalog : `lsst.afw.table.SourceCatalog` 

896 Detector-level catalog extracted from ``visitCatalog``. 

897 """ 

898 # map from dataFrame column to afw table column 

899 mapping = {'x': 'centroid_x', 

900 'y': 'centroid_y', 

901 'xErr': 'centroid_xErr', 

902 'yErr': 'centroid_yErr', 

903 ixxColumns[0]: 'shape_xx', 

904 ixxColumns[1]: 'shape_yy', 

905 ixxColumns[2]: 'shape_xy', 

906 f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux', 

907 f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr', 

908 } 

909 

910 catalog = lsst.afw.table.SourceCatalog(table) 

911 matched = visitCatalog[detectorColumn] == detectorId 

912 catalog.resize(sum(matched)) 

913 view = visitCatalog.loc[matched] 

914 catalog['id'] = view.index 

915 for dfCol, afwCol in mapping.items(): 

916 catalog[afwCol] = view[dfCol] 

917 

918 self.log.debug("%d sources selected in visit %d detector %d", 

919 len(catalog), 

920 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first 

921 detectorId) 

922 return catalog 

923 

924 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, 

925 jointcalControl, camera): 

926 """Read the data that jointcal needs to run. (Gen3 version) 

927 

928 Modifies ``associations`` in-place with the loaded data. 

929 

930 Parameters 

931 ---------- 

932 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

933 References to visit-level DataFrames to load the catalog data from. 

934 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

935 Visit-level exposure summary catalog with metadata. 

936 associations : `lsst.jointcal.Associations` 

937 Object to add the loaded data to by constructing new CcdImages. 

938 jointcalControl : `jointcal.JointcalControl` 

939 Control object for C++ associations management. 

940 camera : `lsst.afw.cameraGeom.Camera` 

941 Camera object for detector geometry. 

942 

943 Returns 

944 ------- 

945 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`] 

946 The original WCS of the input data, to aid in writing tests. 

947 bands : `list` [`str`] 

948 The filter bands of each input dataset. 

949 """ 

950 oldWcsList = [] 

951 filters = [] 

952 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else '' 

953 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

954 table = self._make_schema_table() # every detector catalog has the same layout 

955 # No guarantee that the input is in the same order of visits, so we have to map one of them. 

956 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)} 

957 detectorDict = {detector.getId(): detector for detector in camera} 

958 

959 columns = None 

960 

961 for visitSummaryRef in inputVisitSummary: 

962 visitSummary = visitSummaryRef.get() 

963 

964 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]] 

965 if columns is None: 

966 inColumns = dataRef.get(component='columns') 

967 columns, detColumn, ixxColumns = self._get_sourceTable_visit_columns(inColumns) 

968 visitCatalog = dataRef.get(parameters={'columns': columns}) 

969 

970 selected = self.sourceSelector.run(visitCatalog) 

971 

972 # Build a CcdImage for each detector in this visit. 

973 detectors = {id: index for index, id in enumerate(visitSummary['id'])} 

974 for id, index in detectors.items(): 

975 catalog = self._extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id, 

976 detColumn, ixxColumns) 

977 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict) 

978 result = self._build_ccdImage(data, associations, jointcalControl) 

979 if result is not None: 

980 oldWcsList.append(result.wcs) 

981 # A visit has only one band, so we can just use the first. 

982 filters.append(data.filter) 

983 if len(filters) == 0: 

984 raise RuntimeError("No data to process: did source selector remove all sources?") 

985 filters = collections.Counter(filters) 

986 

987 return oldWcsList, filters 

988 

989 def _make_one_input_data(self, visitRecord, catalog, detectorDict): 

990 """Return a data structure for this detector+visit.""" 

991 return JointcalInputData(visit=visitRecord['visit'], 

992 catalog=catalog, 

993 visitInfo=visitRecord.getVisitInfo(), 

994 detector=detectorDict[visitRecord.getId()], 

995 photoCalib=visitRecord.getPhotoCalib(), 

996 wcs=visitRecord.getWcs(), 

997 bbox=visitRecord.getBBox(), 

998 # ExposureRecord doesn't have a FilterLabel yet, 

999 # so we have to make one. 

1000 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'], 

1001 physical=visitRecord['physical_filter'])) 

1002 

1003 def _get_sourceTable_visit_columns(self, inColumns): 

1004 """ 

1005 Get the sourceTable_visit columns from the config. 

1006 

1007 Parameters 

1008 ---------- 

1009 inColumns : `list` 

1010 List of columns available in the sourceTable_visit 

1011 

1012 Returns 

1013 ------- 

1014 columns : `list` 

1015 List of columns to read from sourceTable_visit. 

1016 detectorColumn : `str` 

1017 Name of the detector column. 

1018 ixxColumns : `list` 

1019 Name of the ixx/iyy/ixy columns. 

1020 """ 

1021 if 'detector' in inColumns: 

1022 # Default name for Gen3. 

1023 detectorColumn = 'detector' 

1024 else: 

1025 # Default name for Gen2 and Gen2 conversions. 

1026 detectorColumn = 'ccd' 

1027 

1028 columns = ['visit', detectorColumn, 

1029 'sourceId', 'x', 'xErr', 'y', 'yErr', 

1030 self.config.sourceFluxType + '_instFlux', self.config.sourceFluxType + '_instFluxErr'] 

1031 

1032 if 'ixx' in inColumns: 

1033 # New columns post-DM-31825 

1034 ixxColumns = ['ixx', 'iyy', 'ixy'] 

1035 else: 

1036 # Old columns pre-DM-31825 

1037 ixxColumns = ['Ixx', 'Iyy', 'Ixy'] 

1038 columns.extend(ixxColumns) 

1039 

1040 if self.sourceSelector.config.doFlags: 

1041 columns.extend(self.sourceSelector.config.flags.bad) 

1042 if self.sourceSelector.config.doUnresolved: 

1043 columns.append(self.sourceSelector.config.unresolved.name) 

1044 if self.sourceSelector.config.doIsolated: 

1045 columns.append(self.sourceSelector.config.isolated.parentName) 

1046 columns.append(self.sourceSelector.config.isolated.nChildName) 

1047 

1048 return columns, detectorColumn, ixxColumns 

1049 

1050 # We don't currently need to persist the metadata. 

1051 # If we do in the future, we will have to add appropriate dataset templates 

1052 # to each obs package (the metadata template should look like `jointcal_wcs`). 

1053 def _getMetadataName(self): 

1054 return None 

1055 

1056 @classmethod 

1057 def _makeArgumentParser(cls): 

1058 """Create an argument parser""" 

1059 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1060 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9", 

1061 ContainerClass=PerTractCcdDataIdContainer) 

1062 return parser 

1063 

1064 def _build_ccdImage(self, data, associations, jointcalControl): 

1065 """ 

1066 Extract the necessary things from this catalog+metadata to add a new 

1067 ccdImage. 

1068 

1069 Parameters 

1070 ---------- 

1071 data : `JointcalInputData` 

1072 The loaded input data. 

1073 associations : `lsst.jointcal.Associations` 

1074 Object to add the info to, to construct a new CcdImage 

1075 jointcalControl : `jointcal.JointcalControl` 

1076 Control object for associations management 

1077 

1078 Returns 

1079 ------ 

1080 namedtuple or `None` 

1081 ``wcs`` 

1082 The TAN WCS of this image, read from the calexp 

1083 (`lsst.afw.geom.SkyWcs`). 

1084 ``key`` 

1085 A key to identify this dataRef by its visit and ccd ids 

1086 (`namedtuple`). 

1087 `None` 

1088 if there are no sources in the loaded catalog. 

1089 """ 

1090 if len(data.catalog) == 0: 

1091 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId()) 

1092 return None 

1093 

1094 associations.createCcdImage(data.catalog, 

1095 data.wcs, 

1096 data.visitInfo, 

1097 data.bbox, 

1098 data.filter.physicalLabel, 

1099 data.photoCalib, 

1100 data.detector, 

1101 data.visit, 

1102 data.detector.getId(), 

1103 jointcalControl) 

1104 

1105 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key')) 

1106 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

1107 return Result(data.wcs, Key(data.visit, data.detector.getId())) 

1108 

1109 def _readDataId(self, butler, dataId): 

1110 """Read all of the data for one dataId from the butler. (gen2 version)""" 

1111 # Not all instruments have `visit` in their dataIds. 

1112 if "visit" in dataId.keys(): 

1113 visit = dataId["visit"] 

1114 else: 

1115 visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0] 

1116 detector = butler.get('calexp_detector', dataId=dataId) 

1117 

1118 catalog = butler.get('src', 

1119 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, 

1120 dataId=dataId) 

1121 goodSrc = self.sourceSelector.run(catalog) 

1122 self.log.debug("%d sources selected in visit %d detector %d", 

1123 len(goodSrc.sourceCat), 

1124 visit, 

1125 detector.getId()) 

1126 return JointcalInputData(visit=visit, 

1127 catalog=goodSrc.sourceCat, 

1128 visitInfo=butler.get('calexp_visitInfo', dataId=dataId), 

1129 detector=detector, 

1130 photoCalib=butler.get('calexp_photoCalib', dataId=dataId), 

1131 wcs=butler.get('calexp_wcs', dataId=dataId), 

1132 bbox=butler.get('calexp_bbox', dataId=dataId), 

1133 filter=butler.get('calexp_filterLabel', dataId=dataId)) 

1134 

1135 def loadData(self, dataRefs, associations, jointcalControl): 

1136 """Read the data that jointcal needs to run. (Gen2 version)""" 

1137 visit_ccd_to_dataRef = {} 

1138 oldWcsList = [] 

1139 filters = [] 

1140 load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else '' 

1141 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

1142 # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models 

1143 camera = dataRefs[0].get('camera', immediate=True) 

1144 self.focalPlaneBBox = camera.getFpBBox() 

1145 for dataRef in dataRefs: 

1146 data = self._readDataId(dataRef.getButler(), dataRef.dataId) 

1147 result = self._build_ccdImage(data, associations, jointcalControl) 

1148 if result is None: 

1149 continue 

1150 oldWcsList.append(result.wcs) 

1151 visit_ccd_to_dataRef[result.key] = dataRef 

1152 filters.append(data.filter) 

1153 if len(filters) == 0: 

1154 raise RuntimeError("No data to process: did source selector remove all sources?") 

1155 filters = collections.Counter(filters) 

1156 

1157 return oldWcsList, filters, visit_ccd_to_dataRef 

1158 

1159 def _getDebugPath(self, filename): 

1160 """Constructs a path to filename using the configured debug path. 

1161 """ 

1162 return os.path.join(self.config.debugOutputPath, filename) 

1163 

1164 def _prep_sky(self, associations, filters): 

1165 """Prepare on-sky and other data that must be computed after data has 

1166 been read. 

1167 """ 

1168 associations.computeCommonTangentPoint() 

1169 

1170 boundingCircle = associations.computeBoundingCircle() 

1171 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

1172 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

1173 

1174 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

1175 

1176 # Determine a default filter band associated with the catalog. See DM-9093 

1177 defaultFilter = filters.most_common(1)[0][0] 

1178 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel) 

1179 

1180 # compute and set the reference epoch of the observations, for proper motion corrections 

1181 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

1182 associations.setEpoch(epoch.jyear) 

1183 

1184 return boundingCircle, center, radius, defaultFilter, epoch 

1185 

1186 @timeMethod 

1187 def runDataRef(self, dataRefs): 

1188 """ 

1189 Jointly calibrate the astrometry and photometry across a set of images. 

1190 

1191 NOTE: this is for gen2 middleware only. 

1192 

1193 Parameters 

1194 ---------- 

1195 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

1196 List of data references to the exposures to be fit. 

1197 

1198 Returns 

1199 ------- 

1200 result : `lsst.pipe.base.Struct` 

1201 Struct of metadata from the fit, containing: 

1202 

1203 ``dataRefs`` 

1204 The provided data references that were fit (with updated WCSs) 

1205 ``oldWcsList`` 

1206 The original WCS from each dataRef 

1207 ``metrics`` 

1208 Dictionary of internally-computed metrics for testing/validation. 

1209 """ 

1210 if len(dataRefs) == 0: 

1211 raise ValueError('Need a non-empty list of data references!') 

1212 

1213 exitStatus = 0 # exit status for shell 

1214 

1215 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,) 

1216 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

1217 associations = lsst.jointcal.Associations() 

1218 

1219 oldWcsList, filters, visit_ccd_to_dataRef = self.loadData(dataRefs, 

1220 associations, 

1221 jointcalControl) 

1222 

1223 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, filters) 

1224 

1225 tract = dataRefs[0].dataId['tract'] 

1226 

1227 if self.config.doAstrometry: 

1228 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

1229 name="astrometry", 

1230 refObjLoader=self.astrometryRefObjLoader, 

1231 referenceSelector=self.astrometryReferenceSelector, 

1232 fit_function=self._fit_astrometry, 

1233 tract=tract, 

1234 epoch=epoch) 

1235 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef) 

1236 else: 

1237 astrometry = Astrometry(None, None, None) 

1238 

1239 if self.config.doPhotometry: 

1240 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

1241 name="photometry", 

1242 refObjLoader=self.photometryRefObjLoader, 

1243 referenceSelector=self.photometryReferenceSelector, 

1244 fit_function=self._fit_photometry, 

1245 tract=tract, 

1246 epoch=epoch, 

1247 reject_bad_fluxes=True) 

1248 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef) 

1249 else: 

1250 photometry = Photometry(None, None) 

1251 

1252 return pipeBase.Struct(dataRefs=dataRefs, 

1253 oldWcsList=oldWcsList, 

1254 job=self.job, 

1255 astrometryRefObjLoader=self.astrometryRefObjLoader, 

1256 photometryRefObjLoader=self.photometryRefObjLoader, 

1257 defaultFilter=defaultFilter, 

1258 epoch=epoch, 

1259 exitStatus=exitStatus) 

1260 

1261 def _get_refcat_coordinate_error_override(self, refCat, name): 

1262 """Check whether we should override the refcat coordinate errors, and 

1263 return the overridden error if necessary. 

1264 

1265 Parameters 

1266 ---------- 

1267 refCat : `lsst.afw.table.SimpleCatalog` 

1268 The reference catalog to check for a ``coord_raErr`` field. 

1269 name : `str` 

1270 Whether we are doing "astrometry" or "photometry". 

1271 

1272 Returns 

1273 ------- 

1274 refCoordErr : `float` 

1275 The refcat coordinate error to use, or NaN if we are not overriding 

1276 those fields. 

1277 

1278 Raises 

1279 ------ 

1280 lsst.pex.config.FieldValidationError 

1281 Raised if the refcat does not contain coordinate errors and 

1282 ``config.astrometryReferenceErr`` is not set. 

1283 """ 

1284 # This value doesn't matter for photometry, so just set something to 

1285 # keep old refcats from causing problems. 

1286 if name.lower() == "photometry": 

1287 if 'coord_raErr' not in refCat.schema: 

1288 return 100 

1289 else: 

1290 return float('nan') 

1291 

1292 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

1293 msg = ("Reference catalog does not contain coordinate errors, " 

1294 "and config.astrometryReferenceErr not supplied.") 

1295 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

1296 self.config, 

1297 msg) 

1298 

1299 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

1300 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

1301 self.config.astrometryReferenceErr) 

1302 

1303 if self.config.astrometryReferenceErr is None: 

1304 return float('nan') 

1305 else: 

1306 return self.config.astrometryReferenceErr 

1307 

1308 def _compute_proper_motion_epoch(self, ccdImageList): 

1309 """Return the proper motion correction epoch of the provided images. 

1310 

1311 Parameters 

1312 ---------- 

1313 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

1314 The images to compute the appropriate epoch for. 

1315 

1316 Returns 

1317 ------- 

1318 epoch : `astropy.time.Time` 

1319 The date to use for proper motion corrections. 

1320 """ 

1321 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]), 

1322 format="jyear", 

1323 scale="tai") 

1324 

1325 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

1326 tract="", match_cut=3.0, 

1327 reject_bad_fluxes=False, *, 

1328 name="", refObjLoader=None, referenceSelector=None, 

1329 fit_function=None, epoch=None): 

1330 """Load reference catalog, perform the fit, and return the result. 

1331 

1332 Parameters 

1333 ---------- 

1334 associations : `lsst.jointcal.Associations` 

1335 The star/reference star associations to fit. 

1336 defaultFilter : `lsst.afw.image.FilterLabel` 

1337 filter to load from reference catalog. 

1338 center : `lsst.geom.SpherePoint` 

1339 ICRS center of field to load from reference catalog. 

1340 radius : `lsst.geom.Angle` 

1341 On-sky radius to load from reference catalog. 

1342 name : `str` 

1343 Name of thing being fit: "astrometry" or "photometry". 

1344 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

1345 Reference object loader to use to load a reference catalog. 

1346 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1347 Selector to use to pick objects from the loaded reference catalog. 

1348 fit_function : callable 

1349 Function to call to perform fit (takes Associations object). 

1350 tract : `str`, optional 

1351 Name of tract currently being fit. 

1352 match_cut : `float`, optional 

1353 Radius in arcseconds to find cross-catalog matches to during 

1354 associations.associateCatalogs. 

1355 reject_bad_fluxes : `bool`, optional 

1356 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

1357 epoch : `astropy.time.Time`, optional 

1358 Epoch to which to correct refcat proper motion and parallax, 

1359 or `None` to not apply such corrections. 

1360 

1361 Returns 

1362 ------- 

1363 result : `Photometry` or `Astrometry` 

1364 Result of `fit_function()` 

1365 """ 

1366 self.log.info("====== Now processing %s...", name) 

1367 # TODO: this should not print "trying to invert a singular transformation:" 

1368 # if it does that, something's not right about the WCS... 

1369 associations.associateCatalogs(match_cut) 

1370 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name, 

1371 associations.fittedStarListSize()) 

1372 

1373 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

1374 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

1375 center, radius, defaultFilter, 

1376 applyColorterms=applyColorterms, 

1377 epoch=epoch) 

1378 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

1379 

1380 associations.collectRefStars(refCat, 

1381 self.config.matchCut*lsst.geom.arcseconds, 

1382 fluxField, 

1383 refCoordinateErr=refCoordErr, 

1384 rejectBadFluxes=reject_bad_fluxes) 

1385 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name, 

1386 associations.refStarListSize()) 

1387 

1388 associations.prepareFittedStars(self.config.minMeasurements) 

1389 

1390 self._check_star_lists(associations, name) 

1391 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name, 

1392 associations.nFittedStarsWithAssociatedRefStar()) 

1393 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name, 

1394 associations.fittedStarListSize()) 

1395 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name, 

1396 associations.nCcdImagesValidForFit()) 

1397 

1398 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else '' 

1399 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel) 

1400 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

1401 result = fit_function(associations, dataName) 

1402 # TODO DM-12446: turn this into a "butler save" somehow. 

1403 # Save reference and measurement chi2 contributions for this data 

1404 if self.config.writeChi2FilesInitialFinal: 

1405 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

1406 result.fit.saveChi2Contributions(baseName+"{type}") 

1407 self.log.info("Wrote chi2 contributions files: %s", baseName) 

1408 

1409 return result 

1410 

1411 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, 

1412 applyColorterms=False, epoch=None): 

1413 """Load the necessary reference catalog sources, convert fluxes to 

1414 correct units, and apply color term corrections if requested. 

1415 

1416 Parameters 

1417 ---------- 

1418 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

1419 The reference catalog loader to use to get the data. 

1420 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1421 Source selector to apply to loaded reference catalog. 

1422 center : `lsst.geom.SpherePoint` 

1423 The center around which to load sources. 

1424 radius : `lsst.geom.Angle` 

1425 The radius around ``center`` to load sources in. 

1426 filterLabel : `lsst.afw.image.FilterLabel` 

1427 The camera filter to load fluxes for. 

1428 applyColorterms : `bool` 

1429 Apply colorterm corrections to the refcat for ``filterName``? 

1430 epoch : `astropy.time.Time`, optional 

1431 Epoch to which to correct refcat proper motion and parallax, 

1432 or `None` to not apply such corrections. 

1433 

1434 Returns 

1435 ------- 

1436 refCat : `lsst.afw.table.SimpleCatalog` 

1437 The loaded reference catalog. 

1438 fluxField : `str` 

1439 The name of the reference catalog flux field appropriate for ``filterName``. 

1440 """ 

1441 skyCircle = refObjLoader.loadSkyCircle(center, 

1442 radius, 

1443 filterLabel.bandLabel, 

1444 epoch=epoch) 

1445 

1446 selected = referenceSelector.run(skyCircle.refCat) 

1447 # Need memory contiguity to get reference filters as a vector. 

1448 if not selected.sourceCat.isContiguous(): 

1449 refCat = selected.sourceCat.copy(deep=True) 

1450 else: 

1451 refCat = selected.sourceCat 

1452 

1453 if applyColorterms: 

1454 refCatName = refObjLoader.ref_dataset_name 

1455 self.log.info("Applying color terms for physical filter=%r reference catalog=%s", 

1456 filterLabel.physicalLabel, refCatName) 

1457 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel, 

1458 refCatName, 

1459 doRaise=True) 

1460 

1461 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

1462 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

1463 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

1464 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

1465 

1466 return refCat, skyCircle.fluxField 

1467 

1468 def _check_star_lists(self, associations, name): 

1469 # TODO: these should be len(blah), but we need this properly wrapped first. 

1470 if associations.nCcdImagesValidForFit() == 0: 

1471 raise RuntimeError('No images in the ccdImageList!') 

1472 if associations.fittedStarListSize() == 0: 

1473 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

1474 if associations.refStarListSize() == 0: 

1475 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

1476 

1477 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

1478 """Compute chi2, log it, validate the model, and return chi2. 

1479 

1480 Parameters 

1481 ---------- 

1482 associations : `lsst.jointcal.Associations` 

1483 The star/reference star associations to fit. 

1484 fit : `lsst.jointcal.FitterBase` 

1485 The fitter to use for minimization. 

1486 model : `lsst.jointcal.Model` 

1487 The model being fit. 

1488 chi2Label : `str` 

1489 Label to describe the chi2 (e.g. "Initialized", "Final"). 

1490 writeChi2Name : `str`, optional 

1491 Filename prefix to write the chi2 contributions to. 

1492 Do not supply an extension: an appropriate one will be added. 

1493 

1494 Returns 

1495 ------- 

1496 chi2: `lsst.jointcal.Chi2Accumulator` 

1497 The chi2 object for the current fitter and model. 

1498 

1499 Raises 

1500 ------ 

1501 FloatingPointError 

1502 Raised if chi2 is infinite or NaN. 

1503 ValueError 

1504 Raised if the model is not valid. 

1505 """ 

1506 if writeChi2Name is not None: 

1507 fullpath = self._getDebugPath(writeChi2Name) 

1508 fit.saveChi2Contributions(fullpath+"{type}") 

1509 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

1510 

1511 chi2 = fit.computeChi2() 

1512 self.log.info("%s %s", chi2Label, chi2) 

1513 self._check_stars(associations) 

1514 if not np.isfinite(chi2.chi2): 

1515 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

1516 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

1517 raise ValueError("Model is not valid: check log messages for warnings.") 

1518 return chi2 

1519 

1520 def _fit_photometry(self, associations, dataName=None): 

1521 """ 

1522 Fit the photometric data. 

1523 

1524 Parameters 

1525 ---------- 

1526 associations : `lsst.jointcal.Associations` 

1527 The star/reference star associations to fit. 

1528 dataName : `str` 

1529 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1530 identifying debugging files. 

1531 

1532 Returns 

1533 ------- 

1534 fit_result : `namedtuple` 

1535 fit : `lsst.jointcal.PhotometryFit` 

1536 The photometric fitter used to perform the fit. 

1537 model : `lsst.jointcal.PhotometryModel` 

1538 The photometric model that was fit. 

1539 """ 

1540 self.log.info("=== Starting photometric fitting...") 

1541 

1542 # TODO: should use pex.config.RegistryField here (see DM-9195) 

1543 if self.config.photometryModel == "constrainedFlux": 

1544 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

1545 self.focalPlaneBBox, 

1546 visitOrder=self.config.photometryVisitOrder, 

1547 errorPedestal=self.config.photometryErrorPedestal) 

1548 # potentially nonlinear problem, so we may need a line search to converge. 

1549 doLineSearch = self.config.allowLineSearch 

1550 elif self.config.photometryModel == "constrainedMagnitude": 

1551 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

1552 self.focalPlaneBBox, 

1553 visitOrder=self.config.photometryVisitOrder, 

1554 errorPedestal=self.config.photometryErrorPedestal) 

1555 # potentially nonlinear problem, so we may need a line search to converge. 

1556 doLineSearch = self.config.allowLineSearch 

1557 elif self.config.photometryModel == "simpleFlux": 

1558 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

1559 errorPedestal=self.config.photometryErrorPedestal) 

1560 doLineSearch = False # purely linear in model parameters, so no line search needed 

1561 elif self.config.photometryModel == "simpleMagnitude": 

1562 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

1563 errorPedestal=self.config.photometryErrorPedestal) 

1564 doLineSearch = False # purely linear in model parameters, so no line search needed 

1565 

1566 fit = lsst.jointcal.PhotometryFit(associations, model) 

1567 # TODO DM-12446: turn this into a "butler save" somehow. 

1568 # Save reference and measurement chi2 contributions for this data 

1569 if self.config.writeChi2FilesInitialFinal: 

1570 baseName = f"photometry_initial_chi2-{dataName}" 

1571 else: 

1572 baseName = None 

1573 if self.config.writeInitialModel: 

1574 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt") 

1575 writeModel(model, fullpath, self.log) 

1576 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

1577 

1578 def getChi2Name(whatToFit): 

1579 if self.config.writeChi2FilesOuterLoop: 

1580 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

1581 else: 

1582 return None 

1583 

1584 # The constrained model needs the visit transform fit first; the chip 

1585 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

1586 if self.config.writeInitMatrix: 

1587 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}") 

1588 else: 

1589 dumpMatrixFile = "" 

1590 if self.config.photometryModel.startswith("constrained"): 

1591 # no line search: should be purely (or nearly) linear, 

1592 # and we want a large step size to initialize with. 

1593 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

1594 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

1595 writeChi2Name=getChi2Name("ModelVisit")) 

1596 dumpMatrixFile = "" # so we don't redo the output on the next step 

1597 

1598 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

1599 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

1600 writeChi2Name=getChi2Name("Model")) 

1601 

1602 fit.minimize("Fluxes") # no line search: always purely linear. 

1603 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

1604 writeChi2Name=getChi2Name("Fluxes")) 

1605 

1606 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

1607 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

1608 writeChi2Name=getChi2Name("ModelFluxes")) 

1609 

1610 model.freezeErrorTransform() 

1611 self.log.debug("Photometry error scales are frozen.") 

1612 

1613 chi2 = self._iterate_fit(associations, 

1614 fit, 

1615 self.config.maxPhotometrySteps, 

1616 "photometry", 

1617 "Model Fluxes", 

1618 doRankUpdate=self.config.photometryDoRankUpdate, 

1619 doLineSearch=doLineSearch, 

1620 dataName=dataName) 

1621 

1622 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

1623 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

1624 return Photometry(fit, model) 

1625 

1626 def _fit_astrometry(self, associations, dataName=None): 

1627 """ 

1628 Fit the astrometric data. 

1629 

1630 Parameters 

1631 ---------- 

1632 associations : `lsst.jointcal.Associations` 

1633 The star/reference star associations to fit. 

1634 dataName : `str` 

1635 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1636 identifying debugging files. 

1637 

1638 Returns 

1639 ------- 

1640 fit_result : `namedtuple` 

1641 fit : `lsst.jointcal.AstrometryFit` 

1642 The astrometric fitter used to perform the fit. 

1643 model : `lsst.jointcal.AstrometryModel` 

1644 The astrometric model that was fit. 

1645 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

1646 The model for the sky to tangent plane projection that was used in the fit. 

1647 """ 

1648 

1649 self.log.info("=== Starting astrometric fitting...") 

1650 

1651 associations.deprojectFittedStars() 

1652 

1653 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1654 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1655 # them so carefully? 

1656 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1657 

1658 if self.config.astrometryModel == "constrained": 

1659 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1660 sky_to_tan_projection, 

1661 chipOrder=self.config.astrometryChipOrder, 

1662 visitOrder=self.config.astrometryVisitOrder) 

1663 elif self.config.astrometryModel == "simple": 

1664 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1665 sky_to_tan_projection, 

1666 self.config.useInputWcs, 

1667 nNotFit=0, 

1668 order=self.config.astrometrySimpleOrder) 

1669 

1670 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1671 # TODO DM-12446: turn this into a "butler save" somehow. 

1672 # Save reference and measurement chi2 contributions for this data 

1673 if self.config.writeChi2FilesInitialFinal: 

1674 baseName = f"astrometry_initial_chi2-{dataName}" 

1675 else: 

1676 baseName = None 

1677 if self.config.writeInitialModel: 

1678 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt") 

1679 writeModel(model, fullpath, self.log) 

1680 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1681 

1682 def getChi2Name(whatToFit): 

1683 if self.config.writeChi2FilesOuterLoop: 

1684 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1685 else: 

1686 return None 

1687 

1688 if self.config.writeInitMatrix: 

1689 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}") 

1690 else: 

1691 dumpMatrixFile = "" 

1692 # The constrained model needs the visit transform fit first; the chip 

1693 # transform is initialized from the detector's cameraGeom, so it's close. 

1694 if self.config.astrometryModel == "constrained": 

1695 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1696 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1697 writeChi2Name=getChi2Name("DistortionsVisit")) 

1698 dumpMatrixFile = "" # so we don't redo the output on the next step 

1699 

1700 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1701 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1702 writeChi2Name=getChi2Name("Distortions")) 

1703 

1704 fit.minimize("Positions") 

1705 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1706 writeChi2Name=getChi2Name("Positions")) 

1707 

1708 fit.minimize("Distortions Positions") 

1709 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1710 writeChi2Name=getChi2Name("DistortionsPositions")) 

1711 

1712 chi2 = self._iterate_fit(associations, 

1713 fit, 

1714 self.config.maxAstrometrySteps, 

1715 "astrometry", 

1716 "Distortions Positions", 

1717 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance, 

1718 doRankUpdate=self.config.astrometryDoRankUpdate, 

1719 dataName=dataName) 

1720 

1721 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1722 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1723 

1724 return Astrometry(fit, model, sky_to_tan_projection) 

1725 

1726 def _check_stars(self, associations): 

1727 """Count measured and reference stars per ccd and warn/log them.""" 

1728 for ccdImage in associations.getCcdImageList(): 

1729 nMeasuredStars, nRefStars = ccdImage.countStars() 

1730 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1731 ccdImage.getName(), nMeasuredStars, nRefStars) 

1732 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1733 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)", 

1734 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1735 if nRefStars < self.config.minRefStarsPerCcd: 

1736 self.log.warning("ccdImage %s has only %s RefStars (desired %s)", 

1737 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1738 

1739 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1740 dataName="", 

1741 sigmaRelativeTolerance=0, 

1742 doRankUpdate=True, 

1743 doLineSearch=False): 

1744 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1745 

1746 Parameters 

1747 ---------- 

1748 associations : `lsst.jointcal.Associations` 

1749 The star/reference star associations to fit. 

1750 fitter : `lsst.jointcal.FitterBase` 

1751 The fitter to use for minimization. 

1752 max_steps : `int` 

1753 Maximum number of steps to run outlier rejection before declaring 

1754 convergence failure. 

1755 name : {'photometry' or 'astrometry'} 

1756 What type of data are we fitting (for logs and debugging files). 

1757 whatToFit : `str` 

1758 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1759 dataName : `str`, optional 

1760 Descriptive name for this dataset (e.g. tract and filter), 

1761 for debugging. 

1762 sigmaRelativeTolerance : `float`, optional 

1763 Convergence tolerance for the fractional change in the chi2 cut 

1764 level for determining outliers. If set to zero, iterations will 

1765 continue until there are no outliers. 

1766 doRankUpdate : `bool`, optional 

1767 Do an Eigen rank update during minimization, or recompute the full 

1768 matrix and gradient? 

1769 doLineSearch : `bool`, optional 

1770 Do a line search for the optimum step during minimization? 

1771 

1772 Returns 

1773 ------- 

1774 chi2: `lsst.jointcal.Chi2Statistic` 

1775 The final chi2 after the fit converges, or is forced to end. 

1776 

1777 Raises 

1778 ------ 

1779 FloatingPointError 

1780 Raised if the fitter fails with a non-finite value. 

1781 RuntimeError 

1782 Raised if the fitter fails for some other reason; 

1783 log messages will provide further details. 

1784 """ 

1785 if self.config.writeInitMatrix: 

1786 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}") 

1787 else: 

1788 dumpMatrixFile = "" 

1789 oldChi2 = lsst.jointcal.Chi2Statistic() 

1790 oldChi2.chi2 = float("inf") 

1791 for i in range(max_steps): 

1792 if self.config.writeChi2FilesOuterLoop: 

1793 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1794 else: 

1795 writeChi2Name = None 

1796 result = fitter.minimize(whatToFit, 

1797 self.config.outlierRejectSigma, 

1798 sigmaRelativeTolerance=sigmaRelativeTolerance, 

1799 doRankUpdate=doRankUpdate, 

1800 doLineSearch=doLineSearch, 

1801 dumpMatrixFile=dumpMatrixFile) 

1802 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1803 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1804 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1805 

1806 if result == MinimizeResult.Converged: 

1807 if doRankUpdate: 

1808 self.log.debug("fit has converged - no more outliers - redo minimization " 

1809 "one more time in case we have lost accuracy in rank update.") 

1810 # Redo minimization one more time in case we have lost accuracy in rank update 

1811 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma, 

1812 sigmaRelativeTolerance=sigmaRelativeTolerance) 

1813 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1814 

1815 # log a message for a large final chi2, TODO: DM-15247 for something better 

1816 if chi2.chi2/chi2.ndof >= 4.0: 

1817 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1818 

1819 break 

1820 elif result == MinimizeResult.Chi2Increased: 

1821 self.log.warning("Still some outliers remaining but chi2 increased - retry") 

1822 # Check whether the increase was large enough to cause trouble. 

1823 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1824 if chi2Ratio > 1.5: 

1825 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1826 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1827 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1828 # appears that chi2 increases more than a factor of ~2 always 

1829 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1830 # Using 10 as the "failure" threshold gives some room between 

1831 # leaving a warning and bailing early. 

1832 if chi2Ratio > 10: 

1833 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1834 " Try setting one or more of the `writeChi2*` config fields and looking" 

1835 " at how individual star chi2-values evolve during the fit.") 

1836 raise RuntimeError(msg) 

1837 oldChi2 = chi2 

1838 elif result == MinimizeResult.NonFinite: 

1839 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1840 # TODO DM-12446: turn this into a "butler save" somehow. 

1841 fitter.saveChi2Contributions(filename+"{type}") 

1842 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1843 raise FloatingPointError(msg.format(filename)) 

1844 elif result == MinimizeResult.Failed: 

1845 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1846 else: 

1847 raise RuntimeError("Unxepected return code from minimize().") 

1848 else: 

1849 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1850 

1851 return chi2 

1852 

1853 def _make_output(self, ccdImageList, model, func): 

1854 """Return the internal jointcal models converted to the afw 

1855 structures that will be saved to disk. 

1856 

1857 Parameters 

1858 ---------- 

1859 ccdImageList : `lsst.jointcal.CcdImageList` 

1860 The list of CcdImages to get the output for. 

1861 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel` 

1862 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`. 

1863 func : `str` 

1864 The name of the function to call on ``model`` to get the converted 

1865 structure. Must accept an `lsst.jointcal.CcdImage`. 

1866 

1867 Returns 

1868 ------- 

1869 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or 

1870 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`] 

1871 The data to be saved, keyed on (visit, detector). 

1872 """ 

1873 output = {} 

1874 for ccdImage in ccdImageList: 

1875 ccd = ccdImage.ccdId 

1876 visit = ccdImage.visit 

1877 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd) 

1878 output[(visit, ccd)] = getattr(model, func)(ccdImage) 

1879 return output 

1880 

1881 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef): 

1882 """ 

1883 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 

1884 

1885 Parameters 

1886 ---------- 

1887 associations : `lsst.jointcal.Associations` 

1888 The star/reference star associations to fit. 

1889 model : `lsst.jointcal.AstrometryModel` 

1890 The astrometric model that was fit. 

1891 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1892 Dict of ccdImage identifiers to dataRefs that were fit. 

1893 """ 

1894 ccdImageList = associations.getCcdImageList() 

1895 output = self._make_output(ccdImageList, model, "makeSkyWcs") 

1896 for key, skyWcs in output.items(): 

1897 dataRef = visit_ccd_to_dataRef[key] 

1898 try: 

1899 dataRef.put(skyWcs, 'jointcal_wcs') 

1900 except pexExceptions.Exception as e: 

1901 self.log.fatal('Failed to write updated Wcs: %s', str(e)) 

1902 raise e 

1903 

1904 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef): 

1905 """ 

1906 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 

1907 

1908 Parameters 

1909 ---------- 

1910 associations : `lsst.jointcal.Associations` 

1911 The star/reference star associations to fit. 

1912 model : `lsst.jointcal.PhotometryModel` 

1913 The photoometric model that was fit. 

1914 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1915 Dict of ccdImage identifiers to dataRefs that were fit. 

1916 """ 

1917 

1918 ccdImageList = associations.getCcdImageList() 

1919 output = self._make_output(ccdImageList, model, "toPhotoCalib") 

1920 for key, photoCalib in output.items(): 

1921 dataRef = visit_ccd_to_dataRef[key] 

1922 try: 

1923 dataRef.put(photoCalib, 'jointcal_photoCalib') 

1924 except pexExceptions.Exception as e: 

1925 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e)) 

1926 raise e