Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import dataclasses 

23import collections 

24import os 

25 

26import astropy.time 

27import numpy as np 

28import astropy.units as u 

29 

30import lsst.geom 

31import lsst.utils 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34from lsst.afw.image import fluxErrFromABMagErr 

35import lsst.pex.exceptions as pexExceptions 

36import lsst.afw.cameraGeom 

37import lsst.afw.table 

38import lsst.log 

39from lsst.obs.base import Instrument 

40from lsst.pipe.tasks.colorterms import ColortermLibrary 

41from lsst.verify import Job, Measurement 

42 

43from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask, 

44 ReferenceObjectLoader) 

45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry 

46 

47from .dataIds import PerTractCcdDataIdContainer 

48 

49import lsst.jointcal 

50from lsst.jointcal import MinimizeResult 

51 

52__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"] 

53 

54Photometry = collections.namedtuple('Photometry', ('fit', 'model')) 

55Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection')) 

56 

57 

58# TODO: move this to MeasurementSet in lsst.verify per DM-12655. 

59def add_measurement(job, name, value): 

60 meas = Measurement(job.metrics[name], value) 

61 job.measurements.insert(meas) 

62 

63 

64class JointcalRunner(pipeBase.ButlerInitializedTaskRunner): 

65 """Subclass of TaskRunner for jointcalTask (gen2) 

66 

67 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 

68 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 

69 single dataRef, are are called repeatedly). This class transforms the processed 

70 arguments generated by the ArgumentParser into the arguments expected by 

71 Jointcal.runDataRef(). 

72 

73 See pipeBase.TaskRunner for more information. 

74 """ 

75 

76 @staticmethod 

77 def getTargetList(parsedCmd, **kwargs): 

78 """ 

79 Return a list of tuples per tract, each containing (dataRefs, kwargs). 

80 

81 Jointcal operates on lists of dataRefs simultaneously. 

82 """ 

83 kwargs['butler'] = parsedCmd.butler 

84 

85 # organize data IDs by tract 

86 refListDict = {} 

87 for ref in parsedCmd.id.refList: 

88 refListDict.setdefault(ref.dataId["tract"], []).append(ref) 

89 # we call runDataRef() once with each tract 

90 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())] 

91 return result 

92 

93 def __call__(self, args): 

94 """ 

95 Parameters 

96 ---------- 

97 args 

98 Arguments for Task.runDataRef() 

99 

100 Returns 

101 ------- 

102 pipe.base.Struct 

103 if self.doReturnResults is False: 

104 

105 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

106 

107 if self.doReturnResults is True: 

108 

109 - ``result``: the result of calling jointcal.runDataRef() 

110 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 

111 """ 

112 exitStatus = 0 # exit status for shell 

113 

114 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef. 

115 dataRefList, kwargs = args 

116 butler = kwargs.pop('butler') 

117 task = self.TaskClass(config=self.config, log=self.log, butler=butler) 

118 result = None 

119 try: 

120 result = task.runDataRef(dataRefList, **kwargs) 

121 exitStatus = result.exitStatus 

122 job_path = butler.get('verify_job_filename') 

123 result.job.write(job_path[0]) 

124 except Exception as e: # catch everything, sort it out later. 

125 if self.doRaise: 

126 raise e 

127 else: 

128 exitStatus = 1 

129 eName = type(e).__name__ 

130 tract = dataRefList[0].dataId['tract'] 

131 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e) 

132 

133 # Put the butler back into kwargs for the other Tasks. 

134 kwargs['butler'] = butler 

135 if self.doReturnResults: 

136 return pipeBase.Struct(result=result, exitStatus=exitStatus) 

137 else: 

138 return pipeBase.Struct(exitStatus=exitStatus) 

139 

140 

141def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections): 

142 """Lookup function that asserts/hopes that a static calibration dataset 

143 exists in a particular collection, since this task can't provide a single 

144 date/time to use to search for one properly. 

145 

146 This is mostly useful for the ``camera`` dataset, in cases where the task's 

147 quantum dimensions do *not* include something temporal, like ``exposure`` 

148 or ``visit``. 

149 

150 Parameters 

151 ---------- 

152 datasetType : `lsst.daf.butler.DatasetType` 

153 Type of dataset being searched for. 

154 registry : `lsst.daf.butler.Registry` 

155 Data repository registry to search. 

156 quantumDataId : `lsst.daf.butler.DataCoordinate` 

157 Data ID of the quantum this camera should match. 

158 collections : `Iterable` [ `str` ] 

159 Collections that should be searched - but this lookup function works 

160 by ignoring this in favor of a more-or-less hard-coded value. 

161 

162 Returns 

163 ------- 

164 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

165 Iterator over dataset references; should have only one element. 

166 

167 Notes 

168 ----- 

169 This implementation duplicates one in fgcmcal, and is at least quite 

170 similar to another in cp_pipe. This duplicate has the most documentation. 

171 Fixing this is DM-29661. 

172 """ 

173 instrument = Instrument.fromName(quantumDataId["instrument"], registry) 

174 unboundedCollection = instrument.makeUnboundedCalibrationRunName() 

175 return registry.queryDatasets(datasetType, 

176 dataId=quantumDataId, 

177 collections=[unboundedCollection], 

178 findFirst=True) 

179 

180 

181def lookupVisitRefCats(datasetType, registry, quantumDataId, collections): 

182 """Lookup function that finds all refcats for all visits that overlap a 

183 tract, rather than just the refcats that directly overlap the tract. 

184 

185 Parameters 

186 ---------- 

187 datasetType : `lsst.daf.butler.DatasetType` 

188 Type of dataset being searched for. 

189 registry : `lsst.daf.butler.Registry` 

190 Data repository registry to search. 

191 quantumDataId : `lsst.daf.butler.DataCoordinate` 

192 Data ID of the quantum; expected to be something we can use as a 

193 constraint to query for overlapping visits. 

194 collections : `Iterable` [ `str` ] 

195 Collections to search. 

196 

197 Returns 

198 ------- 

199 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ] 

200 Iterator over refcat references. 

201 """ 

202 refs = set() 

203 # Use .expanded() on the query methods below because we need data IDs with 

204 # regions, both in the outer loop over visits (queryDatasets will expand 

205 # any data ID we give it, but doing it up-front in bulk is much more 

206 # efficient) and in the data IDs of the DatasetRefs this function yields 

207 # (because the RefCatLoader relies on them to do some of its own 

208 # filtering). 

209 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()): 

210 refs.update( 

211 registry.queryDatasets( 

212 datasetType, 

213 collections=collections, 

214 dataId=visit_data_id, 

215 findFirst=True, 

216 ).expanded() 

217 ) 

218 yield from refs 

219 

220 

221class JointcalTaskConnections(pipeBase.PipelineTaskConnections, 

222 dimensions=("skymap", "tract", "instrument", "physical_filter")): 

223 """Middleware input/output connections for jointcal data.""" 

224 inputCamera = pipeBase.connectionTypes.PrerequisiteInput( 

225 doc="The camera instrument that took these observations.", 

226 name="camera", 

227 storageClass="Camera", 

228 dimensions=("instrument",), 

229 isCalibration=True, 

230 lookupFunction=lookupStaticCalibrations, 

231 ) 

232 inputSourceTableVisit = pipeBase.connectionTypes.Input( 

233 doc="Source table in parquet format, per visit", 

234 name="sourceTable_visit", 

235 storageClass="DataFrame", 

236 dimensions=("instrument", "visit"), 

237 deferLoad=True, 

238 multiple=True, 

239 ) 

240 inputVisitSummary = pipeBase.connectionTypes.Input( 

241 doc=("Per-visit consolidated exposure metadata built from calexps. " 

242 "These catalogs use detector id for the id and must be sorted for " 

243 "fast lookups of a detector."), 

244 name="visitSummary", 

245 storageClass="ExposureCatalog", 

246 dimensions=("instrument", "visit"), 

247 deferLoad=True, 

248 multiple=True, 

249 ) 

250 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

251 doc="The astrometry reference catalog to match to loaded input catalog sources.", 

252 name="gaia_dr2_20200414", 

253 storageClass="SimpleCatalog", 

254 dimensions=("skypix",), 

255 deferLoad=True, 

256 multiple=True, 

257 lookupFunction=lookupVisitRefCats, 

258 ) 

259 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput( 

260 doc="The photometry reference catalog to match to loaded input catalog sources.", 

261 name="ps1_pv3_3pi_20170110", 

262 storageClass="SimpleCatalog", 

263 dimensions=("skypix",), 

264 deferLoad=True, 

265 multiple=True, 

266 lookupFunction=lookupVisitRefCats, 

267 ) 

268 

269 outputWcs = pipeBase.connectionTypes.Output( 

270 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model." 

271 " These catalogs only contain entries for detectors with an output, and use" 

272 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

273 name="jointcalSkyWcsCatalog", 

274 storageClass="ExposureCatalog", 

275 dimensions=("instrument", "visit", "skymap", "tract"), 

276 multiple=True 

277 ) 

278 outputPhotoCalib = pipeBase.connectionTypes.Output( 

279 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model." 

280 " These catalogs only contain entries for detectors with an output, and use" 

281 " the detector id for the catalog id, sorted on id for fast lookups of a detector."), 

282 name="jointcalPhotoCalibCatalog", 

283 storageClass="ExposureCatalog", 

284 dimensions=("instrument", "visit", "skymap", "tract"), 

285 multiple=True 

286 ) 

287 

288 # measurements of metrics 

289 # The vars() trick used here allows us to set class attributes 

290 # programatically. Taken from: 

291 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class 

292 for name in ("astrometry", "photometry"): 

293 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output( 

294 doc=f"The number of cross-matched fittedStars for {name}", 

295 name=f"metricvalue_jointcal_{name}_matched_fittedStars", 

296 storageClass="MetricValue", 

297 dimensions=("instrument", "skymap", "tract"), 

298 ) 

299 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output( 

300 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.", 

301 name=f"metricvalue_jointcal_{name}_collected_refStars", 

302 storageClass="MetricValue", 

303 dimensions=("instrument", "skymap", "tract"), 

304 ) 

305 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output( 

306 doc=f"The number of {name} reference stars matched to fittedStars.", 

307 name=f"metricvalue_jointcal_{name}_prepared_refStars", 

308 storageClass="MetricValue", 

309 dimensions=("instrument", "skymap", "tract"), 

310 ) 

311 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output( 

312 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.", 

313 name=f"metricvalue_jointcal_{name}_prepared_fittedStars", 

314 storageClass="MetricValue", 

315 dimensions=("instrument", "skymap", "tract"), 

316 ) 

317 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output( 

318 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.", 

319 name=f"metricvalue_jointcal_{name}_prepared_ccdImages", 

320 storageClass="MetricValue", 

321 dimensions=("instrument", "skymap", "tract"), 

322 ) 

323 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output( 

324 doc=f"The final chi2 of the {name} fit.", 

325 name=f"metricvalue_jointcal_{name}_final_chi2", 

326 storageClass="MetricValue", 

327 dimensions=("instrument", "skymap", "tract"), 

328 ) 

329 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output( 

330 doc=f"The number of degrees of freedom of the fitted {name}.", 

331 name=f"metricvalue_jointcal_{name}_final_ndof", 

332 storageClass="MetricValue", 

333 dimensions=("instrument", "skymap", "tract"), 

334 ) 

335 

336 def __init__(self, *, config=None): 

337 super().__init__(config=config) 

338 # When we are only doing one of astrometry or photometry, we don't 

339 # need the reference catalog or produce the outputs for the other. 

340 # This informs the middleware of that when the QuantumGraph is 

341 # generated, so we don't block on getting something we won't need or 

342 # create an expectation that downstream tasks will be able to consume 

343 # something we won't produce. 

344 if not config.doAstrometry: 

345 self.prerequisiteInputs.remove("astrometryRefCat") 

346 self.outputs.remove("outputWcs") 

347 for key in list(self.outputs.keys()): 

348 if "metricvalue_jointcal_astrometry" in key: 

349 self.outputs.remove(key) 

350 if not config.doPhotometry: 

351 self.prerequisiteInputs.remove("photometryRefCat") 

352 self.outputs.remove("outputPhotoCalib") 

353 for key in list(self.outputs.keys()): 

354 if "metricvalue_jointcal_photometry" in key: 

355 self.outputs.remove(key) 

356 

357 

358class JointcalConfig(pipeBase.PipelineTaskConfig, 

359 pipelineConnections=JointcalTaskConnections): 

360 """Configuration for JointcalTask""" 

361 

362 doAstrometry = pexConfig.Field( 

363 doc="Fit astrometry and write the fitted result.", 

364 dtype=bool, 

365 default=True 

366 ) 

367 doPhotometry = pexConfig.Field( 

368 doc="Fit photometry and write the fitted result.", 

369 dtype=bool, 

370 default=True 

371 ) 

372 coaddName = pexConfig.Field( 

373 doc="Type of coadd, typically deep or goodSeeing", 

374 dtype=str, 

375 default="deep" 

376 ) 

377 # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal. 

378 sourceFluxType = pexConfig.Field( 

379 dtype=str, 

380 doc="Source flux field to use in source selection and to get fluxes from the catalog.", 

381 default='Calib' 

382 ) 

383 positionErrorPedestal = pexConfig.Field( 

384 doc="Systematic term to apply to the measured position error (pixels)", 

385 dtype=float, 

386 default=0.02, 

387 ) 

388 photometryErrorPedestal = pexConfig.Field( 

389 doc="Systematic term to apply to the measured error on flux or magnitude as a " 

390 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).", 

391 dtype=float, 

392 default=0.0, 

393 ) 

394 # TODO: DM-6885 matchCut should be an geom.Angle 

395 matchCut = pexConfig.Field( 

396 doc="Matching radius between fitted and reference stars (arcseconds)", 

397 dtype=float, 

398 default=3.0, 

399 ) 

400 minMeasurements = pexConfig.Field( 

401 doc="Minimum number of associated measured stars for a fitted star to be included in the fit", 

402 dtype=int, 

403 default=2, 

404 ) 

405 minMeasuredStarsPerCcd = pexConfig.Field( 

406 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

407 dtype=int, 

408 default=100, 

409 ) 

410 minRefStarsPerCcd = pexConfig.Field( 

411 doc="Minimum number of measuredStars per ccdImage before printing warnings", 

412 dtype=int, 

413 default=30, 

414 ) 

415 allowLineSearch = pexConfig.Field( 

416 doc="Allow a line search during minimization, if it is reasonable for the model" 

417 " (models with a significant non-linear component, e.g. constrainedPhotometry).", 

418 dtype=bool, 

419 default=False 

420 ) 

421 astrometrySimpleOrder = pexConfig.Field( 

422 doc="Polynomial order for fitting the simple astrometry model.", 

423 dtype=int, 

424 default=3, 

425 ) 

426 astrometryChipOrder = pexConfig.Field( 

427 doc="Order of the per-chip transform for the constrained astrometry model.", 

428 dtype=int, 

429 default=1, 

430 ) 

431 astrometryVisitOrder = pexConfig.Field( 

432 doc="Order of the per-visit transform for the constrained astrometry model.", 

433 dtype=int, 

434 default=5, 

435 ) 

436 useInputWcs = pexConfig.Field( 

437 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.", 

438 dtype=bool, 

439 default=True, 

440 ) 

441 astrometryModel = pexConfig.ChoiceField( 

442 doc="Type of model to fit to astrometry", 

443 dtype=str, 

444 default="constrained", 

445 allowed={"simple": "One polynomial per ccd", 

446 "constrained": "One polynomial per ccd, and one polynomial per visit"} 

447 ) 

448 photometryModel = pexConfig.ChoiceField( 

449 doc="Type of model to fit to photometry", 

450 dtype=str, 

451 default="constrainedMagnitude", 

452 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.", 

453 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit," 

454 " fitting in flux space.", 

455 "simpleMagnitude": "One constant zeropoint per ccd and visit," 

456 " fitting in magnitude space.", 

457 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit," 

458 " fitting in magnitude space.", 

459 } 

460 ) 

461 applyColorTerms = pexConfig.Field( 

462 doc="Apply photometric color terms to reference stars?" 

463 "Requires that colorterms be set to a ColortermLibrary", 

464 dtype=bool, 

465 default=False 

466 ) 

467 colorterms = pexConfig.ConfigField( 

468 doc="Library of photometric reference catalog name to color term dict.", 

469 dtype=ColortermLibrary, 

470 ) 

471 photometryVisitOrder = pexConfig.Field( 

472 doc="Order of the per-visit polynomial transform for the constrained photometry model.", 

473 dtype=int, 

474 default=7, 

475 ) 

476 photometryDoRankUpdate = pexConfig.Field( 

477 doc=("Do the rank update step during minimization. " 

478 "Skipping this can help deal with models that are too non-linear."), 

479 dtype=bool, 

480 default=True, 

481 ) 

482 astrometryDoRankUpdate = pexConfig.Field( 

483 doc=("Do the rank update step during minimization (should not change the astrometry fit). " 

484 "Skipping this can help deal with models that are too non-linear."), 

485 dtype=bool, 

486 default=True, 

487 ) 

488 outlierRejectSigma = pexConfig.Field( 

489 doc="How many sigma to reject outliers at during minimization.", 

490 dtype=float, 

491 default=5.0, 

492 ) 

493 astrometryOutlierRelativeTolerance = pexConfig.Field( 

494 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will " 

495 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set " 

496 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002" 

497 "as a balance between a shorter minimization runtime and achieving a final fitted model that is" 

498 "close to the solution found when removing all outliers."), 

499 dtype=float, 

500 default=0, 

501 ) 

502 maxPhotometrySteps = pexConfig.Field( 

503 doc="Maximum number of minimize iterations to take when fitting photometry.", 

504 dtype=int, 

505 default=20, 

506 ) 

507 maxAstrometrySteps = pexConfig.Field( 

508 doc="Maximum number of minimize iterations to take when fitting astrometry.", 

509 dtype=int, 

510 default=20, 

511 ) 

512 astrometryRefObjLoader = pexConfig.ConfigurableField( 

513 target=LoadIndexedReferenceObjectsTask, 

514 doc="Reference object loader for astrometric fit", 

515 ) 

516 photometryRefObjLoader = pexConfig.ConfigurableField( 

517 target=LoadIndexedReferenceObjectsTask, 

518 doc="Reference object loader for photometric fit", 

519 ) 

520 sourceSelector = sourceSelectorRegistry.makeField( 

521 doc="How to select sources for cross-matching", 

522 default="astrometry" 

523 ) 

524 astrometryReferenceSelector = pexConfig.ConfigurableField( 

525 target=ReferenceSourceSelectorTask, 

526 doc="How to down-select the loaded astrometry reference catalog.", 

527 ) 

528 photometryReferenceSelector = pexConfig.ConfigurableField( 

529 target=ReferenceSourceSelectorTask, 

530 doc="How to down-select the loaded photometry reference catalog.", 

531 ) 

532 astrometryReferenceErr = pexConfig.Field( 

533 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 

534 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 

535 "If specified, overrides any existing `coord_*Err` values."), 

536 dtype=float, 

537 default=None, 

538 optional=True 

539 ) 

540 

541 # configs for outputting debug information 

542 writeInitMatrix = pexConfig.Field( 

543 dtype=bool, 

544 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 

545 "Output files will be written to `config.debugOutputPath` and will " 

546 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. " 

547 "Note that these files are the dense versions of the matrix, and so may be very large."), 

548 default=False 

549 ) 

550 writeChi2FilesInitialFinal = pexConfig.Field( 

551 dtype=bool, 

552 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. " 

553 "Output files will be written to `config.debugOutputPath` and will " 

554 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."), 

555 default=False 

556 ) 

557 writeChi2FilesOuterLoop = pexConfig.Field( 

558 dtype=bool, 

559 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. " 

560 "Output files will be written to `config.debugOutputPath` and will " 

561 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."), 

562 default=False 

563 ) 

564 writeInitialModel = pexConfig.Field( 

565 dtype=bool, 

566 doc=("Write the pre-initialization model to text files, for debugging. " 

567 "Output files will be written to `config.debugOutputPath` and will be " 

568 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."), 

569 default=False 

570 ) 

571 debugOutputPath = pexConfig.Field( 

572 dtype=str, 

573 default=".", 

574 doc=("Path to write debug output files to. Used by " 

575 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.") 

576 ) 

577 detailedProfile = pexConfig.Field( 

578 dtype=bool, 

579 default=False, 

580 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting" 

581 ) 

582 

583 def validate(self): 

584 super().validate() 

585 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0: 

586 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

587 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg) 

588 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms: 

589 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;" 

590 "applyColorTerms=True will be ignored.") 

591 lsst.log.warning(msg) 

592 

593 def setDefaults(self): 

594 # Use science source selector which can filter on extendedness, SNR, and whether blended 

595 self.sourceSelector.name = 'science' 

596 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing 

597 self.sourceSelector['science'].doUnresolved = True 

598 # with dependable signal to noise ratio. 

599 self.sourceSelector['science'].doSignalToNoise = True 

600 # Min SNR must be > 0 because jointcal cannot handle negative fluxes, 

601 # and S/N > 10 to use sources that are not too faint, and thus better measured. 

602 self.sourceSelector['science'].signalToNoise.minimum = 10. 

603 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive 

604 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux" 

605 self.sourceSelector['science'].signalToNoise.fluxField = fluxField 

606 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err" 

607 # Do not trust blended sources' aperture fluxes which also depend on seeing 

608 self.sourceSelector['science'].doIsolated = True 

609 # Do not trust either flux or centroid measurements with flags, 

610 # chosen from the usual QA flags for stars) 

611 self.sourceSelector['science'].doFlags = True 

612 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 

613 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag', 

614 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter'] 

615 self.sourceSelector['science'].flags.bad = badFlags 

616 

617 # Default to Gaia-DR2 (with proper motions) for astrometry and 

618 # PS1-DR1 for photometry, with a reasonable initial filterMap. 

619 self.astrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414" 

620 self.astrometryRefObjLoader.requireProperMotion = True 

621 self.astrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean' 

622 self.photometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110" 

623 

624 

625def writeModel(model, filename, log): 

626 """Write model to outfile.""" 

627 with open(filename, "w") as file: 

628 file.write(repr(model)) 

629 log.info("Wrote %s to file: %s", model, filename) 

630 

631 

632@dataclasses.dataclass 

633class JointcalInputData: 

634 """The input data jointcal needs for each detector/visit.""" 

635 visit: int 

636 """The visit identifier of this exposure.""" 

637 catalog: lsst.afw.table.SourceCatalog 

638 """The catalog derived from this exposure.""" 

639 visitInfo: lsst.afw.image.VisitInfo 

640 """The VisitInfo of this exposure.""" 

641 detector: lsst.afw.cameraGeom.Detector 

642 """The detector of this exposure.""" 

643 photoCalib: lsst.afw.image.PhotoCalib 

644 """The photometric calibration of this exposure.""" 

645 wcs: lsst.afw.geom.skyWcs 

646 """The WCS of this exposure.""" 

647 bbox: lsst.geom.Box2I 

648 """The bounding box of this exposure.""" 

649 filter: lsst.afw.image.FilterLabel 

650 """The filter of this exposure.""" 

651 

652 

653class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

654 """Astrometricly and photometricly calibrate across multiple visits of the 

655 same field. 

656 

657 Parameters 

658 ---------- 

659 butler : `lsst.daf.persistence.Butler` 

660 The butler is passed to the refObjLoader constructor in case it is 

661 needed. Ignored if the refObjLoader argument provides a loader directly. 

662 Used to initialize the astrometry and photometry refObjLoaders. 

663 initInputs : `dict`, optional 

664 Dictionary used to initialize PipelineTasks (empty for jointcal). 

665 """ 

666 

667 ConfigClass = JointcalConfig 

668 RunnerClass = JointcalRunner 

669 _DefaultName = "jointcal" 

670 

671 def __init__(self, butler=None, initInputs=None, **kwargs): 

672 super().__init__(**kwargs) 

673 self.makeSubtask("sourceSelector") 

674 if self.config.doAstrometry: 

675 if initInputs is None: 

676 # gen3 middleware does refcat things internally (and will not have a butler here) 

677 self.makeSubtask('astrometryRefObjLoader', butler=butler) 

678 self.makeSubtask("astrometryReferenceSelector") 

679 else: 

680 self.astrometryRefObjLoader = None 

681 if self.config.doPhotometry: 

682 if initInputs is None: 

683 # gen3 middleware does refcat things internally (and will not have a butler here) 

684 self.makeSubtask('photometryRefObjLoader', butler=butler) 

685 self.makeSubtask("photometryReferenceSelector") 

686 else: 

687 self.photometryRefObjLoader = None 

688 

689 # To hold various computed metrics for use by tests 

690 self.job = Job.load_metrics_package(subset='jointcal') 

691 

692 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

693 # We override runQuantum to set up the refObjLoaders and write the 

694 # outputs to the correct refs. 

695 inputs = butlerQC.get(inputRefs) 

696 # We want the tract number for writing debug files 

697 tract = butlerQC.quantum.dataId['tract'] 

698 if self.config.doAstrometry: 

699 self.astrometryRefObjLoader = ReferenceObjectLoader( 

700 dataIds=[ref.datasetRef.dataId 

701 for ref in inputRefs.astrometryRefCat], 

702 refCats=inputs.pop('astrometryRefCat'), 

703 config=self.config.astrometryRefObjLoader, 

704 log=self.log) 

705 if self.config.doPhotometry: 

706 self.photometryRefObjLoader = ReferenceObjectLoader( 

707 dataIds=[ref.datasetRef.dataId 

708 for ref in inputRefs.photometryRefCat], 

709 refCats=inputs.pop('photometryRefCat'), 

710 config=self.config.photometryRefObjLoader, 

711 log=self.log) 

712 outputs = self.run(**inputs, tract=tract) 

713 self._put_metrics(butlerQC, outputs.job, outputRefs) 

714 if self.config.doAstrometry: 

715 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs, 

716 inputs['inputCamera'], "setWcs") 

717 if self.config.doPhotometry: 

718 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib, 

719 inputs['inputCamera'], "setPhotoCalib") 

720 

721 def _put_metrics(self, butlerQC, job, outputRefs): 

722 """Persist all measured metrics stored in a job. 

723 

724 Parameters 

725 ---------- 

726 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

727 A butler which is specialized to operate in the context of a 

728 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

729 job : `lsst.verify.job.Job` 

730 Measurements of metrics to persist. 

731 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

732 The DatasetRefs to persist the data to. 

733 """ 

734 for key in job.measurements.keys(): 

735 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', ''))) 

736 

737 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter): 

738 """Persist the output datasets to their appropriate datarefs. 

739 

740 Parameters 

741 ---------- 

742 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

743 A butler which is specialized to operate in the context of a 

744 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`. 

745 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or 

746 `dict` [`tuple, `lsst.afw.image.PhotoCalib`] 

747 The fitted objects to persist. 

748 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`] 

749 The DatasetRefs to persist the data to. 

750 camera : `lsst.afw.cameraGeom.Camera` 

751 The camera for this instrument, to get detector ids from. 

752 setter : `str` 

753 The method to call on the ExposureCatalog to set each output. 

754 """ 

755 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

756 schema.addField('visit', type='I', doc='Visit number') 

757 

758 def new_catalog(visit, size): 

759 """Return an catalog ready to be filled with appropriate output.""" 

760 catalog = lsst.afw.table.ExposureCatalog(schema) 

761 catalog.resize(size) 

762 catalog['visit'] = visit 

763 metadata = lsst.daf.base.PropertyList() 

764 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

765 metadata.add("COMMENT", "Only detectors with data have entries.") 

766 return catalog 

767 

768 # count how many detectors have output for each visit 

769 detectors_per_visit = collections.defaultdict(int) 

770 for key in outputs: 

771 # key is (visit, detector_id), and we only need visit here 

772 detectors_per_visit[key[0]] += 1 

773 

774 for ref in outputRefs: 

775 visit = ref.dataId['visit'] 

776 catalog = new_catalog(visit, detectors_per_visit[visit]) 

777 # Iterate over every detector and skip the ones we don't have output for. 

778 i = 0 

779 for detector in camera: 

780 detectorId = detector.getId() 

781 key = (ref.dataId['visit'], detectorId) 

782 if key not in outputs: 

783 # skip detectors we don't have output for 

784 self.log.debug("No %s output for detector %s in visit %s", 

785 setter[3:], detectorId, visit) 

786 continue 

787 

788 catalog[i].setId(detectorId) 

789 getattr(catalog[i], setter)(outputs[key]) 

790 i += 1 

791 

792 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups 

793 butlerQC.put(catalog, ref) 

794 self.log.info("Wrote %s detectors to %s", i, ref) 

795 

796 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None): 

797 # Docstring inherited. 

798 

799 # We take values out of the Parquet table, and put them in "flux_", 

800 # and the config.sourceFluxType field is used during that extraction, 

801 # so just use "flux" here. 

802 sourceFluxField = "flux" 

803 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

804 associations = lsst.jointcal.Associations() 

805 self.focalPlaneBBox = inputCamera.getFpBBox() 

806 oldWcsList, bands = self._load_data(inputSourceTableVisit, 

807 inputVisitSummary, 

808 associations, 

809 jointcalControl, 

810 inputCamera) 

811 

812 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands) 

813 

814 if self.config.doAstrometry: 

815 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

816 name="astrometry", 

817 refObjLoader=self.astrometryRefObjLoader, 

818 referenceSelector=self.astrometryReferenceSelector, 

819 fit_function=self._fit_astrometry, 

820 tract=tract, 

821 epoch=epoch) 

822 astrometry_output = self._make_output(associations.getCcdImageList(), 

823 astrometry.model, 

824 "makeSkyWcs") 

825 else: 

826 astrometry_output = None 

827 

828 if self.config.doPhotometry: 

829 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

830 name="photometry", 

831 refObjLoader=self.photometryRefObjLoader, 

832 referenceSelector=self.photometryReferenceSelector, 

833 fit_function=self._fit_photometry, 

834 tract=tract, 

835 epoch=epoch, 

836 reject_bad_fluxes=True) 

837 photometry_output = self._make_output(associations.getCcdImageList(), 

838 photometry.model, 

839 "toPhotoCalib") 

840 else: 

841 photometry_output = None 

842 

843 return pipeBase.Struct(outputWcs=astrometry_output, 

844 outputPhotoCalib=photometry_output, 

845 job=self.job, 

846 astrometryRefObjLoader=self.astrometryRefObjLoader, 

847 photometryRefObjLoader=self.photometryRefObjLoader) 

848 

849 def _make_schema_table(self): 

850 """Return an afw SourceTable to use as a base for creating the 

851 SourceCatalog to insert values from the dataFrame into. 

852 

853 Returns 

854 ------- 

855 table : `lsst.afw.table.SourceTable` 

856 Table with schema and slots to use to make SourceCatalogs. 

857 """ 

858 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

859 schema.addField("centroid_x", "D") 

860 schema.addField("centroid_y", "D") 

861 schema.addField("centroid_xErr", "F") 

862 schema.addField("centroid_yErr", "F") 

863 schema.addField("shape_xx", "D") 

864 schema.addField("shape_yy", "D") 

865 schema.addField("shape_xy", "D") 

866 schema.addField("flux_instFlux", "D") 

867 schema.addField("flux_instFluxErr", "D") 

868 table = lsst.afw.table.SourceTable.make(schema) 

869 table.defineCentroid("centroid") 

870 table.defineShape("shape") 

871 return table 

872 

873 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId, 

874 detectorColumn, ixxColumns): 

875 """Return an afw SourceCatalog extracted from a visit-level dataframe, 

876 limited to just one detector. 

877 

878 Parameters 

879 ---------- 

880 table : `lsst.afw.table.SourceTable` 

881 Table factory to use to make the SourceCatalog that will be 

882 populated with data from ``visitCatalog``. 

883 visitCatalog : `pandas.DataFrame` 

884 DataFrame to extract a detector catalog from. 

885 detectorId : `int` 

886 Numeric id of the detector to extract from ``visitCatalog``. 

887 detectorColumn : `str` 

888 Name of the detector column in the catalog. 

889 ixxColumns : `list` [`str`] 

890 Names of the ixx/iyy/ixy columns in the catalog. 

891 

892 Returns 

893 ------- 

894 catalog : `lsst.afw.table.SourceCatalog` 

895 Detector-level catalog extracted from ``visitCatalog``. 

896 """ 

897 # map from dataFrame column to afw table column 

898 mapping = {'x': 'centroid_x', 

899 'y': 'centroid_y', 

900 'xErr': 'centroid_xErr', 

901 'yErr': 'centroid_yErr', 

902 ixxColumns[0]: 'shape_xx', 

903 ixxColumns[1]: 'shape_yy', 

904 ixxColumns[2]: 'shape_xy', 

905 f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux', 

906 f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr', 

907 } 

908 

909 catalog = lsst.afw.table.SourceCatalog(table) 

910 matched = visitCatalog[detectorColumn] == detectorId 

911 catalog.resize(sum(matched)) 

912 view = visitCatalog.loc[matched] 

913 catalog['id'] = view.index 

914 for dfCol, afwCol in mapping.items(): 

915 catalog[afwCol] = view[dfCol] 

916 

917 self.log.debug("%d sources selected in visit %d detector %d", 

918 len(catalog), 

919 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first 

920 detectorId) 

921 return catalog 

922 

923 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, 

924 jointcalControl, camera): 

925 """Read the data that jointcal needs to run. (Gen3 version) 

926 

927 Modifies ``associations`` in-place with the loaded data. 

928 

929 Parameters 

930 ---------- 

931 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

932 References to visit-level DataFrames to load the catalog data from. 

933 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

934 Visit-level exposure summary catalog with metadata. 

935 associations : `lsst.jointcal.Associations` 

936 Object to add the loaded data to by constructing new CcdImages. 

937 jointcalControl : `jointcal.JointcalControl` 

938 Control object for C++ associations management. 

939 camera : `lsst.afw.cameraGeom.Camera` 

940 Camera object for detector geometry. 

941 

942 Returns 

943 ------- 

944 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`] 

945 The original WCS of the input data, to aid in writing tests. 

946 bands : `list` [`str`] 

947 The filter bands of each input dataset. 

948 """ 

949 oldWcsList = [] 

950 filters = [] 

951 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else '' 

952 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

953 table = self._make_schema_table() # every detector catalog has the same layout 

954 # No guarantee that the input is in the same order of visits, so we have to map one of them. 

955 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)} 

956 detectorDict = {detector.getId(): detector for detector in camera} 

957 

958 columns = None 

959 

960 for visitSummaryRef in inputVisitSummary: 

961 visitSummary = visitSummaryRef.get() 

962 

963 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]] 

964 if columns is None: 

965 inColumns = dataRef.get(component='columns') 

966 columns, detColumn, ixxColumns = self._get_sourceTable_visit_columns(inColumns) 

967 visitCatalog = dataRef.get(parameters={'columns': columns}) 

968 

969 selected = self.sourceSelector.run(visitCatalog) 

970 

971 # Build a CcdImage for each detector in this visit. 

972 detectors = {id: index for index, id in enumerate(visitSummary['id'])} 

973 for id, index in detectors.items(): 

974 catalog = self._extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id, 

975 detColumn, ixxColumns) 

976 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict) 

977 result = self._build_ccdImage(data, associations, jointcalControl) 

978 if result is not None: 

979 oldWcsList.append(result.wcs) 

980 # A visit has only one band, so we can just use the first. 

981 filters.append(data.filter) 

982 if len(filters) == 0: 

983 raise RuntimeError("No data to process: did source selector remove all sources?") 

984 filters = collections.Counter(filters) 

985 

986 return oldWcsList, filters 

987 

988 def _make_one_input_data(self, visitRecord, catalog, detectorDict): 

989 """Return a data structure for this detector+visit.""" 

990 return JointcalInputData(visit=visitRecord['visit'], 

991 catalog=catalog, 

992 visitInfo=visitRecord.getVisitInfo(), 

993 detector=detectorDict[visitRecord.getId()], 

994 photoCalib=visitRecord.getPhotoCalib(), 

995 wcs=visitRecord.getWcs(), 

996 bbox=visitRecord.getBBox(), 

997 # ExposureRecord doesn't have a FilterLabel yet, 

998 # so we have to make one. 

999 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'], 

1000 physical=visitRecord['physical_filter'])) 

1001 

1002 def _get_sourceTable_visit_columns(self, inColumns): 

1003 """ 

1004 Get the sourceTable_visit columns from the config. 

1005 

1006 Parameters 

1007 ---------- 

1008 inColumns : `list` 

1009 List of columns available in the sourceTable_visit 

1010 

1011 Returns 

1012 ------- 

1013 columns : `list` 

1014 List of columns to read from sourceTable_visit. 

1015 detectorColumn : `str` 

1016 Name of the detector column. 

1017 ixxColumns : `list` 

1018 Name of the ixx/iyy/ixy columns. 

1019 """ 

1020 if 'detector' in inColumns: 

1021 # Default name for Gen3. 

1022 detectorColumn = 'detector' 

1023 else: 

1024 # Default name for Gen2 and Gen2 conversions. 

1025 detectorColumn = 'ccd' 

1026 

1027 columns = ['visit', detectorColumn, 

1028 'sourceId', 'x', 'xErr', 'y', 'yErr', 

1029 self.config.sourceFluxType + '_instFlux', self.config.sourceFluxType + '_instFluxErr'] 

1030 

1031 if 'ixx' in inColumns: 

1032 # New columns post-DM-31825 

1033 ixxColumns = ['ixx', 'iyy', 'ixy'] 

1034 else: 

1035 # Old columns pre-DM-31825 

1036 ixxColumns = ['Ixx', 'Iyy', 'Ixy'] 

1037 columns.extend(ixxColumns) 

1038 

1039 if self.sourceSelector.config.doFlags: 

1040 columns.extend(self.sourceSelector.config.flags.bad) 

1041 if self.sourceSelector.config.doUnresolved: 

1042 columns.append(self.sourceSelector.config.unresolved.name) 

1043 if self.sourceSelector.config.doIsolated: 

1044 columns.append(self.sourceSelector.config.isolated.parentName) 

1045 columns.append(self.sourceSelector.config.isolated.nChildName) 

1046 

1047 return columns, detectorColumn, ixxColumns 

1048 

1049 # We don't currently need to persist the metadata. 

1050 # If we do in the future, we will have to add appropriate dataset templates 

1051 # to each obs package (the metadata template should look like `jointcal_wcs`). 

1052 def _getMetadataName(self): 

1053 return None 

1054 

1055 @classmethod 

1056 def _makeArgumentParser(cls): 

1057 """Create an argument parser""" 

1058 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

1059 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9", 

1060 ContainerClass=PerTractCcdDataIdContainer) 

1061 return parser 

1062 

1063 def _build_ccdImage(self, data, associations, jointcalControl): 

1064 """ 

1065 Extract the necessary things from this catalog+metadata to add a new 

1066 ccdImage. 

1067 

1068 Parameters 

1069 ---------- 

1070 data : `JointcalInputData` 

1071 The loaded input data. 

1072 associations : `lsst.jointcal.Associations` 

1073 Object to add the info to, to construct a new CcdImage 

1074 jointcalControl : `jointcal.JointcalControl` 

1075 Control object for associations management 

1076 

1077 Returns 

1078 ------ 

1079 namedtuple or `None` 

1080 ``wcs`` 

1081 The TAN WCS of this image, read from the calexp 

1082 (`lsst.afw.geom.SkyWcs`). 

1083 ``key`` 

1084 A key to identify this dataRef by its visit and ccd ids 

1085 (`namedtuple`). 

1086 `None` 

1087 if there are no sources in the loaded catalog. 

1088 """ 

1089 if len(data.catalog) == 0: 

1090 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId()) 

1091 return None 

1092 

1093 associations.createCcdImage(data.catalog, 

1094 data.wcs, 

1095 data.visitInfo, 

1096 data.bbox, 

1097 data.filter.physicalLabel, 

1098 data.photoCalib, 

1099 data.detector, 

1100 data.visit, 

1101 data.detector.getId(), 

1102 jointcalControl) 

1103 

1104 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key')) 

1105 Key = collections.namedtuple('Key', ('visit', 'ccd')) 

1106 return Result(data.wcs, Key(data.visit, data.detector.getId())) 

1107 

1108 def _readDataId(self, butler, dataId): 

1109 """Read all of the data for one dataId from the butler. (gen2 version)""" 

1110 # Not all instruments have `visit` in their dataIds. 

1111 if "visit" in dataId.keys(): 

1112 visit = dataId["visit"] 

1113 else: 

1114 visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0] 

1115 detector = butler.get('calexp_detector', dataId=dataId) 

1116 

1117 catalog = butler.get('src', 

1118 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, 

1119 dataId=dataId) 

1120 goodSrc = self.sourceSelector.run(catalog) 

1121 self.log.debug("%d sources selected in visit %d detector %d", 

1122 len(goodSrc.sourceCat), 

1123 visit, 

1124 detector.getId()) 

1125 return JointcalInputData(visit=visit, 

1126 catalog=goodSrc.sourceCat, 

1127 visitInfo=butler.get('calexp_visitInfo', dataId=dataId), 

1128 detector=detector, 

1129 photoCalib=butler.get('calexp_photoCalib', dataId=dataId), 

1130 wcs=butler.get('calexp_wcs', dataId=dataId), 

1131 bbox=butler.get('calexp_bbox', dataId=dataId), 

1132 filter=butler.get('calexp_filterLabel', dataId=dataId)) 

1133 

1134 def loadData(self, dataRefs, associations, jointcalControl): 

1135 """Read the data that jointcal needs to run. (Gen2 version)""" 

1136 visit_ccd_to_dataRef = {} 

1137 oldWcsList = [] 

1138 filters = [] 

1139 load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else '' 

1140 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

1141 # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models 

1142 camera = dataRefs[0].get('camera', immediate=True) 

1143 self.focalPlaneBBox = camera.getFpBBox() 

1144 for dataRef in dataRefs: 

1145 data = self._readDataId(dataRef.getButler(), dataRef.dataId) 

1146 result = self._build_ccdImage(data, associations, jointcalControl) 

1147 if result is None: 

1148 continue 

1149 oldWcsList.append(result.wcs) 

1150 visit_ccd_to_dataRef[result.key] = dataRef 

1151 filters.append(data.filter) 

1152 if len(filters) == 0: 

1153 raise RuntimeError("No data to process: did source selector remove all sources?") 

1154 filters = collections.Counter(filters) 

1155 

1156 return oldWcsList, filters, visit_ccd_to_dataRef 

1157 

1158 def _getDebugPath(self, filename): 

1159 """Constructs a path to filename using the configured debug path. 

1160 """ 

1161 return os.path.join(self.config.debugOutputPath, filename) 

1162 

1163 def _prep_sky(self, associations, filters): 

1164 """Prepare on-sky and other data that must be computed after data has 

1165 been read. 

1166 """ 

1167 associations.computeCommonTangentPoint() 

1168 

1169 boundingCircle = associations.computeBoundingCircle() 

1170 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

1171 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians) 

1172 

1173 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.") 

1174 

1175 # Determine a default filter band associated with the catalog. See DM-9093 

1176 defaultFilter = filters.most_common(1)[0][0] 

1177 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel) 

1178 

1179 # compute and set the reference epoch of the observations, for proper motion corrections 

1180 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList()) 

1181 associations.setEpoch(epoch.jyear) 

1182 

1183 return boundingCircle, center, radius, defaultFilter, epoch 

1184 

1185 @pipeBase.timeMethod 

1186 def runDataRef(self, dataRefs): 

1187 """ 

1188 Jointly calibrate the astrometry and photometry across a set of images. 

1189 

1190 NOTE: this is for gen2 middleware only. 

1191 

1192 Parameters 

1193 ---------- 

1194 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 

1195 List of data references to the exposures to be fit. 

1196 

1197 Returns 

1198 ------- 

1199 result : `lsst.pipe.base.Struct` 

1200 Struct of metadata from the fit, containing: 

1201 

1202 ``dataRefs`` 

1203 The provided data references that were fit (with updated WCSs) 

1204 ``oldWcsList`` 

1205 The original WCS from each dataRef 

1206 ``metrics`` 

1207 Dictionary of internally-computed metrics for testing/validation. 

1208 """ 

1209 if len(dataRefs) == 0: 

1210 raise ValueError('Need a non-empty list of data references!') 

1211 

1212 exitStatus = 0 # exit status for shell 

1213 

1214 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,) 

1215 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField) 

1216 associations = lsst.jointcal.Associations() 

1217 

1218 oldWcsList, filters, visit_ccd_to_dataRef = self.loadData(dataRefs, 

1219 associations, 

1220 jointcalControl) 

1221 

1222 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, filters) 

1223 

1224 tract = dataRefs[0].dataId['tract'] 

1225 

1226 if self.config.doAstrometry: 

1227 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

1228 name="astrometry", 

1229 refObjLoader=self.astrometryRefObjLoader, 

1230 referenceSelector=self.astrometryReferenceSelector, 

1231 fit_function=self._fit_astrometry, 

1232 tract=tract, 

1233 epoch=epoch) 

1234 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef) 

1235 else: 

1236 astrometry = Astrometry(None, None, None) 

1237 

1238 if self.config.doPhotometry: 

1239 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius, 

1240 name="photometry", 

1241 refObjLoader=self.photometryRefObjLoader, 

1242 referenceSelector=self.photometryReferenceSelector, 

1243 fit_function=self._fit_photometry, 

1244 tract=tract, 

1245 epoch=epoch, 

1246 reject_bad_fluxes=True) 

1247 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef) 

1248 else: 

1249 photometry = Photometry(None, None) 

1250 

1251 return pipeBase.Struct(dataRefs=dataRefs, 

1252 oldWcsList=oldWcsList, 

1253 job=self.job, 

1254 astrometryRefObjLoader=self.astrometryRefObjLoader, 

1255 photometryRefObjLoader=self.photometryRefObjLoader, 

1256 defaultFilter=defaultFilter, 

1257 epoch=epoch, 

1258 exitStatus=exitStatus) 

1259 

1260 def _get_refcat_coordinate_error_override(self, refCat, name): 

1261 """Check whether we should override the refcat coordinate errors, and 

1262 return the overridden error if necessary. 

1263 

1264 Parameters 

1265 ---------- 

1266 refCat : `lsst.afw.table.SimpleCatalog` 

1267 The reference catalog to check for a ``coord_raErr`` field. 

1268 name : `str` 

1269 Whether we are doing "astrometry" or "photometry". 

1270 

1271 Returns 

1272 ------- 

1273 refCoordErr : `float` 

1274 The refcat coordinate error to use, or NaN if we are not overriding 

1275 those fields. 

1276 

1277 Raises 

1278 ------ 

1279 lsst.pex.config.FieldValidationError 

1280 Raised if the refcat does not contain coordinate errors and 

1281 ``config.astrometryReferenceErr`` is not set. 

1282 """ 

1283 # This value doesn't matter for photometry, so just set something to 

1284 # keep old refcats from causing problems. 

1285 if name.lower() == "photometry": 

1286 if 'coord_raErr' not in refCat.schema: 

1287 return 100 

1288 else: 

1289 return float('nan') 

1290 

1291 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema: 

1292 msg = ("Reference catalog does not contain coordinate errors, " 

1293 "and config.astrometryReferenceErr not supplied.") 

1294 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr, 

1295 self.config, 

1296 msg) 

1297 

1298 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema: 

1299 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]", 

1300 self.config.astrometryReferenceErr) 

1301 

1302 if self.config.astrometryReferenceErr is None: 

1303 return float('nan') 

1304 else: 

1305 return self.config.astrometryReferenceErr 

1306 

1307 def _compute_proper_motion_epoch(self, ccdImageList): 

1308 """Return the proper motion correction epoch of the provided images. 

1309 

1310 Parameters 

1311 ---------- 

1312 ccdImageList : `list` [`lsst.jointcal.CcdImage`] 

1313 The images to compute the appropriate epoch for. 

1314 

1315 Returns 

1316 ------- 

1317 epoch : `astropy.time.Time` 

1318 The date to use for proper motion corrections. 

1319 """ 

1320 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]), 

1321 format="jyear", 

1322 scale="tai") 

1323 

1324 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, 

1325 tract="", match_cut=3.0, 

1326 reject_bad_fluxes=False, *, 

1327 name="", refObjLoader=None, referenceSelector=None, 

1328 fit_function=None, epoch=None): 

1329 """Load reference catalog, perform the fit, and return the result. 

1330 

1331 Parameters 

1332 ---------- 

1333 associations : `lsst.jointcal.Associations` 

1334 The star/reference star associations to fit. 

1335 defaultFilter : `lsst.afw.image.FilterLabel` 

1336 filter to load from reference catalog. 

1337 center : `lsst.geom.SpherePoint` 

1338 ICRS center of field to load from reference catalog. 

1339 radius : `lsst.geom.Angle` 

1340 On-sky radius to load from reference catalog. 

1341 name : `str` 

1342 Name of thing being fit: "astrometry" or "photometry". 

1343 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

1344 Reference object loader to use to load a reference catalog. 

1345 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1346 Selector to use to pick objects from the loaded reference catalog. 

1347 fit_function : callable 

1348 Function to call to perform fit (takes Associations object). 

1349 tract : `str`, optional 

1350 Name of tract currently being fit. 

1351 match_cut : `float`, optional 

1352 Radius in arcseconds to find cross-catalog matches to during 

1353 associations.associateCatalogs. 

1354 reject_bad_fluxes : `bool`, optional 

1355 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 

1356 epoch : `astropy.time.Time`, optional 

1357 Epoch to which to correct refcat proper motion and parallax, 

1358 or `None` to not apply such corrections. 

1359 

1360 Returns 

1361 ------- 

1362 result : `Photometry` or `Astrometry` 

1363 Result of `fit_function()` 

1364 """ 

1365 self.log.info("====== Now processing %s...", name) 

1366 # TODO: this should not print "trying to invert a singular transformation:" 

1367 # if it does that, something's not right about the WCS... 

1368 associations.associateCatalogs(match_cut) 

1369 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name, 

1370 associations.fittedStarListSize()) 

1371 

1372 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms 

1373 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector, 

1374 center, radius, defaultFilter, 

1375 applyColorterms=applyColorterms, 

1376 epoch=epoch) 

1377 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name) 

1378 

1379 associations.collectRefStars(refCat, 

1380 self.config.matchCut*lsst.geom.arcseconds, 

1381 fluxField, 

1382 refCoordinateErr=refCoordErr, 

1383 rejectBadFluxes=reject_bad_fluxes) 

1384 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name, 

1385 associations.refStarListSize()) 

1386 

1387 associations.prepareFittedStars(self.config.minMeasurements) 

1388 

1389 self._check_star_lists(associations, name) 

1390 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name, 

1391 associations.nFittedStarsWithAssociatedRefStar()) 

1392 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name, 

1393 associations.fittedStarListSize()) 

1394 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name, 

1395 associations.nCcdImagesValidForFit()) 

1396 

1397 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else '' 

1398 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel) 

1399 with pipeBase.cmdLineTask.profile(load_cat_prof_file): 

1400 result = fit_function(associations, dataName) 

1401 # TODO DM-12446: turn this into a "butler save" somehow. 

1402 # Save reference and measurement chi2 contributions for this data 

1403 if self.config.writeChi2FilesInitialFinal: 

1404 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}") 

1405 result.fit.saveChi2Contributions(baseName+"{type}") 

1406 self.log.info("Wrote chi2 contributions files: %s", baseName) 

1407 

1408 return result 

1409 

1410 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, 

1411 applyColorterms=False, epoch=None): 

1412 """Load the necessary reference catalog sources, convert fluxes to 

1413 correct units, and apply color term corrections if requested. 

1414 

1415 Parameters 

1416 ---------- 

1417 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 

1418 The reference catalog loader to use to get the data. 

1419 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 

1420 Source selector to apply to loaded reference catalog. 

1421 center : `lsst.geom.SpherePoint` 

1422 The center around which to load sources. 

1423 radius : `lsst.geom.Angle` 

1424 The radius around ``center`` to load sources in. 

1425 filterLabel : `lsst.afw.image.FilterLabel` 

1426 The camera filter to load fluxes for. 

1427 applyColorterms : `bool` 

1428 Apply colorterm corrections to the refcat for ``filterName``? 

1429 epoch : `astropy.time.Time`, optional 

1430 Epoch to which to correct refcat proper motion and parallax, 

1431 or `None` to not apply such corrections. 

1432 

1433 Returns 

1434 ------- 

1435 refCat : `lsst.afw.table.SimpleCatalog` 

1436 The loaded reference catalog. 

1437 fluxField : `str` 

1438 The name of the reference catalog flux field appropriate for ``filterName``. 

1439 """ 

1440 skyCircle = refObjLoader.loadSkyCircle(center, 

1441 radius, 

1442 filterLabel.bandLabel, 

1443 epoch=epoch) 

1444 

1445 selected = referenceSelector.run(skyCircle.refCat) 

1446 # Need memory contiguity to get reference filters as a vector. 

1447 if not selected.sourceCat.isContiguous(): 

1448 refCat = selected.sourceCat.copy(deep=True) 

1449 else: 

1450 refCat = selected.sourceCat 

1451 

1452 if applyColorterms: 

1453 refCatName = refObjLoader.ref_dataset_name 

1454 self.log.info("Applying color terms for physical filter=%r reference catalog=%s", 

1455 filterLabel.physicalLabel, refCatName) 

1456 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel, 

1457 refCatName, 

1458 doRaise=True) 

1459 

1460 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

1461 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy) 

1462 # TODO: I didn't want to use this, but I'll deal with it in DM-16903 

1463 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9 

1464 

1465 return refCat, skyCircle.fluxField 

1466 

1467 def _check_star_lists(self, associations, name): 

1468 # TODO: these should be len(blah), but we need this properly wrapped first. 

1469 if associations.nCcdImagesValidForFit() == 0: 

1470 raise RuntimeError('No images in the ccdImageList!') 

1471 if associations.fittedStarListSize() == 0: 

1472 raise RuntimeError('No stars in the {} fittedStarList!'.format(name)) 

1473 if associations.refStarListSize() == 0: 

1474 raise RuntimeError('No stars in the {} reference star list!'.format(name)) 

1475 

1476 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None): 

1477 """Compute chi2, log it, validate the model, and return chi2. 

1478 

1479 Parameters 

1480 ---------- 

1481 associations : `lsst.jointcal.Associations` 

1482 The star/reference star associations to fit. 

1483 fit : `lsst.jointcal.FitterBase` 

1484 The fitter to use for minimization. 

1485 model : `lsst.jointcal.Model` 

1486 The model being fit. 

1487 chi2Label : `str` 

1488 Label to describe the chi2 (e.g. "Initialized", "Final"). 

1489 writeChi2Name : `str`, optional 

1490 Filename prefix to write the chi2 contributions to. 

1491 Do not supply an extension: an appropriate one will be added. 

1492 

1493 Returns 

1494 ------- 

1495 chi2: `lsst.jointcal.Chi2Accumulator` 

1496 The chi2 object for the current fitter and model. 

1497 

1498 Raises 

1499 ------ 

1500 FloatingPointError 

1501 Raised if chi2 is infinite or NaN. 

1502 ValueError 

1503 Raised if the model is not valid. 

1504 """ 

1505 if writeChi2Name is not None: 

1506 fullpath = self._getDebugPath(writeChi2Name) 

1507 fit.saveChi2Contributions(fullpath+"{type}") 

1508 self.log.info("Wrote chi2 contributions files: %s", fullpath) 

1509 

1510 chi2 = fit.computeChi2() 

1511 self.log.info("%s %s", chi2Label, chi2) 

1512 self._check_stars(associations) 

1513 if not np.isfinite(chi2.chi2): 

1514 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}') 

1515 if not model.validate(associations.getCcdImageList(), chi2.ndof): 

1516 raise ValueError("Model is not valid: check log messages for warnings.") 

1517 return chi2 

1518 

1519 def _fit_photometry(self, associations, dataName=None): 

1520 """ 

1521 Fit the photometric data. 

1522 

1523 Parameters 

1524 ---------- 

1525 associations : `lsst.jointcal.Associations` 

1526 The star/reference star associations to fit. 

1527 dataName : `str` 

1528 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1529 identifying debugging files. 

1530 

1531 Returns 

1532 ------- 

1533 fit_result : `namedtuple` 

1534 fit : `lsst.jointcal.PhotometryFit` 

1535 The photometric fitter used to perform the fit. 

1536 model : `lsst.jointcal.PhotometryModel` 

1537 The photometric model that was fit. 

1538 """ 

1539 self.log.info("=== Starting photometric fitting...") 

1540 

1541 # TODO: should use pex.config.RegistryField here (see DM-9195) 

1542 if self.config.photometryModel == "constrainedFlux": 

1543 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(), 

1544 self.focalPlaneBBox, 

1545 visitOrder=self.config.photometryVisitOrder, 

1546 errorPedestal=self.config.photometryErrorPedestal) 

1547 # potentially nonlinear problem, so we may need a line search to converge. 

1548 doLineSearch = self.config.allowLineSearch 

1549 elif self.config.photometryModel == "constrainedMagnitude": 

1550 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(), 

1551 self.focalPlaneBBox, 

1552 visitOrder=self.config.photometryVisitOrder, 

1553 errorPedestal=self.config.photometryErrorPedestal) 

1554 # potentially nonlinear problem, so we may need a line search to converge. 

1555 doLineSearch = self.config.allowLineSearch 

1556 elif self.config.photometryModel == "simpleFlux": 

1557 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(), 

1558 errorPedestal=self.config.photometryErrorPedestal) 

1559 doLineSearch = False # purely linear in model parameters, so no line search needed 

1560 elif self.config.photometryModel == "simpleMagnitude": 

1561 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(), 

1562 errorPedestal=self.config.photometryErrorPedestal) 

1563 doLineSearch = False # purely linear in model parameters, so no line search needed 

1564 

1565 fit = lsst.jointcal.PhotometryFit(associations, model) 

1566 # TODO DM-12446: turn this into a "butler save" somehow. 

1567 # Save reference and measurement chi2 contributions for this data 

1568 if self.config.writeChi2FilesInitialFinal: 

1569 baseName = f"photometry_initial_chi2-{dataName}" 

1570 else: 

1571 baseName = None 

1572 if self.config.writeInitialModel: 

1573 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt") 

1574 writeModel(model, fullpath, self.log) 

1575 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName) 

1576 

1577 def getChi2Name(whatToFit): 

1578 if self.config.writeChi2FilesOuterLoop: 

1579 return f"photometry_init-%s_chi2-{dataName}" % whatToFit 

1580 else: 

1581 return None 

1582 

1583 # The constrained model needs the visit transform fit first; the chip 

1584 # transform is initialized from the singleFrame PhotoCalib, so it's close. 

1585 if self.config.writeInitMatrix: 

1586 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}") 

1587 else: 

1588 dumpMatrixFile = "" 

1589 if self.config.photometryModel.startswith("constrained"): 

1590 # no line search: should be purely (or nearly) linear, 

1591 # and we want a large step size to initialize with. 

1592 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile) 

1593 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit", 

1594 writeChi2Name=getChi2Name("ModelVisit")) 

1595 dumpMatrixFile = "" # so we don't redo the output on the next step 

1596 

1597 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile) 

1598 self._logChi2AndValidate(associations, fit, model, "Initialize Model", 

1599 writeChi2Name=getChi2Name("Model")) 

1600 

1601 fit.minimize("Fluxes") # no line search: always purely linear. 

1602 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes", 

1603 writeChi2Name=getChi2Name("Fluxes")) 

1604 

1605 fit.minimize("Model Fluxes", doLineSearch=doLineSearch) 

1606 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes", 

1607 writeChi2Name=getChi2Name("ModelFluxes")) 

1608 

1609 model.freezeErrorTransform() 

1610 self.log.debug("Photometry error scales are frozen.") 

1611 

1612 chi2 = self._iterate_fit(associations, 

1613 fit, 

1614 self.config.maxPhotometrySteps, 

1615 "photometry", 

1616 "Model Fluxes", 

1617 doRankUpdate=self.config.photometryDoRankUpdate, 

1618 doLineSearch=doLineSearch, 

1619 dataName=dataName) 

1620 

1621 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2) 

1622 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof) 

1623 return Photometry(fit, model) 

1624 

1625 def _fit_astrometry(self, associations, dataName=None): 

1626 """ 

1627 Fit the astrometric data. 

1628 

1629 Parameters 

1630 ---------- 

1631 associations : `lsst.jointcal.Associations` 

1632 The star/reference star associations to fit. 

1633 dataName : `str` 

1634 Name of the data being processed (e.g. "1234_HSC-Y"), for 

1635 identifying debugging files. 

1636 

1637 Returns 

1638 ------- 

1639 fit_result : `namedtuple` 

1640 fit : `lsst.jointcal.AstrometryFit` 

1641 The astrometric fitter used to perform the fit. 

1642 model : `lsst.jointcal.AstrometryModel` 

1643 The astrometric model that was fit. 

1644 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 

1645 The model for the sky to tangent plane projection that was used in the fit. 

1646 """ 

1647 

1648 self.log.info("=== Starting astrometric fitting...") 

1649 

1650 associations.deprojectFittedStars() 

1651 

1652 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected. 

1653 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage 

1654 # them so carefully? 

1655 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList()) 

1656 

1657 if self.config.astrometryModel == "constrained": 

1658 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(), 

1659 sky_to_tan_projection, 

1660 chipOrder=self.config.astrometryChipOrder, 

1661 visitOrder=self.config.astrometryVisitOrder) 

1662 elif self.config.astrometryModel == "simple": 

1663 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(), 

1664 sky_to_tan_projection, 

1665 self.config.useInputWcs, 

1666 nNotFit=0, 

1667 order=self.config.astrometrySimpleOrder) 

1668 

1669 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal) 

1670 # TODO DM-12446: turn this into a "butler save" somehow. 

1671 # Save reference and measurement chi2 contributions for this data 

1672 if self.config.writeChi2FilesInitialFinal: 

1673 baseName = f"astrometry_initial_chi2-{dataName}" 

1674 else: 

1675 baseName = None 

1676 if self.config.writeInitialModel: 

1677 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt") 

1678 writeModel(model, fullpath, self.log) 

1679 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName) 

1680 

1681 def getChi2Name(whatToFit): 

1682 if self.config.writeChi2FilesOuterLoop: 

1683 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit 

1684 else: 

1685 return None 

1686 

1687 if self.config.writeInitMatrix: 

1688 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}") 

1689 else: 

1690 dumpMatrixFile = "" 

1691 # The constrained model needs the visit transform fit first; the chip 

1692 # transform is initialized from the detector's cameraGeom, so it's close. 

1693 if self.config.astrometryModel == "constrained": 

1694 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile) 

1695 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit", 

1696 writeChi2Name=getChi2Name("DistortionsVisit")) 

1697 dumpMatrixFile = "" # so we don't redo the output on the next step 

1698 

1699 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile) 

1700 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions", 

1701 writeChi2Name=getChi2Name("Distortions")) 

1702 

1703 fit.minimize("Positions") 

1704 self._logChi2AndValidate(associations, fit, model, "Initialize Positions", 

1705 writeChi2Name=getChi2Name("Positions")) 

1706 

1707 fit.minimize("Distortions Positions") 

1708 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions", 

1709 writeChi2Name=getChi2Name("DistortionsPositions")) 

1710 

1711 chi2 = self._iterate_fit(associations, 

1712 fit, 

1713 self.config.maxAstrometrySteps, 

1714 "astrometry", 

1715 "Distortions Positions", 

1716 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance, 

1717 doRankUpdate=self.config.astrometryDoRankUpdate, 

1718 dataName=dataName) 

1719 

1720 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2) 

1721 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof) 

1722 

1723 return Astrometry(fit, model, sky_to_tan_projection) 

1724 

1725 def _check_stars(self, associations): 

1726 """Count measured and reference stars per ccd and warn/log them.""" 

1727 for ccdImage in associations.getCcdImageList(): 

1728 nMeasuredStars, nRefStars = ccdImage.countStars() 

1729 self.log.debug("ccdImage %s has %s measured and %s reference stars", 

1730 ccdImage.getName(), nMeasuredStars, nRefStars) 

1731 if nMeasuredStars < self.config.minMeasuredStarsPerCcd: 

1732 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)", 

1733 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd) 

1734 if nRefStars < self.config.minRefStarsPerCcd: 

1735 self.log.warning("ccdImage %s has only %s RefStars (desired %s)", 

1736 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd) 

1737 

1738 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, 

1739 dataName="", 

1740 sigmaRelativeTolerance=0, 

1741 doRankUpdate=True, 

1742 doLineSearch=False): 

1743 """Run fitter.minimize up to max_steps times, returning the final chi2. 

1744 

1745 Parameters 

1746 ---------- 

1747 associations : `lsst.jointcal.Associations` 

1748 The star/reference star associations to fit. 

1749 fitter : `lsst.jointcal.FitterBase` 

1750 The fitter to use for minimization. 

1751 max_steps : `int` 

1752 Maximum number of steps to run outlier rejection before declaring 

1753 convergence failure. 

1754 name : {'photometry' or 'astrometry'} 

1755 What type of data are we fitting (for logs and debugging files). 

1756 whatToFit : `str` 

1757 Passed to ``fitter.minimize()`` to define the parameters to fit. 

1758 dataName : `str`, optional 

1759 Descriptive name for this dataset (e.g. tract and filter), 

1760 for debugging. 

1761 sigmaRelativeTolerance : `float`, optional 

1762 Convergence tolerance for the fractional change in the chi2 cut 

1763 level for determining outliers. If set to zero, iterations will 

1764 continue until there are no outliers. 

1765 doRankUpdate : `bool`, optional 

1766 Do an Eigen rank update during minimization, or recompute the full 

1767 matrix and gradient? 

1768 doLineSearch : `bool`, optional 

1769 Do a line search for the optimum step during minimization? 

1770 

1771 Returns 

1772 ------- 

1773 chi2: `lsst.jointcal.Chi2Statistic` 

1774 The final chi2 after the fit converges, or is forced to end. 

1775 

1776 Raises 

1777 ------ 

1778 FloatingPointError 

1779 Raised if the fitter fails with a non-finite value. 

1780 RuntimeError 

1781 Raised if the fitter fails for some other reason; 

1782 log messages will provide further details. 

1783 """ 

1784 if self.config.writeInitMatrix: 

1785 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}") 

1786 else: 

1787 dumpMatrixFile = "" 

1788 oldChi2 = lsst.jointcal.Chi2Statistic() 

1789 oldChi2.chi2 = float("inf") 

1790 for i in range(max_steps): 

1791 if self.config.writeChi2FilesOuterLoop: 

1792 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}" 

1793 else: 

1794 writeChi2Name = None 

1795 result = fitter.minimize(whatToFit, 

1796 self.config.outlierRejectSigma, 

1797 sigmaRelativeTolerance=sigmaRelativeTolerance, 

1798 doRankUpdate=doRankUpdate, 

1799 doLineSearch=doLineSearch, 

1800 dumpMatrixFile=dumpMatrixFile) 

1801 dumpMatrixFile = "" # clear it so we don't write the matrix again. 

1802 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), 

1803 f"Fit iteration {i}", writeChi2Name=writeChi2Name) 

1804 

1805 if result == MinimizeResult.Converged: 

1806 if doRankUpdate: 

1807 self.log.debug("fit has converged - no more outliers - redo minimization " 

1808 "one more time in case we have lost accuracy in rank update.") 

1809 # Redo minimization one more time in case we have lost accuracy in rank update 

1810 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma, 

1811 sigmaRelativeTolerance=sigmaRelativeTolerance) 

1812 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed") 

1813 

1814 # log a message for a large final chi2, TODO: DM-15247 for something better 

1815 if chi2.chi2/chi2.ndof >= 4.0: 

1816 self.log.error("Potentially bad fit: High chi-squared/ndof.") 

1817 

1818 break 

1819 elif result == MinimizeResult.Chi2Increased: 

1820 self.log.warning("Still some outliers remaining but chi2 increased - retry") 

1821 # Check whether the increase was large enough to cause trouble. 

1822 chi2Ratio = chi2.chi2 / oldChi2.chi2 

1823 if chi2Ratio > 1.5: 

1824 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g', 

1825 chi2.chi2, oldChi2.chi2, chi2Ratio) 

1826 # Based on a variety of HSC jointcal logs (see DM-25779), it 

1827 # appears that chi2 increases more than a factor of ~2 always 

1828 # result in the fit diverging rapidly and ending at chi2 > 1e10. 

1829 # Using 10 as the "failure" threshold gives some room between 

1830 # leaving a warning and bailing early. 

1831 if chi2Ratio > 10: 

1832 msg = ("Large chi2 increase between steps: fit likely cannot converge." 

1833 " Try setting one or more of the `writeChi2*` config fields and looking" 

1834 " at how individual star chi2-values evolve during the fit.") 

1835 raise RuntimeError(msg) 

1836 oldChi2 = chi2 

1837 elif result == MinimizeResult.NonFinite: 

1838 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)) 

1839 # TODO DM-12446: turn this into a "butler save" somehow. 

1840 fitter.saveChi2Contributions(filename+"{type}") 

1841 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 

1842 raise FloatingPointError(msg.format(filename)) 

1843 elif result == MinimizeResult.Failed: 

1844 raise RuntimeError("Chi2 minimization failure, cannot complete fit.") 

1845 else: 

1846 raise RuntimeError("Unxepected return code from minimize().") 

1847 else: 

1848 self.log.error("%s failed to converge after %d steps"%(name, max_steps)) 

1849 

1850 return chi2 

1851 

1852 def _make_output(self, ccdImageList, model, func): 

1853 """Return the internal jointcal models converted to the afw 

1854 structures that will be saved to disk. 

1855 

1856 Parameters 

1857 ---------- 

1858 ccdImageList : `lsst.jointcal.CcdImageList` 

1859 The list of CcdImages to get the output for. 

1860 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel` 

1861 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`. 

1862 func : `str` 

1863 The name of the function to call on ``model`` to get the converted 

1864 structure. Must accept an `lsst.jointcal.CcdImage`. 

1865 

1866 Returns 

1867 ------- 

1868 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or 

1869 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`] 

1870 The data to be saved, keyed on (visit, detector). 

1871 """ 

1872 output = {} 

1873 for ccdImage in ccdImageList: 

1874 ccd = ccdImage.ccdId 

1875 visit = ccdImage.visit 

1876 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd) 

1877 output[(visit, ccd)] = getattr(model, func)(ccdImage) 

1878 return output 

1879 

1880 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef): 

1881 """ 

1882 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 

1883 

1884 Parameters 

1885 ---------- 

1886 associations : `lsst.jointcal.Associations` 

1887 The star/reference star associations to fit. 

1888 model : `lsst.jointcal.AstrometryModel` 

1889 The astrometric model that was fit. 

1890 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1891 Dict of ccdImage identifiers to dataRefs that were fit. 

1892 """ 

1893 ccdImageList = associations.getCcdImageList() 

1894 output = self._make_output(ccdImageList, model, "makeSkyWcs") 

1895 for key, skyWcs in output.items(): 

1896 dataRef = visit_ccd_to_dataRef[key] 

1897 try: 

1898 dataRef.put(skyWcs, 'jointcal_wcs') 

1899 except pexExceptions.Exception as e: 

1900 self.log.fatal('Failed to write updated Wcs: %s', str(e)) 

1901 raise e 

1902 

1903 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef): 

1904 """ 

1905 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 

1906 

1907 Parameters 

1908 ---------- 

1909 associations : `lsst.jointcal.Associations` 

1910 The star/reference star associations to fit. 

1911 model : `lsst.jointcal.PhotometryModel` 

1912 The photoometric model that was fit. 

1913 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 

1914 Dict of ccdImage identifiers to dataRefs that were fit. 

1915 """ 

1916 

1917 ccdImageList = associations.getCcdImageList() 

1918 output = self._make_output(ccdImageList, model, "toPhotoCalib") 

1919 for key, photoCalib in output.items(): 

1920 dataRef = visit_ccd_to_dataRef[key] 

1921 try: 

1922 dataRef.put(photoCalib, 'jointcal_photoCalib') 

1923 except pexExceptions.Exception as e: 

1924 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e)) 

1925 raise e