Coverage for tests/test_calibrateImage.py: 15%

294 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-01 04:34 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23from unittest import mock 

24import tempfile 

25 

26import astropy.units as u 

27from astropy.coordinates import SkyCoord 

28import numpy as np 

29 

30import lsst.afw.image as afwImage 

31import lsst.afw.math as afwMath 

32import lsst.afw.table as afwTable 

33import lsst.daf.base 

34import lsst.daf.butler 

35import lsst.daf.butler.tests as butlerTests 

36import lsst.geom 

37import lsst.meas.algorithms 

38from lsst.meas.algorithms import testUtils 

39import lsst.meas.extensions.psfex 

40import lsst.meas.base 

41import lsst.meas.base.tests 

42import lsst.pipe.base.testUtils 

43from lsst.pipe.tasks.calibrateImage import CalibrateImageTask 

44import lsst.utils.tests 

45 

46 

47class CalibrateImageTaskTests(lsst.utils.tests.TestCase): 

48 

49 def setUp(self): 

50 # Different x/y dimensions so they're easy to distinguish in a plot, 

51 # and non-zero minimum, to help catch xy0 errors. 

52 bbox = lsst.geom.Box2I(lsst.geom.Point2I(5, 4), lsst.geom.Point2I(205, 184)) 

53 self.sky_center = lsst.geom.SpherePoint(245.0, -45.0, lsst.geom.degrees) 

54 self.photo_calib = 12.3 

55 dataset = lsst.meas.base.tests.TestDataset(bbox, crval=self.sky_center, calibration=self.photo_calib) 

56 # sqrt of area of a normalized 2d gaussian 

57 psf_scale = np.sqrt(4*np.pi*(dataset.psfShape.getDeterminantRadius())**2) 

58 noise = 10.0 # stddev of noise per pixel 

59 # Sources ordered from faintest to brightest. 

60 self.fluxes = np.array((6*noise*psf_scale, 

61 12*noise*psf_scale, 

62 45*noise*psf_scale, 

63 150*noise*psf_scale, 

64 400*noise*psf_scale, 

65 1000*noise*psf_scale)) 

66 self.centroids = np.array(((162, 22), 

67 (40, 70), 

68 (100, 160), 

69 (50, 120), 

70 (92, 35), 

71 (175, 154)), dtype=np.float32) 

72 for flux, centroid in zip(self.fluxes, self.centroids): 

73 dataset.addSource(instFlux=flux, centroid=lsst.geom.Point2D(centroid[0], centroid[1])) 

74 

75 # Bright extended source in the center of the image: should not appear 

76 # in any of the output catalogs. 

77 center = lsst.geom.Point2D(100, 100) 

78 shape = lsst.afw.geom.Quadrupole(8, 9, 3) 

79 dataset.addSource(instFlux=500*noise*psf_scale, centroid=center, shape=shape) 

80 

81 schema = dataset.makeMinimalSchema() 

82 self.truth_exposure, self.truth_cat = dataset.realize(noise=noise, schema=schema) 

83 # To make it look like a version=1 (nJy fluxes) refcat 

84 self.truth_cat = self.truth_exposure.photoCalib.calibrateCatalog(self.truth_cat) 

85 self.ref_loader = testUtils.MockReferenceObjectLoaderFromMemory([self.truth_cat]) 

86 metadata = lsst.daf.base.PropertyList() 

87 metadata.set("REFCAT_FORMAT_VERSION", 1) 

88 self.truth_cat.setMetadata(metadata) 

89 

90 # TODO: a cosmic ray (need to figure out how to insert a fake-CR) 

91 # self.truth_exposure.image.array[10, 10] = 100000 

92 # self.truth_exposure.variance.array[10, 10] = 100000/noise 

93 

94 # Copy the truth exposure, because CalibrateImage modifies the input. 

95 # Post-ISR ccds only contain: initial WCS, VisitInfo, filter 

96 self.exposure = afwImage.ExposureF(self.truth_exposure.maskedImage) 

97 self.exposure.setWcs(self.truth_exposure.wcs) 

98 self.exposure.info.setVisitInfo(self.truth_exposure.visitInfo) 

99 # "truth" filter, to match the "truth" refcat. 

100 self.exposure.setFilter(lsst.afw.image.FilterLabel(physical='truth', band="truth")) 

101 

102 # Test-specific configuration: 

103 self.config = CalibrateImageTask.ConfigClass() 

104 # We don't have many sources, so have to fit simpler models. 

105 self.config.psf_detection.background.approxOrderX = 1 

106 self.config.star_detection.background.approxOrderX = 1 

107 # Only insert 2 sky sources, for simplicity. 

108 self.config.star_sky_sources.nSources = 2 

109 # Use PCA psf fitter, as psfex fails if there are only 4 stars. 

110 self.config.psf_measure_psf.psfDeterminer = 'pca' 

111 # We don't have many test points, so can't match on complicated shapes. 

112 self.config.astrometry.matcher.numPointsForShape = 3 

113 # ApFlux has more noise than PsfFlux (the latter unrealistically small 

114 # in this test data), so we need to do magnitude rejection at higher 

115 # sigma, otherwise we can lose otherwise good sources. 

116 # TODO DM-39203: Once we are using Compensated Gaussian Fluxes, we 

117 # will use those fluxes here, and hopefully can remove this. 

118 self.config.astrometry.magnitudeOutlierRejectionNSigma = 9.0 

119 

120 # Make a realistic id generator so that output catalog ids are useful. 

121 # NOTE: The id generator is used to seed the noise replacer during 

122 # measurement, so changes to values here can have subtle effects on 

123 # the centroids and fluxes mesaured on the image, which might cause 

124 # tests to fail. 

125 data_id = lsst.daf.butler.DataCoordinate.standardize( 

126 instrument="I", 

127 visit=self.truth_exposure.visitInfo.id, 

128 detector=12, 

129 universe=lsst.daf.butler.DimensionUniverse(), 

130 ) 

131 self.config.id_generator.packer.name = "observation" 

132 self.config.id_generator.packer["observation"].n_observations = 10000 

133 self.config.id_generator.packer["observation"].n_detectors = 99 

134 self.config.id_generator.n_releases = 8 

135 self.config.id_generator.release_id = 2 

136 self.id_generator = self.config.id_generator.apply(data_id) 

137 

138 # Something about this test dataset prefers a larger threshold here. 

139 self.config.star_selector["science"].unresolved.maximum = 0.2 

140 

141 def _check_run(self, calibrate, result): 

142 """Test the result of CalibrateImage.run(). 

143 

144 Parameters 

145 ---------- 

146 calibrate : `lsst.pipe.tasks.calibrateImage.CalibrateImageTask` 

147 Configured task that had `run` called on it. 

148 result : `lsst.pipe.base.Struct` 

149 Result of calling calibrate.run(). 

150 """ 

151 # Background should have 4 elements: 3 from compute_psf and one from 

152 # re-estimation during source detection. 

153 self.assertEqual(len(result.background), 4) 

154 

155 # Both afw and astropy psf_stars catalogs should be populated. 

156 self.assertEqual(result.psf_stars["calib_psf_used"].sum(), 3) 

157 self.assertEqual(result.psf_stars_footprints["calib_psf_used"].sum(), 3) 

158 

159 # Check that the summary statistics are reasonable. 

160 summary = result.exposure.info.getSummaryStats() 

161 self.assertFloatsAlmostEqual(summary.psfSigma, 2.0, rtol=1e-2) 

162 self.assertFloatsAlmostEqual(summary.ra, self.sky_center.getRa().asDegrees(), rtol=1e-7) 

163 self.assertFloatsAlmostEqual(summary.dec, self.sky_center.getDec().asDegrees(), rtol=1e-7) 

164 

165 # Should have finite sky coordinates in the afw and astropy catalogs. 

166 self.assertTrue(np.isfinite(result.stars_footprints["coord_ra"]).all()) 

167 self.assertTrue(np.isfinite(result.stars["coord_ra"]).all()) 

168 

169 # Returned photoCalib should be the applied value, not the ==1 one on the exposure. 

170 self.assertFloatsAlmostEqual(result.applied_photo_calib.getCalibrationMean(), 

171 self.photo_calib, rtol=2e-3) 

172 # Should have calibrated flux/magnitudes in the afw and astropy catalogs 

173 self.assertIn("slot_PsfFlux_flux", result.stars_footprints.schema) 

174 self.assertIn("slot_PsfFlux_mag", result.stars_footprints.schema) 

175 self.assertEqual(result.stars["slot_PsfFlux_flux"].unit, u.nJy) 

176 self.assertEqual(result.stars["slot_PsfFlux_mag"].unit, u.ABmag) 

177 

178 # Should have detected all S/N >= 10 sources plus 2 sky sources, whether 1 or 2 snaps. 

179 self.assertEqual(len(result.stars), 7) 

180 # Did the psf flags get propagated from the psf_stars catalog? 

181 self.assertEqual(result.stars["calib_psf_used"].sum(), 3) 

182 

183 # Check that all necessary fields are in the output. 

184 lsst.pipe.base.testUtils.assertValidOutput(calibrate, result) 

185 

186 def test_run(self): 

187 """Test that run() returns reasonable values to be butler put. 

188 """ 

189 calibrate = CalibrateImageTask(config=self.config) 

190 calibrate.astrometry.setRefObjLoader(self.ref_loader) 

191 calibrate.photometry.match.setRefObjLoader(self.ref_loader) 

192 result = calibrate.run(exposures=self.exposure) 

193 

194 self._check_run(calibrate, result) 

195 

196 def test_run_2_snaps(self): 

197 """Test that run() returns reasonable values to be butler put, when 

198 passed two exposures to combine as snaps. 

199 """ 

200 calibrate = CalibrateImageTask(config=self.config) 

201 calibrate.astrometry.setRefObjLoader(self.ref_loader) 

202 calibrate.photometry.match.setRefObjLoader(self.ref_loader) 

203 # Halve the flux in each exposure to get the expected visit sum. 

204 self.exposure.image /= 2 

205 self.exposure.variance /= 2 

206 result = calibrate.run(exposures=[self.exposure, self.exposure]) 

207 

208 self._check_run(calibrate, result) 

209 

210 def test_handle_snaps(self): 

211 calibrate = CalibrateImageTask(config=self.config) 

212 self.assertEqual(calibrate._handle_snaps(self.exposure), self.exposure) 

213 self.assertEqual(calibrate._handle_snaps((self.exposure, )), self.exposure) 

214 self.assertEqual(calibrate._handle_snaps(self.exposure), self.exposure) 

215 with self.assertRaisesRegex(RuntimeError, "Can only process 1 or 2 snaps, not 0."): 

216 calibrate._handle_snaps([]) 

217 with self.assertRaisesRegex(RuntimeError, "Can only process 1 or 2 snaps, not 3."): 

218 calibrate._handle_snaps(3*[self.exposure]) 

219 

220 def test_compute_psf(self): 

221 """Test that our brightest sources are found by _compute_psf(), 

222 that a PSF is assigned to the expopsure. 

223 """ 

224 calibrate = CalibrateImageTask(config=self.config) 

225 psf_stars, background, candidates = calibrate._compute_psf(self.exposure, self.id_generator) 

226 

227 # Catalog ids should be very large from this id generator. 

228 self.assertTrue(all(psf_stars['id'] > 1000000000)) 

229 

230 # Background should have 3 elements: initial subtraction, and two from 

231 # re-estimation during the two detection passes. 

232 self.assertEqual(len(background), 3) 

233 

234 # Only the point-sources with S/N > 50 should be in this output. 

235 self.assertEqual(psf_stars["calib_psf_used"].sum(), 3) 

236 # Sort in order of brightness, to easily compare with expected positions. 

237 psf_stars.sort(psf_stars.getPsfFluxSlot().getMeasKey()) 

238 for record, flux, center in zip(psf_stars[::-1], self.fluxes, self.centroids[self.fluxes > 50]): 

239 self.assertFloatsAlmostEqual(record.getX(), center[0], rtol=0.01) 

240 self.assertFloatsAlmostEqual(record.getY(), center[1], rtol=0.01) 

241 # PsfFlux should match the values inserted. 

242 self.assertFloatsAlmostEqual(record["slot_PsfFlux_instFlux"], flux, rtol=0.01) 

243 

244 # TODO: While debugging DM-32701, we're using PCA instead of psfex. 

245 # Check that we got a useable PSF. 

246 # self.assertIsInstance(self.exposure.psf, lsst.meas.extensions.psfex.PsfexPsf) 

247 self.assertIsInstance(self.exposure.psf, lsst.meas.algorithms.PcaPsf) 

248 # TestDataset sources have PSF radius=2 pixels. 

249 radius = self.exposure.psf.computeShape(self.exposure.psf.getAveragePosition()).getDeterminantRadius() 

250 self.assertFloatsAlmostEqual(radius, 2.0, rtol=1e-2) 

251 

252 # To look at images for debugging (`setup display_ds9` and run ds9): 

253 # import lsst.afw.display 

254 # display = lsst.afw.display.getDisplay() 

255 # display.mtv(self.exposure) 

256 

257 def test_measure_aperture_correction(self): 

258 """Test that _measure_aperture_correction() assigns an ApCorrMap to the 

259 exposure. 

260 """ 

261 calibrate = CalibrateImageTask(config=self.config) 

262 psf_stars, background, candidates = calibrate._compute_psf(self.exposure, self.id_generator) 

263 

264 # First check that the exposure doesn't have an ApCorrMap. 

265 self.assertIsNone(self.exposure.apCorrMap) 

266 calibrate._measure_aperture_correction(self.exposure, psf_stars) 

267 self.assertIsInstance(self.exposure.apCorrMap, afwImage.ApCorrMap) 

268 

269 def test_find_stars(self): 

270 """Test that _find_stars() correctly identifies the S/N>10 stars 

271 in the image and returns them in the output catalog. 

272 """ 

273 calibrate = CalibrateImageTask(config=self.config) 

274 psf_stars, background, candidates = calibrate._compute_psf(self.exposure, self.id_generator) 

275 calibrate._measure_aperture_correction(self.exposure, psf_stars) 

276 

277 stars = calibrate._find_stars(self.exposure, background, self.id_generator) 

278 

279 # Catalog ids should be very large from this id generator. 

280 self.assertTrue(all(stars['id'] > 1000000000)) 

281 

282 # Background should have 4 elements: 3 from compute_psf and one from 

283 # re-estimation during source detection. 

284 self.assertEqual(len(background), 4) 

285 

286 # Only 5 psf-like sources with S/N>10 should be in the output catalog, 

287 # plus two sky sources. 

288 self.assertEqual(len(stars), 7) 

289 self.assertTrue(stars.isContiguous()) 

290 # Sort in order of brightness, to easily compare with expected positions. 

291 stars.sort(stars.getPsfFluxSlot().getMeasKey()) 

292 for record, flux, center in zip(stars[::-1], self.fluxes, self.centroids[self.fluxes > 50]): 

293 self.assertFloatsAlmostEqual(record.getX(), center[0], rtol=0.01) 

294 self.assertFloatsAlmostEqual(record.getY(), center[1], rtol=0.01) 

295 self.assertFloatsAlmostEqual(record["slot_PsfFlux_instFlux"], flux, rtol=0.01) 

296 

297 def test_astrometry(self): 

298 """Test that the fitted WCS gives good catalog coordinates. 

299 """ 

300 calibrate = CalibrateImageTask(config=self.config) 

301 calibrate.astrometry.setRefObjLoader(self.ref_loader) 

302 psf_stars, background, candidates = calibrate._compute_psf(self.exposure, self.id_generator) 

303 calibrate._measure_aperture_correction(self.exposure, psf_stars) 

304 stars = calibrate._find_stars(self.exposure, background, self.id_generator) 

305 

306 calibrate._fit_astrometry(self.exposure, stars) 

307 

308 # Check that we got reliable matches with the truth coordinates. 

309 sky = stars["sky_source"] 

310 fitted = SkyCoord(stars[~sky]['coord_ra'], stars[~sky]['coord_dec'], unit="radian") 

311 truth = SkyCoord(self.truth_cat['coord_ra'], self.truth_cat['coord_dec'], unit="radian") 

312 idx, d2d, _ = fitted.match_to_catalog_sky(truth) 

313 np.testing.assert_array_less(d2d.to_value(u.milliarcsecond), 35.0) 

314 

315 def test_photometry(self): 

316 """Test that the fitted photoCalib matches the one we generated, 

317 and that the exposure is calibrated. 

318 """ 

319 calibrate = CalibrateImageTask(config=self.config) 

320 calibrate.astrometry.setRefObjLoader(self.ref_loader) 

321 calibrate.photometry.match.setRefObjLoader(self.ref_loader) 

322 psf_stars, background, candidates = calibrate._compute_psf(self.exposure, self.id_generator) 

323 calibrate._measure_aperture_correction(self.exposure, psf_stars) 

324 stars = calibrate._find_stars(self.exposure, background, self.id_generator) 

325 calibrate._fit_astrometry(self.exposure, stars) 

326 

327 stars, matches, meta, photoCalib = calibrate._fit_photometry(self.exposure, stars) 

328 

329 # NOTE: With this test data, PhotoCalTask returns calibrationErr==0, 

330 # so we can't check that the photoCal error has been set. 

331 self.assertFloatsAlmostEqual(photoCalib.getCalibrationMean(), self.photo_calib, rtol=2e-3) 

332 # The exposure should be calibrated by the applied photoCalib. 

333 self.assertFloatsAlmostEqual(self.exposure.image.array/self.truth_exposure.image.array, 

334 self.photo_calib, rtol=2e-3) 

335 # PhotoCalib on the exposure must be identically 1. 

336 self.assertEqual(self.exposure.photoCalib.getCalibrationMean(), 1.0) 

337 

338 # Check that we got reliable magnitudes and fluxes vs. truth, ignoring 

339 # sky sources. 

340 sky = stars["sky_source"] 

341 fitted = SkyCoord(stars[~sky]['coord_ra'], stars[~sky]['coord_dec'], unit="radian") 

342 truth = SkyCoord(self.truth_cat['coord_ra'], self.truth_cat['coord_dec'], unit="radian") 

343 idx, _, _ = fitted.match_to_catalog_sky(truth) 

344 # Because the input variance image does not include contributions from 

345 # the sources, we can't use fluxErr as a bound on the measurement 

346 # quality here. 

347 self.assertFloatsAlmostEqual(stars[~sky]['slot_PsfFlux_flux'], 

348 self.truth_cat['truth_flux'][idx], 

349 rtol=0.1) 

350 self.assertFloatsAlmostEqual(stars[~sky]['slot_PsfFlux_mag'], 

351 self.truth_cat['truth_mag'][idx], 

352 rtol=0.01) 

353 

354 def test_match_psf_stars(self): 

355 """Test that _match_psf_stars() flags the correct stars as psf stars 

356 and candidates. 

357 """ 

358 calibrate = CalibrateImageTask(config=self.config) 

359 psf_stars, background, candidates = calibrate._compute_psf(self.exposure, self.id_generator) 

360 calibrate._measure_aperture_correction(self.exposure, psf_stars) 

361 stars = calibrate._find_stars(self.exposure, background, self.id_generator) 

362 

363 # There should be no psf-related flags set at first. 

364 self.assertEqual(stars["calib_psf_candidate"].sum(), 0) 

365 self.assertEqual(stars["calib_psf_used"].sum(), 0) 

366 self.assertEqual(stars["calib_psf_reserved"].sum(), 0) 

367 

368 # Reorder stars to be out of order with psf_stars (putting the sky 

369 # sources in front); this tests that I get the indexing right. 

370 stars.sort(stars.getCentroidSlot().getMeasKey().getX()) 

371 stars = stars.copy(deep=True) 

372 # Re-number the ids: the matcher requires sorted ids: this is always 

373 # true in the code itself, but we've permuted them by sorting on 

374 # flux. We don't care what the actual ids themselves are here. 

375 stars["id"] = np.arange(len(stars)) 

376 

377 calibrate._match_psf_stars(psf_stars, stars) 

378 

379 # Check that the three brightest stars have the psf flags transfered 

380 # from the psf_stars catalog by sorting in order of brightness. 

381 stars.sort(stars.getPsfFluxSlot().getMeasKey()) 

382 # sort() above leaves the catalog non-contiguous. 

383 stars = stars.copy(deep=True) 

384 np.testing.assert_array_equal(stars["calib_psf_candidate"], 

385 [False, False, False, False, True, True, True]) 

386 np.testing.assert_array_equal(stars["calib_psf_used"], [False, False, False, False, True, True, True]) 

387 # Too few sources to reserve any in these tests. 

388 self.assertEqual(stars["calib_psf_reserved"].sum(), 0) 

389 

390 def test_match_psf_stars_no_matches(self): 

391 """Check that _match_psf_stars handles the case of no cross-matches. 

392 """ 

393 calibrate = CalibrateImageTask(config=self.config) 

394 # Make two catalogs that cannot have matches. 

395 stars = self.truth_cat[2:].copy(deep=True) 

396 psf_stars = self.truth_cat[:2].copy(deep=True) 

397 

398 with self.assertRaisesRegex(RuntimeError, "0 psf_stars out of 2 matched"): 

399 calibrate._match_psf_stars(psf_stars, stars) 

400 

401 

402class CalibrateImageTaskRunQuantumTests(lsst.utils.tests.TestCase): 

403 """Tests of ``CalibrateImageTask.runQuantum``, which need a test butler, 

404 but do not need real images. 

405 """ 

406 def setUp(self): 

407 instrument = "testCam" 

408 exposure0 = 101 

409 exposure1 = 102 

410 visit = 100101 

411 detector = 42 

412 

413 # Create a and populate a test butler for runQuantum tests. 

414 self.repo_path = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) 

415 self.repo = butlerTests.makeTestRepo(self.repo_path.name) 

416 

417 # A complete instrument record is necessary for the id generator. 

418 instrumentRecord = self.repo.dimensions["instrument"].RecordClass( 

419 name=instrument, visit_max=1e6, exposure_max=1e6, detector_max=128, 

420 class_name="lsst.obs.base.instrument_tests.DummyCam", 

421 ) 

422 self.repo.registry.syncDimensionData("instrument", instrumentRecord) 

423 

424 # dataIds for fake data 

425 butlerTests.addDataIdValue(self.repo, "detector", detector) 

426 butlerTests.addDataIdValue(self.repo, "exposure", exposure0) 

427 butlerTests.addDataIdValue(self.repo, "exposure", exposure1) 

428 butlerTests.addDataIdValue(self.repo, "visit", visit) 

429 

430 # inputs 

431 butlerTests.addDatasetType(self.repo, "postISRCCD", {"instrument", "exposure", "detector"}, 

432 "ExposureF") 

433 butlerTests.addDatasetType(self.repo, "gaia_dr3_20230707", {"htm7"}, "SimpleCatalog") 

434 butlerTests.addDatasetType(self.repo, "ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog") 

435 

436 # outputs 

437 butlerTests.addDatasetType(self.repo, "initial_pvi", {"instrument", "visit", "detector"}, 

438 "ExposureF") 

439 butlerTests.addDatasetType(self.repo, "initial_stars_footprints_detector", 

440 {"instrument", "visit", "detector"}, 

441 "SourceCatalog") 

442 butlerTests.addDatasetType(self.repo, "initial_stars_detector", 

443 {"instrument", "visit", "detector"}, 

444 "ArrowAstropy") 

445 butlerTests.addDatasetType(self.repo, "initial_photoCalib_detector", 

446 {"instrument", "visit", "detector"}, 

447 "PhotoCalib") 

448 # optional outputs 

449 butlerTests.addDatasetType(self.repo, "initial_pvi_background", {"instrument", "visit", "detector"}, 

450 "Background") 

451 butlerTests.addDatasetType(self.repo, "initial_psf_stars_footprints_detector", 

452 {"instrument", "visit", "detector"}, 

453 "SourceCatalog") 

454 butlerTests.addDatasetType(self.repo, "initial_psf_stars_detector", 

455 {"instrument", "visit", "detector"}, 

456 "ArrowAstropy") 

457 butlerTests.addDatasetType(self.repo, 

458 "initial_astrometry_match_detector", 

459 {"instrument", "visit", "detector"}, 

460 "Catalog") 

461 butlerTests.addDatasetType(self.repo, 

462 "initial_photometry_match_detector", 

463 {"instrument", "visit", "detector"}, 

464 "Catalog") 

465 

466 # dataIds 

467 self.exposure0_id = self.repo.registry.expandDataId( 

468 {"instrument": instrument, "exposure": exposure0, "detector": detector}) 

469 self.exposure1_id = self.repo.registry.expandDataId( 

470 {"instrument": instrument, "exposure": exposure1, "detector": detector}) 

471 self.visit_id = self.repo.registry.expandDataId( 

472 {"instrument": instrument, "visit": visit, "detector": detector}) 

473 self.htm_id = self.repo.registry.expandDataId({"htm7": 42}) 

474 

475 # put empty data 

476 self.butler = butlerTests.makeTestCollection(self.repo) 

477 self.butler.put(afwImage.ExposureF(), "postISRCCD", self.exposure0_id) 

478 self.butler.put(afwImage.ExposureF(), "postISRCCD", self.exposure1_id) 

479 self.butler.put(afwTable.SimpleCatalog(), "gaia_dr3_20230707", self.htm_id) 

480 self.butler.put(afwTable.SimpleCatalog(), "ps1_pv3_3pi_20170110", self.htm_id) 

481 

482 def tearDown(self): 

483 self.repo_path.cleanup() 

484 

485 def test_runQuantum(self): 

486 task = CalibrateImageTask() 

487 lsst.pipe.base.testUtils.assertValidInitOutput(task) 

488 

489 quantum = lsst.pipe.base.testUtils.makeQuantum( 

490 task, self.butler, self.visit_id, 

491 {"exposures": [self.exposure0_id], 

492 "astrometry_ref_cat": [self.htm_id], 

493 "photometry_ref_cat": [self.htm_id], 

494 # outputs 

495 "exposure": self.visit_id, 

496 "stars": self.visit_id, 

497 "stars_footprints": self.visit_id, 

498 "background": self.visit_id, 

499 "psf_stars": self.visit_id, 

500 "psf_stars_footprints": self.visit_id, 

501 "applied_photo_calib": self.visit_id, 

502 "initial_pvi_background": self.visit_id, 

503 "astrometry_matches": self.visit_id, 

504 "photometry_matches": self.visit_id, 

505 }) 

506 mock_run = lsst.pipe.base.testUtils.runTestQuantum(task, self.butler, quantum) 

507 

508 # Ensure the reference loaders have been configured. 

509 self.assertEqual(task.astrometry.refObjLoader.name, "gaia_dr3_20230707") 

510 self.assertEqual(task.photometry.match.refObjLoader.name, "ps1_pv3_3pi_20170110") 

511 # Check that the proper kwargs are passed to run(). 

512 self.assertEqual(mock_run.call_args.kwargs.keys(), {"exposures", "result", "id_generator"}) 

513 

514 def test_runQuantum_2_snaps(self): 

515 task = CalibrateImageTask() 

516 lsst.pipe.base.testUtils.assertValidInitOutput(task) 

517 

518 quantum = lsst.pipe.base.testUtils.makeQuantum( 

519 task, self.butler, self.visit_id, 

520 {"exposures": [self.exposure0_id, self.exposure1_id], 

521 "astrometry_ref_cat": [self.htm_id], 

522 "photometry_ref_cat": [self.htm_id], 

523 # outputs 

524 "exposure": self.visit_id, 

525 "stars": self.visit_id, 

526 "stars_footprints": self.visit_id, 

527 "background": self.visit_id, 

528 "psf_stars": self.visit_id, 

529 "psf_stars_footprints": self.visit_id, 

530 "applied_photo_calib": self.visit_id, 

531 "initial_pvi_background": self.visit_id, 

532 "astrometry_matches": self.visit_id, 

533 "photometry_matches": self.visit_id, 

534 }) 

535 mock_run = lsst.pipe.base.testUtils.runTestQuantum(task, self.butler, quantum) 

536 

537 # Ensure the reference loaders have been configured. 

538 self.assertEqual(task.astrometry.refObjLoader.name, "gaia_dr3_20230707") 

539 self.assertEqual(task.photometry.match.refObjLoader.name, "ps1_pv3_3pi_20170110") 

540 # Check that the proper kwargs are passed to run(). 

541 self.assertEqual(mock_run.call_args.kwargs.keys(), {"exposures", "result", "id_generator"}) 

542 

543 def test_runQuantum_no_optional_outputs(self): 

544 config = CalibrateImageTask.ConfigClass() 

545 config.optional_outputs = None 

546 task = CalibrateImageTask(config=config) 

547 lsst.pipe.base.testUtils.assertValidInitOutput(task) 

548 

549 quantum = lsst.pipe.base.testUtils.makeQuantum( 

550 task, self.butler, self.visit_id, 

551 {"exposures": [self.exposure0_id], 

552 "astrometry_ref_cat": [self.htm_id], 

553 "photometry_ref_cat": [self.htm_id], 

554 # outputs 

555 "exposure": self.visit_id, 

556 "stars": self.visit_id, 

557 "stars_footprints": self.visit_id, 

558 "applied_photo_calib": self.visit_id, 

559 "background": self.visit_id, 

560 }) 

561 mock_run = lsst.pipe.base.testUtils.runTestQuantum(task, self.butler, quantum) 

562 

563 # Ensure the reference loaders have been configured. 

564 self.assertEqual(task.astrometry.refObjLoader.name, "gaia_dr3_20230707") 

565 self.assertEqual(task.photometry.match.refObjLoader.name, "ps1_pv3_3pi_20170110") 

566 # Check that the proper kwargs are passed to run(). 

567 self.assertEqual(mock_run.call_args.kwargs.keys(), {"exposures", "result", "id_generator"}) 

568 

569 def test_lintConnections(self): 

570 """Check that the connections are self-consistent. 

571 """ 

572 Connections = CalibrateImageTask.ConfigClass.ConnectionsClass 

573 lsst.pipe.base.testUtils.lintConnections(Connections) 

574 

575 def test_runQuantum_exception(self): 

576 """Test exception handling in runQuantum. 

577 """ 

578 task = CalibrateImageTask() 

579 lsst.pipe.base.testUtils.assertValidInitOutput(task) 

580 

581 quantum = lsst.pipe.base.testUtils.makeQuantum( 

582 task, self.butler, self.visit_id, 

583 {"exposures": [self.exposure0_id], 

584 "astrometry_ref_cat": [self.htm_id], 

585 "photometry_ref_cat": [self.htm_id], 

586 # outputs 

587 "exposure": self.visit_id, 

588 "stars": self.visit_id, 

589 "stars_footprints": self.visit_id, 

590 "background": self.visit_id, 

591 "psf_stars": self.visit_id, 

592 "psf_stars_footprints": self.visit_id, 

593 "applied_photo_calib": self.visit_id, 

594 "initial_pvi_background": self.visit_id, 

595 "astrometry_matches": self.visit_id, 

596 "photometry_matches": self.visit_id, 

597 }) 

598 

599 # A generic exception should raise directly. 

600 msg = "mocked run exception" 

601 with ( 

602 mock.patch.object(task, "run", side_effect=ValueError(msg)), 

603 self.assertRaisesRegex(ValueError, "mocked run exception") 

604 ): 

605 lsst.pipe.base.testUtils.runTestQuantum(task, self.butler, quantum, mockRun=False) 

606 

607 # A AlgorimthError should write annotated partial outputs. 

608 error = lsst.meas.algorithms.MeasureApCorrError(name="test", nSources=100, ndof=101) 

609 

610 def mock_run(exposures, result=None, id_generator=None): 

611 """Mock success through compute_psf, but failure after. 

612 """ 

613 result.exposure = afwImage.ExposureF(10, 10) 

614 result.psf_stars_footprints = afwTable.SourceCatalog() 

615 result.psf_stars = afwTable.SourceCatalog().asAstropy() 

616 result.background = afwMath.BackgroundList() 

617 raise error 

618 

619 with ( 

620 mock.patch.object(task, "run", side_effect=mock_run), 

621 self.assertRaises(lsst.pipe.base.AnnotatedPartialOutputsError), 

622 lsst.log.UsePythonLogging(), # so that assertLogs works with lsst.log 

623 ): 

624 with self.assertLogs("lsst.calibrateImage", level="ERROR") as cm: 

625 lsst.pipe.base.testUtils.runTestQuantum(task, 

626 self.butler, 

627 quantum, 

628 mockRun=False) 

629 

630 logged = "\n".join(cm.output) 

631 self.assertIn("Task failed with only partial outputs", logged) 

632 self.assertIn("MeasureApCorrError", logged) 

633 

634 # NOTE: This is an integration test of afw Exposure & SourceCatalog 

635 # metadata with the error annotation system in pipe_base. 

636 # Check that we did get the annotated partial outputs... 

637 pvi = self.butler.get("initial_pvi", self.visit_id) 

638 self.assertIn("Unable to measure aperture correction", pvi.metadata["failure.message"]) 

639 self.assertIn("MeasureApCorrError", pvi.metadata["failure.type"]) 

640 self.assertEqual(pvi.metadata["failure.metadata.ndof"], 101) 

641 stars = self.butler.get("initial_psf_stars_footprints_detector", self.visit_id) 

642 self.assertIn("Unable to measure aperture correction", stars.metadata["failure.message"]) 

643 self.assertIn("MeasureApCorrError", stars.metadata["failure.type"]) 

644 self.assertEqual(stars.metadata["failure.metadata.ndof"], 101) 

645 # ... but not the un-produced outputs. 

646 with self.assertRaises(FileNotFoundError): 

647 self.butler.get("initial_stars_footprints_detector", self.visit_id) 

648 

649 

650def setup_module(module): 

651 lsst.utils.tests.init() 

652 

653 

654class MemoryTestCase(lsst.utils.tests.MemoryTestCase): 

655 pass 

656 

657 

658if __name__ == "__main__": 658 ↛ 659line 658 didn't jump to line 659, because the condition on line 658 was never true

659 lsst.utils.tests.init() 

660 unittest.main()