Coverage for python/lsst/meas/base/tests.py: 13%

401 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-09 11:35 +0000

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24import numpy as np 

25 

26import lsst.geom 

27import lsst.afw.table 

28import lsst.afw.image 

29import lsst.afw.detection 

30import lsst.afw.geom 

31import lsst.pex.exceptions 

32 

33from .sfm import SingleFrameMeasurementTask 

34from .forcedMeasurement import ForcedMeasurementTask 

35from . import CentroidResultKey 

36 

37__all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase", 

38 "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper", 

39 "FluxTransformTestCase", "CentroidTransformTestCase") 

40 

41 

42class BlendContext: 

43 """Context manager which adds multiple overlapping sources and a parent. 

44 

45 Notes 

46 ----- 

47 This is used as the return value for `TestDataset.addBlend`, and this is 

48 the only way it should be used. 

49 """ 

50 

51 def __init__(self, owner): 

52 self.owner = owner 

53 self.parentRecord = self.owner.catalog.addNew() 

54 self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox()) 

55 self.children = [] 

56 

57 def __enter__(self): 

58 # BlendContext is its own context manager, so we just return self. 

59 return self 

60 

61 def addChild(self, instFlux, centroid, shape=None): 

62 """Add a child to the blend; return corresponding truth catalog record. 

63 

64 instFlux : `float` 

65 Total instFlux of the source to be added. 

66 centroid : `lsst.geom.Point2D` 

67 Position of the source to be added. 

68 shape : `lsst.afw.geom.Quadrupole` 

69 Second moments of the source before PSF convolution. Note that 

70 the truth catalog records post-convolution moments) 

71 """ 

72 record, image = self.owner.addSource(instFlux, centroid, shape) 

73 record.set(self.owner.keys["parent"], self.parentRecord.getId()) 

74 self.parentImage += image 

75 self.children.append((record, image)) 

76 return record 

77 

78 def __exit__(self, type_, value, tb): 

79 # We're not using the context manager for any kind of exception safety 

80 # or guarantees; we just want the nice "with" statement syntax. 

81 

82 if type_ is not None: 

83 # exception was raised; just skip all this and let it propagate 

84 return 

85 

86 # On exit, compute and set the truth values for the parent object. 

87 self.parentRecord.set(self.owner.keys["nChild"], len(self.children)) 

88 # Compute instFlux from sum of component fluxes 

89 instFlux = 0.0 

90 for record, image in self.children: 

91 instFlux += record.get(self.owner.keys["instFlux"]) 

92 self.parentRecord.set(self.owner.keys["instFlux"], instFlux) 

93 # Compute centroid from instFlux-weighted mean of component centroids 

94 x = 0.0 

95 y = 0.0 

96 for record, image in self.children: 

97 w = record.get(self.owner.keys["instFlux"])/instFlux 

98 x += record.get(self.owner.keys["centroid"].getX())*w 

99 y += record.get(self.owner.keys["centroid"].getY())*w 

100 self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y)) 

101 # Compute shape from instFlux-weighted mean of offset component shapes 

102 xx = 0.0 

103 yy = 0.0 

104 xy = 0.0 

105 for record, image in self.children: 

106 w = record.get(self.owner.keys["instFlux"])/instFlux 

107 dx = record.get(self.owner.keys["centroid"].getX()) - x 

108 dy = record.get(self.owner.keys["centroid"].getY()) - y 

109 xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w 

110 yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w 

111 xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w 

112 self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy)) 

113 # Run detection on the parent image to get the parent Footprint. 

114 self.owner._installFootprint(self.parentRecord, self.parentImage) 

115 # Create perfect HeavyFootprints for all children; these will need to 

116 # be modified later to account for the noise we'll add to the image. 

117 deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.getMaskedImage(), True) 

118 for record, image in self.children: 

119 deblend.getImage().getArray()[:, :] = image.getArray() 

120 heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend) 

121 record.setFootprint(heavyFootprint) 

122 

123 

124class TestDataset: 

125 """A simulated dataset consisuting of test image and truth catalog. 

126 

127 TestDataset creates an idealized image made of pure Gaussians (including a 

128 Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints 

129 that simulated the outputs of detection and deblending. Multiple noise 

130 realizations can be created from the same underlying sources, allowing 

131 uncertainty estimates to be verified via Monte Carlo. 

132 

133 Parameters 

134 ---------- 

135 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

136 Bounding box of the test image. 

137 threshold : `float` 

138 Threshold absolute value used to determine footprints for 

139 simulated sources. This thresholding will be applied before noise is 

140 actually added to images (or before the noise level is even known), so 

141 this will necessarily produce somewhat artificial footprints. 

142 exposure : `lsst.afw.image.ExposureF` 

143 The image to which test sources should be added. Ownership should 

144 be considered transferred from the caller to the TestDataset. 

145 Must have a Gaussian PSF for truth catalog shapes to be exact. 

146 **kwds 

147 Keyword arguments forwarded to makeEmptyExposure if exposure is `None`. 

148 

149 Notes 

150 ----- 

151 Typical usage: 

152 

153 .. code-block: py 

154 

155 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100, 

156 100)) 

157 dataset = TestDataset(bbox) 

158 dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26)) 

159 dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24), 

160 shape=lsst.afw.geom.Quadrupole(8, 7, 2)) 

161 with dataset.addBlend() as family: 

162 family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72)) 

163 family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74)) 

164 exposure, catalog = dataset.realize(noise=100.0, 

165 schema=TestDataset.makeMinimalSchema()) 

166 """ 

167 

168 @classmethod 

169 def makeMinimalSchema(cls): 

170 """Return the minimal schema needed to hold truth catalog fields. 

171 

172 Notes 

173 ----- 

174 When `TestDataset.realize` is called, the schema must include at least 

175 these fields. Usually it will include additional fields for 

176 measurement algorithm outputs, allowing the same catalog to be used 

177 for both truth values (the fields from the minimal schema) and the 

178 measurements. 

179 """ 

180 if not hasattr(cls, "_schema"): 

181 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

182 cls.keys = {} 

183 cls.keys["parent"] = schema.find("parent").key 

184 cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32) 

185 cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64, 

186 doc="true instFlux", units="count") 

187 cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields( 

188 schema, "truth", "true simulated centroid", "pixel" 

189 ) 

190 cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields( 

191 schema, "truth", ['x', 'y'], "pixel" 

192 ) 

193 cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag", 

194 doc="set if the object is a star") 

195 cls.keys["shape"] = lsst.afw.table.QuadrupoleKey.addFields( 

196 schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL 

197 ) 

198 cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag", 

199 doc="set if the object is a star") 

200 schema.getAliasMap().set("slot_Shape", "truth") 

201 schema.getAliasMap().set("slot_Centroid", "truth") 

202 schema.getAliasMap().set("slot_ModelFlux", "truth") 

203 cls._schema = schema 

204 schema = lsst.afw.table.Schema(cls._schema) 

205 schema.disconnectAliases() 

206 return schema 

207 

208 @staticmethod 

209 def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, 

210 minRotation=None, maxRotation=None, 

211 minRefShift=None, maxRefShift=None, 

212 minPixShift=2.0, maxPixShift=4.0, randomSeed=1): 

213 """Return a perturbed version of the input WCS. 

214 

215 Create a new undistorted TAN WCS that is similar but not identical to 

216 another, with random scaling, rotation, and offset (in both pixel 

217 position and reference position). 

218 

219 Parameters 

220 ---------- 

221 oldWcs : `lsst.afw.geom.SkyWcs` 

222 The input WCS. 

223 minScaleFactor : `float` 

224 Minimum scale factor to apply to the input WCS. 

225 maxScaleFactor : `float` 

226 Maximum scale factor to apply to the input WCS. 

227 minRotation : `lsst.geom.Angle` or `None` 

228 Minimum rotation to apply to the input WCS. If `None`, defaults to 

229 30 degrees. 

230 maxRotation : `lsst.geom.Angle` or `None` 

231 Minimum rotation to apply to the input WCS. If `None`, defaults to 

232 60 degrees. 

233 minRefShift : `lsst.geom.Angle` or `None` 

234 Miniumum shift to apply to the input WCS reference value. If 

235 `None`, defaults to 0.5 arcsec. 

236 maxRefShift : `lsst.geom.Angle` or `None` 

237 Miniumum shift to apply to the input WCS reference value. If 

238 `None`, defaults to 1.0 arcsec. 

239 minPixShift : `float` 

240 Minimum shift to apply to the input WCS reference pixel. 

241 maxPixShift : `float` 

242 Maximum shift to apply to the input WCS reference pixel. 

243 randomSeed : `int` 

244 Random seed. 

245 

246 Returns 

247 ------- 

248 newWcs : `lsst.afw.geom.SkyWcs` 

249 A perturbed version of the input WCS. 

250 

251 Notes 

252 ----- 

253 The maximum and minimum arguments are interpreted as absolute values 

254 for a split range that covers both positive and negative values (as 

255 this method is used in testing, it is typically most important to 

256 avoid perturbations near zero). Scale factors are treated somewhat 

257 differently: the actual scale factor is chosen between 

258 ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``) 

259 and (``1/minScaleFactor``). 

260 

261 The default range for rotation is 30-60 degrees, and the default range 

262 for reference shift is 0.5-1.0 arcseconds (these cannot be safely 

263 included directly as default values because Angle objects are 

264 mutable). 

265 

266 The random number generator is primed with the seed given. If 

267 `None`, a seed is automatically chosen. 

268 """ 

269 random_state = np.random.RandomState(randomSeed) 

270 if minRotation is None: 

271 minRotation = 30.0*lsst.geom.degrees 

272 if maxRotation is None: 

273 maxRotation = 60.0*lsst.geom.degrees 

274 if minRefShift is None: 

275 minRefShift = 0.5*lsst.geom.arcseconds 

276 if maxRefShift is None: 

277 maxRefShift = 1.0*lsst.geom.arcseconds 

278 

279 def splitRandom(min1, max1, min2=None, max2=None): 

280 if min2 is None: 

281 min2 = -max1 

282 if max2 is None: 

283 max2 = -min1 

284 if random_state.uniform() > 0.5: 

285 return float(random_state.uniform(min1, max1)) 

286 else: 

287 return float(random_state.uniform(min2, max2)) 

288 # Generate random perturbations 

289 scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor) 

290 rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians 

291 refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

292 refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

293 pixShiftX = splitRandom(minPixShift, maxPixShift) 

294 pixShiftY = splitRandom(minPixShift, maxPixShift) 

295 # Compute new CD matrix 

296 oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix()) 

297 rTransform = lsst.geom.LinearTransform.makeRotation(rotation) 

298 sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor) 

299 newTransform = oldTransform*rTransform*sTransform 

300 matrix = newTransform.getMatrix() 

301 # Compute new coordinate reference pixel (CRVAL) 

302 oldSkyOrigin = oldWcs.getSkyOrigin() 

303 newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa, 

304 oldSkyOrigin.getDec() + refShiftDec) 

305 # Compute new pixel reference pixel (CRPIX) 

306 oldPixOrigin = oldWcs.getPixelOrigin() 

307 newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX, 

308 oldPixOrigin.getY() + pixShiftY) 

309 return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix) 

310 

311 @staticmethod 

312 def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4): 

313 """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values. 

314 

315 Parameters 

316 ---------- 

317 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

318 Bounding box of the image in image coordinates. 

319 wcs : `lsst.afw.geom.SkyWcs`, optional 

320 New WCS for the exposure (created from CRVAL and CDELT if `None`). 

321 crval : `lsst.afw.geom.SpherePoint`, optional 

322 ICRS center of the TAN WCS attached to the image. If `None`, (45 

323 degrees, 45 degrees) is assumed. 

324 cdelt : `lsst.geom.Angle`, optional 

325 Pixel scale of the image. If `None`, 0.2 arcsec is assumed. 

326 psfSigma : `float`, optional 

327 Radius (sigma) of the Gaussian PSF attached to the image 

328 psfDim : `int`, optional 

329 Width and height of the image's Gaussian PSF attached to the image 

330 calibration : `float`, optional 

331 The spatially-constant calibration (in nJy/count) to set the 

332 PhotoCalib of the exposure. 

333 

334 Returns 

335 ------- 

336 exposure : `lsst.age.image.ExposureF` 

337 An empty image. 

338 """ 

339 if wcs is None: 

340 if crval is None: 

341 crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees) 

342 if cdelt is None: 

343 cdelt = 0.2*lsst.geom.arcseconds 

344 crpix = lsst.geom.Box2D(bbox).getCenter() 

345 wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, 

346 cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt)) 

347 exposure = lsst.afw.image.ExposureF(bbox) 

348 psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma) 

349 photoCalib = lsst.afw.image.PhotoCalib(calibration) 

350 exposure.setWcs(wcs) 

351 exposure.setPsf(psf) 

352 exposure.setPhotoCalib(photoCalib) 

353 return exposure 

354 

355 @staticmethod 

356 def drawGaussian(bbox, instFlux, ellipse): 

357 """Create an image of an elliptical Gaussian. 

358 

359 Parameters 

360 ---------- 

361 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

362 Bounding box of image to create. 

363 instFlux : `float` 

364 Total instrumental flux of the Gaussian (normalized analytically, 

365 not using pixel values). 

366 ellipse : `lsst.afw.geom.Ellipse` 

367 Defines the centroid and shape. 

368 

369 Returns 

370 ------- 

371 image : `lsst.afw.image.ImageF` 

372 An image of the Gaussian. 

373 """ 

374 x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()), 

375 np.arange(bbox.getBeginY(), bbox.getEndY())) 

376 t = ellipse.getGridTransform() 

377 xt = t[t.XX] * x + t[t.XY] * y + t[t.X] 

378 yt = t[t.YX] * x + t[t.YY] * y + t[t.Y] 

379 image = lsst.afw.image.ImageF(bbox) 

380 image.getArray()[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea()) 

381 return image 

382 

383 def __init__(self, bbox, threshold=10.0, exposure=None, **kwds): 

384 if exposure is None: 

385 exposure = self.makeEmptyExposure(bbox, **kwds) 

386 self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE) 

387 self.exposure = exposure 

388 self.psfShape = self.exposure.getPsf().computeShape(bbox.getCenter()) 

389 self.schema = self.makeMinimalSchema() 

390 self.catalog = lsst.afw.table.SourceCatalog(self.schema) 

391 

392 def _installFootprint(self, record, image, setPeakSignificance=True): 

393 """Create simulated Footprint and add it to a truth catalog record. 

394 """ 

395 schema = lsst.afw.detection.PeakTable.makeMinimalSchema() 

396 if setPeakSignificance: 

397 schema.addField("significance", type=float, 

398 doc="Ratio of peak value to configured standard deviation.") 

399 # Run detection on the single-source image 

400 fpSet = lsst.afw.detection.FootprintSet(image, self.threshold, peakSchema=schema) 

401 # the call below to the FootprintSet ctor is actually a grow operation 

402 fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True) 

403 if setPeakSignificance: 

404 # This isn't a traditional significance, since we're using the VALUE 

405 # threshold type, but it's the best we can do in that case. 

406 for footprint in fpSet.getFootprints(): 

407 footprint.updatePeakSignificance(self.threshold.getValue()) 

408 # Update the full exposure's mask plane to indicate the detection 

409 fpSet.setMask(self.exposure.getMaskedImage().getMask(), "DETECTED") 

410 # Attach the new footprint to the exposure 

411 if len(fpSet.getFootprints()) > 1: 

412 raise RuntimeError("Threshold value results in multiple Footprints for a single object") 

413 if len(fpSet.getFootprints()) == 0: 

414 raise RuntimeError("Threshold value results in zero Footprints for object") 

415 record.setFootprint(fpSet.getFootprints()[0]) 

416 

417 def addSource(self, instFlux, centroid, shape=None, setPeakSignificance=True): 

418 """Add a source to the simulation. 

419 

420 Parameters 

421 ---------- 

422 instFlux : `float` 

423 Total instFlux of the source to be added. 

424 centroid : `lsst.geom.Point2D` 

425 Position of the source to be added. 

426 shape : `lsst.afw.geom.Quadrupole` 

427 Second moments of the source before PSF convolution. Note that the 

428 truth catalog records post-convolution moments. If `None`, a point 

429 source will be added. 

430 setPeakSignificance : `bool` 

431 Set the ``significance`` field for peaks in the footprints? 

432 See ``lsst.meas.algorithms.SourceDetectionTask.setPeakSignificance`` 

433 for how this field is computed for real datasets. 

434 

435 Returns 

436 ------- 

437 record : `lsst.afw.table.SourceRecord` 

438 A truth catalog record. 

439 image : `lsst.afw.image.ImageF` 

440 Single-source image corresponding to the new source. 

441 """ 

442 # Create and set the truth catalog fields 

443 record = self.catalog.addNew() 

444 record.set(self.keys["instFlux"], instFlux) 

445 record.set(self.keys["centroid"], centroid) 

446 covariance = np.random.normal(0, 0.1, 4).reshape(2, 2) 

447 covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov 

448 record.set(self.keys["centroid_sigma"], covariance.astype(np.float32)) 

449 if shape is None: 

450 record.set(self.keys["isStar"], True) 

451 fullShape = self.psfShape 

452 else: 

453 record.set(self.keys["isStar"], False) 

454 fullShape = shape.convolve(self.psfShape) 

455 record.set(self.keys["shape"], fullShape) 

456 # Create an image containing just this source 

457 image = self.drawGaussian(self.exposure.getBBox(), instFlux, 

458 lsst.afw.geom.Ellipse(fullShape, centroid)) 

459 # Generate a footprint for this source 

460 self._installFootprint(record, image, setPeakSignificance) 

461 # Actually add the source to the full exposure 

462 self.exposure.getMaskedImage().getImage().getArray()[:, :] += image.getArray() 

463 return record, image 

464 

465 def addBlend(self): 

466 """Return a context manager which can add a blend of multiple sources. 

467 

468 Notes 

469 ----- 

470 Note that nothing stops you from creating overlapping sources just using the addSource() method, 

471 but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type 

472 produced by the detection and deblending pipelines. 

473 

474 Examples 

475 -------- 

476 .. code-block: py 

477 d = TestDataset(...) 

478 with d.addBlend() as b: 

479 b.addChild(flux1, centroid1) 

480 b.addChild(flux2, centroid2, shape2) 

481 """ 

482 return BlendContext(self) 

483 

484 def transform(self, wcs, **kwds): 

485 """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib. 

486 

487 Parameters 

488 ---------- 

489 wcs : `lsst.afw.geom.SkyWcs` 

490 WCS for the new dataset. 

491 **kwds 

492 Additional keyword arguments passed on to 

493 `TestDataset.makeEmptyExposure`. If not specified, these revert 

494 to the defaults for `~TestDataset.makeEmptyExposure`, not the 

495 values in the current dataset. 

496 

497 Returns 

498 ------- 

499 newDataset : `TestDataset` 

500 Transformed copy of this dataset. 

501 """ 

502 bboxD = lsst.geom.Box2D() 

503 xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs) 

504 for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners(): 

505 bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner))) 

506 bboxI = lsst.geom.Box2I(bboxD) 

507 result = TestDataset(bbox=bboxI, wcs=wcs, **kwds) 

508 oldPhotoCalib = self.exposure.getPhotoCalib() 

509 newPhotoCalib = result.exposure.getPhotoCalib() 

510 oldPsfShape = self.exposure.getPsf().computeShape(bboxD.getCenter()) 

511 for record in self.catalog: 

512 if record.get(self.keys["nChild"]): 

513 raise NotImplementedError("Transforming blended sources in TestDatasets is not supported") 

514 magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"])) 

515 newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude) 

516 oldCentroid = record.get(self.keys["centroid"]) 

517 newCentroid = xyt.applyForward(oldCentroid) 

518 if record.get(self.keys["isStar"]): 

519 newDeconvolvedShape = None 

520 else: 

521 affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid) 

522 oldFullShape = record.get(self.keys["shape"]) 

523 oldDeconvolvedShape = lsst.afw.geom.Quadrupole( 

524 oldFullShape.getIxx() - oldPsfShape.getIxx(), 

525 oldFullShape.getIyy() - oldPsfShape.getIyy(), 

526 oldFullShape.getIxy() - oldPsfShape.getIxy(), 

527 False 

528 ) 

529 newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear()) 

530 result.addSource(newFlux, newCentroid, newDeconvolvedShape) 

531 return result 

532 

533 def realize(self, noise, schema, randomSeed=1): 

534 r"""Simulate an exposure and detection catalog for this dataset. 

535 

536 The simulation includes noise, and the detection catalog includes 

537 `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s. 

538 

539 Parameters 

540 ---------- 

541 noise : `float` 

542 Standard deviation of noise to be added to the exposure. The 

543 noise will be Gaussian and constant, appropriate for the 

544 sky-limited regime. 

545 schema : `lsst.afw.table.Schema` 

546 Schema of the new catalog to be created. Must start with 

547 ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be 

548 `True`), but typically contains fields for already-configured 

549 measurement algorithms as well. 

550 randomSeed : `int`, optional 

551 Seed for the random number generator. 

552 If `None`, a seed is chosen automatically. 

553 

554 Returns 

555 ------- 

556 `exposure` : `lsst.afw.image.ExposureF` 

557 Simulated image. 

558 `catalog` : `lsst.afw.table.SourceCatalog` 

559 Simulated detection catalog. 

560 """ 

561 random_state = np.random.RandomState(randomSeed) 

562 assert schema.contains(self.schema) 

563 mapper = lsst.afw.table.SchemaMapper(self.schema) 

564 mapper.addMinimalSchema(self.schema, True) 

565 exposure = self.exposure.clone() 

566 exposure.getMaskedImage().getVariance().getArray()[:, :] = noise**2 

567 exposure.getMaskedImage().getImage().getArray()[:, :] \ 

568 += random_state.randn(exposure.getHeight(), exposure.getWidth())*noise 

569 catalog = lsst.afw.table.SourceCatalog(schema) 

570 catalog.extend(self.catalog, mapper=mapper) 

571 # Loop over sources and generate new HeavyFootprints that divide up 

572 # the noisy pixels, not the ideal no-noise pixels. 

573 for record in catalog: 

574 # parent objects have non-Heavy Footprints, which don't need to be 

575 # updated after adding noise. 

576 if record.getParent() == 0: 

577 continue 

578 # get flattened arrays that correspond to the no-noise and noisy 

579 # parent images 

580 parent = catalog.find(record.getParent()) 

581 footprint = parent.getFootprint() 

582 parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32) 

583 footprint.spans.flatten(parentFluxArrayNoNoise, 

584 self.exposure.getMaskedImage().getImage().getArray(), 

585 self.exposure.getXY0()) 

586 parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32) 

587 footprint.spans.flatten(parentFluxArrayNoisy, 

588 exposure.getMaskedImage().getImage().getArray(), 

589 exposure.getXY0()) 

590 oldHeavy = record.getFootprint() 

591 fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise) 

592 # N.B. this isn't a copy ctor - it's a copy from a vanilla 

593 # Footprint, so it doesn't copy the arrays we don't want to 

594 # change, and hence we have to do that ourselves below. 

595 newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy) 

596 newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction 

597 newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray() 

598 newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray() 

599 record.setFootprint(newHeavy) 

600 return exposure, catalog 

601 

602 

603class AlgorithmTestCase: 

604 """Base class for tests of measurement tasks. 

605 """ 

606 def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()): 

607 """Create an instance of `SingleFrameMeasurementTask.ConfigClass`. 

608 

609 Only the specified plugin and its dependencies will be run; the 

610 Centroid, Shape, and ModelFlux slots will be set to the truth fields 

611 generated by the `TestDataset` class. 

612 

613 Parameters 

614 ---------- 

615 plugin : `str` 

616 Name of measurement plugin to enable. 

617 dependencies : iterable of `str`, optional 

618 Names of dependencies of the measurement plugin. 

619 

620 Returns 

621 ------- 

622 config : `SingleFrameMeasurementTask.ConfigClass` 

623 The resulting task configuration. 

624 """ 

625 config = SingleFrameMeasurementTask.ConfigClass() 

626 with warnings.catch_warnings(): 

627 warnings.filterwarnings("ignore", message="ignoreSlotPluginChecks", category=FutureWarning) 

628 config = SingleFrameMeasurementTask.ConfigClass(ignoreSlotPluginChecks=True) 

629 config.slots.centroid = "truth" 

630 config.slots.shape = "truth" 

631 config.slots.modelFlux = None 

632 config.slots.apFlux = None 

633 config.slots.psfFlux = None 

634 config.slots.gaussianFlux = None 

635 config.slots.calibFlux = None 

636 config.plugins.names = (plugin,) + tuple(dependencies) 

637 return config 

638 

639 def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, 

640 algMetadata=None): 

641 """Create a configured instance of `SingleFrameMeasurementTask`. 

642 

643 Parameters 

644 ---------- 

645 plugin : `str`, optional 

646 Name of measurement plugin to enable. If `None`, a configuration 

647 must be supplied as the ``config`` parameter. If both are 

648 specified, ``config`` takes precedence. 

649 dependencies : iterable of `str`, optional 

650 Names of dependencies of the specified measurement plugin. 

651 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

652 Configuration for the task. If `None`, a measurement plugin must 

653 be supplied as the ``plugin`` paramter. If both are specified, 

654 ``config`` takes precedence. 

655 schema : `lsst.afw.table.Schema`, optional 

656 Measurement table schema. If `None`, a default schema is 

657 generated. 

658 algMetadata : `lsst.daf.base.PropertyList`, optional 

659 Measurement algorithm metadata. If `None`, a default container 

660 will be generated. 

661 

662 Returns 

663 ------- 

664 task : `SingleFrameMeasurementTask` 

665 A configured instance of the measurement task. 

666 """ 

667 if config is None: 

668 if plugin is None: 

669 raise ValueError("Either plugin or config argument must not be None") 

670 config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies) 

671 if schema is None: 

672 schema = TestDataset.makeMinimalSchema() 

673 # Clear all aliases so only those defined by config are set. 

674 schema.setAliasMap(None) 

675 if algMetadata is None: 

676 algMetadata = lsst.daf.base.PropertyList() 

677 return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config) 

678 

679 def makeForcedMeasurementConfig(self, plugin=None, dependencies=()): 

680 """Create an instance of `ForcedMeasurementTask.ConfigClass`. 

681 

682 In addition to the plugins specified in the plugin and dependencies 

683 arguments, the `TransformedCentroid` and `TransformedShape` plugins 

684 will be run and used as the centroid and shape slots; these simply 

685 transform the reference catalog centroid and shape to the measurement 

686 coordinate system. 

687 

688 Parameters 

689 ---------- 

690 plugin : `str` 

691 Name of measurement plugin to enable. 

692 dependencies : iterable of `str`, optional 

693 Names of dependencies of the measurement plugin. 

694 

695 Returns 

696 ------- 

697 config : `ForcedMeasurementTask.ConfigClass` 

698 The resulting task configuration. 

699 """ 

700 

701 config = ForcedMeasurementTask.ConfigClass() 

702 config.slots.centroid = "base_TransformedCentroid" 

703 config.slots.shape = "base_TransformedShape" 

704 config.slots.modelFlux = None 

705 config.slots.apFlux = None 

706 config.slots.psfFlux = None 

707 config.slots.gaussianFlux = None 

708 config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid", 

709 "base_TransformedShape") 

710 return config 

711 

712 def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, 

713 algMetadata=None): 

714 """Create a configured instance of `ForcedMeasurementTask`. 

715 

716 Parameters 

717 ---------- 

718 plugin : `str`, optional 

719 Name of measurement plugin to enable. If `None`, a configuration 

720 must be supplied as the ``config`` parameter. If both are 

721 specified, ``config`` takes precedence. 

722 dependencies : iterable of `str`, optional 

723 Names of dependencies of the specified measurement plugin. 

724 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

725 Configuration for the task. If `None`, a measurement plugin must 

726 be supplied as the ``plugin`` paramter. If both are specified, 

727 ``config`` takes precedence. 

728 refSchema : `lsst.afw.table.Schema`, optional 

729 Reference table schema. If `None`, a default schema is 

730 generated. 

731 algMetadata : `lsst.daf.base.PropertyList`, optional 

732 Measurement algorithm metadata. If `None`, a default container 

733 will be generated. 

734 

735 Returns 

736 ------- 

737 task : `ForcedMeasurementTask` 

738 A configured instance of the measurement task. 

739 """ 

740 if config is None: 

741 if plugin is None: 

742 raise ValueError("Either plugin or config argument must not be None") 

743 config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies) 

744 if refSchema is None: 

745 refSchema = TestDataset.makeMinimalSchema() 

746 if algMetadata is None: 

747 algMetadata = lsst.daf.base.PropertyList() 

748 return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config) 

749 

750 

751class TransformTestCase: 

752 """Base class for testing measurement transformations. 

753 

754 Notes 

755 ----- 

756 We test both that the transform itself operates successfully (fluxes are 

757 converted to magnitudes, flags are propagated properly) and that the 

758 transform is registered as the default for the appropriate measurement 

759 algorithms. 

760 

761 In the simple case of one-measurement-per-transformation, the developer 

762 need not directly write any tests themselves: simply customizing the class 

763 variables is all that is required. More complex measurements (e.g. 

764 multiple aperture fluxes) require extra effort. 

765 """ 

766 name = "MeasurementTransformTest" 

767 """The name used for the measurement algorithm (str). 

768 

769 Notes 

770 ----- 

771 This determines the names of the fields in the resulting catalog. This 

772 default should generally be fine, but subclasses can override if 

773 required. 

774 """ 

775 

776 # These should be customized by subclassing. 

777 controlClass = None 

778 algorithmClass = None 

779 transformClass = None 

780 

781 flagNames = ("flag",) 

782 """Flags which may be set by the algorithm being tested (iterable of `str`). 

783 """ 

784 

785 # The plugin being tested should be registered under these names for 

786 # single frame and forced measurement. Should be customized by 

787 # subclassing. 

788 singleFramePlugins = () 

789 forcedPlugins = () 

790 

791 def setUp(self): 

792 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200)) 

793 self.calexp = TestDataset.makeEmptyExposure(bbox) 

794 self._setupTransform() 

795 

796 def tearDown(self): 

797 del self.calexp 

798 del self.inputCat 

799 del self.mapper 

800 del self.transform 

801 del self.outputCat 

802 

803 def _populateCatalog(self, baseNames): 

804 records = [] 

805 for flagValue in (True, False): 

806 records.append(self.inputCat.addNew()) 

807 for baseName in baseNames: 

808 for flagName in self.flagNames: 

809 if records[-1].schema.join(baseName, flagName) in records[-1].schema: 

810 records[-1].set(records[-1].schema.join(baseName, flagName), flagValue) 

811 self._setFieldsInRecords(records, baseName) 

812 

813 def _checkOutput(self, baseNames): 

814 for inSrc, outSrc in zip(self.inputCat, self.outputCat): 

815 for baseName in baseNames: 

816 self._compareFieldsInRecords(inSrc, outSrc, baseName) 

817 for flagName in self.flagNames: 

818 keyName = outSrc.schema.join(baseName, flagName) 

819 if keyName in inSrc.schema: 

820 self.assertEqual(outSrc.get(keyName), inSrc.get(keyName)) 

821 else: 

822 self.assertFalse(keyName in outSrc.schema) 

823 

824 def _runTransform(self, doExtend=True): 

825 if doExtend: 

826 self.outputCat.extend(self.inputCat, mapper=self.mapper) 

827 self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib()) 

828 

829 def testTransform(self, baseNames=None): 

830 """Test the transformation on a catalog containing random data. 

831 

832 Parameters 

833 ---------- 

834 baseNames : iterable of `str` 

835 Iterable of the initial parts of measurement field names. 

836 

837 Notes 

838 ----- 

839 We check that: 

840 

841 - An appropriate exception is raised on an attempt to transform 

842 between catalogs with different numbers of rows; 

843 - Otherwise, all appropriate conversions are properly appled and that 

844 flags have been propagated. 

845 

846 The ``baseNames`` argument requires some explanation. This should be 

847 an iterable of the leading parts of the field names for each 

848 measurement; that is, everything that appears before ``_instFlux``, 

849 ``_flag``, etc. In the simple case of a single measurement per plugin, 

850 this is simply equal to ``self.name`` (thus measurements are stored as 

851 ``self.name + "_instFlux"``, etc). More generally, the developer may 

852 specify whatever iterable they require. For example, to handle 

853 multiple apertures, we could have ``(self.name + "_0", self.name + 

854 "_1", ...)``. 

855 """ 

856 baseNames = baseNames or [self.name] 

857 self._populateCatalog(baseNames) 

858 self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False) 

859 self._runTransform() 

860 self._checkOutput(baseNames) 

861 

862 def _checkRegisteredTransform(self, registry, name): 

863 # If this is a Python-based transform, we can compare directly; if 

864 # it's wrapped C++, we need to compare the wrapped class. 

865 self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass) 

866 

867 def testRegistration(self): 

868 """Test that the transformation is appropriately registered. 

869 """ 

870 for pluginName in self.singleFramePlugins: 

871 self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName) 

872 for pluginName in self.forcedPlugins: 

873 self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName) 

874 

875 

876class SingleFramePluginTransformSetupHelper: 

877 

878 def _setupTransform(self): 

879 self.control = self.controlClass() 

880 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema() 

881 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

882 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

883 # defining the schema. 

884 inputSchema.getAliasMap().set("slot_Centroid", "dummy") 

885 inputSchema.getAliasMap().set("slot_Shape", "dummy") 

886 self.algorithmClass(self.control, self.name, inputSchema) 

887 inputSchema.getAliasMap().erase("slot_Centroid") 

888 inputSchema.getAliasMap().erase("slot_Shape") 

889 self.inputCat = lsst.afw.table.SourceCatalog(inputSchema) 

890 self.mapper = lsst.afw.table.SchemaMapper(inputSchema) 

891 self.transform = self.transformClass(self.control, self.name, self.mapper) 

892 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

893 

894 

895class ForcedPluginTransformSetupHelper: 

896 

897 def _setupTransform(self): 

898 self.control = self.controlClass() 

899 inputMapper = lsst.afw.table.SchemaMapper(lsst.afw.table.SourceTable.makeMinimalSchema(), 

900 lsst.afw.table.SourceTable.makeMinimalSchema()) 

901 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

902 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

903 # defining the schema. 

904 inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy") 

905 inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy") 

906 self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList()) 

907 inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid") 

908 inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape") 

909 self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema()) 

910 self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema()) 

911 self.transform = self.transformClass(self.control, self.name, self.mapper) 

912 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

913 

914 

915class FluxTransformTestCase(TransformTestCase): 

916 

917 def _setFieldsInRecords(self, records, name): 

918 for record in records: 

919 record[record.schema.join(name, 'instFlux')] = np.random.random() 

920 record[record.schema.join(name, 'instFluxErr')] = np.random.random() 

921 

922 # Negative instFluxes should be converted to NaNs. 

923 assert len(records) > 1 

924 records[0][record.schema.join(name, 'instFlux')] = -1 

925 

926 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

927 instFluxName = inSrc.schema.join(name, 'instFlux') 

928 instFluxErrName = inSrc.schema.join(name, 'instFluxErr') 

929 if inSrc[instFluxName] > 0: 

930 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

931 inSrc[instFluxErrName]) 

932 self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value) 

933 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

934 else: 

935 # negative instFlux results in NaN magnitude, but can still have finite error 

936 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')])) 

937 if np.isnan(inSrc[instFluxErrName]): 

938 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')])) 

939 else: 

940 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

941 inSrc[instFluxErrName]) 

942 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

943 

944 

945class CentroidTransformTestCase(TransformTestCase): 

946 

947 def _setFieldsInRecords(self, records, name): 

948 for record in records: 

949 record[record.schema.join(name, 'x')] = np.random.random() 

950 record[record.schema.join(name, 'y')] = np.random.random() 

951 # Some algorithms set no errors; some set only sigma on x & y; some provide 

952 # a full covariance matrix. Set only those which exist in the schema. 

953 for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'): 

954 fieldName = record.schema.join(name, fieldSuffix) 

955 if fieldName in record.schema: 

956 record[fieldName] = np.random.random() 

957 

958 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

959 centroidResultKey = CentroidResultKey(inSrc.schema[self.name]) 

960 centroidResult = centroidResultKey.get(inSrc) 

961 

962 coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc) 

963 coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid()) 

964 self.assertEqual(coordTruth, coord) 

965 

966 # If the centroid has an associated uncertainty matrix, the coordinate 

967 # must have one too, and vice versa. 

968 try: 

969 coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name], 

970 ["ra", "dec"]).get(outSrc) 

971 except lsst.pex.exceptions.NotFoundError: 

972 self.assertFalse(centroidResultKey.getCentroidErr().isValid()) 

973 else: 

974 transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians) 

975 coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(), 

976 centroidResult.getCentroidErr()), 

977 transform.getLinear().getMatrix().transpose()) 

978 np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)