Coverage for python/lsst/meas/base/tests.py: 15%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

389 statements  

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23 

24import lsst.geom 

25import lsst.afw.table 

26import lsst.afw.image 

27import lsst.afw.detection 

28import lsst.afw.geom 

29import lsst.pex.exceptions 

30 

31from .sfm import SingleFrameMeasurementTask 

32from .forcedMeasurement import ForcedMeasurementTask 

33from . import CentroidResultKey 

34 

35__all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase", 

36 "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper", 

37 "FluxTransformTestCase", "CentroidTransformTestCase") 

38 

39 

40class BlendContext: 

41 """Context manager which adds multiple overlapping sources and a parent. 

42 

43 Notes 

44 ----- 

45 This is used as the return value for `TestDataset.addBlend`, and this is 

46 the only way it should be used. 

47 """ 

48 

49 def __init__(self, owner): 

50 self.owner = owner 

51 self.parentRecord = self.owner.catalog.addNew() 

52 self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox()) 

53 self.children = [] 

54 

55 def __enter__(self): 

56 # BlendContext is its own context manager, so we just return self. 

57 return self 

58 

59 def addChild(self, instFlux, centroid, shape=None): 

60 """Add a child to the blend; return corresponding truth catalog record. 

61 

62 instFlux : `float` 

63 Total instFlux of the source to be added. 

64 centroid : `lsst.geom.Point2D` 

65 Position of the source to be added. 

66 shape : `lsst.afw.geom.Quadrupole` 

67 Second moments of the source before PSF convolution. Note that 

68 the truth catalog records post-convolution moments) 

69 """ 

70 record, image = self.owner.addSource(instFlux, centroid, shape) 

71 record.set(self.owner.keys["parent"], self.parentRecord.getId()) 

72 self.parentImage += image 

73 self.children.append((record, image)) 

74 return record 

75 

76 def __exit__(self, type_, value, tb): 

77 # We're not using the context manager for any kind of exception safety 

78 # or guarantees; we just want the nice "with" statement syntax. 

79 

80 if type_ is not None: 

81 # exception was raised; just skip all this and let it propagate 

82 return 

83 

84 # On exit, compute and set the truth values for the parent object. 

85 self.parentRecord.set(self.owner.keys["nChild"], len(self.children)) 

86 # Compute instFlux from sum of component fluxes 

87 instFlux = 0.0 

88 for record, image in self.children: 

89 instFlux += record.get(self.owner.keys["instFlux"]) 

90 self.parentRecord.set(self.owner.keys["instFlux"], instFlux) 

91 # Compute centroid from instFlux-weighted mean of component centroids 

92 x = 0.0 

93 y = 0.0 

94 for record, image in self.children: 

95 w = record.get(self.owner.keys["instFlux"])/instFlux 

96 x += record.get(self.owner.keys["centroid"].getX())*w 

97 y += record.get(self.owner.keys["centroid"].getY())*w 

98 self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y)) 

99 # Compute shape from instFlux-weighted mean of offset component shapes 

100 xx = 0.0 

101 yy = 0.0 

102 xy = 0.0 

103 for record, image in self.children: 

104 w = record.get(self.owner.keys["instFlux"])/instFlux 

105 dx = record.get(self.owner.keys["centroid"].getX()) - x 

106 dy = record.get(self.owner.keys["centroid"].getY()) - y 

107 xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w 

108 yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w 

109 xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w 

110 self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy)) 

111 # Run detection on the parent image to get the parent Footprint. 

112 self.owner._installFootprint(self.parentRecord, self.parentImage) 

113 # Create perfect HeavyFootprints for all children; these will need to 

114 # be modified later to account for the noise we'll add to the image. 

115 deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.getMaskedImage(), True) 

116 for record, image in self.children: 

117 deblend.getImage().getArray()[:, :] = image.getArray() 

118 heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend) 

119 record.setFootprint(heavyFootprint) 

120 

121 

122class TestDataset: 

123 """A simulated dataset consisuting of test image and truth catalog. 

124 

125 TestDataset creates an idealized image made of pure Gaussians (including a 

126 Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints 

127 that simulated the outputs of detection and deblending. Multiple noise 

128 realizations can be created from the same underlying sources, allowing 

129 uncertainty estimates to be verified via Monte Carlo. 

130 

131 Parameters 

132 ---------- 

133 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

134 Bounding box of the test image. 

135 threshold : `float` 

136 Threshold absolute value used to determine footprints for 

137 simulated sources. This thresholding will be applied before noise is 

138 actually added to images (or before the noise level is even known), so 

139 this will necessarily produce somewhat artificial footprints. 

140 exposure : `lsst.afw.image.ExposureF` 

141 The image to which test sources should be added. Ownership should 

142 be considered transferred from the caller to the TestDataset. 

143 Must have a Gaussian PSF for truth catalog shapes to be exact. 

144 **kwds 

145 Keyword arguments forwarded to makeEmptyExposure if exposure is `None`. 

146 

147 Notes 

148 ----- 

149 Typical usage: 

150 

151 .. code-block: py 

152 

153 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100, 

154 100)) 

155 dataset = TestDataset(bbox) 

156 dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26)) 

157 dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24), 

158 shape=lsst.afw.geom.Quadrupole(8, 7, 2)) 

159 with dataset.addBlend() as family: 

160 family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72)) 

161 family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74)) 

162 exposure, catalog = dataset.realize(noise=100.0, 

163 schema=TestDataset.makeMinimalSchema()) 

164 """ 

165 

166 @classmethod 

167 def makeMinimalSchema(cls): 

168 """Return the minimal schema needed to hold truth catalog fields. 

169 

170 Notes 

171 ----- 

172 When `TestDataset.realize` is called, the schema must include at least 

173 these fields. Usually it will include additional fields for 

174 measurement algorithm outputs, allowing the same catalog to be used 

175 for both truth values (the fields from the minimal schema) and the 

176 measurements. 

177 """ 

178 if not hasattr(cls, "_schema"): 

179 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

180 cls.keys = {} 

181 cls.keys["parent"] = schema.find("parent").key 

182 cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32) 

183 cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64, 

184 doc="true instFlux", units="count") 

185 cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields( 

186 schema, "truth", "true simulated centroid", "pixel" 

187 ) 

188 cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields( 

189 schema, "truth", ['x', 'y'], "pixel" 

190 ) 

191 cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag", 

192 doc="set if the object is a star") 

193 cls.keys["shape"] = lsst.afw.table.QuadrupoleKey.addFields( 

194 schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL 

195 ) 

196 cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag", 

197 doc="set if the object is a star") 

198 schema.getAliasMap().set("slot_Shape", "truth") 

199 schema.getAliasMap().set("slot_Centroid", "truth") 

200 schema.getAliasMap().set("slot_ModelFlux", "truth") 

201 cls._schema = schema 

202 schema = lsst.afw.table.Schema(cls._schema) 

203 schema.disconnectAliases() 

204 return schema 

205 

206 @staticmethod 

207 def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, 

208 minRotation=None, maxRotation=None, 

209 minRefShift=None, maxRefShift=None, 

210 minPixShift=2.0, maxPixShift=4.0, randomSeed=1): 

211 """Return a perturbed version of the input WCS. 

212 

213 Create a new undistorted TAN WCS that is similar but not identical to 

214 another, with random scaling, rotation, and offset (in both pixel 

215 position and reference position). 

216 

217 Parameters 

218 ---------- 

219 oldWcs : `lsst.afw.geom.SkyWcs` 

220 The input WCS. 

221 minScaleFactor : `float` 

222 Minimum scale factor to apply to the input WCS. 

223 maxScaleFactor : `float` 

224 Maximum scale factor to apply to the input WCS. 

225 minRotation : `lsst.geom.Angle` or `None` 

226 Minimum rotation to apply to the input WCS. If `None`, defaults to 

227 30 degrees. 

228 maxRotation : `lsst.geom.Angle` or `None` 

229 Minimum rotation to apply to the input WCS. If `None`, defaults to 

230 60 degrees. 

231 minRefShift : `lsst.geom.Angle` or `None` 

232 Miniumum shift to apply to the input WCS reference value. If 

233 `None`, defaults to 0.5 arcsec. 

234 maxRefShift : `lsst.geom.Angle` or `None` 

235 Miniumum shift to apply to the input WCS reference value. If 

236 `None`, defaults to 1.0 arcsec. 

237 minPixShift : `float` 

238 Minimum shift to apply to the input WCS reference pixel. 

239 maxPixShift : `float` 

240 Maximum shift to apply to the input WCS reference pixel. 

241 randomSeed : `int` 

242 Random seed. 

243 

244 Returns 

245 ------- 

246 newWcs : `lsst.afw.geom.SkyWcs` 

247 A perturbed version of the input WCS. 

248 

249 Notes 

250 ----- 

251 The maximum and minimum arguments are interpreted as absolute values 

252 for a split range that covers both positive and negative values (as 

253 this method is used in testing, it is typically most important to 

254 avoid perturbations near zero). Scale factors are treated somewhat 

255 differently: the actual scale factor is chosen between 

256 ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``) 

257 and (``1/minScaleFactor``). 

258 

259 The default range for rotation is 30-60 degrees, and the default range 

260 for reference shift is 0.5-1.0 arcseconds (these cannot be safely 

261 included directly as default values because Angle objects are 

262 mutable). 

263 

264 The random number generator is primed with the seed given. If 

265 `None`, a seed is automatically chosen. 

266 """ 

267 random_state = np.random.RandomState(randomSeed) 

268 if minRotation is None: 

269 minRotation = 30.0*lsst.geom.degrees 

270 if maxRotation is None: 

271 maxRotation = 60.0*lsst.geom.degrees 

272 if minRefShift is None: 

273 minRefShift = 0.5*lsst.geom.arcseconds 

274 if maxRefShift is None: 

275 maxRefShift = 1.0*lsst.geom.arcseconds 

276 

277 def splitRandom(min1, max1, min2=None, max2=None): 

278 if min2 is None: 

279 min2 = -max1 

280 if max2 is None: 

281 max2 = -min1 

282 if random_state.uniform() > 0.5: 

283 return float(random_state.uniform(min1, max1)) 

284 else: 

285 return float(random_state.uniform(min2, max2)) 

286 # Generate random perturbations 

287 scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor) 

288 rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians 

289 refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

290 refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

291 pixShiftX = splitRandom(minPixShift, maxPixShift) 

292 pixShiftY = splitRandom(minPixShift, maxPixShift) 

293 # Compute new CD matrix 

294 oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix()) 

295 rTransform = lsst.geom.LinearTransform.makeRotation(rotation) 

296 sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor) 

297 newTransform = oldTransform*rTransform*sTransform 

298 matrix = newTransform.getMatrix() 

299 # Compute new coordinate reference pixel (CRVAL) 

300 oldSkyOrigin = oldWcs.getSkyOrigin() 

301 newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa, 

302 oldSkyOrigin.getDec() + refShiftDec) 

303 # Compute new pixel reference pixel (CRPIX) 

304 oldPixOrigin = oldWcs.getPixelOrigin() 

305 newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX, 

306 oldPixOrigin.getY() + pixShiftY) 

307 return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix) 

308 

309 @staticmethod 

310 def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4): 

311 """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values. 

312 

313 Parameters 

314 ---------- 

315 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

316 Bounding box of the image in image coordinates. 

317 wcs : `lsst.afw.geom.SkyWcs`, optional 

318 New WCS for the exposure (created from CRVAL and CDELT if `None`). 

319 crval : `lsst.afw.geom.SpherePoint`, optional 

320 ICRS center of the TAN WCS attached to the image. If `None`, (45 

321 degrees, 45 degrees) is assumed. 

322 cdelt : `lsst.geom.Angle`, optional 

323 Pixel scale of the image. If `None`, 0.2 arcsec is assumed. 

324 psfSigma : `float`, optional 

325 Radius (sigma) of the Gaussian PSF attached to the image 

326 psfDim : `int`, optional 

327 Width and height of the image's Gaussian PSF attached to the image 

328 calibration : `float`, optional 

329 The spatially-constant calibration (in nJy/count) to set the 

330 PhotoCalib of the exposure. 

331 

332 Returns 

333 ------- 

334 exposure : `lsst.age.image.ExposureF` 

335 An empty image. 

336 """ 

337 if wcs is None: 

338 if crval is None: 

339 crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees) 

340 if cdelt is None: 

341 cdelt = 0.2*lsst.geom.arcseconds 

342 crpix = lsst.geom.Box2D(bbox).getCenter() 

343 wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, 

344 cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt)) 

345 exposure = lsst.afw.image.ExposureF(bbox) 

346 psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma) 

347 photoCalib = lsst.afw.image.PhotoCalib(calibration) 

348 exposure.setWcs(wcs) 

349 exposure.setPsf(psf) 

350 exposure.setPhotoCalib(photoCalib) 

351 return exposure 

352 

353 @staticmethod 

354 def drawGaussian(bbox, instFlux, ellipse): 

355 """Create an image of an elliptical Gaussian. 

356 

357 Parameters 

358 ---------- 

359 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

360 Bounding box of image to create. 

361 instFlux : `float` 

362 Total instrumental flux of the Gaussian (normalized analytically, 

363 not using pixel values). 

364 ellipse : `lsst.afw.geom.Ellipse` 

365 Defines the centroid and shape. 

366 

367 Returns 

368 ------- 

369 image : `lsst.afw.image.ImageF` 

370 An image of the Gaussian. 

371 """ 

372 x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()), 

373 np.arange(bbox.getBeginY(), bbox.getEndY())) 

374 t = ellipse.getGridTransform() 

375 xt = t[t.XX] * x + t[t.XY] * y + t[t.X] 

376 yt = t[t.YX] * x + t[t.YY] * y + t[t.Y] 

377 image = lsst.afw.image.ImageF(bbox) 

378 image.getArray()[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea()) 

379 return image 

380 

381 def __init__(self, bbox, threshold=10.0, exposure=None, **kwds): 

382 if exposure is None: 

383 exposure = self.makeEmptyExposure(bbox, **kwds) 

384 self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE) 

385 self.exposure = exposure 

386 self.psfShape = self.exposure.getPsf().computeShape(bbox.getCenter()) 

387 self.schema = self.makeMinimalSchema() 

388 self.catalog = lsst.afw.table.SourceCatalog(self.schema) 

389 

390 def _installFootprint(self, record, image): 

391 """Create simulated Footprint and add it to a truth catalog record. 

392 """ 

393 # Run detection on the single-source image 

394 fpSet = lsst.afw.detection.FootprintSet(image, self.threshold) 

395 # the call below to the FootprintSet ctor is actually a grow operation 

396 fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True) 

397 # Update the full exposure's mask plane to indicate the detection 

398 fpSet.setMask(self.exposure.getMaskedImage().getMask(), "DETECTED") 

399 # Attach the new footprint to the exposure 

400 if len(fpSet.getFootprints()) > 1: 

401 raise RuntimeError("Threshold value results in multiple Footprints for a single object") 

402 if len(fpSet.getFootprints()) == 0: 

403 raise RuntimeError("Threshold value results in zero Footprints for object") 

404 record.setFootprint(fpSet.getFootprints()[0]) 

405 

406 def addSource(self, instFlux, centroid, shape=None): 

407 """Add a source to the simulation. 

408 

409 Parameters 

410 ---------- 

411 instFlux : `float` 

412 Total instFlux of the source to be added. 

413 centroid : `lsst.geom.Point2D` 

414 Position of the source to be added. 

415 shape : `lsst.afw.geom.Quadrupole` 

416 Second moments of the source before PSF convolution. Note that the 

417 truth catalog records post-convolution moments. If `None`, a point 

418 source will be added. 

419 

420 Returns 

421 ------- 

422 record : `lsst.afw.table.SourceRecord` 

423 A truth catalog record. 

424 image : `lsst.afw.image.ImageF` 

425 Single-source image corresponding to the new source. 

426 """ 

427 # Create and set the truth catalog fields 

428 record = self.catalog.addNew() 

429 record.set(self.keys["instFlux"], instFlux) 

430 record.set(self.keys["centroid"], centroid) 

431 covariance = np.random.normal(0, 0.1, 4).reshape(2, 2) 

432 covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov 

433 record.set(self.keys["centroid_sigma"], covariance.astype(np.float32)) 

434 if shape is None: 

435 record.set(self.keys["isStar"], True) 

436 fullShape = self.psfShape 

437 else: 

438 record.set(self.keys["isStar"], False) 

439 fullShape = shape.convolve(self.psfShape) 

440 record.set(self.keys["shape"], fullShape) 

441 # Create an image containing just this source 

442 image = self.drawGaussian(self.exposure.getBBox(), instFlux, 

443 lsst.afw.geom.Ellipse(fullShape, centroid)) 

444 # Generate a footprint for this source 

445 self._installFootprint(record, image) 

446 # Actually add the source to the full exposure 

447 self.exposure.getMaskedImage().getImage().getArray()[:, :] += image.getArray() 

448 return record, image 

449 

450 def addBlend(self): 

451 """Return a context manager which can add a blend of multiple sources. 

452 

453 Notes 

454 ----- 

455 Note that nothing stops you from creating overlapping sources just using the addSource() method, 

456 but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type 

457 produced by the detection and deblending pipelines. 

458 

459 Examples 

460 -------- 

461 .. code-block: py 

462 d = TestDataset(...) 

463 with d.addBlend() as b: 

464 b.addChild(flux1, centroid1) 

465 b.addChild(flux2, centroid2, shape2) 

466 """ 

467 return BlendContext(self) 

468 

469 def transform(self, wcs, **kwds): 

470 """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib. 

471 

472 Parameters 

473 ---------- 

474 wcs : `lsst.afw.geom.SkyWcs` 

475 WCS for the new dataset. 

476 **kwds 

477 Additional keyword arguments passed on to 

478 `TestDataset.makeEmptyExposure`. If not specified, these revert 

479 to the defaults for `~TestDataset.makeEmptyExposure`, not the 

480 values in the current dataset. 

481 

482 Returns 

483 ------- 

484 newDataset : `TestDataset` 

485 Transformed copy of this dataset. 

486 """ 

487 bboxD = lsst.geom.Box2D() 

488 xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs) 

489 for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners(): 

490 bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner))) 

491 bboxI = lsst.geom.Box2I(bboxD) 

492 result = TestDataset(bbox=bboxI, wcs=wcs, **kwds) 

493 oldPhotoCalib = self.exposure.getPhotoCalib() 

494 newPhotoCalib = result.exposure.getPhotoCalib() 

495 oldPsfShape = self.exposure.getPsf().computeShape(bboxD.getCenter()) 

496 for record in self.catalog: 

497 if record.get(self.keys["nChild"]): 

498 raise NotImplementedError("Transforming blended sources in TestDatasets is not supported") 

499 magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"])) 

500 newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude) 

501 oldCentroid = record.get(self.keys["centroid"]) 

502 newCentroid = xyt.applyForward(oldCentroid) 

503 if record.get(self.keys["isStar"]): 

504 newDeconvolvedShape = None 

505 else: 

506 affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid) 

507 oldFullShape = record.get(self.keys["shape"]) 

508 oldDeconvolvedShape = lsst.afw.geom.Quadrupole( 

509 oldFullShape.getIxx() - oldPsfShape.getIxx(), 

510 oldFullShape.getIyy() - oldPsfShape.getIyy(), 

511 oldFullShape.getIxy() - oldPsfShape.getIxy(), 

512 False 

513 ) 

514 newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear()) 

515 result.addSource(newFlux, newCentroid, newDeconvolvedShape) 

516 return result 

517 

518 def realize(self, noise, schema, randomSeed=1): 

519 r"""Simulate an exposure and detection catalog for this dataset. 

520 

521 The simulation includes noise, and the detection catalog includes 

522 `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s. 

523 

524 Parameters 

525 ---------- 

526 noise : `float` 

527 Standard deviation of noise to be added to the exposure. The 

528 noise will be Gaussian and constant, appropriate for the 

529 sky-limited regime. 

530 schema : `lsst.afw.table.Schema` 

531 Schema of the new catalog to be created. Must start with 

532 ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be 

533 `True`), but typically contains fields for already-configured 

534 measurement algorithms as well. 

535 randomSeed : `int`, optional 

536 Seed for the random number generator. 

537 If `None`, a seed is chosen automatically. 

538 

539 Returns 

540 ------- 

541 `exposure` : `lsst.afw.image.ExposureF` 

542 Simulated image. 

543 `catalog` : `lsst.afw.table.SourceCatalog` 

544 Simulated detection catalog. 

545 """ 

546 random_state = np.random.RandomState(randomSeed) 

547 assert schema.contains(self.schema) 

548 mapper = lsst.afw.table.SchemaMapper(self.schema) 

549 mapper.addMinimalSchema(self.schema, True) 

550 exposure = self.exposure.clone() 

551 exposure.getMaskedImage().getVariance().getArray()[:, :] = noise**2 

552 exposure.getMaskedImage().getImage().getArray()[:, :] \ 

553 += random_state.randn(exposure.getHeight(), exposure.getWidth())*noise 

554 catalog = lsst.afw.table.SourceCatalog(schema) 

555 catalog.extend(self.catalog, mapper=mapper) 

556 # Loop over sources and generate new HeavyFootprints that divide up 

557 # the noisy pixels, not the ideal no-noise pixels. 

558 for record in catalog: 

559 # parent objects have non-Heavy Footprints, which don't need to be 

560 # updated after adding noise. 

561 if record.getParent() == 0: 

562 continue 

563 # get flattened arrays that correspond to the no-noise and noisy 

564 # parent images 

565 parent = catalog.find(record.getParent()) 

566 footprint = parent.getFootprint() 

567 parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32) 

568 footprint.spans.flatten(parentFluxArrayNoNoise, 

569 self.exposure.getMaskedImage().getImage().getArray(), 

570 self.exposure.getXY0()) 

571 parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32) 

572 footprint.spans.flatten(parentFluxArrayNoisy, 

573 exposure.getMaskedImage().getImage().getArray(), 

574 exposure.getXY0()) 

575 oldHeavy = record.getFootprint() 

576 fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise) 

577 # N.B. this isn't a copy ctor - it's a copy from a vanilla 

578 # Footprint, so it doesn't copy the arrays we don't want to 

579 # change, and hence we have to do that ourselves below. 

580 newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy) 

581 newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction 

582 newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray() 

583 newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray() 

584 record.setFootprint(newHeavy) 

585 return exposure, catalog 

586 

587 

588class AlgorithmTestCase: 

589 

590 def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()): 

591 """Create an instance of `SingleFrameMeasurementTask.ConfigClass`. 

592 

593 Only the specified plugin and its dependencies will be run; the 

594 Centroid, Shape, and ModelFlux slots will be set to the truth fields 

595 generated by the `TestDataset` class. 

596 

597 Parameters 

598 ---------- 

599 plugin : `str` 

600 Name of measurement plugin to enable. 

601 dependencies : iterable of `str`, optional 

602 Names of dependencies of the measurement plugin. 

603 

604 Returns 

605 ------- 

606 config : `SingleFrameMeasurementTask.ConfigClass` 

607 The resulting task configuration. 

608 """ 

609 config = SingleFrameMeasurementTask.ConfigClass() 

610 config.slots.centroid = "truth" 

611 config.slots.shape = "truth" 

612 config.slots.modelFlux = None 

613 config.slots.apFlux = None 

614 config.slots.psfFlux = None 

615 config.slots.gaussianFlux = None 

616 config.slots.calibFlux = None 

617 config.plugins.names = (plugin,) + tuple(dependencies) 

618 return config 

619 

620 def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, 

621 algMetadata=None): 

622 """Create a configured instance of `SingleFrameMeasurementTask`. 

623 

624 Parameters 

625 ---------- 

626 plugin : `str`, optional 

627 Name of measurement plugin to enable. If `None`, a configuration 

628 must be supplied as the ``config`` parameter. If both are 

629 specified, ``config`` takes precedence. 

630 dependencies : iterable of `str`, optional 

631 Names of dependencies of the specified measurement plugin. 

632 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

633 Configuration for the task. If `None`, a measurement plugin must 

634 be supplied as the ``plugin`` paramter. If both are specified, 

635 ``config`` takes precedence. 

636 schema : `lsst.afw.table.Schema`, optional 

637 Measurement table schema. If `None`, a default schema is 

638 generated. 

639 algMetadata : `lsst.daf.base.PropertyList`, optional 

640 Measurement algorithm metadata. If `None`, a default container 

641 will be generated. 

642 

643 Returns 

644 ------- 

645 task : `SingleFrameMeasurementTask` 

646 A configured instance of the measurement task. 

647 """ 

648 if config is None: 

649 if plugin is None: 

650 raise ValueError("Either plugin or config argument must not be None") 

651 config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies) 

652 if schema is None: 

653 schema = TestDataset.makeMinimalSchema() 

654 # Clear all aliases so only those defined by config are set. 

655 schema.setAliasMap(None) 

656 if algMetadata is None: 

657 algMetadata = lsst.daf.base.PropertyList() 

658 return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config) 

659 

660 def makeForcedMeasurementConfig(self, plugin=None, dependencies=()): 

661 """Create an instance of `ForcedMeasurementTask.ConfigClass`. 

662 

663 In addition to the plugins specified in the plugin and dependencies 

664 arguments, the `TransformedCentroid` and `TransformedShape` plugins 

665 will be run and used as the centroid and shape slots; these simply 

666 transform the reference catalog centroid and shape to the measurement 

667 coordinate system. 

668 

669 Parameters 

670 ---------- 

671 plugin : `str` 

672 Name of measurement plugin to enable. 

673 dependencies : iterable of `str`, optional 

674 Names of dependencies of the measurement plugin. 

675 

676 Returns 

677 ------- 

678 config : `ForcedMeasurementTask.ConfigClass` 

679 The resulting task configuration. 

680 """ 

681 

682 config = ForcedMeasurementTask.ConfigClass() 

683 config.slots.centroid = "base_TransformedCentroid" 

684 config.slots.shape = "base_TransformedShape" 

685 config.slots.modelFlux = None 

686 config.slots.apFlux = None 

687 config.slots.psfFlux = None 

688 config.slots.gaussianFlux = None 

689 config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid", 

690 "base_TransformedShape") 

691 return config 

692 

693 def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, 

694 algMetadata=None): 

695 """Create a configured instance of `ForcedMeasurementTask`. 

696 

697 Parameters 

698 ---------- 

699 plugin : `str`, optional 

700 Name of measurement plugin to enable. If `None`, a configuration 

701 must be supplied as the ``config`` parameter. If both are 

702 specified, ``config`` takes precedence. 

703 dependencies : iterable of `str`, optional 

704 Names of dependencies of the specified measurement plugin. 

705 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

706 Configuration for the task. If `None`, a measurement plugin must 

707 be supplied as the ``plugin`` paramter. If both are specified, 

708 ``config`` takes precedence. 

709 refSchema : `lsst.afw.table.Schema`, optional 

710 Reference table schema. If `None`, a default schema is 

711 generated. 

712 algMetadata : `lsst.daf.base.PropertyList`, optional 

713 Measurement algorithm metadata. If `None`, a default container 

714 will be generated. 

715 

716 Returns 

717 ------- 

718 task : `ForcedMeasurementTask` 

719 A configured instance of the measurement task. 

720 """ 

721 if config is None: 

722 if plugin is None: 

723 raise ValueError("Either plugin or config argument must not be None") 

724 config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies) 

725 if refSchema is None: 

726 refSchema = TestDataset.makeMinimalSchema() 

727 if algMetadata is None: 

728 algMetadata = lsst.daf.base.PropertyList() 

729 return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config) 

730 

731 

732class TransformTestCase: 

733 """Base class for testing measurement transformations. 

734 

735 Notes 

736 ----- 

737 We test both that the transform itself operates successfully (fluxes are 

738 converted to magnitudes, flags are propagated properly) and that the 

739 transform is registered as the default for the appropriate measurement 

740 algorithms. 

741 

742 In the simple case of one-measurement-per-transformation, the developer 

743 need not directly write any tests themselves: simply customizing the class 

744 variables is all that is required. More complex measurements (e.g. 

745 multiple aperture fluxes) require extra effort. 

746 """ 

747 name = "MeasurementTransformTest" 

748 """The name used for the measurement algorithm (str). 

749 

750 Notes 

751 ----- 

752 This determines the names of the fields in the resulting catalog. This 

753 default should generally be fine, but subclasses can override if 

754 required. 

755 """ 

756 

757 # These should be customized by subclassing. 

758 controlClass = None 

759 algorithmClass = None 

760 transformClass = None 

761 

762 flagNames = ("flag",) 

763 """Flags which may be set by the algorithm being tested (iterable of `str`). 

764 """ 

765 

766 # The plugin being tested should be registered under these names for 

767 # single frame and forced measurement. Should be customized by 

768 # subclassing. 

769 singleFramePlugins = () 

770 forcedPlugins = () 

771 

772 def setUp(self): 

773 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200)) 

774 self.calexp = TestDataset.makeEmptyExposure(bbox) 

775 self._setupTransform() 

776 

777 def tearDown(self): 

778 del self.calexp 

779 del self.inputCat 

780 del self.mapper 

781 del self.transform 

782 del self.outputCat 

783 

784 def _populateCatalog(self, baseNames): 

785 records = [] 

786 for flagValue in (True, False): 

787 records.append(self.inputCat.addNew()) 

788 for baseName in baseNames: 

789 for flagName in self.flagNames: 

790 if records[-1].schema.join(baseName, flagName) in records[-1].schema: 

791 records[-1].set(records[-1].schema.join(baseName, flagName), flagValue) 

792 self._setFieldsInRecords(records, baseName) 

793 

794 def _checkOutput(self, baseNames): 

795 for inSrc, outSrc in zip(self.inputCat, self.outputCat): 

796 for baseName in baseNames: 

797 self._compareFieldsInRecords(inSrc, outSrc, baseName) 

798 for flagName in self.flagNames: 

799 keyName = outSrc.schema.join(baseName, flagName) 

800 if keyName in inSrc.schema: 

801 self.assertEqual(outSrc.get(keyName), inSrc.get(keyName)) 

802 else: 

803 self.assertFalse(keyName in outSrc.schema) 

804 

805 def _runTransform(self, doExtend=True): 

806 if doExtend: 

807 self.outputCat.extend(self.inputCat, mapper=self.mapper) 

808 self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib()) 

809 

810 def testTransform(self, baseNames=None): 

811 """Test the transformation on a catalog containing random data. 

812 

813 Parameters 

814 ---------- 

815 baseNames : iterable of `str` 

816 Iterable of the initial parts of measurement field names. 

817 

818 Notes 

819 ----- 

820 We check that: 

821 

822 - An appropriate exception is raised on an attempt to transform 

823 between catalogs with different numbers of rows; 

824 - Otherwise, all appropriate conversions are properly appled and that 

825 flags have been propagated. 

826 

827 The ``baseNames`` argument requires some explanation. This should be 

828 an iterable of the leading parts of the field names for each 

829 measurement; that is, everything that appears before ``_instFlux``, 

830 ``_flag``, etc. In the simple case of a single measurement per plugin, 

831 this is simply equal to ``self.name`` (thus measurements are stored as 

832 ``self.name + "_instFlux"``, etc). More generally, the developer may 

833 specify whatever iterable they require. For example, to handle 

834 multiple apertures, we could have ``(self.name + "_0", self.name + 

835 "_1", ...)``. 

836 """ 

837 baseNames = baseNames or [self.name] 

838 self._populateCatalog(baseNames) 

839 self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False) 

840 self._runTransform() 

841 self._checkOutput(baseNames) 

842 

843 def _checkRegisteredTransform(self, registry, name): 

844 # If this is a Python-based transform, we can compare directly; if 

845 # it's wrapped C++, we need to compare the wrapped class. 

846 self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass) 

847 

848 def testRegistration(self): 

849 """Test that the transformation is appropriately registered. 

850 """ 

851 for pluginName in self.singleFramePlugins: 

852 self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName) 

853 for pluginName in self.forcedPlugins: 

854 self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName) 

855 

856 

857class SingleFramePluginTransformSetupHelper: 

858 

859 def _setupTransform(self): 

860 self.control = self.controlClass() 

861 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema() 

862 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

863 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

864 # defining the schema. 

865 inputSchema.getAliasMap().set("slot_Centroid", "dummy") 

866 inputSchema.getAliasMap().set("slot_Shape", "dummy") 

867 self.algorithmClass(self.control, self.name, inputSchema) 

868 inputSchema.getAliasMap().erase("slot_Centroid") 

869 inputSchema.getAliasMap().erase("slot_Shape") 

870 self.inputCat = lsst.afw.table.SourceCatalog(inputSchema) 

871 self.mapper = lsst.afw.table.SchemaMapper(inputSchema) 

872 self.transform = self.transformClass(self.control, self.name, self.mapper) 

873 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

874 

875 

876class ForcedPluginTransformSetupHelper: 

877 

878 def _setupTransform(self): 

879 self.control = self.controlClass() 

880 inputMapper = lsst.afw.table.SchemaMapper(lsst.afw.table.SourceTable.makeMinimalSchema(), 

881 lsst.afw.table.SourceTable.makeMinimalSchema()) 

882 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

883 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

884 # defining the schema. 

885 inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy") 

886 inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy") 

887 self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList()) 

888 inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid") 

889 inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape") 

890 self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema()) 

891 self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema()) 

892 self.transform = self.transformClass(self.control, self.name, self.mapper) 

893 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

894 

895 

896class FluxTransformTestCase(TransformTestCase): 

897 

898 def _setFieldsInRecords(self, records, name): 

899 for record in records: 

900 record[record.schema.join(name, 'instFlux')] = np.random.random() 

901 record[record.schema.join(name, 'instFluxErr')] = np.random.random() 

902 

903 # Negative instFluxes should be converted to NaNs. 

904 assert len(records) > 1 

905 records[0][record.schema.join(name, 'instFlux')] = -1 

906 

907 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

908 instFluxName = inSrc.schema.join(name, 'instFlux') 

909 instFluxErrName = inSrc.schema.join(name, 'instFluxErr') 

910 if inSrc[instFluxName] > 0: 

911 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

912 inSrc[instFluxErrName]) 

913 self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value) 

914 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

915 else: 

916 # negative instFlux results in NaN magnitude, but can still have finite error 

917 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')])) 

918 if np.isnan(inSrc[instFluxErrName]): 

919 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')])) 

920 else: 

921 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

922 inSrc[instFluxErrName]) 

923 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

924 

925 

926class CentroidTransformTestCase(TransformTestCase): 

927 

928 def _setFieldsInRecords(self, records, name): 

929 for record in records: 

930 record[record.schema.join(name, 'x')] = np.random.random() 

931 record[record.schema.join(name, 'y')] = np.random.random() 

932 # Some algorithms set no errors; some set only sigma on x & y; some provide 

933 # a full covariance matrix. Set only those which exist in the schema. 

934 for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'): 

935 fieldName = record.schema.join(name, fieldSuffix) 

936 if fieldName in record.schema: 

937 record[fieldName] = np.random.random() 

938 

939 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

940 centroidResultKey = CentroidResultKey(inSrc.schema[self.name]) 

941 centroidResult = centroidResultKey.get(inSrc) 

942 

943 coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc) 

944 coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid()) 

945 self.assertEqual(coordTruth, coord) 

946 

947 # If the centroid has an associated uncertainty matrix, the coordinate 

948 # must have one too, and vice versa. 

949 try: 

950 coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name], 

951 ["ra", "dec"]).get(outSrc) 

952 except lsst.pex.exceptions.NotFoundError: 

953 self.assertFalse(centroidResultKey.getCentroidErr().isValid()) 

954 else: 

955 transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians) 

956 coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(), 

957 centroidResult.getCentroidErr()), 

958 transform.getLinear().getMatrix().transpose()) 

959 np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)