Coverage for python/lsst/meas/base/tests.py: 14%

407 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-13 10:06 +0000

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24import numpy as np 

25 

26import lsst.geom 

27import lsst.afw.table 

28import lsst.afw.image 

29import lsst.afw.detection 

30import lsst.afw.geom 

31import lsst.afw.coord 

32import lsst.daf.base 

33import lsst.pex.exceptions 

34 

35from .sfm import SingleFrameMeasurementTask 

36from .forcedMeasurement import ForcedMeasurementTask 

37from ._measBaseLib import CentroidResultKey 

38 

39__all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase", 

40 "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper", 

41 "FluxTransformTestCase", "CentroidTransformTestCase") 

42 

43 

44class BlendContext: 

45 """Context manager which adds multiple overlapping sources and a parent. 

46 

47 Notes 

48 ----- 

49 This is used as the return value for `TestDataset.addBlend`, and this is 

50 the only way it should be used. 

51 """ 

52 

53 def __init__(self, owner): 

54 self.owner = owner 

55 self.parentRecord = self.owner.catalog.addNew() 

56 self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox()) 

57 self.children = [] 

58 

59 def __enter__(self): 

60 # BlendContext is its own context manager, so we just return self. 

61 return self 

62 

63 def addChild(self, instFlux, centroid, shape=None): 

64 """Add a child to the blend; return corresponding truth catalog record. 

65 

66 instFlux : `float` 

67 Total instFlux of the source to be added. 

68 centroid : `lsst.geom.Point2D` 

69 Position of the source to be added. 

70 shape : `lsst.afw.geom.Quadrupole` 

71 Second moments of the source before PSF convolution. Note that 

72 the truth catalog records post-convolution moments) 

73 """ 

74 record, image = self.owner.addSource(instFlux, centroid, shape) 

75 record.set(self.owner.keys["parent"], self.parentRecord.getId()) 

76 self.parentImage += image 

77 self.children.append((record, image)) 

78 return record 

79 

80 def __exit__(self, type_, value, tb): 

81 # We're not using the context manager for any kind of exception safety 

82 # or guarantees; we just want the nice "with" statement syntax. 

83 

84 if type_ is not None: 

85 # exception was raised; just skip all this and let it propagate 

86 return 

87 

88 # On exit, compute and set the truth values for the parent object. 

89 self.parentRecord.set(self.owner.keys["nChild"], len(self.children)) 

90 # Compute instFlux from sum of component fluxes 

91 instFlux = 0.0 

92 for record, image in self.children: 

93 instFlux += record.get(self.owner.keys["instFlux"]) 

94 self.parentRecord.set(self.owner.keys["instFlux"], instFlux) 

95 # Compute centroid from instFlux-weighted mean of component centroids 

96 x = 0.0 

97 y = 0.0 

98 for record, image in self.children: 

99 w = record.get(self.owner.keys["instFlux"])/instFlux 

100 x += record.get(self.owner.keys["centroid"].getX())*w 

101 y += record.get(self.owner.keys["centroid"].getY())*w 

102 self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y)) 

103 # Compute shape from instFlux-weighted mean of offset component shapes 

104 xx = 0.0 

105 yy = 0.0 

106 xy = 0.0 

107 for record, image in self.children: 

108 w = record.get(self.owner.keys["instFlux"])/instFlux 

109 dx = record.get(self.owner.keys["centroid"].getX()) - x 

110 dy = record.get(self.owner.keys["centroid"].getY()) - y 

111 xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w 

112 yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w 

113 xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w 

114 self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy)) 

115 # Run detection on the parent image to get the parent Footprint. 

116 self.owner._installFootprint(self.parentRecord, self.parentImage) 

117 # Create perfect HeavyFootprints for all children; these will need to 

118 # be modified later to account for the noise we'll add to the image. 

119 deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.maskedImage, True) 

120 for record, image in self.children: 

121 deblend.image.array[:, :] = image.array 

122 heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend) 

123 record.setFootprint(heavyFootprint) 

124 

125 

126class TestDataset: 

127 """A simulated dataset consisuting of test image and truth catalog. 

128 

129 TestDataset creates an idealized image made of pure Gaussians (including a 

130 Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints 

131 that simulated the outputs of detection and deblending. Multiple noise 

132 realizations can be created from the same underlying sources, allowing 

133 uncertainty estimates to be verified via Monte Carlo. 

134 

135 Parameters 

136 ---------- 

137 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

138 Bounding box of the test image. 

139 threshold : `float` 

140 Threshold absolute value used to determine footprints for 

141 simulated sources. This thresholding will be applied before noise is 

142 actually added to images (or before the noise level is even known), so 

143 this will necessarily produce somewhat artificial footprints. 

144 exposure : `lsst.afw.image.ExposureF` 

145 The image to which test sources should be added. Ownership should 

146 be considered transferred from the caller to the TestDataset. 

147 Must have a Gaussian PSF for truth catalog shapes to be exact. 

148 **kwds 

149 Keyword arguments forwarded to makeEmptyExposure if exposure is `None`. 

150 

151 Notes 

152 ----- 

153 Typical usage: 

154 

155 .. code-block: py 

156 

157 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100, 

158 100)) 

159 dataset = TestDataset(bbox) 

160 dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26)) 

161 dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24), 

162 shape=lsst.afw.geom.Quadrupole(8, 7, 2)) 

163 with dataset.addBlend() as family: 

164 family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72)) 

165 family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74)) 

166 exposure, catalog = dataset.realize(noise=100.0, 

167 schema=TestDataset.makeMinimalSchema()) 

168 """ 

169 

170 def __init__(self, bbox, threshold=10.0, exposure=None, **kwds): 

171 if exposure is None: 

172 exposure = self.makeEmptyExposure(bbox, **kwds) 

173 self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE) 

174 self.exposure = exposure 

175 self.psfShape = self.exposure.getPsf().computeShape(bbox.getCenter()) 

176 self.schema = self.makeMinimalSchema() 

177 self.catalog = lsst.afw.table.SourceCatalog(self.schema) 

178 

179 @classmethod 

180 def makeMinimalSchema(cls): 

181 """Return the minimal schema needed to hold truth catalog fields. 

182 

183 Notes 

184 ----- 

185 When `TestDataset.realize` is called, the schema must include at least 

186 these fields. Usually it will include additional fields for 

187 measurement algorithm outputs, allowing the same catalog to be used 

188 for both truth values (the fields from the minimal schema) and the 

189 measurements. 

190 """ 

191 if not hasattr(cls, "_schema"): 

192 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

193 cls.keys = {} 

194 cls.keys["parent"] = schema.find("parent").key 

195 cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32) 

196 cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64, 

197 doc="true instFlux", units="count") 

198 cls.keys["instFluxErr"] = schema.addField("truth_instFluxErr", type=np.float64, 

199 doc="true instFluxErr", units="count") 

200 cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields( 

201 schema, "truth", "true simulated centroid", "pixel" 

202 ) 

203 cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields( 

204 schema, "truth", ['x', 'y'], "pixel" 

205 ) 

206 cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag", 

207 doc="set if the object is a star") 

208 cls.keys["shape"] = lsst.afw.table.QuadrupoleKey.addFields( 

209 schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL 

210 ) 

211 cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag", 

212 doc="set if the object is a star") 

213 schema.getAliasMap().set("slot_Shape", "truth") 

214 schema.getAliasMap().set("slot_Centroid", "truth") 

215 schema.getAliasMap().set("slot_ModelFlux", "truth") 

216 cls._schema = schema 

217 schema = lsst.afw.table.Schema(cls._schema) 

218 schema.disconnectAliases() 

219 return schema 

220 

221 @staticmethod 

222 def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, 

223 minRotation=None, maxRotation=None, 

224 minRefShift=None, maxRefShift=None, 

225 minPixShift=2.0, maxPixShift=4.0, randomSeed=1): 

226 """Return a perturbed version of the input WCS. 

227 

228 Create a new undistorted TAN WCS that is similar but not identical to 

229 another, with random scaling, rotation, and offset (in both pixel 

230 position and reference position). 

231 

232 Parameters 

233 ---------- 

234 oldWcs : `lsst.afw.geom.SkyWcs` 

235 The input WCS. 

236 minScaleFactor : `float` 

237 Minimum scale factor to apply to the input WCS. 

238 maxScaleFactor : `float` 

239 Maximum scale factor to apply to the input WCS. 

240 minRotation : `lsst.geom.Angle` or `None` 

241 Minimum rotation to apply to the input WCS. If `None`, defaults to 

242 30 degrees. 

243 maxRotation : `lsst.geom.Angle` or `None` 

244 Minimum rotation to apply to the input WCS. If `None`, defaults to 

245 60 degrees. 

246 minRefShift : `lsst.geom.Angle` or `None` 

247 Miniumum shift to apply to the input WCS reference value. If 

248 `None`, defaults to 0.5 arcsec. 

249 maxRefShift : `lsst.geom.Angle` or `None` 

250 Miniumum shift to apply to the input WCS reference value. If 

251 `None`, defaults to 1.0 arcsec. 

252 minPixShift : `float` 

253 Minimum shift to apply to the input WCS reference pixel. 

254 maxPixShift : `float` 

255 Maximum shift to apply to the input WCS reference pixel. 

256 randomSeed : `int` 

257 Random seed. 

258 

259 Returns 

260 ------- 

261 newWcs : `lsst.afw.geom.SkyWcs` 

262 A perturbed version of the input WCS. 

263 

264 Notes 

265 ----- 

266 The maximum and minimum arguments are interpreted as absolute values 

267 for a split range that covers both positive and negative values (as 

268 this method is used in testing, it is typically most important to 

269 avoid perturbations near zero). Scale factors are treated somewhat 

270 differently: the actual scale factor is chosen between 

271 ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``) 

272 and (``1/minScaleFactor``). 

273 

274 The default range for rotation is 30-60 degrees, and the default range 

275 for reference shift is 0.5-1.0 arcseconds (these cannot be safely 

276 included directly as default values because Angle objects are 

277 mutable). 

278 

279 The random number generator is primed with the seed given. If 

280 `None`, a seed is automatically chosen. 

281 """ 

282 random_state = np.random.RandomState(randomSeed) 

283 if minRotation is None: 

284 minRotation = 30.0*lsst.geom.degrees 

285 if maxRotation is None: 

286 maxRotation = 60.0*lsst.geom.degrees 

287 if minRefShift is None: 

288 minRefShift = 0.5*lsst.geom.arcseconds 

289 if maxRefShift is None: 

290 maxRefShift = 1.0*lsst.geom.arcseconds 

291 

292 def splitRandom(min1, max1, min2=None, max2=None): 

293 if min2 is None: 

294 min2 = -max1 

295 if max2 is None: 

296 max2 = -min1 

297 if random_state.uniform() > 0.5: 

298 return float(random_state.uniform(min1, max1)) 

299 else: 

300 return float(random_state.uniform(min2, max2)) 

301 # Generate random perturbations 

302 scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor) 

303 rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians 

304 refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

305 refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

306 pixShiftX = splitRandom(minPixShift, maxPixShift) 

307 pixShiftY = splitRandom(minPixShift, maxPixShift) 

308 # Compute new CD matrix 

309 oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix()) 

310 rTransform = lsst.geom.LinearTransform.makeRotation(rotation) 

311 sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor) 

312 newTransform = oldTransform*rTransform*sTransform 

313 matrix = newTransform.getMatrix() 

314 # Compute new coordinate reference pixel (CRVAL) 

315 oldSkyOrigin = oldWcs.getSkyOrigin() 

316 newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa, 

317 oldSkyOrigin.getDec() + refShiftDec) 

318 # Compute new pixel reference pixel (CRPIX) 

319 oldPixOrigin = oldWcs.getPixelOrigin() 

320 newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX, 

321 oldPixOrigin.getY() + pixShiftY) 

322 return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix) 

323 

324 @staticmethod 

325 def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4): 

326 """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values. 

327 

328 Parameters 

329 ---------- 

330 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

331 Bounding box of the image in image coordinates. 

332 wcs : `lsst.afw.geom.SkyWcs`, optional 

333 New WCS for the exposure (created from CRVAL and CDELT if `None`). 

334 crval : `lsst.afw.geom.SpherePoint`, optional 

335 ICRS center of the TAN WCS attached to the image. If `None`, (45 

336 degrees, 45 degrees) is assumed. 

337 cdelt : `lsst.geom.Angle`, optional 

338 Pixel scale of the image. If `None`, 0.2 arcsec is assumed. 

339 psfSigma : `float`, optional 

340 Radius (sigma) of the Gaussian PSF attached to the image 

341 psfDim : `int`, optional 

342 Width and height of the image's Gaussian PSF attached to the image 

343 calibration : `float`, optional 

344 The spatially-constant calibration (in nJy/count) to set the 

345 PhotoCalib of the exposure. 

346 

347 Returns 

348 ------- 

349 exposure : `lsst.age.image.ExposureF` 

350 An empty image. 

351 """ 

352 if wcs is None: 

353 if crval is None: 

354 crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees) 

355 if cdelt is None: 

356 cdelt = 0.2*lsst.geom.arcseconds 

357 crpix = lsst.geom.Box2D(bbox).getCenter() 

358 wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, 

359 cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt)) 

360 exposure = lsst.afw.image.ExposureF(bbox) 

361 psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma) 

362 photoCalib = lsst.afw.image.PhotoCalib(calibration) 

363 visitInfo = lsst.afw.image.VisitInfo(id=1234, 

364 exposureTime=30.0, 

365 date=lsst.daf.base.DateTime(60000.0), 

366 observatory=lsst.afw.coord.Observatory(11.1*lsst.geom.degrees, 

367 22.2*lsst.geom.degrees, 

368 0.333)) 

369 exposure.setWcs(wcs) 

370 exposure.setPsf(psf) 

371 exposure.setPhotoCalib(photoCalib) 

372 exposure.info.setVisitInfo(visitInfo) 

373 return exposure 

374 

375 @staticmethod 

376 def drawGaussian(bbox, instFlux, ellipse): 

377 """Create an image of an elliptical Gaussian. 

378 

379 Parameters 

380 ---------- 

381 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

382 Bounding box of image to create. 

383 instFlux : `float` 

384 Total instrumental flux of the Gaussian (normalized analytically, 

385 not using pixel values). 

386 ellipse : `lsst.afw.geom.Ellipse` 

387 Defines the centroid and shape. 

388 

389 Returns 

390 ------- 

391 image : `lsst.afw.image.ImageF` 

392 An image of the Gaussian. 

393 """ 

394 x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()), 

395 np.arange(bbox.getBeginY(), bbox.getEndY())) 

396 t = ellipse.getGridTransform() 

397 xt = t[t.XX] * x + t[t.XY] * y + t[t.X] 

398 yt = t[t.YX] * x + t[t.YY] * y + t[t.Y] 

399 image = lsst.afw.image.ImageF(bbox) 

400 image.array[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea()) 

401 return image 

402 

403 def _installFootprint(self, record, image, setPeakSignificance=True): 

404 """Create simulated Footprint and add it to a truth catalog record. 

405 """ 

406 schema = lsst.afw.detection.PeakTable.makeMinimalSchema() 

407 if setPeakSignificance: 

408 schema.addField("significance", type=float, 

409 doc="Ratio of peak value to configured standard deviation.") 

410 # Run detection on the single-source image 

411 fpSet = lsst.afw.detection.FootprintSet(image, self.threshold, peakSchema=schema) 

412 # the call below to the FootprintSet ctor is actually a grow operation 

413 fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True) 

414 if setPeakSignificance: 

415 # This isn't a traditional significance, since we're using the VALUE 

416 # threshold type, but it's the best we can do in that case. 

417 for footprint in fpSet.getFootprints(): 

418 footprint.updatePeakSignificance(self.threshold.getValue()) 

419 # Update the full exposure's mask plane to indicate the detection 

420 fpSet.setMask(self.exposure.mask, "DETECTED") 

421 # Attach the new footprint to the exposure 

422 if len(fpSet.getFootprints()) > 1: 

423 raise RuntimeError("Threshold value results in multiple Footprints for a single object") 

424 if len(fpSet.getFootprints()) == 0: 

425 raise RuntimeError("Threshold value results in zero Footprints for object") 

426 record.setFootprint(fpSet.getFootprints()[0]) 

427 

428 def addSource(self, instFlux, centroid, shape=None, setPeakSignificance=True): 

429 """Add a source to the simulation. 

430 

431 To insert a point source with a given signal-to-noise (sn), the total 

432 ``instFlux`` should be: ``sn*noise*psf_scale``, where ``noise`` is the 

433 noise you will pass to ``realize()``, and 

434 ``psf_scale=sqrt(4*pi*r^2)``, where ``r`` is the width of the PSF. 

435 

436 Parameters 

437 ---------- 

438 instFlux : `float` 

439 Total instFlux of the source to be added. 

440 centroid : `lsst.geom.Point2D` 

441 Position of the source to be added. 

442 shape : `lsst.afw.geom.Quadrupole` 

443 Second moments of the source before PSF convolution. Note that the 

444 truth catalog records post-convolution moments. If `None`, a point 

445 source will be added. 

446 setPeakSignificance : `bool` 

447 Set the ``significance`` field for peaks in the footprints? 

448 See ``lsst.meas.algorithms.SourceDetectionTask.setPeakSignificance`` 

449 for how this field is computed for real datasets. 

450 

451 Returns 

452 ------- 

453 record : `lsst.afw.table.SourceRecord` 

454 A truth catalog record. 

455 image : `lsst.afw.image.ImageF` 

456 Single-source image corresponding to the new source. 

457 """ 

458 # Create and set the truth catalog fields 

459 record = self.catalog.addNew() 

460 record.set(self.keys["instFlux"], instFlux) 

461 record.set(self.keys["instFluxErr"], 0) 

462 record.set(self.keys["centroid"], centroid) 

463 covariance = np.random.normal(0, 0.1, 4).reshape(2, 2) 

464 covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov 

465 record.set(self.keys["centroid_sigma"], covariance.astype(np.float32)) 

466 if shape is None: 

467 record.set(self.keys["isStar"], True) 

468 fullShape = self.psfShape 

469 else: 

470 record.set(self.keys["isStar"], False) 

471 fullShape = shape.convolve(self.psfShape) 

472 record.set(self.keys["shape"], fullShape) 

473 # Create an image containing just this source 

474 image = self.drawGaussian(self.exposure.getBBox(), instFlux, 

475 lsst.afw.geom.Ellipse(fullShape, centroid)) 

476 # Generate a footprint for this source 

477 self._installFootprint(record, image, setPeakSignificance) 

478 # Actually add the source to the full exposure 

479 self.exposure.image.array[:, :] += image.array 

480 return record, image 

481 

482 def addBlend(self): 

483 """Return a context manager which can add a blend of multiple sources. 

484 

485 Notes 

486 ----- 

487 Note that nothing stops you from creating overlapping sources just using the addSource() method, 

488 but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type 

489 produced by the detection and deblending pipelines. 

490 

491 Examples 

492 -------- 

493 .. code-block: py 

494 d = TestDataset(...) 

495 with d.addBlend() as b: 

496 b.addChild(flux1, centroid1) 

497 b.addChild(flux2, centroid2, shape2) 

498 """ 

499 return BlendContext(self) 

500 

501 def transform(self, wcs, **kwds): 

502 """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib. 

503 

504 Parameters 

505 ---------- 

506 wcs : `lsst.afw.geom.SkyWcs` 

507 WCS for the new dataset. 

508 **kwds 

509 Additional keyword arguments passed on to 

510 `TestDataset.makeEmptyExposure`. If not specified, these revert 

511 to the defaults for `~TestDataset.makeEmptyExposure`, not the 

512 values in the current dataset. 

513 

514 Returns 

515 ------- 

516 newDataset : `TestDataset` 

517 Transformed copy of this dataset. 

518 """ 

519 bboxD = lsst.geom.Box2D() 

520 xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs) 

521 for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners(): 

522 bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner))) 

523 bboxI = lsst.geom.Box2I(bboxD) 

524 result = TestDataset(bbox=bboxI, wcs=wcs, **kwds) 

525 oldPhotoCalib = self.exposure.getPhotoCalib() 

526 newPhotoCalib = result.exposure.getPhotoCalib() 

527 oldPsfShape = self.exposure.getPsf().computeShape(bboxD.getCenter()) 

528 for record in self.catalog: 

529 if record.get(self.keys["nChild"]): 

530 raise NotImplementedError("Transforming blended sources in TestDatasets is not supported") 

531 magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"])) 

532 newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude) 

533 oldCentroid = record.get(self.keys["centroid"]) 

534 newCentroid = xyt.applyForward(oldCentroid) 

535 if record.get(self.keys["isStar"]): 

536 newDeconvolvedShape = None 

537 else: 

538 affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid) 

539 oldFullShape = record.get(self.keys["shape"]) 

540 oldDeconvolvedShape = lsst.afw.geom.Quadrupole( 

541 oldFullShape.getIxx() - oldPsfShape.getIxx(), 

542 oldFullShape.getIyy() - oldPsfShape.getIyy(), 

543 oldFullShape.getIxy() - oldPsfShape.getIxy(), 

544 False 

545 ) 

546 newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear()) 

547 result.addSource(newFlux, newCentroid, newDeconvolvedShape) 

548 return result 

549 

550 def realize(self, noise, schema, randomSeed=1): 

551 r"""Simulate an exposure and detection catalog for this dataset. 

552 

553 The simulation includes noise, and the detection catalog includes 

554 `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s. 

555 

556 Parameters 

557 ---------- 

558 noise : `float` 

559 Standard deviation of noise to be added to the exposure. The 

560 noise will be Gaussian and constant, appropriate for the 

561 sky-limited regime. 

562 schema : `lsst.afw.table.Schema` 

563 Schema of the new catalog to be created. Must start with 

564 ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be 

565 `True`), but typically contains fields for already-configured 

566 measurement algorithms as well. 

567 randomSeed : `int`, optional 

568 Seed for the random number generator. 

569 If `None`, a seed is chosen automatically. 

570 

571 Returns 

572 ------- 

573 `exposure` : `lsst.afw.image.ExposureF` 

574 Simulated image. 

575 `catalog` : `lsst.afw.table.SourceCatalog` 

576 Simulated detection catalog. 

577 """ 

578 random_state = np.random.RandomState(randomSeed) 

579 assert schema.contains(self.schema) 

580 mapper = lsst.afw.table.SchemaMapper(self.schema) 

581 mapper.addMinimalSchema(self.schema, True) 

582 exposure = self.exposure.clone() 

583 exposure.variance.array[:, :] = noise**2 

584 exposure.image.array[:, :] += random_state.randn(exposure.height, exposure.width)*noise 

585 catalog = lsst.afw.table.SourceCatalog(schema) 

586 catalog.extend(self.catalog, mapper=mapper) 

587 # Loop over sources and generate new HeavyFootprints that divide up 

588 # the noisy pixels, not the ideal no-noise pixels. 

589 for record in catalog: 

590 # parent objects have non-Heavy Footprints, which don't need to be 

591 # updated after adding noise. 

592 if record.getParent() == 0: 

593 continue 

594 # get flattened arrays that correspond to the no-noise and noisy 

595 # parent images 

596 parent = catalog.find(record.getParent()) 

597 footprint = parent.getFootprint() 

598 parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32) 

599 footprint.spans.flatten(parentFluxArrayNoNoise, self.exposure.image.array, self.exposure.getXY0()) 

600 parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32) 

601 footprint.spans.flatten(parentFluxArrayNoisy, exposure.image.array, exposure.getXY0()) 

602 oldHeavy = record.getFootprint() 

603 fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise) 

604 # N.B. this isn't a copy ctor - it's a copy from a vanilla 

605 # Footprint, so it doesn't copy the arrays we don't want to 

606 # change, and hence we have to do that ourselves below. 

607 newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy) 

608 newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction 

609 newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray() 

610 newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray() 

611 record.setFootprint(newHeavy) 

612 return exposure, catalog 

613 

614 

615class AlgorithmTestCase: 

616 """Base class for tests of measurement tasks. 

617 """ 

618 def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()): 

619 """Create an instance of `SingleFrameMeasurementTask.ConfigClass`. 

620 

621 Only the specified plugin and its dependencies will be run; the 

622 Centroid, Shape, and ModelFlux slots will be set to the truth fields 

623 generated by the `TestDataset` class. 

624 

625 Parameters 

626 ---------- 

627 plugin : `str` 

628 Name of measurement plugin to enable. 

629 dependencies : iterable of `str`, optional 

630 Names of dependencies of the measurement plugin. 

631 

632 Returns 

633 ------- 

634 config : `SingleFrameMeasurementTask.ConfigClass` 

635 The resulting task configuration. 

636 """ 

637 config = SingleFrameMeasurementTask.ConfigClass() 

638 with warnings.catch_warnings(): 

639 warnings.filterwarnings("ignore", message="ignoreSlotPluginChecks", category=FutureWarning) 

640 config = SingleFrameMeasurementTask.ConfigClass(ignoreSlotPluginChecks=True) 

641 config.slots.centroid = "truth" 

642 config.slots.shape = "truth" 

643 config.slots.modelFlux = None 

644 config.slots.apFlux = None 

645 config.slots.psfFlux = None 

646 config.slots.gaussianFlux = None 

647 config.slots.calibFlux = None 

648 config.plugins.names = (plugin,) + tuple(dependencies) 

649 return config 

650 

651 def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, 

652 algMetadata=None): 

653 """Create a configured instance of `SingleFrameMeasurementTask`. 

654 

655 Parameters 

656 ---------- 

657 plugin : `str`, optional 

658 Name of measurement plugin to enable. If `None`, a configuration 

659 must be supplied as the ``config`` parameter. If both are 

660 specified, ``config`` takes precedence. 

661 dependencies : iterable of `str`, optional 

662 Names of dependencies of the specified measurement plugin. 

663 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

664 Configuration for the task. If `None`, a measurement plugin must 

665 be supplied as the ``plugin`` paramter. If both are specified, 

666 ``config`` takes precedence. 

667 schema : `lsst.afw.table.Schema`, optional 

668 Measurement table schema. If `None`, a default schema is 

669 generated. 

670 algMetadata : `lsst.daf.base.PropertyList`, optional 

671 Measurement algorithm metadata. If `None`, a default container 

672 will be generated. 

673 

674 Returns 

675 ------- 

676 task : `SingleFrameMeasurementTask` 

677 A configured instance of the measurement task. 

678 """ 

679 if config is None: 

680 if plugin is None: 

681 raise ValueError("Either plugin or config argument must not be None") 

682 config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies) 

683 if schema is None: 

684 schema = TestDataset.makeMinimalSchema() 

685 # Clear all aliases so only those defined by config are set. 

686 schema.setAliasMap(None) 

687 if algMetadata is None: 

688 algMetadata = lsst.daf.base.PropertyList() 

689 return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config) 

690 

691 def makeForcedMeasurementConfig(self, plugin=None, dependencies=()): 

692 """Create an instance of `ForcedMeasurementTask.ConfigClass`. 

693 

694 In addition to the plugins specified in the plugin and dependencies 

695 arguments, the `TransformedCentroid` and `TransformedShape` plugins 

696 will be run and used as the centroid and shape slots; these simply 

697 transform the reference catalog centroid and shape to the measurement 

698 coordinate system. 

699 

700 Parameters 

701 ---------- 

702 plugin : `str` 

703 Name of measurement plugin to enable. 

704 dependencies : iterable of `str`, optional 

705 Names of dependencies of the measurement plugin. 

706 

707 Returns 

708 ------- 

709 config : `ForcedMeasurementTask.ConfigClass` 

710 The resulting task configuration. 

711 """ 

712 

713 config = ForcedMeasurementTask.ConfigClass() 

714 config.slots.centroid = "base_TransformedCentroid" 

715 config.slots.shape = "base_TransformedShape" 

716 config.slots.modelFlux = None 

717 config.slots.apFlux = None 

718 config.slots.psfFlux = None 

719 config.slots.gaussianFlux = None 

720 config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid", 

721 "base_TransformedShape") 

722 return config 

723 

724 def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, 

725 algMetadata=None): 

726 """Create a configured instance of `ForcedMeasurementTask`. 

727 

728 Parameters 

729 ---------- 

730 plugin : `str`, optional 

731 Name of measurement plugin to enable. If `None`, a configuration 

732 must be supplied as the ``config`` parameter. If both are 

733 specified, ``config`` takes precedence. 

734 dependencies : iterable of `str`, optional 

735 Names of dependencies of the specified measurement plugin. 

736 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

737 Configuration for the task. If `None`, a measurement plugin must 

738 be supplied as the ``plugin`` paramter. If both are specified, 

739 ``config`` takes precedence. 

740 refSchema : `lsst.afw.table.Schema`, optional 

741 Reference table schema. If `None`, a default schema is 

742 generated. 

743 algMetadata : `lsst.daf.base.PropertyList`, optional 

744 Measurement algorithm metadata. If `None`, a default container 

745 will be generated. 

746 

747 Returns 

748 ------- 

749 task : `ForcedMeasurementTask` 

750 A configured instance of the measurement task. 

751 """ 

752 if config is None: 

753 if plugin is None: 

754 raise ValueError("Either plugin or config argument must not be None") 

755 config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies) 

756 if refSchema is None: 

757 refSchema = TestDataset.makeMinimalSchema() 

758 if algMetadata is None: 

759 algMetadata = lsst.daf.base.PropertyList() 

760 return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config) 

761 

762 

763class TransformTestCase: 

764 """Base class for testing measurement transformations. 

765 

766 Notes 

767 ----- 

768 We test both that the transform itself operates successfully (fluxes are 

769 converted to magnitudes, flags are propagated properly) and that the 

770 transform is registered as the default for the appropriate measurement 

771 algorithms. 

772 

773 In the simple case of one-measurement-per-transformation, the developer 

774 need not directly write any tests themselves: simply customizing the class 

775 variables is all that is required. More complex measurements (e.g. 

776 multiple aperture fluxes) require extra effort. 

777 """ 

778 name = "MeasurementTransformTest" 

779 """The name used for the measurement algorithm (str). 

780 

781 Notes 

782 ----- 

783 This determines the names of the fields in the resulting catalog. This 

784 default should generally be fine, but subclasses can override if 

785 required. 

786 """ 

787 

788 # These should be customized by subclassing. 

789 controlClass = None 

790 algorithmClass = None 

791 transformClass = None 

792 

793 flagNames = ("flag",) 

794 """Flags which may be set by the algorithm being tested (iterable of `str`). 

795 """ 

796 

797 # The plugin being tested should be registered under these names for 

798 # single frame and forced measurement. Should be customized by 

799 # subclassing. 

800 singleFramePlugins = () 

801 forcedPlugins = () 

802 

803 def setUp(self): 

804 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200)) 

805 self.calexp = TestDataset.makeEmptyExposure(bbox) 

806 self._setupTransform() 

807 

808 def tearDown(self): 

809 del self.calexp 

810 del self.inputCat 

811 del self.mapper 

812 del self.transform 

813 del self.outputCat 

814 

815 def _populateCatalog(self, baseNames): 

816 records = [] 

817 for flagValue in (True, False): 

818 records.append(self.inputCat.addNew()) 

819 for baseName in baseNames: 

820 for flagName in self.flagNames: 

821 if records[-1].schema.join(baseName, flagName) in records[-1].schema: 

822 records[-1].set(records[-1].schema.join(baseName, flagName), flagValue) 

823 self._setFieldsInRecords(records, baseName) 

824 

825 def _checkOutput(self, baseNames): 

826 for inSrc, outSrc in zip(self.inputCat, self.outputCat): 

827 for baseName in baseNames: 

828 self._compareFieldsInRecords(inSrc, outSrc, baseName) 

829 for flagName in self.flagNames: 

830 keyName = outSrc.schema.join(baseName, flagName) 

831 if keyName in inSrc.schema: 

832 self.assertEqual(outSrc.get(keyName), inSrc.get(keyName)) 

833 else: 

834 self.assertFalse(keyName in outSrc.schema) 

835 

836 def _runTransform(self, doExtend=True): 

837 if doExtend: 

838 self.outputCat.extend(self.inputCat, mapper=self.mapper) 

839 self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib()) 

840 

841 def testTransform(self, baseNames=None): 

842 """Test the transformation on a catalog containing random data. 

843 

844 Parameters 

845 ---------- 

846 baseNames : iterable of `str` 

847 Iterable of the initial parts of measurement field names. 

848 

849 Notes 

850 ----- 

851 We check that: 

852 

853 - An appropriate exception is raised on an attempt to transform 

854 between catalogs with different numbers of rows; 

855 - Otherwise, all appropriate conversions are properly appled and that 

856 flags have been propagated. 

857 

858 The ``baseNames`` argument requires some explanation. This should be 

859 an iterable of the leading parts of the field names for each 

860 measurement; that is, everything that appears before ``_instFlux``, 

861 ``_flag``, etc. In the simple case of a single measurement per plugin, 

862 this is simply equal to ``self.name`` (thus measurements are stored as 

863 ``self.name + "_instFlux"``, etc). More generally, the developer may 

864 specify whatever iterable they require. For example, to handle 

865 multiple apertures, we could have ``(self.name + "_0", self.name + 

866 "_1", ...)``. 

867 """ 

868 baseNames = baseNames or [self.name] 

869 self._populateCatalog(baseNames) 

870 self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False) 

871 self._runTransform() 

872 self._checkOutput(baseNames) 

873 

874 def _checkRegisteredTransform(self, registry, name): 

875 # If this is a Python-based transform, we can compare directly; if 

876 # it's wrapped C++, we need to compare the wrapped class. 

877 self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass) 

878 

879 def testRegistration(self): 

880 """Test that the transformation is appropriately registered. 

881 """ 

882 for pluginName in self.singleFramePlugins: 

883 self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName) 

884 for pluginName in self.forcedPlugins: 

885 self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName) 

886 

887 

888class SingleFramePluginTransformSetupHelper: 

889 

890 def _setupTransform(self): 

891 self.control = self.controlClass() 

892 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema() 

893 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

894 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

895 # defining the schema. 

896 inputSchema.getAliasMap().set("slot_Centroid", "dummy") 

897 inputSchema.getAliasMap().set("slot_Shape", "dummy") 

898 self.algorithmClass(self.control, self.name, inputSchema) 

899 inputSchema.getAliasMap().erase("slot_Centroid") 

900 inputSchema.getAliasMap().erase("slot_Shape") 

901 self.inputCat = lsst.afw.table.SourceCatalog(inputSchema) 

902 self.mapper = lsst.afw.table.SchemaMapper(inputSchema) 

903 self.transform = self.transformClass(self.control, self.name, self.mapper) 

904 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

905 

906 

907class ForcedPluginTransformSetupHelper: 

908 

909 def _setupTransform(self): 

910 self.control = self.controlClass() 

911 inputMapper = lsst.afw.table.SchemaMapper(lsst.afw.table.SourceTable.makeMinimalSchema(), 

912 lsst.afw.table.SourceTable.makeMinimalSchema()) 

913 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

914 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

915 # defining the schema. 

916 inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy") 

917 inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy") 

918 self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList()) 

919 inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid") 

920 inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape") 

921 self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema()) 

922 self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema()) 

923 self.transform = self.transformClass(self.control, self.name, self.mapper) 

924 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

925 

926 

927class FluxTransformTestCase(TransformTestCase): 

928 

929 def _setFieldsInRecords(self, records, name): 

930 for record in records: 

931 record[record.schema.join(name, 'instFlux')] = np.random.random() 

932 record[record.schema.join(name, 'instFluxErr')] = np.random.random() 

933 

934 # Negative instFluxes should be converted to NaNs. 

935 assert len(records) > 1 

936 records[0][record.schema.join(name, 'instFlux')] = -1 

937 

938 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

939 instFluxName = inSrc.schema.join(name, 'instFlux') 

940 instFluxErrName = inSrc.schema.join(name, 'instFluxErr') 

941 if inSrc[instFluxName] > 0: 

942 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

943 inSrc[instFluxErrName]) 

944 self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value) 

945 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

946 else: 

947 # negative instFlux results in NaN magnitude, but can still have finite error 

948 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')])) 

949 if np.isnan(inSrc[instFluxErrName]): 

950 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')])) 

951 else: 

952 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

953 inSrc[instFluxErrName]) 

954 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

955 

956 

957class CentroidTransformTestCase(TransformTestCase): 

958 

959 def _setFieldsInRecords(self, records, name): 

960 for record in records: 

961 record[record.schema.join(name, 'x')] = np.random.random() 

962 record[record.schema.join(name, 'y')] = np.random.random() 

963 # Some algorithms set no errors; some set only sigma on x & y; some provide 

964 # a full covariance matrix. Set only those which exist in the schema. 

965 for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'): 

966 fieldName = record.schema.join(name, fieldSuffix) 

967 if fieldName in record.schema: 

968 record[fieldName] = np.random.random() 

969 

970 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

971 centroidResultKey = CentroidResultKey(inSrc.schema[self.name]) 

972 centroidResult = centroidResultKey.get(inSrc) 

973 

974 coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc) 

975 coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid()) 

976 self.assertEqual(coordTruth, coord) 

977 

978 # If the centroid has an associated uncertainty matrix, the coordinate 

979 # must have one too, and vice versa. 

980 try: 

981 coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name], 

982 ["ra", "dec"]).get(outSrc) 

983 except lsst.pex.exceptions.NotFoundError: 

984 self.assertFalse(centroidResultKey.getCentroidErr().isValid()) 

985 else: 

986 transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians) 

987 coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(), 

988 centroidResult.getCentroidErr()), 

989 transform.getLinear().getMatrix().transpose()) 

990 np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)