Coverage for python/lsst/meas/base/tests.py: 14%

409 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-18 11:04 +0000

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24import numpy as np 

25 

26import lsst.geom 

27import lsst.afw.table 

28import lsst.afw.image 

29import lsst.afw.detection 

30import lsst.afw.geom 

31import lsst.afw.coord 

32import lsst.daf.base 

33import lsst.pex.exceptions 

34 

35from .sfm import SingleFrameMeasurementTask 

36from .forcedMeasurement import ForcedMeasurementTask 

37from ._measBaseLib import CentroidResultKey 

38 

39__all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase", 

40 "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper", 

41 "FluxTransformTestCase", "CentroidTransformTestCase") 

42 

43 

44class BlendContext: 

45 """Context manager which adds multiple overlapping sources and a parent. 

46 

47 Notes 

48 ----- 

49 This is used as the return value for `TestDataset.addBlend`, and this is 

50 the only way it should be used. 

51 """ 

52 

53 def __init__(self, owner): 

54 self.owner = owner 

55 self.parentRecord = self.owner.catalog.addNew() 

56 self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox()) 

57 self.children = [] 

58 

59 def __enter__(self): 

60 # BlendContext is its own context manager, so we just return self. 

61 return self 

62 

63 def addChild(self, instFlux, centroid, shape=None): 

64 """Add a child to the blend; return corresponding truth catalog record. 

65 

66 instFlux : `float` 

67 Total instFlux of the source to be added. 

68 centroid : `lsst.geom.Point2D` 

69 Position of the source to be added. 

70 shape : `lsst.afw.geom.Quadrupole` 

71 Second moments of the source before PSF convolution. Note that 

72 the truth catalog records post-convolution moments) 

73 """ 

74 record, image = self.owner.addSource(instFlux, centroid, shape) 

75 record.set(self.owner.keys["parent"], self.parentRecord.getId()) 

76 self.parentImage += image 

77 self.children.append((record, image)) 

78 return record 

79 

80 def __exit__(self, type_, value, tb): 

81 # We're not using the context manager for any kind of exception safety 

82 # or guarantees; we just want the nice "with" statement syntax. 

83 

84 if type_ is not None: 

85 # exception was raised; just skip all this and let it propagate 

86 return 

87 

88 # On exit, compute and set the truth values for the parent object. 

89 self.parentRecord.set(self.owner.keys["nChild"], len(self.children)) 

90 # Compute instFlux from sum of component fluxes 

91 instFlux = 0.0 

92 for record, image in self.children: 

93 instFlux += record.get(self.owner.keys["instFlux"]) 

94 self.parentRecord.set(self.owner.keys["instFlux"], instFlux) 

95 # Compute centroid from instFlux-weighted mean of component centroids 

96 x = 0.0 

97 y = 0.0 

98 for record, image in self.children: 

99 w = record.get(self.owner.keys["instFlux"])/instFlux 

100 x += record.get(self.owner.keys["centroid"].getX())*w 

101 y += record.get(self.owner.keys["centroid"].getY())*w 

102 self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y)) 

103 # Compute shape from instFlux-weighted mean of offset component shapes 

104 xx = 0.0 

105 yy = 0.0 

106 xy = 0.0 

107 for record, image in self.children: 

108 w = record.get(self.owner.keys["instFlux"])/instFlux 

109 dx = record.get(self.owner.keys["centroid"].getX()) - x 

110 dy = record.get(self.owner.keys["centroid"].getY()) - y 

111 xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w 

112 yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w 

113 xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w 

114 self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy)) 

115 # Run detection on the parent image to get the parent Footprint. 

116 self.owner._installFootprint(self.parentRecord, self.parentImage) 

117 # Create perfect HeavyFootprints for all children; these will need to 

118 # be modified later to account for the noise we'll add to the image. 

119 deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.maskedImage, True) 

120 for record, image in self.children: 

121 deblend.image.array[:, :] = image.array 

122 heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend) 

123 record.setFootprint(heavyFootprint) 

124 

125 

126class TestDataset: 

127 """A simulated dataset consisuting of test image and truth catalog. 

128 

129 TestDataset creates an idealized image made of pure Gaussians (including a 

130 Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints 

131 that simulated the outputs of detection and deblending. Multiple noise 

132 realizations can be created from the same underlying sources, allowing 

133 uncertainty estimates to be verified via Monte Carlo. 

134 

135 Parameters 

136 ---------- 

137 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

138 Bounding box of the test image. 

139 threshold : `float` 

140 Threshold absolute value used to determine footprints for 

141 simulated sources. This thresholding will be applied before noise is 

142 actually added to images (or before the noise level is even known), so 

143 this will necessarily produce somewhat artificial footprints. 

144 exposure : `lsst.afw.image.ExposureF` 

145 The image to which test sources should be added. Ownership should 

146 be considered transferred from the caller to the TestDataset. 

147 Must have a Gaussian PSF for truth catalog shapes to be exact. 

148 **kwds 

149 Keyword arguments forwarded to makeEmptyExposure if exposure is `None`. 

150 

151 Notes 

152 ----- 

153 Typical usage: 

154 

155 .. code-block: py 

156 

157 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100, 

158 100)) 

159 dataset = TestDataset(bbox) 

160 dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26)) 

161 dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24), 

162 shape=lsst.afw.geom.Quadrupole(8, 7, 2)) 

163 with dataset.addBlend() as family: 

164 family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72)) 

165 family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74)) 

166 exposure, catalog = dataset.realize(noise=100.0, 

167 schema=TestDataset.makeMinimalSchema()) 

168 """ 

169 

170 def __init__(self, bbox, threshold=10.0, exposure=None, **kwds): 

171 if exposure is None: 

172 exposure = self.makeEmptyExposure(bbox, **kwds) 

173 self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE) 

174 self.exposure = exposure 

175 self.psfShape = self.exposure.getPsf().computeShape(bbox.getCenter()) 

176 self.schema = self.makeMinimalSchema() 

177 self.catalog = lsst.afw.table.SourceCatalog(self.schema) 

178 

179 @classmethod 

180 def makeMinimalSchema(cls): 

181 """Return the minimal schema needed to hold truth catalog fields. 

182 

183 Notes 

184 ----- 

185 When `TestDataset.realize` is called, the schema must include at least 

186 these fields. Usually it will include additional fields for 

187 measurement algorithm outputs, allowing the same catalog to be used 

188 for both truth values (the fields from the minimal schema) and the 

189 measurements. 

190 """ 

191 if not hasattr(cls, "_schema"): 

192 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

193 cls.keys = {} 

194 cls.keys["coordErr"] = lsst.afw.table.CoordKey.addErrorFields(schema) 

195 cls.keys["parent"] = schema.find("parent").key 

196 cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32) 

197 cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64, 

198 doc="true instFlux", units="count") 

199 cls.keys["instFluxErr"] = schema.addField("truth_instFluxErr", type=np.float64, 

200 doc="true instFluxErr", units="count") 

201 cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields( 

202 schema, "truth", "true simulated centroid", "pixel" 

203 ) 

204 cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields( 

205 schema, "truth", ['x', 'y'], "pixel" 

206 ) 

207 cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag", 

208 doc="set if the object is a star") 

209 cls.keys["shape"] = lsst.afw.table.QuadrupoleKey.addFields( 

210 schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL 

211 ) 

212 cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag", 

213 doc="set if the object is a star") 

214 schema.getAliasMap().set("slot_Shape", "truth") 

215 schema.getAliasMap().set("slot_Centroid", "truth") 

216 schema.getAliasMap().set("slot_ModelFlux", "truth") 

217 cls._schema = schema 

218 schema = lsst.afw.table.Schema(cls._schema) 

219 schema.disconnectAliases() 

220 return schema 

221 

222 @staticmethod 

223 def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, 

224 minRotation=None, maxRotation=None, 

225 minRefShift=None, maxRefShift=None, 

226 minPixShift=2.0, maxPixShift=4.0, randomSeed=1): 

227 """Return a perturbed version of the input WCS. 

228 

229 Create a new undistorted TAN WCS that is similar but not identical to 

230 another, with random scaling, rotation, and offset (in both pixel 

231 position and reference position). 

232 

233 Parameters 

234 ---------- 

235 oldWcs : `lsst.afw.geom.SkyWcs` 

236 The input WCS. 

237 minScaleFactor : `float` 

238 Minimum scale factor to apply to the input WCS. 

239 maxScaleFactor : `float` 

240 Maximum scale factor to apply to the input WCS. 

241 minRotation : `lsst.geom.Angle` or `None` 

242 Minimum rotation to apply to the input WCS. If `None`, defaults to 

243 30 degrees. 

244 maxRotation : `lsst.geom.Angle` or `None` 

245 Minimum rotation to apply to the input WCS. If `None`, defaults to 

246 60 degrees. 

247 minRefShift : `lsst.geom.Angle` or `None` 

248 Miniumum shift to apply to the input WCS reference value. If 

249 `None`, defaults to 0.5 arcsec. 

250 maxRefShift : `lsst.geom.Angle` or `None` 

251 Miniumum shift to apply to the input WCS reference value. If 

252 `None`, defaults to 1.0 arcsec. 

253 minPixShift : `float` 

254 Minimum shift to apply to the input WCS reference pixel. 

255 maxPixShift : `float` 

256 Maximum shift to apply to the input WCS reference pixel. 

257 randomSeed : `int` 

258 Random seed. 

259 

260 Returns 

261 ------- 

262 newWcs : `lsst.afw.geom.SkyWcs` 

263 A perturbed version of the input WCS. 

264 

265 Notes 

266 ----- 

267 The maximum and minimum arguments are interpreted as absolute values 

268 for a split range that covers both positive and negative values (as 

269 this method is used in testing, it is typically most important to 

270 avoid perturbations near zero). Scale factors are treated somewhat 

271 differently: the actual scale factor is chosen between 

272 ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``) 

273 and (``1/minScaleFactor``). 

274 

275 The default range for rotation is 30-60 degrees, and the default range 

276 for reference shift is 0.5-1.0 arcseconds (these cannot be safely 

277 included directly as default values because Angle objects are 

278 mutable). 

279 

280 The random number generator is primed with the seed given. If 

281 `None`, a seed is automatically chosen. 

282 """ 

283 random_state = np.random.RandomState(randomSeed) 

284 if minRotation is None: 

285 minRotation = 30.0*lsst.geom.degrees 

286 if maxRotation is None: 

287 maxRotation = 60.0*lsst.geom.degrees 

288 if minRefShift is None: 

289 minRefShift = 0.5*lsst.geom.arcseconds 

290 if maxRefShift is None: 

291 maxRefShift = 1.0*lsst.geom.arcseconds 

292 

293 def splitRandom(min1, max1, min2=None, max2=None): 

294 if min2 is None: 

295 min2 = -max1 

296 if max2 is None: 

297 max2 = -min1 

298 if random_state.uniform() > 0.5: 

299 return float(random_state.uniform(min1, max1)) 

300 else: 

301 return float(random_state.uniform(min2, max2)) 

302 # Generate random perturbations 

303 scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor) 

304 rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians 

305 refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

306 refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

307 pixShiftX = splitRandom(minPixShift, maxPixShift) 

308 pixShiftY = splitRandom(minPixShift, maxPixShift) 

309 # Compute new CD matrix 

310 oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix()) 

311 rTransform = lsst.geom.LinearTransform.makeRotation(rotation) 

312 sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor) 

313 newTransform = oldTransform*rTransform*sTransform 

314 matrix = newTransform.getMatrix() 

315 # Compute new coordinate reference pixel (CRVAL) 

316 oldSkyOrigin = oldWcs.getSkyOrigin() 

317 newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa, 

318 oldSkyOrigin.getDec() + refShiftDec) 

319 # Compute new pixel reference pixel (CRPIX) 

320 oldPixOrigin = oldWcs.getPixelOrigin() 

321 newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX, 

322 oldPixOrigin.getY() + pixShiftY) 

323 return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix) 

324 

325 @staticmethod 

326 def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4): 

327 """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values. 

328 

329 Parameters 

330 ---------- 

331 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

332 Bounding box of the image in image coordinates. 

333 wcs : `lsst.afw.geom.SkyWcs`, optional 

334 New WCS for the exposure (created from CRVAL and CDELT if `None`). 

335 crval : `lsst.afw.geom.SpherePoint`, optional 

336 ICRS center of the TAN WCS attached to the image. If `None`, (45 

337 degrees, 45 degrees) is assumed. 

338 cdelt : `lsst.geom.Angle`, optional 

339 Pixel scale of the image. If `None`, 0.2 arcsec is assumed. 

340 psfSigma : `float`, optional 

341 Radius (sigma) of the Gaussian PSF attached to the image 

342 psfDim : `int`, optional 

343 Width and height of the image's Gaussian PSF attached to the image 

344 calibration : `float`, optional 

345 The spatially-constant calibration (in nJy/count) to set the 

346 PhotoCalib of the exposure. 

347 

348 Returns 

349 ------- 

350 exposure : `lsst.age.image.ExposureF` 

351 An empty image. 

352 """ 

353 if wcs is None: 

354 if crval is None: 

355 crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees) 

356 if cdelt is None: 

357 cdelt = 0.2*lsst.geom.arcseconds 

358 crpix = lsst.geom.Box2D(bbox).getCenter() 

359 wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, 

360 cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt)) 

361 exposure = lsst.afw.image.ExposureF(bbox) 

362 psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma) 

363 photoCalib = lsst.afw.image.PhotoCalib(calibration) 

364 visitInfo = lsst.afw.image.VisitInfo(id=1234, 

365 exposureTime=30.0, 

366 date=lsst.daf.base.DateTime(60000.0), 

367 observatory=lsst.afw.coord.Observatory(11.1*lsst.geom.degrees, 

368 22.2*lsst.geom.degrees, 

369 0.333)) 

370 exposure.setWcs(wcs) 

371 exposure.setPsf(psf) 

372 exposure.setPhotoCalib(photoCalib) 

373 exposure.info.setVisitInfo(visitInfo) 

374 return exposure 

375 

376 @staticmethod 

377 def drawGaussian(bbox, instFlux, ellipse): 

378 """Create an image of an elliptical Gaussian. 

379 

380 Parameters 

381 ---------- 

382 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

383 Bounding box of image to create. 

384 instFlux : `float` 

385 Total instrumental flux of the Gaussian (normalized analytically, 

386 not using pixel values). 

387 ellipse : `lsst.afw.geom.Ellipse` 

388 Defines the centroid and shape. 

389 

390 Returns 

391 ------- 

392 image : `lsst.afw.image.ImageF` 

393 An image of the Gaussian. 

394 """ 

395 x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()), 

396 np.arange(bbox.getBeginY(), bbox.getEndY())) 

397 t = ellipse.getGridTransform() 

398 xt = t[t.XX] * x + t[t.XY] * y + t[t.X] 

399 yt = t[t.YX] * x + t[t.YY] * y + t[t.Y] 

400 image = lsst.afw.image.ImageF(bbox) 

401 image.array[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea()) 

402 return image 

403 

404 def _installFootprint(self, record, image, setPeakSignificance=True): 

405 """Create simulated Footprint and add it to a truth catalog record. 

406 """ 

407 schema = lsst.afw.detection.PeakTable.makeMinimalSchema() 

408 if setPeakSignificance: 

409 schema.addField("significance", type=float, 

410 doc="Ratio of peak value to configured standard deviation.") 

411 # Run detection on the single-source image 

412 fpSet = lsst.afw.detection.FootprintSet(image, self.threshold, peakSchema=schema) 

413 # the call below to the FootprintSet ctor is actually a grow operation 

414 fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True) 

415 if setPeakSignificance: 

416 # This isn't a traditional significance, since we're using the VALUE 

417 # threshold type, but it's the best we can do in that case. 

418 for footprint in fpSet.getFootprints(): 

419 footprint.updatePeakSignificance(self.threshold.getValue()) 

420 # Update the full exposure's mask plane to indicate the detection 

421 fpSet.setMask(self.exposure.mask, "DETECTED") 

422 # Attach the new footprint to the exposure 

423 if len(fpSet.getFootprints()) > 1: 

424 raise RuntimeError("Threshold value results in multiple Footprints for a single object") 

425 if len(fpSet.getFootprints()) == 0: 

426 raise RuntimeError("Threshold value results in zero Footprints for object") 

427 record.setFootprint(fpSet.getFootprints()[0]) 

428 

429 def addSource(self, instFlux, centroid, shape=None, setPeakSignificance=True): 

430 """Add a source to the simulation. 

431 

432 To insert a point source with a given signal-to-noise (sn), the total 

433 ``instFlux`` should be: ``sn*noise*psf_scale``, where ``noise`` is the 

434 noise you will pass to ``realize()``, and 

435 ``psf_scale=sqrt(4*pi*r^2)``, where ``r`` is the width of the PSF. 

436 

437 Parameters 

438 ---------- 

439 instFlux : `float` 

440 Total instFlux of the source to be added. 

441 centroid : `lsst.geom.Point2D` 

442 Position of the source to be added. 

443 shape : `lsst.afw.geom.Quadrupole` 

444 Second moments of the source before PSF convolution. Note that the 

445 truth catalog records post-convolution moments. If `None`, a point 

446 source will be added. 

447 setPeakSignificance : `bool` 

448 Set the ``significance`` field for peaks in the footprints? 

449 See ``lsst.meas.algorithms.SourceDetectionTask.setPeakSignificance`` 

450 for how this field is computed for real datasets. 

451 

452 Returns 

453 ------- 

454 record : `lsst.afw.table.SourceRecord` 

455 A truth catalog record. 

456 image : `lsst.afw.image.ImageF` 

457 Single-source image corresponding to the new source. 

458 """ 

459 # Create and set the truth catalog fields 

460 record = self.catalog.addNew() 

461 record.set(self.keys["instFlux"], instFlux) 

462 record.set(self.keys["instFluxErr"], 0) 

463 record.set(self.keys["centroid"], centroid) 

464 covariance = np.random.normal(0, 0.1, 4).reshape(2, 2) 

465 covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov 

466 record.set(self.keys["centroid_sigma"], covariance.astype(np.float32)) 

467 if shape is None: 

468 record.set(self.keys["isStar"], True) 

469 fullShape = self.psfShape 

470 else: 

471 record.set(self.keys["isStar"], False) 

472 fullShape = shape.convolve(self.psfShape) 

473 record.set(self.keys["shape"], fullShape) 

474 # Create an image containing just this source 

475 image = self.drawGaussian(self.exposure.getBBox(), instFlux, 

476 lsst.afw.geom.Ellipse(fullShape, centroid)) 

477 # Generate a footprint for this source 

478 self._installFootprint(record, image, setPeakSignificance) 

479 # Actually add the source to the full exposure 

480 self.exposure.image.array[:, :] += image.array 

481 return record, image 

482 

483 def addBlend(self): 

484 """Return a context manager which can add a blend of multiple sources. 

485 

486 Notes 

487 ----- 

488 Note that nothing stops you from creating overlapping sources just using the addSource() method, 

489 but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type 

490 produced by the detection and deblending pipelines. 

491 

492 Examples 

493 -------- 

494 .. code-block: py 

495 d = TestDataset(...) 

496 with d.addBlend() as b: 

497 b.addChild(flux1, centroid1) 

498 b.addChild(flux2, centroid2, shape2) 

499 """ 

500 return BlendContext(self) 

501 

502 def transform(self, wcs, **kwds): 

503 """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib. 

504 

505 Parameters 

506 ---------- 

507 wcs : `lsst.afw.geom.SkyWcs` 

508 WCS for the new dataset. 

509 **kwds 

510 Additional keyword arguments passed on to 

511 `TestDataset.makeEmptyExposure`. If not specified, these revert 

512 to the defaults for `~TestDataset.makeEmptyExposure`, not the 

513 values in the current dataset. 

514 

515 Returns 

516 ------- 

517 newDataset : `TestDataset` 

518 Transformed copy of this dataset. 

519 """ 

520 bboxD = lsst.geom.Box2D() 

521 xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs) 

522 for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners(): 

523 bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner))) 

524 bboxI = lsst.geom.Box2I(bboxD) 

525 result = TestDataset(bbox=bboxI, wcs=wcs, **kwds) 

526 oldPhotoCalib = self.exposure.getPhotoCalib() 

527 newPhotoCalib = result.exposure.getPhotoCalib() 

528 oldPsfShape = self.exposure.getPsf().computeShape(bboxD.getCenter()) 

529 for record in self.catalog: 

530 if record.get(self.keys["nChild"]): 

531 raise NotImplementedError("Transforming blended sources in TestDatasets is not supported") 

532 magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"])) 

533 newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude) 

534 oldCentroid = record.get(self.keys["centroid"]) 

535 newCentroid = xyt.applyForward(oldCentroid) 

536 if record.get(self.keys["isStar"]): 

537 newDeconvolvedShape = None 

538 else: 

539 affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid) 

540 oldFullShape = record.get(self.keys["shape"]) 

541 oldDeconvolvedShape = lsst.afw.geom.Quadrupole( 

542 oldFullShape.getIxx() - oldPsfShape.getIxx(), 

543 oldFullShape.getIyy() - oldPsfShape.getIyy(), 

544 oldFullShape.getIxy() - oldPsfShape.getIxy(), 

545 False 

546 ) 

547 newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear()) 

548 result.addSource(newFlux, newCentroid, newDeconvolvedShape) 

549 return result 

550 

551 def realize(self, noise, schema, randomSeed=1): 

552 r"""Simulate an exposure and detection catalog for this dataset. 

553 

554 The simulation includes noise, and the detection catalog includes 

555 `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s. 

556 

557 Parameters 

558 ---------- 

559 noise : `float` 

560 Standard deviation of noise to be added to the exposure. The 

561 noise will be Gaussian and constant, appropriate for the 

562 sky-limited regime. 

563 schema : `lsst.afw.table.Schema` 

564 Schema of the new catalog to be created. Must start with 

565 ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be 

566 `True`), but typically contains fields for already-configured 

567 measurement algorithms as well. 

568 randomSeed : `int`, optional 

569 Seed for the random number generator. 

570 If `None`, a seed is chosen automatically. 

571 

572 Returns 

573 ------- 

574 `exposure` : `lsst.afw.image.ExposureF` 

575 Simulated image. 

576 `catalog` : `lsst.afw.table.SourceCatalog` 

577 Simulated detection catalog. 

578 """ 

579 random_state = np.random.RandomState(randomSeed) 

580 assert schema.contains(self.schema) 

581 mapper = lsst.afw.table.SchemaMapper(self.schema) 

582 mapper.addMinimalSchema(self.schema, True) 

583 exposure = self.exposure.clone() 

584 exposure.variance.array[:, :] = noise**2 

585 exposure.image.array[:, :] += random_state.randn(exposure.height, exposure.width)*noise 

586 catalog = lsst.afw.table.SourceCatalog(schema) 

587 catalog.extend(self.catalog, mapper=mapper) 

588 # Loop over sources and generate new HeavyFootprints that divide up 

589 # the noisy pixels, not the ideal no-noise pixels. 

590 for record in catalog: 

591 # parent objects have non-Heavy Footprints, which don't need to be 

592 # updated after adding noise. 

593 if record.getParent() == 0: 

594 continue 

595 # get flattened arrays that correspond to the no-noise and noisy 

596 # parent images 

597 parent = catalog.find(record.getParent()) 

598 footprint = parent.getFootprint() 

599 parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32) 

600 footprint.spans.flatten(parentFluxArrayNoNoise, self.exposure.image.array, self.exposure.getXY0()) 

601 parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32) 

602 footprint.spans.flatten(parentFluxArrayNoisy, exposure.image.array, exposure.getXY0()) 

603 oldHeavy = record.getFootprint() 

604 fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise) 

605 # N.B. this isn't a copy ctor - it's a copy from a vanilla 

606 # Footprint, so it doesn't copy the arrays we don't want to 

607 # change, and hence we have to do that ourselves below. 

608 newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy) 

609 newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction 

610 newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray() 

611 newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray() 

612 record.setFootprint(newHeavy) 

613 lsst.afw.table.updateSourceCoords(exposure.wcs, catalog) 

614 return exposure, catalog 

615 

616 

617class AlgorithmTestCase: 

618 """Base class for tests of measurement tasks. 

619 """ 

620 def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()): 

621 """Create an instance of `SingleFrameMeasurementTask.ConfigClass`. 

622 

623 Only the specified plugin and its dependencies will be run; the 

624 Centroid, Shape, and ModelFlux slots will be set to the truth fields 

625 generated by the `TestDataset` class. 

626 

627 Parameters 

628 ---------- 

629 plugin : `str` 

630 Name of measurement plugin to enable. 

631 dependencies : iterable of `str`, optional 

632 Names of dependencies of the measurement plugin. 

633 

634 Returns 

635 ------- 

636 config : `SingleFrameMeasurementTask.ConfigClass` 

637 The resulting task configuration. 

638 """ 

639 config = SingleFrameMeasurementTask.ConfigClass() 

640 with warnings.catch_warnings(): 

641 warnings.filterwarnings("ignore", message="ignoreSlotPluginChecks", category=FutureWarning) 

642 config = SingleFrameMeasurementTask.ConfigClass(ignoreSlotPluginChecks=True) 

643 config.slots.centroid = "truth" 

644 config.slots.shape = "truth" 

645 config.slots.modelFlux = None 

646 config.slots.apFlux = None 

647 config.slots.psfFlux = None 

648 config.slots.gaussianFlux = None 

649 config.slots.calibFlux = None 

650 config.plugins.names = (plugin,) + tuple(dependencies) 

651 return config 

652 

653 def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, 

654 algMetadata=None): 

655 """Create a configured instance of `SingleFrameMeasurementTask`. 

656 

657 Parameters 

658 ---------- 

659 plugin : `str`, optional 

660 Name of measurement plugin to enable. If `None`, a configuration 

661 must be supplied as the ``config`` parameter. If both are 

662 specified, ``config`` takes precedence. 

663 dependencies : iterable of `str`, optional 

664 Names of dependencies of the specified measurement plugin. 

665 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

666 Configuration for the task. If `None`, a measurement plugin must 

667 be supplied as the ``plugin`` paramter. If both are specified, 

668 ``config`` takes precedence. 

669 schema : `lsst.afw.table.Schema`, optional 

670 Measurement table schema. If `None`, a default schema is 

671 generated. 

672 algMetadata : `lsst.daf.base.PropertyList`, optional 

673 Measurement algorithm metadata. If `None`, a default container 

674 will be generated. 

675 

676 Returns 

677 ------- 

678 task : `SingleFrameMeasurementTask` 

679 A configured instance of the measurement task. 

680 """ 

681 if config is None: 

682 if plugin is None: 

683 raise ValueError("Either plugin or config argument must not be None") 

684 config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies) 

685 if schema is None: 

686 schema = TestDataset.makeMinimalSchema() 

687 # Clear all aliases so only those defined by config are set. 

688 schema.setAliasMap(None) 

689 if algMetadata is None: 

690 algMetadata = lsst.daf.base.PropertyList() 

691 return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config) 

692 

693 def makeForcedMeasurementConfig(self, plugin=None, dependencies=()): 

694 """Create an instance of `ForcedMeasurementTask.ConfigClass`. 

695 

696 In addition to the plugins specified in the plugin and dependencies 

697 arguments, the `TransformedCentroid` and `TransformedShape` plugins 

698 will be run and used as the centroid and shape slots; these simply 

699 transform the reference catalog centroid and shape to the measurement 

700 coordinate system. 

701 

702 Parameters 

703 ---------- 

704 plugin : `str` 

705 Name of measurement plugin to enable. 

706 dependencies : iterable of `str`, optional 

707 Names of dependencies of the measurement plugin. 

708 

709 Returns 

710 ------- 

711 config : `ForcedMeasurementTask.ConfigClass` 

712 The resulting task configuration. 

713 """ 

714 

715 config = ForcedMeasurementTask.ConfigClass() 

716 config.slots.centroid = "base_TransformedCentroid" 

717 config.slots.shape = "base_TransformedShape" 

718 config.slots.modelFlux = None 

719 config.slots.apFlux = None 

720 config.slots.psfFlux = None 

721 config.slots.gaussianFlux = None 

722 config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid", 

723 "base_TransformedShape") 

724 return config 

725 

726 def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, 

727 algMetadata=None): 

728 """Create a configured instance of `ForcedMeasurementTask`. 

729 

730 Parameters 

731 ---------- 

732 plugin : `str`, optional 

733 Name of measurement plugin to enable. If `None`, a configuration 

734 must be supplied as the ``config`` parameter. If both are 

735 specified, ``config`` takes precedence. 

736 dependencies : iterable of `str`, optional 

737 Names of dependencies of the specified measurement plugin. 

738 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

739 Configuration for the task. If `None`, a measurement plugin must 

740 be supplied as the ``plugin`` paramter. If both are specified, 

741 ``config`` takes precedence. 

742 refSchema : `lsst.afw.table.Schema`, optional 

743 Reference table schema. If `None`, a default schema is 

744 generated. 

745 algMetadata : `lsst.daf.base.PropertyList`, optional 

746 Measurement algorithm metadata. If `None`, a default container 

747 will be generated. 

748 

749 Returns 

750 ------- 

751 task : `ForcedMeasurementTask` 

752 A configured instance of the measurement task. 

753 """ 

754 if config is None: 

755 if plugin is None: 

756 raise ValueError("Either plugin or config argument must not be None") 

757 config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies) 

758 if refSchema is None: 

759 refSchema = TestDataset.makeMinimalSchema() 

760 if algMetadata is None: 

761 algMetadata = lsst.daf.base.PropertyList() 

762 return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config) 

763 

764 

765class TransformTestCase: 

766 """Base class for testing measurement transformations. 

767 

768 Notes 

769 ----- 

770 We test both that the transform itself operates successfully (fluxes are 

771 converted to magnitudes, flags are propagated properly) and that the 

772 transform is registered as the default for the appropriate measurement 

773 algorithms. 

774 

775 In the simple case of one-measurement-per-transformation, the developer 

776 need not directly write any tests themselves: simply customizing the class 

777 variables is all that is required. More complex measurements (e.g. 

778 multiple aperture fluxes) require extra effort. 

779 """ 

780 name = "MeasurementTransformTest" 

781 """The name used for the measurement algorithm (str). 

782 

783 Notes 

784 ----- 

785 This determines the names of the fields in the resulting catalog. This 

786 default should generally be fine, but subclasses can override if 

787 required. 

788 """ 

789 

790 # These should be customized by subclassing. 

791 controlClass = None 

792 algorithmClass = None 

793 transformClass = None 

794 

795 flagNames = ("flag",) 

796 """Flags which may be set by the algorithm being tested (iterable of `str`). 

797 """ 

798 

799 # The plugin being tested should be registered under these names for 

800 # single frame and forced measurement. Should be customized by 

801 # subclassing. 

802 singleFramePlugins = () 

803 forcedPlugins = () 

804 

805 def setUp(self): 

806 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200)) 

807 self.calexp = TestDataset.makeEmptyExposure(bbox) 

808 self._setupTransform() 

809 

810 def tearDown(self): 

811 del self.calexp 

812 del self.inputCat 

813 del self.mapper 

814 del self.transform 

815 del self.outputCat 

816 

817 def _populateCatalog(self, baseNames): 

818 records = [] 

819 for flagValue in (True, False): 

820 records.append(self.inputCat.addNew()) 

821 for baseName in baseNames: 

822 for flagName in self.flagNames: 

823 if records[-1].schema.join(baseName, flagName) in records[-1].schema: 

824 records[-1].set(records[-1].schema.join(baseName, flagName), flagValue) 

825 self._setFieldsInRecords(records, baseName) 

826 

827 def _checkOutput(self, baseNames): 

828 for inSrc, outSrc in zip(self.inputCat, self.outputCat): 

829 for baseName in baseNames: 

830 self._compareFieldsInRecords(inSrc, outSrc, baseName) 

831 for flagName in self.flagNames: 

832 keyName = outSrc.schema.join(baseName, flagName) 

833 if keyName in inSrc.schema: 

834 self.assertEqual(outSrc.get(keyName), inSrc.get(keyName)) 

835 else: 

836 self.assertFalse(keyName in outSrc.schema) 

837 

838 def _runTransform(self, doExtend=True): 

839 if doExtend: 

840 self.outputCat.extend(self.inputCat, mapper=self.mapper) 

841 self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib()) 

842 

843 def testTransform(self, baseNames=None): 

844 """Test the transformation on a catalog containing random data. 

845 

846 Parameters 

847 ---------- 

848 baseNames : iterable of `str` 

849 Iterable of the initial parts of measurement field names. 

850 

851 Notes 

852 ----- 

853 We check that: 

854 

855 - An appropriate exception is raised on an attempt to transform 

856 between catalogs with different numbers of rows; 

857 - Otherwise, all appropriate conversions are properly appled and that 

858 flags have been propagated. 

859 

860 The ``baseNames`` argument requires some explanation. This should be 

861 an iterable of the leading parts of the field names for each 

862 measurement; that is, everything that appears before ``_instFlux``, 

863 ``_flag``, etc. In the simple case of a single measurement per plugin, 

864 this is simply equal to ``self.name`` (thus measurements are stored as 

865 ``self.name + "_instFlux"``, etc). More generally, the developer may 

866 specify whatever iterable they require. For example, to handle 

867 multiple apertures, we could have ``(self.name + "_0", self.name + 

868 "_1", ...)``. 

869 """ 

870 baseNames = baseNames or [self.name] 

871 self._populateCatalog(baseNames) 

872 self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False) 

873 self._runTransform() 

874 self._checkOutput(baseNames) 

875 

876 def _checkRegisteredTransform(self, registry, name): 

877 # If this is a Python-based transform, we can compare directly; if 

878 # it's wrapped C++, we need to compare the wrapped class. 

879 self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass) 

880 

881 def testRegistration(self): 

882 """Test that the transformation is appropriately registered. 

883 """ 

884 for pluginName in self.singleFramePlugins: 

885 self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName) 

886 for pluginName in self.forcedPlugins: 

887 self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName) 

888 

889 

890class SingleFramePluginTransformSetupHelper: 

891 

892 def _setupTransform(self): 

893 self.control = self.controlClass() 

894 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema() 

895 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

896 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

897 # defining the schema. 

898 inputSchema.getAliasMap().set("slot_Centroid", "dummy") 

899 inputSchema.getAliasMap().set("slot_Shape", "dummy") 

900 self.algorithmClass(self.control, self.name, inputSchema) 

901 inputSchema.getAliasMap().erase("slot_Centroid") 

902 inputSchema.getAliasMap().erase("slot_Shape") 

903 self.inputCat = lsst.afw.table.SourceCatalog(inputSchema) 

904 self.mapper = lsst.afw.table.SchemaMapper(inputSchema) 

905 self.transform = self.transformClass(self.control, self.name, self.mapper) 

906 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

907 

908 

909class ForcedPluginTransformSetupHelper: 

910 

911 def _setupTransform(self): 

912 self.control = self.controlClass() 

913 inputMapper = lsst.afw.table.SchemaMapper(lsst.afw.table.SourceTable.makeMinimalSchema(), 

914 lsst.afw.table.SourceTable.makeMinimalSchema()) 

915 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

916 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

917 # defining the schema. 

918 inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy") 

919 inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy") 

920 self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList()) 

921 inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid") 

922 inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape") 

923 self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema()) 

924 self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema()) 

925 self.transform = self.transformClass(self.control, self.name, self.mapper) 

926 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

927 

928 

929class FluxTransformTestCase(TransformTestCase): 

930 

931 def _setFieldsInRecords(self, records, name): 

932 for record in records: 

933 record[record.schema.join(name, 'instFlux')] = np.random.random() 

934 record[record.schema.join(name, 'instFluxErr')] = np.random.random() 

935 

936 # Negative instFluxes should be converted to NaNs. 

937 assert len(records) > 1 

938 records[0][record.schema.join(name, 'instFlux')] = -1 

939 

940 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

941 instFluxName = inSrc.schema.join(name, 'instFlux') 

942 instFluxErrName = inSrc.schema.join(name, 'instFluxErr') 

943 if inSrc[instFluxName] > 0: 

944 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

945 inSrc[instFluxErrName]) 

946 self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value) 

947 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

948 else: 

949 # negative instFlux results in NaN magnitude, but can still have finite error 

950 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')])) 

951 if np.isnan(inSrc[instFluxErrName]): 

952 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')])) 

953 else: 

954 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

955 inSrc[instFluxErrName]) 

956 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

957 

958 

959class CentroidTransformTestCase(TransformTestCase): 

960 

961 def _setFieldsInRecords(self, records, name): 

962 for record in records: 

963 record[record.schema.join(name, 'x')] = np.random.random() 

964 record[record.schema.join(name, 'y')] = np.random.random() 

965 # Some algorithms set no errors; some set only sigma on x & y; some provide 

966 # a full covariance matrix. Set only those which exist in the schema. 

967 for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'): 

968 fieldName = record.schema.join(name, fieldSuffix) 

969 if fieldName in record.schema: 

970 record[fieldName] = np.random.random() 

971 

972 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

973 centroidResultKey = CentroidResultKey(inSrc.schema[self.name]) 

974 centroidResult = centroidResultKey.get(inSrc) 

975 

976 coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc) 

977 coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid()) 

978 self.assertEqual(coordTruth, coord) 

979 

980 # If the centroid has an associated uncertainty matrix, the coordinate 

981 # must have one too, and vice versa. 

982 try: 

983 coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name], 

984 ["ra", "dec"]).get(outSrc) 

985 except lsst.pex.exceptions.NotFoundError: 

986 self.assertFalse(centroidResultKey.getCentroidErr().isValid()) 

987 else: 

988 transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians) 

989 coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(), 

990 centroidResult.getCentroidErr()), 

991 transform.getLinear().getMatrix().transpose()) 

992 np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)