Coverage for python/lsst/meas/base/tests.py: 14%

409 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-26 03:00 -0700

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24import numpy as np 

25 

26import lsst.geom 

27import lsst.afw.table 

28import lsst.afw.image 

29import lsst.afw.detection 

30import lsst.afw.geom 

31import lsst.afw.coord 

32import lsst.daf.base 

33import lsst.pex.exceptions 

34 

35from .sfm import SingleFrameMeasurementTask 

36from .forcedMeasurement import ForcedMeasurementTask 

37from ._measBaseLib import CentroidResultKey 

38 

39__all__ = ("BlendContext", "TestDataset", "AlgorithmTestCase", "TransformTestCase", 

40 "SingleFramePluginTransformSetupHelper", "ForcedPluginTransformSetupHelper", 

41 "FluxTransformTestCase", "CentroidTransformTestCase") 

42 

43 

44class BlendContext: 

45 """Context manager which adds multiple overlapping sources and a parent. 

46 

47 Notes 

48 ----- 

49 This is used as the return value for `TestDataset.addBlend`, and this is 

50 the only way it should be used. 

51 """ 

52 

53 def __init__(self, owner): 

54 self.owner = owner 

55 self.parentRecord = self.owner.catalog.addNew() 

56 self.parentImage = lsst.afw.image.ImageF(self.owner.exposure.getBBox()) 

57 self.children = [] 

58 

59 def __enter__(self): 

60 # BlendContext is its own context manager, so we just return self. 

61 return self 

62 

63 def addChild(self, instFlux, centroid, shape=None): 

64 """Add a child to the blend; return corresponding truth catalog record. 

65 

66 instFlux : `float` 

67 Total instFlux of the source to be added. 

68 centroid : `lsst.geom.Point2D` 

69 Position of the source to be added. 

70 shape : `lsst.afw.geom.Quadrupole` 

71 Second moments of the source before PSF convolution. Note that 

72 the truth catalog records post-convolution moments) 

73 """ 

74 record, image = self.owner.addSource(instFlux, centroid, shape) 

75 record.set(self.owner.keys["parent"], self.parentRecord.getId()) 

76 self.parentImage += image 

77 self.children.append((record, image)) 

78 return record 

79 

80 def __exit__(self, type_, value, tb): 

81 # We're not using the context manager for any kind of exception safety 

82 # or guarantees; we just want the nice "with" statement syntax. 

83 

84 if type_ is not None: 

85 # exception was raised; just skip all this and let it propagate 

86 return 

87 

88 # On exit, compute and set the truth values for the parent object. 

89 self.parentRecord.set(self.owner.keys["nChild"], len(self.children)) 

90 # Compute instFlux from sum of component fluxes 

91 instFlux = 0.0 

92 for record, image in self.children: 

93 instFlux += record.get(self.owner.keys["instFlux"]) 

94 self.parentRecord.set(self.owner.keys["instFlux"], instFlux) 

95 # Compute centroid from instFlux-weighted mean of component centroids 

96 x = 0.0 

97 y = 0.0 

98 for record, image in self.children: 

99 w = record.get(self.owner.keys["instFlux"])/instFlux 

100 x += record.get(self.owner.keys["centroid"].getX())*w 

101 y += record.get(self.owner.keys["centroid"].getY())*w 

102 self.parentRecord.set(self.owner.keys["centroid"], lsst.geom.Point2D(x, y)) 

103 # Compute shape from instFlux-weighted mean of offset component shapes 

104 xx = 0.0 

105 yy = 0.0 

106 xy = 0.0 

107 for record, image in self.children: 

108 w = record.get(self.owner.keys["instFlux"])/instFlux 

109 dx = record.get(self.owner.keys["centroid"].getX()) - x 

110 dy = record.get(self.owner.keys["centroid"].getY()) - y 

111 xx += (record.get(self.owner.keys["shape"].getIxx()) + dx**2)*w 

112 yy += (record.get(self.owner.keys["shape"].getIyy()) + dy**2)*w 

113 xy += (record.get(self.owner.keys["shape"].getIxy()) + dx*dy)*w 

114 self.parentRecord.set(self.owner.keys["shape"], lsst.afw.geom.Quadrupole(xx, yy, xy)) 

115 # Run detection on the parent image to get the parent Footprint. 

116 self.owner._installFootprint(self.parentRecord, self.parentImage) 

117 # Create perfect HeavyFootprints for all children; these will need to 

118 # be modified later to account for the noise we'll add to the image. 

119 deblend = lsst.afw.image.MaskedImageF(self.owner.exposure.maskedImage, True) 

120 for record, image in self.children: 

121 deblend.image.array[:, :] = image.array 

122 heavyFootprint = lsst.afw.detection.HeavyFootprintF(self.parentRecord.getFootprint(), deblend) 

123 record.setFootprint(heavyFootprint) 

124 

125 

126class TestDataset: 

127 """A simulated dataset consisuting of test image and truth catalog. 

128 

129 TestDataset creates an idealized image made of pure Gaussians (including a 

130 Gaussian PSF), with simple noise and idealized Footprints/HeavyFootprints 

131 that simulated the outputs of detection and deblending. Multiple noise 

132 realizations can be created from the same underlying sources, allowing 

133 uncertainty estimates to be verified via Monte Carlo. 

134 

135 Parameters 

136 ---------- 

137 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

138 Bounding box of the test image. 

139 threshold : `float` 

140 Threshold absolute value used to determine footprints for 

141 simulated sources. This thresholding will be applied before noise is 

142 actually added to images (or before the noise level is even known), so 

143 this will necessarily produce somewhat artificial footprints. 

144 exposure : `lsst.afw.image.ExposureF` 

145 The image to which test sources should be added. Ownership should 

146 be considered transferred from the caller to the TestDataset. 

147 Must have a Gaussian PSF for truth catalog shapes to be exact. 

148 **kwds 

149 Keyword arguments forwarded to makeEmptyExposure if exposure is `None`. 

150 

151 Notes 

152 ----- 

153 Typical usage: 

154 

155 .. code-block: py 

156 

157 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0,0), lsst.geom.Point2I(100, 

158 100)) 

159 dataset = TestDataset(bbox) 

160 dataset.addSource(instFlux=1E5, centroid=lsst.geom.Point2D(25, 26)) 

161 dataset.addSource(instFlux=2E5, centroid=lsst.geom.Point2D(75, 24), 

162 shape=lsst.afw.geom.Quadrupole(8, 7, 2)) 

163 with dataset.addBlend() as family: 

164 family.addChild(instFlux=2E5, centroid=lsst.geom.Point2D(50, 72)) 

165 family.addChild(instFlux=1.5E5, centroid=lsst.geom.Point2D(51, 74)) 

166 exposure, catalog = dataset.realize(noise=100.0, 

167 schema=TestDataset.makeMinimalSchema()) 

168 """ 

169 

170 def __init__(self, bbox, threshold=10.0, exposure=None, **kwds): 

171 if exposure is None: 

172 exposure = self.makeEmptyExposure(bbox, **kwds) 

173 self.threshold = lsst.afw.detection.Threshold(threshold, lsst.afw.detection.Threshold.VALUE) 

174 self.exposure = exposure 

175 self.psfShape = self.exposure.getPsf().computeShape(bbox.getCenter()) 

176 self.schema = self.makeMinimalSchema() 

177 self.catalog = lsst.afw.table.SourceCatalog(self.schema) 

178 

179 @classmethod 

180 def makeMinimalSchema(cls): 

181 """Return the minimal schema needed to hold truth catalog fields. 

182 

183 Notes 

184 ----- 

185 When `TestDataset.realize` is called, the schema must include at least 

186 these fields. Usually it will include additional fields for 

187 measurement algorithm outputs, allowing the same catalog to be used 

188 for both truth values (the fields from the minimal schema) and the 

189 measurements. 

190 """ 

191 if not hasattr(cls, "_schema"): 

192 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

193 cls.keys = {} 

194 cls.keys["coordErr"] = lsst.afw.table.CoordKey.addErrorFields(schema) 

195 cls.keys["parent"] = schema.find("parent").key 

196 cls.keys["nChild"] = schema.addField("deblend_nChild", type=np.int32) 

197 cls.keys["instFlux"] = schema.addField("truth_instFlux", type=np.float64, 

198 doc="true instFlux", units="count") 

199 cls.keys["instFluxErr"] = schema.addField("truth_instFluxErr", type=np.float64, 

200 doc="true instFluxErr", units="count") 

201 cls.keys["centroid"] = lsst.afw.table.Point2DKey.addFields( 

202 schema, "truth", "true simulated centroid", "pixel" 

203 ) 

204 cls.keys["centroid_sigma"] = lsst.afw.table.CovarianceMatrix2fKey.addFields( 

205 schema, "truth", ['x', 'y'], "pixel" 

206 ) 

207 cls.keys["centroid_flag"] = schema.addField("truth_flag", type="Flag", 

208 doc="set if the object is a star") 

209 cls.keys["shape"] = lsst.afw.table.QuadrupoleKey.addFields( 

210 schema, "truth", "true shape after PSF convolution", lsst.afw.table.CoordinateType.PIXEL 

211 ) 

212 cls.keys["isStar"] = schema.addField("truth_isStar", type="Flag", 

213 doc="set if the object is a star") 

214 schema.getAliasMap().set("slot_Shape", "truth") 

215 schema.getAliasMap().set("slot_Centroid", "truth") 

216 schema.getAliasMap().set("slot_ModelFlux", "truth") 

217 cls._schema = schema 

218 schema = lsst.afw.table.Schema(cls._schema) 

219 schema.disconnectAliases() 

220 return schema 

221 

222 @staticmethod 

223 def makePerturbedWcs(oldWcs, minScaleFactor=1.2, maxScaleFactor=1.5, 

224 minRotation=None, maxRotation=None, 

225 minRefShift=None, maxRefShift=None, 

226 minPixShift=2.0, maxPixShift=4.0, randomSeed=1): 

227 """Return a perturbed version of the input WCS. 

228 

229 Create a new undistorted TAN WCS that is similar but not identical to 

230 another, with random scaling, rotation, and offset (in both pixel 

231 position and reference position). 

232 

233 Parameters 

234 ---------- 

235 oldWcs : `lsst.afw.geom.SkyWcs` 

236 The input WCS. 

237 minScaleFactor : `float` 

238 Minimum scale factor to apply to the input WCS. 

239 maxScaleFactor : `float` 

240 Maximum scale factor to apply to the input WCS. 

241 minRotation : `lsst.geom.Angle` or `None` 

242 Minimum rotation to apply to the input WCS. If `None`, defaults to 

243 30 degrees. 

244 maxRotation : `lsst.geom.Angle` or `None` 

245 Minimum rotation to apply to the input WCS. If `None`, defaults to 

246 60 degrees. 

247 minRefShift : `lsst.geom.Angle` or `None` 

248 Miniumum shift to apply to the input WCS reference value. If 

249 `None`, defaults to 0.5 arcsec. 

250 maxRefShift : `lsst.geom.Angle` or `None` 

251 Miniumum shift to apply to the input WCS reference value. If 

252 `None`, defaults to 1.0 arcsec. 

253 minPixShift : `float` 

254 Minimum shift to apply to the input WCS reference pixel. 

255 maxPixShift : `float` 

256 Maximum shift to apply to the input WCS reference pixel. 

257 randomSeed : `int` 

258 Random seed. 

259 

260 Returns 

261 ------- 

262 newWcs : `lsst.afw.geom.SkyWcs` 

263 A perturbed version of the input WCS. 

264 

265 Notes 

266 ----- 

267 The maximum and minimum arguments are interpreted as absolute values 

268 for a split range that covers both positive and negative values (as 

269 this method is used in testing, it is typically most important to 

270 avoid perturbations near zero). Scale factors are treated somewhat 

271 differently: the actual scale factor is chosen between 

272 ``minScaleFactor`` and ``maxScaleFactor`` OR (``1/maxScaleFactor``) 

273 and (``1/minScaleFactor``). 

274 

275 The default range for rotation is 30-60 degrees, and the default range 

276 for reference shift is 0.5-1.0 arcseconds (these cannot be safely 

277 included directly as default values because Angle objects are 

278 mutable). 

279 

280 The random number generator is primed with the seed given. If 

281 `None`, a seed is automatically chosen. 

282 """ 

283 random_state = np.random.RandomState(randomSeed) 

284 if minRotation is None: 

285 minRotation = 30.0*lsst.geom.degrees 

286 if maxRotation is None: 

287 maxRotation = 60.0*lsst.geom.degrees 

288 if minRefShift is None: 

289 minRefShift = 0.5*lsst.geom.arcseconds 

290 if maxRefShift is None: 

291 maxRefShift = 1.0*lsst.geom.arcseconds 

292 

293 def splitRandom(min1, max1, min2=None, max2=None): 

294 if min2 is None: 

295 min2 = -max1 

296 if max2 is None: 

297 max2 = -min1 

298 if random_state.uniform() > 0.5: 

299 return float(random_state.uniform(min1, max1)) 

300 else: 

301 return float(random_state.uniform(min2, max2)) 

302 # Generate random perturbations 

303 scaleFactor = splitRandom(minScaleFactor, maxScaleFactor, 1.0/maxScaleFactor, 1.0/minScaleFactor) 

304 rotation = splitRandom(minRotation.asRadians(), maxRotation.asRadians())*lsst.geom.radians 

305 refShiftRa = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

306 refShiftDec = splitRandom(minRefShift.asRadians(), maxRefShift.asRadians())*lsst.geom.radians 

307 pixShiftX = splitRandom(minPixShift, maxPixShift) 

308 pixShiftY = splitRandom(minPixShift, maxPixShift) 

309 # Compute new CD matrix 

310 oldTransform = lsst.geom.LinearTransform(oldWcs.getCdMatrix()) 

311 rTransform = lsst.geom.LinearTransform.makeRotation(rotation) 

312 sTransform = lsst.geom.LinearTransform.makeScaling(scaleFactor) 

313 newTransform = oldTransform*rTransform*sTransform 

314 matrix = newTransform.getMatrix() 

315 # Compute new coordinate reference pixel (CRVAL) 

316 oldSkyOrigin = oldWcs.getSkyOrigin() 

317 newSkyOrigin = lsst.geom.SpherePoint(oldSkyOrigin.getRa() + refShiftRa, 

318 oldSkyOrigin.getDec() + refShiftDec) 

319 # Compute new pixel reference pixel (CRPIX) 

320 oldPixOrigin = oldWcs.getPixelOrigin() 

321 newPixOrigin = lsst.geom.Point2D(oldPixOrigin.getX() + pixShiftX, 

322 oldPixOrigin.getY() + pixShiftY) 

323 return lsst.afw.geom.makeSkyWcs(crpix=newPixOrigin, crval=newSkyOrigin, cdMatrix=matrix) 

324 

325 @staticmethod 

326 def makeEmptyExposure(bbox, wcs=None, crval=None, cdelt=None, psfSigma=2.0, psfDim=17, calibration=4, 

327 visitId=1234, mjd=60000.0): 

328 """Create an Exposure, with a PhotoCalib, Wcs, and Psf, but no pixel values. 

329 

330 Parameters 

331 ---------- 

332 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

333 Bounding box of the image in image coordinates. 

334 wcs : `lsst.afw.geom.SkyWcs`, optional 

335 New WCS for the exposure (created from CRVAL and CDELT if `None`). 

336 crval : `lsst.afw.geom.SpherePoint`, optional 

337 ICRS center of the TAN WCS attached to the image. If `None`, (45 

338 degrees, 45 degrees) is assumed. 

339 cdelt : `lsst.geom.Angle`, optional 

340 Pixel scale of the image. If `None`, 0.2 arcsec is assumed. 

341 psfSigma : `float`, optional 

342 Radius (sigma) of the Gaussian PSF attached to the image 

343 psfDim : `int`, optional 

344 Width and height of the image's Gaussian PSF attached to the image 

345 calibration : `float`, optional 

346 The spatially-constant calibration (in nJy/count) to set the 

347 PhotoCalib of the exposure. 

348 visitId : `int`, optional 

349 Visit id to store in VisitInfo. 

350 mjd : `float`, optional 

351 Modified Julian Date of this exposure to store in VisitInfo. 

352 

353 Returns 

354 ------- 

355 exposure : `lsst.age.image.ExposureF` 

356 An empty image. 

357 """ 

358 if wcs is None: 

359 if crval is None: 

360 crval = lsst.geom.SpherePoint(45.0, 45.0, lsst.geom.degrees) 

361 if cdelt is None: 

362 cdelt = 0.2*lsst.geom.arcseconds 

363 crpix = lsst.geom.Box2D(bbox).getCenter() 

364 wcs = lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, 

365 cdMatrix=lsst.afw.geom.makeCdMatrix(scale=cdelt)) 

366 exposure = lsst.afw.image.ExposureF(bbox) 

367 psf = lsst.afw.detection.GaussianPsf(psfDim, psfDim, psfSigma) 

368 photoCalib = lsst.afw.image.PhotoCalib(calibration) 

369 visitInfo = lsst.afw.image.VisitInfo(id=visitId, 

370 exposureTime=30.0, 

371 date=lsst.daf.base.DateTime(mjd), 

372 observatory=lsst.afw.coord.Observatory(11.1*lsst.geom.degrees, 

373 22.2*lsst.geom.degrees, 

374 0.333), 

375 hasSimulatedContent=True) 

376 exposure.setWcs(wcs) 

377 exposure.setPsf(psf) 

378 exposure.setPhotoCalib(photoCalib) 

379 exposure.info.setVisitInfo(visitInfo) 

380 return exposure 

381 

382 @staticmethod 

383 def drawGaussian(bbox, instFlux, ellipse): 

384 """Create an image of an elliptical Gaussian. 

385 

386 Parameters 

387 ---------- 

388 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D` 

389 Bounding box of image to create. 

390 instFlux : `float` 

391 Total instrumental flux of the Gaussian (normalized analytically, 

392 not using pixel values). 

393 ellipse : `lsst.afw.geom.Ellipse` 

394 Defines the centroid and shape. 

395 

396 Returns 

397 ------- 

398 image : `lsst.afw.image.ImageF` 

399 An image of the Gaussian. 

400 """ 

401 x, y = np.meshgrid(np.arange(bbox.getBeginX(), bbox.getEndX()), 

402 np.arange(bbox.getBeginY(), bbox.getEndY())) 

403 t = ellipse.getGridTransform() 

404 xt = t[t.XX] * x + t[t.XY] * y + t[t.X] 

405 yt = t[t.YX] * x + t[t.YY] * y + t[t.Y] 

406 image = lsst.afw.image.ImageF(bbox) 

407 image.array[:, :] = np.exp(-0.5*(xt**2 + yt**2))*instFlux/(2.0*ellipse.getCore().getArea()) 

408 return image 

409 

410 def _installFootprint(self, record, image, setPeakSignificance=True): 

411 """Create simulated Footprint and add it to a truth catalog record. 

412 """ 

413 schema = lsst.afw.detection.PeakTable.makeMinimalSchema() 

414 if setPeakSignificance: 

415 schema.addField("significance", type=float, 

416 doc="Ratio of peak value to configured standard deviation.") 

417 # Run detection on the single-source image 

418 fpSet = lsst.afw.detection.FootprintSet(image, self.threshold, peakSchema=schema) 

419 # the call below to the FootprintSet ctor is actually a grow operation 

420 fpSet = lsst.afw.detection.FootprintSet(fpSet, int(self.psfShape.getDeterminantRadius() + 1.0), True) 

421 if setPeakSignificance: 

422 # This isn't a traditional significance, since we're using the VALUE 

423 # threshold type, but it's the best we can do in that case. 

424 for footprint in fpSet.getFootprints(): 

425 footprint.updatePeakSignificance(self.threshold.getValue()) 

426 # Update the full exposure's mask plane to indicate the detection 

427 fpSet.setMask(self.exposure.mask, "DETECTED") 

428 # Attach the new footprint to the exposure 

429 if len(fpSet.getFootprints()) > 1: 

430 raise RuntimeError("Threshold value results in multiple Footprints for a single object") 

431 if len(fpSet.getFootprints()) == 0: 

432 raise RuntimeError("Threshold value results in zero Footprints for object") 

433 record.setFootprint(fpSet.getFootprints()[0]) 

434 

435 def addSource(self, instFlux, centroid, shape=None, setPeakSignificance=True): 

436 """Add a source to the simulation. 

437 

438 To insert a point source with a given signal-to-noise (sn), the total 

439 ``instFlux`` should be: ``sn*noise*psf_scale``, where ``noise`` is the 

440 noise you will pass to ``realize()``, and 

441 ``psf_scale=sqrt(4*pi*r^2)``, where ``r`` is the width of the PSF. 

442 

443 Parameters 

444 ---------- 

445 instFlux : `float` 

446 Total instFlux of the source to be added. 

447 centroid : `lsst.geom.Point2D` 

448 Position of the source to be added. 

449 shape : `lsst.afw.geom.Quadrupole` 

450 Second moments of the source before PSF convolution. Note that the 

451 truth catalog records post-convolution moments. If `None`, a point 

452 source will be added. 

453 setPeakSignificance : `bool` 

454 Set the ``significance`` field for peaks in the footprints? 

455 See ``lsst.meas.algorithms.SourceDetectionTask.setPeakSignificance`` 

456 for how this field is computed for real datasets. 

457 

458 Returns 

459 ------- 

460 record : `lsst.afw.table.SourceRecord` 

461 A truth catalog record. 

462 image : `lsst.afw.image.ImageF` 

463 Single-source image corresponding to the new source. 

464 """ 

465 # Create and set the truth catalog fields 

466 record = self.catalog.addNew() 

467 record.set(self.keys["instFlux"], instFlux) 

468 record.set(self.keys["instFluxErr"], 0) 

469 record.set(self.keys["centroid"], centroid) 

470 covariance = np.random.normal(0, 0.1, 4).reshape(2, 2) 

471 covariance[0, 1] = covariance[1, 0] # CovarianceMatrixKey assumes symmetric x_y_Cov 

472 record.set(self.keys["centroid_sigma"], covariance.astype(np.float32)) 

473 if shape is None: 

474 record.set(self.keys["isStar"], True) 

475 fullShape = self.psfShape 

476 else: 

477 record.set(self.keys["isStar"], False) 

478 fullShape = shape.convolve(self.psfShape) 

479 record.set(self.keys["shape"], fullShape) 

480 # Create an image containing just this source 

481 image = self.drawGaussian(self.exposure.getBBox(), instFlux, 

482 lsst.afw.geom.Ellipse(fullShape, centroid)) 

483 # Generate a footprint for this source 

484 self._installFootprint(record, image, setPeakSignificance) 

485 # Actually add the source to the full exposure 

486 self.exposure.image.array[:, :] += image.array 

487 return record, image 

488 

489 def addBlend(self): 

490 """Return a context manager which can add a blend of multiple sources. 

491 

492 Notes 

493 ----- 

494 Note that nothing stops you from creating overlapping sources just using the addSource() method, 

495 but addBlend() is necesssary to create a parent object and deblended HeavyFootprints of the type 

496 produced by the detection and deblending pipelines. 

497 

498 Examples 

499 -------- 

500 .. code-block: py 

501 d = TestDataset(...) 

502 with d.addBlend() as b: 

503 b.addChild(flux1, centroid1) 

504 b.addChild(flux2, centroid2, shape2) 

505 """ 

506 return BlendContext(self) 

507 

508 def transform(self, wcs, **kwds): 

509 """Copy this dataset transformed to a new WCS, with new Psf and PhotoCalib. 

510 

511 Parameters 

512 ---------- 

513 wcs : `lsst.afw.geom.SkyWcs` 

514 WCS for the new dataset. 

515 **kwds 

516 Additional keyword arguments passed on to 

517 `TestDataset.makeEmptyExposure`. If not specified, these revert 

518 to the defaults for `~TestDataset.makeEmptyExposure`, not the 

519 values in the current dataset. 

520 

521 Returns 

522 ------- 

523 newDataset : `TestDataset` 

524 Transformed copy of this dataset. 

525 """ 

526 bboxD = lsst.geom.Box2D() 

527 xyt = lsst.afw.geom.makeWcsPairTransform(self.exposure.getWcs(), wcs) 

528 for corner in lsst.geom.Box2D(self.exposure.getBBox()).getCorners(): 

529 bboxD.include(xyt.applyForward(lsst.geom.Point2D(corner))) 

530 bboxI = lsst.geom.Box2I(bboxD) 

531 result = TestDataset(bbox=bboxI, wcs=wcs, **kwds) 

532 oldPhotoCalib = self.exposure.getPhotoCalib() 

533 newPhotoCalib = result.exposure.getPhotoCalib() 

534 oldPsfShape = self.exposure.getPsf().computeShape(bboxD.getCenter()) 

535 for record in self.catalog: 

536 if record.get(self.keys["nChild"]): 

537 raise NotImplementedError("Transforming blended sources in TestDatasets is not supported") 

538 magnitude = oldPhotoCalib.instFluxToMagnitude(record.get(self.keys["instFlux"])) 

539 newFlux = newPhotoCalib.magnitudeToInstFlux(magnitude) 

540 oldCentroid = record.get(self.keys["centroid"]) 

541 newCentroid = xyt.applyForward(oldCentroid) 

542 if record.get(self.keys["isStar"]): 

543 newDeconvolvedShape = None 

544 else: 

545 affine = lsst.afw.geom.linearizeTransform(xyt, oldCentroid) 

546 oldFullShape = record.get(self.keys["shape"]) 

547 oldDeconvolvedShape = lsst.afw.geom.Quadrupole( 

548 oldFullShape.getIxx() - oldPsfShape.getIxx(), 

549 oldFullShape.getIyy() - oldPsfShape.getIyy(), 

550 oldFullShape.getIxy() - oldPsfShape.getIxy(), 

551 False 

552 ) 

553 newDeconvolvedShape = oldDeconvolvedShape.transform(affine.getLinear()) 

554 result.addSource(newFlux, newCentroid, newDeconvolvedShape) 

555 return result 

556 

557 def realize(self, noise, schema, randomSeed=1): 

558 r"""Simulate an exposure and detection catalog for this dataset. 

559 

560 The simulation includes noise, and the detection catalog includes 

561 `~lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s. 

562 

563 Parameters 

564 ---------- 

565 noise : `float` 

566 Standard deviation of noise to be added to the exposure. The 

567 noise will be Gaussian and constant, appropriate for the 

568 sky-limited regime. 

569 schema : `lsst.afw.table.Schema` 

570 Schema of the new catalog to be created. Must start with 

571 ``self.schema`` (i.e. ``schema.contains(self.schema)`` must be 

572 `True`), but typically contains fields for already-configured 

573 measurement algorithms as well. 

574 randomSeed : `int`, optional 

575 Seed for the random number generator. 

576 If `None`, a seed is chosen automatically. 

577 

578 Returns 

579 ------- 

580 `exposure` : `lsst.afw.image.ExposureF` 

581 Simulated image. 

582 `catalog` : `lsst.afw.table.SourceCatalog` 

583 Simulated detection catalog. 

584 """ 

585 random_state = np.random.RandomState(randomSeed) 

586 assert schema.contains(self.schema) 

587 mapper = lsst.afw.table.SchemaMapper(self.schema) 

588 mapper.addMinimalSchema(self.schema, True) 

589 exposure = self.exposure.clone() 

590 exposure.variance.array[:, :] = noise**2 

591 exposure.image.array[:, :] += random_state.randn(exposure.height, exposure.width)*noise 

592 catalog = lsst.afw.table.SourceCatalog(schema) 

593 catalog.extend(self.catalog, mapper=mapper) 

594 # Loop over sources and generate new HeavyFootprints that divide up 

595 # the noisy pixels, not the ideal no-noise pixels. 

596 for record in catalog: 

597 # parent objects have non-Heavy Footprints, which don't need to be 

598 # updated after adding noise. 

599 if record.getParent() == 0: 

600 continue 

601 # get flattened arrays that correspond to the no-noise and noisy 

602 # parent images 

603 parent = catalog.find(record.getParent()) 

604 footprint = parent.getFootprint() 

605 parentFluxArrayNoNoise = np.zeros(footprint.getArea(), dtype=np.float32) 

606 footprint.spans.flatten(parentFluxArrayNoNoise, self.exposure.image.array, self.exposure.getXY0()) 

607 parentFluxArrayNoisy = np.zeros(footprint.getArea(), dtype=np.float32) 

608 footprint.spans.flatten(parentFluxArrayNoisy, exposure.image.array, exposure.getXY0()) 

609 oldHeavy = record.getFootprint() 

610 fraction = (oldHeavy.getImageArray() / parentFluxArrayNoNoise) 

611 # N.B. this isn't a copy ctor - it's a copy from a vanilla 

612 # Footprint, so it doesn't copy the arrays we don't want to 

613 # change, and hence we have to do that ourselves below. 

614 newHeavy = lsst.afw.detection.HeavyFootprintF(oldHeavy) 

615 newHeavy.getImageArray()[:] = parentFluxArrayNoisy*fraction 

616 newHeavy.getMaskArray()[:] = oldHeavy.getMaskArray() 

617 newHeavy.getVarianceArray()[:] = oldHeavy.getVarianceArray() 

618 record.setFootprint(newHeavy) 

619 lsst.afw.table.updateSourceCoords(exposure.wcs, catalog) 

620 return exposure, catalog 

621 

622 

623class AlgorithmTestCase: 

624 """Base class for tests of measurement tasks. 

625 """ 

626 def makeSingleFrameMeasurementConfig(self, plugin=None, dependencies=()): 

627 """Create an instance of `SingleFrameMeasurementTask.ConfigClass`. 

628 

629 Only the specified plugin and its dependencies will be run; the 

630 Centroid, Shape, and ModelFlux slots will be set to the truth fields 

631 generated by the `TestDataset` class. 

632 

633 Parameters 

634 ---------- 

635 plugin : `str` 

636 Name of measurement plugin to enable. 

637 dependencies : iterable of `str`, optional 

638 Names of dependencies of the measurement plugin. 

639 

640 Returns 

641 ------- 

642 config : `SingleFrameMeasurementTask.ConfigClass` 

643 The resulting task configuration. 

644 """ 

645 config = SingleFrameMeasurementTask.ConfigClass() 

646 with warnings.catch_warnings(): 

647 warnings.filterwarnings("ignore", message="ignoreSlotPluginChecks", category=FutureWarning) 

648 config = SingleFrameMeasurementTask.ConfigClass(ignoreSlotPluginChecks=True) 

649 config.slots.centroid = "truth" 

650 config.slots.shape = "truth" 

651 config.slots.modelFlux = None 

652 config.slots.apFlux = None 

653 config.slots.psfFlux = None 

654 config.slots.gaussianFlux = None 

655 config.slots.calibFlux = None 

656 config.plugins.names = (plugin,) + tuple(dependencies) 

657 return config 

658 

659 def makeSingleFrameMeasurementTask(self, plugin=None, dependencies=(), config=None, schema=None, 

660 algMetadata=None): 

661 """Create a configured instance of `SingleFrameMeasurementTask`. 

662 

663 Parameters 

664 ---------- 

665 plugin : `str`, optional 

666 Name of measurement plugin to enable. If `None`, a configuration 

667 must be supplied as the ``config`` parameter. If both are 

668 specified, ``config`` takes precedence. 

669 dependencies : iterable of `str`, optional 

670 Names of dependencies of the specified measurement plugin. 

671 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

672 Configuration for the task. If `None`, a measurement plugin must 

673 be supplied as the ``plugin`` paramter. If both are specified, 

674 ``config`` takes precedence. 

675 schema : `lsst.afw.table.Schema`, optional 

676 Measurement table schema. If `None`, a default schema is 

677 generated. 

678 algMetadata : `lsst.daf.base.PropertyList`, optional 

679 Measurement algorithm metadata. If `None`, a default container 

680 will be generated. 

681 

682 Returns 

683 ------- 

684 task : `SingleFrameMeasurementTask` 

685 A configured instance of the measurement task. 

686 """ 

687 if config is None: 

688 if plugin is None: 

689 raise ValueError("Either plugin or config argument must not be None") 

690 config = self.makeSingleFrameMeasurementConfig(plugin=plugin, dependencies=dependencies) 

691 if schema is None: 

692 schema = TestDataset.makeMinimalSchema() 

693 # Clear all aliases so only those defined by config are set. 

694 schema.setAliasMap(None) 

695 if algMetadata is None: 

696 algMetadata = lsst.daf.base.PropertyList() 

697 return SingleFrameMeasurementTask(schema=schema, algMetadata=algMetadata, config=config) 

698 

699 def makeForcedMeasurementConfig(self, plugin=None, dependencies=()): 

700 """Create an instance of `ForcedMeasurementTask.ConfigClass`. 

701 

702 In addition to the plugins specified in the plugin and dependencies 

703 arguments, the `TransformedCentroid` and `TransformedShape` plugins 

704 will be run and used as the centroid and shape slots; these simply 

705 transform the reference catalog centroid and shape to the measurement 

706 coordinate system. 

707 

708 Parameters 

709 ---------- 

710 plugin : `str` 

711 Name of measurement plugin to enable. 

712 dependencies : iterable of `str`, optional 

713 Names of dependencies of the measurement plugin. 

714 

715 Returns 

716 ------- 

717 config : `ForcedMeasurementTask.ConfigClass` 

718 The resulting task configuration. 

719 """ 

720 

721 config = ForcedMeasurementTask.ConfigClass() 

722 config.slots.centroid = "base_TransformedCentroid" 

723 config.slots.shape = "base_TransformedShape" 

724 config.slots.modelFlux = None 

725 config.slots.apFlux = None 

726 config.slots.psfFlux = None 

727 config.slots.gaussianFlux = None 

728 config.plugins.names = (plugin,) + tuple(dependencies) + ("base_TransformedCentroid", 

729 "base_TransformedShape") 

730 return config 

731 

732 def makeForcedMeasurementTask(self, plugin=None, dependencies=(), config=None, refSchema=None, 

733 algMetadata=None): 

734 """Create a configured instance of `ForcedMeasurementTask`. 

735 

736 Parameters 

737 ---------- 

738 plugin : `str`, optional 

739 Name of measurement plugin to enable. If `None`, a configuration 

740 must be supplied as the ``config`` parameter. If both are 

741 specified, ``config`` takes precedence. 

742 dependencies : iterable of `str`, optional 

743 Names of dependencies of the specified measurement plugin. 

744 config : `SingleFrameMeasurementTask.ConfigClass`, optional 

745 Configuration for the task. If `None`, a measurement plugin must 

746 be supplied as the ``plugin`` paramter. If both are specified, 

747 ``config`` takes precedence. 

748 refSchema : `lsst.afw.table.Schema`, optional 

749 Reference table schema. If `None`, a default schema is 

750 generated. 

751 algMetadata : `lsst.daf.base.PropertyList`, optional 

752 Measurement algorithm metadata. If `None`, a default container 

753 will be generated. 

754 

755 Returns 

756 ------- 

757 task : `ForcedMeasurementTask` 

758 A configured instance of the measurement task. 

759 """ 

760 if config is None: 

761 if plugin is None: 

762 raise ValueError("Either plugin or config argument must not be None") 

763 config = self.makeForcedMeasurementConfig(plugin=plugin, dependencies=dependencies) 

764 if refSchema is None: 

765 refSchema = TestDataset.makeMinimalSchema() 

766 if algMetadata is None: 

767 algMetadata = lsst.daf.base.PropertyList() 

768 return ForcedMeasurementTask(refSchema=refSchema, algMetadata=algMetadata, config=config) 

769 

770 

771class TransformTestCase: 

772 """Base class for testing measurement transformations. 

773 

774 Notes 

775 ----- 

776 We test both that the transform itself operates successfully (fluxes are 

777 converted to magnitudes, flags are propagated properly) and that the 

778 transform is registered as the default for the appropriate measurement 

779 algorithms. 

780 

781 In the simple case of one-measurement-per-transformation, the developer 

782 need not directly write any tests themselves: simply customizing the class 

783 variables is all that is required. More complex measurements (e.g. 

784 multiple aperture fluxes) require extra effort. 

785 """ 

786 name = "MeasurementTransformTest" 

787 """The name used for the measurement algorithm (str). 

788 

789 Notes 

790 ----- 

791 This determines the names of the fields in the resulting catalog. This 

792 default should generally be fine, but subclasses can override if 

793 required. 

794 """ 

795 

796 # These should be customized by subclassing. 

797 controlClass = None 

798 algorithmClass = None 

799 transformClass = None 

800 

801 flagNames = ("flag",) 

802 """Flags which may be set by the algorithm being tested (iterable of `str`). 

803 """ 

804 

805 # The plugin being tested should be registered under these names for 

806 # single frame and forced measurement. Should be customized by 

807 # subclassing. 

808 singleFramePlugins = () 

809 forcedPlugins = () 

810 

811 def setUp(self): 

812 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(200, 200)) 

813 self.calexp = TestDataset.makeEmptyExposure(bbox) 

814 self._setupTransform() 

815 

816 def tearDown(self): 

817 del self.calexp 

818 del self.inputCat 

819 del self.mapper 

820 del self.transform 

821 del self.outputCat 

822 

823 def _populateCatalog(self, baseNames): 

824 records = [] 

825 for flagValue in (True, False): 

826 records.append(self.inputCat.addNew()) 

827 for baseName in baseNames: 

828 for flagName in self.flagNames: 

829 if records[-1].schema.join(baseName, flagName) in records[-1].schema: 

830 records[-1].set(records[-1].schema.join(baseName, flagName), flagValue) 

831 self._setFieldsInRecords(records, baseName) 

832 

833 def _checkOutput(self, baseNames): 

834 for inSrc, outSrc in zip(self.inputCat, self.outputCat): 

835 for baseName in baseNames: 

836 self._compareFieldsInRecords(inSrc, outSrc, baseName) 

837 for flagName in self.flagNames: 

838 keyName = outSrc.schema.join(baseName, flagName) 

839 if keyName in inSrc.schema: 

840 self.assertEqual(outSrc.get(keyName), inSrc.get(keyName)) 

841 else: 

842 self.assertFalse(keyName in outSrc.schema) 

843 

844 def _runTransform(self, doExtend=True): 

845 if doExtend: 

846 self.outputCat.extend(self.inputCat, mapper=self.mapper) 

847 self.transform(self.inputCat, self.outputCat, self.calexp.getWcs(), self.calexp.getPhotoCalib()) 

848 

849 def testTransform(self, baseNames=None): 

850 """Test the transformation on a catalog containing random data. 

851 

852 Parameters 

853 ---------- 

854 baseNames : iterable of `str` 

855 Iterable of the initial parts of measurement field names. 

856 

857 Notes 

858 ----- 

859 We check that: 

860 

861 - An appropriate exception is raised on an attempt to transform 

862 between catalogs with different numbers of rows; 

863 - Otherwise, all appropriate conversions are properly appled and that 

864 flags have been propagated. 

865 

866 The ``baseNames`` argument requires some explanation. This should be 

867 an iterable of the leading parts of the field names for each 

868 measurement; that is, everything that appears before ``_instFlux``, 

869 ``_flag``, etc. In the simple case of a single measurement per plugin, 

870 this is simply equal to ``self.name`` (thus measurements are stored as 

871 ``self.name + "_instFlux"``, etc). More generally, the developer may 

872 specify whatever iterable they require. For example, to handle 

873 multiple apertures, we could have ``(self.name + "_0", self.name + 

874 "_1", ...)``. 

875 """ 

876 baseNames = baseNames or [self.name] 

877 self._populateCatalog(baseNames) 

878 self.assertRaises(lsst.pex.exceptions.LengthError, self._runTransform, False) 

879 self._runTransform() 

880 self._checkOutput(baseNames) 

881 

882 def _checkRegisteredTransform(self, registry, name): 

883 # If this is a Python-based transform, we can compare directly; if 

884 # it's wrapped C++, we need to compare the wrapped class. 

885 self.assertEqual(registry[name].PluginClass.getTransformClass(), self.transformClass) 

886 

887 def testRegistration(self): 

888 """Test that the transformation is appropriately registered. 

889 """ 

890 for pluginName in self.singleFramePlugins: 

891 self._checkRegisteredTransform(lsst.meas.base.SingleFramePlugin.registry, pluginName) 

892 for pluginName in self.forcedPlugins: 

893 self._checkRegisteredTransform(lsst.meas.base.ForcedPlugin.registry, pluginName) 

894 

895 

896class SingleFramePluginTransformSetupHelper: 

897 

898 def _setupTransform(self): 

899 self.control = self.controlClass() 

900 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema() 

901 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

902 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

903 # defining the schema. 

904 inputSchema.getAliasMap().set("slot_Centroid", "dummy") 

905 inputSchema.getAliasMap().set("slot_Shape", "dummy") 

906 self.algorithmClass(self.control, self.name, inputSchema) 

907 inputSchema.getAliasMap().erase("slot_Centroid") 

908 inputSchema.getAliasMap().erase("slot_Shape") 

909 self.inputCat = lsst.afw.table.SourceCatalog(inputSchema) 

910 self.mapper = lsst.afw.table.SchemaMapper(inputSchema) 

911 self.transform = self.transformClass(self.control, self.name, self.mapper) 

912 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

913 

914 

915class ForcedPluginTransformSetupHelper: 

916 

917 def _setupTransform(self): 

918 self.control = self.controlClass() 

919 inputMapper = lsst.afw.table.SchemaMapper(lsst.afw.table.SourceTable.makeMinimalSchema(), 

920 lsst.afw.table.SourceTable.makeMinimalSchema()) 

921 # Trick algorithms that depend on the slot centroid or alias into thinking they've been defined; 

922 # it doesn't matter for this test since we won't actually use the plugins for anything besides 

923 # defining the schema. 

924 inputMapper.editOutputSchema().getAliasMap().set("slot_Centroid", "dummy") 

925 inputMapper.editOutputSchema().getAliasMap().set("slot_Shape", "dummy") 

926 self.algorithmClass(self.control, self.name, inputMapper, lsst.daf.base.PropertyList()) 

927 inputMapper.editOutputSchema().getAliasMap().erase("slot_Centroid") 

928 inputMapper.editOutputSchema().getAliasMap().erase("slot_Shape") 

929 self.inputCat = lsst.afw.table.SourceCatalog(inputMapper.getOutputSchema()) 

930 self.mapper = lsst.afw.table.SchemaMapper(inputMapper.getOutputSchema()) 

931 self.transform = self.transformClass(self.control, self.name, self.mapper) 

932 self.outputCat = lsst.afw.table.BaseCatalog(self.mapper.getOutputSchema()) 

933 

934 

935class FluxTransformTestCase(TransformTestCase): 

936 

937 def _setFieldsInRecords(self, records, name): 

938 for record in records: 

939 record[record.schema.join(name, 'instFlux')] = np.random.random() 

940 record[record.schema.join(name, 'instFluxErr')] = np.random.random() 

941 

942 # Negative instFluxes should be converted to NaNs. 

943 assert len(records) > 1 

944 records[0][record.schema.join(name, 'instFlux')] = -1 

945 

946 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

947 instFluxName = inSrc.schema.join(name, 'instFlux') 

948 instFluxErrName = inSrc.schema.join(name, 'instFluxErr') 

949 if inSrc[instFluxName] > 0: 

950 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

951 inSrc[instFluxErrName]) 

952 self.assertEqual(outSrc[outSrc.schema.join(name, 'mag')], mag.value) 

953 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

954 else: 

955 # negative instFlux results in NaN magnitude, but can still have finite error 

956 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'mag')])) 

957 if np.isnan(inSrc[instFluxErrName]): 

958 self.assertTrue(np.isnan(outSrc[outSrc.schema.join(name, 'magErr')])) 

959 else: 

960 mag = self.calexp.getPhotoCalib().instFluxToMagnitude(inSrc[instFluxName], 

961 inSrc[instFluxErrName]) 

962 self.assertEqual(outSrc[outSrc.schema.join(name, 'magErr')], mag.error) 

963 

964 

965class CentroidTransformTestCase(TransformTestCase): 

966 

967 def _setFieldsInRecords(self, records, name): 

968 for record in records: 

969 record[record.schema.join(name, 'x')] = np.random.random() 

970 record[record.schema.join(name, 'y')] = np.random.random() 

971 # Some algorithms set no errors; some set only sigma on x & y; some provide 

972 # a full covariance matrix. Set only those which exist in the schema. 

973 for fieldSuffix in ('xErr', 'yErr', 'x_y_Cov'): 

974 fieldName = record.schema.join(name, fieldSuffix) 

975 if fieldName in record.schema: 

976 record[fieldName] = np.random.random() 

977 

978 def _compareFieldsInRecords(self, inSrc, outSrc, name): 

979 centroidResultKey = CentroidResultKey(inSrc.schema[self.name]) 

980 centroidResult = centroidResultKey.get(inSrc) 

981 

982 coord = lsst.afw.table.CoordKey(outSrc.schema[self.name]).get(outSrc) 

983 coordTruth = self.calexp.getWcs().pixelToSky(centroidResult.getCentroid()) 

984 self.assertEqual(coordTruth, coord) 

985 

986 # If the centroid has an associated uncertainty matrix, the coordinate 

987 # must have one too, and vice versa. 

988 try: 

989 coordErr = lsst.afw.table.CovarianceMatrix2fKey(outSrc.schema[self.name], 

990 ["ra", "dec"]).get(outSrc) 

991 except lsst.pex.exceptions.NotFoundError: 

992 self.assertFalse(centroidResultKey.getCentroidErr().isValid()) 

993 else: 

994 transform = self.calexp.getWcs().linearizePixelToSky(coordTruth, lsst.geom.radians) 

995 coordErrTruth = np.dot(np.dot(transform.getLinear().getMatrix(), 

996 centroidResult.getCentroidErr()), 

997 transform.getLinear().getMatrix().transpose()) 

998 np.testing.assert_array_almost_equal(np.array(coordErrTruth), coordErr)