Coverage for python/lsst/jointcal/testUtils.py: 17%

99 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-27 03:04 -0700

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Functions to help create jointcal tests by generating fake data.""" 

23 

24__all__ = ['createFakeCatalog', 'createTwoFakeCcdImages', 'getMeasuredStarsFromCatalog'] 

25 

26import os 

27import unittest 

28 

29import numpy as np 

30 

31import lsst.afw.geom 

32import lsst.afw.table 

33import lsst.daf.butler 

34import lsst.pipe.base 

35 

36import lsst.jointcal.star 

37 

38 

39def canRunTests(): 

40 """Returns True if the necessary packages and files are available. 

41 

42 We need ``obs_cfht`` to load the test/data/cfht_minimal dataset, which 

43 includes the metadata that is used to build the fake catalogs. 

44 """ 

45 try: 

46 import lsst.obs.cfht # noqa: F401 

47 return True 

48 except ImportError: 

49 return False 

50 

51 

52def createTwoFakeCcdImages(num1=4, num2=4, seed=100, fakeDetectorId=12, 

53 photoCalibMean1=1e-2, photoCalibMean2=1.2e-2, 

54 fakeWcses=(None, None), 

55 fakeVisitInfos=(None, None)): 

56 """Return two fake ccdImages built on CFHT Megacam metadata. 

57 

58 If ``num1 == num2``, the catalogs will align on-sky so each source will 

59 have a match in the other catalog. 

60 

61 This uses the butler dataset stored in `tests/data/cfht_minimal` to 

62 bootstrap the metadata. 

63 

64 Parameters 

65 ---------- 

66 num1, num2 : `int`, optional 

67 Number of sources to put in the first and second catalogs. Should be 

68 a square, to have sqrt(num) centroids on a grid. 

69 seed : `int`, optional 

70 Seed value for np.random. 

71 fakeDetectorId : `int`, optional 

72 Sensor identifier to use for both CcdImages. The wcs, bbox, photoCalib, etc. 

73 will still be drawn from the CFHT ccd=12 files, as that is the only 

74 testdata that is included in this simple test dataset. 

75 photoCalibMean1, photoCalibMean2: `float`, optional 

76 The mean photometric calibration to pass to each ccdImage construction. 

77 Note: this value is 1/instFluxMag0, so it should be less than 1. 

78 fakeWcses : `list` [`lsst.afw.geom.SkyWcs`], optional 

79 The SkyWcses to use instead of the ones read from disk. 

80 fakeWcses : `list` [`lsst.afw.image.VisitInfo`], optional 

81 The VisitInfos to use instead of the ones read from disk. 

82 

83 Returns 

84 ------- 

85 struct : `lsst.pipe.base.Struct` 

86 Result struct with components: 

87 

88 - `camera` : Camera representing these catalogs 

89 (`lsst.afw.cameraGeom.Camera`). 

90 - `catalogs` : Catalogs containing fake sources 

91 (`list` of `lsst.afw.table.SourceCatalog`). 

92 - `ccdImageList` : CcdImages containing the metadata and fake sources 

93 (`list` of `lsst.jointcal.CcdImage`). 

94 - `bbox` : Bounding Box of the image (`lsst.geom.Box2I`). 

95 - 'fluxFieldName' : name of the instFlux field in the catalogs ('str'). 

96 """ 

97 if not canRunTests(): 

98 msg = "Necessary packages not available to run tests that use the cfht_minimal dataset." 

99 raise unittest.SkipTest(msg) 

100 

101 np.random.seed(seed) 

102 

103 visit1 = 849375 

104 visit2 = 850587 

105 fluxFieldName = "SomeFlux" 

106 

107 # Load or fake the necessary metadata for each CcdImage 

108 dataDir = lsst.utils.getPackageDir('jointcal') 

109 inputDir = os.path.join(dataDir, 'tests/data/cfht_minimal/repo') 

110 # Ensure this butler is not writeable, so that we don't mess up the repo accidentally. 

111 butler = lsst.daf.butler.Butler(inputDir, collections=["singleFrame"], writeable=False) 

112 

113 # so we can access parts of the camera later (e.g. focal plane) 

114 camera = butler.get('camera', instrument="MegaPrime") 

115 

116 struct1 = createFakeCcdImage(butler, visit1, num1, fluxFieldName, 

117 photoCalibMean=photoCalibMean1, photoCalibErr=1.0, 

118 fakeDetectorId=fakeDetectorId, 

119 fakeWcs=fakeWcses[0], fakeVisitInfo=fakeVisitInfos[0]) 

120 struct2 = createFakeCcdImage(butler, visit2, num2, fluxFieldName, 

121 photoCalibMean=photoCalibMean2, photoCalibErr=5.0, 

122 fakeDetectorId=fakeDetectorId, 

123 fakeWcs=fakeWcses[1], fakeVisitInfo=fakeVisitInfos[1]) 

124 

125 return lsst.pipe.base.Struct(camera=camera, 

126 catalogs=[struct1.catalog, struct2.catalog], 

127 ccdImageList=[struct1.ccdImage, struct2.ccdImage], 

128 bbox=struct1.bbox, 

129 skyWcs=[struct1.skyWcs, struct2.skyWcs], 

130 fluxFieldName=fluxFieldName) 

131 

132 

133def createFakeCcdImage(butler, visit, num, fluxFieldName, 

134 photoCalibMean=1e-2, photoCalibErr=1.0, fakeDetectorId=12, 

135 fakeWcs=None, fakeVisitInfo=None): 

136 """Create a fake CcdImage by making a fake catalog. 

137 

138 Parameters 

139 ---------- 

140 butler : `lsst.daf.butler.Butler` 

141 Butler to load metadata from. 

142 visit : `int` 

143 Visit identifier to build a butler dataId. 

144 num : `int` 

145 Number of sources to put in the catalogs. Should be 

146 a square, to have sqrt(num) centroids on a grid. 

147 fluxFieldName : `str` 

148 Name of the flux field to populate in the catalog, without `_instFlux` 

149 (e.g. "slot_CalibFlux"). 

150 photoCalibMean : `float`, optional 

151 Value to set for calibrationMean in the created PhotoCalib. 

152 Note: this value is 1/instFluxMag0, so it should be less than 1. 

153 photoCalibErr : `float`, optional 

154 Value to set for calibrationErr in the created PhotoCalib. 

155 fakeDetectorId : `int`, optional 

156 Use this as the detectorId in the returned CcdImage. 

157 fakeWcs : `lsst.afw.geom.SkyWcs`, optional 

158 A SkyWcs to use instead of one read from disk. 

159 fakeVisitInfo : `lsst.afw.image.VisitInfo`, optional 

160 A VisitInfo to use instead of one read from disk. 

161 

162 Returns 

163 ------- 

164 struct : `lsst.pipe.base.Struct` 

165 Result struct with components: 

166 

167 - `catalog` : Catalogs containing fake sources 

168 (`lsst.afw.table.SourceCatalog`). 

169 - `ccdImage` : CcdImage containing the metadata and fake sources 

170 (`lsst.jointcal.CcdImage`). 

171 - `bbox` : Bounding Box of the image (`lsst.geom.Box2I`). 

172 - `skyWcs` : SkyWcs of the image (`lsst.afw.geom.SkyWcs`). 

173 """ 

174 detectorId = 12 # we only have data for detector=12 

175 

176 dataId = dict(visit=visit, detector=detectorId, instrument="MegaPrime") 

177 skyWcs = fakeWcs if fakeWcs is not None else butler.get('calexp.wcs', dataId=dataId) 

178 visitInfo = fakeVisitInfo if fakeVisitInfo is not None else butler.get('calexp.visitInfo', dataId=dataId) 

179 bbox = butler.get('calexp.bbox', dataId=dataId) 

180 detector = butler.get('calexp.detector', dataId=dataId) 

181 filt = butler.get("calexp.filter", dataId=dataId).bandLabel 

182 photoCalib = lsst.afw.image.PhotoCalib(photoCalibMean, photoCalibErr) 

183 

184 catalog = createFakeCatalog(num, bbox, fluxFieldName, skyWcs=skyWcs) 

185 ccdImage = lsst.jointcal.ccdImage.CcdImage(catalog, skyWcs, visitInfo, bbox, filt, photoCalib, 

186 detector, visit, fakeDetectorId, fluxFieldName) 

187 

188 return lsst.pipe.base.Struct(catalog=catalog, ccdImage=ccdImage, bbox=bbox, skyWcs=skyWcs) 

189 

190 

191def createFakeCatalog(num, bbox, fluxFieldName, skyWcs=None, refCat=False): 

192 """Return a fake minimally-useful catalog for jointcal. 

193 

194 Parameters 

195 ---------- 

196 num : `int` 

197 Number of sources to put in the catalogs. Should be 

198 a square, to have sqrt(num) centroids on a grid. 

199 bbox : `lsst.geom.Box2I` 

200 Bounding Box of the detector to populate. 

201 fluxFieldName : `str` 

202 Name of the flux field to populate in the catalog, without `_instFlux` 

203 (e.g. "slot_CalibFlux"). 

204 skyWcs : `lsst.afw.geom.SkyWcs` or None, optional 

205 If supplied, use this to fill in coordinates from centroids. 

206 refCat : `bool`, optional 

207 Return a ``SimpleCatalog`` so that it behaves like a reference catalog? 

208 

209 Returns 

210 ------- 

211 catalog : `lsst.afw.table.SourceCatalog` 

212 A populated source catalog. 

213 """ 

214 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

215 # centroid 

216 centroidKey = lsst.afw.table.Point2DKey.addFields(schema, "centroid", "centroid", "pixels") 

217 xErrKey = schema.addField("centroid_xErr", type="F") 

218 yErrKey = schema.addField("centroid_yErr", type="F") 

219 # shape 

220 shapeKey = lsst.afw.table.QuadrupoleKey.addFields(schema, "shape", "", 

221 lsst.afw.table.CoordinateType.PIXEL) 

222 # Put the fake sources in the minimal catalog. 

223 schema.addField(fluxFieldName+"_instFlux", type="D", doc="post-ISR instFlux") 

224 schema.addField(fluxFieldName+"_instFluxErr", type="D", doc="post-ISR instFlux stddev") 

225 schema.addField(fluxFieldName+"_flux", type="D", doc="source flux (nJy)") 

226 schema.addField(fluxFieldName+"_fluxErr", type="D", doc="flux stddev (nJy)") 

227 schema.addField(fluxFieldName+"_mag", type="D", doc="magnitude") 

228 schema.addField(fluxFieldName+"_magErr", type="D", doc="magnitude stddev") 

229 return fillCatalog(schema, num, bbox, 

230 centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName, 

231 skyWcs=skyWcs, refCat=refCat) 

232 

233 

234def fillCatalog(schema, num, bbox, 

235 centroidKey, xErrKey, yErrKey, shapeKey, fluxFieldName, 

236 skyWcs=None, fluxErrFraction=0.05, refCat=False): 

237 """Return a catalog populated with fake, but reasonable, sources. 

238 

239 Centroids are placed on a uniform grid, errors are normally distributed. 

240 

241 Parameters 

242 ---------- 

243 schema : `lsst.afw.table.Schema` 

244 Pre-built schema to make the catalog from. 

245 num : `int` 

246 Number of sources to put in the catalog. 

247 bbox : `lsst.geom.Box2I` 

248 Bounding box of the ccd to put sources in. 

249 centroidKey : `lsst.afw.table.Key` 

250 Key for the centroid field to populate. 

251 xErrKey : `lsst.afw.table.Key` 

252 Key for the xErr field to populate. 

253 yErrKey : `lsst.afw.table.Key` 

254 Key for the yErr field to populate. 

255 shapeKey : `lsst.afw.table.Key` 

256 Key for the shape field to populate. 

257 fluxFieldName : `str` 

258 Name of the flux field to populate in the catalog, without `_instFlux` 

259 (e.g. "slot_CalibFlux"). 

260 skyWcs : `lsst.afw.geom.SkyWcs` or None, optional 

261 If supplied, use this to fill in coordinates from centroids. 

262 fluxErrFraction : `float`, optional 

263 Fraction of instFlux to use for the instFluxErr. 

264 refCat : `bool`, optional 

265 Return a ``SimpleCatalog`` so that it behaves like a reference catalog? 

266 

267 Returns 

268 ------- 

269 catalog : `lsst.afw.table.SourceCatalog` 

270 The filled catalog. 

271 """ 

272 table = lsst.afw.table.SourceTable.make(schema) 

273 table.defineCentroid('centroid') 

274 table.defineShape('shape') 

275 table.defineCalibFlux(fluxFieldName) 

276 if refCat: 

277 catalog = lsst.afw.table.SimpleCatalog(table) 

278 else: 

279 catalog = lsst.afw.table.SourceCatalog(table) 

280 

281 instFlux = np.random.random(num)*10000 

282 instFluxErr = np.abs(instFlux * np.random.normal(fluxErrFraction, scale=0.1, size=num)) 

283 xx = np.linspace(bbox.getMinX(), bbox.getMaxX(), int(np.sqrt(num))) 

284 yy = np.linspace(bbox.getMinY(), bbox.getMaxY(), int(np.sqrt(num))) 

285 xv, yv = np.meshgrid(xx, yy) 

286 vx = np.random.normal(scale=0.1, size=num) 

287 vy = np.random.normal(scale=0.1, size=num) 

288 

289 # make all the sources perfectly spherical, for simplicity. 

290 mxx = 1 

291 myy = 1 

292 mxy = 0 

293 

294 for i, (x, y) in enumerate(zip(xv.ravel(), yv.ravel())): 

295 record = catalog.addNew() 

296 record.set('id', i) 

297 record.set(centroidKey, lsst.geom.Point2D(x, y)) 

298 record.set(shapeKey, lsst.afw.geom.ellipses.Quadrupole(mxx, myy, mxy)) 

299 

300 if skyWcs is not None: 

301 lsst.afw.table.updateSourceCoords(skyWcs, catalog) 

302 

303 catalog[xErrKey] = vx 

304 catalog[yErrKey] = vy 

305 catalog[fluxFieldName + '_instFlux'] = instFlux 

306 catalog[fluxFieldName + '_instFluxErr'] = instFluxErr 

307 

308 return catalog 

309 

310 

311def getMeasuredStarsFromCatalog(catalog, pixToFocal): 

312 """Return a list of measuredStars built from a catalog. 

313 

314 Parameters 

315 ---------- 

316 catalog : `lsst.afw.table.SourceCatalog` 

317 The table to get sources from. 

318 pixToFocal : `lsst.afw.geom.TransformPoint2ToPoint2` 

319 Transform that goes from pixel to focal plane coordinates, to set the 

320 MeasuredStar x/y focal points. 

321 

322 Returns 

323 ------- 

324 stars : `list` of `lsst.jointcal.MeasuredStar` 

325 MeasuredStars built from the catalog sources. 

326 """ 

327 stars = [] 

328 for record in catalog: 

329 star = lsst.jointcal.star.MeasuredStar() 

330 star.x = record.getX() 

331 star.y = record.getY() 

332 star.setInstFluxAndErr(record.getCalibInstFlux(), record.getCalibInstFluxErr()) 

333 # TODO: cleanup after DM-4044 

334 point = lsst.geom.Point2D(star.x, star.y) 

335 pointFocal = pixToFocal.applyForward(point) 

336 star.setXFocal(pointFocal.getX()) 

337 star.setYFocal(pointFocal.getY()) 

338 stars.append(star) 

339 

340 return stars