Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ap_association. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import numpy as np 

24import pandas as pd 

25import tempfile 

26import unittest 

27 

28from lsst.afw.cameraGeom.testUtils import DetectorWrapper 

29import lsst.afw.geom as afwGeom 

30import lsst.afw.image as afwImage 

31from lsst.ap.association import (LoadDiaCatalogsTask, 

32 LoadDiaCatalogsConfig, 

33 make_dia_source_schema, 

34 make_dia_object_schema) 

35import lsst.daf.base as dafBase 

36from lsst.dax.apdb import Apdb, ApdbConfig 

37import lsst.geom as geom 

38import lsst.sphgeom as sphgeom 

39from lsst.utils import getPackageDir 

40import lsst.utils.tests 

41 

42 

43def _data_file_name(basename, module_name): 

44 """Return path name of a data file. 

45 

46 Parameters 

47 ---------- 

48 basename : `str` 

49 Name of the file to add to the path string. 

50 module_name : `str` 

51 Name of lsst stack package environment variable. 

52 

53 Returns 

54 ------- 

55 data_file_path : `str` 

56 Full path of the file to load from the "data" directory in a given 

57 repository. 

58 """ 

59 return os.path.join(getPackageDir(module_name), "data", basename) 

60 

61 

62def makeExposure(flipX=False, flipY=False): 

63 """Create an exposure and flip the x or y (or both) coordinates. 

64 

65 Returns bounding boxes that are right or left handed around the bounding 

66 polygon. 

67 

68 Parameters 

69 ---------- 

70 flipX : `bool` 

71 Flip the x coordinate in the WCS. 

72 flipY : `bool` 

73 Flip the y coordinate in the WCS. 

74 

75 Returns 

76 ------- 

77 exposure : `lsst.afw.image.Exposure` 

78 Exposure with a valid bounding box and wcs. 

79 """ 

80 metadata = dafBase.PropertySet() 

81 

82 metadata.set("SIMPLE", "T") 

83 metadata.set("BITPIX", -32) 

84 metadata.set("NAXIS", 2) 

85 metadata.set("NAXIS1", 1024) 

86 metadata.set("NAXIS2", 1153) 

87 metadata.set("RADECSYS", 'FK5') 

88 metadata.set("EQUINOX", 2000.) 

89 

90 metadata.setDouble("CRVAL1", 215.604025685476) 

91 metadata.setDouble("CRVAL2", 53.1595451514076) 

92 metadata.setDouble("CRPIX1", 1109.99981456774) 

93 metadata.setDouble("CRPIX2", 560.018167811613) 

94 metadata.set("CTYPE1", 'RA---SIN') 

95 metadata.set("CTYPE2", 'DEC--SIN') 

96 

97 xFlip = 1 

98 if flipX: 

99 xFlip = -1 

100 yFlip = 1 

101 if flipY: 

102 yFlip = -1 

103 metadata.setDouble("CD1_1", xFlip * 5.10808596133527E-05) 

104 metadata.setDouble("CD1_2", yFlip * 1.85579539217196E-07) 

105 metadata.setDouble("CD2_2", yFlip * -5.10281493481982E-05) 

106 metadata.setDouble("CD2_1", xFlip * -8.27440751733828E-07) 

107 

108 wcs = afwGeom.makeSkyWcs(metadata) 

109 exposure = afwImage.makeExposure( 

110 afwImage.makeMaskedImageFromArrays(np.ones((1024, 1153))), wcs) 

111 detector = DetectorWrapper(id=23, bbox=exposure.getBBox()).detector 

112 visit = afwImage.VisitInfo( 

113 exposureId=1234, 

114 exposureTime=200., 

115 date=dafBase.DateTime("2014-05-13T17:00:00.000000000", 

116 dafBase.DateTime.Timescale.TAI)) 

117 exposure.setDetector(detector) 

118 exposure.getInfo().setVisitInfo(visit) 

119 exposure.setFilterLabel(afwImage.FilterLabel(band='g')) 

120 

121 return exposure 

122 

123 

124def makeDiaObjects(nObjects, exposure, pixelator): 

125 """Make a test set of DiaObjects. 

126 

127 Parameters 

128 ---------- 

129 nObjects : `int` 

130 Number of objects to create. 

131 exposure : `lsst.afw.image.Exposure` 

132 Exposure to create objects over. 

133 pixelator : `lsst.sphgeom.HtmPixelization` 

134 Object to compute spatial indicies from. 

135 

136 Returns 

137 ------- 

138 diaObjects : `pandas.DataFrame` 

139 DiaObjects generated across the exposure. 

140 """ 

141 bbox = geom.Box2D(exposure.getBBox()) 

142 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nObjects) 

143 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nObjects) 

144 

145 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get( 

146 system=dafBase.DateTime.MJD) 

147 

148 wcs = exposure.getWcs() 

149 

150 data = [] 

151 for idx, (x, y) in enumerate(zip(rand_x, rand_y)): 

152 coord = wcs.pixelToSky(x, y) 

153 htmIdx = pixelator.index(coord.getVector()) 

154 newObject = {"ra": coord.getRa().asDegrees(), 

155 "decl": coord.getRa().asDegrees(), 

156 "radecTai": midPointTaiMJD, 

157 "diaObjectId": idx, 

158 "pixelId": htmIdx, 

159 "pmParallaxNdata": 0, 

160 "nearbyObj1": 0, 

161 "nearbyObj2": 0, 

162 "nearbyObj3": 0} 

163 for f in ["u", "g", "r", "i", "z", "y"]: 

164 newObject["%sPSFluxNdata" % f] = 0 

165 data.append(newObject) 

166 

167 return pd.DataFrame(data=data) 

168 

169 

170def makeDiaSources(nSources, diaObjectIds, exposure, pixelator): 

171 """Make a test set of DiaSources. 

172 

173 Parameters 

174 ---------- 

175 nSources : `int` 

176 Number of sources to create. 

177 diaObjectIds : `numpy.ndarray` 

178 Integer Ids of diaobjects to "associate" with the DiaSources. 

179 exposure : `lsst.afw.image.Exposure` 

180 Exposure to create sources over. 

181 pixelator : `lsst.sphgeom.HtmPixelization` 

182 Object to compute spatial indicies from. 

183 

184 Returns 

185 ------- 

186 diaSources : `pandas.DataFrame` 

187 DiaSources generated across the exposure. 

188 """ 

189 bbox = geom.Box2D(exposure.getBBox()) 

190 rand_x = np.random.uniform(bbox.getMinX(), bbox.getMaxX(), size=nSources) 

191 rand_y = np.random.uniform(bbox.getMinY(), bbox.getMaxY(), size=nSources) 

192 rand_ids = diaObjectIds[np.random.randint(len(diaObjectIds), size=nSources)] 

193 

194 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get( 

195 system=dafBase.DateTime.MJD) 

196 

197 wcs = exposure.getWcs() 

198 

199 data = [] 

200 for idx, (x, y, objId) in enumerate(zip(rand_x, rand_y, rand_ids)): 

201 coord = wcs.pixelToSky(x, y) 

202 htmIdx = pixelator.index(coord.getVector()) 

203 data.append({"ra": coord.getRa().asDegrees(), 

204 "decl": coord.getRa().asDegrees(), 

205 "diaObjectId": objId, 

206 "diaSourceId": idx, 

207 "pixelId": htmIdx, 

208 "midPointTai": midPointTaiMJD}) 

209 

210 return pd.DataFrame(data=data) 

211 

212 

213def makeDiaForcedSources(nForcedSources, diaObjectIds, exposure): 

214 """Make a test set of DiaForcedSources. 

215 

216 Parameters 

217 ---------- 

218 nForcedSources : `int` 

219 Number of sources to create. 

220 diaObjectIds : `numpy.ndarray` 

221 Integer Ids of diaobjects to "associate" with the DiaSources. 

222 exposure : `lsst.afw.image.Exposure` 

223 Exposure to create sources over. 

224 

225 Returns 

226 ------- 

227 diaForcedSources : `pandas.DataFrame` 

228 DiaForcedSources generated across the exposure. 

229 """ 

230 rand_ids = diaObjectIds[ 

231 np.random.randint(len(diaObjectIds), size=nForcedSources)] 

232 

233 midPointTaiMJD = exposure.getInfo().getVisitInfo().getDate().get( 

234 system=dafBase.DateTime.MJD) 

235 

236 data = [] 

237 for idx, objId in enumerate(rand_ids): 

238 data.append({"diaObjectId": objId, 

239 "diaForcedSourceId": idx, 

240 "ccdVisitId": idx, 

241 "midPointTai": midPointTaiMJD}) 

242 

243 return pd.DataFrame(data=data) 

244 

245 

246class TestLoadDiaCatalogs(unittest.TestCase): 

247 

248 def setUp(self): 

249 np.random.seed(1234) 

250 

251 self.db_file_fd, self.db_file = tempfile.mkstemp( 

252 dir=os.path.dirname(__file__)) 

253 

254 self.apdbConfig = ApdbConfig() 

255 self.apdbConfig.db_url = "sqlite:///" + self.db_file 

256 self.apdbConfig.isolation_level = "READ_UNCOMMITTED" 

257 self.apdbConfig.dia_object_index = "baseline" 

258 self.apdbConfig.dia_object_columns = [] 

259 self.apdbConfig.schema_file = _data_file_name( 

260 "apdb-schema.yaml", "dax_apdb") 

261 self.apdbConfig.column_map = _data_file_name( 

262 "apdb-ap-pipe-afw-map.yaml", "ap_association") 

263 self.apdbConfig.extra_schema_file = _data_file_name( 

264 "apdb-ap-pipe-schema-extra.yaml", "ap_association") 

265 

266 self.apdb = Apdb(config=self.apdbConfig, 

267 afw_schemas=dict(DiaObject=make_dia_object_schema(), 

268 DiaSource=make_dia_source_schema())) 

269 self.apdb.makeSchema() 

270 

271 # Expected HTM pixel ranges for max range=4 and level = 20. This 

272 # set of pixels should be same for the WCS created by default in 

273 # makeExposure and for one with a flipped y axis. 

274 self.ranges = np.sort(np.array([15154776375296, 15154779521024, 

275 15154788958208, 15154792103936])) 

276 

277 self.pixelator = sphgeom.HtmPixelization(20) 

278 self.exposure = makeExposure(False, False) 

279 

280 self.diaObjects = makeDiaObjects(20, self.exposure, self.pixelator) 

281 self.diaSources = makeDiaSources( 

282 100, 

283 self.diaObjects["diaObjectId"].to_numpy(), 

284 self.exposure, 

285 self.pixelator) 

286 self.diaForcedSources = makeDiaForcedSources( 

287 200, 

288 self.diaObjects["diaObjectId"].to_numpy(), 

289 self.exposure) 

290 

291 self.apdb.storeDiaSources(self.diaSources) 

292 self.apdb.storeDiaForcedSources(self.diaForcedSources) 

293 self.dateTime = \ 

294 self.exposure.getInfo().getVisitInfo().getDate().toPython() 

295 self.apdb.storeDiaObjects(self.diaObjects, 

296 self.dateTime) 

297 

298 def tearDown(self): 

299 os.close(self.db_file_fd) 

300 os.remove(self.db_file) 

301 

302 def testRun(self): 

303 """Test the full run method for the loader. 

304 """ 

305 diaLoader = LoadDiaCatalogsTask() 

306 result = diaLoader.run(self.exposure, self.apdb) 

307 

308 self.assertEqual(len(result.diaObjects), len(self.diaObjects)) 

309 self.assertEqual(len(result.diaSources), len(self.diaSources)) 

310 self.assertEqual(len(result.diaForcedSources), 

311 len(self.diaForcedSources)) 

312 

313 def testLoadDiaObjects(self): 

314 """Test that the correct number of diaObjects are loaded. 

315 """ 

316 diaLoader = LoadDiaCatalogsTask() 

317 normPixels = diaLoader._getPixelRanges(self.exposure) 

318 diaObjects = diaLoader.loadDiaObjects(normPixels, 

319 self.apdb) 

320 self.assertEqual(len(diaObjects), len(self.diaObjects)) 

321 

322 def testLoadDiaForcedSources(self): 

323 """Test that the correct number of diaForcedSources are loaded. 

324 """ 

325 diaLoader = LoadDiaCatalogsTask() 

326 diaForcedSources = diaLoader.loadDiaForcedSources( 

327 self.diaObjects, 

328 self.dateTime, 

329 self.apdb) 

330 self.assertEqual(len(diaForcedSources), len(self.diaForcedSources)) 

331 

332 def testLoadDiaSourcesByPixelId(self): 

333 """Test that the correct number of diaSources are loaded. 

334 

335 Also check that they can be properly loaded both by location and 

336 ``diaObjectId``. 

337 """ 

338 self._testLoadDiaSources(True) 

339 

340 def testLoadDiaSourcesByDiaObjectId(self): 

341 """Test that the correct number of diaSources are loaded. 

342 

343 Also check that they can be properly loaded both by location and 

344 ``diaObjectId``. 

345 """ 

346 self._testLoadDiaSources(False) 

347 

348 def _testLoadDiaSources(self, loadByPixelId): 

349 """Test that DiaSources are loaded correctly. 

350 

351 Parameters 

352 ---------- 

353 loadByPixelId : `bool` 

354 Load DiaSources by ``pixelId`` if ``True`` and by ``diaObjectId`` 

355 if ``False``. 

356 """ 

357 diaConfig = LoadDiaCatalogsConfig() 

358 diaConfig.loadDiaSourcesByPixelId = loadByPixelId 

359 diaLoader = LoadDiaCatalogsTask(config=diaConfig) 

360 

361 normPixels = diaLoader._getPixelRanges(self.exposure) 

362 diaSources = diaLoader.loadDiaSources(self.diaObjects, 

363 normPixels, 

364 self.dateTime, 

365 self.apdb) 

366 self.assertEqual(len(diaSources), len(self.diaSources)) 

367 

368 def testGetPixelRanges(self): 

369 """Test the same pixels are returned for flips/ordering changes in 

370 the WCS. 

371 """ 

372 diaConfig = LoadDiaCatalogsConfig() 

373 diaConfig.pixelMargin = 300 # overriding to value tests were conditioned on 

374 diaConfig.htmMaxRanges = 4 

375 diaLoader = LoadDiaCatalogsTask(config=diaConfig) 

376 

377 # Make two exposures, one with a flipped y axis to get left vs. right 

378 # handed. 

379 exposure = makeExposure(False, False) 

380 exposureFlip = makeExposure(False, True) 

381 

382 normPixels = diaLoader._getPixelRanges(exposure) 

383 flipPixels = diaLoader._getPixelRanges(exposureFlip) 

384 

385 for normPix, flipPix, testPix in zip( 

386 np.sort(np.array(normPixels).flatten()), 

387 np.sort(np.array(flipPixels).flatten()), 

388 self.ranges): 

389 self.assertEqual(normPix, flipPix) 

390 self.assertEqual(normPix, testPix) 

391 self.assertEqual(flipPix, testPix) 

392 

393 

394class MemoryTester(lsst.utils.tests.MemoryTestCase): 

395 pass 

396 

397 

398def setup_module(module): 

399 lsst.utils.tests.init() 

400 

401 

402if __name__ == "__main__": 402 ↛ 403line 402 didn't jump to line 403, because the condition on line 402 was never true

403 lsst.utils.tests.init() 

404 unittest.main()