Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of dax_apdb. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit test for Apdb class. 

23""" 

24 

25import pandas 

26import random 

27from typing import Iterator 

28import unittest 

29 

30from lsst.daf.base import DateTime 

31from lsst.dax.apdb import ApdbSql, ApdbSqlConfig 

32from lsst.sphgeom import Angle, Circle, LonLat, Region, UnitVector3d 

33from lsst.geom import SpherePoint 

34import lsst.utils.tests 

35 

36 

37def _makeRegion() -> Region: 

38 """Generate pixel ID ranges for some envelope region""" 

39 pointing_v = UnitVector3d(1., 1., -1.) 

40 fov = 0.05 # radians 

41 region = Circle(pointing_v, Angle(fov/2)) 

42 return region 

43 

44 

45def _makeVectors(region: Region, count: int = 1) -> Iterator[SpherePoint]: 

46 """Generate bunch of SpherePoints inside given region. 

47 

48 Returned vectors are random but not necessarily uniformly distributed. 

49 """ 

50 bbox = region.getBoundingBox() 

51 center = bbox.getCenter() 

52 center_lon = center.getLon().asRadians() 

53 center_lat = center.getLat().asRadians() 

54 width = bbox.getWidth().asRadians() 

55 height = bbox.getHeight().asRadians() 

56 while count > 0: 

57 lon = random.uniform(center_lon - width / 2, center_lon + width / 2) 

58 lat = random.uniform(center_lat - height / 2, center_lat + height / 2) 

59 lonlat = LonLat.fromRadians(lon, lat) 

60 uv3d = UnitVector3d(lonlat) 

61 if region.contains(uv3d): 

62 yield SpherePoint(lonlat) 

63 count -= 1 

64 

65 

66def _makeObjectCatalogPandas(region, count: int, config: ApdbSqlConfig): 

67 """Make a catalog containing a bunch of DiaObjects inside region. 

68 

69 The number of created records will be equal to the number of ranges (one 

70 object per pixel range). Coordinates of the created objects are not usable. 

71 """ 

72 data_list = [] 

73 # 0 id'ed DiaObjects don't exist and is used as a Null value for the id. 

74 for oid, sp in enumerate(_makeVectors(region, count)): 

75 tmp_dict = {"diaObjectId": oid + 1, 

76 "ra": sp.getRa().asDegrees(), 

77 "decl": sp.getDec().asDegrees()} 

78 data_list.append(tmp_dict) 

79 

80 df = pandas.DataFrame(data=data_list) 

81 return df 

82 

83 

84def _makeSourceCatalogPandas(objects, visit_time, start_id=0): 

85 """Make a catalog containing a bunch of DiaSources associated with the 

86 input diaObjects. 

87 """ 

88 # make some sources 

89 catalog = [] 

90 midPointTai = visit_time.get(system=DateTime.MJD) 

91 for index, obj in objects.iterrows(): 

92 catalog.append({"diaSourceId": start_id, 

93 "ccdVisitId": 1, 

94 "diaObjectId": obj["diaObjectId"], 

95 "parentDiaSourceId": 0, 

96 "ra": obj["ra"], 

97 "decl": obj["decl"], 

98 "midPointTai": midPointTai, 

99 "flags": 0}) 

100 start_id += 1 

101 return pandas.DataFrame(data=catalog) 

102 

103 

104def _makeForcedSourceCatalogPandas(objects, visit_time, ccdVisitId=1): 

105 """Make a catalog containing a bunch of DiaFourceSources associated with 

106 the input diaObjects. 

107 """ 

108 # make some sources 

109 catalog = [] 

110 midPointTai = visit_time.get(system=DateTime.MJD) 

111 for index, obj in objects.iterrows(): 

112 catalog.append({"diaObjectId": obj["diaObjectId"], 

113 "ccdVisitId": ccdVisitId, 

114 "midPointTai": midPointTai, 

115 "flags": 0}) 

116 return pandas.DataFrame(data=catalog) 

117 

118 

119class ApdbTestCase(unittest.TestCase): 

120 """A test case for Apdb class 

121 """ 

122 

123 data_type = pandas.DataFrame 

124 

125 def _assertCatalog(self, catalog, size, type=pandas.DataFrame): 

126 """Validate catalog type and size 

127 

128 Parameters 

129 ---------- 

130 calalog : `object` 

131 Expected type of this is ``type``. 

132 size : int 

133 Expected catalog size 

134 type : `type`, optional 

135 Expected catalog type 

136 """ 

137 self.assertIsInstance(catalog, type) 

138 self.assertEqual(len(catalog), size) 

139 

140 def test_makeSchema(self): 

141 """Test for making an instance of Apdb using in-memory sqlite engine. 

142 """ 

143 # sqlite does not support default READ_COMMITTED, for in-memory 

144 # database have to use connection pool 

145 config = ApdbSqlConfig(db_url="sqlite://") 

146 apdb = ApdbSql(config) 

147 # the essence of a test here is that there are no exceptions. 

148 apdb.makeSchema() 

149 

150 def test_emptyGetsBaseline0months(self): 

151 """Test for getting data from empty database. 

152 

153 All get() methods should return empty results, only useful for 

154 checking that code is not broken. 

155 """ 

156 

157 # set read_sources_months to 0 so that Forced/Sources are None 

158 config = ApdbSqlConfig(db_url="sqlite:///", 

159 read_sources_months=0, 

160 read_forced_sources_months=0) 

161 apdb = ApdbSql(config) 

162 apdb.makeSchema() 

163 

164 region = _makeRegion() 

165 visit_time = DateTime.now() 

166 

167 # get objects by region 

168 res = apdb.getDiaObjects(region) 

169 self._assertCatalog(res, 0, type=self.data_type) 

170 

171 # get sources by region 

172 res = apdb.getDiaSources(region, None, visit_time) 

173 self.assertIs(res, None) 

174 

175 # get sources by object ID, empty object list 

176 res = apdb.getDiaSources(region, [], visit_time) 

177 self.assertIs(res, None) 

178 

179 # get forced sources by object ID, empty object list 

180 res = apdb.getDiaForcedSources(region, [], visit_time) 

181 self.assertIs(res, None) 

182 

183 def test_emptyGetsBaseline(self): 

184 """Test for getting data from empty database. 

185 

186 All get() methods should return empty results, only useful for 

187 checking that code is not broken. 

188 """ 

189 

190 # use non-zero months for Forced/Source fetching 

191 config = ApdbSqlConfig(db_url="sqlite:///", 

192 read_sources_months=12, 

193 read_forced_sources_months=12) 

194 apdb = ApdbSql(config) 

195 apdb.makeSchema() 

196 

197 region = _makeRegion() 

198 visit_time = DateTime.now() 

199 

200 # get objects by region 

201 res = apdb.getDiaObjects(region) 

202 self._assertCatalog(res, 0, type=self.data_type) 

203 

204 # get sources by region 

205 res = apdb.getDiaSources(region, None, visit_time) 

206 self._assertCatalog(res, 0, type=self.data_type) 

207 

208 res = apdb.getDiaSources(region, [], visit_time) 

209 self._assertCatalog(res, 0, type=self.data_type) 

210 

211 # get sources by object ID, non-empty object list 

212 res = apdb.getDiaSources(region, [1, 2, 3], visit_time) 

213 self._assertCatalog(res, 0, type=self.data_type) 

214 

215 # get forced sources by object ID, empty object list 

216 res = apdb.getDiaForcedSources(region, [], visit_time) 

217 self._assertCatalog(res, 0, type=self.data_type) 

218 

219 # get sources by object ID, non-empty object list 

220 res = apdb.getDiaForcedSources(region, [1, 2, 3], visit_time) 

221 self._assertCatalog(res, 0, type=self.data_type) 

222 

223 # SQL implementation needs ID list 

224 with self.assertRaises(NotImplementedError): 

225 apdb.getDiaForcedSources(region, None, visit_time) 

226 

227 def test_emptyGetsObjectLast(self): 

228 """Test for getting DiaObjects from empty database using DiaObjectLast 

229 table. 

230 

231 All get() methods should return empty results, only useful for 

232 checking that code is not broken. 

233 """ 

234 

235 # don't care about sources. 

236 config = ApdbSqlConfig(db_url="sqlite:///", 

237 dia_object_index="last_object_table") 

238 apdb = ApdbSql(config) 

239 apdb.makeSchema() 

240 

241 region = _makeRegion() 

242 

243 # get objects by region 

244 res = apdb.getDiaObjects(region) 

245 self._assertCatalog(res, 0, type=self.data_type) 

246 

247 def test_storeObjectsBaseline(self): 

248 """Store and retrieve DiaObjects.""" 

249 

250 # don't care about sources. 

251 config = ApdbSqlConfig(db_url="sqlite:///", 

252 dia_object_index="baseline") 

253 apdb = ApdbSql(config) 

254 apdb.makeSchema() 

255 

256 region = _makeRegion() 

257 visit_time = DateTime.now() 

258 

259 # make catalog with Objects 

260 catalog = _makeObjectCatalogPandas(region, 100, config) 

261 

262 # store catalog 

263 apdb.store(visit_time, catalog) 

264 

265 # read it back and check sizes 

266 res = apdb.getDiaObjects(region) 

267 self._assertCatalog(res, len(catalog), type=self.data_type) 

268 

269 def test_storeObjectsLast(self): 

270 """Store and retrieve DiaObjects using DiaObjectLast table.""" 

271 # don't care about sources. 

272 config = ApdbSqlConfig(db_url="sqlite:///", 

273 dia_object_index="last_object_table", 

274 object_last_replace=True) 

275 apdb = ApdbSql(config) 

276 apdb.makeSchema() 

277 

278 region = _makeRegion() 

279 visit_time = DateTime.now() 

280 

281 # make catalog with Objects 

282 catalog = _makeObjectCatalogPandas(region, 100, config) 

283 

284 # store catalog 

285 apdb.store(visit_time, catalog) 

286 

287 # read it back and check sizes 

288 res = apdb.getDiaObjects(region) 

289 self._assertCatalog(res, len(catalog), type=self.data_type) 

290 

291 def test_storeSources(self): 

292 """Store and retrieve DiaSources.""" 

293 config = ApdbSqlConfig(db_url="sqlite:///", 

294 read_sources_months=12, 

295 read_forced_sources_months=12) 

296 apdb = ApdbSql(config) 

297 apdb.makeSchema() 

298 

299 region = _makeRegion() 

300 visit_time = DateTime.now() 

301 

302 # have to store Objects first 

303 objects = _makeObjectCatalogPandas(region, 100, config) 

304 oids = list(objects["diaObjectId"]) 

305 sources = _makeSourceCatalogPandas(objects, visit_time) 

306 

307 # save the objects and sources 

308 apdb.store(visit_time, objects, sources) 

309 

310 # read it back, no ID filtering 

311 res = apdb.getDiaSources(region, None, visit_time) 

312 self._assertCatalog(res, len(sources), type=self.data_type) 

313 

314 # read it back and filter by ID 

315 res = apdb.getDiaSources(region, oids, visit_time) 

316 self._assertCatalog(res, len(sources), type=self.data_type) 

317 

318 # read it back to get schema 

319 res = apdb.getDiaSources(region, [], visit_time) 

320 self._assertCatalog(res, 0, type=self.data_type) 

321 

322 def test_storeForcedSources(self): 

323 """Store and retrieve DiaForcedSources.""" 

324 

325 config = ApdbSqlConfig(db_url="sqlite:///", 

326 read_sources_months=12, 

327 read_forced_sources_months=12) 

328 apdb = ApdbSql(config) 

329 apdb.makeSchema() 

330 

331 region = _makeRegion() 

332 visit_time = DateTime.now() 

333 

334 # have to store Objects first 

335 objects = _makeObjectCatalogPandas(region, 100, config) 

336 oids = list(objects["diaObjectId"]) 

337 catalog = _makeForcedSourceCatalogPandas(objects, visit_time) 

338 

339 apdb.store(visit_time, objects, forced_sources=catalog) 

340 

341 # read it back and check sizes 

342 res = apdb.getDiaForcedSources(region, oids, visit_time) 

343 self._assertCatalog(res, len(catalog), type=self.data_type) 

344 

345 # read it back to get schema 

346 res = apdb.getDiaForcedSources(region, [], visit_time) 

347 self._assertCatalog(res, 0, type=self.data_type) 

348 

349 def test_midPointTai_src(self): 

350 """Test for time filtering of DiaSources. 

351 """ 

352 config = ApdbSqlConfig(db_url="sqlite:///", 

353 read_sources_months=12, 

354 read_forced_sources_months=12) 

355 apdb = ApdbSql(config) 

356 apdb.makeSchema() 

357 

358 region = _makeRegion() 

359 # 2021-01-01 plus 360 days is 2021-12-27 

360 src_time1 = DateTime(2021, 1, 1, 0, 0, 0, DateTime.TAI) 

361 src_time2 = DateTime(2021, 1, 1, 0, 0, 2, DateTime.TAI) 

362 visit_time0 = DateTime(2021, 12, 26, 23, 59, 59, DateTime.TAI) 

363 visit_time1 = DateTime(2021, 12, 27, 0, 0, 1, DateTime.TAI) 

364 visit_time2 = DateTime(2021, 12, 27, 0, 0, 3, DateTime.TAI) 

365 

366 objects = _makeObjectCatalogPandas(region, 100, config) 

367 oids = list(objects["diaObjectId"]) 

368 sources = _makeSourceCatalogPandas(objects, src_time1, 0) 

369 apdb.store(src_time1, objects, sources) 

370 

371 sources = _makeSourceCatalogPandas(objects, src_time2, 100) 

372 apdb.store(src_time2, objects, sources) 

373 

374 # reading at time of last save should read all 

375 res = apdb.getDiaSources(region, oids, src_time2) 

376 self._assertCatalog(res, 200, type=self.data_type) 

377 

378 # one second before 12 months 

379 res = apdb.getDiaSources(region, oids, visit_time0) 

380 self._assertCatalog(res, 200, type=self.data_type) 

381 

382 # reading at later time of last save should only read a subset 

383 res = apdb.getDiaSources(region, oids, visit_time1) 

384 self._assertCatalog(res, 100, type=self.data_type) 

385 

386 # reading at later time of last save should only read a subset 

387 res = apdb.getDiaSources(region, oids, visit_time2) 

388 self._assertCatalog(res, 0, type=self.data_type) 

389 

390 def test_midPointTai_fsrc(self): 

391 """Test for time filtering of DiaForcedSources. 

392 """ 

393 config = ApdbSqlConfig(db_url="sqlite:///", 

394 read_sources_months=12, 

395 read_forced_sources_months=12) 

396 apdb = ApdbSql(config) 

397 apdb.makeSchema() 

398 

399 region = _makeRegion() 

400 src_time1 = DateTime(2021, 1, 1, 0, 0, 0, DateTime.TAI) 

401 src_time2 = DateTime(2021, 1, 1, 0, 0, 2, DateTime.TAI) 

402 visit_time0 = DateTime(2021, 12, 26, 23, 59, 59, DateTime.TAI) 

403 visit_time1 = DateTime(2021, 12, 27, 0, 0, 1, DateTime.TAI) 

404 visit_time2 = DateTime(2021, 12, 27, 0, 0, 3, DateTime.TAI) 

405 

406 objects = _makeObjectCatalogPandas(region, 100, config) 

407 oids = list(objects["diaObjectId"]) 

408 sources = _makeForcedSourceCatalogPandas(objects, src_time1, 1) 

409 apdb.store(src_time1, objects, forced_sources=sources) 

410 

411 sources = _makeForcedSourceCatalogPandas(objects, src_time2, 2) 

412 apdb.store(src_time2, objects, forced_sources=sources) 

413 

414 # reading at time of last save should read all 

415 res = apdb.getDiaForcedSources(region, oids, src_time2) 

416 self._assertCatalog(res, 200, type=self.data_type) 

417 

418 # one second before 12 months 

419 res = apdb.getDiaForcedSources(region, oids, visit_time0) 

420 self._assertCatalog(res, 200, type=self.data_type) 

421 

422 # reading at later time of last save should only read a subset 

423 res = apdb.getDiaForcedSources(region, oids, visit_time1) 

424 self._assertCatalog(res, 100, type=self.data_type) 

425 

426 # reading at later time of last save should only read a subset 

427 res = apdb.getDiaForcedSources(region, oids, visit_time2) 

428 self._assertCatalog(res, 0, type=self.data_type) 

429 

430 

431class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

432 pass 

433 

434 

435def setup_module(module): 

436 lsst.utils.tests.init() 

437 

438 

439if __name__ == "__main__": 439 ↛ 440line 439 didn't jump to line 440, because the condition on line 439 was never true

440 lsst.utils.tests.init() 

441 unittest.main()