Coverage for tests/test_apdb.py: 17%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

196 statements  

1# This file is part of dax_apdb. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit test for Apdb class. 

23""" 

24 

25import datetime 

26import pandas 

27import unittest 

28 

29import lsst.afw.table as afwTable 

30from lsst.dax.apdb import (Apdb, ApdbConfig, make_minimal_dia_object_schema, 

31 make_minimal_dia_source_schema) 

32from lsst.sphgeom import Angle, Circle, HtmPixelization, Vector3d, UnitVector3d 

33from lsst.geom import SpherePoint 

34import lsst.utils.tests 

35 

36 

37# HTM indexing level used in the unit tests 

38HTM_LEVEL = 20 

39 

40 

41def _makePixelRanges(): 

42 """Generate pixel ID ranges for some envelope region""" 

43 pointing_v = UnitVector3d(1., 1., -1.) 

44 fov = 0.05 # radians 

45 region = Circle(pointing_v, Angle(fov/2)) 

46 pixelator = HtmPixelization(HTM_LEVEL) 

47 indices = pixelator.envelope(region, 128) 

48 return indices.ranges() 

49 

50 

51def _makeObjectCatalog(pixel_ranges): 

52 """Make a catalog containing a bunch of DiaObjects inside pixel envelope. 

53 

54 The number of created records will be equal number of ranges (one object 

55 per pixel range). Coordinates of the created objects are not usable. 

56 """ 

57 # make afw catalog 

58 schema = make_minimal_dia_object_schema() 

59 catalog = afwTable.SourceCatalog(schema) 

60 

61 # make small bunch of records, one entry per one pixel range, 

62 # we do not care about coordinates here, in current implementation 

63 # they are not used in any query 

64 v3d = Vector3d(1., 1., -1.) 

65 sp = SpherePoint(v3d) 

66 for oid, (start, end) in enumerate(pixel_ranges): 

67 record = catalog.addNew() 

68 record.set("id", oid) 

69 record.set("pixelId", start) 

70 record.set("coord_ra", sp.getRa()) 

71 record.set("coord_dec", sp.getDec()) 

72 

73 return catalog 

74 

75 

76def _makeObjectCatalogPandas(pixel_ranges): 

77 """Make a catalog containing a bunch of DiaObjects inside pixel envelope. 

78 

79 The number of created records will be equal to the number of ranges (one 

80 object per pixel range). Coordinates of the created objects are not usable. 

81 """ 

82 v3d = Vector3d(1., 1., -1.) 

83 sp = SpherePoint(v3d) 

84 data_list = [] 

85 for oid, (start, end) in enumerate(pixel_ranges): 

86 tmp_dict = {"diaObjectId": oid, 

87 "pixelId": start, 

88 "ra": sp.getRa().asDegrees(), 

89 "decl": sp.getDec().asDegrees()} 

90 data_list.append(tmp_dict) 

91 

92 df = pandas.DataFrame(data=data_list) 

93 return df 

94 

95 

96def _makeSourceCatalog(objects): 

97 """Make a catalog containing a bunch of DiaSources associated with the 

98 input diaObjects. 

99 """ 

100 # make some sources 

101 schema = make_minimal_dia_source_schema() 

102 catalog = afwTable.BaseCatalog(schema) 

103 oids = [] 

104 for sid, obj in enumerate(objects): 

105 record = catalog.addNew() 

106 record.set("id", sid) 

107 record.set("ccdVisitId", 1) 

108 record.set("diaObjectId", obj["id"]) 

109 record.set("parent", 0) 

110 record.set("coord_ra", obj["coord_ra"]) 

111 record.set("coord_dec", obj["coord_dec"]) 

112 record.set("flags", 0) 

113 record.set("pixelId", obj["pixelId"]) 

114 oids.append(obj["id"]) 

115 

116 return catalog, oids 

117 

118 

119def _makeSourceCatalogPandas(objects): 

120 """Make a catalog containing a bunch of DiaSources associated with the 

121 input diaObjects. 

122 """ 

123 # make some sources 

124 catalog = [] 

125 oids = [] 

126 for sid, (index, obj) in enumerate(objects.iterrows()): 

127 catalog.append({"diaSourceId": sid, 

128 "ccdVisitId": 1, 

129 "diaObjectId": obj["diaObjectId"], 

130 "parentDiaSourceId": 0, 

131 "ra": obj["ra"], 

132 "decl": obj["decl"], 

133 "flags": 0, 

134 "pixelId": obj["pixelId"]}) 

135 oids.append(obj["diaObjectId"]) 

136 return pandas.DataFrame(data=catalog), oids 

137 

138 

139def _makeForcedSourceCatalog(objects): 

140 """Make a catalog containing a bunch of DiaFourceSources associated with 

141 the input diaObjects. 

142 """ 

143 # make some sources 

144 schema = afwTable.Schema() 

145 schema.addField("diaObjectId", "L") 

146 schema.addField("ccdVisitId", "L") 

147 schema.addField("flags", "L") 

148 catalog = afwTable.BaseCatalog(schema) 

149 oids = [] 

150 for obj in objects: 

151 record = catalog.addNew() 

152 record.set("diaObjectId", obj["id"]) 

153 record.set("ccdVisitId", 1) 

154 record.set("flags", 0) 

155 oids.append(obj["id"]) 

156 

157 return catalog, oids 

158 

159 

160def _makeForcedSourceCatalogPandas(objects): 

161 """Make a catalog containing a bunch of DiaFourceSources associated with 

162 the input diaObjects. 

163 """ 

164 # make some sources 

165 catalog = [] 

166 oids = [] 

167 for index, obj in objects.iterrows(): 

168 catalog.append({"diaObjectId": obj["diaObjectId"], 

169 "ccdVisitId": 1, 

170 "flags": 0}) 

171 oids.append(obj["diaObjectId"]) 

172 return pandas.DataFrame(data=catalog), oids 

173 

174 

175class ApdbTestCase(unittest.TestCase): 

176 """A test case for Apdb class 

177 """ 

178 

179 use_pandas = False 

180 data_type = afwTable.SourceCatalog 

181 

182 def _assertCatalog(self, catalog, size, type=afwTable.SourceCatalog): 

183 """Validate catalog type and size 

184 

185 Parameters 

186 ---------- 

187 calalog : `lsst.afw.table.SourceCatalog` 

188 size : int 

189 Expected catalog size 

190 type : `type`, optional 

191 Expected catalog type 

192 """ 

193 self.assertIsInstance(catalog, type) 

194 self.assertEqual(len(catalog), size) 

195 

196 def test_makeSchema(self): 

197 """Test for making an instance of Apdb using in-memory sqlite engine. 

198 """ 

199 # sqlite does not support default READ_COMMITTED, for in-memory 

200 # database have to use connection pool 

201 config = ApdbConfig(db_url="sqlite://", 

202 isolation_level="READ_UNCOMMITTED") 

203 apdb = Apdb(config) 

204 # the essence of a test here is that there are no exceptions. 

205 apdb.makeSchema() 

206 

207 def test_emptyGetsBaseline0months(self): 

208 """Test for getting data from empty database. 

209 

210 All get() methods should return empty results, only useful for 

211 checking that code is not broken. 

212 """ 

213 

214 # set read_sources_months to 0 so that Forced/Sources are None 

215 config = ApdbConfig(db_url="sqlite:///", 

216 isolation_level="READ_UNCOMMITTED", 

217 read_sources_months=0, 

218 read_forced_sources_months=0) 

219 apdb = Apdb(config) 

220 apdb.makeSchema() 

221 

222 pixel_ranges = _makePixelRanges() 

223 visit_time = datetime.datetime.now() 

224 

225 # get objects by region 

226 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas) 

227 self._assertCatalog(res, 0, type=self.data_type) 

228 

229 # get sources by region 

230 res = apdb.getDiaSourcesInRegion(pixel_ranges, visit_time, return_pandas=self.use_pandas) 

231 self.assertIs(res, None) 

232 

233 # get sources by object ID, empty object list 

234 res = apdb.getDiaSources([], visit_time, return_pandas=self.use_pandas) 

235 

236 # get forced sources by object ID, empty object list 

237 res = apdb.getDiaForcedSources([], visit_time, return_pandas=self.use_pandas) 

238 self.assertIs(res, None) 

239 

240 def test_emptyGetsBaseline(self): 

241 """Test for getting data from empty database. 

242 

243 All get() methods should return empty results, only useful for 

244 checking that code is not broken. 

245 """ 

246 

247 # use non-zero months for Forced/Source fetching 

248 config = ApdbConfig(db_url="sqlite:///", 

249 isolation_level="READ_UNCOMMITTED", 

250 read_sources_months=12, 

251 read_forced_sources_months=12) 

252 apdb = Apdb(config) 

253 apdb.makeSchema() 

254 

255 pixel_ranges = _makePixelRanges() 

256 visit_time = datetime.datetime.now() 

257 

258 # get objects by region 

259 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas) 

260 self._assertCatalog(res, 0, type=self.data_type) 

261 

262 # get sources by region 

263 res = apdb.getDiaSourcesInRegion(pixel_ranges, visit_time, return_pandas=self.use_pandas) 

264 self._assertCatalog(res, 0, type=self.data_type) 

265 

266 # get sources by object ID, empty object list, should return None 

267 res = apdb.getDiaSources([], visit_time, return_pandas=self.use_pandas) 

268 self.assertIs(res, None) 

269 

270 # get sources by object ID, non-empty object list 

271 res = apdb.getDiaSources([1, 2, 3], visit_time, return_pandas=self.use_pandas) 

272 self._assertCatalog(res, 0, type=self.data_type) 

273 

274 # get forced sources by object ID, empty object list 

275 res = apdb.getDiaForcedSources([], visit_time, return_pandas=self.use_pandas) 

276 self.assertIs(res, None) 

277 

278 # get sources by object ID, non-empty object list 

279 res = apdb.getDiaForcedSources([1, 2, 3], visit_time, return_pandas=self.use_pandas) 

280 self._assertCatalog(res, 0, type=self.data_type) 

281 

282 def test_emptyGetsObjectLast(self): 

283 """Test for getting DiaObjects from empty database using DiaObjectLast 

284 table. 

285 

286 All get() methods should return empty results, only useful for 

287 checking that code is not broken. 

288 """ 

289 

290 # don't care about sources. 

291 config = ApdbConfig(db_url="sqlite:///", 

292 isolation_level="READ_UNCOMMITTED", 

293 dia_object_index="last_object_table") 

294 apdb = Apdb(config) 

295 apdb.makeSchema() 

296 

297 pixel_ranges = _makePixelRanges() 

298 

299 # get objects by region 

300 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas) 

301 self._assertCatalog(res, 0, type=self.data_type) 

302 

303 def test_storeObjectsBaseline(self): 

304 """Store and retrieve DiaObjects.""" 

305 

306 # don't care about sources. 

307 config = ApdbConfig(db_url="sqlite:///", 

308 isolation_level="READ_UNCOMMITTED", 

309 dia_object_index="baseline") 

310 apdb = Apdb(config) 

311 apdb.makeSchema() 

312 

313 pixel_ranges = _makePixelRanges() 

314 visit_time = datetime.datetime.now() 

315 

316 # make afw catalog with Objects 

317 if self.use_pandas: 

318 catalog = _makeObjectCatalogPandas(pixel_ranges) 

319 else: 

320 catalog = _makeObjectCatalog(pixel_ranges) 

321 

322 # store catalog 

323 apdb.storeDiaObjects(catalog, visit_time) 

324 

325 # read it back and check sizes 

326 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas) 

327 self._assertCatalog(res, len(catalog), type=self.data_type) 

328 

329 def test_storeObjectsLast(self): 

330 """Store and retrieve DiaObjects using DiaObjectLast table.""" 

331 # don't care about sources. 

332 config = ApdbConfig(db_url="sqlite:///", 

333 isolation_level="READ_UNCOMMITTED", 

334 dia_object_index="last_object_table", 

335 object_last_replace=True) 

336 apdb = Apdb(config) 

337 apdb.makeSchema() 

338 

339 pixel_ranges = _makePixelRanges() 

340 visit_time = datetime.datetime.now() 

341 

342 # make afw catalog with Objects 

343 if self.use_pandas: 

344 catalog = _makeObjectCatalogPandas(pixel_ranges) 

345 else: 

346 catalog = _makeObjectCatalog(pixel_ranges) 

347 

348 # store catalog 

349 apdb.storeDiaObjects(catalog, visit_time) 

350 

351 # read it back and check sizes 

352 res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas) 

353 self._assertCatalog(res, len(catalog), type=self.data_type) 

354 

355 def test_storeSources(self): 

356 """Store and retrieve DiaSources.""" 

357 config = ApdbConfig(db_url="sqlite:///", 

358 isolation_level="READ_UNCOMMITTED", 

359 read_sources_months=12, 

360 read_forced_sources_months=12) 

361 apdb = Apdb(config) 

362 apdb.makeSchema() 

363 

364 pixel_ranges = _makePixelRanges() 

365 visit_time = datetime.datetime.now() 

366 

367 # have to store Objects first 

368 if self.use_pandas: 

369 objects = _makeObjectCatalogPandas(pixel_ranges) 

370 catalog, oids = _makeSourceCatalogPandas(objects) 

371 else: 

372 objects = _makeObjectCatalog(pixel_ranges) 

373 catalog, oids = _makeSourceCatalog(objects) 

374 

375 # save the objects 

376 apdb.storeDiaObjects(objects, visit_time) 

377 

378 # save the sources 

379 apdb.storeDiaSources(catalog) 

380 

381 # read it back and check sizes 

382 res = apdb.getDiaSourcesInRegion(pixel_ranges, visit_time, self.use_pandas) 

383 self._assertCatalog(res, len(catalog), type=self.data_type) 

384 

385 # read it back using different method 

386 res = apdb.getDiaSources(oids, visit_time, self.use_pandas) 

387 self._assertCatalog(res, len(catalog), type=self.data_type) 

388 

389 def test_storeForcedSources(self): 

390 """Store and retrieve DiaForcedSources.""" 

391 

392 config = ApdbConfig(db_url="sqlite:///", 

393 isolation_level="READ_UNCOMMITTED", 

394 read_sources_months=12, 

395 read_forced_sources_months=12) 

396 apdb = Apdb(config) 

397 apdb.makeSchema() 

398 

399 pixel_ranges = _makePixelRanges() 

400 visit_time = datetime.datetime.now() 

401 

402 # have to store Objects first 

403 if self.use_pandas: 

404 objects = _makeObjectCatalogPandas(pixel_ranges) 

405 catalog, oids = _makeForcedSourceCatalogPandas(objects) 

406 else: 

407 objects = _makeObjectCatalog(pixel_ranges) 

408 catalog, oids = _makeForcedSourceCatalog(objects) 

409 

410 apdb.storeDiaObjects(objects, visit_time) 

411 

412 # save them 

413 apdb.storeDiaForcedSources(catalog) 

414 

415 # read it back and check sizes 

416 res = apdb.getDiaForcedSources(oids, visit_time, return_pandas=self.use_pandas) 

417 self._assertCatalog(res, len(catalog), type=self.data_type) 

418 

419 

420class ApdbPandasTestCase(ApdbTestCase): 

421 """A test case for Apdb using Pandas as the input/output""" 

422 

423 use_pandas = True 

424 data_type = pandas.DataFrame 

425 

426 

427class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

428 pass 

429 

430 

431def setup_module(module): 

432 lsst.utils.tests.init() 

433 

434 

435if __name__ == "__main__": 435 ↛ 436line 435 didn't jump to line 436, because the condition on line 435 was never true

436 lsst.utils.tests.init() 

437 unittest.main()