Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558

559

560

561

562

563

564

565

566

567

568

569

570

571

572

573

574

575

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596

597

598

599

600

601

602

603

604

# This file is part of ap_association. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (https://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <https://www.gnu.org/licenses/>. 

 

"""Simple sqlite3 interface for storing and retrieving DIAObjects and 

DIASources. This class is mainly for testing purposes in the context of 

ap_pipe/ap_verify. 

""" 

 

__all__ = ["AssociationDBSqliteConfig", 

"AssociationDBSqliteTask", 

"SqliteDBConverter"] 

 

import sqlite3 

 

from lsst.meas.algorithms.indexerRegistry import IndexerRegistry 

import lsst.afw.table as afwTable 

import lsst.afw.geom as afwGeom 

import lsst.pex.config as pexConfig 

import lsst.pipe.base as pipeBase 

from .afwUtils import \ 

make_minimal_dia_object_schema, \ 

make_minimal_dia_source_schema, \ 

getCcdVisitSchemaSql, \ 

add_dia_source_aliases_to_catalog, \ 

get_ccd_visit_info_from_exposure, \ 

make_overwrite_dict 

 

 

class SqliteDBConverter(object): 

"""Class for defining conversions to and from an sqlite database and afw 

SourceRecord objects. 

 

Parameters 

---------- 

schema : `lsst.afw.table.Schema` 

Schema defining the SourceRecord objects to be converted. 

table_name : `str` 

Name of the sqlite table this converter is to be used for. 

""" 

 

def __init__(self, schema, table_name): 

self._schema = schema 

self._table_name = table_name 

self._afw_to_db_types = { 

"Angle": "REAL", 

"D": "REAL", 

"L": "INTEGER", 

"String": "TEXT", 

} 

 

@property 

def table_name(self): 

"""Return name of the sqlite table this catalog is for 

""" 

return self._table_name 

 

@property 

def schema(self): 

"""Return the internal catalog schema. 

""" 

return self._schema 

 

def make_table_from_afw_schema(self, table_name): 

"""Convert the schema into a sqlite CREATE TABLE command. 

 

Parameters 

---------- 

table_name : `str` 

Name of the new table to create 

 

Returns 

------- 

sql_query : `str` 

A string of the query command to create the new table in sqlite. 

""" 

name_type_string = "" 

for sub_schema in self._schema: 

tmp_name = sub_schema.getField().getName() 

tmp_type = self._afw_to_db_types[ 

sub_schema.getField().getTypeString()] 

if tmp_name == 'id': 

tmp_type += " PRIMARY KEY" 

name_type_string += "%s %s," % (tmp_name, tmp_type) 

name_type_string = name_type_string[:-1] 

 

return "CREATE TABLE IF NOT EXISTS %s (%s)" % (table_name, name_type_string) 

 

def source_record_from_db_row(self, db_row): 

"""Create a source record from the values stored in a database row. 

 

Parameters 

---------- 

db_row : `list` of ``values`` 

Retrieved values from the database to convert into a SourceRecord. 

 

Returns 

------- 

record : `lsst.afw.table.SourceRecord` 

Converted source record. 

""" 

 

output_source_record = afwTable.SourceTable.makeRecord( 

afwTable.SourceTable.make(self._schema)) 

 

for sub_schema, value in zip(self._schema, db_row): 

if value is None: 

if sub_schema.getField().getTypeString() == 'L': 

value = -1 

elif sub_schema.getField().getTypeString() == 'String': 

value = 'NULL' 

else: 

continue 

if sub_schema.getField().getTypeString() == 'Angle': 

output_source_record.set( 

sub_schema.getKey(), value * afwGeom.degrees) 

else: 

output_source_record.set(sub_schema.getKey(), value) 

return output_source_record 

 

def source_record_to_value_list(self, source_record, overwrite_dict={}): 

"""Convert a source record object into a list of its internal values. 

 

Parameters 

---------- 

source_record : `lsst.afw.table.SourceRecord` 

SourceRecord to convert. 

overwrite_dict : `dict` (optional) 

Mapping specifying the names of columns to overwrite with 

specified values. 

 

Returns 

------- 

source_list : `list` of ``values`` 

Extracted values from ``source_record`` in `list` form. 

""" 

values = [] 

for sub_schema in self._schema: 

field_name = sub_schema.getField().getName() 

if field_name in overwrite_dict: 

values.append(overwrite_dict[field_name]) 

elif field_name in source_record.getSchema().getNames(): 

if sub_schema.getField().getTypeString() == 'Angle': 

values.append( 

source_record.get(sub_schema.getKey()).asDegrees()) 

else: 

values.append(source_record.get(sub_schema.getKey())) 

else: 

values.append(None) 

return values 

 

 

class AssociationDBSqliteConfig(pexConfig.Config): 

"""Configuration parameters for the AssociationDBSqliteTask 

""" 

db_name = pexConfig.Field( 

dtype=str, 

doc='Location on disk and name of the sqlite3 database for storing ' 

'and loading DIASources and DIAObjects.', 

default=':memory:' 

) 

filter_names = pexConfig.ListField( 

dtype=str, 

doc='List of filter names to store and expect from in this DB.', 

default=[], 

) 

indexer = IndexerRegistry.makeField( 

doc='Select the spatial indexer to use within the database.', 

default='HTM' 

) 

 

 

class AssociationDBSqliteTask(pipeBase.Task): 

"""Enable storage of and reading of DIAObjects and DIASources from a 

sqlite database. 

 

Create a simple sqlite database and implement wrappers to store and 

retrieve DIAObjects and DIASources from within that database. This task 

functions as a testing ground for the L1 database and should mimic this 

database's eventual functionality. This specific database implementation is 

useful for the verification packages which may not be run with access to 

L1 database. 

""" 

 

ConfigClass = AssociationDBSqliteConfig 

_DefaultName = "association_db_sqlite" 

 

def __init__(self, **kwargs): 

 

pipeBase.Task.__init__(self, **kwargs) 

self.indexer = IndexerRegistry[self.config.indexer.name]( 

self.config.indexer.active) 

self._db_connection = sqlite3.connect(self.config.db_name, 

timeout=60) 

 

self._dia_object_converter = SqliteDBConverter( 

make_minimal_dia_object_schema(self.config.filter_names), 

"dia_objects") 

self._dia_source_converter = SqliteDBConverter( 

make_minimal_dia_source_schema(), 

"dia_sources") 

self._ccd_visit_schema = getCcdVisitSchemaSql() 

 

def _commit(self): 

"""Save changes to the sqlite database. 

""" 

self._db_connection.commit() 

 

def close(self): 

"""Close the connection to the sqlite database. 

""" 

self._db_connection.close() 

 

def create_tables(self): 

"""If no sqlite database with the correct tables exists we can create 

one using this method. 

 

Returns 

------- 

succeeded : `bool` 

Successfully created a new database with specified tables. 

""" 

# Create tables to store the individual DIAObjects and DIASources 

with self._db_connection as conn: 

conn.execute("BEGIN") 

conn.execute( 

self._dia_object_converter.make_table_from_afw_schema( 

"dia_objects")) 

conn.execute( 

"CREATE INDEX IF NOT EXISTS indexer_id_index ON " 

"dia_objects(pixelId)") 

conn.execute( 

self._dia_source_converter.make_table_from_afw_schema( 

"dia_sources")) 

conn.execute( 

"CREATE INDEX IF NOT EXISTS diaObjectId_index ON " 

"dia_sources(diaObjectId)") 

 

table_schema = ",".join( 

"%s %s" % (key, self._ccd_visit_schema[key]) 

for key in self._ccd_visit_schema.keys()) 

conn.execute( 

"CREATE TABLE IF NOT EXISTS CcdVisit (%s)" % table_schema) 

 

return True 

 

@pipeBase.timeMethod 

def load_dia_objects(self, exposure): 

"""Load all DIAObjects within the exposure. 

 

Parameters 

---------- 

exposure : `lsst.afw.image.Exposure` 

An exposure with a solved WCS representing the area on the sky to 

load DIAObjects. 

 

Returns 

------- 

dia_objects : `lsst.afw.table.SourceCatalog` 

Catalog of DIAObjects that are contained with the the bounding 

box defined by expMd. 

""" 

bbox = afwGeom.Box2D(exposure.getBBox()) 

wcs = exposure.getWcs() 

 

ctr_coord = wcs.pixelToSky(bbox.getCenter()) 

max_radius = max( 

ctr_coord.separation(wcs.pixelToSky(pp)) 

for pp in bbox.getCorners()) 

 

indexer_indices, on_boundry = self.indexer.getShardIds( 

ctr_coord, max_radius) 

 

dia_objects = self._get_dia_object_catalog(indexer_indices, bbox, wcs) 

 

return dia_objects 

 

@pipeBase.timeMethod 

def load_dia_sources(self, dia_obj_ids): 

"""Retrieve all DIASources associated with this collection of DIAObject 

ids. 

 

Parameters 

---------- 

dia_obj_ids : array-like of `int` 

Id of the DIAObject that is associated with the DIASources 

of interest. 

 

Returns 

------- 

dia_sources : `lsst.afw.table.SourceCatalog` 

SourceCatalog of DIASources 

""" 

 

dia_source_schema = make_minimal_dia_source_schema() 

output_dia_sources = afwTable.SourceCatalog(dia_source_schema) 

 

rows = self._query_dia_sources(dia_obj_ids) 

for row in rows: 

output_dia_sources.append( 

self._dia_source_converter.source_record_from_db_row(row)) 

 

return output_dia_sources.copy(deep=True) 

 

@pipeBase.timeMethod 

def store_dia_objects(self, 

dia_objects, 

compute_spatial_index=False, 

exposure=None): 

"""Store all DIAObjects in this SourceCatalog. 

 

Parameters 

---------- 

dia_objects : `lsst.afw.table.SourceCatalog` 

Catalog of DIAObjects to store. 

compute_spatial_index : `bool` 

If True, compute the spatial search indices using the 

indexer specified at class instantiation. 

exposure: `lsst.afw.image.Exposure` (optional) 

CcdExposure associated with these DIAObjects being inserted. 

Inserts the CcdVisitInfo for this exposure in the CcdVisitTable. 

""" 

if compute_spatial_index: 

for dia_object in dia_objects: 

sphPoint = dia_object.getCoord() 

dia_object.set('pixelId', 

self.compute_indexer_id(sphPoint)) 

self._store_catalog(dia_objects, self._dia_object_converter, None) 

 

if exposure is not None: 

self.store_ccd_visit_info(exposure) 

 

def compute_indexer_id(self, sphere_point): 

"""Compute the pixel index of the given point. 

 

Parameters 

---------- 

sphere_point : `lsst.afw.geom.SpherePoint` 

Point to compute pixel index for. 

 

Returns 

------- 

index : `int` 

Index of the pixel the point is contained in. 

""" 

return self.indexer.indexPoints( 

[sphere_point.getRa().asDegrees()], 

[sphere_point.getDec().asDegrees()])[0] 

 

@pipeBase.timeMethod 

def store_dia_sources(self, 

dia_sources, 

associated_ids=None, 

exposure=None): 

"""Store all DIASources in this SourceCatalog. 

 

Parameters 

---------- 

dia_sources : `lsst.afw.table.SourceCatalog` 

Catalog of DIASources to store. 

associated_ids : array-like of `int` (optional) 

DIAObject ids that have been associated with these DIASources 

exposure : `lsst.afw.image.Exposure` 

Exposure object the DIASources were detected in. 

""" 

self._store_catalog(dia_sources, 

self._dia_source_converter, 

associated_ids, 

exposure) 

 

def store_ccd_visit_info(self, exposure): 

"""Store information describing the exposure for this ccd, visit. 

 

Parameters 

---------- 

exposure : `lsst.afw.image.Exposure` 

Exposure to store information from. 

""" 

 

values = get_ccd_visit_info_from_exposure(exposure) 

with self._db_connection as conn: 

conn.execute( 

"INSERT OR REPLACE INTO CcdVisit VALUES (%s)" % 

",".join("?" for idx in range(len(self._ccd_visit_schema))), 

[values[key] for key in self._ccd_visit_schema.keys()]) 

 

def _get_dia_object_catalog(self, indexer_indices, bbox, wcs): 

"""Retrieve the DIAObjects from the database whose indexer indices 

are with the specified list of indices. 

 

Retrieves a list of DIAObjects that are covered by the pixels with 

indices, indexer_indices. Use this to retrieve complete DIAObjects. 

 

Parameters 

---------- 

indexer_indices : array-like of `int` 

Pixelized indexer indices from which to load. 

bbox : `lsst.geom.Box2D` 

Bounding box of exposure. 

wcs : `lsst.geom.SkyWcs` 

WCS of exposure 

 

Returns 

------- 

dia_objects : `lsst.afw.table.SourceCatalog` 

Catalog of DIAObjects with the specified indexer index and 

contained within the bounding box. 

""" 

dia_object_rows = self._query_dia_objects(indexer_indices) 

 

output_dia_objects = afwTable.SourceCatalog( 

self._dia_object_converter.schema) 

 

for row in dia_object_rows: 

dia_object_record = \ 

self._dia_object_converter.source_record_from_db_row(row) 

if self._check_dia_object_position(dia_object_record, bbox, wcs): 

output_dia_objects.append(dia_object_record) 

 

return output_dia_objects.copy(deep=True) 

 

def _query_dia_objects(self, indexer_indices): 

"""Query the database for the stored DIAObjects given a set of 

indices in the indexer. 

 

Parameters 

---------- 

indexer_indices : array-like of `int` 

Spatial indices in the indexer specifying the area on the sky 

to load DIAObjects for. 

 

Returns 

------- 

dia_objects : `list` of `tuples` containing ``dia_object`` ``values`` 

Query result containing the catalog values of the DIAObject. 

""" 

with self._db_connection as conn: 

conn.execute("BEGIN") 

conn.execute( 

"CREATE TEMPORARY TABLE tmp_indexer_indices " 

"(pixelId INTEGER PRIMARY KEY)") 

 

conn.executemany( 

"INSERT OR REPLACE INTO tmp_indexer_indices VALUES (?)", 

[(int(indexer_index),) for indexer_index in indexer_indices]) 

 

cursor = conn.execute( 

"SELECT o.* FROM dia_objects AS o " 

"INNER JOIN tmp_indexer_indices AS i " 

"ON o.pixelId = i.pixelId") 

 

output_rows = cursor.fetchall() 

 

conn.execute("DROP TABLE tmp_indexer_indices") 

 

return output_rows 

 

def _check_dia_object_position(self, dia_object_record, bbox, wcs): 

"""Check the RA, DEC position of the current dia_object_record against 

the bounding box of the exposure. 

 

Parameters 

---------- 

dia_object_record : `lsst.afw.table.SourceRecord` 

A SourceRecord object containing the DIAObject we would like to 

test against our bounding box. 

bbox : `lsst.geom.Box2D` 

Bounding box of exposure. 

wcs : `lsst.geom.SkyWcs` 

WCS of exposure. 

 

Return 

------ 

is_contained : `bool` 

Object position is contained within the bounding box of expMd. 

""" 

point = wcs.skyToPixel(dia_object_record.getCoord()) 

return bbox.contains(point) 

 

def _query_dia_sources(self, dia_object_ids): 

"""Query the database for the stored DIASources given a set of 

DIAObject ids. 

 

Parameters 

---------- 

dia_object_ids : array-like of `int` 

Spatial indices in the indexer specifying the area on the sky 

to load DIAObjects for. 

 

Return 

------ 

dia_objects : `list` of `tuples` 

Query result containing the values representing DIASources 

""" 

with self._db_connection as conn: 

conn.execute("BEGIN") 

conn.execute( 

"CREATE TEMPORARY TABLE tmp_object_ids " 

"(diaObjectId INTEGER PRIMARY KEY)") 

 

conn.executemany( 

"INSERT OR REPLACE INTO tmp_object_ids VALUES (?)", 

[(int(dia_object_id),) for dia_object_id in dia_object_ids]) 

 

cursor = conn.execute( 

"SELECT s.* FROM dia_sources AS s " 

"INNER JOIN tmp_object_ids AS o " 

"ON s.diaObjectId = o.diaObjectId") 

 

output_rows = cursor.fetchall() 

 

conn.execute("DROP TABLE tmp_object_ids") 

 

return output_rows 

 

def _store_catalog(self, source_catalog, converter, obj_ids=None, exposure=None): 

""" Store a SourceCatalog into the database. 

 

Parameters 

---------- 

source_catalog : `lsst.afw.table.SourceCatalog` 

SourceCatalog to store in the database table specified by 

converter. 

converter : `lsst.ap.association.SqliteDBConverter` 

A converter object specifying the correct database table to write 

into. 

obj_id : array-like of `int` (optional) 

Ids of the DIAObjects these objects are associated with. Use only 

when storing DIASources. 

exposure : `lsst.afw.image.Exposure` (optional) 

Exposure that the sources in source_catalog were detected in. If 

set the fluxes are calibrated and stored using the exposure Calib 

object. The filter the exposure was taken in as well as the 

ccdVisitId are also stored. 

""" 

values = [] 

 

if converter.table_name == 'dia_sources': 

# Create aliases to appropriate flux fields if they exist. 

add_dia_source_aliases_to_catalog(source_catalog) 

 

if exposure is None: 

exp_dict = None 

else: 

exp_dict = get_ccd_visit_info_from_exposure(exposure) 

 

for src_idx, source_record in enumerate(source_catalog): 

if obj_ids is None: 

obj_id = None 

else: 

obj_id = obj_ids[src_idx] 

 

overwrite_dict = make_overwrite_dict(source_record, 

obj_id, 

exp_dict) 

 

values.append(converter.source_record_to_value_list( 

source_record, overwrite_dict)) 

 

insert_string = ("?," * len(values[0]))[:-1] 

 

with self._db_connection as conn: 

conn.executemany( 

"INSERT OR REPLACE INTO %s VALUES (%s)" % 

(converter.table_name, insert_string), values) 

 

@property 

def dia_object_afw_schema(self): 

"""Retrieve the Schema of the DIAObjects in this database. 

 

Returns 

------- 

schema : `lsst.afw.table.Schema` 

Schema of the DIAObjects in this database. 

""" 

return self._dia_object_converter.schema 

 

@property 

def dia_source_afw_schema(self): 

"""Retrieve the Schema of the DIASources in this database. 

 

Returns 

------- 

schema : `lsst.afw.table.Schema` 

Schema of the DIASources in this database. 

""" 

return self._dia_source_converter.schema