41 # define __attribute__(x) 46 #include "boost/regex.hpp" 57 #include <mysql/mysql.h> 71 LOG_LOGGER _log =
LOG_GET(
"daf.persistence.DbStorage");
76 namespace persistence {
91 template<> enum_field_types
93 template<> enum_field_types
95 template<> enum_field_types
97 template<> enum_field_types
100 template <
typename N> enum_field_types
104 template<> enum_field_types
120 template<> enum_field_types
124 template<> enum_field_types
128 template<> enum_field_types
132 template<> enum_field_types
145 lsst::daf::base::Citizen(typeid(*this)), _data(0) {
168 dafPer::DbStorageImpl::DbStorageImpl(
void) :
196 void dafPer::DbStorageImpl::startSession(
std::string const& location) {
198 setenv(
"TZ",
"UTC", 1);
207 unsigned int port = strtoul(dbloc.
getPort().
c_str(), 0, 10);
208 if (mysql_real_connect(_db,
214 error(
"Unable to connect to MySQL database: " + _location);
221 void dafPer::DbStorageImpl::setPersistLocation(
LogicalLocation const& location) {
229 void dafPer::DbStorageImpl::setRetrieveLocation(
LogicalLocation const& location) {
236 void dafPer::DbStorageImpl::startTransaction(
void) {
237 if (_db == 0) error(
"Database session not initialized " 238 "in DbStorage::startTransaction()",
false);
239 if (mysql_autocommit(_db,
false)) error(
"Unable to turn off autocommit");
244 void dafPer::DbStorageImpl::endTransaction(
void) {
245 if (_db == 0) error(
"Database session not initialized " 246 "in DbStorage::endTransaction()",
false);
247 if (mysql_commit(_db)) error(
"Unable to commit transaction");
248 if (mysql_autocommit(_db,
true)) error(
"Unable to turn on autocommit");
257 void dafPer::DbStorageImpl::executeQuery(
std::string const& query) {
259 error(
"No DB connection for query: " + query,
false);
262 if (mysql_query(_db, query.
c_str()) != 0) {
263 error(
"Unable to execute query: " + query);
270 std::string::size_type pos = name.
find(
'.');
271 if (pos == std::string::npos)
return '`' + name +
'`';
276 void dafPer::DbStorageImpl::stError(
std::string const& text) {
277 error(text +
" - * " + mysql_stmt_error(_statement),
false);
280 void dafPer::DbStorageImpl::error(
std::string const& text,
bool mysqlCause) {
289 void* dafPer::DbStorageImpl::allocateMemory(
size_t size) {
290 boost::shared_array<char> mem(
new char[size]);
291 _bindingMemory.push_back(mem);
304 void dafPer::DbStorageImpl::createTableFromTemplate(
std::string const& tableName,
306 bool mayAlreadyExist) {
308 if (mayAlreadyExist) query +=
"IF NOT EXISTS ";
309 query += quote(tableName) +
" LIKE " + quote(templateName);
316 void dafPer::DbStorageImpl::dropTable(
std::string const& tableName) {
317 executeQuery(
"DROP TABLE " + quote(tableName));
323 void dafPer::DbStorageImpl::truncateTable(
std::string const& tableName) {
324 executeQuery(
"TRUNCATE TABLE " + quote(tableName));
331 void dafPer::DbStorageImpl::executeSql(
std::string const& sqlStatement) {
332 executeQuery(sqlStatement);
342 void dafPer::DbStorageImpl::setTableForInsert(
std::string const& tableName) {
344 error(
"Attempt to insert into read-only database",
false);
346 _insertTable = tableName;
354 template <
typename T>
355 void dafPer::DbStorageImpl::setColumn(
std::string const& columnName,
357 BoundVarMap::iterator bv = _inputVars.find(columnName);
358 size_t size =
sizeof(T);
359 if (bv == _inputVars.end()) {
360 bv = _inputVars.insert(
361 BoundVarMap::value_type(columnName,
362 BoundVar(allocateMemory(size)))).first;
364 else if (bv->second._length != size) {
365 bv->second._data = allocateMemory(size);
368 bv->second._isNull =
false;
370 bv->second._length = size;
371 memcpy(bv->second._data, &value, size);
375 void dafPer::DbStorageImpl::setColumn(
std::string const& columnName,
377 BoundVarMap::iterator bv = _inputVars.find(columnName);
378 size_t size = value.
length();
379 if (bv == _inputVars.end()) {
380 bv = _inputVars.insert(
381 BoundVarMap::value_type(columnName,
382 BoundVar(allocateMemory(size)))).first;
384 else if (bv->second._length != size) {
385 bv->second._data = allocateMemory(size);
388 bv->second._isNull =
false;
390 bv->second._length = size;
391 memcpy(bv->second._data, value.
data(), size);
395 void dafPer::DbStorageImpl::setColumn(
std::string const& columnName,
397 BoundVarMap::iterator bv = _inputVars.find(columnName);
398 size_t size =
sizeof(MYSQL_TIME);
399 if (bv == _inputVars.end()) {
400 bv = _inputVars.insert(
401 BoundVarMap::value_type(columnName,
402 BoundVar(allocateMemory(size)))).first;
404 else if (bv->second._length != size) {
405 bv->second._data = allocateMemory(size);
408 bv->second._isNull =
false;
410 bv->second._length = size;
412 MYSQL_TIME* t =
reinterpret_cast<MYSQL_TIME*
>(bv->second._data);
413 t->year = v.tm_year + 1900;
414 t->month = v.tm_mon + 1;
417 t->minute = v.tm_min;
418 t->second = v.tm_sec;
421 static_cast<unsigned long>((value.
nsecs() % 1000000000LL) / 1000);
427 void dafPer::DbStorageImpl::setColumnToNull(
std::string const& columnName) {
428 BoundVarMap::iterator bv = _inputVars.find(columnName);
429 if (bv == _inputVars.end()) {
430 bv = _inputVars.insert(
431 BoundVarMap::value_type(columnName,
432 BoundVar(allocateMemory(1)))).first;
434 bv->second._isNull =
true;
435 bv->second._length = 1;
441 void dafPer::DbStorageImpl::insertRow(
void) {
443 error(
"Attempt to insert into read-only database",
false);
445 if (_insertTable.empty()) error(
"Insert table not initialized in DbStorage::insertRow()",
false);
446 if (_inputVars.empty()) error(
"No values to insert",
false);
448 std::string query =
"INSERT INTO " + quote(_insertTable) +
" (";
451 memset(binder.get(), 0, _inputVars.size() *
sizeof(MYSQL_BIND));
454 for (BoundVarMap::iterator it = _inputVars.begin();
455 it != _inputVars.end(); ++it) {
456 if (it != _inputVars.begin()) {
459 query += quote(it->first);
462 MYSQL_BIND& bind(binder[i]);
465 bind.buffer_type = MYSQL_TYPE_NULL;
468 bind.buffer_type = bv.
_type;
469 bind.buffer = bv.
_data;
470 bind.buffer_length = bv.
_length;
478 query +=
") VALUES (";
479 for (
size_t i = 0; i < _inputVars.size(); ++i) {
489 _statement = mysql_stmt_init(_db);
490 if (_statement == 0) {
491 error(
"Unable to initialize statement: " + query);
493 if (mysql_stmt_prepare(_statement, query.
c_str(), query.
length()) != 0) {
494 stError(
"Unable to prepare statement: " + query);
496 if (mysql_stmt_bind_param(_statement, binder.get())) {
497 stError(
"Unable to bind variables in: " + query);
499 if (mysql_stmt_execute(_statement) != 0) {
500 stError(
"Unable to execute statement: " + query);
502 mysql_stmt_close(_statement);
514 void dafPer::DbStorageImpl::setTableForQuery(
std::string const& tableName,
516 if (_db == 0) error(
"Database session not initialized in DbStorage::setTableForQuery()",
false);
517 _queryTables.clear();
518 _queryTables.push_back(isExpr ? tableName : quote(tableName));
522 _whereClause.clear();
531 void dafPer::DbStorageImpl::setTableListForQuery(
533 if (_db == 0) error(
"Database session not initialized in DbStorage::setTableListForQuery()",
false);
535 it != tableNameList.
end(); ++it) {
536 _queryTables.push_back(quote(*it));
541 _whereClause.clear();
554 void dafPer::DbStorageImpl::outColumn(
std::string const& columnName,
556 std::string col = isExpr ? columnName : quote(columnName);
557 _outColumns.push_back(col);
568 template <
typename T>
569 void dafPer::DbStorageImpl::outParam(
std::string const& columnName,
570 T* location,
bool isExpr) {
571 std::string col = isExpr ? columnName : quote(columnName);
572 _outColumns.push_back(col);
573 size_t size =
sizeof(T);
575 BoundVarMap::value_type(col,
BoundVar(location)));
577 error(
"Duplicate column name requested: " + columnName,
false);
587 void dafPer::DbStorageImpl::outParam(
std::string const& columnName,
589 std::string col = isExpr ? columnName : quote(columnName);
590 _outColumns.push_back(col);
593 BoundVarMap::value_type(
596 error(
"Duplicate column name requested: " + columnName,
false);
607 void dafPer::DbStorageImpl::outParam(
std::string const& columnName,
610 std::string col = isExpr ? columnName : quote(columnName);
611 _outColumns.push_back(col);
612 size_t size =
sizeof(MYSQL_TIME);
614 BoundVarMap::value_type(
617 error(
"Duplicate column name requested: " + columnName,
false);
632 template <
typename T>
633 void dafPer::DbStorageImpl::condParam(
std::string const& paramName, T
const& value) {
634 setColumn<T>(paramName, value);
641 void dafPer::DbStorageImpl::orderBy(
std::string const& expression) {
642 if (!_orderBy.empty()) {
645 _orderBy += expression;
651 void dafPer::DbStorageImpl::groupBy(
std::string const& expression) {
652 if (!_groupBy.empty()) {
655 _groupBy += expression;
663 void dafPer::DbStorageImpl::setQueryWhere(
std::string const& whereClause) {
664 _whereClause = whereClause;
669 void dafPer::DbStorageImpl::query(
void) {
670 if (_outColumns.empty()) error(
"No output columns for query",
false);
678 it != _outColumns.end(); ++it) {
679 if (it != _outColumns.begin()) {
688 it != _queryTables.end(); ++it) {
689 if (it != _queryTables.begin()) {
697 if (!_whereClause.empty()) {
698 boost::regex re(
":([A-Za-z_]+)");
701 boost::regex_iterator<std::string::iterator>
m;
702 for (boost::regex_iterator<std::string::iterator> i(
703 _whereClause.begin(), _whereClause.end(), re);
704 i != boost::regex_iterator<std::string::iterator>(); ++i) {
706 std::copy(m->prefix().first, m->prefix().second, out);
708 assert(m->size() == 2);
711 if (m != boost::regex_iterator<std::string::iterator>()) {
712 std::copy(m->suffix().first, m->suffix().second, out);
715 std::copy(_whereClause.begin(), _whereClause.end(), out);
717 query +=
" WHERE " + result;
721 if (!_groupBy.empty()) query +=
" GROUP BY " + _groupBy;
724 if (!_orderBy.empty()) query +=
" ORDER BY " + _orderBy;
730 new MYSQL_BIND[whereBindings.
size()]);
731 memset(inBinder.get(), 0, whereBindings.
size() *
sizeof(MYSQL_BIND));
732 for (
size_t i = 0; i < whereBindings.
size(); ++i) {
733 MYSQL_BIND& bind(inBinder[i]);
734 BoundVarMap::iterator it = _inputVars.find(whereBindings[i]);
735 if (it == _inputVars.end()) {
736 error(
"Unbound variable in WHERE clause: " + whereBindings[i],
740 bind.buffer_type = bv.
_type;
741 bind.buffer = bv.
_data;
742 bind.buffer_length = bv.
_length;
751 _statement = mysql_stmt_init(_db);
753 error(
"Unable to initialize prepared statement");
756 if (mysql_stmt_prepare(_statement, query.
c_str(), query.
length()) != 0) {
757 stError(
"Unable to prepare statement: " + query);
762 unsigned int params = mysql_stmt_param_count(_statement);
763 if (_whereClause.empty()) {
765 error(
"Unbound WHERE clause parameters: " + query,
false);
769 if (params != whereBindings.
size()) {
770 error(
"Mismatch in number of WHERE clause parameters: " + query,
773 if (mysql_stmt_bind_param(_statement, inBinder.get())) {
774 stError(
"Unable to bind WHERE parameters: " + query);
779 MYSQL_RES* queryMetadata = mysql_stmt_result_metadata(_statement);
780 if (!queryMetadata) {
781 stError(
"No query metadata: " + query);
783 _numResultFields = mysql_num_fields(queryMetadata);
784 if (static_cast<unsigned int>(_numResultFields) != _outColumns.size()) {
785 error(
"Mismatch in number of SELECT items: " + query,
false);
791 if (mysql_stmt_execute(_statement) != 0) {
792 stError(
"MySQL query failed: " + query);
798 _resultFields = mysql_fetch_fields(queryMetadata);
801 memset(outBinder.
get(), 0, _numResultFields *
sizeof(MYSQL_BIND));
802 _fieldLengths.reset(
new unsigned long[_numResultFields]);
803 _fieldNulls.reset(
new my_bool[_numResultFields]);
805 for (
int i = 0; i < _numResultFields; ++i) {
806 MYSQL_BIND& bind(outBinder[i]);
807 if (_outputVars.empty()) {
808 bind.buffer_type = MYSQL_TYPE_STRING;
810 bind.buffer_length = 0;
811 bind.length = &(_fieldLengths[i]);
812 bind.is_null = &(_fieldNulls[i]);
813 bind.is_unsigned = (_resultFields[i].flags & UNSIGNED_FLAG) != 0;
817 BoundVarMap::iterator it = _outputVars.find(_outColumns[i]);
818 if (it == _outputVars.end()) {
819 error(
"Unbound variable in SELECT clause: " + _outColumns[i],
824 bind.buffer_type = bv.
_type;
826 bind.buffer =
reinterpret_cast<char*
>(bv.
_data) +
830 bind.buffer =
reinterpret_cast<char*
>(bv.
_data) +
834 bind.buffer = bv.
_data;
836 bind.buffer_length = bv.
_length;
837 bind.length = &(_fieldLengths[i]);
838 bind.is_null = &(_fieldNulls[i]);
843 if (mysql_stmt_bind_result(_statement, outBinder.
get())) {
844 stError(
"Unable to bind results: " + query);
851 bool dafPer::DbStorageImpl::next(
void) {
852 if (_statement == 0) {
853 error(
"Statement not initialized in DbStorage::next()",
false);
855 int ret = mysql_stmt_fetch(_statement);
858 if (!_outputVars.empty()) {
859 for (
size_t i = 0; i < _outColumns.size(); ++i) {
860 BoundVarMap::iterator bvit = _outputVars.find(_outColumns[i]);
861 if (bvit == _outputVars.end()) {
862 error(
"Unbound variable in SELECT clause: " +
863 _outColumns[i],
false);
873 char* cp =
reinterpret_cast<char*
>(bv.
_data) +
875 MYSQL_TIME* t =
reinterpret_cast<MYSQL_TIME*
>(cp);
878 t->hour, t->minute, t->second,
885 if (ret == MYSQL_NO_DATA)
return false;
886 if (ret == MYSQL_DATA_TRUNCATED && _outputVars.empty())
return true;
887 stError(
"Error fetching next row");
895 template <
typename T>
896 T
const& dafPer::DbStorageImpl::getColumnByPos(
int pos) {
897 if (pos > _numResultFields) {
899 os <<
"Nonexistent column: " << pos;
900 error(os.
str(),
false);
903 memset(&bind, 0,
sizeof(MYSQL_BIND));
908 bind.buffer_length =
sizeof(T);
909 bind.length = &(_fieldLengths[pos]);
910 bind.is_null = &(_fieldNulls[pos]);
911 if (mysql_stmt_fetch_column(_statement, &bind, pos, 0)) {
913 os <<
"Error fetching column: " << pos;
914 error(os.
str(),
false);
920 std::string const& dafPer::DbStorageImpl::getColumnByPos(
int pos) {
921 if (pos > _numResultFields) {
923 os <<
"Nonexistent column: " << pos;
924 error(os.
str(),
false);
927 memset(&bind, 0,
sizeof(MYSQL_BIND));
928 if (_resultFields[pos].type == MYSQL_TYPE_BIT) {
929 error(
"Invalid type for string retrieval",
false);
934 bind.buffer = t.
get();
935 bind.buffer_length = _fieldLengths[pos];
936 bind.length = &(_fieldLengths[pos]);
937 bind.is_null = &(_fieldNulls[pos]);
938 if (mysql_stmt_fetch_column(_statement, &bind, pos, 0)) {
940 os <<
"Error fetching string column: " << pos;
950 if (pos > _numResultFields) {
952 os <<
"Nonexistent column: " << pos;
953 error(os.
str(),
false);
956 memset(&bind, 0,
sizeof(MYSQL_BIND));
957 if (_resultFields[pos].type != MYSQL_TYPE_TIME &&
958 _resultFields[pos].type != MYSQL_TYPE_DATE &&
959 _resultFields[pos].type != MYSQL_TYPE_DATETIME &&
960 _resultFields[pos].type != MYSQL_TYPE_TIMESTAMP) {
961 error(
"Invalid type for DateTime retrieval",
false);
967 bind.buffer_length =
sizeof(MYSQL_TIME);
968 bind.length = &(_fieldLengths[pos]);
969 bind.is_null = &(_fieldNulls[pos]);
970 if (mysql_stmt_fetch_column(_statement, &bind, pos, 0)) {
972 os <<
"Error fetching DateTime column: " << pos;
985 bool dafPer::DbStorageImpl::columnIsNull(
int pos) {
986 if (pos > _numResultFields) {
988 os <<
"Nonexistent column: " << pos;
989 error(os.
str(),
false);
991 return _fieldNulls[pos];
996 void dafPer::DbStorageImpl::finishQuery(
void) {
997 mysql_stmt_close(_statement);
1005 template void dafPer::DbStorageImpl::setColumn<>(
std::string const& columnName,
char const& value);
1006 template void dafPer::DbStorageImpl::setColumn<>(
std::string const& columnName,
short const& value);
1007 template void dafPer::DbStorageImpl::setColumn<>(
std::string const& columnName,
int const& value);
1008 template void dafPer::DbStorageImpl::setColumn<>(
std::string const& columnName,
long const& value);
1009 template void dafPer::DbStorageImpl::setColumn<>(
std::string const& columnName,
long long const& value);
1010 template void dafPer::DbStorageImpl::setColumn<>(
std::string const& columnName,
float const& value);
1011 template void dafPer::DbStorageImpl::setColumn<>(
std::string const& columnName,
double const& value);
1012 template void dafPer::DbStorageImpl::setColumn<>(
std::string const& columnName,
std::string const& value);
1013 template void dafPer::DbStorageImpl::setColumn<>(
std::string const& columnName,
bool const& value);
1016 template void dafPer::DbStorageImpl::outParam<>(
std::string const& columnName,
char* location,
bool isExpr);
1017 template void dafPer::DbStorageImpl::outParam<>(
std::string const& columnName,
short* location,
bool isExpr);
1018 template void dafPer::DbStorageImpl::outParam<>(
std::string const& columnName,
int* location,
bool isExpr);
1019 template void dafPer::DbStorageImpl::outParam<>(
std::string const& columnName,
long* location,
bool isExpr);
1020 template void dafPer::DbStorageImpl::outParam<>(
std::string const& columnName,
long long* location,
bool isExpr);
1021 template void dafPer::DbStorageImpl::outParam<>(
std::string const& columnName,
float* location,
bool isExpr);
1022 template void dafPer::DbStorageImpl::outParam<>(
std::string const& columnName,
double* location,
bool isExpr);
1023 template void dafPer::DbStorageImpl::outParam<>(
std::string const& columnName,
std::string* location,
bool isExpr);
1024 template void dafPer::DbStorageImpl::outParam<>(
std::string const& columnName,
bool* location,
bool isExpr);
1027 template void dafPer::DbStorageImpl::condParam<>(
std::string const& paramName,
char const& value);
1028 template void dafPer::DbStorageImpl::condParam<>(
std::string const& paramName,
short const& value);
1029 template void dafPer::DbStorageImpl::condParam<>(
std::string const& paramName,
int const& value);
1030 template void dafPer::DbStorageImpl::condParam<>(
std::string const& paramName,
long const& value);
1031 template void dafPer::DbStorageImpl::condParam<>(
std::string const& paramName,
long long const& value);
1032 template void dafPer::DbStorageImpl::condParam<>(
std::string const& paramName,
float const& value);
1033 template void dafPer::DbStorageImpl::condParam<>(
std::string const& paramName,
double const& value);
1034 template void dafPer::DbStorageImpl::condParam<>(
std::string const& paramName,
std::string const& value);
1035 template void dafPer::DbStorageImpl::condParam<>(
std::string const& paramName,
bool const& value);
1038 template char const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
1039 template short const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
1040 template int const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
1041 template long const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
1042 template long long const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
1043 template float const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
1044 template double const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
1045 template std::string const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
1046 template bool const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
1047 template dafBase::DateTime const& dafPer::DbStorageImpl::getColumnByPos<>(
int pos);
std::shared_ptr< Policy > Ptr
virtual std::string const & getDbName(void) const
Accessor for database name.
struct tm gmtime(Timescale scale) const
Class for logical location of a persisted Persistable instance.
std::string const & locString(void) const
Accessor.
virtual std::string const & getHostname(void) const
Accessor for database hostname.
virtual std::string const & getPassword(void) const
Accessor for password.
BoundVar(void)
Default constructor.
virtual ~DbStorageImpl(void)
Destructor.
Location of a persisted Persistable instance in a database.
virtual std::string const & getUsername(void) const
Accessor for username.
long long nsecs(Timescale scale=TAI) const
virtual std::string const & getPort(void) const
Accessor for database port number.
#define LSST_EXCEPT(type,...)
static enum_field_types mysqlType
Citizen(const std::type_info &)
#define LOGLS_DEBUG(logger, message)
Interface for DbStorageImpl class.
static enum_field_types mysqlType
Interface for LogicalLocation class.
Interface for DbStorageLocation class.