Drizzled Public API Documentation

drizzledump_data.cc
00001 /* -*- mode: c++; c-basic-offset: 2; indent-tabs-mode: nil; -*-
00002  *  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
00003  *
00004  *  Copyright (C) 2010 Andrew Hutchings
00005  *
00006  *  This program is free software; you can redistribute it and/or modify
00007  *  it under the terms of the GNU General Public License as published by
00008  *  the Free Software Foundation; version 2 of the License.
00009  *
00010  *  This program is distributed in the hope that it will be useful,
00011  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
00012  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00013  *  GNU General Public License for more details.
00014  *
00015  *  You should have received a copy of the GNU General Public License
00016  *  along with this program; if not, write to the Free Software
00017  *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
00018  */
00019 
00020 #include "drizzledump_data.h"
00021 #include "client_priv.h"
00022 #include <drizzled/gettext.h>
00023 #include <string>
00024 #include <iostream>
00025 #include <boost/regex.hpp>
00026 #include <boost/unordered_set.hpp>
00027 
00028 #define EX_DRIZZLEERR 2
00029 
00030 extern bool opt_no_create_info;
00031 extern bool opt_no_data;
00032 extern bool opt_create_db;
00033 extern bool opt_disable_keys;
00034 extern bool extended_insert;
00035 extern bool opt_replace_into;
00036 extern bool opt_drop;
00037 extern bool verbose;
00038 extern bool opt_databases;
00039 extern bool opt_alldbs;
00040 extern uint32_t show_progress_size;
00041 extern bool opt_ignore;
00042 extern bool opt_compress;
00043 extern bool opt_drop_database;
00044 extern bool opt_autocommit;
00045 extern bool ignore_errors;
00046 extern std::string opt_destination_database;
00047 
00048 extern boost::unordered_set<std::string> ignore_table;
00049 extern void maybe_exit(int error);
00050 
00051 enum destinations {
00052   DESTINATION_DB,
00053   DESTINATION_FILES,
00054   DESTINATION_STDOUT
00055 };
00056 
00057 extern int opt_destination;
00058 
00059 // returns true on keep, false on ignore Olaf: sounds backwards/wrong, shouldn't it be the other way around?
00060 bool DrizzleDumpDatabase::ignoreTable(std::string tableName)
00061 {
00062   return ignore_table.find(databaseName + "." + tableName) == ignore_table.end();
00063 }
00064 
00065 void DrizzleDumpDatabase::cleanTableName(std::string &tableName)
00066 {
00067   std::string replace("``");
00068   std::string find("`");
00069   size_t j = 0;
00070   for (;(j = tableName.find(find, j)) != std::string::npos;)
00071   {
00072     tableName.replace(j, find.length(), replace);
00073     j+= replace.length();
00074   }
00075 
00076 }
00077 
00078 std::ostream& operator <<(std::ostream &os, const DrizzleDumpForeignKey &obj)
00079 {
00080   os << "  CONSTRAINT `" << obj.constraintName << "` FOREIGN KEY ("
00081     << obj.parentColumns << ") REFERENCES `" << obj.childTable << "` ("
00082     << obj.childColumns << ")";
00083 
00084   if (not obj.deleteRule.empty())
00085     os << " ON DELETE " << obj.deleteRule;
00086 
00087   if (not obj.updateRule.empty())
00088     os << " ON UPDATE " << obj.updateRule;
00089 
00090   return os;
00091 }
00092 
00093 std::ostream& operator <<(std::ostream &os, const DrizzleDumpIndex &obj)
00094 {
00095   if (obj.isPrimary)
00096   {
00097     os << "  PRIMARY KEY ";
00098   }
00099   else if (obj.isUnique)
00100   {
00101     os << "  UNIQUE KEY `" << obj.indexName << "` ";
00102   }
00103   else
00104   {
00105     os << "  KEY `" << obj.indexName << "` ";
00106   }
00107 
00108   os << "(";
00109   
00110   std::vector<DrizzleDumpIndex::columnData>::iterator i;
00111   std::vector<DrizzleDumpIndex::columnData> fields = obj.columns;
00112   for (i= fields.begin(); i != fields.end(); ++i)
00113   {
00114     if (i != fields.begin())
00115       os << ",";
00116     os << "`" << (*i).first << "`";
00117     if ((*i).second > 0)
00118       os << "(" << (*i).second << ")";
00119   }
00120 
00121   os << ")";
00122 
00123   return os;
00124 }
00125 
00126 std::ostream& operator <<(std::ostream &os, const DrizzleDumpField &obj)
00127 {
00128   os << "  `" << obj.fieldName << "` ";
00129   os << obj.type;
00130   if (((obj.type.compare("VARCHAR") == 0) or
00131    (obj.type.compare("VARBINARY") == 0)) and
00132    (obj.length > 0))
00133   {
00134     os << "(" << obj.length << ")";
00135   }
00136   else if (((obj.type.compare("DECIMAL") == 0) or
00137     (obj.type.compare("DOUBLE") == 0)) and
00138     ((obj.decimalPrecision + obj.decimalScale) > 0))
00139   {
00140     os << "(" << obj.decimalPrecision << "," << obj.decimalScale << ")";
00141   }
00142   else if (obj.type.compare("ENUM") == 0)
00143   {
00144     os << "(" << obj.enumValues << ")";
00145   }
00146 
00147   if (not obj.isNull)
00148   {
00149     os << " NOT NULL";
00150   }
00151 
00152   if ((not obj.collation.empty()) and (obj.collation.compare("binary") != 0))
00153   {
00154     os << " COLLATE " << obj.collation;
00155   }
00156 
00157   if (obj.isAutoIncrement)
00158     os << " AUTO_INCREMENT";
00159 
00160   if (not obj.defaultValue.empty())
00161   {
00162     if (obj.defaultValue.compare("CURRENT_TIMESTAMP") != 0)
00163     {
00164       if (obj.defaultValue.compare(0, 2, "b'") == 0)
00165       {
00166         os << " DEFAULT " << obj.defaultValue;
00167       }
00168       else
00169       {
00170         os << " DEFAULT '" << obj.defaultValue << "'";
00171       }
00172     }
00173     else
00174     {
00175      os << " DEFAULT CURRENT_TIMESTAMP";
00176     }
00177   }
00178   else if ((obj.defaultIsNull))
00179   {
00180     os << " DEFAULT NULL";
00181   }
00182 
00183   if (not obj.comment.empty())
00184   {
00185     os << " COMMENT '" << DrizzleDumpData::escape(obj.comment.c_str(), obj.comment.length()) << "'";
00186   }
00187 
00188   return os;
00189 }
00190 
00191 std::ostream& operator <<(std::ostream &os, const DrizzleDumpDatabase &obj)
00192 {
00193   if ((opt_destination == DESTINATION_DB) or opt_databases or opt_alldbs)
00194   {
00195     if (verbose)
00196     {
00197       std::cerr << "--" << std::endl
00198         << "-- Current Database: `" << obj.databaseName << "`" << std::endl
00199         << "--" << std::endl << std::endl;
00200     }
00201 
00202     /* Love that this variable is the opposite of its name */
00203     if (not opt_create_db)
00204     {
00205       if (opt_drop_database)
00206       {
00207         os << "DROP DATABASE IF EXISTS `"
00208           << ((opt_destination_database.empty()) ? obj.databaseName
00209           : opt_destination_database) << "`" << std::endl;
00210       }
00211 
00212       os << "CREATE DATABASE IF NOT EXISTS `"
00213         << ((opt_destination_database.empty()) ? obj.databaseName
00214         : opt_destination_database) << "`";
00215       if (not obj.collate.empty())
00216        os << " COLLATE = " << obj.collate;
00217 
00218       os << ";" << std::endl << std::endl;
00219     }
00220     os << "USE `" << ((opt_destination_database.empty()) ? obj.databaseName
00221       : opt_destination_database) << "`;" << std::endl << std::endl;
00222   }
00223 
00224   std::vector<DrizzleDumpTable*>::iterator i;
00225   std::vector<DrizzleDumpTable*> output_tables = obj.tables;
00226   for (i= output_tables.begin(); i != output_tables.end(); ++i)
00227   {
00228     DrizzleDumpTable *table= *i;
00229     if (not opt_no_create_info)
00230       os << *table;
00231     if (not opt_no_data)
00232     {
00233       obj.dcon->setDB(obj.databaseName);
00234       DrizzleDumpData *data= table->getData();
00235       if (data == NULL)
00236       {
00237         std::cerr << "Error: Could not get data for table " << table->displayName << std::endl;
00238         if (not ignore_errors)
00239           maybe_exit(EX_DRIZZLEERR);
00240         else
00241           continue;
00242       }
00243       os << *data;
00244       delete data;
00245     }
00246   }
00247 
00248   return os;
00249 }
00250 
00251 
00252 std::ostream& operator <<(std::ostream &os, const DrizzleDumpData &obj)
00253 {
00254   bool new_insert= true;
00255   bool first= true;
00256   uint64_t rownr= 0;
00257   size_t byte_counter= 0;
00258 
00259   drizzle_row_t row;
00260 
00261   if (verbose)
00262     std::cerr << _("-- Retrieving data for ") << obj.table->displayName << "..." << std::endl;
00263 
00264   if (drizzle_result_row_count(obj.result) < 1)
00265   {
00266     if (verbose)
00267     {
00268       std::cerr << "--" << std::endl
00269         << "-- No data to dump for table `" << obj.table->displayName << "`"
00270         << std::endl << "--" << std::endl << std::endl;
00271     }
00272     return os;
00273   }
00274   else if (verbose)
00275   {
00276     std::cerr << "--" << std::endl
00277       << "-- Dumping data for table `" << obj.table->displayName << "`"
00278       << std::endl << "--" << std::endl << std::endl;
00279   }
00280   if (opt_disable_keys)
00281     os << "ALTER TABLE `" << obj.table->displayName << "` DISABLE KEYS;" << std::endl;
00282 
00283   /* Another option that does the opposite of its name, makes me sad :( */
00284   if (opt_autocommit)
00285     os << "START TRANSACTION;" << std::endl;
00286 
00287   std::streampos out_position= os.tellp();
00288 
00289   while((row= drizzle_row_next(obj.result)))
00290   {
00291     rownr++;
00292     if (verbose and (rownr % show_progress_size) == 0)
00293     {
00294       std::cerr << "-- " << rownr << _(" rows dumped for table ") << obj.table->displayName << std::endl;
00295     }
00296 
00297     size_t* row_sizes= drizzle_row_field_sizes(obj.result);
00298     for (uint32_t i= 0; i < drizzle_result_column_count(obj.result); i++)
00299       byte_counter+= row_sizes[i];
00300 
00301     if (not first and not new_insert)
00302     {
00303       if (extended_insert)
00304         os << "),(";
00305       else
00306         os << ");" << std::endl;
00307       byte_counter+= 3;
00308     }
00309     else
00310       first= false;
00311 
00312     if (new_insert)
00313     {
00314       if (opt_replace_into)
00315         os << "REPLACE ";
00316       else
00317       {
00318         os << "INSERT ";
00319         if (opt_ignore)
00320           os << "IGNORE ";
00321       }
00322       os << "INTO `" << obj.table->displayName << "` VALUES (";
00323       byte_counter+= 28 + obj.table->displayName.length();
00324       if (extended_insert)
00325         new_insert= false;
00326     }
00327     for (uint32_t i= 0; i < drizzle_result_column_count(obj.result); i++)
00328     {
00329       if (not row[i])
00330       {
00331         os << "NULL";
00332         if (i != obj.table->fields.size() - 1)
00333           os << ",";
00334         continue;
00335       }
00336 
00337       if ((obj.table->fields[i]->rangeCheck) and
00338         (obj.table->fields[i]->type.compare("BIGINT") == 0) and
00339         (boost::lexical_cast<uint64_t>(row[i]) > INT64_MAX))
00340       {
00341         std::cerr << "Error: Data for column " << obj.table->fields[i]->fieldName << " is greater than max BIGINT, cannot migrate automatically" << std::endl;
00342         if (not ignore_errors)
00343           maybe_exit(EX_DRIZZLEERR);
00344         else
00345           continue;
00346       }
00347 
00348       /* time/date conversion for MySQL connections */
00349       else if (obj.table->fields[i]->convertDateTime)
00350       {
00351         os << obj.checkDateTime(row[i], i);
00352       }
00353       else
00354       {
00355         if ((obj.table->fields[i]->type.compare("INT") != 0) and
00356           (obj.table->fields[i]->type.compare("BIGINT") != 0))
00357         {
00358           /* Hex blob processing or escape text */
00359           if (((obj.table->fields[i]->type.compare("BLOB") == 0) or
00360             (obj.table->fields[i]->type.compare("VARBINARY") == 0)))
00361           {
00362             os << obj.convertHex((unsigned char*)row[i], row_sizes[i]);
00363             byte_counter+= row_sizes[i];
00364           }
00365           else if ((obj.table->fields[i]->type.compare("ENUM") == 0) and
00366             (strcmp(row[i], "") == 0))
00367           {
00368             os << "NULL";
00369           }
00370           else if (obj.table->fields[i]->type.compare("BOOLEAN") == 0)
00371           {
00372             if (strncmp(row[i], "1", 1) == 0)
00373               os << "TRUE";
00374             else
00375               os << "FALSE";
00376           }
00377           else
00378             os << "'" << DrizzleDumpData::escape(row[i], row_sizes[i]) << "'";
00379           byte_counter+= 3;
00380         }
00381         else
00382           os << row[i];
00383       }
00384       if (i != obj.table->fields.size() - 1)
00385         os << ",";
00386     }
00387     /* Break insert up if it is too long */
00388     if ((extended_insert and
00389       (byte_counter >= DRIZZLE_MAX_LINE_LENGTH)) or (not extended_insert))
00390     {
00391       os << ");" << std::endl;
00392       new_insert= true;
00393       byte_counter= 0;
00394     }
00395   }
00396   if (not new_insert)
00397     os << ");" << std::endl;
00398 
00399   if (opt_autocommit)
00400     os << "COMMIT;" << std::endl;
00401 
00402   if (opt_disable_keys)
00403     os << "ALTER TABLE `" << obj.table->tableName << "` ENABLE KEYS;" << std::endl;
00404 
00405   os << std::endl;
00406 
00407   return os;
00408 }
00409 
00410 std::string DrizzleDumpData::convertHex(const unsigned char* from, size_t from_size) const
00411 {
00412   std::ostringstream output;
00413   if (from_size > 0)
00414     output << "0x";
00415   else
00416     output << "''";
00417 
00418   while (from_size > 0)
00419   {
00420     /* Would be nice if std::hex liked uint8_t, ah well */
00421     output << std::uppercase << std::hex << std::setw(2) << std::setfill('0') << (unsigned short)(*from);
00422     (void) *from++;
00423     from_size--;
00424   }
00425 
00426   return output.str();
00427 }
00428 
00429 /* Ripped out of libdrizzle, hopefully a little safer */
00430 std::string DrizzleDumpData::escape(const char* from, size_t from_size)
00431 {
00432   std::string output;
00433 
00434   while (from_size > 0)
00435   {
00436     if (!(*from & 0x80))
00437     {
00438       switch (*from)
00439       {
00440          case 0:
00441            output.append("\\0");
00442            break;
00443          case '\n':
00444            output.append("\\n");
00445            break;
00446          case '\r':
00447            output.append("\\r");
00448            break;
00449          case '\\':
00450            output.append("\\\\");
00451            break;
00452          case '\'':
00453            output.append("\\'");
00454            break;
00455          case '"':
00456            output.append("\\\"");
00457            break;
00458          case '\032':
00459            output.append("\\Z");
00460            break;
00461          default:
00462            output.push_back(*from);
00463            break;
00464        }
00465     }
00466     else
00467       output.push_back(*from);
00468     (void) *from++;
00469     from_size--;
00470   }
00471 
00472   return output;
00473 }
00474 
00475 std::ostream& operator <<(std::ostream &os, const DrizzleDumpTable &obj)
00476 {
00477   if (verbose)
00478   {
00479     std::cerr << "--" << std::endl
00480       << "-- Table structure for table `" << obj.displayName << "`" << std::endl
00481       << "--" << std::endl << std::endl;
00482   }
00483 
00484   if (opt_drop)
00485     os << "DROP TABLE IF EXISTS `" << obj.displayName <<  "`;" << std::endl;
00486 
00487   os << "CREATE TABLE `" << obj.displayName << "` (" << std::endl;
00488   std::vector<DrizzleDumpField*>::iterator i;
00489   std::vector<DrizzleDumpField*> output_fields = obj.fields;
00490   for (i= output_fields.begin(); i != output_fields.end(); ++i)
00491   {
00492     if (i != output_fields.begin())
00493       os << "," << std::endl;
00494     DrizzleDumpField *field= *i;
00495     os << *field;
00496   }
00497 
00498   std::vector<DrizzleDumpIndex*>::iterator j;
00499   std::vector<DrizzleDumpIndex*> output_indexes = obj.indexes;
00500   for (j= output_indexes.begin(); j != output_indexes.end(); ++j)
00501   {
00502     os << "," << std::endl;
00503     DrizzleDumpIndex *index= *j;
00504     os << *index;
00505   }
00506 
00507   std::vector<DrizzleDumpForeignKey*>::iterator k;
00508   std::vector<DrizzleDumpForeignKey*> output_fkeys = obj.fkeys;
00509   for (k= output_fkeys.begin(); k != output_fkeys.end(); ++k)
00510   {
00511     os << "," << std::endl;
00512     DrizzleDumpForeignKey *fkey= *k;
00513     os << *fkey;
00514   }
00515 
00516   os << std::endl;
00517   os << ") ENGINE='" << obj.engineName << "' ";
00518   if (obj.autoIncrement > 0)
00519   {
00520     os << "AUTO_INCREMENT=" << obj.autoIncrement << " ";
00521   }
00522 
00523   os << "COLLATE='" << obj.collate << "'";
00524 
00525   if (not obj.comment.empty())
00526   {
00527     os << " COMMENT='" << obj.comment << "'";
00528   }
00529 
00530   if (not obj.replicate)
00531   {
00532     os << " REPLICATE=FALSE";
00533   }
00534 
00535   os << ";" << std::endl << std::endl;
00536 
00537   return os;
00538 }
00539 
00540 DrizzleDumpConnection::DrizzleDumpConnection(std::string &host, uint16_t port, 
00541   std::string &username, std::string &password, bool drizzle_protocol) :
00542   hostName(host),
00543   drizzleProtocol(drizzle_protocol)
00544 {
00545   drizzle_return_t ret;
00546 
00547   if (host.empty())
00548     host= "localhost";
00549 
00550   std::string protocol= (drizzle_protocol) ? "Drizzle" : "MySQL";
00551   if (verbose)
00552   {
00553     std::cerr << _("-- Connecting to ") << host  << _(" using protocol ")
00554       << protocol << "..." << std::endl;
00555   }
00556   drizzle_create(&drizzle);
00557   drizzle_con_create(&drizzle, &connection);
00558   drizzle_con_set_tcp(&connection, (char *)host.c_str(), port);
00559   drizzle_con_set_auth(&connection, (char *)username.c_str(),
00560     (char *)password.c_str());
00561   drizzle_con_add_options(&connection, 
00562     drizzle_protocol ? DRIZZLE_CON_EXPERIMENTAL : DRIZZLE_CON_MYSQL);
00563   ret= drizzle_con_connect(&connection);
00564   if (ret != DRIZZLE_RETURN_OK)
00565   {
00566     errorHandler(NULL, ret, "when trying to connect");
00567     throw std::exception();
00568   }
00569 
00570   ServerDetect server_detect= ServerDetect(&connection);
00571 
00572   serverType= server_detect.getServerType();
00573   serverVersion= server_detect.getServerVersion();
00574 }
00575 
00576 drizzle_result_st* DrizzleDumpConnection::query(std::string &str_query)
00577 {
00578   drizzle_return_t ret;
00579   drizzle_result_st* result= new drizzle_result_st;
00580   if (drizzle_query_str(&connection, result, str_query.c_str(), &ret) == NULL ||
00581       ret != DRIZZLE_RETURN_OK)
00582   {
00583     if (ret == DRIZZLE_RETURN_ERROR_CODE)
00584     {
00585       std::cerr << _("Error executing query: ") <<
00586         drizzle_result_error(result) << std::endl;
00587       drizzle_result_free(result);
00588     }
00589     else
00590     {
00591       std::cerr << _("Error executing query: ") <<
00592         drizzle_con_error(&connection) << std::endl;
00593     }
00594     return NULL;
00595   }
00596 
00597   if (drizzle_result_buffer(result) != DRIZZLE_RETURN_OK)
00598   {
00599     std::cerr << _("Could not buffer result: ") <<
00600         drizzle_con_error(&connection) << std::endl;
00601     return NULL;
00602   }
00603   return result;
00604 }
00605 
00606 void DrizzleDumpConnection::freeResult(drizzle_result_st* result)
00607 {
00608   drizzle_result_free(result);
00609   delete result;
00610 }
00611 
00612 bool DrizzleDumpConnection::queryNoResult(std::string &str_query)
00613 {
00614   drizzle_return_t ret;
00615   drizzle_result_st result;
00616 
00617   if (drizzle_query_str(&connection, &result, str_query.c_str(), &ret) == NULL ||
00618       ret != DRIZZLE_RETURN_OK)
00619   {
00620     if (ret == DRIZZLE_RETURN_ERROR_CODE)
00621     {
00622       std::cerr << _("Error executing query: ") <<
00623         drizzle_result_error(&result) << std::endl;
00624       drizzle_result_free(&result);
00625     }
00626     else
00627     {
00628       std::cerr << _("Error executing query: ") <<
00629         drizzle_con_error(&connection) << std::endl;
00630     }
00631     return false;
00632   }
00633 
00634   drizzle_result_free(&result);
00635   return true;
00636 }
00637 
00638 bool DrizzleDumpConnection::setDB(std::string databaseName)
00639 {
00640   drizzle_return_t ret;
00641   drizzle_result_st result;
00642   if (drizzle_select_db(&connection, &result, databaseName.c_str(), &ret) == 
00643     NULL || ret != DRIZZLE_RETURN_OK)
00644   {
00645     std::cerr << _("Error: Could not set db '") << databaseName << "'" << std::endl;
00646     if (ret == DRIZZLE_RETURN_ERROR_CODE)
00647       drizzle_result_free(&result);
00648     return false;
00649   }
00650   drizzle_result_free(&result);
00651   return true;
00652 }
00653 
00654 void DrizzleDumpConnection::errorHandler(drizzle_result_st *res,
00655   drizzle_return_t ret, const char *when)
00656 {
00657   if (res == NULL)
00658   {
00659     std::cerr << _("Got error: ") << drizzle_con_error(&connection) << " "
00660       << when << std::endl;
00661   }
00662   else if (ret == DRIZZLE_RETURN_ERROR_CODE)
00663   {
00664     std::cerr << _("Got error: ") << drizzle_result_error(res)
00665       << " (" << drizzle_result_error_code(res) << ") " << when << std::endl;
00666     drizzle_result_free(res);
00667   }
00668   else
00669   {
00670     std::cerr << _("Got error: ") << ret << " " << when << std::endl;
00671   }
00672 
00673   return;
00674 }
00675 
00676 DrizzleDumpConnection::~DrizzleDumpConnection()
00677 {
00678   if (verbose)
00679     std::cerr << _("-- Disconnecting from ") << hostName << "..." << std::endl;
00680   drizzle_con_free(&connection);
00681   drizzle_free(&drizzle);
00682 }