Drizzled Public API Documentation

cloud_ms.cc
00001 /* Copyright (C) 2009 PrimeBase Technologies GmbH, Germany
00002  *
00003  * PrimeBase Media Stream for MySQL
00004  *
00005  * This program is free software; you can redistribute it and/or modify
00006  * it under the terms of the GNU General Public License as published by
00007  * the Free Software Foundation; either version 2 of the License, or
00008  * (at your option) any later version.
00009  *
00010  * This program is distributed in the hope that it will be useful,
00011  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00012  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00013  * GNU General Public License for more details.
00014  *
00015  * You should have received a copy of the GNU General Public License
00016  * along with this program; if not, write to the Free Software
00017  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
00018  *
00019  *  Created by Barry Leslie on 3/20/09.
00020  *
00021  */
00022  
00023 #ifdef DRIZZLED
00024 #include <config.h>
00025 #include <drizzled/common.h>
00026 #include <drizzled/session.h>
00027 #include <drizzled/table.h>
00028 #include <drizzled/message/table.pb.h>
00029 #include <drizzled/charset_info.h>
00030 #include <drizzled/table_proto.h>
00031 #include <drizzled/session.h>
00032 #include <drizzled/field.h>
00033 #endif
00034 
00035 #include "cslib/CSConfig.h"
00036 
00037 #include <stdlib.h>
00038 #include <ctype.h>
00039 #include <string.h>
00040 
00041 #include "cslib/CSGlobal.h"
00042 #include "cslib/CSThread.h"
00043 #include "cslib/CSLog.h"
00044 #include "cslib/CSPath.h"
00045 #include "cslib/CSFile.h"
00046 #include "cslib/CSString.h"
00047 #include "cslib/CSStrUtil.h"
00048 #include "cslib/CSStorage.h"
00049 #include "cslib/CSEncode.h"
00050 #include "cslib/CSS3Protocol.h"
00051 
00052 #include "backup_ms.h"
00053 #include "cloud_ms.h"
00054 
00055 CSSyncSparseArray *MSCloudInfo::gCloudInfo;
00056 uint32_t      MSCloudInfo::gMaxInfoRef;
00057 
00058 uint32_t      CloudDB::gKeyIndex;
00059 CSMutex     CloudDB::gCloudKeyLock;
00060 
00061 //==============================
00062 MSCloudInfo::MSCloudInfo(uint32_t id,
00063             const char *server,
00064             const char *bucket_arg,
00065             const char *publicKey,
00066             const char *privateKey
00067             ):
00068   cloudRefId(id),
00069   bucket(NULL),
00070   s3Prot(NULL)
00071 {   
00072   new_(s3Prot, CSS3Protocol());
00073   s3Prot->s3_setServer(server);
00074   s3Prot->s3_setPublicKey(publicKey);
00075   s3Prot->s3_setPrivateKey(privateKey);
00076   
00077   bucket = CSString::newString(bucket_arg);   
00078 }
00079     
00080 //-------------------------------
00081 MSCloudInfo::~MSCloudInfo()
00082 {
00083   if (bucket)
00084     bucket->release();
00085     
00086   if (s3Prot)
00087     s3Prot->release();
00088 }
00089   
00090 //-------------------------------
00091 const char *MSCloudInfo::getServer() 
00092 { 
00093   return s3Prot->s3_getServer();
00094 }
00095 
00096 //-------------------------------
00097 const char *MSCloudInfo::getBucket() 
00098 { 
00099   return bucket->getCString();
00100 }
00101 
00102 //-------------------------------
00103 const char *MSCloudInfo::getPublicKey() 
00104 { 
00105   return s3Prot->s3_getPublicKey();
00106 }
00107 
00108 //-------------------------------
00109 const char *MSCloudInfo::getPrivateKey() 
00110 { 
00111   return s3Prot->s3_getPrivateKey();
00112 }
00113 
00114 //-------------------------------
00115 CSString *MSCloudInfo::getSignature(const char *key, const char *content_type, uint32_t *s3AuthorizationTime)
00116 {
00117   return s3Prot->s3_getAuthorization(bucket->getCString(), key, content_type, s3AuthorizationTime);
00118 }
00119 
00120 //-------------------------------
00121 CSString *MSCloudInfo::getDataURL(const char *key, int keep_alive)
00122 {
00123   return s3Prot->s3_getDataURL(bucket->getCString(), key, keep_alive);
00124 }
00125 
00126 //-------------------------------
00127 void MSCloudInfo::send(CSInputStream *input, const char *key, off64_t size)
00128 {
00129   CSVector *headers;
00130   headers = s3Prot->s3_send(input, bucket->getCString(), key, size);
00131   headers->release();
00132 }
00133 
00134 //-------------------------------
00135 CSVector *MSCloudInfo::list(const char *key_prefix, uint32_t max)
00136 {
00137   return s3Prot->s3_list(bucket->getCString(), key_prefix, max);
00138 }
00139 
00140 //-------------------------------
00141 void MSCloudInfo::receive(CSOutputStream *output, const char *key)
00142 {
00143   bool found;
00144   CSVector *headers;
00145   
00146   headers = s3Prot->s3_receive(output, bucket->getCString(), key, &found);
00147   headers->release();
00148   if (!found) {
00149     CSStringBuffer *err;
00150     enter_();
00151     
00152     new_(err, CSStringBuffer());
00153     push_(err);
00154     err->append("S3 object not found: ");
00155     err->append(getServer());
00156     err->append("/");
00157     err->append(bucket->getCString());
00158     err->append("/");
00159     err->append(key);
00160 
00161     CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, err->getCString());
00162     release_(err);
00163     outer_();
00164   }
00165 }
00166 
00167 //-------------------------------
00168 void MSCloudInfo::cDelete(const char *key)
00169 {
00170   s3Prot->s3_delete(bucket->getCString(), key);
00171 }
00172 
00173 //-------------------------------
00174 void MSCloudInfo::copy(MSCloudInfo *dst_cloud, const char *dst_key, const char *src_key)
00175 {
00176   enter_();
00177   push_(dst_cloud);
00178   
00179   s3Prot->s3_copy(dst_cloud->getServer() ,dst_cloud->bucket->getCString(), dst_key, bucket->getCString(), src_key);
00180 
00181   release_(dst_cloud);
00182   exit_();
00183 }
00184 
00185 //==============================
00186 CloudDB::CloudDB(uint32_t db_id):
00187   dfltCloudRefId(0),
00188   keep_alive(5 * 60),// default URL keep alive in seconds.
00189   blob_recovery_no(0),
00190   blob_db_id(db_id),
00191   isBackup(false),
00192   backupInfo(NULL),
00193   backupCloud(NULL),
00194   clObjectKey(NULL)
00195 {
00196   enter_();
00197 
00198   new_(clObjectKey, CSStringBuffer());
00199   clObjectKey->setLength(base_key_size);
00200     
00201   exit_();
00202 }
00203 
00204 //-------------------------------
00205 CloudDB::~CloudDB()
00206 {
00207   
00208   if (backupInfo)
00209     backupInfo->release();
00210     
00211   if (backupCloud)
00212     backupCloud->release();
00213     
00214   if (clObjectKey)
00215     clObjectKey->release();
00216     
00217 }
00218 //-------------------------------
00219 MSBackupInfo *CloudDB::cl_getBackupInfo()
00220 { 
00221   if (backupInfo)
00222     backupInfo->retain();
00223     
00224   return backupInfo;
00225 }
00226 
00227 //-------------------------------
00228 void CloudDB::cl_clearBackupInfo(){ backupInfo->release(); backupInfo = NULL;}
00229 
00230 //-------------------------------
00231 void CloudDB::cl_createDB()
00232 {
00233 // This is a no-op. 
00234 }
00235 
00236 //-------------------------------
00237 // Restore all the 
00238 void CloudDB::cl_restoreDB()
00239 {
00240   CSVector *list = NULL;
00241   CSString *key = NULL;
00242   CloudObjectKey *src_objectKey = NULL, *dst_objectKey = NULL;
00243   CloudKeyRec   cloudKey;
00244   uint32_t    src_cloudRef, dst_cloudRef = 0;
00245   MSBackupInfo  *backup_info = NULL;
00246   MSCloudInfo   *src_cloud = NULL, *dst_cloud = NULL;
00247   enter_();
00248 
00249   if (!blob_recovery_no)
00250     exit_(); // nothing to do.
00251  
00252   backup_info = MSBackupInfo::getBackupInfo(blob_recovery_no);
00253   push_(backup_info);
00254   
00255   src_cloudRef = backup_info->getcloudRef();
00256   src_cloud = MSCloudInfo::getCloudInfo(src_cloudRef);
00257   push_(src_cloud);
00258   
00259   new_(dst_objectKey, CloudObjectKey(blob_db_id));
00260   push_(dst_objectKey);
00261   
00262   // Get the key for the backup BLOB
00263   new_(src_objectKey, CloudObjectKey(blob_db_id));
00264   push_(src_objectKey);
00265   src_objectKey->setObjectKey(NULL, backup_info->getcloudBackupNo(), backup_info->getDatabaseId()); 
00266 
00267   // Get a list of all the BLOBs that were backed up.
00268   list = src_cloud->list(src_objectKey->getCString());
00269   release_(src_objectKey);
00270   push_(list);
00271   
00272   
00273   // Go through the list copying the keys.
00274   dst_cloudRef = src_cloudRef;
00275   dst_cloud = src_cloud;
00276   dst_cloud->retain();
00277   
00278   push_ref_(dst_cloud); // Push a reference to dst_cloud so that what ever it references will be released.
00279 
00280   while ((key = (CSString*)(list->take(0))) ) {
00281     push_(key);
00282 
00283     // The source key name must be parsed to get its
00284     // destination cloud reference. The destination for
00285     // the BLOBs may not all be in the same cloud. 
00286     CloudObjectKey::parseObjectKey(key->getCString(), &cloudKey);
00287     
00288     // Reset the destination cloud if required.
00289     if (cloudKey.cloud_ref != dst_cloudRef) {
00290       if (dst_cloud) {
00291         dst_cloud->release();
00292         dst_cloud = NULL;
00293       }
00294       dst_cloudRef =  cloudKey.cloud_ref;
00295       dst_cloud = MSCloudInfo::getCloudInfo(dst_cloudRef);
00296     }
00297 
00298     // Copy the BLOB to the recovered database.
00299     dst_objectKey->setObjectKey(&cloudKey);
00300     src_cloud->copy(RETAIN(dst_cloud), dst_objectKey->getCString(), key->getCString());
00301     release_(key);
00302     
00303   }
00304   
00305   release_(dst_cloud);
00306   
00307   blob_recovery_no = 0;
00308   release_(list);
00309   release_(dst_objectKey);  
00310   release_(src_cloud);
00311   release_(backup_info);
00312   exit_();  
00313 }
00314 
00315 //-------------------------------
00316 uint32_t CloudDB::cl_getNextBackupNumber(uint32_t cloud_ref)
00317 {
00318   CloudObjectKey *objectKey;
00319   CSVector *list;
00320   uint32_t backup_no = 0, size = 1;
00321   MSCloudInfo *s3Cloud;
00322   enter_();
00323 
00324   s3Cloud = MSCloudInfo::getCloudInfo((cloud_ref)?cloud_ref:dfltCloudRefId);
00325   push_(s3Cloud);
00326   
00327   new_(objectKey, CloudObjectKey(blob_db_id));
00328   push_(objectKey);
00329 
00330   // Find the next available backup number
00331   while (size) {
00332     backup_no++;
00333     objectKey->setObjectKey(NULL, backup_no); // use the key prefix with the backup number for listing.
00334     list = s3Cloud->list(objectKey->getCString(), 1);
00335     size = list->size();
00336     list->release();
00337   }
00338   
00339   release_(objectKey);
00340   release_(s3Cloud);
00341 
00342   return_(backup_no);
00343 }
00344 
00345 //-------------------------------
00346 void CloudDB::cl_backupBLOB(CloudKeyPtr key)
00347 {
00348   CloudObjectKey *src_objectKey, *dst_objectKey;
00349   uint32_t cloudRef, backupNo;
00350   MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
00351   enter_();
00352 
00353   ASSERT(backupInfo);
00354   
00355   if ((cloudRef = backupInfo->getcloudRef()) == 0) {
00356     backupInfo->setcloudRef(dfltCloudRefId);
00357     cloudRef = dfltCloudRefId;
00358   }
00359     
00360   if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {   
00361     backupNo = cl_getNextBackupNumber(cloudRef);
00362     backupInfo->setcloudBackupNo(backupNo);
00363   }
00364   
00365   // Set the source object's key
00366   new_(src_objectKey, CloudObjectKey(blob_db_id));
00367   push_(src_objectKey);
00368   src_objectKey->setObjectKey(key);
00369 
00370   // Set the destination object's key
00371   new_(dst_objectKey, CloudObjectKey(blob_db_id));
00372   push_(dst_objectKey);
00373   dst_objectKey->setObjectKey(key, backupNo);
00374 
00375   // Get the source cloud
00376   src_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
00377   push_(src_cloud);
00378   
00379   // Copy the object to the destination cloud
00380   dst_cloud = MSCloudInfo::getCloudInfo(cloudRef);
00381   src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
00382 
00383   release_(src_cloud);
00384   release_(dst_objectKey);
00385   release_(src_objectKey);
00386   exit_();
00387 }
00388 
00389 //-------------------------------
00390 void CloudDB::cl_restoreBLOB(CloudKeyPtr key, uint32_t backup_db_id)
00391 {
00392   CloudObjectKey *src_objectKey, *dst_objectKey;
00393   uint32_t cloudRef, backupNo;
00394   MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
00395   enter_();
00396 
00397   ASSERT(backupInfo);
00398   
00399   if ((cloudRef = backupInfo->getcloudRef()) == 0) {
00400     backupInfo->setcloudRef(dfltCloudRefId);
00401     cloudRef = dfltCloudRefId;
00402   }
00403     
00404   if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {   
00405     backupNo = cl_getNextBackupNumber(cloudRef);
00406     backupInfo->setcloudBackupNo(backupNo);
00407   }
00408   
00409   // Set the source object's key
00410   new_(src_objectKey, CloudObjectKey(backup_db_id));
00411   push_(src_objectKey);
00412   src_objectKey->setObjectKey(key, backupNo);
00413 
00414   // Set the destination object's key
00415   new_(dst_objectKey, CloudObjectKey(blob_db_id));
00416   push_(dst_objectKey);
00417   dst_objectKey->setObjectKey(key);
00418 
00419   // Get the source cloud
00420   src_cloud = MSCloudInfo::getCloudInfo(cloudRef);
00421   push_(src_cloud);
00422   
00423   // Copy the object to the destination cloud
00424   dst_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
00425   src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
00426 
00427   release_(src_cloud);
00428   release_(dst_objectKey);
00429   release_(src_objectKey);
00430   exit_();
00431 }
00432 
00433 //-------------------------------
00434 // Drop database deletes all objects with the database key prefix
00435 void CloudDB::cl_dropDB()
00436 {
00437   CSVector *list;
00438   CSString *key;
00439   CloudObjectKey *objectKey;  
00440   MSCloudInfo *s3Cloud = NULL;
00441   int i;
00442   const char *key_str;
00443   
00444   enter_();
00445   new_(objectKey, CloudObjectKey(blob_db_id));
00446   push_(objectKey);
00447   
00448   lock_(MSCloudInfo::gCloudInfo);
00449 
00450   if (isBackup) {
00451     uint32_t backup_no;
00452     if (backupInfo && (backup_no = backupInfo->getcloudBackupNo())) {
00453       objectKey->setObjectKey(NULL, backup_no); // use the key prefix for the backup for listing.
00454       if ((s3Cloud = MSCloudInfo::getCloudInfo(backupInfo->getcloudRef())))
00455         push_(s3Cloud);
00456     }
00457   } else {
00458     objectKey->setObjectKey(); // use the key prefix for listing.
00459     i = 0;
00460     s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++); // <-- unreferenced object 
00461   }
00462     
00463   key_str = objectKey->getCString();
00464 
00465   // For non backup BLOBs all known clouds must be searched 
00466   // for possible BLOBs and deleted. The BLOBs belonging to a backup
00467   // will ever only be in one cloud storage location.
00468   while (s3Cloud) {
00469     list = s3Cloud->list(key_str);
00470     push_(list);
00471     
00472     // Go through the list deleting the keys.
00473     while ((key = (CSString*)(list->take(0))) ) {
00474       push_(key);
00475       s3Cloud->cDelete(key->getCString());
00476       release_(key);
00477     }
00478     
00479     release_(list);
00480     if (isBackup) {
00481       release_(s3Cloud); // Only the backup s3Cloud needs to be released.
00482       s3Cloud = NULL;
00483     } else
00484       s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++);// <-- unreferenced object
00485   }
00486   
00487   unlock_(MSCloudInfo::gCloudInfo);
00488   release_(objectKey);
00489   exit_();
00490 }
00491 
00492 //-------------------------------
00493 void CloudDB::cl_putData(CloudKeyPtr key, CSInputStream *stream, off64_t size)
00494 {
00495   CloudObjectKey *objectKey;
00496   MSCloudInfo *s3Cloud;
00497   
00498   enter_();
00499   
00500   push_(stream);
00501   
00502   new_(objectKey, CloudObjectKey(blob_db_id));
00503   push_(objectKey);
00504   
00505   objectKey->setObjectKey(key);
00506   
00507   s3Cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
00508   push_(s3Cloud);
00509   s3Cloud->send(RETAIN(stream), objectKey->getCString(), size);
00510   release_(s3Cloud);
00511   
00512   release_(objectKey);
00513   release_(stream);
00514   
00515   exit_();
00516 }
00517 
00518 //-------------------------------
00519 off64_t CloudDB::cl_getData(CloudKeyPtr key,  char *buffer, off64_t size)
00520 { 
00521   CloudObjectKey *objectKey;
00522   CSStaticMemoryOutputStream *output;
00523   MSCloudInfo *s3Cloud;
00524   enter_();
00525   
00526   new_(objectKey, CloudObjectKey(blob_db_id));
00527   push_(objectKey);
00528   
00529   s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
00530   push_(s3Cloud);
00531 
00532   new_(output, CSStaticMemoryOutputStream((u_char *)buffer, size));
00533   push_(output);
00534   
00535   objectKey->setObjectKey(key);
00536   
00537   s3Cloud->receive(RETAIN(output), objectKey->getCString());  
00538   size = output->getSize();
00539   release_(output);
00540   
00541   release_(s3Cloud);
00542   release_(objectKey);
00543   return_(size);
00544 }
00545 
00546 //-------------------------------
00547 void CloudDB::cl_deleteData(CloudKeyPtr key)
00548 {
00549   MSCloudInfo *s3Cloud;
00550   CloudObjectKey *objectKey;
00551   enter_();
00552   
00553   new_(objectKey, CloudObjectKey(blob_db_id));
00554   push_(objectKey);
00555   
00556   s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
00557   push_(s3Cloud);
00558 
00559   objectKey->setObjectKey(key);
00560 
00561   s3Cloud->cDelete(objectKey->getCString());  
00562   
00563   release_(s3Cloud);
00564   release_(objectKey);
00565 
00566   exit_();
00567 }
00568 
00569 //-------------------------------
00570 CSString *CloudDB::cl_getDataURL(CloudKeyPtr key)
00571 {
00572   CloudObjectKey *objectKey;
00573   CSString *url;
00574   MSCloudInfo *s3Cloud;
00575   enter_();
00576   
00577   new_(objectKey, CloudObjectKey(blob_db_id));
00578   push_(objectKey);
00579   
00580   objectKey->setObjectKey(key);
00581   
00582   s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);  
00583   push_(s3Cloud);
00584     
00585   url = s3Cloud->getDataURL(objectKey->getCString(), keep_alive);
00586   
00587   release_(s3Cloud);
00588   release_(objectKey);
00589 
00590   return_(url);
00591 }
00592 
00593 //-------------------------------
00594 CSString *CloudDB::cl_getSignature(CloudKeyPtr key, CSString *content_type_arg, uint32_t *s3AuthorizationTime)
00595 {
00596   CSString *signature;
00597   CloudObjectKey *objectKey;
00598   const char *content_type = NULL;
00599   MSCloudInfo *s3Cloud;
00600   enter_();
00601   
00602   new_(objectKey, CloudObjectKey(blob_db_id));
00603   push_(objectKey);
00604   
00605   if (content_type_arg) {
00606     push_(content_type_arg);
00607     content_type = content_type_arg->getCString();
00608   }
00609   
00610   objectKey->setObjectKey(key);
00611   s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);  
00612   push_(s3Cloud);
00613   
00614   signature = s3Cloud->getSignature(objectKey->getCString(), content_type, s3AuthorizationTime);
00615   
00616   if (content_type_arg) 
00617     release_(content_type_arg);
00618 
00619   release_(s3Cloud);
00620   release_(objectKey);
00621   
00622   return_(signature);
00623 }
00624 
00625 //==============================
00626