Drizzled Public API Documentation

xtrabackup.cc
1 /******************************************************
2 XtraBackup: The another hot backup tool for InnoDB
3 (c) 2009 Percona Inc.
4 (C) 2011 Stewart Smith
5 Created 3/3/2009 Yasufumi Kinoshita
6 
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; version 2 of the License.
10 
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15 
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 
20 *******************************************************/
21 
22 #ifndef XTRABACKUP_VERSION
23 #define XTRABACKUP_VERSION "undefined"
24 #endif
25 #ifndef XTRABACKUP_REVISION
26 #define XTRABACKUP_REVISION "undefined"
27 #endif
28 
29 #include <config.h>
30 #include <string>
31 #include <drizzled/internal/my_sys.h>
32 #include <drizzled/charset.h>
33 #include <drizzled/gettext.h>
34 #include <drizzled/constrained_value.h>
35 #include <drizzled/configmake.h>
36 #include <drizzled/error/priority_t.h>
37 
38 #include "ha_prototypes.h"
39  //#define XTRABACKUP_TARGET_IS_PLUGIN
40 #include <boost/program_options.hpp>
41 #include <boost/scoped_array.hpp>
42 #include <boost/scoped_ptr.hpp>
43 
45 
46 #define my_progname "xtrabackup"
47 
48 #define MYSQL_VERSION_ID 50507 /* Drizzle is much greater */
49 
50 #include <univ.i>
51 #include <os0file.h>
52 #include <os0thread.h>
53 #include <srv0start.h>
54 #include <srv0srv.h>
55 #include <trx0roll.h>
56 #include <trx0trx.h>
57 #include <trx0sys.h>
58 #include <mtr0mtr.h>
59 #include <row0ins.h>
60 #include <row0mysql.h>
61 #include <row0sel.h>
62 #include <row0upd.h>
63 #include <log0log.h>
64 #include <log0recv.h>
65 #include <lock0lock.h>
66 #include <dict0crea.h>
67 #include <btr0cur.h>
68 #include <btr0btr.h>
69 #include <btr0sea.h>
70 #include <fsp0fsp.h>
71 #include <sync0sync.h>
72 #include <fil0fil.h>
73 #include <trx0xa.h>
74 
75 #ifdef INNODB_VERSION_SHORT
76 #include <ibuf0ibuf.h>
77 #else
78 #error ENOCOOL
79 #endif
80 
81 #define IB_INT64 ib_int64_t
82 #define LSN64 ib_uint64_t
83 #define MACH_READ_64 mach_read_from_8
84 #define MACH_WRITE_64 mach_write_to_8
85 #define OS_MUTEX_CREATE() os_mutex_create()
86 #define ut_dulint_zero 0
87 #define ut_dulint_cmp(A, B) (A > B ? 1 : (A == B ? 0 : -1))
88 #define ut_dulint_add(A, B) (A + B)
89 #define ut_dulint_minus(A, B) (A - B)
90 #define ut_dulint_align_down(A, B) (A & ~((ib_int64_t)B - 1))
91 #define ut_dulint_align_up(A, B) ((A + B - 1) & ~((ib_int64_t)B - 1))
92 
93 #ifdef __WIN__
94 #define SRV_PATH_SEPARATOR '\\'
95 #define SRV_PATH_SEPARATOR_STR "\\"
96 #else
97 #define SRV_PATH_SEPARATOR '/'
98 #define SRV_PATH_SEPARATOR_STR "/"
99 #endif
100 
101 #ifndef UNIV_PAGE_SIZE_MAX
102 #define UNIV_PAGE_SIZE_MAX UNIV_PAGE_SIZE
103 #endif
104 #ifndef UNIV_PAGE_SIZE_SHIFT_MAX
105 #define UNIV_PAGE_SIZE_SHIFT_MAX UNIV_PAGE_SIZE_SHIFT
106 #endif
107 
108 using namespace drizzled;
109 namespace po=boost::program_options;
110 
111 namespace drizzled {
112  bool errmsg_printf (error::priority_t, char const *format, ...);
113 
114  bool errmsg_printf (error::priority_t, char const *format, ...)
115  {
116  bool rv;
117  va_list args;
118  va_start(args, format);
119  rv= vfprintf(stderr, format, args);
120  va_end(args);
121  fprintf(stderr, "\n");
122  return rv;
123  }
124 
125 }
126 
127 #include "xtrabackup_api.h"
128 
129  #include <fcntl.h>
130  #include <regex.h>
131 
132  #ifdef POSIX_FADV_NORMAL
133  #define USE_POSIX_FADVISE
134  #endif
135 
136  /* ==start === definition at fil0fil.c === */
137  // ##################################################################
138  // NOTE: We should check the following definitions fit to the source.
139  // ##################################################################
140 
141  //Plugin ?
143  struct fil_node_struct {
144  fil_space_t* space;
146  char* name;
147  ibool open;
148  os_file_t handle;
149  ibool is_raw_disk;
151  ulint size;
154  ulint n_pending;
158  ulint n_pending_flushes;
162  ib_int64_t modification_counter;
164  ib_int64_t flush_counter;
167  UT_LIST_NODE_T(fil_node_t) chain;
171  ulint magic_n;
172  };
173 
174  struct fil_space_struct {
175  char* name;
177  ulint id;
178  ib_int64_t tablespace_version;
184  ibool mark;
188  ibool stop_ios;
192  ibool stop_ibuf_merges;
195  ibool is_being_deleted;
202  ulint purpose;
206  ulint size;
210  ulint flags;
211  ulint n_reserved_extents;
214  ulint n_pending_flushes;
217  ulint n_pending_ibuf_merges;
223  hash_node_t hash;
224  hash_node_t name_hash;
225  #ifndef UNIV_HOTBACKUP
226  rw_lock_t latch;
228  #endif /* !UNIV_HOTBACKUP */
229  UT_LIST_NODE_T(fil_space_t) unflushed_spaces;
232  ibool is_in_unflushed_spaces;
234  #ifdef XTRADB_BASED
235  ibool is_corrupt;
236  #endif
237  UT_LIST_NODE_T(fil_space_t) space_list;
239  ulint magic_n;
240  };
241 
242  typedef struct fil_system_struct fil_system_t;
243 
244  struct fil_system_struct {
245  #ifndef UNIV_HOTBACKUP
246  mutex_t mutex;
247  #ifdef XTRADB55
248  mutex_t file_extend_mutex;
249  #endif
250  #endif /* !UNIV_HOTBACKUP */
251  hash_table_t* spaces;
254  hash_table_t* name_hash;
267  UT_LIST_BASE_NODE_T(fil_space_t) unflushed_spaces;
273  ulint n_open;
274  ulint max_n_open;
276  ib_int64_t modification_counter;
278  ulint max_assigned_id;
284  ib_int64_t tablespace_version;
292  UT_LIST_BASE_NODE_T(fil_space_t) space_list;
294  };
295 
296  typedef struct {
297  ulint page_size;
298  } xb_delta_info_t;
299 
300  extern fil_system_t* fil_system;
301 
302  /* ==end=== definition at fil0fil.c === */
303 
304 
305  bool innodb_inited= 0;
306 
307  /* === xtrabackup specific options === */
308  char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/";
309  const char *xtrabackup_target_dir= xtrabackup_real_target_dir;
310  bool xtrabackup_backup = false;
311  bool xtrabackup_stats = false;
312  bool xtrabackup_prepare = false;
313  bool xtrabackup_print_param = false;
314 
315  bool xtrabackup_export = false;
316  bool xtrabackup_apply_log_only = false;
317 
318  bool xtrabackup_suspend_at_end = false;
319  uint64_t xtrabackup_use_memory = 100*1024*1024L;
320  bool xtrabackup_create_ib_logfile = false;
321 
322  long xtrabackup_throttle = 0; /* 0:unlimited */
323  lint io_ticket;
324  os_event_t wait_throttle = NULL;
325 
326  bool xtrabackup_stream = false;
327 const char *xtrabackup_incremental = NULL;
328  LSN64 incremental_lsn;
329  LSN64 incremental_to_lsn;
330  LSN64 incremental_last_lsn;
331  byte* incremental_buffer = NULL;
332  byte* incremental_buffer_base = NULL;
333 
334 const char *xtrabackup_incremental_basedir = NULL; /* for --backup */
335 const char *xtrabackup_extra_lsndir = NULL; /* for --backup with --extra-lsndir */
336 char *xtrabackup_incremental_dir = NULL; /* for --prepare */
337 
338  char *xtrabackup_tables = NULL;
339  int tables_regex_num;
340  regex_t *tables_regex;
341  regmatch_t tables_regmatch[1];
342 
343 const char *xtrabackup_tables_file = NULL;
344  hash_table_t* tables_hash;
345 
347  char* name;
348  hash_node_t name_hash;
349  };
351 
352  #ifdef XTRADB_BASED
353  static ulint thread_nr[SRV_MAX_N_IO_THREADS + 6 + 64];
354  static os_thread_id_t thread_ids[SRV_MAX_N_IO_THREADS + 6 + 64];
355  #else
356  static ulint thread_nr[SRV_MAX_N_IO_THREADS + 6];
357  static os_thread_id_t thread_ids[SRV_MAX_N_IO_THREADS + 6];
358  #endif
359 
360  LSN64 checkpoint_lsn_start;
361  LSN64 checkpoint_no_start;
362  LSN64 log_copy_scanned_lsn;
363  IB_INT64 log_copy_offset = 0;
364  ibool log_copying = TRUE;
365  ibool log_copying_running = FALSE;
366  ibool log_copying_succeed = FALSE;
367 
368  ibool xtrabackup_logfile_is_renamed = FALSE;
369 
370  uint parallel;
371 
372  /* === metadata of backup === */
373  #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints"
374  char metadata_type[30] = ""; /*[full-backuped|full-prepared|incremental]*/
375 
376  ib_uint64_t metadata_from_lsn = 0;
377  ib_uint64_t metadata_to_lsn = 0;
378  ib_uint64_t metadata_last_lsn = 0;
379 
380  #define XB_DELTA_INFO_SUFFIX ".meta"
381 
382  /* === sharing with thread === */
383  os_file_t dst_log = -1;
384  char dst_log_path[FN_REFLEN];
385 
386  /* === some variables from mysqld === */
387  char mysql_real_data_home[FN_REFLEN] = "./";
388  char *mysql_data_home= mysql_real_data_home;
389 std::string mysql_data_home_arg;
390  static char mysql_data_home_buff[2];
391 
392 const char *opt_mysql_tmpdir = NULL;
393 
394 /* === static parameters in ha_innodb.cc */
395 
396 #define HA_INNOBASE_ROWS_IN_TABLE 10000 /* to get optimization right */
397 #define HA_INNOBASE_RANGE_COUNT 100
398 
399 ulong innobase_large_page_size = 0;
400 
401 /* The default values for the following, type long or longlong, start-up
402 parameters are declared in mysqld.cc: */
403 
404 long innobase_additional_mem_pool_size = 1*1024*1024L;
405 long innobase_buffer_pool_awe_mem_mb = 0;
406 long innobase_file_io_threads = 4;
407 long innobase_read_io_threads = 4;
408 long innobase_write_io_threads = 4;
409 long innobase_force_recovery = 0;
410 long innobase_lock_wait_timeout = 50;
411 long innobase_log_buffer_size = 1024*1024L;
412 long innobase_log_files_in_group = 2;
413 long innobase_log_files_in_group_backup;
414 long innobase_mirrored_log_groups = 1;
415 long innobase_open_files = 300L;
416 
417 long innobase_page_size = (1 << 14); /* 16KB */
418 static ulong innobase_log_block_size = 512;
419 bool innobase_fast_checksum = false;
420 bool innobase_extra_undoslots = false;
421 char* innobase_doublewrite_file = NULL;
422 
423 uint64_t innobase_buffer_pool_size = 8*1024*1024L;
424 
426 static log_file_constraint innobase_log_file_size;
427 
428 uint64_t innobase_log_file_size_backup;
429 
430 /* The default values for the following char* start-up parameters
431 are determined in innobase_init below: */
432 
433 char* innobase_data_home_dir = NULL;
434 const char* innobase_data_file_path = NULL;
435 char* innobase_log_group_home_dir = NULL;
436 char* innobase_log_group_home_dir_backup = NULL;
437 char* innobase_log_arch_dir = NULL;/* unused */
438 /* The following has a misleading name: starting from 4.0.5, this also
439 affects Windows: */
440 char* innobase_unix_file_flush_method = NULL;
441 
442 /* Below we have boolean-valued start-up parameters, and their default
443 values */
444 
445 ulong innobase_fast_shutdown = 1;
446 bool innobase_log_archive = FALSE;/* unused */
447 bool innobase_use_doublewrite = TRUE;
448 bool innobase_use_checksums = TRUE;
449 bool innobase_use_large_pages = FALSE;
450 bool innobase_file_per_table = FALSE;
451 bool innobase_locks_unsafe_for_binlog = FALSE;
452 bool innobase_rollback_on_timeout = FALSE;
453 bool innobase_create_status_file = FALSE;
454 bool innobase_adaptive_hash_index = TRUE;
455 
456 static char *internal_innobase_data_file_path = NULL;
457 
458 /* The following counter is used to convey information to InnoDB
459 about server activity: in selects it is not sensible to call
460 srv_active_wake_master_thread after each fetch or search, we only do
461 it every INNOBASE_WAKE_INTERVAL'th step. */
462 
463 #define INNOBASE_WAKE_INTERVAL 32
464 ulong innobase_active_counter = 0;
465 
466 UNIV_INTERN
467 bool
468 innobase_isspace(
469  const void *cs,
470  char char_to_test)
471 {
472  return static_cast<const CHARSET_INFO*>(cs)->isspace(char_to_test);
473 }
474 
475 UNIV_INTERN
476 void
478  Table* ,
479  const rec_t* ,
480  const dict_index_t* ,
481  const ulint* );
482 
483 UNIV_INTERN
484 void
486 /*==================*/
487  Table* ,
488  const rec_t* ,
489  const dict_index_t* ,
490  const ulint* )
492 {
493  fprintf(stderr, "ERROR: innobase_rec_to_mysql called\n");
494  return;
495 }
496 
497 UNIV_INTERN
498 void
500 
501 UNIV_INTERN
502 void
504 /*===============*/
505  Table* )
506 {
507  fprintf(stderr, "ERROR: innobase_rec_reset called\n");
508  return;
509 }
510 
511 UNIV_INTERN
512 void
514 /*===================*/
516  ulint )
517 {
518  return;
519 }
520 
521 
522 
523 /* ======== Datafiles iterator ======== */
524 typedef struct {
525  fil_system_t *system;
526  fil_space_t *space;
527  fil_node_t *node;
528  ibool started;
529  os_mutex_t mutex;
531 
532 static
534 datafiles_iter_new(fil_system_t *f_system)
535 {
536  datafiles_iter_t *it;
537 
539  it->mutex = OS_MUTEX_CREATE();
540 
541  it->system = f_system;
542  it->space = NULL;
543  it->node = NULL;
544  it->started = FALSE;
545 
546  return it;
547 }
548 
549 static
550 fil_node_t *
551 datafiles_iter_next(datafiles_iter_t *it, ibool *space_changed)
552 {
553  os_mutex_enter(it->mutex);
554 
555  *space_changed = FALSE;
556 
557  if (it->node == NULL) {
558  if (it->started)
559  goto end;
560  it->started = TRUE;
561  } else {
562  it->node = UT_LIST_GET_NEXT(chain, it->node);
563  if (it->node != NULL)
564  goto end;
565  }
566 
567  it->space = (it->space == NULL) ?
568  UT_LIST_GET_FIRST(it->system->space_list) :
569  UT_LIST_GET_NEXT(space_list, it->space);
570 
571  while (it->space != NULL &&
572  (it->space->purpose != FIL_TABLESPACE ||
573  UT_LIST_GET_LEN(it->space->chain) == 0))
574  it->space = UT_LIST_GET_NEXT(space_list, it->space);
575  if (it->space == NULL)
576  goto end;
577  *space_changed = TRUE;
578 
579  it->node = UT_LIST_GET_FIRST(it->space->chain);
580 
581 end:
582  os_mutex_exit(it->mutex);
583 
584  return it->node;
585 }
586 
587 static
588 void
589 datafiles_iter_free(datafiles_iter_t *it)
590 {
591  os_mutex_free(it->mutex);
592  ut_free(it);
593 }
594 
595 /* ======== Date copying thread context ======== */
596 
597 typedef struct {
598  datafiles_iter_t *it;
599  uint num;
600  uint *count;
601  os_mutex_t count_mutex;
602  os_thread_id_t id;
604 
605 static void print_version(void)
606 {
607  printf("%s Ver %s Rev %s for %s %s (%s)\n" ,my_progname,
608  XTRABACKUP_VERSION, XTRABACKUP_REVISION, "Drizzle",
609  TARGET_OS, TARGET_CPU);
610 }
611 
612 static void usage(void)
613 {
614  puts("Open source backup tool for InnoDB and XtraDB\n\
615 \n\
616 Copyright (C) 2009 Percona Inc.\n\
617 \n\
618 This program is free software; you can redistribute it and/or\n\
619 modify it under the terms of the GNU General Public License\n\
620 as published by the Free Software Foundation version 2\n\
621 of the License.\n\
622 \n\
623 This program is distributed in the hope that it will be useful,\n\
624 but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
625 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\
626 GNU General Public License for more details.\n\
627 \n\
628 You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n");
629 
630  printf("Usage: [%s [--defaults-file=#] --backup | %s [--defaults-file=#] --prepare] [OPTIONS]\n",my_progname,my_progname);
631  // FIXME: print what variables we have
632 }
633 
634 /* ================ Dummys =================== */
635 
636 UNIV_INTERN
637 ibool
639 /*============================*/
641 {
642  fprintf(stderr, "xtrabackup: thd_is_replication_slave_thread() is called\n");
643  return(FALSE);
644 }
645 
646 UNIV_INTERN
647 ibool
649 /*===========================*/
651 {
652  fprintf(stderr, "xtrabackup: thd_has_edited_nontrans_tables() is called\n");
653  return(FALSE);
654 }
655 
656 UNIV_INTERN
657 ibool
659 /*==========*/
660  const drizzled::Session *)
661 {
662  fprintf(stderr, "xtrabackup: thd_is_select() is called\n");
663  return(false);
664 }
665 
666 UNIV_INTERN
667 void
669  FILE*,
671  uint)
672 {
673  fprintf(stderr, "xtrabackup: innobase_mysql_print_thd() is called\n");
674 }
675 
676 void
678  ulint cset,
679  ulint* mbminlen,
680  ulint* mbmaxlen)
681 {
682  CHARSET_INFO* cs;
683  ut_ad(cset < 256);
684  ut_ad(mbminlen);
685  ut_ad(mbmaxlen);
686 
687  cs = all_charsets[cset];
688  if (cs) {
689  *mbminlen = cs->mbminlen;
690  *mbmaxlen = cs->mbmaxlen;
691  } else {
692  ut_a(cset == 0);
693  *mbminlen = *mbmaxlen = 0;
694  }
695 }
696 
697 UNIV_INTERN
698 void
700 /*===========================*/
701  const void*,
702  char* ,
703  const char* ,
704  ulint )
705 {
706  fprintf(stderr, "xtrabackup: innobase_convert_from_table_id() is called\n");
707 }
708 
709 UNIV_INTERN
710 void
712 /*=====================*/
713  const void*,
714  char* ,
715  const char* ,
716  ulint )
717 {
718  fprintf(stderr, "xtrabackup: innobase_convert_from_id() is called\n");
719 }
720 
721 int
723  const char* a,
724  const char* b)
725 {
726  return my_charset_utf8_general_ci.strcasecmp(a, b);
727 }
728 
729 void
731  char* a)
732 {
733  my_charset_utf8_general_ci.casedn_str(a);
734 }
735 
736 UNIV_INTERN
737 const char*
739 /*==============*/
741  size_t* )
742 {
743  fprintf(stderr, "xtrabackup: innobase_get_stmt() is called\n");
744  return("nothing");
745 }
746 
747 int
749 {
750  char filename[FN_REFLEN];
751  int fd2 = -1;
752  int fd = internal::create_temp_file(filename, opt_mysql_tmpdir,
753  "ib",
754 #ifdef __WIN__
755  O_BINARY | O_TRUNC | O_SEQUENTIAL |
756  O_TEMPORARY | O_SHORT_LIVED |
757 #endif /* __WIN__ */
758  O_CREAT | O_EXCL | O_RDWR);
759  if (fd >= 0) {
760 #ifndef __WIN__
761  /* On Windows, open files cannot be removed, but files can be
762  created with the O_TEMPORARY flag to the same effect
763  ("delete on close"). */
764  unlink(filename);
765 #endif /* !__WIN__ */
766  /* Copy the file descriptor, so that the additional resources
767  allocated by create_temp_file() can be freed by invoking
768  my_close().
769 
770  Because the file descriptor returned by this function
771  will be passed to fdopen(), it will be closed by invoking
772  fclose(), which in turn will invoke close() instead of
773  my_close(). */
774  fd2 = dup(fd);
775  if (fd2 < 0) {
776  fprintf(stderr, "xtrabackup: Got error %d on dup\n",fd2);
777  }
778  close(fd);
779  }
780  return(fd2);
781 }
782 
783 void
785  trx_t* ,
786  const char* ,
787  ulint )
788 {
789  /* do nothing */
790 }
791 
792 /*****************************************************************/
796 static
797 char*
799 /*========================*/
800  char* buf,
801  ulint buflen,
802  const char* id,
803  ulint idlen,
804  void* ,
805  ibool )
807 {
808  const char* s = id;
809  int q;
810 
811  /* See if the identifier needs to be quoted. */
812  q = '"';
813 
814  if (q == EOF) {
815  if (UNIV_UNLIKELY(idlen > buflen)) {
816  idlen = buflen;
817  }
818  memcpy(buf, s, idlen);
819  return(buf + idlen);
820  }
821 
822  /* Quote the identifier. */
823  if (buflen < 2) {
824  return(buf);
825  }
826 
827  *buf++ = q;
828  buflen--;
829 
830  for (; idlen; idlen--) {
831  int c = *s++;
832  if (UNIV_UNLIKELY(c == q)) {
833  if (UNIV_UNLIKELY(buflen < 3)) {
834  break;
835  }
836 
837  *buf++ = c;
838  *buf++ = c;
839  buflen -= 2;
840  } else {
841  if (UNIV_UNLIKELY(buflen < 2)) {
842  break;
843  }
844 
845  *buf++ = c;
846  buflen--;
847  }
848  }
849 
850  *buf++ = q;
851  return(buf);
852 }
853 
854 /*****************************************************************/
858 UNIV_INTERN
859 char*
861 /*==================*/
862  char* buf,
863  ulint buflen,
864  const char* id,
865  ulint idlen,
866  drizzled::Session *session,
867  ibool table_id)
869 {
870  char* s = buf;
871  const char* bufend = buf + buflen;
872 
873  if (table_id) {
874  const char* slash = (const char*) memchr(id, '/', idlen);
875  if (!slash) {
876 
877  goto no_db_name;
878  }
879 
880  /* Print the database name and table name separately. */
881  s = innobase_convert_identifier(s, bufend - s, id, slash - id,
882  session, TRUE);
883  if (UNIV_LIKELY(s < bufend)) {
884  *s++ = '.';
885  s = innobase_convert_identifier(s, bufend - s,
886  slash + 1, idlen
887  - (slash - id) - 1,
888  session, TRUE);
889  }
890  } else if (UNIV_UNLIKELY(*id == TEMP_INDEX_PREFIX)) {
891  /* Temporary index name (smart ALTER TABLE) */
892  const char temp_index_suffix[]= "--temporary--";
893 
894  s = innobase_convert_identifier(buf, buflen, id + 1, idlen - 1,
895  session, FALSE);
896  if (s - buf + (sizeof temp_index_suffix - 1) < buflen) {
897  memcpy(s, temp_index_suffix,
898  sizeof temp_index_suffix - 1);
899  s += sizeof temp_index_suffix - 1;
900  }
901  } else {
902 no_db_name:
903  s = innobase_convert_identifier(buf, buflen, id, idlen,
904  session, table_id);
905  }
906 
907  return(s);
908 
909 }
910 
911 ibool
913  trx_t* )
914 {
915  /* There are no mysql_thd */
916  return(FALSE);
917 }
918 
919 UNIV_INTERN int
921 /*===============*/
922  int mysql_type,
923  uint charset_number,
924  const unsigned char* a,
925  unsigned int a_length,
927  const unsigned char* b, /* in: data field */
928  unsigned int b_length); /* in: data field length,
929  not UNIV_SQL_NULL */
930 
931 int
933 /*===============*/
934  /* out: 1, 0, -1, if a is greater, equal, less than b, respectively */
935  int mysql_type, /* in: MySQL type */
936  uint charset_number, /* in: number of the charset */
937  const unsigned char* a, /* in: data field */
938  unsigned int a_length, /* in: data field length, not UNIV_SQL_NULL */
939  const unsigned char* b, /* in: data field */
940  unsigned int b_length) /* in: data field length, not UNIV_SQL_NULL */
941 {
942  const CHARSET_INFO* charset;
943  enum_field_types mysql_tp;
944  int ret;
945 
946  assert(a_length != UNIV_SQL_NULL);
947  assert(b_length != UNIV_SQL_NULL);
948 
949  mysql_tp = (enum_field_types) mysql_type;
950 
951  switch (mysql_tp) {
952 
953  case DRIZZLE_TYPE_BLOB:
954  case DRIZZLE_TYPE_VARCHAR:
955  /* Use the charset number to pick the right charset struct for
956  the comparison. Since the MySQL function get_charset may be
957  slow before Bar removes the mutex operation there, we first
958  look at 2 common charsets directly. */
959 
960  if (charset_number == default_charset_info->number) {
961  charset = default_charset_info;
962  } else {
963  charset = get_charset(charset_number);
964 
965  if (charset == NULL) {
966  fprintf(stderr, "xtrabackup needs charset %lu for doing "
967  "a comparison, but MySQL cannot "
968  "find that charset.",
969  (ulong) charset_number);
970  ut_a(0);
971  }
972  }
973 
974  /* Starting from 4.1.3, we use strnncollsp() in comparisons of
975  non-latin1_swedish_ci strings. NOTE that the collation order
976  changes then: 'b\0\0...' is ordered BEFORE 'b ...'. Users
977  having indexes on such data need to rebuild their tables! */
978 
979  ret = charset->coll->strnncollsp(charset,
980  a, a_length,
981  b, b_length, 0);
982  if (ret < 0) {
983  return(-1);
984  } else if (ret > 0) {
985  return(1);
986  } else {
987  return(0);
988  }
989  default:
990  ut_error;
991  }
992 
993  return(0);
994 }
995 
996 ulint
998  ulint charset_id,
999  ulint prefix_len,
1000  ulint data_len,
1001  const char* str)
1002 {
1003  ulint char_length; /* character length in bytes */
1004  ulint n_chars; /* number of characters in prefix */
1005  const CHARSET_INFO* charset; /* charset used in the field */
1006 
1007  charset = get_charset((uint) charset_id);
1008 
1009  ut_ad(charset);
1010  ut_ad(charset->mbmaxlen);
1011 
1012  /* Calculate how many characters at most the prefix index contains */
1013 
1014  n_chars = prefix_len / charset->mbmaxlen;
1015 
1016  /* If the charset is multi-byte, then we must find the length of the
1017  first at most n chars in the string. If the string contains less
1018  characters than n, then we return the length to the end of the last
1019  character. */
1020 
1021  if (charset->mbmaxlen > 1) {
1022  /* my_charpos() returns the byte length of the first n_chars
1023  characters, or a value bigger than the length of str, if
1024  there were not enough full characters in str.
1025 
1026  Why does the code below work:
1027  Suppose that we are looking for n UTF-8 characters.
1028 
1029  1) If the string is long enough, then the prefix contains at
1030  least n complete UTF-8 characters + maybe some extra
1031  characters + an incomplete UTF-8 character. No problem in
1032  this case. The function returns the pointer to the
1033  end of the nth character.
1034 
1035  2) If the string is not long enough, then the string contains
1036  the complete value of a column, that is, only complete UTF-8
1037  characters, and we can store in the column prefix index the
1038  whole string. */
1039 
1040  char_length = my_charpos(charset, str,
1041  str + data_len, (int) n_chars);
1042  if (char_length > data_len) {
1043  char_length = data_len;
1044  }
1045  } else {
1046  if (data_len < prefix_len) {
1047  char_length = data_len;
1048  } else {
1049  char_length = prefix_len;
1050  }
1051  }
1052 
1053  return(char_length);
1054 }
1055 
1056 UNIV_INTERN
1057 ulint
1059 /*================*/
1060  const char* ,
1061  ulint ,
1063  ulint ,
1064  char* ,
1065  ulint )
1067 {
1068  fprintf(stderr, "xtrabackup: innobase_raw_format() is called\n");
1069  return(0);
1070 }
1071 
1072 UNIV_INTERN
1073 ulong
1075 /*==================*/
1078 {
1079  return(innobase_lock_wait_timeout);
1080 }
1081 
1082 UNIV_INTERN
1083 ibool
1085 /*============*/
1086  drizzled::Session* )
1088 {
1089  return(FALSE);
1090 }
1091 
1092 ibool
1094 /*==========*/
1095  trx_t*)
1096 {
1097  return(FALSE);
1098 }
1099 
1100 #ifdef XTRADB_BASED
1101 trx_t*
1102 innobase_get_trx()
1103 {
1104  return(NULL);
1105 }
1106 
1107 ibool
1108 innobase_get_slow_log()
1109 {
1110  return(FALSE);
1111 }
1112 #endif
1113 
1114 /***********************************************************************
1115 Computes bit shift for a given value. If the argument is not a power
1116 of 2, returns 0.*/
1117 UNIV_INLINE
1118 ulint
1119 get_bit_shift(ulint value)
1120 {
1121  ulint shift;
1122 
1123  if (value == 0)
1124  return 0;
1125 
1126  for (shift = 0; !(value & 1UL); shift++) {
1127  value >>= 1;
1128  }
1129  return (value >> 1) ? 0 : shift;
1130 }
1131 
1132 static bool
1133 innodb_init_param(void)
1134 {
1135  /* innobase_init */
1136  static char current_dir[3]; /* Set if using current lib */
1137  bool ret;
1138  char *default_path;
1139 
1140  /* dummy for initialize all_charsets[] */
1141  get_charset_name(0);
1142 
1143 #ifdef XTRADB_BASED
1144  srv_page_size = 0;
1145  srv_page_size_shift = 0;
1146 
1147  if (innobase_page_size != (1 << 14)) {
1148  int n_shift = get_bit_shift(innobase_page_size);
1149 
1150  if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) {
1151  fprintf(stderr,
1152  "InnoDB: Warning: innodb_page_size has been "
1153  "changed from default value 16384.\n");
1154  srv_page_size_shift = n_shift;
1155  srv_page_size = 1 << n_shift;
1156  fprintf(stderr,
1157  "InnoDB: The universal page size of the "
1158  "database is set to %lu.\n", srv_page_size);
1159  } else {
1160  fprintf(stderr, "InnoDB: Error: invalid value of "
1161  "innobase_page_size: %lu", innobase_page_size);
1162  exit(EXIT_FAILURE);
1163  }
1164  } else {
1165  srv_page_size_shift = 14;
1166  srv_page_size = (1 << srv_page_size_shift);
1167  }
1168 
1169  srv_log_block_size = 0;
1170  if (innobase_log_block_size != 512) {
1171  uint n_shift = get_bit_shift(innobase_log_block_size);;
1172 
1173  fprintf(stderr,
1174  "InnoDB: Warning: innodb_log_block_size has "
1175  "been changed from its default value. "
1176  "(###EXPERIMENTAL### operation)\n");
1177  if (n_shift > 0) {
1178  srv_log_block_size = (1 << n_shift);
1179  fprintf(stderr,
1180  "InnoDB: The log block size is set to %lu.\n",
1181  srv_log_block_size);
1182  }
1183  } else {
1184  srv_log_block_size = 512;
1185  }
1186 
1187  if (!srv_log_block_size) {
1188  fprintf(stderr,
1189  "InnoDB: Error: %lu is not valid value for "
1190  "innodb_log_block_size.\n", innobase_log_block_size);
1191  goto error;
1192  }
1193 
1194  srv_fast_checksum = (ibool) innobase_fast_checksum;
1195 #endif
1196 
1197  /* Check that values don't overflow on 32-bit systems. */
1198  if (sizeof(ulint) == 4) {
1199  if (xtrabackup_use_memory > UINT32_MAX) {
1200  fprintf(stderr,
1201  "xtrabackup: use-memory can't be over 4GB"
1202  " on 32-bit systems\n");
1203  }
1204 
1205  if (innobase_buffer_pool_size > UINT32_MAX) {
1206  fprintf(stderr,
1207  "xtrabackup: innobase_buffer_pool_size can't be over 4GB"
1208  " on 32-bit systems\n");
1209 
1210  goto error;
1211  }
1212 
1213  if (innobase_log_file_size > UINT32_MAX) {
1214  fprintf(stderr,
1215  "xtrabackup: innobase_log_file_size can't be over 4GB"
1216  " on 32-bit systemsi\n");
1217 
1218  goto error;
1219  }
1220  }
1221 
1222  os_innodb_umask = (ulint)0664;
1223 
1224  /* First calculate the default path for innodb_data_home_dir etc.,
1225  in case the user has not given any value.
1226 
1227  Note that when using the embedded server, the datadirectory is not
1228  necessarily the current directory of this program. */
1229 
1230  /* It's better to use current lib, to keep paths short */
1231  current_dir[0] = FN_CURLIB;
1232  current_dir[1] = FN_LIBCHAR;
1233  current_dir[2] = 0;
1234  default_path = current_dir;
1235 
1236  ut_a(default_path);
1237 
1238  /* Set InnoDB initialization parameters according to the values
1239  read from MySQL .cnf file */
1240 
1241  if (xtrabackup_backup || xtrabackup_stats) {
1242  fprintf(stderr, "xtrabackup: Target instance is assumed as followings.\n");
1243  } else {
1244  fprintf(stderr, "xtrabackup: Temporary instance for recovery is set as followings.\n");
1245  }
1246 
1247  /*--------------- Data files -------------------------*/
1248 
1249  /* The default dir for data files is the datadir of MySQL */
1250 
1251  srv_data_home = ((xtrabackup_backup || xtrabackup_stats) && innobase_data_home_dir
1252  ? innobase_data_home_dir : default_path);
1253  fprintf(stderr, "xtrabackup: innodb_data_home_dir = %s\n", srv_data_home);
1254 
1255  /* Set default InnoDB data file size to 10 MB and let it be
1256  auto-extending. Thus users can use InnoDB in >= 4.0 without having
1257  to specify any startup options. */
1258 
1259  if (!innobase_data_file_path) {
1260  innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
1261  }
1262  fprintf(stderr, "xtrabackup: innodb_data_file_path = %s\n",
1263  innobase_data_file_path);
1264 
1265  /* Since InnoDB edits the argument in the next call, we make another
1266  copy of it: */
1267 
1268  internal_innobase_data_file_path = strdup(innobase_data_file_path);
1269 
1271  internal_innobase_data_file_path);
1272  if (ret == FALSE) {
1273  fprintf(stderr,
1274  "xtrabackup: syntax error in innodb_data_file_path\n");
1275 mem_free_and_error:
1276  free(internal_innobase_data_file_path);
1277  goto error;
1278  }
1279 
1280  if (xtrabackup_prepare) {
1281  /* "--prepare" needs filenames only */
1282  ulint i;
1283 
1284  for (i=0; i < srv_n_data_files; i++) {
1285  char *p;
1286 
1287  p = srv_data_file_names[i];
1288  while ((p = strstr(p, SRV_PATH_SEPARATOR_STR)) != NULL)
1289  {
1290  p++;
1291  srv_data_file_names[i] = p;
1292  }
1293  }
1294  }
1295 
1296 #ifdef XTRADB_BASED
1297  srv_doublewrite_file = innobase_doublewrite_file;
1298 #ifndef XTRADB55
1299  srv_extra_undoslots = (ibool) innobase_extra_undoslots;
1300 #endif
1301 #endif
1302 
1303  /* -------------- Log files ---------------------------*/
1304 
1305  /* The default dir for log files is the datadir of MySQL */
1306 
1307  if (!((xtrabackup_backup || xtrabackup_stats) && innobase_log_group_home_dir)) {
1308  innobase_log_group_home_dir = default_path;
1309  }
1310  if (xtrabackup_prepare && xtrabackup_incremental_dir) {
1311  innobase_log_group_home_dir = xtrabackup_incremental_dir;
1312  }
1313  fprintf(stderr, "xtrabackup: innodb_log_group_home_dir = %s\n",
1314  innobase_log_group_home_dir);
1315 
1316 #ifdef UNIV_LOG_ARCHIVE
1317  /* Since innodb_log_arch_dir has no relevance under MySQL,
1318  starting from 4.0.6 we always set it the same as
1319  innodb_log_group_home_dir: */
1320 
1321  innobase_log_arch_dir = innobase_log_group_home_dir;
1322 
1323  srv_arch_dir = innobase_log_arch_dir;
1324 #endif /* UNIG_LOG_ARCHIVE */
1325 
1326  ret = (bool)
1327  srv_parse_log_group_home_dirs(innobase_log_group_home_dir);
1328 
1329  if (ret == FALSE || innobase_mirrored_log_groups != 1) {
1330  fprintf(stderr, "xtrabackup: syntax error in innodb_log_group_home_dir, or a "
1331  "wrong number of mirrored log groups\n");
1332 
1333  goto mem_free_and_error;
1334  }
1335 
1336  srv_adaptive_flushing = FALSE;
1337  srv_use_sys_malloc = TRUE;
1338  srv_file_format = 1; /* Barracuda */
1340 
1341  /* --------------------------------------------------*/
1342 
1343  srv_file_flush_method_str = innobase_unix_file_flush_method;
1344 
1345  srv_n_log_groups = (ulint) innobase_mirrored_log_groups;
1346  srv_n_log_files = (ulint) innobase_log_files_in_group;
1347  srv_log_file_size = (ulint) innobase_log_file_size;
1348  fprintf(stderr, "xtrabackup: innodb_log_files_in_group = %ld\n",
1349  srv_n_log_files);
1350  fprintf(stderr, "xtrabackup: innodb_log_file_size = %ld\n",
1351  srv_log_file_size);
1352 
1353 #ifdef UNIV_LOG_ARCHIVE
1354  srv_log_archive_on = (ulint) innobase_log_archive;
1355 #endif /* UNIV_LOG_ARCHIVE */
1356  srv_log_buffer_size = (ulint) innobase_log_buffer_size;
1357 
1358  /* We set srv_pool_size here in units of 1 kB. InnoDB internally
1359  changes the value so that it becomes the number of database pages. */
1360 
1361  //srv_buf_pool_size = (ulint) innobase_buffer_pool_size;
1362  srv_buf_pool_size = (ulint) xtrabackup_use_memory;
1363 
1364  srv_mem_pool_size = (ulint) innobase_additional_mem_pool_size;
1365 
1366  srv_n_file_io_threads = (ulint) innobase_file_io_threads;
1367 
1368  srv_n_read_io_threads = (ulint) innobase_read_io_threads;
1369  srv_n_write_io_threads = (ulint) innobase_write_io_threads;
1370 
1371  srv_force_recovery = (ulint) innobase_force_recovery;
1372 
1373  srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite;
1374  srv_use_checksums = (ibool) innobase_use_checksums;
1375 
1376  btr_search_enabled = innobase_adaptive_hash_index ? true : false;
1377 
1378  os_use_large_pages = (ibool) innobase_use_large_pages;
1379  os_large_page_size = (ulint) innobase_large_page_size;
1380 
1381  row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout;
1382 
1383  srv_file_per_table = innobase_file_per_table ? true : false;
1384 
1385  srv_locks_unsafe_for_binlog = (ibool) innobase_locks_unsafe_for_binlog;
1386 
1387  srv_max_n_open_files = (ulint) innobase_open_files;
1388  srv_innodb_status = (ibool) innobase_create_status_file;
1389 
1390  srv_print_verbose_log = 1;
1391 
1392  /* Store the default charset-collation number of this MySQL
1393  installation */
1394 
1395  /* We cannot treat characterset here for now!! */
1396  data_mysql_default_charset_coll = (ulint)default_charset_info->number;
1397 
1398  ut_a(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number);
1399 
1400  //innobase_commit_concurrency_init_default();
1401 
1402  /* Since we in this module access directly the fields of a trx
1403  struct, and due to different headers and flags it might happen that
1404  mutex_t has a different size in this module and in InnoDB
1405  modules, we check at run time that the size is the same in
1406  these compilation modules. */
1407 
1408 
1409  /* On 5.5 srv_use_native_aio is TRUE by default. It is later reset
1410  if it is not supported by the platform in
1411  innobase_start_or_create_for_mysql(). As we don't call it in xtrabackup,
1412  we have to duplicate checks from that function here. */
1413 
1414 #ifdef __WIN__
1415  switch (os_get_os_version()) {
1416  case OS_WIN95:
1417  case OS_WIN31:
1418  case OS_WINNT:
1419  /* On Win 95, 98, ME, Win32 subsystem for Windows 3.1,
1420  and NT use simulated aio. In NT Windows provides async i/o,
1421  but when run in conjunction with InnoDB Hot Backup, it seemed
1422  to corrupt the data files. */
1423 
1424  srv_use_native_aio = FALSE;
1425  break;
1426 
1427  case OS_WIN2000:
1428  case OS_WINXP:
1429  /* On 2000 and XP, async IO is available. */
1430  srv_use_native_aio = TRUE;
1431  break;
1432 
1433  default:
1434  /* Vista and later have both async IO and condition variables */
1435  srv_use_native_aio = TRUE;
1436  srv_use_native_conditions = TRUE;
1437  break;
1438  }
1439 
1440 #elif defined(LINUX_NATIVE_AIO)
1441 
1442  if (srv_use_native_aio) {
1443  ut_print_timestamp(stderr);
1444  fprintf(stderr,
1445  " InnoDB: Using Linux native AIO\n");
1446  }
1447 #else
1448  /* Currently native AIO is supported only on windows and linux
1449  and that also when the support is compiled in. In all other
1450  cases, we ignore the setting of innodb_use_native_aio. */
1451  srv_use_native_aio = FALSE;
1452 
1453 #endif
1454 
1455  return(FALSE);
1456 
1457 error:
1458  fprintf(stderr, "xtrabackup: innodb_init_param(): Error occured.\n");
1459  return(TRUE);
1460 }
1461 
1462 static bool
1463 innodb_init(void)
1464 {
1465  int err;
1466 
1468 
1469  if (err != DB_SUCCESS) {
1470  free(internal_innobase_data_file_path);
1471  goto error;
1472  }
1473 
1474  /* They may not be needed for now */
1475 // (void) hash_init(&innobase_open_tables,system_charset_info, 32, 0, 0,
1476 // (hash_get_key) innobase_get_key, 0, 0);
1477 // pthread_mutex_init(&innobase_share_mutex, MY_MUTEX_INIT_FAST);
1478 // pthread_mutex_init(&prepare_commit_mutex, MY_MUTEX_INIT_FAST);
1479 // pthread_mutex_init(&commit_threads_m, MY_MUTEX_INIT_FAST);
1480 // pthread_mutex_init(&commit_cond_m, MY_MUTEX_INIT_FAST);
1481 // pthread_cond_init(&commit_cond, NULL);
1482 
1483  innodb_inited= 1;
1484 
1485  return(FALSE);
1486 
1487 error:
1488  fprintf(stderr, "xtrabackup: innodb_init(): Error occured.\n");
1489  return(TRUE);
1490 }
1491 
1492 static bool
1493 innodb_end(void)
1494 {
1495  srv_fast_shutdown = (ulint) innobase_fast_shutdown;
1496  innodb_inited = 0;
1497 
1498  fprintf(stderr, "xtrabackup: starting shutdown with innodb_fast_shutdown = %lu\n",
1499  srv_fast_shutdown);
1500 
1501  if (innobase_shutdown_for_mysql() != DB_SUCCESS) {
1502  goto error;
1503  }
1504  free(internal_innobase_data_file_path);
1505 
1506  /* They may not be needed for now */
1507 // hash_free(&innobase_open_tables);
1508 // pthread_mutex_destroy(&innobase_share_mutex);
1509 // pthread_mutex_destroy(&prepare_commit_mutex);
1510 // pthread_mutex_destroy(&commit_threads_m);
1511 // pthread_mutex_destroy(&commit_cond_m);
1512 // pthread_cond_destroy(&commit_cond);
1513 
1514  return(FALSE);
1515 
1516 error:
1517  fprintf(stderr, "xtrabackup: innodb_end(): Error occured.\n");
1518  return(TRUE);
1519 }
1520 
1521 /* ================= common ================= */
1522 static bool
1523 xtrabackup_read_metadata(char *filename)
1524 {
1525  FILE *fp;
1526 
1527  fp = fopen(filename,"r");
1528  if(!fp) {
1529  fprintf(stderr, "xtrabackup: Error: cannot open %s\n", filename);
1530  return(TRUE);
1531  }
1532 
1533  if (fscanf(fp, "backup_type = %29s\n", metadata_type)
1534  != 1)
1535  return(TRUE);
1536  if (fscanf(fp, "from_lsn = %"PRIu64"\n", &metadata_from_lsn)
1537  != 1)
1538  return(TRUE);
1539  if (fscanf(fp, "to_lsn = %"PRIu64"\n", &metadata_to_lsn)
1540  != 1)
1541  return(TRUE);
1542  if (fscanf(fp, "last_lsn = %"PRIu64"\n", &metadata_last_lsn)
1543  != 1) {
1544  metadata_last_lsn = 0;
1545  }
1546 
1547  fclose(fp);
1548 
1549  return(FALSE);
1550 }
1551 
1552 static bool
1553 xtrabackup_write_metadata(char *filename)
1554 {
1555  FILE *fp;
1556 
1557  fp = fopen(filename,"w");
1558  if(!fp) {
1559  fprintf(stderr, "xtrabackup: Error: cannot open %s\n", filename);
1560  return(TRUE);
1561  }
1562 
1563  if (fprintf(fp, "backup_type = %s\n", metadata_type)
1564  < 0)
1565  return(TRUE);
1566  if (fprintf(fp, "from_lsn = %"PRIu64"\n", metadata_from_lsn)
1567  < 0)
1568  return(TRUE);
1569  if (fprintf(fp, "to_lsn = %"PRIu64"\n", metadata_to_lsn)
1570  < 0)
1571  return(TRUE);
1572  if (fprintf(fp, "last_lsn = %"PRIu64"\n", metadata_last_lsn)
1573  < 0)
1574  return(TRUE);
1575 
1576  fclose(fp);
1577 
1578  return(FALSE);
1579 }
1580 
1581 /***********************************************************************
1582 Read meta info for an incremental delta.
1583 @return TRUE on success, FALSE on failure. */
1584 static bool
1585 xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info)
1586 {
1587  FILE *fp;
1588 
1589  memset(info, 0, sizeof(xb_delta_info_t));
1590 
1591  fp = fopen(filepath, "r");
1592  if (!fp) {
1593  /* Meta files for incremental deltas are optional */
1594  return(TRUE);
1595  }
1596 
1597  if (fscanf(fp, "page_size = %lu\n", &info->page_size) != 1)
1598  return(FALSE);
1599 
1600  fclose(fp);
1601 
1602  return(TRUE);
1603 }
1604 
1605 /***********************************************************************
1606 Write meta info for an incremental delta.
1607 @return TRUE on success, FALSE on failure. */
1608 static bool
1609 xb_write_delta_metadata(const char *filepath, const xb_delta_info_t *info)
1610 {
1611  FILE *fp;
1612 
1613  fp = fopen(filepath, "w");
1614  if (!fp) {
1615  fprintf(stderr, "xtrabackup: Error: cannot open %s\n", filepath);
1616  return(FALSE);
1617  }
1618 
1619  if (fprintf(fp, "page_size = %lu\n", info->page_size) < 0)
1620  return(FALSE);
1621 
1622  fclose(fp);
1623 
1624  return(TRUE);
1625 }
1626 
1627 /* ================= backup ================= */
1628 static void
1629 xtrabackup_io_throttling(void)
1630 {
1631  if (xtrabackup_throttle && (io_ticket--) < 0) {
1632  os_event_reset(wait_throttle);
1633  os_event_wait(wait_throttle);
1634  }
1635 }
1636 
1637 
1638 /* TODO: We may tune the behavior (e.g. by fil_aio)*/
1639 #define COPY_CHUNK 64
1640 
1641 static bool
1642 xtrabackup_copy_datafile(fil_node_t* node, uint thread_n)
1643 {
1644  os_file_t src_file = -1;
1645  os_file_t dst_file = -1;
1646  char dst_path[FN_REFLEN];
1647  char meta_path[FN_REFLEN];
1648  ibool success;
1649  byte* page;
1650  byte* buf2 = NULL;
1651  IB_INT64 file_size;
1652  IB_INT64 offset;
1653  ulint page_in_buffer= 0;
1654  ulint incremental_buffers = 0;
1655  ulint page_size;
1656  ulint page_size_shift;
1657  ulint zip_size;
1658  xb_delta_info_t info;
1659 
1660  info.page_size = 0;
1661 
1662 #ifdef XTRADB_BASED
1663  if (xtrabackup_tables && (!trx_sys_sys_space(node->space->id)))
1664 #else
1665  if (xtrabackup_tables && (node->space->id != 0))
1666 #endif
1667  { /* must backup id==0 */
1668  char *p;
1669  int p_len, regres= 0;
1670  char *next, *prev;
1671  char tmp;
1672  int i;
1673 
1674  p = node->name;
1675  prev = NULL;
1676  while ((next = strstr(p, SRV_PATH_SEPARATOR_STR)) != NULL)
1677  {
1678  prev = p;
1679  p = next + 1;
1680  }
1681  p_len = strlen(p) - strlen(".ibd");
1682 
1683  if (p_len < 1) {
1684  /* unknown situation: skip filtering */
1685  goto skip_filter;
1686  }
1687 
1688  /* TODO: Fix this lazy implementation... */
1689  tmp = p[p_len];
1690  p[p_len] = 0;
1691  *(p - 1) = '.';
1692 
1693  for (i = 0; i < tables_regex_num; i++) {
1694  regres = regexec(&tables_regex[i], prev, 1, tables_regmatch, 0);
1695  if (regres != REG_NOMATCH)
1696  break;
1697  }
1698 
1699  p[p_len] = tmp;
1700  *(p - 1) = SRV_PATH_SEPARATOR;
1701 
1702  if ( regres == REG_NOMATCH ) {
1703  printf("[%02u] Copying %s is skipped.\n",
1704  thread_n, node->name);
1705  return(FALSE);
1706  }
1707  }
1708 
1709 #ifdef XTRADB_BASED
1710  if (xtrabackup_tables_file && (!trx_sys_sys_space(node->space->id)))
1711 #else
1712  if (xtrabackup_tables_file && (node->space->id != 0))
1713 #endif
1714  { /* must backup id==0 */
1715  xtrabackup_tables_t* table;
1716  char *p;
1717  int p_len;
1718  char *next, *prev;
1719  char tmp;
1720 
1721  p = node->name;
1722  prev = NULL;
1723  while ((next = strstr(p, SRV_PATH_SEPARATOR_STR)) != NULL)
1724  {
1725  prev = p;
1726  p = next + 1;
1727  }
1728  p_len = strlen(p) - strlen(".ibd");
1729 
1730  if (p_len < 1) {
1731  /* unknown situation: skip filtering */
1732  goto skip_filter;
1733  }
1734 
1735  /* TODO: Fix this lazy implementation... */
1736  tmp = p[p_len];
1737  p[p_len] = 0;
1738 
1739  HASH_SEARCH(name_hash, tables_hash, ut_fold_string(prev),
1741  table,
1742  ut_ad(table->name),
1743  !strcmp(table->name, prev));
1744 
1745  p[p_len] = tmp;
1746 
1747  if (!table) {
1748  printf("[%02u] Copying %s is skipped.\n",
1749  thread_n, node->name);
1750  return(FALSE);
1751  }
1752  }
1753 
1754 skip_filter:
1755  zip_size = fil_space_get_zip_size(node->space->id);
1756  if (zip_size == ULINT_UNDEFINED) {
1757  fprintf(stderr, "[%02u] xtrabackup: Warning: "
1758  "Failed to determine page size for %s.\n"
1759  "[%02u] xtrabackup: Warning: We assume the table was "
1760  "dropped during xtrabackup execution and ignore the "
1761  "file.\n", thread_n, node->name, thread_n);
1762  goto skip;
1763  } else if (zip_size) {
1764  page_size = zip_size;
1765  page_size_shift = get_bit_shift(page_size);
1766  fprintf(stderr, "[%02u] %s is compressed with page size = "
1767  "%lu bytes\n", thread_n, node->name, page_size);
1768  if (page_size_shift < 10 || page_size_shift > 14) {
1769  fprintf(stderr, "[%02u] xtrabackup: Error: Invalid "
1770  "page size.\n", thread_n);
1771  ut_error;
1772  }
1773  } else {
1774  page_size = UNIV_PAGE_SIZE;
1775  page_size_shift = UNIV_PAGE_SIZE_SHIFT;
1776  }
1777 
1778 #ifdef XTRADB_BASED
1779  if (trx_sys_sys_space(node->space->id))
1780 #else
1781  if (node->space->id == 0)
1782 #endif
1783  {
1784  char *next, *p;
1785  /* system datafile "/fullpath/datafilename.ibd" or "./datafilename.ibd" */
1786  p = node->name;
1787  while ((next = strstr(p, SRV_PATH_SEPARATOR_STR)) != NULL)
1788  {
1789  p = next + 1;
1790  }
1791  sprintf(dst_path, "%s/%s", xtrabackup_target_dir, p);
1792  } else {
1793  /* file per table style "./database/table.ibd" */
1794  sprintf(dst_path, "%s%s", xtrabackup_target_dir, strstr(node->name, SRV_PATH_SEPARATOR_STR));
1795  }
1796 
1797  if (xtrabackup_incremental) {
1798  snprintf(meta_path, sizeof(meta_path),
1799  "%s%s", dst_path, XB_DELTA_INFO_SUFFIX);
1800  strcat(dst_path, ".delta");
1801 
1802  /* clear buffer */
1803  bzero(incremental_buffer, (page_size/4) * page_size);
1804  page_in_buffer = 0;
1805  mach_write_to_4(incremental_buffer, 0x78747261UL);/*"xtra"*/
1806  page_in_buffer++;
1807 
1808  info.page_size = page_size;
1809  }
1810 
1811  /* open src_file*/
1812  if (!node->open) {
1813  src_file = os_file_create_simple_no_error_handling(
1814  0 /* dummy of innodb_file_data_key */,
1815  node->name, OS_FILE_OPEN,
1816  OS_FILE_READ_ONLY, &success);
1817  if (!success) {
1818  /* The following call prints an error message */
1819  os_file_get_last_error(TRUE);
1820 
1821  fprintf(stderr,
1822  "[%02u] xtrabackup: Warning: cannot open %s\n"
1823  "[%02u] xtrabackup: Warning: We assume the "
1824  "table was dropped during xtrabackup execution "
1825  "and ignore the file.\n",
1826  thread_n, node->name, thread_n);
1827  goto skip;
1828  }
1829 
1830  if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT) {
1831  os_file_set_nocache(src_file, node->name, "OPEN");
1832  }
1833  } else {
1834  src_file = node->handle;
1835  }
1836 
1837 #ifdef USE_POSIX_FADVISE
1838  posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL);
1839  posix_fadvise(src_file, 0, 0, POSIX_FADV_DONTNEED);
1840 #endif
1841 
1842  /* open dst_file */
1843  /* os_file_create reads srv_unix_file_flush_method */
1844  dst_file = os_file_create(
1845  0 /* dummy of innodb_file_data_key */,
1846  dst_path, OS_FILE_CREATE,
1847  OS_FILE_NORMAL, OS_DATA_FILE, &success);
1848  if (!success) {
1849  /* The following call prints an error message */
1850  os_file_get_last_error(TRUE);
1851 
1852  fprintf(stderr,"[%02u] xtrabackup: error: "
1853  "cannot open %s\n", thread_n, dst_path);
1854  goto error;
1855  }
1856 
1857 #ifdef USE_POSIX_FADVISE
1858  posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
1859 #endif
1860 
1861  /* copy : TODO: tune later */
1862  printf("[%02u] Copying %s \n to %s\n", thread_n,
1863  node->name, dst_path);
1864 
1865  buf2 = (unsigned char*) ut_malloc(COPY_CHUNK * page_size + UNIV_PAGE_SIZE);
1866  page = (unsigned char*) ut_align(buf2, UNIV_PAGE_SIZE);
1867 
1868  success = os_file_read(src_file, page, 0, 0, UNIV_PAGE_SIZE);
1869  if (!success) {
1870  goto error;
1871  }
1872 
1873  file_size = os_file_get_size_as_iblonglong(src_file);
1874 
1875  for (offset = 0; offset < file_size; offset += COPY_CHUNK * page_size) {
1876  ulint chunk;
1877  ulint chunk_offset;
1878  ulint retry_count = 10;
1879 //copy_loop:
1880  if ((ulint)(file_size - offset) > COPY_CHUNK * page_size) {
1881  chunk = COPY_CHUNK * page_size;
1882  } else {
1883  chunk = (ulint)(file_size - offset);
1884  }
1885 
1886 read_retry:
1887  xtrabackup_io_throttling();
1888 
1889  success = os_file_read(src_file, page,
1890  (ulint)(offset & 0xFFFFFFFFUL),
1891  (ulint)(offset >> 32), chunk);
1892  if (!success) {
1893  goto error;
1894  }
1895 
1896  /* check corruption and retry */
1897  for (chunk_offset = 0; chunk_offset < chunk; chunk_offset += page_size) {
1898  if (buf_page_is_corrupted(page + chunk_offset, zip_size))
1899  {
1900  if (
1901 #ifdef XTRADB_BASED
1902  trx_sys_sys_space(node->space->id)
1903 #else
1904  node->space->id == 0
1905 #endif
1906  && ((offset + (IB_INT64)chunk_offset) >> page_size_shift)
1907  >= FSP_EXTENT_SIZE
1908  && ((offset + (IB_INT64)chunk_offset) >> page_size_shift)
1909  < FSP_EXTENT_SIZE * 3) {
1910  /* double write buffer may have old data in the end
1911  or it may contain the other format page like COMPRESSED.
1912  So, we can pass the check of double write buffer.*/
1913  ut_a(page_size == UNIV_PAGE_SIZE);
1914  fprintf(stderr, "[%02u] xtrabackup: "
1915  "Page %lu seems double write "
1916  "buffer. passing the check.\n",
1917  thread_n,
1918  (ulint)((offset +
1919  (IB_INT64)chunk_offset) >>
1920  page_size_shift));
1921  } else {
1922  retry_count--;
1923  if (retry_count == 0) {
1924  fprintf(stderr,
1925  "[%02u] xtrabackup: "
1926  "Error: 10 retries "
1927  "resulted in fail. This"
1928  "file seems to be "
1929  "corrupted.\n",
1930  thread_n);
1931  goto error;
1932  }
1933  fprintf(stderr, "[%02u] xtrabackup: "
1934  "Database page corruption "
1935  "detected at page %lu. "
1936  "retrying...\n",
1937  thread_n,
1938  (ulint)((offset +
1939  (IB_INT64)chunk_offset)
1940  >> page_size_shift));
1941  goto read_retry;
1942  }
1943  }
1944  }
1945 
1946  if (xtrabackup_incremental) {
1947  for (chunk_offset = 0; chunk_offset < chunk; chunk_offset += page_size) {
1948  /* newer page */
1949  /* This condition may be OK for header, ibuf and fsp */
1950  if (ut_dulint_cmp(incremental_lsn,
1951  MACH_READ_64(page + chunk_offset + FIL_PAGE_LSN)) < 0) {
1952  /* ========================================= */
1953  IB_INT64 offset_on_page;
1954 
1955  if (page_in_buffer == page_size/4) {
1956  /* flush buffer */
1957  success = os_file_write(dst_path, dst_file, incremental_buffer,
1958  ((incremental_buffers * (page_size/4))
1959  << page_size_shift) & 0xFFFFFFFFUL,
1960  (incremental_buffers * (page_size/4))
1961  >> (32 - page_size_shift),
1962  page_in_buffer * page_size);
1963  if (!success) {
1964  goto error;
1965  }
1966 
1967  incremental_buffers++;
1968 
1969  /* clear buffer */
1970  bzero(incremental_buffer, (page_size/4) * page_size);
1971  page_in_buffer = 0;
1972  mach_write_to_4(incremental_buffer, 0x78747261UL);/*"xtra"*/
1973  page_in_buffer++;
1974  }
1975 
1976  offset_on_page = ((offset + (IB_INT64)chunk_offset) >> page_size_shift);
1977  ut_a(offset_on_page >> 32 == 0);
1978 
1979  mach_write_to_4(incremental_buffer + page_in_buffer * 4, (ulint)offset_on_page);
1980  memcpy(incremental_buffer + page_in_buffer * page_size,
1981  page + chunk_offset, page_size);
1982 
1983  page_in_buffer++;
1984  /* ========================================= */
1985  }
1986  }
1987  } else {
1988  success = os_file_write(dst_path, dst_file, page,
1989  (ulint)(offset & 0xFFFFFFFFUL),
1990  (ulint)(offset >> 32), chunk);
1991  if (!success) {
1992  goto error;
1993  }
1994  }
1995 
1996  }
1997 
1998  if (xtrabackup_incremental) {
1999  /* termination */
2000  if (page_in_buffer != page_size/4) {
2001  mach_write_to_4(incremental_buffer + page_in_buffer * 4, 0xFFFFFFFFUL);
2002  }
2003 
2004  mach_write_to_4(incremental_buffer, 0x58545241UL);/*"XTRA"*/
2005 
2006  /* flush buffer */
2007  success = os_file_write(dst_path, dst_file, incremental_buffer,
2008  ((incremental_buffers * (page_size/4))
2009  << page_size_shift) & 0xFFFFFFFFUL,
2010  (incremental_buffers * (page_size/4))
2011  >> (32 - page_size_shift),
2012  page_in_buffer * page_size);
2013  if (!success) {
2014  goto error;
2015  }
2016  if (!xb_write_delta_metadata(meta_path, &info)) {
2017  fprintf(stderr, "[%02u] xtrabackup: Error: "
2018  "failed to write meta info for %s\n",
2019  thread_n, dst_path);
2020  goto error;
2021  }
2022  }
2023 
2024  success = os_file_flush(dst_file);
2025  if (!success) {
2026  goto error;
2027  }
2028 
2029 
2030  /* check size again */
2031  /* TODO: but is it needed ?? */
2032 // if (file_size < os_file_get_size_as_iblonglong(src_file)) {
2033 // offset -= COPY_CHUNK * page_size;
2034 // file_size = os_file_get_size_as_iblonglong(src_file);
2035 // goto copy_loop;
2036 // }
2037 
2038  /* TODO: How should we treat double_write_buffer here? */
2039  /* (currently, don't care about. Because,
2040  the blocks is newer than the last checkpoint anyway.) */
2041 
2042  /* close */
2043  printf("[%02u] ...done\n", thread_n);
2044  if (!node->open) {
2045  os_file_close(src_file);
2046  }
2047  os_file_close(dst_file);
2048  ut_free(buf2);
2049  return(FALSE);
2050 error:
2051  if (src_file != -1 && !node->open)
2052  os_file_close(src_file);
2053  if (dst_file != -1)
2054  os_file_close(dst_file);
2055  if (buf2)
2056  ut_free(buf2);
2057  fprintf(stderr, "[%02u] xtrabackup: Error: "
2058  "xtrabackup_copy_datafile() failed.\n", thread_n);
2059  return(TRUE); /*ERROR*/
2060 
2061 skip:
2062  if (src_file != -1 && !node->open)
2063  os_file_close(src_file);
2064  if (dst_file != -1)
2065  os_file_close(dst_file);
2066  if (buf2)
2067  ut_free(buf2);
2068  fprintf(stderr, "[%02u] xtrabackup: Warning: skipping file %s.\n",
2069  thread_n, node->name);
2070  return(FALSE);
2071 }
2072 
2073 static bool
2074 xtrabackup_copy_logfile(LSN64 from_lsn, bool is_last)
2075 {
2076  /* definition from recv_recovery_from_checkpoint_start() */
2077  log_group_t* group;
2078  LSN64 group_scanned_lsn;
2079  LSN64 contiguous_lsn;
2080 
2081  ibool success;
2082 
2083  if (!xtrabackup_stream)
2084  ut_a(dst_log != -1);
2085 
2086  /* read from checkpoint_lsn_start to current */
2087  contiguous_lsn = ut_dulint_align_down(from_lsn,
2089 
2090  /* TODO: We must check the contiguous_lsn still exists in log file.. */
2091 
2092  group = UT_LIST_GET_FIRST(log_sys->log_groups);
2093 
2094  while (group) {
2095  ibool finished;
2096  LSN64 start_lsn;
2097  LSN64 end_lsn;
2098 
2099 
2100  /* reference recv_group_scan_log_recs() */
2101  finished = FALSE;
2102 
2103  start_lsn = contiguous_lsn;
2104 
2105  while (!finished) {
2106  end_lsn = ut_dulint_add(start_lsn, RECV_SCAN_SIZE);
2107 
2108  xtrabackup_io_throttling();
2109 
2110  log_group_read_log_seg(LOG_RECOVER, log_sys->buf,
2111  group, start_lsn, end_lsn);
2112 
2113  //printf("log read from (%lu %lu) to (%lu %lu)\n",
2114  // start_lsn.high, start_lsn.low, end_lsn.high, end_lsn.low);
2115 
2116  /* reference recv_scan_log_recs() */
2117  {
2118  byte* log_block;
2119  ulint no;
2120  LSN64 scanned_lsn;
2121  ulint data_len;
2122 
2123  ulint scanned_checkpoint_no = 0;
2124 
2125  finished = FALSE;
2126 
2127  log_block = log_sys->buf;
2128  scanned_lsn = start_lsn;
2129 
2130  while (log_block < log_sys->buf + RECV_SCAN_SIZE && !finished) {
2131 
2132  no = log_block_get_hdr_no(log_block);
2133 
2134  if (no != log_block_convert_lsn_to_no(scanned_lsn)
2135  || !log_block_checksum_is_ok_or_old_format(log_block)) {
2136 
2137  if (no > log_block_convert_lsn_to_no(scanned_lsn)
2138  && log_block_checksum_is_ok_or_old_format(log_block)) {
2139  fprintf(stderr,
2140 ">> ###Warning###: The copying transaction log migh be overtaken already by the target.\n"
2141 ">> : Waiting log block no %lu, but the bumber is already %lu.\n"
2142 ">> : If the number equals %lu + n * %lu, it should be overtaken already.\n",
2143  (ulong) log_block_convert_lsn_to_no(scanned_lsn),
2144  (ulong) no,
2145  (ulong) log_block_convert_lsn_to_no(scanned_lsn),
2146  (ulong) (log_block_convert_lsn_to_no(
2147  log_group_get_capacity(group)
2148  ) - 1));
2149 
2150  } else if (no == log_block_convert_lsn_to_no(scanned_lsn)
2151  && !log_block_checksum_is_ok_or_old_format(
2152  log_block)) {
2153  fprintf(stderr,
2154 "xtrabackup: Log block no %lu at lsn %"PRIu64" has\n"
2155 "xtrabackup: ok header, but checksum field contains %lu, should be %lu\n",
2156  (ulong) no,
2157  scanned_lsn,
2158  (ulong) log_block_get_checksum(log_block),
2159  (ulong) log_block_calc_checksum(log_block));
2160  }
2161 
2162  /* Garbage or an incompletely written log block */
2163 
2164  finished = TRUE;
2165 
2166  break;
2167  }
2168 
2169  if (log_block_get_flush_bit(log_block)) {
2170  /* This block was a start of a log flush operation:
2171  we know that the previous flush operation must have
2172  been completed for all log groups before this block
2173  can have been flushed to any of the groups. Therefore,
2174  we know that log data is contiguous up to scanned_lsn
2175  in all non-corrupt log groups. */
2176 
2177  if (ut_dulint_cmp(scanned_lsn, contiguous_lsn) > 0) {
2178  contiguous_lsn = scanned_lsn;
2179  }
2180  }
2181 
2182  data_len = log_block_get_data_len(log_block);
2183 
2184  if (
2185  (scanned_checkpoint_no > 0)
2186  && (log_block_get_checkpoint_no(log_block)
2187  < scanned_checkpoint_no)
2188  && (scanned_checkpoint_no
2189  - log_block_get_checkpoint_no(log_block)
2190  > 0x80000000UL)) {
2191 
2192  /* Garbage from a log buffer flush which was made
2193  before the most recent database recovery */
2194 
2195  finished = TRUE;
2196  break;
2197  }
2198 
2199  scanned_lsn = ut_dulint_add(scanned_lsn, data_len);
2200  scanned_checkpoint_no = log_block_get_checkpoint_no(log_block);
2201 
2202  if (data_len < OS_FILE_LOG_BLOCK_SIZE) {
2203  /* Log data for this group ends here */
2204 
2205  finished = TRUE;
2206  } else {
2207  log_block += OS_FILE_LOG_BLOCK_SIZE;
2208  }
2209  } /* while (log_block < log_sys->buf + RECV_SCAN_SIZE && !finished) */
2210 
2211  group_scanned_lsn = scanned_lsn;
2212 
2213 
2214 
2215  }
2216 
2217  /* ===== write log to 'xtrabackup_logfile' ====== */
2218  {
2219  ulint write_size;
2220 
2221  if (!finished) {
2222  write_size = RECV_SCAN_SIZE;
2223  } else {
2224  write_size = ut_dulint_minus(
2225  ut_dulint_align_up(group_scanned_lsn, OS_FILE_LOG_BLOCK_SIZE),
2226  start_lsn);
2227  }
2228 
2229  //printf("Wrinting offset= %lld, size= %lu\n", log_copy_offset, write_size);
2230 
2231  if (!xtrabackup_stream) {
2232  success = os_file_write(dst_log_path, dst_log, log_sys->buf,
2233  (ulint)(log_copy_offset & 0xFFFFFFFFUL),
2234  (ulint)(log_copy_offset >> 32), write_size);
2235  } else {
2236  ulint ret;
2237  ulint stdout_write_size = write_size;
2238  if (finished && !is_last
2239  && group_scanned_lsn % OS_FILE_LOG_BLOCK_SIZE
2240  )
2241  stdout_write_size -= OS_FILE_LOG_BLOCK_SIZE;
2242  if (stdout_write_size) {
2243  ret = write(fileno(stdout), log_sys->buf, stdout_write_size);
2244  if (ret == stdout_write_size) {
2245  success = TRUE;
2246  } else {
2247  fprintf(stderr, "write: %lu > %lu\n", stdout_write_size, ret);
2248  success = FALSE;
2249  }
2250  } else {
2251  success = TRUE; /* do nothing */
2252  }
2253  }
2254 
2255  log_copy_offset += write_size;
2256 
2257  if (finished && group_scanned_lsn % OS_FILE_LOG_BLOCK_SIZE)
2258  {
2259  /* if continue, it will start from align_down(group_scanned_lsn) */
2260  log_copy_offset -= OS_FILE_LOG_BLOCK_SIZE;
2261  }
2262 
2263  if(!success) {
2264  if (!xtrabackup_stream) {
2265  fprintf(stderr, "xtrabackup: Error: os_file_write to %s\n", dst_log_path);
2266  } else {
2267  fprintf(stderr, "xtrabackup: Error: write to stdout\n");
2268  }
2269  goto error;
2270  }
2271 
2272 
2273  }
2274 
2275 
2276 
2277 
2278 
2279  start_lsn = end_lsn;
2280  }
2281 
2282 
2283 
2284  group->scanned_lsn = group_scanned_lsn;
2285 
2286 
2287  fprintf(stderr, ">> log scanned up to (%"PRIu64")\n",group->scanned_lsn);
2288 
2289  group = UT_LIST_GET_NEXT(log_groups, group);
2290 
2291  /* update global variable*/
2292  log_copy_scanned_lsn = group_scanned_lsn;
2293 
2294  /* innodb_mirrored_log_groups must be 1, no other groups */
2295  ut_a(group == NULL);
2296  }
2297 
2298 
2299  if (!xtrabackup_stream) {
2300  success = os_file_flush(dst_log);
2301  } else {
2302  fflush(stdout);
2303  success = TRUE;
2304  }
2305 
2306  if(!success) {
2307  goto error;
2308  }
2309 
2310  return(FALSE);
2311 
2312 error:
2313  if (!xtrabackup_stream)
2314  os_file_close(dst_log);
2315  fprintf(stderr, "xtrabackup: Error: xtrabackup_copy_logfile() failed.\n");
2316  return(TRUE);
2317 }
2318 
2319 /* copying logfile in background */
2320 #define SLEEPING_PERIOD 5
2321 
2322 static
2323 #ifndef __WIN__
2324 void*
2325 #else
2326 ulint
2327 #endif
2328 log_copying_thread(
2329  void* )
2330 {
2331  ulint counter = 0;
2332 
2333  if (!xtrabackup_stream)
2334  ut_a(dst_log != -1);
2335 
2336  log_copying_running = TRUE;
2337 
2338  while(log_copying) {
2339  os_thread_sleep(200000); /*0.2 sec*/
2340 
2341  counter++;
2342  if(counter >= SLEEPING_PERIOD * 5) {
2343  if(xtrabackup_copy_logfile(log_copy_scanned_lsn, FALSE))
2344  goto end;
2345  counter = 0;
2346  }
2347  }
2348 
2349  /* last copying */
2350  if(xtrabackup_copy_logfile(log_copy_scanned_lsn, TRUE))
2351  goto end;
2352 
2353  log_copying_succeed = TRUE;
2354 end:
2355  log_copying_running = FALSE;
2356  os_thread_exit(NULL);
2357 
2358  return(0);
2359 }
2360 
2361 /* io throttle watching (rough) */
2362 static
2363 #ifndef __WIN__
2364 void*
2365 #else
2366 ulint
2367 #endif
2368 io_watching_thread(
2369  void* )
2370 {
2371  /* currently, for --backup only */
2372  ut_a(xtrabackup_backup);
2373 
2374  while (log_copying) {
2375  os_thread_sleep(1000000); /*1 sec*/
2376 
2377  //for DEBUG
2378  //if (io_ticket == xtrabackup_throttle) {
2379  // fprintf(stderr, "There seem to be no IO...?\n");
2380  //}
2381 
2382  io_ticket = xtrabackup_throttle;
2383  os_event_set(wait_throttle);
2384  }
2385 
2386  /* stop io throttle */
2387  xtrabackup_throttle = 0;
2388  os_event_set(wait_throttle);
2389 
2390  os_thread_exit(NULL);
2391 
2392  return(0);
2393 }
2394 
2395 /************************************************************************
2396 I/o-handler thread function. */
2397 static
2398 
2399 #ifndef __WIN__
2400 void*
2401 #else
2402 ulint
2403 #endif
2404 io_handler_thread(
2405 /*==============*/
2406  void* arg)
2407 {
2408  ulint segment;
2409  ulint i;
2410 
2411  segment = *((ulint*)arg);
2412 
2413  for (i = 0;; i++) {
2414  fil_aio_wait(segment);
2415  }
2416 
2417  /* We count the number of threads in os_thread_exit(). A created
2418  thread should always use that to exit and not use return() to exit.
2419  The thread actually never comes here because it is exited in an
2420  os_event_wait(). */
2421 
2422  os_thread_exit(NULL);
2423 
2424 #ifndef __WIN__
2425  return(NULL); /* Not reached */
2426 #else
2427  return(0);
2428 #endif
2429 }
2430 
2431 /***************************************************************************
2432 Creates an output directory for a given tablespace, if it does not exist */
2433 static
2434 int
2435 xtrabackup_create_output_dir(
2436 /*==========================*/
2437  /* out: 0 if succes, -1 if failure */
2438  fil_space_t *space) /* in: tablespace */
2439 {
2440  char path[FN_REFLEN];
2441  char *ptr1, *ptr2;
2442 
2443  /* mkdir if not exist */
2444  ptr1 = strstr(space->name, SRV_PATH_SEPARATOR_STR);
2445  if (ptr1) {
2446  ptr2 = strstr(ptr1 + 1, SRV_PATH_SEPARATOR_STR);
2447  } else {
2448  ptr2 = NULL;
2449  }
2450 #ifdef XTRADB_BASED
2451  if(!trx_sys_sys_space(space->id) && ptr2)
2452 #else
2453  if(space->id && ptr2)
2454 #endif
2455  {
2456  /* single table space */
2457  *ptr2 = 0; /* temporary (it's my lazy..)*/
2458  snprintf(path, sizeof(path), "%s%s", xtrabackup_target_dir,
2459  ptr1);
2460  *ptr2 = SRV_PATH_SEPARATOR;
2461 
2462  if (mkdir(path, 0777) != 0 && errno != EEXIST) {
2463  fprintf(stderr,
2464  "xtrabackup: Error: cannot mkdir %d: %s\n",
2465  errno, path);
2466  return -1;
2467  }
2468  }
2469  return 0;
2470 }
2471 
2472 /**************************************************************************
2473 Datafiles copying thread.*/
2474 static
2475 os_thread_ret_t
2476 data_copy_thread_func(
2477 /*==================*/
2478  void *arg) /* thread context */
2479 {
2480  data_thread_ctxt_t *ctxt = (data_thread_ctxt_t *) arg;
2481  uint num = ctxt->num;
2482  fil_space_t* space;
2483  ibool space_changed;
2484  fil_node_t* node;
2485 
2486  while ((node = datafiles_iter_next(ctxt->it, &space_changed)) != NULL) {
2487  space = node->space;
2488 
2489  if (space_changed && xtrabackup_create_output_dir(space))
2490  exit(EXIT_FAILURE);
2491 
2492  /* copy the datafile */
2493  if(xtrabackup_copy_datafile(node, num)) {
2494  fprintf(stderr, "[%02u] xtrabackup: Error: "
2495  "failed to copy datafile.\n",
2496  num);
2497  exit(EXIT_FAILURE);
2498  }
2499  }
2500 
2501  os_mutex_enter(ctxt->count_mutex);
2502  (*ctxt->count)--;
2503  os_mutex_exit(ctxt->count_mutex);
2504 
2505  os_thread_exit(NULL);
2506  OS_THREAD_DUMMY_RETURN;
2507 }
2508 
2509 /* CAUTION(?): Don't rename file_per_table during backup */
2510 static void
2511 xtrabackup_backup_func(void)
2512 {
2513  struct stat stat_info;
2514  LSN64 latest_cp;
2515 
2516 #ifdef USE_POSIX_FADVISE
2517  fprintf(stderr, "xtrabackup: uses posix_fadvise().\n");
2518 #endif
2519 
2520  /* cd to datadir */
2521 
2522  if (chdir(mysql_real_data_home) != 0)
2523  {
2524  fprintf(stderr, "xtrabackup: cannot my_setwd %s\n", mysql_real_data_home);
2525  exit(EXIT_FAILURE);
2526  }
2527  fprintf(stderr, "xtrabackup: cd to %s\n", mysql_real_data_home);
2528 
2529  mysql_data_home= mysql_data_home_buff;
2530  mysql_data_home[0]=FN_CURLIB; // all paths are relative from here
2531  mysql_data_home[1]=0;
2532 
2533  /* set read only */
2534  srv_read_only = TRUE;
2535 
2536  /* initialize components */
2537  if(innodb_init_param())
2538  exit(EXIT_FAILURE);
2539 
2540  if (srv_file_flush_method_str == NULL) {
2541  /* These are the default options */
2542  srv_unix_file_flush_method = SRV_UNIX_FSYNC;
2543 
2544  srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED;
2545 #ifndef __WIN__
2546  } else if (0 == ut_strcmp(srv_file_flush_method_str, "fsync")) {
2547  srv_unix_file_flush_method = SRV_UNIX_FSYNC;
2548 
2549  } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DSYNC")) {
2550  srv_unix_file_flush_method = SRV_UNIX_O_DSYNC;
2551 
2552  } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DIRECT")) {
2553  srv_unix_file_flush_method = SRV_UNIX_O_DIRECT;
2554  fprintf(stderr,"xtrabackup: use O_DIRECT\n");
2555  } else if (0 == ut_strcmp(srv_file_flush_method_str, "littlesync")) {
2556  srv_unix_file_flush_method = SRV_UNIX_LITTLESYNC;
2557 
2558  } else if (0 == ut_strcmp(srv_file_flush_method_str, "nosync")) {
2559  srv_unix_file_flush_method = SRV_UNIX_NOSYNC;
2560 #else
2561  } else if (0 == ut_strcmp(srv_file_flush_method_str, "normal")) {
2562  srv_win_file_flush_method = SRV_WIN_IO_NORMAL;
2563  os_aio_use_native_aio = FALSE;
2564 
2565  } else if (0 == ut_strcmp(srv_file_flush_method_str, "unbuffered")) {
2566  srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED;
2567  os_aio_use_native_aio = FALSE;
2568 
2569  } else if (0 == ut_strcmp(srv_file_flush_method_str,
2570  "async_unbuffered")) {
2571  srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED;
2572 #endif
2573  } else {
2574  fprintf(stderr,
2575  "xtrabackup: Unrecognized value %s for innodb_flush_method\n",
2576  srv_file_flush_method_str);
2577  exit(EXIT_FAILURE);
2578  }
2579 
2580  if (srv_buf_pool_size >= 1000 * 1024 * 1024) {
2581  /* Here we still have srv_pool_size counted
2582  in kilobytes (in 4.0 this was in bytes)
2583  srv_boot() converts the value to
2584  pages; if buffer pool is less than 1000 MB,
2585  assume fewer threads. */
2586  srv_max_n_threads = 50000;
2587 
2588 
2589  } else if (srv_buf_pool_size >= 8 * 1024 * 1024) {
2590 
2591  srv_max_n_threads = 10000;
2592  } else {
2593  srv_max_n_threads = 1000; /* saves several MB of memory,
2594  especially in 64-bit
2595  computers */
2596  }
2597 
2598  {
2599  ulint nr;
2600  ulint i;
2601 
2602  nr = srv_n_data_files;
2603 
2604  for (i = 0; i < nr; i++) {
2605  srv_data_file_sizes[i] = srv_data_file_sizes[i]
2606  * ((1024 * 1024) / UNIV_PAGE_SIZE);
2607  }
2608 
2609  srv_last_file_size_max = srv_last_file_size_max
2610  * ((1024 * 1024) / UNIV_PAGE_SIZE);
2611 
2612  srv_log_file_size = srv_log_file_size / UNIV_PAGE_SIZE;
2613 
2614  srv_log_buffer_size = srv_log_buffer_size / UNIV_PAGE_SIZE;
2615 
2616  srv_lock_table_size = 5 * (srv_buf_pool_size / UNIV_PAGE_SIZE);
2617  }
2618 
2619  os_sync_mutex = NULL;
2620  srv_general_init();
2621 
2622  {
2623  ibool create_new_db;
2624 #ifdef XTRADB_BASED
2625  ibool create_new_doublewrite_file;
2626 #endif
2627  ibool log_file_created;
2628  ibool log_created = FALSE;
2629  ibool log_opened = FALSE;
2630  LSN64 min_flushed_lsn;
2631  LSN64 max_flushed_lsn;
2632  ulint sum_of_new_sizes;
2633  ulint err;
2634  ulint i;
2635 
2636 
2637 
2638 
2639 #define SRV_N_PENDING_IOS_PER_THREAD OS_AIO_N_PENDING_IOS_PER_THREAD
2640 #define SRV_MAX_N_PENDING_SYNC_IOS 100
2641 
2642  srv_n_file_io_threads = 2 + srv_n_read_io_threads + srv_n_write_io_threads;
2643 
2644  os_aio_init(8 * SRV_N_PENDING_IOS_PER_THREAD,
2645  srv_n_read_io_threads,
2646  srv_n_write_io_threads,
2647  SRV_MAX_N_PENDING_SYNC_IOS);
2648 
2649  fil_init(srv_file_per_table ? 50000 : 5000,
2650  srv_max_n_open_files);
2651 
2652  fsp_init();
2653  log_init();
2654 
2655  lock_sys_create(srv_lock_table_size);
2656 
2657  for (i = 0; i < srv_n_file_io_threads; i++) {
2658  thread_nr[i] = i;
2659 
2660  os_thread_create(io_handler_thread, thread_nr + i, thread_ids + i);
2661  }
2662 
2663  os_thread_sleep(200000); /*0.2 sec*/
2664 
2665  err = open_or_create_data_files(&create_new_db,
2666 #ifdef XTRADB_BASED
2667  &create_new_doublewrite_file,
2668 #endif
2669  &min_flushed_lsn, &max_flushed_lsn,
2670  &sum_of_new_sizes);
2671  if (err != DB_SUCCESS) {
2672  fprintf(stderr,
2673 "xtrabackup: Could not open or create data files.\n"
2674 "xtrabackup: If you tried to add new data files, and it failed here,\n"
2675 "xtrabackup: you should now edit innodb_data_file_path in my.cnf back\n"
2676 "xtrabackup: to what it was, and remove the new ibdata files InnoDB created\n"
2677 "xtrabackup: in this failed attempt. InnoDB only wrote those files full of\n"
2678 "xtrabackup: zeros, but did not yet use them in any way. But be careful: do not\n"
2679 "xtrabackup: remove old data files which contain your precious data!\n");
2680 
2681  //return((int) err);
2682  exit(EXIT_FAILURE);
2683  }
2684 
2685  /* create_new_db must not be TRUE.. */
2686  if (create_new_db) {
2687  fprintf(stderr, "xtrabackup: Something wrong with source files...\n");
2688  exit(EXIT_FAILURE);
2689  }
2690 
2691  for (i = 0; i < srv_n_log_files; i++) {
2692  err = open_or_create_log_file(create_new_db, &log_file_created,
2693  log_opened, 0, i);
2694  if (err != DB_SUCCESS) {
2695 
2696  //return((int) err);
2697  exit(EXIT_FAILURE);
2698  }
2699 
2700  if (log_file_created) {
2701  log_created = TRUE;
2702  } else {
2703  log_opened = TRUE;
2704  }
2705  if ((log_opened && create_new_db)
2706  || (log_opened && log_created)) {
2707  fprintf(stderr,
2708  "xtrabackup: Error: all log files must be created at the same time.\n"
2709  "xtrabackup: All log files must be created also in database creation.\n"
2710  "xtrabackup: If you want bigger or smaller log files, shut down the\n"
2711  "xtrabackup: database and make sure there were no errors in shutdown.\n"
2712  "xtrabackup: Then delete the existing log files. Edit the .cnf file\n"
2713  "xtrabackup: and start the database again.\n");
2714 
2715  //return(DB_ERROR);
2716  exit(EXIT_FAILURE);
2717  }
2718  }
2719 
2720  /* log_file_created must not be TRUE, if online */
2721  if (log_file_created) {
2722  fprintf(stderr, "xtrabackup: Something wrong with source files...\n");
2723  exit(EXIT_FAILURE);
2724  }
2725 
2726  fil_load_single_table_tablespaces();
2727 
2728  }
2729 
2730  /* create extra LSN dir if it does not exist. */
2731  if (xtrabackup_extra_lsndir
2732  && (stat(xtrabackup_extra_lsndir,&stat_info) != 0)
2733  && (mkdir(xtrabackup_extra_lsndir,0777) != 0)){
2734  fprintf(stderr,"xtrabackup: Error: cannot mkdir %d: %s\n",errno,xtrabackup_extra_lsndir);
2735  exit(EXIT_FAILURE);
2736  }
2737 
2738 
2739  if (!xtrabackup_stream) {
2740 
2741  /* create target dir if not exist */
2742  if (stat(xtrabackup_target_dir,&stat_info) != 0
2743  && (mkdir(xtrabackup_target_dir,0777) != 0)){
2744  fprintf(stderr,"xtrabackup: Error: cannot mkdir %d: %s\n",errno,xtrabackup_target_dir);
2745  exit(EXIT_FAILURE);
2746  }
2747 
2748  } else {
2749  fprintf(stderr,"xtrabackup: Stream mode.\n");
2750  /* stdout can treat binary at Linux */
2751  //setmode(fileno(stdout), O_BINARY);
2752  }
2753 
2754  {
2755  fil_system_t* f_system = fil_system;
2756 
2757  /* definition from recv_recovery_from_checkpoint_start() */
2758  log_group_t* max_cp_group;
2759  ulint max_cp_field;
2760  byte* buf;
2761  boost::scoped_array<byte> log_hdr_buf_(
2762  new byte[LOG_FILE_HDR_SIZE + OS_FILE_LOG_BLOCK_SIZE]);
2763  byte* log_hdr_buf;
2764  ulint err;
2765 
2766  ibool success;
2767 
2768  /* start back ground thread to copy newer log */
2769  os_thread_id_t log_copying_thread_id;
2770  datafiles_iter_t *it;
2771 
2772  log_hdr_buf = (byte*)ut_align(log_hdr_buf_.get(),
2774 
2775  /* log space */
2776  //space = UT_LIST_GET_NEXT(space_list, UT_LIST_GET_FIRST(f_system->space_list));
2777  //printf("space: name=%s, id=%d, purpose=%d, size=%d\n",
2778  // space->name, space->id, space->purpose, space->size);
2779 
2780  /* get current checkpoint_lsn */
2781  /* Look for the latest checkpoint from any of the log groups */
2782 
2783  err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field);
2784 
2785  if (err != DB_SUCCESS) {
2786 
2787  exit(EXIT_FAILURE);
2788  }
2789 
2790  log_group_read_checkpoint_info(max_cp_group, max_cp_field);
2791  buf = log_sys->checkpoint_buf;
2792 
2793  checkpoint_lsn_start = MACH_READ_64(buf + LOG_CHECKPOINT_LSN);
2794  checkpoint_no_start = MACH_READ_64(buf + LOG_CHECKPOINT_NO);
2795 
2796 reread_log_header:
2797  fil_io(OS_FILE_READ | OS_FILE_LOG, TRUE, max_cp_group->space_id,
2798  0,
2799  0, 0, LOG_FILE_HDR_SIZE,
2800  log_hdr_buf, max_cp_group);
2801 
2802  /* check consistency of log file header to copy */
2803  err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field);
2804 
2805  if (err != DB_SUCCESS) {
2806 
2807  exit(EXIT_FAILURE);
2808  }
2809 
2810  log_group_read_checkpoint_info(max_cp_group, max_cp_field);
2811  buf = log_sys->checkpoint_buf;
2812 
2813  if(ut_dulint_cmp(checkpoint_no_start,
2814  MACH_READ_64(buf + LOG_CHECKPOINT_NO)) != 0) {
2815  checkpoint_lsn_start = MACH_READ_64(buf + LOG_CHECKPOINT_LSN);
2816  checkpoint_no_start = MACH_READ_64(buf + LOG_CHECKPOINT_NO);
2817  goto reread_log_header;
2818  }
2819 
2820  if (!xtrabackup_stream) {
2821 
2822  /* open 'xtrabackup_logfile' */
2823  sprintf(dst_log_path, "%s%s", xtrabackup_target_dir, "/xtrabackup_logfile");
2824  srv_normalize_path_for_win(dst_log_path);
2825  /* os_file_create reads srv_unix_file_flush_method for OS_DATA_FILE*/
2826  dst_log = os_file_create(
2827  0 /* dummy of innodb_file_data_key */,
2828  dst_log_path, OS_FILE_CREATE,
2829  OS_FILE_NORMAL, OS_DATA_FILE, &success);
2830 
2831  if (!success) {
2832  /* The following call prints an error message */
2833  os_file_get_last_error(TRUE);
2834 
2835  fprintf(stderr,
2836 "xtrabackup: error: cannot open %s\n",
2837  dst_log_path);
2838  exit(EXIT_FAILURE);
2839  }
2840 
2841 #ifdef USE_POSIX_FADVISE
2842  posix_fadvise(dst_log, 0, 0, POSIX_FADV_DONTNEED);
2843 #endif
2844 
2845  }
2846 
2847  /* label it */
2848  strcpy((char*) log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP,
2849  "xtrabkup ");
2851  (char*) log_hdr_buf + (LOG_FILE_WAS_CREATED_BY_HOT_BACKUP
2852  + (sizeof "xtrabkup ") - 1));
2853 
2854  if (!xtrabackup_stream) {
2855  success = os_file_write(dst_log_path, dst_log, log_hdr_buf,
2856  0, 0, LOG_FILE_HDR_SIZE);
2857  } else {
2858  /* Stream */
2859  if ((write(fileno(stdout), log_hdr_buf, LOG_FILE_HDR_SIZE)
2860  - LOG_FILE_HDR_SIZE) == 0)
2861  {
2862  success = TRUE;
2863  } else {
2864  success = FALSE;
2865  }
2866  }
2867 
2868  log_copy_offset += LOG_FILE_HDR_SIZE;
2869  if (!success) {
2870  if (dst_log != -1)
2871  os_file_close(dst_log);
2872  exit(EXIT_FAILURE);
2873  }
2874 
2875  /* start flag */
2876  log_copying = TRUE;
2877 
2878  /* start io throttle */
2879  if(xtrabackup_throttle) {
2880  os_thread_id_t io_watching_thread_id;
2881 
2882  io_ticket = xtrabackup_throttle;
2883  wait_throttle = os_event_create(NULL);
2884 
2885  os_thread_create(io_watching_thread, NULL, &io_watching_thread_id);
2886  }
2887 
2888 
2889  /* copy log file by current position */
2890  if(xtrabackup_copy_logfile(checkpoint_lsn_start, FALSE))
2891  exit(EXIT_FAILURE);
2892 
2893 
2894  os_thread_create(log_copying_thread, NULL, &log_copying_thread_id);
2895 
2896 
2897 
2898  if (!xtrabackup_stream) { /* stream mode is transaction log only */
2899  uint i;
2900  uint count;
2901  os_mutex_t count_mutex;
2902  data_thread_ctxt_t *data_threads;
2903 
2904  ut_a(parallel > 0);
2905 
2906  if (parallel > 1)
2907  printf("xtrabackup: Starting %u threads for parallel "
2908  "data files transfer\n", parallel);
2909 
2910  it = datafiles_iter_new(f_system);
2911  if (it == NULL) {
2912  fprintf(stderr,
2913  "xtrabackup: Error: "
2914  "datafiles_iter_new() failed.\n");
2915  exit(EXIT_FAILURE);
2916  }
2917 
2918  /* Create data copying threads */
2919  ut_a(parallel > 0);
2920 
2921  data_threads = (data_thread_ctxt_t *)
2922  ut_malloc(sizeof(data_thread_ctxt_t) * parallel);
2923  count = parallel;
2924  count_mutex = OS_MUTEX_CREATE();
2925 
2926  for (i = 0; i < parallel; i++) {
2927  data_threads[i].it = it;
2928  data_threads[i].num = i+1;
2929  data_threads[i].count = &count;
2930  data_threads[i].count_mutex = count_mutex;
2931  os_thread_create(data_copy_thread_func,
2932  data_threads + i,
2933  &data_threads[i].id);
2934  }
2935 
2936  /* Wait for threads to exit */
2937  while (1) {
2938  os_thread_sleep(1000000);
2939  os_mutex_enter(count_mutex);
2940  if (count == 0) {
2941  os_mutex_exit(count_mutex);
2942  break;
2943  }
2944  os_mutex_exit(count_mutex);
2945  }
2946  /* NOTE: It may not needed at "--backup" for now */
2947  /* mutex_enter(&(f_system->mutex)); */
2948 
2949  os_mutex_free(count_mutex);
2950  datafiles_iter_free(it);
2951 
2952  } //if (!xtrabackup_stream)
2953 
2954  //mutex_exit(&(f_system->mutex));
2955  }
2956 
2957 
2958  /* suspend-at-end */
2959  if (xtrabackup_suspend_at_end) {
2960  os_file_t suspend_file = -1;
2961  char suspend_path[FN_REFLEN];
2962  ibool success, exists;
2963  os_file_type_t type;
2964 
2965  sprintf(suspend_path, "%s%s", xtrabackup_target_dir,
2966  "/xtrabackup_suspended");
2967 
2968  srv_normalize_path_for_win(suspend_path);
2969  /* os_file_create reads srv_unix_file_flush_method */
2970  suspend_file = os_file_create(
2971  0 /* dummy of innodb_file_data_key */,
2972  suspend_path, OS_FILE_OVERWRITE,
2973  OS_FILE_NORMAL, OS_DATA_FILE, &success);
2974 
2975  if (!success) {
2976  fprintf(stderr, "xtrabackup: Error: failed to create file 'xtrabackup_suspended'\n");
2977  }
2978 
2979  if (suspend_file != -1)
2980  os_file_close(suspend_file);
2981 
2982  exists = TRUE;
2983  while (exists) {
2984  os_thread_sleep(200000); /*0.2 sec*/
2985  success = os_file_status(suspend_path, &exists, &type);
2986  if (!success)
2987  break;
2988  }
2989  xtrabackup_suspend_at_end = FALSE; /* suspend is 1 time */
2990  }
2991 
2992  /* read the latest checkpoint lsn */
2993  latest_cp = ut_dulint_zero;
2994  {
2995  log_group_t* max_cp_group;
2996  ulint max_cp_field;
2997  ulint err;
2998 
2999  err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field);
3000 
3001  if (err != DB_SUCCESS) {
3002  fprintf(stderr, "xtrabackup: Error: recv_find_max_checkpoint() failed.\n");
3003  goto skip_last_cp;
3004  }
3005 
3006  log_group_read_checkpoint_info(max_cp_group, max_cp_field);
3007 
3008  latest_cp = MACH_READ_64(log_sys->checkpoint_buf + LOG_CHECKPOINT_LSN);
3009 
3010  if (!xtrabackup_stream) {
3011  printf("xtrabackup: The latest check point (for incremental): '%"PRIu64"'\n",
3012  latest_cp);
3013  } else {
3014  fprintf(stderr, "xtrabackup: The latest check point (for incremental): '%"PRIu64"'\n",
3015  latest_cp);
3016  }
3017  }
3018 skip_last_cp:
3019  /* stop log_copying_thread */
3020  log_copying = FALSE;
3021  if (!xtrabackup_stream) {
3022  printf("xtrabackup: Stopping log copying thread");
3023  while (log_copying_running) {
3024  printf(".");
3025  os_thread_sleep(200000); /*0.2 sec*/
3026  }
3027  printf("\n");
3028  } else {
3029  while (log_copying_running)
3030  os_thread_sleep(200000); /*0.2 sec*/
3031  }
3032 
3033  /* output to metadata file */
3034  {
3035  char filename[FN_REFLEN];
3036 
3037  if(!xtrabackup_incremental) {
3038  strcpy(metadata_type, "full-backuped");
3039  metadata_from_lsn = ut_dulint_zero;
3040  } else {
3041  strcpy(metadata_type, "incremental");
3042  metadata_from_lsn = incremental_lsn;
3043  }
3044  metadata_to_lsn = latest_cp;
3045  metadata_last_lsn = log_copy_scanned_lsn;
3046 
3047  sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME);
3048  if (xtrabackup_write_metadata(filename))
3049  fprintf(stderr, "xtrabackup: error: xtrabackup_write_metadata(xtrabackup_target_dir)\n");
3050 
3051  if(xtrabackup_extra_lsndir) {
3052  sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME);
3053  if (xtrabackup_write_metadata(filename))
3054  fprintf(stderr, "xtrabackup: error: xtrabackup_write_metadata(xtrabackup_extra_lsndir)\n");
3055  }
3056  }
3057 
3058  if (!log_copying_succeed) {
3059  fprintf(stderr, "xtrabackup: Error: log_copying_thread failed.\n");
3060  exit(EXIT_FAILURE);
3061  }
3062 
3063  if (!xtrabackup_stream)
3064  os_file_close(dst_log);
3065 
3066  if (wait_throttle)
3067  os_event_free(wait_throttle);
3068 
3069  if (!xtrabackup_stream) {
3070  printf("xtrabackup: Transaction log of lsn (%"PRIu64") to (%"PRIu64") was copied.\n",
3071  checkpoint_lsn_start, log_copy_scanned_lsn);
3072  } else {
3073  fprintf(stderr, "xtrabackup: Transaction log of lsn (%"PRIu64") to (%"PRIu64") was copied.\n",
3074  checkpoint_lsn_start, log_copy_scanned_lsn);
3075  if(xtrabackup_extra_lsndir) {
3076  char filename[FN_REFLEN];
3077  sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME);
3078  if (xtrabackup_write_metadata(filename))
3079  fprintf(stderr, "xtrabackup: error: xtrabackup_write_metadata(xtrabackup_extra_lsndir)\n");
3080  }
3081  }
3082 }
3083 
3084 /* ================= stats ================= */
3085 static bool
3086 xtrabackup_stats_level(
3087  dict_index_t* index,
3088  ulint level)
3089 {
3090  ulint space;
3091  page_t* page;
3092 
3093  rec_t* node_ptr;
3094 
3095  ulint right_page_no;
3096 
3097  page_cur_t cursor;
3098 
3099  mtr_t mtr;
3100  mem_heap_t* heap = mem_heap_create(256);
3101 
3102  ulint* offsets = NULL;
3103 
3104  uint64_t n_pages, n_pages_extern;
3105  uint64_t sum_data, sum_data_extern;
3106  uint64_t n_recs;
3107  ulint page_size;
3108 
3109  n_pages = sum_data = n_recs = 0;
3110  n_pages_extern = sum_data_extern = 0;
3111 
3112  buf_block_t* block;
3113  ulint zip_size;
3114 
3115  if (level == 0)
3116  fprintf(stdout, " leaf pages: ");
3117  else
3118  fprintf(stdout, " level %lu pages: ", level);
3119 
3120  mtr_start(&mtr);
3121 
3122  mtr_x_lock(&(index->lock), &mtr);
3123  block = btr_root_block_get(index, &mtr);
3124  page = buf_block_get_frame(block);
3125 
3126  space = page_get_space_id(page);
3127  zip_size = fil_space_get_zip_size(space);
3128 
3129  while (level != btr_page_get_level(page, &mtr)) {
3130 
3131  ut_a(space == buf_block_get_space(block));
3132  ut_a(space == page_get_space_id(page));
3133  ut_a(!page_is_leaf(page));
3134 
3135  page_cur_set_before_first(block, &cursor);
3136  page_cur_move_to_next(&cursor);
3137 
3138  node_ptr = page_cur_get_rec(&cursor);
3139  offsets = rec_get_offsets(node_ptr, index, offsets,
3140  ULINT_UNDEFINED, &heap);
3141 
3142  block = btr_node_ptr_get_child(node_ptr, index, offsets, &mtr);
3143  page = buf_block_get_frame(block);
3144  }
3145 
3146 loop:
3147  mem_heap_empty(heap);
3148  offsets = NULL;
3149  mtr_x_lock(&(index->lock), &mtr);
3150 
3151  right_page_no = btr_page_get_next(page, &mtr);
3152 
3153 
3154  /*=================================*/
3155  //fprintf(stdout, "%lu ", (ulint) buf_frame_get_page_no(page));
3156 
3157  n_pages++;
3158  sum_data += page_get_data_size(page);
3159  n_recs += page_get_n_recs(page);
3160 
3161 
3162  if (level == 0) {
3163  page_cur_t cur;
3164  ulint n_fields;
3165  ulint i;
3166  mem_heap_t* local_heap = NULL;
3167  ulint offsets_[REC_OFFS_NORMAL_SIZE];
3168  ulint* local_offsets = offsets_;
3169 
3170  *offsets_ = (sizeof offsets_) / sizeof *offsets_;
3171 
3172  page_cur_set_before_first(block, &cur);
3173 
3174  page_cur_move_to_next(&cur);
3175 
3176  for (;;) {
3177  if (page_cur_is_after_last(&cur)) {
3178  break;
3179  }
3180 
3181  local_offsets = rec_get_offsets(cur.rec, index, local_offsets,
3182  ULINT_UNDEFINED, &local_heap);
3183  n_fields = rec_offs_n_fields(local_offsets);
3184 
3185  for (i = 0; i < n_fields; i++) {
3186  if (rec_offs_nth_extern(local_offsets, i)) {
3187  page_t* local_page;
3188  ulint space_id;
3189  ulint page_no;
3190  ulint offset;
3191  byte* blob_header;
3192  ulint part_len;
3193  mtr_t local_mtr;
3194  ulint local_len;
3195  byte* data;
3196  buf_block_t* local_block;
3197 
3198  data = rec_get_nth_field(cur.rec, local_offsets, i, &local_len);
3199 
3200  ut_a(local_len >= BTR_EXTERN_FIELD_REF_SIZE);
3201  local_len -= BTR_EXTERN_FIELD_REF_SIZE;
3202 
3203  space_id = mach_read_from_4(data + local_len + BTR_EXTERN_SPACE_ID);
3204  page_no = mach_read_from_4(data + local_len + BTR_EXTERN_PAGE_NO);
3205  offset = mach_read_from_4(data + local_len + BTR_EXTERN_OFFSET);
3206 
3207  if (offset != FIL_PAGE_DATA)
3208  fprintf(stderr, "\nWarning: several record may share same external page.\n");
3209 
3210  for (;;) {
3211  mtr_start(&local_mtr);
3212 
3213  local_block = btr_block_get(space_id, zip_size, page_no, RW_S_LATCH, &local_mtr);
3214  local_page = buf_block_get_frame(local_block);
3215 
3216  blob_header = local_page + offset;
3217 #define BTR_BLOB_HDR_PART_LEN 0
3218 #define BTR_BLOB_HDR_NEXT_PAGE_NO 4
3219  //part_len = btr_blob_get_part_len(blob_header);
3220  part_len = mach_read_from_4(blob_header + BTR_BLOB_HDR_PART_LEN);
3221 
3222  //page_no = btr_blob_get_next_page_no(blob_header);
3223  page_no = mach_read_from_4(blob_header + BTR_BLOB_HDR_NEXT_PAGE_NO);
3224 
3225  offset = FIL_PAGE_DATA;
3226 
3227 
3228 
3229 
3230  /*=================================*/
3231  //fprintf(stdout, "[%lu] ", (ulint) buf_frame_get_page_no(page));
3232 
3233  n_pages_extern++;
3234  sum_data_extern += part_len;
3235 
3236 
3237  mtr_commit(&local_mtr);
3238 
3239  if (page_no == FIL_NULL)
3240  break;
3241  }
3242  }
3243  }
3244 
3245  page_cur_move_to_next(&cur);
3246  }
3247  }
3248 
3249 
3250 
3251 
3252  mtr_commit(&mtr);
3253  if (right_page_no != FIL_NULL) {
3254  mtr_start(&mtr);
3255  block = btr_block_get(space, zip_size, right_page_no, RW_X_LATCH, &mtr);
3256  page = buf_block_get_frame(block);
3257  goto loop;
3258  }
3259  mem_heap_free(heap);
3260 
3261  if (zip_size) {
3262  page_size = zip_size;
3263  } else {
3264  page_size = UNIV_PAGE_SIZE;
3265  }
3266 
3267  if (level == 0)
3268  fprintf(stdout, "recs=%"PRIu64", ", n_recs);
3269 
3270  fprintf(stdout, "pages=%"PRIu64", data=%"PRIu64" bytes, data/pages=%"PRIu64"%%",
3271  n_pages, sum_data,
3272  ((sum_data * 100)/ page_size)/n_pages);
3273 
3274 
3275  if (level == 0 && n_pages_extern) {
3276  putc('\n', stdout);
3277  /* also scan blob pages*/
3278  fprintf(stdout, " external pages: ");
3279 
3280  fprintf(stdout, "pages=%"PRIu64", data=%"PRIu64" bytes, data/pages=%"PRIu64"%%",
3281  n_pages_extern, sum_data_extern,
3282  ((sum_data_extern * 100)/ page_size)/n_pages_extern);
3283  }
3284 
3285  putc('\n', stdout);
3286 
3287  if (level > 0) {
3288  xtrabackup_stats_level(index, level - 1);
3289  }
3290 
3291  return(TRUE);
3292 }
3293 
3294 static void
3295 xtrabackup_stats_func(void)
3296 {
3297  ulint n;
3298 
3299  /* cd to datadir */
3300 
3301  if (chdir(mysql_real_data_home) != 0)
3302  {
3303  fprintf(stderr, "xtrabackup: cannot my_setwd %s\n", mysql_real_data_home);
3304  exit(EXIT_FAILURE);
3305  }
3306  fprintf(stderr, "xtrabackup: cd to %s\n", mysql_real_data_home);
3307 
3308  mysql_data_home= mysql_data_home_buff;
3309  mysql_data_home[0]=FN_CURLIB; // all paths are relative from here
3310  mysql_data_home[1]=0;
3311 
3312  /* set read only */
3313  srv_read_only = TRUE;
3314  srv_fake_write = TRUE;
3315 
3316  /* initialize components */
3317  if(innodb_init_param())
3318  exit(EXIT_FAILURE);
3319 
3320  /* Check if the log files have been created, otherwise innodb_init()
3321  will crash when called with srv_read_only == TRUE */
3322  for (n = 0; n < srv_n_log_files; n++) {
3323  char logname[FN_REFLEN];
3324  ibool exists;
3325  os_file_type_t type;
3326 
3327  sprintf(logname, "ib_logfile%lu", (ulong) n);
3328  if (!os_file_status(logname, &exists, &type) || !exists ||
3329  type != OS_FILE_TYPE_FILE) {
3330  fprintf(stderr, "xtrabackup: Error: "
3331  "Cannot find log file %s.\n", logname);
3332  fprintf(stderr, "xtrabackup: Error: "
3333  "to use the statistics feature, you need a "
3334  "clean copy of the database including "
3335  "correctly sized log files, so you need to "
3336  "execute with --prepare twice to use this "
3337  "functionality on a backup.\n");
3338  exit(EXIT_FAILURE);
3339  }
3340  }
3341 
3342  fprintf(stderr, "xtrabackup: Starting 'read-only' InnoDB instance to gather index statistics.\n"
3343  "xtrabackup: Using %"PRIu64" bytes for buffer pool (set by --use-memory parameter)\n",
3344  xtrabackup_use_memory);
3345 
3346  if(innodb_init())
3347  exit(EXIT_FAILURE);
3348 
3349  fprintf(stdout, "\n\n<INDEX STATISTICS>\n");
3350 
3351  /* gather stats */
3352 
3353  {
3354  dict_table_t* sys_tables;
3355  dict_index_t* sys_index;
3356  dict_table_t* table;
3357  btr_pcur_t pcur;
3358  rec_t* rec;
3359  byte* field;
3360  ulint len;
3361  mtr_t mtr;
3362 
3363  /* Enlarge the fatal semaphore wait timeout during the InnoDB table
3364  monitor printout */
3365 
3366  mutex_enter(&kernel_mutex);
3367  srv_fatal_semaphore_wait_threshold += 72000; /* 20 hours */
3368  mutex_exit(&kernel_mutex);
3369 
3370  mutex_enter(&(dict_sys->mutex));
3371 
3372  mtr_start(&mtr);
3373 
3374  sys_tables = dict_table_get_low("SYS_TABLES");
3375  sys_index = UT_LIST_GET_FIRST(sys_tables->indexes);
3376 
3377  btr_pcur_open_at_index_side(TRUE, sys_index, BTR_SEARCH_LEAF, &pcur,
3378  TRUE, &mtr);
3379 loop:
3380  btr_pcur_move_to_next_user_rec(&pcur, &mtr);
3381 
3382  rec = btr_pcur_get_rec(&pcur);
3383 
3384  if (!btr_pcur_is_on_user_rec(&pcur))
3385  {
3386  /* end of index */
3387 
3388  btr_pcur_close(&pcur);
3389  mtr_commit(&mtr);
3390 
3391  mutex_exit(&(dict_sys->mutex));
3392 
3393  /* Restore the fatal semaphore wait timeout */
3394 
3395  mutex_enter(&kernel_mutex);
3396  srv_fatal_semaphore_wait_threshold -= 72000; /* 20 hours */
3397  mutex_exit(&kernel_mutex);
3398 
3399  goto end;
3400  }
3401 
3402  field = rec_get_nth_field_old(rec, 0, &len);
3403 
3404  if (!rec_get_deleted_flag(rec, 0))
3405  {
3406 
3407  /* We found one */
3408 
3409  char* table_name = mem_strdupl((char*) field, len);
3410 
3411  btr_pcur_store_position(&pcur, &mtr);
3412 
3413  mtr_commit(&mtr);
3414 
3415  table = dict_table_get_low(table_name);
3416  mem_free(table_name);
3417 
3418 
3419  if (xtrabackup_tables) {
3420  char *p;
3421  int regres= 0;
3422  int i;
3423 
3424  p = strstr(table->name, SRV_PATH_SEPARATOR_STR);
3425 
3426  if (p)
3427  *p = '.';
3428 
3429  for (i = 0; i < tables_regex_num; i++) {
3430  regres = regexec(&tables_regex[i], table->name, 1, tables_regmatch, 0);
3431  if (regres != REG_NOMATCH)
3432  break;
3433  }
3434 
3435  if (p)
3436  *p = SRV_PATH_SEPARATOR;
3437 
3438  if ( regres == REG_NOMATCH )
3439  goto skip;
3440  }
3441 
3442  if (xtrabackup_tables_file) {
3443  xtrabackup_tables_t* xtable;
3444 
3445  HASH_SEARCH(name_hash, tables_hash, ut_fold_string(table->name),
3447  xtable,
3448  ut_ad(xtable->name),
3449  !strcmp(xtable->name, table->name));
3450 
3451  if (!xtable)
3452  goto skip;
3453  }
3454 
3455 
3456  if (table == NULL) {
3457  fputs("InnoDB: Failed to load table ", stderr);
3458  ut_print_namel(stderr, NULL, TRUE, (char*) field, len);
3459  putc('\n', stderr);
3460  } else {
3461  dict_index_t* index;
3462 
3463  /* The table definition was corrupt if there
3464  is no index */
3465 
3466  if (dict_table_get_first_index(table)) {
3467 #ifdef XTRADB_BASED
3468  dict_update_statistics(table, TRUE, FALSE);
3469 #elif defined(INNODB_VERSION_SHORT)
3470  dict_update_statistics(table, TRUE);
3471 #else
3472  dict_update_statistics_low(table, TRUE);
3473 #endif
3474  }
3475 
3476  //dict_table_print_low(table);
3477 
3478  index = UT_LIST_GET_FIRST(table->indexes);
3479  while (index != NULL) {
3480 {
3481  IB_INT64 n_vals;
3482 
3483  if (index->n_user_defined_cols > 0) {
3484  n_vals = index->stat_n_diff_key_vals[
3485  index->n_user_defined_cols];
3486  } else {
3487  n_vals = index->stat_n_diff_key_vals[1];
3488  }
3489 
3490  fprintf(stdout,
3491  " table: %s, index: %s, space id: %lu, root page: %lu"
3492  ", zip size: %lu"
3493  "\n estimated statistics in dictionary:\n"
3494  " key vals: %lu, leaf pages: %lu, size pages: %lu\n"
3495  " real statistics:\n",
3496  table->name, index->name,
3497  (ulong) index->space,
3498  (ulong) index->page,
3499  (ulong) fil_space_get_zip_size(index->space),
3500  (ulong) n_vals,
3501  (ulong) index->stat_n_leaf_pages,
3502  (ulong) index->stat_index_size);
3503 
3504  {
3505  mtr_t local_mtr;
3506  page_t* root;
3507  ulint page_level;
3508 
3509  mtr_start(&local_mtr);
3510 
3511  mtr_x_lock(&(index->lock), &local_mtr);
3512  root = btr_root_get(index, &local_mtr);
3513 
3514  page_level = btr_page_get_level(root, &local_mtr);
3515 
3516  xtrabackup_stats_level(index, page_level);
3517 
3518  mtr_commit(&local_mtr);
3519  }
3520 
3521  putc('\n', stdout);
3522 }
3523  index = UT_LIST_GET_NEXT(indexes, index);
3524  }
3525  }
3526 
3527 skip:
3528  mtr_start(&mtr);
3529 
3530  btr_pcur_restore_position(BTR_SEARCH_LEAF, &pcur, &mtr);
3531  }
3532 
3533  goto loop;
3534  }
3535 
3536 end:
3537  putc('\n', stdout);
3538 
3539  /* shutdown InnoDB */
3540  if(innodb_end())
3541  exit(EXIT_FAILURE);
3542 }
3543 
3544 /* ================= prepare ================= */
3545 
3546 static bool
3547 xtrabackup_init_temp_log(void)
3548 {
3549  os_file_t src_file = -1;
3550  char src_path[FN_REFLEN];
3551  char dst_path[FN_REFLEN];
3552  ibool success;
3553 
3554  ulint field;
3555  byte* log_buf;
3556  byte* log_buf_ = NULL;
3557 
3558  IB_INT64 file_size;
3559 
3560  LSN64 max_no;
3561  LSN64 max_lsn= 0;
3562  LSN64 checkpoint_no;
3563 
3564  ulint fold;
3565 
3566  max_no = ut_dulint_zero;
3567 
3568  if(!xtrabackup_incremental_dir) {
3569  sprintf(dst_path, "%s%s", xtrabackup_target_dir, "/ib_logfile0");
3570  sprintf(src_path, "%s%s", xtrabackup_target_dir, "/xtrabackup_logfile");
3571  } else {
3572  sprintf(dst_path, "%s%s", xtrabackup_incremental_dir, "/ib_logfile0");
3573  sprintf(src_path, "%s%s", xtrabackup_incremental_dir, "/xtrabackup_logfile");
3574  }
3575 
3576  srv_normalize_path_for_win(dst_path);
3577  srv_normalize_path_for_win(src_path);
3578 retry:
3579  src_file = os_file_create_simple_no_error_handling(
3580  0 /* dummy of innodb_file_data_key */,
3581  src_path, OS_FILE_OPEN,
3582  OS_FILE_READ_WRITE /* OS_FILE_READ_ONLY */, &success);
3583  if (!success) {
3584  /* The following call prints an error message */
3585  os_file_get_last_error(TRUE);
3586 
3587  fprintf(stderr,
3588 "xtrabackup: Warning: cannot open %s. will try to find.\n",
3589  src_path);
3590 
3591  /* check if ib_logfile0 may be xtrabackup_logfile */
3592  src_file = os_file_create_simple_no_error_handling(
3593  0 /* dummy of innodb_file_data_key */,
3594  dst_path, OS_FILE_OPEN,
3595  OS_FILE_READ_WRITE /* OS_FILE_READ_ONLY */, &success);
3596  if (!success) {
3597  os_file_get_last_error(TRUE);
3598  fprintf(stderr,
3599 " xtrabackup: Fatal error: cannot find %s.\n",
3600  src_path);
3601 
3602  goto error;
3603  }
3604 
3605  log_buf_ = (unsigned char*) ut_malloc(LOG_FILE_HDR_SIZE * 2);
3606  log_buf = (unsigned char*) ut_align(log_buf_, LOG_FILE_HDR_SIZE);
3607 
3608  success = os_file_read(src_file, log_buf, 0, 0, LOG_FILE_HDR_SIZE);
3609  if (!success) {
3610  goto error;
3611  }
3612 
3613  if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP,
3614  (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) == 0) {
3615  fprintf(stderr,
3616 " xtrabackup: 'ib_logfile0' seems to be 'xtrabackup_logfile'. will retry.\n");
3617 
3618  ut_free(log_buf_);
3619  log_buf_ = NULL;
3620 
3621  os_file_close(src_file);
3622  src_file = -1;
3623 
3624  /* rename and try again */
3625  success = os_file_rename(
3626  0 /* dummy of innodb_file_data_key */,
3627  dst_path, src_path);
3628  if (!success) {
3629  goto error;
3630  }
3631 
3632  goto retry;
3633  }
3634 
3635  fprintf(stderr,
3636 " xtrabackup: Fatal error: cannot find %s.\n",
3637  src_path);
3638 
3639  ut_free(log_buf_);
3640  log_buf_ = NULL;
3641 
3642  os_file_close(src_file);
3643  src_file = -1;
3644 
3645  goto error;
3646  }
3647 
3648 #ifdef USE_POSIX_FADVISE
3649  posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL);
3650  posix_fadvise(src_file, 0, 0, POSIX_FADV_DONTNEED);
3651 #endif
3652 
3653  if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT) {
3654  os_file_set_nocache(src_file, src_path, "OPEN");
3655  }
3656 
3657  file_size = os_file_get_size_as_iblonglong(src_file);
3658 
3659 
3660  /* TODO: We should skip the following modifies, if it is not the first time. */
3661  log_buf_ = (unsigned char*) ut_malloc(UNIV_PAGE_SIZE * 129);
3662  log_buf = (unsigned char*) ut_align(log_buf_, UNIV_PAGE_SIZE);
3663 
3664  /* read log file header */
3665  success = os_file_read(src_file, log_buf, 0, 0, LOG_FILE_HDR_SIZE);
3666  if (!success) {
3667  goto error;
3668  }
3669 
3670  if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP,
3671  (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) != 0 ) {
3672  printf("xtrabackup: notice: xtrabackup_logfile was already used to '--prepare'.\n");
3673  goto skip_modify;
3674  } else {
3675  /* clear it later */
3676  //memset(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP,
3677  // ' ', 4);
3678  }
3679 
3680  /* read last checkpoint lsn */
3681  for (field = LOG_CHECKPOINT_1; field <= LOG_CHECKPOINT_2;
3682  field += LOG_CHECKPOINT_2 - LOG_CHECKPOINT_1) {
3683  if (!recv_check_cp_is_consistent(log_buf + field))
3684  goto not_consistent;
3685 
3686  checkpoint_no = MACH_READ_64(log_buf + field + LOG_CHECKPOINT_NO);
3687 
3688  if (ut_dulint_cmp(checkpoint_no, max_no) >= 0) {
3689  max_no = checkpoint_no;
3690  max_lsn = MACH_READ_64(log_buf + field + LOG_CHECKPOINT_LSN);
3691 /*
3692  mach_write_to_4(log_buf + field + LOG_CHECKPOINT_OFFSET,
3693  LOG_FILE_HDR_SIZE + ut_dulint_minus(max_lsn,
3694  ut_dulint_align_down(max_lsn,OS_FILE_LOG_BLOCK_SIZE)));
3695 
3696  ulint fold;
3697  fold = ut_fold_binary(log_buf + field, LOG_CHECKPOINT_CHECKSUM_1);
3698  mach_write_to_4(log_buf + field + LOG_CHECKPOINT_CHECKSUM_1, fold);
3699 
3700  fold = ut_fold_binary(log_buf + field + LOG_CHECKPOINT_LSN,
3701  LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN);
3702  mach_write_to_4(log_buf + field + LOG_CHECKPOINT_CHECKSUM_2, fold);
3703 */
3704  }
3705 not_consistent:
3706  ;
3707  }
3708 
3709  if (ut_dulint_cmp(max_no, ut_dulint_zero) == 0) {
3710  fprintf(stderr, "xtrabackup: No valid checkpoint found.\n");
3711  goto error;
3712  }
3713 
3714 
3715  /* It seems to be needed to overwrite the both checkpoint area. */
3716  MACH_WRITE_64(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN, max_lsn);
3717  mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_OFFSET,
3718  LOG_FILE_HDR_SIZE + ut_dulint_minus(max_lsn,
3719  ut_dulint_align_down(max_lsn,OS_FILE_LOG_BLOCK_SIZE)));
3720 #ifdef XTRADB_BASED
3721  MACH_WRITE_64(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_ARCHIVED_LSN,
3722  (ib_uint64_t)(LOG_FILE_HDR_SIZE + ut_dulint_minus(max_lsn,
3723  ut_dulint_align_down(max_lsn,OS_FILE_LOG_BLOCK_SIZE))));
3724 #endif
3725  fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1, LOG_CHECKPOINT_CHECKSUM_1);
3726  mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_1, fold);
3727 
3728  fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN,
3729  LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN);
3730  mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_2, fold);
3731 
3732  MACH_WRITE_64(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN, max_lsn);
3733  mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_OFFSET,
3734  LOG_FILE_HDR_SIZE + ut_dulint_minus(max_lsn,
3735  ut_dulint_align_down(max_lsn,OS_FILE_LOG_BLOCK_SIZE)));
3736 #ifdef XTRADB_BASED
3737  MACH_WRITE_64(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_ARCHIVED_LSN,
3738  (ib_uint64_t)(LOG_FILE_HDR_SIZE + ut_dulint_minus(max_lsn,
3739  ut_dulint_align_down(max_lsn,OS_FILE_LOG_BLOCK_SIZE))));
3740 #endif
3741  fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2, LOG_CHECKPOINT_CHECKSUM_1);
3742  mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_1, fold);
3743 
3744  fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN,
3745  LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN);
3746  mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_2, fold);
3747 
3748 
3749  success = os_file_write(src_path, src_file, log_buf, 0, 0, LOG_FILE_HDR_SIZE);
3750  if (!success) {
3751  goto error;
3752  }
3753 
3754  /* expand file size (9/8) and align to UNIV_PAGE_SIZE */
3755 
3756  if (file_size % UNIV_PAGE_SIZE) {
3757  memset(log_buf, 0, UNIV_PAGE_SIZE);
3758  success = os_file_write(src_path, src_file, log_buf,
3759  (ulint)(file_size & 0xFFFFFFFFUL),
3760  (ulint)(file_size >> 32),
3761  UNIV_PAGE_SIZE - (file_size % UNIV_PAGE_SIZE));
3762  if (!success) {
3763  goto error;
3764  }
3765 
3766  file_size = os_file_get_size_as_iblonglong(src_file);
3767  }
3768 
3769  /* TODO: We should judge whether the file is already expanded or not... */
3770  {
3771  ulint expand;
3772 
3773  memset(log_buf, 0, UNIV_PAGE_SIZE * 128);
3774  expand = file_size / UNIV_PAGE_SIZE / 8;
3775 
3776  for (; expand > 128; expand -= 128) {
3777  success = os_file_write(src_path, src_file, log_buf,
3778  (ulint)(file_size & 0xFFFFFFFFUL),
3779  (ulint)(file_size >> 32),
3780  UNIV_PAGE_SIZE * 128);
3781  if (!success) {
3782  goto error;
3783  }
3784  file_size += UNIV_PAGE_SIZE * 128;
3785  }
3786 
3787  if (expand) {
3788  success = os_file_write(src_path, src_file, log_buf,
3789  (ulint)(file_size & 0xFFFFFFFFUL),
3790  (ulint)(file_size >> 32),
3791  expand * UNIV_PAGE_SIZE);
3792  if (!success) {
3793  goto error;
3794  }
3795  file_size += UNIV_PAGE_SIZE * expand;
3796  }
3797  }
3798 
3799  /* make larger than 2MB */
3800  if (file_size < 2*1024*1024L) {
3801  memset(log_buf, 0, UNIV_PAGE_SIZE);
3802  while (file_size < 2*1024*1024L) {
3803  success = os_file_write(src_path, src_file, log_buf,
3804  (ulint)(file_size & 0xFFFFFFFFUL),
3805  (ulint)(file_size >> 32),
3806  UNIV_PAGE_SIZE);
3807  if (!success) {
3808  goto error;
3809  }
3810  file_size += UNIV_PAGE_SIZE;
3811  }
3812  file_size = os_file_get_size_as_iblonglong(src_file);
3813  }
3814 
3815  printf("xtrabackup: xtrabackup_logfile detected: size=%"PRIu64", start_lsn=(%"PRIu64")\n",
3816  file_size, max_lsn);
3817 
3818  os_file_close(src_file);
3819  src_file = -1;
3820 
3821  /* Backup log parameters */
3822  innobase_log_group_home_dir_backup = innobase_log_group_home_dir;
3823  innobase_log_file_size_backup = innobase_log_file_size;
3824  innobase_log_files_in_group_backup = innobase_log_files_in_group;
3825 
3826  /* fake InnoDB */
3827  innobase_log_group_home_dir = NULL;
3828  innobase_log_file_size = file_size;
3829  innobase_log_files_in_group = 1;
3830 
3831  srv_thread_concurrency = 0;
3832 
3833  /* rename 'xtrabackup_logfile' to 'ib_logfile0' */
3834  success = os_file_rename(
3835  0 /* dummy of innodb_file_data_key */,
3836  src_path, dst_path);
3837  if (!success) {
3838  goto error;
3839  }
3840  xtrabackup_logfile_is_renamed = TRUE;
3841 
3842  ut_free(log_buf_);
3843 
3844  return(FALSE);
3845 
3846 skip_modify:
3847  os_file_close(src_file);
3848  src_file = -1;
3849  ut_free(log_buf_);
3850  return(FALSE);
3851 
3852 error:
3853  if (src_file != -1)
3854  os_file_close(src_file);
3855  if (log_buf_)
3856  ut_free(log_buf_);
3857  fprintf(stderr, "xtrabackup: Error: xtrabackup_init_temp_log() failed.\n");
3858  return(TRUE); /*ERROR*/
3859 }
3860 
3861 /***********************************************************************
3862 Generates path to the meta file path from a given path to an incremental .delta
3863 by replacing trailing ".delta" with ".meta", or returns error if 'delta_path'
3864 does not end with the ".delta" character sequence.
3865 @return TRUE on success, FALSE on error. */
3866 static
3867 ibool
3868 get_meta_path(
3869  const char *delta_path, /* in: path to a .delta file */
3870  char *meta_path) /* out: path to the corresponding .meta
3871  file */
3872 {
3873  size_t len = strlen(delta_path);
3874 
3875  if (len <= 6 || strcmp(delta_path + len - 6, ".delta")) {
3876  return FALSE;
3877  }
3878  memcpy(meta_path, delta_path, len - 6);
3879  strcpy(meta_path + len - 6, XB_DELTA_INFO_SUFFIX);
3880 
3881  return TRUE;
3882 }
3883 
3884 static void
3885 xtrabackup_apply_delta(
3886  const char* dirname, /* in: dir name of incremental */
3887  const char* dbname, /* in: database name (ibdata: NULL) */
3888  const char* filename, /* in: file name (not a path),
3889  including the .delta extension */
3890  bool )
3891 {
3892  os_file_t src_file = -1;
3893  os_file_t dst_file = -1;
3894  char src_path[FN_REFLEN];
3895  char dst_path[FN_REFLEN];
3896  char meta_path[FN_REFLEN];
3897  ibool success;
3898 
3899  ibool last_buffer = FALSE;
3900  ulint page_in_buffer;
3901  ulint incremental_buffers = 0;
3902 
3903  xb_delta_info_t info;
3904  ulint page_size;
3905  ulint page_size_shift;
3906 
3907  ut_a(xtrabackup_incremental);
3908 
3909  if (dbname) {
3910  snprintf(src_path, sizeof(src_path), "%s/%s/%s",
3911  dirname, dbname, filename);
3912  snprintf(dst_path, sizeof(dst_path), "%s/%s/%s",
3913  xtrabackup_real_target_dir, dbname, filename);
3914  } else {
3915  snprintf(src_path, sizeof(src_path), "%s/%s",
3916  dirname, filename);
3917  snprintf(dst_path, sizeof(dst_path), "%s/%s",
3918  xtrabackup_real_target_dir, filename);
3919  }
3920  dst_path[strlen(dst_path) - 6] = '\0';
3921 
3922  if (!get_meta_path(src_path, meta_path)) {
3923  goto error;
3924  }
3925 
3926  srv_normalize_path_for_win(dst_path);
3927  srv_normalize_path_for_win(src_path);
3928  srv_normalize_path_for_win(meta_path);
3929 
3930  if (!xb_read_delta_metadata(meta_path, &info)) {
3931  goto error;
3932  }
3933 
3934  page_size = info.page_size;
3935  page_size_shift = get_bit_shift(page_size);
3936  fprintf(stderr, "xtrabackup: page size for %s is %lu bytes\n",
3937  src_path, page_size);
3938  if (page_size_shift < 10 ||
3939  page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) {
3940  fprintf(stderr,
3941  "xtrabackup: error: invalid value of page_size "
3942  "(%lu bytes) read from %s\n", page_size, meta_path);
3943  goto error;
3944  }
3945 
3946  src_file = os_file_create_simple_no_error_handling(
3947  0 /* dummy of innodb_file_data_key */,
3948  src_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, &success);
3949  if (!success) {
3950  os_file_get_last_error(TRUE);
3951  fprintf(stderr,
3952  "xtrabackup: error: cannot open %s\n",
3953  src_path);
3954  goto error;
3955  }
3956 
3957 #ifdef USE_POSIX_FADVISE
3958  posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL);
3959  posix_fadvise(src_file, 0, 0, POSIX_FADV_DONTNEED);
3960 #endif
3961 
3962  if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT) {
3963  os_file_set_nocache(src_file, src_path, "OPEN");
3964  }
3965 
3966  dst_file = os_file_create_simple_no_error_handling(
3967  0 /* dummy of innodb_file_data_key */,
3968  dst_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, &success);
3969  if (!success) {
3970  os_file_get_last_error(TRUE);
3971  fprintf(stderr,
3972  "xtrabackup: error: cannot open %s\n",
3973  dst_path);
3974  goto error;
3975  }
3976 
3977 #ifdef USE_POSIX_FADVISE
3978  posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
3979 #endif
3980 
3981  if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT) {
3982  os_file_set_nocache(dst_file, dst_path, "OPEN");
3983  }
3984 
3985  printf("Applying %s ...\n", src_path);
3986 
3987  while (!last_buffer) {
3988  ulint cluster_header;
3989 
3990  /* read to buffer */
3991  /* first block of block cluster */
3992  success = os_file_read(src_file, incremental_buffer,
3993  ((incremental_buffers * (page_size / 4))
3994  << page_size_shift) & 0xFFFFFFFFUL,
3995  (incremental_buffers * (page_size / 4))
3996  >> (32 - page_size_shift),
3997  page_size);
3998  if (!success) {
3999  goto error;
4000  }
4001 
4002  cluster_header = mach_read_from_4(incremental_buffer);
4003  switch(cluster_header) {
4004  case 0x78747261UL: /*"xtra"*/
4005  break;
4006  case 0x58545241UL: /*"XTRA"*/
4007  last_buffer = TRUE;
4008  break;
4009  default:
4010  fprintf(stderr,
4011  "xtrabackup: error: %s seems not .delta file.\n",
4012  src_path);
4013  goto error;
4014  }
4015 
4016  for (page_in_buffer = 1; page_in_buffer < page_size / 4;
4017  page_in_buffer++) {
4018  if (mach_read_from_4(incremental_buffer + page_in_buffer * 4)
4019  == 0xFFFFFFFFUL)
4020  break;
4021  }
4022 
4023  ut_a(last_buffer || page_in_buffer == page_size / 4);
4024 
4025  /* read whole of the cluster */
4026  success = os_file_read(src_file, incremental_buffer,
4027  ((incremental_buffers * (page_size / 4))
4028  << page_size_shift) & 0xFFFFFFFFUL,
4029  (incremental_buffers * (page_size / 4))
4030  >> (32 - page_size_shift),
4031  page_in_buffer * page_size);
4032  if (!success) {
4033  goto error;
4034  }
4035 
4036  for (page_in_buffer = 1; page_in_buffer < page_size / 4;
4037  page_in_buffer++) {
4038  ulint offset_on_page;
4039 
4040  offset_on_page = mach_read_from_4(incremental_buffer + page_in_buffer * 4);
4041 
4042  if (offset_on_page == 0xFFFFFFFFUL)
4043  break;
4044 
4045  /* apply blocks in the cluster */
4046 // if (ut_dulint_cmp(incremental_lsn,
4047 // MACH_READ_64(incremental_buffer
4048 // + page_in_buffer * page_size
4049 // + FIL_PAGE_LSN)) >= 0)
4050 // continue;
4051 
4052  success = os_file_write(dst_path, dst_file,
4053  incremental_buffer +
4054  page_in_buffer * page_size,
4055  (offset_on_page << page_size_shift) &
4056  0xFFFFFFFFUL,
4057  offset_on_page >> (32 - page_size_shift),
4058  page_size);
4059  if (!success) {
4060  goto error;
4061  }
4062  }
4063 
4064  incremental_buffers++;
4065  }
4066 
4067  if (src_file != -1)
4068  os_file_close(src_file);
4069  if (dst_file != -1)
4070  os_file_close(dst_file);
4071  return;
4072 
4073 error:
4074  if (src_file != -1)
4075  os_file_close(src_file);
4076  if (dst_file != -1)
4077  os_file_close(dst_file);
4078  fprintf(stderr, "xtrabackup: Error: xtrabackup_apply_delta() failed.\n");
4079  return;
4080 }
4081 
4082 static void
4083 xtrabackup_apply_deltas(bool check_newer)
4084 {
4085  int ret;
4086  char dbpath[FN_REFLEN];
4087  os_file_dir_t dir;
4088  os_file_dir_t dbdir;
4089  os_file_stat_t dbinfo;
4090  os_file_stat_t fileinfo;
4091  ulint err = DB_SUCCESS;
4092  static char current_dir[2];
4093 
4094  current_dir[0] = FN_CURLIB;
4095  current_dir[1] = 0;
4096  srv_data_home = current_dir;
4097 
4098  /* datafile */
4099  dbdir = os_file_opendir(xtrabackup_incremental_dir, FALSE);
4100 
4101  if (dbdir != NULL) {
4102  ret = fil_file_readdir_next_file(&err, xtrabackup_incremental_dir, dbdir,
4103  &fileinfo);
4104  while (ret == 0) {
4105  if (fileinfo.type == OS_FILE_TYPE_DIR) {
4106  goto next_file_item_1;
4107  }
4108 
4109  if (strlen(fileinfo.name) > 6
4110  && 0 == strcmp(fileinfo.name +
4111  strlen(fileinfo.name) - 6,
4112  ".delta")) {
4113  xtrabackup_apply_delta(
4114  xtrabackup_incremental_dir, NULL,
4115  fileinfo.name, check_newer);
4116  }
4117 next_file_item_1:
4118  ret = fil_file_readdir_next_file(&err,
4119  xtrabackup_incremental_dir, dbdir,
4120  &fileinfo);
4121  }
4122 
4123  os_file_closedir(dbdir);
4124  } else {
4125  fprintf(stderr, "xtrabackup: Cannot open dir %s\n", xtrabackup_incremental_dir);
4126  }
4127 
4128  /* single table tablespaces */
4129  dir = os_file_opendir(xtrabackup_incremental_dir, FALSE);
4130 
4131  if (dir == NULL) {
4132  fprintf(stderr, "xtrabackup: Cannot open dir %s\n", xtrabackup_incremental_dir);
4133  }
4134 
4135  ret = fil_file_readdir_next_file(&err, xtrabackup_incremental_dir, dir,
4136  &dbinfo);
4137  while (ret == 0) {
4138  if (dbinfo.type == OS_FILE_TYPE_FILE
4139  || dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
4140 
4141  goto next_datadir_item;
4142  }
4143 
4144  sprintf(dbpath, "%s/%s", xtrabackup_incremental_dir,
4145  dbinfo.name);
4147 
4148  dbdir = os_file_opendir(dbpath, FALSE);
4149 
4150  if (dbdir != NULL) {
4151 
4152  ret = fil_file_readdir_next_file(&err, dbpath, dbdir,
4153  &fileinfo);
4154  while (ret == 0) {
4155 
4156  if (fileinfo.type == OS_FILE_TYPE_DIR) {
4157 
4158  goto next_file_item_2;
4159  }
4160 
4161  if (strlen(fileinfo.name) > 6
4162  && 0 == strcmp(fileinfo.name +
4163  strlen(fileinfo.name) - 6,
4164  ".delta")) {
4165  /* The name ends in .delta; try opening
4166  the file */
4167  xtrabackup_apply_delta(
4168  xtrabackup_incremental_dir, dbinfo.name,
4169  fileinfo.name, check_newer);
4170  }
4171 next_file_item_2:
4172  ret = fil_file_readdir_next_file(&err,
4173  dbpath, dbdir,
4174  &fileinfo);
4175  }
4176 
4177  os_file_closedir(dbdir);
4178  }
4179 next_datadir_item:
4180  ret = fil_file_readdir_next_file(&err,
4181  xtrabackup_incremental_dir,
4182  dir, &dbinfo);
4183  }
4184 
4185  os_file_closedir(dir);
4186 
4187 }
4188 
4189 static bool
4190 xtrabackup_close_temp_log(bool clear_flag)
4191 {
4192  os_file_t src_file = -1;
4193  char src_path[FN_REFLEN];
4194  char dst_path[FN_REFLEN];
4195  ibool success;
4196 
4197  byte* log_buf;
4198  byte* log_buf_ = NULL;
4199 
4200 
4201  if (!xtrabackup_logfile_is_renamed)
4202  return(FALSE);
4203 
4204  /* Restore log parameters */
4205  innobase_log_group_home_dir = innobase_log_group_home_dir_backup;
4206  innobase_log_file_size = innobase_log_file_size_backup;
4207  innobase_log_files_in_group = innobase_log_files_in_group_backup;
4208 
4209  /* rename 'ib_logfile0' to 'xtrabackup_logfile' */
4210  if(!xtrabackup_incremental_dir) {
4211  sprintf(dst_path, "%s%s", xtrabackup_target_dir, "/ib_logfile0");
4212  sprintf(src_path, "%s%s", xtrabackup_target_dir, "/xtrabackup_logfile");
4213  } else {
4214  sprintf(dst_path, "%s%s", xtrabackup_incremental_dir, "/ib_logfile0");
4215  sprintf(src_path, "%s%s", xtrabackup_incremental_dir, "/xtrabackup_logfile");
4216  }
4217 
4218  srv_normalize_path_for_win(dst_path);
4219  srv_normalize_path_for_win(src_path);
4220 
4221  success = os_file_rename(
4222  0 /* dummy of innodb_file_data_key */,
4223  dst_path, src_path);
4224  if (!success) {
4225  goto error;
4226  }
4227  xtrabackup_logfile_is_renamed = FALSE;
4228 
4229  if (!clear_flag)
4230  return(FALSE);
4231 
4232  /* clear LOG_FILE_WAS_CREATED_BY_HOT_BACKUP field */
4233  src_file = os_file_create_simple_no_error_handling(
4234  0 /* dummy of innodb_file_data_key */,
4235  src_path, OS_FILE_OPEN,
4236  OS_FILE_READ_WRITE, &success);
4237  if (!success) {
4238  goto error;
4239  }
4240 
4241 #ifdef USE_POSIX_FADVISE
4242  posix_fadvise(src_file, 0, 0, POSIX_FADV_DONTNEED);
4243 #endif
4244 
4245  if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT) {
4246  os_file_set_nocache(src_file, src_path, "OPEN");
4247  }
4248 
4249  log_buf_ = (unsigned char*) ut_malloc(LOG_FILE_HDR_SIZE * 2);
4250  log_buf = (unsigned char*) ut_align(log_buf_, LOG_FILE_HDR_SIZE);
4251 
4252  success = os_file_read(src_file, log_buf, 0, 0, LOG_FILE_HDR_SIZE);
4253  if (!success) {
4254  goto error;
4255  }
4256 
4257  memset(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, ' ', 4);
4258 
4259  success = os_file_write(src_path, src_file, log_buf, 0, 0, LOG_FILE_HDR_SIZE);
4260  if (!success) {
4261  goto error;
4262  }
4263 
4264  os_file_close(src_file);
4265  src_file = -1;
4266 
4267  return(FALSE);
4268 error:
4269  if (src_file != -1)
4270  os_file_close(src_file);
4271  if (log_buf_)
4272  ut_free(log_buf_);
4273  fprintf(stderr, "xtrabackup: Error: xtrabackup_close_temp_log() failed.\n");
4274  return(TRUE); /*ERROR*/
4275 }
4276 
4277 static void
4278 xtrabackup_prepare_func(void)
4279 {
4280  /* cd to target-dir */
4281 
4282  if (chdir(xtrabackup_real_target_dir) != 0)
4283  {
4284  fprintf(stderr, "xtrabackup: cannot my_setwd %s\n", xtrabackup_real_target_dir);
4285  exit(EXIT_FAILURE);
4286  }
4287  fprintf(stderr, "xtrabackup: cd to %s\n", xtrabackup_real_target_dir);
4288 
4289  xtrabackup_target_dir= mysql_data_home_buff;
4290  mysql_data_home_buff[0]=FN_CURLIB; // all paths are relative from here
4291  mysql_data_home_buff[1]=0;
4292 
4293  /* read metadata of target */
4294  {
4295  char filename[FN_REFLEN];
4296 
4297  sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME);
4298 
4299  if (xtrabackup_read_metadata(filename))
4300  fprintf(stderr, "xtrabackup: error: xtrabackup_read_metadata()\n");
4301 
4302  if (!strcmp(metadata_type, "full-backuped")) {
4303  fprintf(stderr, "xtrabackup: This target seems to be not prepared yet.\n");
4304  } else if (!strcmp(metadata_type, "full-prepared")) {
4305  fprintf(stderr, "xtrabackup: This target seems to be already prepared.\n");
4306  goto skip_check;
4307  } else {
4308  fprintf(stderr, "xtrabackup: This target seems not to have correct metadata...\n");
4309  }
4310 
4311  if (xtrabackup_incremental) {
4312  fprintf(stderr,
4313  "xtrabackup: error: applying incremental backup needs target prepared.\n");
4314  exit(EXIT_FAILURE);
4315  }
4316 skip_check:
4317  if (xtrabackup_incremental
4318  && ut_dulint_cmp(metadata_to_lsn, incremental_lsn) != 0) {
4319  fprintf(stderr,
4320  "xtrabackup: error: This incremental backup seems not to be proper for the target.\n"
4321  "xtrabackup: Check 'to_lsn' of the target and 'from_lsn' of the incremental.\n");
4322  exit(EXIT_FAILURE);
4323  }
4324  }
4325 
4326  /* Create logfiles for recovery from 'xtrabackup_logfile', before start InnoDB */
4327  srv_max_n_threads = 1000;
4328  os_sync_mutex = NULL;
4329  ut_mem_init();
4330 #ifdef XTRADB_BASED
4331  /* temporally dummy value to avoid crash */
4332  srv_page_size_shift = 14;
4333  srv_page_size = (1 << srv_page_size_shift);
4334 #endif
4335  os_sync_init();
4336  sync_init();
4338  if(xtrabackup_init_temp_log())
4339  goto error;
4340 
4341  if(xtrabackup_incremental)
4342  xtrabackup_apply_deltas(TRUE);
4343 
4344  sync_close();
4345  sync_initialized = FALSE;
4346  os_sync_free();
4347  os_sync_mutex = NULL;
4348  ut_free_all_mem();
4349 
4350  /* check the accessibility of target-dir */
4351  /* ############# TODO ##################### */
4352 
4353  if(innodb_init_param())
4354  goto error;
4355 
4356  srv_apply_log_only = (ibool) xtrabackup_apply_log_only;
4357 
4358  /* increase IO threads */
4359  if(srv_n_file_io_threads < 10) {
4360  srv_n_file_io_threads = 10;
4361  }
4362 
4363  fprintf(stderr, "xtrabackup: Starting InnoDB instance for recovery.\n"
4364  "xtrabackup: Using %"PRIu64" bytes for buffer pool (set by --use-memory parameter)\n",
4365  xtrabackup_use_memory);
4366 
4367  if(innodb_init())
4368  goto error;
4369 
4370  //printf("Hello InnoDB world!\n");
4371 
4372  /* TEST: innodb status*/
4373 /*
4374  ulint trx_list_start = ULINT_UNDEFINED;
4375  ulint trx_list_end = ULINT_UNDEFINED;
4376  srv_printf_innodb_monitor(stdout, &trx_list_start, &trx_list_end);
4377 */
4378  /* TEST: list of datafiles and transaction log files and LSN*/
4379 /*
4380  {
4381  fil_system_t* f_system = fil_system;
4382  fil_space_t* space;
4383  fil_node_t* node;
4384 
4385  mutex_enter(&(f_system->mutex));
4386 
4387  space = UT_LIST_GET_FIRST(f_system->space_list);
4388 
4389  while (space != NULL) {
4390  printf("space: name=%s, id=%d, purpose=%d, size=%d\n",
4391  space->name, space->id, space->purpose, space->size);
4392 
4393  node = UT_LIST_GET_FIRST(space->chain);
4394 
4395  while (node != NULL) {
4396  printf("node: name=%s, open=%d, size=%d\n",
4397  node->name, node->open, node->size);
4398 
4399  node = UT_LIST_GET_NEXT(chain, node);
4400  }
4401  space = UT_LIST_GET_NEXT(space_list, space);
4402  }
4403 
4404  mutex_exit(&(f_system->mutex));
4405  }
4406 */
4407  /* align space sizes along with fsp header */
4408  {
4409  fil_system_t* f_system = fil_system;
4410  fil_space_t* space;
4411 
4412  mutex_enter(&(f_system->mutex));
4413  space = UT_LIST_GET_FIRST(f_system->space_list);
4414 
4415  while (space != NULL) {
4416  byte* header;
4417  ulint size;
4418  ulint actual_size;
4419  mtr_t mtr;
4420  buf_block_t* block;
4421  ulint flags;
4422 
4423  if (space->purpose == FIL_TABLESPACE) {
4424  mutex_exit(&(f_system->mutex));
4425 
4426  mtr_start(&mtr);
4427 
4428  mtr_s_lock(fil_space_get_latch(space->id, &flags), &mtr);
4429 
4430  block = buf_page_get(space->id,
4432  0, RW_S_LATCH, &mtr);
4433  header = FIL_PAGE_DATA /*FSP_HEADER_OFFSET*/
4434  + buf_block_get_frame(block);
4435 
4436  size = mtr_read_ulint(header + 8 /* FSP_SIZE */, MLOG_4BYTES, &mtr);
4437 
4438  mtr_commit(&mtr);
4439 
4440  //printf("%d, %d\n", space->id, size);
4441 
4442  fil_extend_space_to_desired_size(&actual_size, space->id, size);
4443 
4444  mutex_enter(&(f_system->mutex));
4445  }
4446 
4447  space = UT_LIST_GET_NEXT(space_list, space);
4448  }
4449 
4450  mutex_exit(&(f_system->mutex));
4451  }
4452 
4453 
4454 
4455  if (xtrabackup_export) {
4456  printf("xtrabackup: export option is specified.\n");
4457  if (innobase_file_per_table) {
4458  fil_system_t* f_system = fil_system;
4459  fil_space_t* space;
4460  fil_node_t* node;
4461  os_file_t info_file = -1;
4462  char info_file_path[FN_REFLEN];
4463  ibool success;
4464  char table_name[FN_REFLEN];
4465 
4466  byte* page;
4467  byte* buf = NULL;
4468 
4469  buf = (byte*) ut_malloc(UNIV_PAGE_SIZE * 2);
4470  page = (byte*) ut_align(buf, UNIV_PAGE_SIZE);
4471 
4472  /* flush insert buffer at shutdwon */
4473  innobase_fast_shutdown = 0;
4474 
4475  mutex_enter(&(f_system->mutex));
4476 
4477  space = UT_LIST_GET_FIRST(f_system->space_list);
4478  while (space != NULL) {
4479  /* treat file_per_table only */
4480  if (space->purpose != FIL_TABLESPACE
4481 #ifdef XTRADB_BASED
4482  || trx_sys_sys_space(space->id)
4483 #else
4484  || space->id == 0
4485 #endif
4486  )
4487  {
4488  space = UT_LIST_GET_NEXT(space_list, space);
4489  continue;
4490  }
4491 
4492  node = UT_LIST_GET_FIRST(space->chain);
4493  while (node != NULL) {
4494  int len;
4495  char *next, *prev, *p;
4496  dict_table_t* table;
4497  dict_index_t* index;
4498  ulint n_index;
4499 
4500  /* node exist == file exist, here */
4501  strncpy(info_file_path, node->name, FN_REFLEN);
4502  len = strlen(info_file_path);
4503  info_file_path[len - 3] = 'e';
4504  info_file_path[len - 2] = 'x';
4505  info_file_path[len - 1] = 'p';
4506 
4507  p = info_file_path;
4508  prev = NULL;
4509  while ((next = strstr(p, SRV_PATH_SEPARATOR_STR)) != NULL)
4510  {
4511  prev = p;
4512  p = next + 1;
4513  }
4514  info_file_path[len - 4] = 0;
4515  strncpy(table_name, prev, FN_REFLEN);
4516 
4517  info_file_path[len - 4] = '.';
4518 
4519  mutex_exit(&(f_system->mutex));
4520  mutex_enter(&(dict_sys->mutex));
4521 
4522  table = dict_table_get_low(table_name);
4523  if (!table) {
4524  fprintf(stderr,
4525 "xtrabackup: error: cannot find dictionary record of table %s\n", table_name);
4526  goto next_node;
4527  }
4528  index = dict_table_get_first_index(table);
4529  n_index = UT_LIST_GET_LEN(table->indexes);
4530  if (n_index > 31) {
4531  fprintf(stderr,
4532 "xtrabackup: error: sorry, cannot export over 31 indexes for now.\n");
4533  goto next_node;
4534  }
4535 
4536  /* init exp file */
4537  bzero(page, UNIV_PAGE_SIZE);
4538  mach_write_to_4(page , 0x78706f72UL);
4539  mach_write_to_4(page + 4, 0x74696e66UL);/*"xportinf"*/
4540  mach_write_to_4(page + 8, n_index);
4541  strncpy((char*)page + 12, table_name, 500);
4542 
4543  printf(
4544 "xtrabackup: export metadata of table '%s' to file `%s` (%lu indexes)\n",
4545  table_name, info_file_path, n_index);
4546 
4547  n_index = 1;
4548  while (index) {
4549  mach_write_to_8(page + n_index * 512, index->id);
4550  mach_write_to_4(page + n_index * 512 + 8,
4551  index->page);
4552 
4553  strncpy((char*)page + n_index * 512 + 12, index->name, 500);
4554 
4555  printf(
4556 "xtrabackup: name=%s, id.low=%lu, page=%lu\n",
4557  index->name,
4558  (ulint)(index->id & 0xFFFFFFFFUL),
4559 
4560  (ulint) index->page);
4561 
4562  index = dict_table_get_next_index(index);
4563  n_index++;
4564  }
4565 
4566  srv_normalize_path_for_win(info_file_path);
4567  info_file = os_file_create(
4568  0 /* dummy of innodb_file_data_key */,
4569  info_file_path, OS_FILE_OVERWRITE,
4570  OS_FILE_NORMAL, OS_DATA_FILE, &success);
4571  if (!success) {
4572  os_file_get_last_error(TRUE);
4573  goto next_node;
4574  }
4575  success = os_file_write(info_file_path, info_file, page,
4576  0, 0, UNIV_PAGE_SIZE);
4577  if (!success) {
4578  os_file_get_last_error(TRUE);
4579  goto next_node;
4580  }
4581  success = os_file_flush(info_file);
4582  if (!success) {
4583  os_file_get_last_error(TRUE);
4584  goto next_node;
4585  }
4586 next_node:
4587  if (info_file != -1) {
4588  os_file_close(info_file);
4589  info_file = -1;
4590  }
4591  mutex_exit(&(dict_sys->mutex));
4592  mutex_enter(&(f_system->mutex));
4593 
4594  node = UT_LIST_GET_NEXT(chain, node);
4595  }
4596 
4597  space = UT_LIST_GET_NEXT(space_list, space);
4598  }
4599  mutex_exit(&(f_system->mutex));
4600 
4601  ut_free(buf);
4602  } else {
4603  printf("xtrabackup: export option is for file_per_table only, disabled.\n");
4604  }
4605  }
4606 
4607  /* print binlog position (again?) */
4608  printf("\n[notice (again)]\n"
4609  " If you use binary log and don't use any hack of group commit,\n"
4610  " the binary log position seems to be:\n");
4611 // FIXME: trx_sys_print_mysql_binlog_offset();
4612  printf("\n");
4613 
4614  /* output to xtrabackup_binlog_pos_innodb */
4615  if (false) {
4616 //FIXME if (*trx_sys_mysql_bin_log_name != '\0') {
4617  FILE *fp;
4618 
4619  fp = fopen("xtrabackup_binlog_pos_innodb", "w");
4620  if (fp) {
4621  fprintf(fp, "%s\t%llu\n",
4622  "none", 0ULL);
4623  /* FIXME
4624  trx_sys_mysql_bin_log_name,
4625  trx_sys_mysql_bin_log_pos);*/
4626  fclose(fp);
4627  } else {
4628  printf("xtrabackup: failed to open 'xtrabackup_binlog_pos_innodb'\n");
4629  }
4630  }
4631 
4632  /* Check whether the log is applied enough or not. */
4633  if ((xtrabackup_incremental
4634  && ut_dulint_cmp(srv_start_lsn, incremental_last_lsn) < 0)
4635  ||(!xtrabackup_incremental
4636  && ut_dulint_cmp(srv_start_lsn, metadata_last_lsn) < 0)) {
4637  printf( "xtrabackup: ########################################################\n"
4638  "xtrabackup: # !!WARNING!! #\n"
4639  "xtrabackup: # The transaction log file should be wrong or corrupt. #\n"
4640  "xtrabackup: # The log was not applied to the intended LSN! #\n"
4641  "xtrabackup: ########################################################\n");
4642  if (xtrabackup_incremental) {
4643  printf("xtrabackup: The intended lsn is %"PRIu64"\n",
4644  incremental_last_lsn);
4645  } else {
4646  printf("xtrabackup: The intended lsn is %"PRIu64"\n",
4647  metadata_last_lsn);
4648  }
4649  }
4650 
4651  if(innodb_end())
4652  goto error;
4653 
4654  sync_initialized = FALSE;
4655  os_sync_mutex = NULL;
4656 
4657  /* re-init necessary components */
4658  ut_mem_init();
4659 
4660  os_sync_init();
4661  sync_init();
4663 
4664  if(xtrabackup_close_temp_log(TRUE))
4665  exit(EXIT_FAILURE);
4666 
4667  /* output to metadata file */
4668  {
4669  char filename[FN_REFLEN];
4670 
4671  strcpy(metadata_type, "full-prepared");
4672 
4673  if(xtrabackup_incremental
4674  && ut_dulint_cmp(metadata_to_lsn, incremental_to_lsn) < 0)
4675  {
4676  metadata_to_lsn = incremental_to_lsn;
4677  metadata_last_lsn = incremental_last_lsn;
4678  }
4679 
4680  sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME);
4681  if (xtrabackup_write_metadata(filename))
4682  fprintf(stderr, "xtrabackup: error: xtrabackup_write_metadata(xtrabackup_target_dir)\n");
4683 
4684  if(xtrabackup_extra_lsndir) {
4685  sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME);
4686  if (xtrabackup_write_metadata(filename))
4687  fprintf(stderr, "xtrabackup: error: xtrabackup_write_metadata(xtrabackup_extra_lsndir)\n");
4688  }
4689  }
4690 
4691  if(!xtrabackup_create_ib_logfile)
4692  return;
4693 
4694  /* TODO: make more smart */
4695 
4696  printf("\n[notice]\nWe cannot call InnoDB second time during the process lifetime.\n");
4697  printf("Please re-execte to create ib_logfile*. Sorry.\n");
4698 /*
4699  printf("Restart InnoDB to create ib_logfile*.\n");
4700 
4701  if(innodb_init_param())
4702  goto error;
4703 
4704  if(innodb_init())
4705  goto error;
4706 
4707  if(innodb_end())
4708  goto error;
4709 */
4710 
4711  return;
4712 
4713 error:
4714  xtrabackup_close_temp_log(FALSE);
4715 
4716  exit(EXIT_FAILURE);
4717 }
4718 
4719 /* ================= main =================== */
4720 
4721 int main(int argc, char **argv)
4722 {
4723  po::options_description commandline_options(_("Options used only in command line"));
4724  commandline_options.add_options()
4725  ("target-dir", po::value<std::string>(), _("destination directory"))
4726  ("backup", po::value<bool>(&xtrabackup_backup)->default_value(false)->zero_tokens(), _("take backup to target-dir"))
4727  ("stats", po::value<bool>(&xtrabackup_stats)->default_value(false)->zero_tokens(), _("calc statistic of datadir (offline mysqld is recommended)"))
4728  ("prepare", po::value<bool>(&xtrabackup_prepare)->default_value(false)->zero_tokens(), _("prepare a backup for starting mysql server on the backup."))
4729  ("export", po::value<bool>(&xtrabackup_export)->default_value(false)->zero_tokens(), _("create files to import to another database when prepare."))
4730  ("apply-log-only", po::value<bool>(&xtrabackup_apply_log_only)->default_value(false)->zero_tokens(), _("stop recovery process not to progress LSN after applying log when prepare."))
4731  ("print-param", po::value<bool>(&xtrabackup_print_param)->default_value(false)->zero_tokens(), _("print parameter of mysqld needed for copyback."))
4732  ("use-memory", po::value<uint64_t>(&xtrabackup_use_memory)->default_value(100*1024*1024), _("The value is used instead of buffer_pool_size"))
4733  ("suspend-at-end", po::value<bool>(&xtrabackup_suspend_at_end)->default_value(false)->zero_tokens(), _("creates a file 'xtrabackup_suspended' and waits until the user deletes that file at the end of '--backup'"))
4734  ("throttle", po::value<long>(&xtrabackup_throttle), _("limit count of IO operations (pairs of read&write) per second to IOS values (for '--backup')"))
4735  ("log-stream", po::value<bool>(&xtrabackup_stream)->default_value(false)->zero_tokens(), _("outputs the contents of 'xtrabackup_logfile' to stdout only until the file 'xtrabackup_suspended' deleted (for '--backup')."))
4736  ("extra-lsndir", po::value<std::string>(), _("(for --backup): save an extra copy of the xtrabackup_checkpoints file in this directory."))
4737  ("incremental-lsn", po::value<std::string>(), _("(for --backup): copy only .ibd pages newer than specified LSN 'high:low'. ##ATTENTION##: checkpoint lsn must be used. anyone can detect your mistake. be carefully!"))
4738  ("incremental-basedir", po::value<std::string>(), _("(for --backup): copy only .ibd pages newer than backup at specified directory."))
4739  ("incremental-dir", po::value<std::string>(), _("(for --prepare): apply .delta files and logfile in the specified directory."))
4740  ("tables", po::value<std::string>(), _("filtering by regexp for table names."))
4741  ("tables-file", po::value<std::string>(), _("filtering by list of the exact database.table name in the file."))
4742  ("create-ib-logfile", po::value<bool>(&xtrabackup_create_ib_logfile), _("** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###"))
4743  ("datadir,h", po::value<std::string>(), _("Path to the database root."))
4744  ("tmpdir,t", po::value<std::string>(), _("Path for temporary files. Several paths may be specified, separated by a colon (:), in this case they are used in a round-robin fashion."))
4745  ("parallel", po::value<uint32_t>(&parallel)->default_value(1), _("Number of threads to use for parallel datafiles transfer. Does not have any effect in the stream mode. The default value is 1."))
4746  ("innodb-adaptive-hash-index", po::value<bool>(&innobase_adaptive_hash_index)->default_value(true), _("Enable InnoDB adaptive hash index (enabled by default). Disable with --skip-innodb-adaptive-hash-index."))
4747  ("innodb-additional-mem-pool-size", po::value<long>(&innobase_additional_mem_pool_size)->default_value(1*1024*1024), _("Size of a memory pool InnoDB uses to store data dictionary information and other internal data structures."))
4748  ("innodb-autoextend-increment", po::value<uint32_t>(&srv_auto_extend_increment)->default_value(8), _("Data file autoextend increment in megabytes"))
4749  ("innodb-buffer-pool-size", po::value<uint64_t>(&innobase_buffer_pool_size)->default_value(8*1024*1024), _("The size of the memory buffer InnoDB uses to cache data and indexes of its tables."))
4750  ("innodb-checksums", po::value<bool>(&innobase_use_checksums)->default_value(true), _("Enable InnoDB checksums validation (enabled by default). Disable with --skip-innodb-checksums."))
4751  ("innodb-data-file-path", po::value<std::string>(), _("Path to individual files and their sizes."))
4752  ("innodb-data-home-dir", po::value<std::string>(), _("The common part for InnoDB table spaces."))
4753  ("innodb-doublewrite", po::value<bool>(&innobase_use_doublewrite)->default_value(true), _("Enable InnoDB doublewrite buffer (enabled by default). Disable with --skip-innodb-doublewrite."))
4754  ("innodb-file-io-threads", po::value<long>(&innobase_file_io_threads)->default_value(4), _("Number of file I/O threads in InnoDB."))
4755  ("innodb-file-per-table", po::value<bool>(&innobase_file_per_table), _("Stores each InnoDB table to an .ibd file in the database dir."))
4756  ("innodb-flush-log-at-trx-commit", po::value<ulong>(&srv_flush_log_at_trx_commit)->default_value(1), _("Set to 0 (write and flush once per second), 1 (write and flush at each commit) or 2 (write at commit, flush once per second)."))
4757  ("innodb-flush-method", po::value<std::string>(), _("With which method to flush data."))
4758 /* ####### Should we use this option? ####### */
4759  ("innodb-force-recovery", po::value<long>(&innobase_force_recovery)->default_value(0), _("Helps to save your data in case the disk image of the database becomes corrupt."))
4760  ("innodb-lock-wait-timeout", po::value<long>(&innobase_lock_wait_timeout)->default_value(50), _("Timeout in seconds an InnoDB transaction may wait for a lock before being rolled back."))
4761  ("innodb-log-buffer-size", po::value<long>(&innobase_log_buffer_size)->default_value(1024*1024), _("The size of the buffer which InnoDB uses to write log to the log files on disk."))
4762  ("innodb-log-file-size", po::value<log_file_constraint>(&innobase_log_file_size)->default_value(20*1024*1024L), _("Size of each log file in a log group."))
4763  ("innodb-log-files-in-group", po::value<long>(&innobase_log_files_in_group)->default_value(2), _("Number of log files in the log group. InnoDB writes to the files in a circular fashion. Value 3 is recommended here."))
4764  ("innodb-log-group-home-dir", po::value<std::string>(), _("Path to InnoDB log files."))
4765  ("innodb-max_dirty-pages-pct", po::value<ulong>(&srv_max_buf_pool_modified_pct)->default_value(90), _("Percentage of dirty pages allowed in bufferpool."))
4766  ("innodb-open-files", po::value<long>(&innobase_open_files)->default_value(300), _("How many files at the maximum InnoDB keeps open at the same time."))
4767 #ifdef XTRADB_BASED
4768  ("innodb-page-size", po::value<uint32_t>(&innobase_page_size)->default_value(1 << 14), _("The universal page size of the database."))
4769  ("innodb-log-block-size", po::value<uint32_t>(&innobase_log_block_size)->default_value(512), _("###EXPERIMENTAL###: The log block size of the transaction log file. Changing for created log file is not supported. Use on your own risk!"))
4770  ("innodb-fast-checksum", po::value<bool>(&innobase_fast_checksum), _("Change the algorithm of checksum for the whole of datapage to 4-bytes word based."))
4771  ("innodb-extra-undoslots", po::value<bool>(&innobase_extra_undoslots), _("Enable to use about 4000 undo slots instead of default 1024. Not recommended to use, Because it is not change back to disable, once it is used."))
4772  ("innodb-doublewrite-file", po::value<char *>(&innobase_doublewrite_file), _("Path to special datafile for doublewrite buffer. (default is "": not used)"))
4773 #endif
4774  ;
4775 
4776  po::variables_map vm;
4777  // Disable allow_guessing, it is evil and broken
4778  int style = po::command_line_style::default_style & ~po::command_line_style::allow_guessing;
4779  po::store(po::command_line_parser(argc, argv).options(commandline_options).style(style).run(), vm);
4780  po::notify(vm);
4781 
4782  if (vm.count("target-dir"))
4783  xtrabackup_target_dir= vm["target-dir"].as<std::string>().c_str();
4784 
4785  if (vm.count("extra-lsndir"))
4786  xtrabackup_extra_lsndir= vm["extra-lsndir"].as<std::string>().c_str();
4787 
4788  if (vm.count("incremental-lsn"))
4789  xtrabackup_incremental= vm["incremental-lsn"].as<std::string>().c_str();
4790 
4791  if (vm.count("incremental-basedir"))
4792  xtrabackup_incremental_basedir= vm["incremental-basedir"].as<std::string>().c_str();
4793 
4794  boost::scoped_ptr<char> xtrabackup_tables_autoptr(new char[(vm.count("tables")) ? vm["tables"].as<std::string>().length() + 1: 0]);
4795  if (vm.count("tables"))
4796  {
4797  xtrabackup_tables= xtrabackup_tables_autoptr.get();
4798  strcpy(xtrabackup_tables, vm["tables"].as<std::string>().c_str());
4799  }
4800 
4801  if (vm.count("tables-file"))
4802  xtrabackup_tables_file= vm["tables-file"].as<std::string>().c_str();
4803 
4804  if (vm.count("tmpdir"))
4805  opt_mysql_tmpdir= vm["tmpdir"].as<std::string>().c_str();
4806 
4807  if (vm.count("innodb-data-file-path"))
4808  innobase_data_file_path= vm["innodb-data-file-path"].as<std::string>().c_str();
4809 
4810  boost::scoped_ptr<char> xtrabackup_incremental_dir_autoptr(new char[(vm.count("incremental-dir")) ? vm["incremental-dir"].as<std::string>().length() + 1: 0]);
4811  if (vm.count("incremental-dir"))
4812  {
4813  xtrabackup_incremental_dir= xtrabackup_incremental_dir_autoptr.get();
4814  strcpy(xtrabackup_incremental_dir, vm["incremental-dir"].as<std::string>().c_str());
4815  }
4816 
4817  boost::scoped_ptr<char> innobase_data_home_dir_autoptr(new char[(vm.count("innodb-data-home-dir")) ? vm["innodb-data-home-dir"].as<std::string>().length() + 1 : 0]);
4818  if (vm.count("innodb-data-home-dir"))
4819  {
4820  innobase_data_home_dir= innobase_data_home_dir_autoptr.get();
4821  strcpy(innobase_data_home_dir, vm["innodb-data-home-dir"].as<std::string>().c_str());
4822  }
4823 
4824  boost::scoped_ptr<char> innobase_flush_method_autoptr(new char[(vm.count("innodb-flush-method")) ? vm["innodb-flush-method"].as<std::string>().length() + 1 : 0]);
4825  if (vm.count("innodb-flush-method"))
4826  {
4827  innobase_unix_file_flush_method= innobase_flush_method_autoptr.get();
4828  strcpy(innobase_unix_file_flush_method, vm["innodb-flush-method"].as<std::string>().c_str());
4829  }
4830 
4831  boost::scoped_ptr<char> innobase_log_group_home_dir_autoptr(new char[(vm.count("innodb-log-group-home-dir")) ? vm["innodb-log-group-home-dir"].as<std::string>().length() + 1: 0]);
4832 
4833  if (vm.count("innodb-log-group-home-dir"))
4834  {
4835  innobase_log_group_home_dir= innobase_log_group_home_dir_autoptr.get();
4836  strcpy(innobase_log_group_home_dir, vm["innodb-log-group-home-dir"].as<std::string>().c_str());
4837  }
4838 
4839  xtrabackup_use_memory-= xtrabackup_use_memory % (1024*1024);
4840  if (xtrabackup_use_memory < (1024*1024)) {
4841  fprintf(stderr, "xtrabackup: use-memory out of range\n");
4842  exit(EXIT_FAILURE);
4843  }
4844 
4845  if (parallel < 1) {
4846  fprintf(stderr, "xtrabackup: parallel needs to be greater than 0\n");
4847  exit(EXIT_FAILURE);
4848  }
4849 
4850  innobase_additional_mem_pool_size-= innobase_additional_mem_pool_size % 1024;
4851  if (innobase_additional_mem_pool_size < (512*1024)) {
4852  fprintf(stderr, "xtrabackup: innodb-additional-mem-pool-size out of range\n");
4853  exit(EXIT_FAILURE);
4854  }
4855 
4856  if ((srv_auto_extend_increment < 1) || (srv_auto_extend_increment > 8)) {
4857  fprintf(stderr, "xtrabackup: innodb-auto-extend-increment out of range\n");
4858  exit(EXIT_FAILURE);
4859  }
4860 
4861  innobase_buffer_pool_size-= innobase_buffer_pool_size % (1024*1024);
4862  if (innobase_buffer_pool_size < (1024*1024)) {
4863  fprintf(stderr, "xtrabackup: innodb-buffer-pool-size out of range\n");
4864  exit(EXIT_FAILURE);
4865  }
4866 
4867  if ((innobase_file_io_threads < 4) || (innobase_file_io_threads > 64)) {
4868  fprintf(stderr, "xtrabackup: innodb-file-io-threads out of range\n");
4869  exit(EXIT_FAILURE);
4870  }
4871 
4872  if (srv_flush_log_at_trx_commit > 2) {
4873  fprintf(stderr, "xtrabackup: innodb-flush-log-at-trx-commit out of range\n");
4874  exit(EXIT_FAILURE);
4875  }
4876 
4877  if (innobase_force_recovery > 6) {
4878  fprintf(stderr, "xtrabackup: innodb-force-recovery out of range\n");
4879  exit(EXIT_FAILURE);
4880  }
4881 
4882  if ((innobase_lock_wait_timeout < 1) || (innobase_lock_wait_timeout > (1024*1024*1024))) {
4883  fprintf(stderr, "xtrabackup: innodb-lock-wait-timeout out of range\n");
4884  exit(EXIT_FAILURE);
4885  }
4886 
4887  innobase_log_buffer_size-= innobase_log_buffer_size % 1024;
4888  if (innobase_additional_mem_pool_size < (256*1024)) {
4889  fprintf(stderr, "xtrabackup: innodb-log-buffer-size out of range\n");
4890  exit(EXIT_FAILURE);
4891  }
4892 
4893  if (innobase_additional_mem_pool_size < (1024*1024)) {
4894  fprintf(stderr, "xtrabackup: innodb-log-file-size out of range\n");
4895  exit(EXIT_FAILURE);
4896  }
4897 
4898  if ((innobase_log_files_in_group < 2) || (innobase_log_files_in_group > (100))) {
4899  fprintf(stderr, "xtrabackup: innodb-log-files-in-group out of range\n");
4900  exit(EXIT_FAILURE);
4901  }
4902 
4903  if (srv_max_buf_pool_modified_pct > 100) {
4904  fprintf(stderr, "xtrabackup: innodb-max-buf-pool-modified-pct out of range\n");
4905  exit(EXIT_FAILURE);
4906  }
4907 
4908  if (innobase_open_files < 10) {
4909  fprintf(stderr, "xtrabackup: innodb-open-files out of range\n");
4910  exit(EXIT_FAILURE);
4911  }
4912 
4913  if ((innobase_page_size < (1 << 12)) || (innobase_page_size > (1 << UNIV_PAGE_SIZE_SHIFT_MAX))) {
4914  fprintf(stderr, "xtrabackup: innodb-page-size out of range\n");
4915  exit(EXIT_FAILURE);
4916  }
4917 
4918  if ((innobase_log_block_size < (512)) || (innobase_log_block_size > (1 << UNIV_PAGE_SIZE_SHIFT_MAX))) {
4919  fprintf(stderr, "xtrabackup: innodb-log-block-size out of range\n");
4920  exit(EXIT_FAILURE);
4921  }
4922 
4923  if (vm.count("datadir"))
4924  {
4925  mysql_data_home_arg.assign(vm["datadir"].as<std::string>());
4926  }
4927  else
4928  {
4929  mysql_data_home_arg.assign(LOCALSTATEDIR);
4930  }
4931 
4932  mysql_data_home= (char*)malloc(mysql_data_home_arg.length());
4933  strcpy(mysql_data_home, mysql_data_home_arg.c_str());
4934 
4935  if ((!xtrabackup_prepare) && (strcmp(mysql_data_home, "./") == 0)) {
4936  if (!xtrabackup_print_param)
4937  usage();
4938  printf("\nxtrabackup: Error: Please set parameter 'datadir'\n");
4939  exit(EXIT_FAILURE);
4940  }
4941 
4942  if (xtrabackup_tables) {
4943  /* init regexp */
4944  char *p, *next;
4945  int i;
4946  char errbuf[100];
4947 
4948  tables_regex_num = 1;
4949 
4950  p = xtrabackup_tables;
4951  while ((p = strchr(p, ',')) != NULL) {
4952  p++;
4953  tables_regex_num++;
4954  }
4955 
4956  tables_regex = (regex_t*) ut_malloc(sizeof(regex_t) * tables_regex_num);
4957 
4958  p = xtrabackup_tables;
4959  for (i=0; i < tables_regex_num; i++) {
4960  next = strchr(p, ',');
4961  ut_a(next || i == tables_regex_num - 1);
4962 
4963  next++;
4964  if (i != tables_regex_num - 1)
4965  *(next - 1) = '\0';
4966 
4967  regerror(regcomp(&tables_regex[i],p,REG_EXTENDED),
4968  &tables_regex[i],errbuf,sizeof(errbuf));
4969  fprintf(stderr, "xtrabackup: tables regcomp(%s): %s\n",p,errbuf);
4970 
4971  if (i != tables_regex_num - 1)
4972  *(next - 1) = ',';
4973  p = next;
4974  }
4975  }
4976 
4977  if (xtrabackup_tables_file) {
4978  char name_buf[NAME_LEN*2+2];
4979  FILE *fp;
4980 
4981  if (xtrabackup_stream) {
4982  fprintf(stderr, "xtrabackup: Warning: --tables_file option doesn't affect with --stream.\n");
4983  xtrabackup_tables_file = NULL;
4984  goto skip_tables_file_register;
4985  }
4986 
4987  name_buf[NAME_LEN*2+1] = '\0';
4988 
4989  /* init tables_hash */
4990  tables_hash = hash_create(1000);
4991 
4992  /* read and store the filenames */
4993  fp = fopen(xtrabackup_tables_file,"r");
4994  if (!fp) {
4995  fprintf(stderr, "xtrabackup: cannot open %s\n", xtrabackup_tables_file);
4996  exit(EXIT_FAILURE);
4997  }
4998  for (;;) {
4999  xtrabackup_tables_t* table;
5000  char* p = name_buf;
5001 
5002  if ( fgets(name_buf, NAME_LEN*2+1, fp) == 0 ) {
5003  break;
5004  }
5005 
5006  while (*p != '\0') {
5007  if (*p == '.') {
5008  *p = '/';
5009  }
5010  p++;
5011  }
5012  p = strchr(name_buf, '\n');
5013  if (p)
5014  {
5015  *p = '\0';
5016  }
5017 
5018  table = (xtrabackup_tables_t*) malloc(sizeof(xtrabackup_tables_t) + strlen(name_buf) + 1);
5019  memset(table, '\0', sizeof(xtrabackup_tables_t) + strlen(name_buf) + 1);
5020  table->name = ((char*)table) + sizeof(xtrabackup_tables_t);
5021  strcpy(table->name, name_buf);
5022 
5023  HASH_INSERT(xtrabackup_tables_t, name_hash, tables_hash,
5024  ut_fold_string(table->name), table);
5025 
5026  printf("xtrabackup: table '%s' is registered to the list.\n", table->name);
5027  }
5028  }
5029 skip_tables_file_register:
5030 
5031 #ifdef XTRADB_BASED
5032  /* temporary setting of enough size */
5033  srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_MAX;
5034  srv_page_size = UNIV_PAGE_SIZE_MAX;
5035  srv_log_block_size = 512;
5036 #endif
5037  if (xtrabackup_backup && xtrabackup_incremental) {
5038  /* direct specification is only for --backup */
5039  /* and the lsn is prior to the other option */
5040 
5041  char* endchar;
5042  int error = 0;
5043 
5044  incremental_lsn = strtoll(xtrabackup_incremental, &endchar, 10);
5045  if (*endchar != '\0')
5046  error = 1;
5047 
5048  if (error) {
5049  fprintf(stderr, "xtrabackup: value '%s' may be wrong format for incremental option.\n",
5050  xtrabackup_incremental);
5051  exit(EXIT_FAILURE);
5052  }
5053 
5054  /* allocate buffer for incremental backup (4096 pages) */
5055  incremental_buffer_base = (byte*) malloc((UNIV_PAGE_SIZE_MAX / 4 + 1) *
5056  UNIV_PAGE_SIZE_MAX);
5057  incremental_buffer = (byte*) ut_align(incremental_buffer_base,
5058  UNIV_PAGE_SIZE_MAX);
5059  } else if (xtrabackup_backup && xtrabackup_incremental_basedir) {
5060  char filename[FN_REFLEN];
5061 
5062  sprintf(filename, "%s/%s", xtrabackup_incremental_basedir, XTRABACKUP_METADATA_FILENAME);
5063 
5064  if (xtrabackup_read_metadata(filename)) {
5065  fprintf(stderr,
5066  "xtrabackup: error: failed to read metadata from %s\n",
5067  filename);
5068  exit(EXIT_FAILURE);
5069  }
5070 
5071  incremental_lsn = metadata_to_lsn;
5072  xtrabackup_incremental = xtrabackup_incremental_basedir; //dummy
5073 
5074  /* allocate buffer for incremental backup (4096 pages) */
5075  incremental_buffer_base = (byte*) malloc((UNIV_PAGE_SIZE_MAX / 4 + 1) *
5076  UNIV_PAGE_SIZE_MAX);
5077  incremental_buffer = (byte*) ut_align(incremental_buffer_base,
5078  UNIV_PAGE_SIZE_MAX);
5079  } else if (xtrabackup_prepare && xtrabackup_incremental_dir) {
5080  char filename[FN_REFLEN];
5081 
5082  sprintf(filename, "%s/%s", xtrabackup_incremental_dir, XTRABACKUP_METADATA_FILENAME);
5083 
5084  if (xtrabackup_read_metadata(filename)) {
5085  fprintf(stderr,
5086  "xtrabackup: error: failed to read metadata from %s\n",
5087  filename);
5088  exit(EXIT_FAILURE);
5089  }
5090 
5091  incremental_lsn = metadata_from_lsn;
5092  incremental_to_lsn = metadata_to_lsn;
5093  incremental_last_lsn = metadata_last_lsn;
5094  xtrabackup_incremental = xtrabackup_incremental_dir; //dummy
5095 
5096  /* allocate buffer for incremental backup (4096 pages) */
5097  incremental_buffer_base = (byte*) malloc((UNIV_PAGE_SIZE / 4 + 1) *
5098  UNIV_PAGE_SIZE);
5099  incremental_buffer = (byte*) ut_align(incremental_buffer_base,
5100  UNIV_PAGE_SIZE);
5101  } else {
5102  /* allocate buffer for applying incremental (for header page only) */
5103  incremental_buffer_base = (byte*) malloc((1 + 1) * UNIV_PAGE_SIZE_MAX);
5104  incremental_buffer = (byte*) ut_align(incremental_buffer_base,
5105  UNIV_PAGE_SIZE_MAX);
5106 
5107  xtrabackup_incremental = NULL;
5108  }
5109 
5110  /* --print-param */
5111  if (xtrabackup_print_param) {
5112  printf("# This MySQL options file was generated by XtraBackup.\n");
5113  printf("[mysqld]\n");
5114  printf("datadir = \"%s\"\n", mysql_data_home);
5115  printf("tmpdir = \"%s\"\n", opt_mysql_tmpdir);
5116  printf("innodb_data_home_dir = \"%s\"\n",
5117  innobase_data_home_dir ? innobase_data_home_dir : mysql_data_home);
5118  printf("innodb_data_file_path = \"%s\"\n",
5119  innobase_data_file_path ? innobase_data_file_path : "ibdata1:10M:autoextend");
5120  printf("innodb_log_group_home_dir = \"%s\"\n",
5121  innobase_log_group_home_dir ? innobase_log_group_home_dir : mysql_data_home);
5122  printf("innodb_log_files_in_group = %ld\n", innobase_log_files_in_group);
5123  printf("innodb_log_file_size = %"PRIu64"\n", (uint64_t)innobase_log_file_size);
5124  printf("innodb_flush_method = \"%s\"\n",
5125  (innobase_unix_file_flush_method != NULL) ?
5126  innobase_unix_file_flush_method : "");
5127  exit(EXIT_SUCCESS);
5128  }
5129 
5130  if (!xtrabackup_stream) {
5131  print_version();
5132  if (xtrabackup_incremental) {
5133  printf("incremental backup from %"PRIu64" is enabled.\n",
5134  incremental_lsn);
5135  }
5136  } else {
5137  if (xtrabackup_backup) {
5138  xtrabackup_suspend_at_end = TRUE;
5139  fprintf(stderr, "xtrabackup: suspend-at-end is enabled.\n");
5140  }
5141  }
5142 
5143  /* cannot execute both for now */
5144  {
5145  int num = 0;
5146 
5147  if (xtrabackup_backup) num++;
5148  if (xtrabackup_stats) num++;
5149  if (xtrabackup_prepare) num++;
5150  if (num != 1) { /* !XOR (for now) */
5151  usage();
5152  exit(EXIT_FAILURE);
5153  }
5154  }
5155 
5156  /* --backup */
5157  if (xtrabackup_backup)
5158  xtrabackup_backup_func();
5159 
5160  /* --stats */
5161  if (xtrabackup_stats)
5162  xtrabackup_stats_func();
5163 
5164  /* --prepare */
5165  if (xtrabackup_prepare)
5166  xtrabackup_prepare_func();
5167 
5168  free(incremental_buffer_base);
5169 
5170  if (xtrabackup_tables) {
5171  /* free regexp */
5172  int i;
5173 
5174  for (i = 0; i < tables_regex_num; i++) {
5175  regfree(&tables_regex[i]);
5176  }
5177  ut_free(tables_regex);
5178  }
5179 
5180  if (xtrabackup_tables_file) {
5181  ulint i;
5182 
5183  /* free the hash elements */
5184  for (i = 0; i < hash_get_n_cells(tables_hash); i++) {
5185  xtrabackup_tables_t* table;
5186 
5187  table = (xtrabackup_tables_t*)
5188  HASH_GET_FIRST(tables_hash, i);
5189 
5190  while (table) {
5191  xtrabackup_tables_t* prev_table = table;
5192 
5193  table = (xtrabackup_tables_t*)
5194  HASH_GET_NEXT(name_hash, prev_table);
5195 
5196  HASH_DELETE(xtrabackup_tables_t, name_hash, tables_hash,
5197  ut_fold_string(prev_table->name), prev_table);
5198  free(prev_table);
5199  }
5200  }
5201 
5202  /* free tables_hash */
5203  hash_table_free(tables_hash);
5204  }
5205 
5206  exit(EXIT_SUCCESS);
5207 }