Drizzled Public API Documentation

mi_check.cc
1 /* Copyright (C) 2000-2006 MySQL AB
2 
3  This program is free software; you can redistribute it and/or modify
4  it under the terms of the GNU General Public License as published by
5  the Free Software Foundation; version 2 of the License.
6 
7  This program is distributed in the hope that it will be useful,
8  but WITHOUT ANY WARRANTY; without even the implied warranty of
9  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10  GNU General Public License for more details.
11 
12  You should have received a copy of the GNU General Public License
13  along with this program; if not, write to the Free Software
14  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
15 
16 /* Describe, check and repair of MyISAM tables */
17 
18 /*
19  About checksum calculation.
20 
21  There are two types of checksums. Table checksum and row checksum.
22 
23  Row checksum is an additional byte at the end of dynamic length
24  records. It must be calculated if the table is configured for them.
25  Otherwise they must not be used. The variable
26  MYISAM_SHARE::calc_checksum determines if row checksums are used.
27  MI_INFO::checksum is used as temporary storage during row handling.
28  For parallel repair we must assure that only one thread can use this
29  variable. There is no problem on the write side as this is done by one
30  thread only. But when checking a record after read this could go
31  wrong. But since all threads read through a common read buffer, it is
32  sufficient if only one thread checks it.
33 
34  Table checksum is an eight byte value in the header of the index file.
35  It can be calculated even if row checksums are not used. The variable
36  MI_CHECK::glob_crc is calculated over all records.
37  MI_SORT_PARAM::calc_checksum determines if this should be done. This
38  variable is not part of MI_CHECK because it must be set per thread for
39  parallel repair. The global glob_crc must be changed by one thread
40  only. And it is sufficient to calculate the checksum once only.
41 */
42 
43 #include "myisam_priv.h"
44 #include <drizzled/internal/m_string.h>
45 #include <stdarg.h>
46 #ifdef HAVE_SYS_VADVISE_H
47 #include <sys/vadvise.h>
48 #endif
49 #ifdef HAVE_SYS_TYPES
50 #include <sys/types.h>
51 #endif
52 #ifdef HAVE_SYS_MMAN_H
53 #include <sys/mman.h>
54 #endif
55 #include <drizzled/util/test.h>
56 #include <drizzled/error.h>
57 
58 #include <algorithm>
59 
60 using namespace std;
61 using namespace drizzled;
62 using namespace drizzled::internal;
63 
64 
65 #define my_off_t2double(A) ((double) (my_off_t) (A))
66 
67 /* Functions defined in this file */
68 
69 static int check_k_link(MI_CHECK *param, MI_INFO *info,uint32_t nr);
70 static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
71  my_off_t page, unsigned char *buff, ha_rows *keys,
72  ha_checksum *key_checksum, uint32_t level);
73 static uint32_t isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
74 static ha_checksum calc_checksum(ha_rows count);
75 static int writekeys(MI_SORT_PARAM *sort_param);
76 static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
77  my_off_t pagepos, int new_file);
78 int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
79 int sort_get_next_record(MI_SORT_PARAM *sort_param);
80 int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
81 int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
82 my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
83  unsigned char *key);
84 int sort_insert_key(MI_SORT_PARAM *sort_param,
85  register SORT_KEY_BLOCKS *key_block,
86  unsigned char *key, my_off_t prev_block);
87 int sort_delete_record(MI_SORT_PARAM *sort_param);
88 
89 /*static int flush_pending_blocks(MI_CHECK *param);*/
90 static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
91  uint32_t buffer_length);
92 static ha_checksum mi_byte_checksum(const unsigned char *buf, uint32_t length);
93 static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
94 
95 void myisamchk_init(MI_CHECK *param)
96 {
97  memset(param, 0, sizeof(*param));
98  param->opt_follow_links=1;
99  param->keys_in_use= ~(uint64_t) 0;
100  param->search_after_block=HA_OFFSET_ERROR;
101  param->auto_increment_value= 0;
102  param->use_buffers=USE_BUFFER_INIT;
103  param->read_buffer_length=READ_BUFFER_INIT;
104  param->write_buffer_length=READ_BUFFER_INIT;
105  param->sort_buffer_length=SORT_BUFFER_INIT;
106  param->sort_key_blocks=BUFFERS_WHEN_SORTING;
107  param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
108  param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
109  param->start_check_pos=0;
110  param->max_record_length= INT64_MAX;
111  param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
112  param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
113 }
114 
115  /* Check the status flags for the table */
116 
117 int chk_status(MI_CHECK *param, register MI_INFO *info)
118 {
119  MYISAM_SHARE *share=info->s;
120 
121  if (mi_is_crashed_on_repair(info))
122  mi_check_print_warning(param,
123  "Table is marked as crashed and last repair failed");
124  else if (mi_is_crashed(info))
125  mi_check_print_warning(param,
126  "Table is marked as crashed");
127  if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
128  {
129  /* Don't count this as a real warning, as check can correct this ! */
130  uint32_t save=param->warning_printed;
131  mi_check_print_warning(param,
132  share->state.open_count==1 ?
133  "%d client is using or hasn't closed the table properly" :
134  "%d clients are using or haven't closed the table properly",
135  share->state.open_count);
136  /* If this will be fixed by the check, forget the warning */
137  if (param->testflag & T_UPDATE_STATE)
138  param->warning_printed=save;
139  }
140  return 0;
141 }
142 
143  /* Check delete links */
144 
145 int chk_del(MI_CHECK *param, register MI_INFO *info, uint32_t test_flag)
146 {
147  uint32_t delete_link_length;
148  my_off_t empty, next_link, old_link= 0;
149  char buff[22],buff2[22];
150 
151  param->record_checksum=0;
152  delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 : info->s->rec_reflength+1);
153 
154  if (!(test_flag & T_SILENT))
155  puts("- check record delete-chain");
156 
157  next_link=info->s->state.dellink;
158  if (info->state->del == 0)
159  {
160  if (test_flag & T_VERBOSE)
161  {
162  puts("No recordlinks");
163  }
164  }
165  else
166  {
167  if (test_flag & T_VERBOSE)
168  printf("Recordlinks: ");
169  empty=0;
170  ha_rows i= info->state->del;
171  for (; i > 0 && next_link != HA_OFFSET_ERROR; i--)
172  {
173  if (*killed_ptr(param))
174  return(1);
175  if (test_flag & T_VERBOSE)
176  printf(" %9s",llstr(next_link,buff));
177  if (next_link >= info->state->data_file_length)
178  goto wrong;
179  if (my_pread(info->dfile, (unsigned char*) buff,delete_link_length, next_link,MYF(MY_NABP)))
180  {
181  if (test_flag & T_VERBOSE) puts("");
182  mi_check_print_error(param,"Can't read delete-link at filepos: %s",
183  llstr(next_link,buff));
184  return(1);
185  }
186  if (*buff != '\0')
187  {
188  if (test_flag & T_VERBOSE) puts("");
189  mi_check_print_error(param,"Record at pos: %s is not remove-marked",
190  llstr(next_link,buff));
191  goto wrong;
192  }
193  if (info->s->options & HA_OPTION_PACK_RECORD)
194  {
195  my_off_t prev_link=mi_sizekorr(buff+12);
196  if (empty && prev_link != old_link)
197  {
198  if (test_flag & T_VERBOSE) puts("");
199  mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
200  goto wrong;
201  }
202  old_link=next_link;
203  next_link=mi_sizekorr(buff+4);
204  empty+=mi_uint3korr(buff+1);
205  }
206  else
207  {
208  param->record_checksum+=(ha_checksum) next_link;
209  next_link=_mi_rec_pos(info->s,(unsigned char*) buff+1);
210  empty+=info->s->base.pack_reclength;
211  }
212  }
213  if (test_flag & T_VERBOSE)
214  puts("\n");
215  if (empty != info->state->empty)
216  {
217  mi_check_print_warning(param,
218  "Found %s deleted space in delete link chain. Should be %s",
219  llstr(empty,buff2),
220  llstr(info->state->empty,buff));
221  }
222  if (next_link != HA_OFFSET_ERROR)
223  {
224  mi_check_print_error(param,
225  "Found more than the expected %s deleted rows in delete link chain",
226  llstr(info->state->del, buff));
227  goto wrong;
228  }
229  if (i != 0)
230  {
231  mi_check_print_error(param, "Found %s deleted rows in delete link chain. Should be %s", llstr(info->state->del - i, buff2), llstr(info->state->del, buff));
232  goto wrong;
233  }
234  }
235  return 0;
236 
237 wrong:
238  param->testflag|=T_RETRY_WITHOUT_QUICK;
239  if (test_flag & T_VERBOSE) puts("");
240  mi_check_print_error(param,"record delete-link-chain corrupted");
241  return 1;
242 } /* chk_del */
243 
244 
245  /* Check delete links in index file */
246 
247 static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint32_t nr)
248 {
249  my_off_t next_link;
250  uint32_t block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
251  ha_rows records;
252  char llbuff[21], llbuff2[21];
253  unsigned char *buff;
254 
255  if (param->testflag & T_VERBOSE)
256  printf("block_size %4u:", block_size);
257 
258  next_link=info->s->state.key_del[nr];
259  records= (ha_rows) (info->state->key_file_length / block_size);
260  while (next_link != HA_OFFSET_ERROR && records > 0)
261  {
262  if (*killed_ptr(param))
263  return(1);
264  if (param->testflag & T_VERBOSE)
265  printf("%16s",llstr(next_link,llbuff));
266 
267  /* Key blocks must lay within the key file length entirely. */
268  if (next_link + block_size > info->state->key_file_length)
269  {
270  mi_check_print_error(param, "Invalid key block position: %s "
271  "key block size: %u file_length: %s",
272  llstr(next_link, llbuff), block_size,
273  llstr(info->state->key_file_length, llbuff2));
274  return(1);
275  }
276 
277  /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
278  if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
279  {
280  mi_check_print_error(param, "Mis-aligned key block: %s "
281  "minimum key block length: %u",
282  llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
283  return(1);
284  }
285 
286  /*
287  Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
288  If the key cache block size is smaller than block_size, we can so
289  avoid unecessary eviction of cache block.
290  */
291 
292  if (not pread(info->s->kfile, info->buff, MI_MIN_KEY_BLOCK_LENGTH, next_link))
293  {
294  mi_check_print_error(param, "key cache read error for block: %s", llstr(next_link,llbuff));
295  return(1);
296  }
297  buff= info->buff;
298  next_link=mi_sizekorr(buff);
299  records--;
300  param->key_file_blocks+=block_size;
301  }
302  if (param->testflag & T_VERBOSE)
303  {
304  if (next_link != HA_OFFSET_ERROR)
305  printf("%16s\n",llstr(next_link,llbuff));
306  else
307  puts("");
308  }
309  return (next_link != HA_OFFSET_ERROR);
310 } /* check_k_link */
311 
312 
313  /* Check sizes of files */
314 
315 int chk_size(MI_CHECK *param, register MI_INFO *info)
316 {
317  int error=0;
318  register my_off_t skr,size;
319  char buff[22],buff2[22];
320 
321  if (!(param->testflag & T_SILENT)) puts("- check file-size");
322 
323  size= lseek(info->s->kfile, 0, SEEK_END);
324  if ((skr=(my_off_t) info->state->key_file_length) != size)
325  {
326  /* Don't give error if file generated by myisampack */
327  if (skr > size && mi_is_any_key_active(info->s->state.key_map))
328  {
329  error=1;
330  mi_check_print_error(param,
331  "Size of indexfile is: %-8s Should be: %s",
332  llstr(size,buff), llstr(skr,buff2));
333  }
334  else
335  mi_check_print_warning(param,
336  "Size of indexfile is: %-8s Should be: %s",
337  llstr(size,buff), llstr(skr,buff2));
338  }
339  if (!(param->testflag & T_VERY_SILENT) &&
340  ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
341  uint64_t2double(info->state->key_file_length) >
342  uint64_t2double(info->s->base.margin_key_file_length)*0.9)
343  mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
344  llstr(info->state->key_file_length,buff),
345  llstr(info->s->base.max_key_file_length-1,buff));
346 
347  size=lseek(info->dfile,0L,SEEK_END);
348  skr=(my_off_t) info->state->data_file_length;
349  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
350  skr+= MEMMAP_EXTRA_MARGIN;
351 #ifdef USE_RELOC
352  if (info->data_file_type == STATIC_RECORD &&
353  skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
354  skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
355 #endif
356  if (skr != size)
357  {
358  info->state->data_file_length=size; /* Skip other errors */
359  if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
360  {
361  error=1;
362  mi_check_print_error(param,"Size of datafile is: %-9s Should be: %s",
363  llstr(size,buff), llstr(skr,buff2));
364  param->testflag|=T_RETRY_WITHOUT_QUICK;
365  }
366  else
367  {
368  mi_check_print_warning(param,
369  "Size of datafile is: %-9s Should be: %s",
370  llstr(size,buff), llstr(skr,buff2));
371  }
372  }
373  if (!(param->testflag & T_VERY_SILENT) &&
374  !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
375  uint64_t2double(info->state->data_file_length) >
376  (uint64_t2double(info->s->base.max_data_file_length)*0.9))
377  mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
378  llstr(info->state->data_file_length,buff),
379  llstr(info->s->base.max_data_file_length-1,buff2));
380  return(error);
381 } /* chk_size */
382 
383 
384  /* Check keys */
385 
386 int chk_key(MI_CHECK *param, register MI_INFO *info)
387 {
388  uint32_t key,found_keys=0,full_text_keys=0,result=0;
389  ha_rows keys;
390  ha_checksum old_record_checksum,init_checksum;
391  my_off_t all_keydata,all_totaldata,key_totlength,length;
392  ulong *rec_per_key_part;
393  MYISAM_SHARE *share=info->s;
394  MI_KEYDEF *keyinfo;
395  char buff[22],buff2[22];
396 
397  if (!(param->testflag & T_SILENT))
398  puts("- check key delete-chain");
399 
400  param->key_file_blocks=info->s->base.keystart;
401  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
402  if (check_k_link(param,info,key))
403  {
404  if (param->testflag & T_VERBOSE) puts("");
405  mi_check_print_error(param,"key delete-link-chain corrupted");
406  return(-1);
407  }
408 
409  if (!(param->testflag & T_SILENT)) puts("- check index reference");
410 
411  all_keydata=all_totaldata=key_totlength=0;
412  old_record_checksum=0;
413  init_checksum=param->record_checksum;
414  if (!(share->options &
415  (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
416  old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
417  share->base.pack_reclength;
418  rec_per_key_part= param->rec_per_key_part;
419  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
420  rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
421  {
422  param->key_crc[key]=0;
423  if (! mi_is_key_active(share->state.key_map, key))
424  {
425  /* Remember old statistics for key */
426  assert(rec_per_key_part >= param->rec_per_key_part);
427  memcpy(rec_per_key_part,
428  (share->state.rec_per_key_part +
429  (rec_per_key_part - param->rec_per_key_part)),
430  keyinfo->keysegs*sizeof(*rec_per_key_part));
431  continue;
432  }
433  found_keys++;
434 
435  param->record_checksum=init_checksum;
436 
437  memset(&param->unique_count, 0, sizeof(param->unique_count));
438  memset(&param->notnull_count, 0, sizeof(param->notnull_count));
439 
440  if ((!(param->testflag & T_SILENT)))
441  printf ("- check data record references index: %d\n",key+1);
442  if (share->state.key_root[key] == HA_OFFSET_ERROR && (info->state->records == 0))
443  goto do_stat;
444  if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
445  DFLT_INIT_HITS,info->buff,0))
446  {
447  mi_check_print_error(param,"Can't read indexpage from filepos: %s",
448  llstr(share->state.key_root[key],buff));
449  if (!(param->testflag & T_INFO))
450  return(-1);
451  result= UINT32_MAX;
452  continue;
453  }
454  param->key_file_blocks+=keyinfo->block_length;
455  keys=0;
456  param->keydata=param->totaldata=0;
457  param->key_blocks=0;
458  param->max_level=0;
459  if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
460  &keys, param->key_crc+key,1))
461  return(-1);
462  if(!(0))
463  {
464  if (keys != info->state->records)
465  {
466  mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
467  llstr(info->state->records,buff2));
468  if (!(param->testflag & T_INFO))
469  return(-1);
470  result= UINT32_MAX;
471  continue;
472  }
473  if (found_keys - full_text_keys == 1 &&
474  ((share->options &
475  (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
476  (param->testflag & T_DONT_CHECK_CHECKSUM)))
477  old_record_checksum=param->record_checksum;
478  else if (old_record_checksum != param->record_checksum)
479  {
480  if (key)
481  mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
482  key+1);
483  else
484  mi_check_print_error(param,"Key 1 doesn't point at all records");
485  if (!(param->testflag & T_INFO))
486  return(-1);
487  result= UINT32_MAX;
488  continue;
489  }
490  }
491  if ((uint) share->base.auto_key -1 == key)
492  {
493  /* Check that auto_increment key is bigger than max key value */
494  uint64_t auto_increment;
495  info->lastinx=key;
496  _mi_read_key_record(info, 0L, info->rec_buff);
497  auto_increment= retrieve_auto_increment(info, info->rec_buff);
498  if (auto_increment > info->s->state.auto_increment)
499  {
500  mi_check_print_warning(param, "Auto-increment value: %s is smaller "
501  "than max used value: %s",
502  llstr(info->s->state.auto_increment,buff2),
503  llstr(auto_increment, buff));
504  }
505  if (param->testflag & T_AUTO_INC)
506  {
507  set_if_bigger(info->s->state.auto_increment,
508  auto_increment);
509  set_if_bigger(info->s->state.auto_increment,
510  param->auto_increment_value);
511  }
512 
513  /* Check that there isn't a row with auto_increment = 0 in the table */
514  mi_extra(info,HA_EXTRA_KEYREAD,0);
515  memset(info->lastkey, 0, keyinfo->seg->length);
516  if (!mi_rkey(info, info->rec_buff, key, (const unsigned char*) info->lastkey,
517  (key_part_map)1, HA_READ_KEY_EXACT))
518  {
519  /* Don't count this as a real warning, as myisamchk can't correct it */
520  uint32_t save=param->warning_printed;
521  mi_check_print_warning(param, "Found row where the auto_increment "
522  "column has the value 0");
523  param->warning_printed=save;
524  }
525  mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
526  }
527 
528  length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
529  if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
530  printf("Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n",
531  key+1,
532  (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
533  (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
534  my_off_t2double(length)),
535  param->max_level);
536  all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
537 
538 do_stat:
539  if (param->testflag & T_STATISTICS)
540  update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
541  param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
542  param->notnull_count: NULL,
543  (uint64_t)info->state->records);
544  }
545  if (param->testflag & T_INFO)
546  {
547  if (all_totaldata != 0L && found_keys > 0)
548  printf("Total: Keyblocks used: %3d%% Packed: %4d%%\n\n",
549  (int) (my_off_t2double(all_keydata)*100.0/
550  my_off_t2double(all_totaldata)),
551  (int) ((my_off_t2double(key_totlength) -
552  my_off_t2double(all_keydata))*100.0/
553  my_off_t2double(key_totlength)));
554  else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
555  puts("");
556  }
557  if (param->key_file_blocks != info->state->key_file_length &&
558  param->keys_in_use != ~(uint64_t) 0)
559  mi_check_print_warning(param, "Some data are unreferenced in keyfile");
560  if (found_keys != full_text_keys)
561  param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */
562  else
563  param->record_checksum=0;
564  return(result);
565 } /* chk_key */
566 
567 
568 static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
569  my_off_t page, unsigned char *buff, ha_rows *keys,
570  ha_checksum *key_checksum, uint32_t level)
571 {
572  char llbuff[22],llbuff2[22];
573 
574  /* Key blocks must lay within the key file length entirely. */
575  if (page + keyinfo->block_length > info->state->key_file_length)
576  {
577  /* Give it a chance to fit in the real file size. */
578  my_off_t max_length= lseek(info->s->kfile, 0, SEEK_END);
579  mi_check_print_error(param, "Invalid key block position: %s "
580  "key block size: %u file_length: %s",
581  llstr(page, llbuff), keyinfo->block_length,
582  llstr(info->state->key_file_length, llbuff2));
583  if (page + keyinfo->block_length > max_length)
584  goto err;
585  /* Fix the remebered key file length. */
586  info->state->key_file_length= (max_length &
587  ~ (my_off_t) (keyinfo->block_length - 1));
588  }
589 
590  /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
591  if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
592  {
593  mi_check_print_error(param, "Mis-aligned key block: %s "
594  "minimum key block length: %u",
595  llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
596  goto err;
597  }
598 
599  if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
600  {
601  mi_check_print_error(param,"Can't read key from filepos: %s",
602  llstr(page,llbuff));
603  goto err;
604  }
605  param->key_file_blocks+=keyinfo->block_length;
606  if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
607  goto err;
608 
609  return(0);
610 
611 err:
612  return(1);
613 }
614 
615 
616 /*
617  "Ignore NULLs" statistics collection method: process first index tuple.
618 
619  SYNOPSIS
620  mi_collect_stats_nonulls_first()
621  keyseg IN Array of key part descriptions
622  notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
623  tuples that don't contain NULLs)
624  key IN Key values tuple
625 
626  DESCRIPTION
627  Process the first index tuple - find out which prefix tuples don't
628  contain NULLs, and update the array of notnull counters accordingly.
629 */
630 
631 static
632 void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, uint64_t *notnull,
633  unsigned char *key)
634 {
635  uint32_t first_null, kp;
636  first_null= ha_find_null(keyseg, key) - keyseg;
637  /*
638  All prefix tuples that don't include keypart_{first_null} are not-null
639  tuples (and all others aren't), increment counters for them.
640  */
641  for (kp= 0; kp < first_null; kp++)
642  notnull[kp]++;
643 }
644 
645 
646 /*
647  "Ignore NULLs" statistics collection method: process next index tuple.
648 
649  SYNOPSIS
650  mi_collect_stats_nonulls_next()
651  keyseg IN Array of key part descriptions
652  notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
653  tuples that don't contain NULLs)
654  prev_key IN Previous key values tuple
655  last_key IN Next key values tuple
656 
657  DESCRIPTION
658  Process the next index tuple:
659  1. Find out which prefix tuples of last_key don't contain NULLs, and
660  update the array of notnull counters accordingly.
661  2. Find the first keypart number where the prev_key and last_key tuples
662  are different(A), or last_key has NULL value(B), and return it, so the
663  caller can count number of unique tuples for each key prefix. We don't
664  need (B) to be counted, and that is compensated back in
665  update_key_parts().
666 
667  RETURN
668  1 + number of first keypart where values differ or last_key tuple has NULL
669 */
670 
671 static
672 int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, uint64_t *notnull,
673  unsigned char *prev_key, unsigned char *last_key)
674 {
675  uint32_t diffs[2];
676  uint32_t first_null_seg, kp;
677  HA_KEYSEG *seg;
678 
679  /*
680  Find the first keypart where values are different or either of them is
681  NULL. We get results in diffs array:
682  diffs[0]= 1 + number of first different keypart
683  diffs[1]=offset: (last_key + diffs[1]) points to first value in
684  last_key that is NULL or different from corresponding
685  value in prev_key.
686  */
687  ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
688  SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
689  seg= keyseg + diffs[0] - 1;
690 
691  /* Find first NULL in last_key */
692  first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
693  for (kp= 0; kp < first_null_seg; kp++)
694  notnull[kp]++;
695 
696  /*
697  Return 1+ number of first key part where values differ. Don't care if
698  these were NULLs and not .... We compensate for that in
699  update_key_parts.
700  */
701  return diffs[0];
702 }
703 
704 
705  /* Check if index is ok */
706 
707 static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
708  my_off_t page, unsigned char *buff, ha_rows *keys,
709  ha_checksum *key_checksum, uint32_t level)
710 {
711  int flag;
712  uint32_t used_length,comp_flag,nod_flag,key_length=0;
713  unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*endpos;
714  my_off_t next_page,record;
715  char llbuff[22];
716  uint32_t diff_pos[2];
717 
718  if (!(temp_buff=(unsigned char*) malloc(keyinfo->block_length)))
719  {
720  mi_check_print_error(param,"Not enough memory for keyblock");
721  return(-1);
722  }
723 
724  if (keyinfo->flag & HA_NOSAME)
725  comp_flag=SEARCH_FIND | SEARCH_UPDATE; /* Not real duplicates */
726  else
727  comp_flag=SEARCH_SAME; /* Keys in positionorder */
728  nod_flag=mi_test_if_nod(buff);
729  used_length=mi_getint(buff);
730  keypos=buff+2+nod_flag;
731  endpos=buff+used_length;
732 
733  param->keydata+=used_length; param->totaldata+=keyinfo->block_length; /* INFO */
734  param->key_blocks++;
735  if (level > param->max_level)
736  param->max_level=level;
737 
738  if (used_length > keyinfo->block_length)
739  {
740  mi_check_print_error(param,"Wrong pageinfo at page: %s",
741  llstr(page,llbuff));
742  goto err;
743  }
744  for ( ;; )
745  {
746  if (*killed_ptr(param))
747  goto err;
748  memcpy(info->lastkey,key,key_length);
749  info->lastkey_length=key_length;
750  if (nod_flag)
751  {
752  next_page=_mi_kpos(nod_flag,keypos);
753  if (chk_index_down(param,info,keyinfo,next_page,
754  temp_buff,keys,key_checksum,level+1))
755  goto err;
756  }
757  if (keypos >= endpos ||
758  (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
759  break;
760  assert(key_length <= sizeof(key));
761  if (keypos > endpos)
762  {
763  mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
764  goto err;
765  }
766  if ((*keys)++ &&
767  (flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
768  comp_flag, diff_pos)) >=0)
769  {
770  if (comp_flag & SEARCH_FIND && flag == 0)
771  mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
772  else
773  mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
774  goto err;
775  }
776  if (param->testflag & T_STATISTICS)
777  {
778  if (*keys != 1L) /* not first_key */
779  {
780  if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
781  ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
782  SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
783  diff_pos);
784  else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
785  {
786  diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg,
787  param->notnull_count,
788  info->lastkey, key);
789  }
790  param->unique_count[diff_pos[0]-1]++;
791  }
792  else
793  {
794  if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
795  mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
796  key);
797  }
798  }
799  (*key_checksum)+= mi_byte_checksum((unsigned char*) key,
800  key_length- info->s->rec_reflength);
801  record= _mi_dpos(info,0,key+key_length);
802  if (record >= info->state->data_file_length)
803  {
804  mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
805  goto err;
806  }
807  param->record_checksum+=(ha_checksum) record;
808  }
809  if (keypos != endpos)
810  {
811  mi_check_print_error(param,"Keyblock size at page %s is not correct. Block length: %d key length: %d",
812  llstr(page,llbuff), used_length, (keypos - buff));
813  goto err;
814  }
815  free(temp_buff);
816  return(0);
817  err:
818  free(temp_buff);
819  return(1);
820 } /* chk_index */
821 
822 
823  /* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */
824 
825 static ha_checksum calc_checksum(ha_rows count)
826 {
827  uint64_t sum,a,b;
828 
829  sum=0;
830  a=count; b=count+1;
831  if (a & 1)
832  b>>=1;
833  else
834  a>>=1;
835  while (b)
836  {
837  if (b & 1)
838  sum+=a;
839  a<<=1; b>>=1;
840  }
841  return((ha_checksum) sum);
842 } /* calc_checksum */
843 
844 
845  /* Calc length of key in normal isam */
846 
847 static uint32_t isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
848 {
849  uint32_t length;
850  HA_KEYSEG *keyseg;
851 
852  length= info->s->rec_reflength;
853  for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
854  length+= keyseg->length;
855 
856  return(length);
857 } /* key_length */
858 
859 
860  /* Check that record-link is ok */
861 
862 int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
863 {
864  int error,got_error,flag;
865  uint key, left_length= 0, b_type;
866  ha_rows records, del_blocks;
867  my_off_t used, empty, pos, splits, start_recpos= 0,
868  del_length, link_used, start_block;
869  unsigned char *record= NULL, *to= NULL;
870  char llbuff[22],llbuff2[22],llbuff3[22];
871  ha_checksum intern_record_checksum;
872  ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
873  MI_KEYDEF *keyinfo;
874  MI_BLOCK_INFO block_info;
875 
876  if (!(param->testflag & T_SILENT))
877  {
878  if (extend)
879  puts("- check records and index references");
880  else
881  puts("- check record links");
882  }
883 
884  if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
885  {
886  mi_check_print_error(param,"Not enough memory for record");
887  return(-1);
888  }
889  records=del_blocks=0;
890  used=link_used=splits=del_length=0;
891  intern_record_checksum=param->glob_crc=0;
892  got_error=error=0;
893  empty=info->s->pack.header_length;
894 
895  pos= param->read_cache.tell();
896  memset(key_checksum, 0, info->s->base.keys * sizeof(key_checksum[0]));
897  while (pos < info->state->data_file_length)
898  {
899  if (*killed_ptr(param))
900  goto err2;
901  switch (info->s->data_file_type) {
902  case STATIC_RECORD:
903  if (param->read_cache.read(record, info->s->base.pack_reclength))
904  goto err;
905  start_recpos=pos;
906  pos+=info->s->base.pack_reclength;
907  splits++;
908  if (*record == '\0')
909  {
910  del_blocks++;
911  del_length+=info->s->base.pack_reclength;
912  continue; /* Record removed */
913  }
914  param->glob_crc+= mi_static_checksum(info,record);
915  used+=info->s->base.pack_reclength;
916  break;
917  case DYNAMIC_RECORD:
918  flag=block_info.second_read=0;
919  block_info.next_filepos=pos;
920  do
921  {
922  if (_mi_read_cache(&param->read_cache,(unsigned char*) block_info.header,
923  (start_block=block_info.next_filepos),
924  sizeof(block_info.header),
925  (flag ? 0 : READING_NEXT) | READING_HEADER))
926  goto err;
927  if (start_block & (MI_DYN_ALIGN_SIZE-1))
928  {
929  mi_check_print_error(param,"Wrong aligned block at %s",
930  llstr(start_block,llbuff));
931  goto err2;
932  }
933  b_type=_mi_get_block_info(&block_info,-1,start_block);
934  if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
935  BLOCK_FATAL_ERROR))
936  {
937  if (b_type & BLOCK_SYNC_ERROR)
938  {
939  if (flag)
940  {
941  mi_check_print_error(param,"Unexpected byte: %d at link: %s",
942  (int) block_info.header[0],
943  llstr(start_block,llbuff));
944  goto err2;
945  }
946  pos=block_info.filepos+block_info.block_len;
947  goto next;
948  }
949  if (b_type & BLOCK_DELETED)
950  {
951  if (block_info.block_len < info->s->base.min_block_length)
952  {
953  mi_check_print_error(param,
954  "Deleted block with impossible length %lu at %s",
955  block_info.block_len,llstr(pos,llbuff));
956  goto err2;
957  }
958  if ((block_info.next_filepos != HA_OFFSET_ERROR &&
959  block_info.next_filepos >= info->state->data_file_length) ||
960  (block_info.prev_filepos != HA_OFFSET_ERROR &&
961  block_info.prev_filepos >= info->state->data_file_length))
962  {
963  mi_check_print_error(param,"Delete link points outside datafile at %s",
964  llstr(pos,llbuff));
965  goto err2;
966  }
967  del_blocks++;
968  del_length+=block_info.block_len;
969  pos=block_info.filepos+block_info.block_len;
970  splits++;
971  goto next;
972  }
973  mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
974  block_info.header[0],block_info.header[1],
975  block_info.header[2],
976  llstr(start_block,llbuff));
977  goto err2;
978  }
979  if (info->state->data_file_length < block_info.filepos+
980  block_info.block_len)
981  {
982  mi_check_print_error(param,
983  "Recordlink that points outside datafile at %s",
984  llstr(pos,llbuff));
985  got_error=1;
986  break;
987  }
988  splits++;
989  if (!flag++) /* First block */
990  {
991  start_recpos=pos;
992  pos=block_info.filepos+block_info.block_len;
993  if (block_info.rec_len > (uint) info->s->base.max_pack_length)
994  {
995  mi_check_print_error(param,"Found too long record (%lu) at %s",
996  (ulong) block_info.rec_len,
997  llstr(start_recpos,llbuff));
998  got_error=1;
999  break;
1000  }
1001  if (info->s->base.blobs)
1002  {
1003  if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1004  &info->rec_buff)))
1005  {
1006  mi_check_print_error(param,
1007  "Not enough memory (%lu) for blob at %s",
1008  (ulong) block_info.rec_len,
1009  llstr(start_recpos,llbuff));
1010  got_error=1;
1011  break;
1012  }
1013  }
1014  else
1015  to= info->rec_buff;
1016  left_length=block_info.rec_len;
1017  }
1018  if (left_length < block_info.data_len)
1019  {
1020  mi_check_print_error(param,"Found too long record (%lu) at %s",
1021  (ulong) block_info.data_len,
1022  llstr(start_recpos,llbuff));
1023  got_error=1;
1024  break;
1025  }
1026  if (_mi_read_cache(&param->read_cache,(unsigned char*) to,block_info.filepos,
1027  (uint) block_info.data_len,
1028  flag == 1 ? READING_NEXT : 0))
1029  goto err;
1030  to+=block_info.data_len;
1031  link_used+= block_info.filepos-start_block;
1032  used+= block_info.filepos - start_block + block_info.data_len;
1033  empty+=block_info.block_len-block_info.data_len;
1034  left_length-=block_info.data_len;
1035  if (left_length)
1036  {
1037  if (b_type & BLOCK_LAST)
1038  {
1039  mi_check_print_error(param,
1040  "Wrong record length %s of %s at %s",
1041  llstr(block_info.rec_len-left_length,llbuff),
1042  llstr(block_info.rec_len, llbuff2),
1043  llstr(start_recpos,llbuff3));
1044  got_error=1;
1045  break;
1046  }
1047  if (info->state->data_file_length < block_info.next_filepos)
1048  {
1049  mi_check_print_error(param,
1050  "Found next-recordlink that points outside datafile at %s",
1051  llstr(block_info.filepos,llbuff));
1052  got_error=1;
1053  break;
1054  }
1055  }
1056  } while (left_length);
1057  if (! got_error)
1058  {
1059  if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
1060  MY_FILE_ERROR)
1061  {
1062  mi_check_print_error(param,"Found wrong record at %s",
1063  llstr(start_recpos,llbuff));
1064  got_error=1;
1065  }
1066  else
1067  {
1068  info->checksum=mi_checksum(info,record);
1069  if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
1070  {
1071  if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
1072  test(info->s->calc_checksum)))
1073  {
1074  mi_check_print_error(param,"Found wrong packed record at %s",
1075  llstr(start_recpos,llbuff));
1076  got_error=1;
1077  }
1078  }
1079  if (!got_error)
1080  param->glob_crc+= info->checksum;
1081  }
1082  }
1083  else if (!flag)
1084  pos=block_info.filepos+block_info.block_len;
1085  break;
1086  case COMPRESSED_RECORD:
1087  case BLOCK_RECORD:
1088  assert(0); /* Impossible */
1089  } /* switch */
1090  if (! got_error)
1091  {
1092  intern_record_checksum+=(ha_checksum) start_recpos;
1093  records++;
1094  if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
1095  {
1096  printf("%s\r", llstr(records,llbuff)); fflush(stdout);
1097  }
1098 
1099  /* Check if keys match the record */
1100 
1101  for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
1102  key++,keyinfo++)
1103  {
1104  if (mi_is_key_active(info->s->state.key_map, key))
1105  {
1106  {
1107  uint32_t key_length=_mi_make_key(info,key,info->lastkey,record,
1108  start_recpos);
1109  if (extend)
1110  {
1111  /* We don't need to lock the key tree here as we don't allow
1112  concurrent threads when running myisamchk
1113  */
1114  int search_result=
1115  _mi_search(info,keyinfo,info->lastkey,key_length,
1116  SEARCH_SAME, info->s->state.key_root[key]);
1117  if (search_result)
1118  {
1119  mi_check_print_error(param,"Record at: %10s "
1120  "Can't find key for index: %2d",
1121  llstr(start_recpos,llbuff),key+1);
1122  if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1123  goto err2;
1124  }
1125  }
1126  else
1127  key_checksum[key]+=mi_byte_checksum((unsigned char*) info->lastkey,
1128  key_length);
1129  }
1130  }
1131  }
1132  }
1133  else
1134  {
1135  got_error=0;
1136  if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1137  goto err2;
1138  }
1139  next:; /* Next record */
1140  }
1141  if (param->testflag & T_WRITE_LOOP)
1142  {
1143  fputs(" \r",stdout); fflush(stdout);
1144  }
1145  if (records != info->state->records)
1146  {
1147  mi_check_print_error(param,"Record-count is not ok; is %-10s Should be: %s",
1148  llstr(records,llbuff), llstr(info->state->records,llbuff2));
1149  error=1;
1150  }
1151  else if (param->record_checksum &&
1152  param->record_checksum != intern_record_checksum)
1153  {
1154  mi_check_print_error(param,
1155  "Keypointers and record positions doesn't match");
1156  error=1;
1157  }
1158  else if (param->glob_crc != info->state->checksum &&
1159  (info->s->options &
1160  (HA_OPTION_COMPRESS_RECORD)))
1161  {
1162  mi_check_print_warning(param,
1163  "Record checksum is not the same as checksum stored in the index file\n");
1164  error=1;
1165  }
1166  else if (!extend)
1167  {
1168  for (key=0 ; key < info->s->base.keys; key++)
1169  {
1170  if (key_checksum[key] != param->key_crc[key])
1171  {
1172  mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
1173  key+1);
1174  error=1;
1175  }
1176  }
1177  }
1178 
1179  if (del_length != info->state->empty)
1180  {
1181  mi_check_print_warning(param,
1182  "Found %s deleted space. Should be %s",
1183  llstr(del_length,llbuff2),
1184  llstr(info->state->empty,llbuff));
1185  }
1186  if (used+empty+del_length != info->state->data_file_length)
1187  {
1188  mi_check_print_warning(param,
1189  "Found %s record-data and %s unused data and %s deleted-data",
1190  llstr(used,llbuff),llstr(empty,llbuff2),
1191  llstr(del_length,llbuff3));
1192  mi_check_print_warning(param,
1193  "Total %s, Should be: %s",
1194  llstr((used+empty+del_length),llbuff),
1195  llstr(info->state->data_file_length,llbuff2));
1196  }
1197  if (del_blocks != info->state->del)
1198  {
1199  mi_check_print_warning(param,
1200  "Found %10s deleted blocks Should be: %s",
1201  llstr(del_blocks,llbuff),
1202  llstr(info->state->del,llbuff2));
1203  }
1204  if (splits != info->s->state.split)
1205  {
1206  mi_check_print_warning(param,
1207  "Found %10s parts Should be: %s parts",
1208  llstr(splits,llbuff),
1209  llstr(info->s->state.split,llbuff2));
1210  }
1211  if (param->testflag & T_INFO)
1212  {
1213  if (param->warning_printed || param->error_printed)
1214  puts("");
1215  if (used != 0 && ! param->error_printed)
1216  {
1217  printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n",
1218  llstr(records,llbuff), (long)((used-link_used)/records),
1219  (info->s->base.blobs ? 0.0 :
1220  (uint64_t2double((uint64_t) info->s->base.reclength*records)-
1221  my_off_t2double(used))/
1222  uint64_t2double((uint64_t) info->s->base.reclength*records)*100.0));
1223  printf("Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n",
1224  (uint64_t2double(used-link_used)/uint64_t2double(used-link_used+empty)*100.0),
1225  (!records ? 100 : (int) (uint64_t2double(del_length+empty)/
1226  my_off_t2double(used)*100.0)),
1227  uint64_t2double(splits - del_blocks) / records);
1228  }
1229  printf("Record blocks:%12s Delete blocks:%10s\n",
1230  llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
1231  printf("Record data: %12s Deleted data: %10s\n",
1232  llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
1233  printf("Lost space: %12s Linkdata: %10s\n",
1234  llstr(empty,llbuff),llstr(link_used,llbuff2));
1235  }
1236  free(mi_get_rec_buff_ptr(info, record));
1237  return (error);
1238  err:
1239  mi_check_print_error(param,"got error: %d when reading datafile at record: %s",errno, llstr(records,llbuff));
1240  err2:
1241  free(mi_get_rec_buff_ptr(info, record));
1242  param->testflag|=T_RETRY_WITHOUT_QUICK;
1243  return(1);
1244 } /* chk_data_link */
1245 
1246 
1300 static void mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, bool force)
1301 {
1302  MYISAM_SHARE *share= info->s;
1303  MI_STATE_INFO *state= &share->state;
1304  uint32_t i;
1305 
1306  /*
1307  If any of the disabled indexes has a key block assigned, we must
1308  drop and recreate all indexes to avoid losing index blocks.
1309 
1310  If we want to recreate disabled indexes only _and_ all of these
1311  indexes are empty, we don't need to recreate the existing indexes.
1312  */
1313  if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
1314  {
1315  for (i= 0; i < share->base.keys; i++)
1316  {
1317  if ((state->key_root[i] != HA_OFFSET_ERROR) &&
1318  !mi_is_key_active(state->key_map, i))
1319  {
1320  /*
1321  This index has at least one key block and it is disabled.
1322  We would lose its block(s) if would just recreate it.
1323  So we need to drop and recreate all indexes.
1324  */
1325  break;
1326  }
1327  }
1328  if (i >= share->base.keys)
1329  {
1330  /*
1331  All of the disabled indexes are empty. We can just recreate them.
1332  Flush dirty blocks of this index file from key cache and remove
1333  all blocks of this index file from key cache.
1334  */
1335  return;
1336  }
1337  /*
1338  We do now drop all indexes and declare them disabled. With the
1339  T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
1340  disabled indexes and enable them.
1341  */
1342  mi_clear_all_keys_active(state->key_map);
1343  }
1344 
1345  /* Clear index root block pointers. */
1346  for (i= 0; i < share->base.keys; i++)
1347  state->key_root[i]= HA_OFFSET_ERROR;
1348 
1349  /* Clear the delete chains. */
1350  for (i= 0; i < state->header.max_block_size_index; i++)
1351  state->key_del[i]= HA_OFFSET_ERROR;
1352 
1353  /* Reset index file length to end of index file header. */
1354  info->state->key_file_length= share->base.keystart;
1355 }
1356 
1357 
1358  /* Recover old table by reading each record and writing all keys */
1359  /* Save new datafile-name in temp_filename */
1360 
1361 int mi_repair(MI_CHECK *param, register MI_INFO *info,
1362  char * name, int rep_quick)
1363 {
1364  int error,got_error;
1365  ha_rows start_records,new_header_length;
1366  my_off_t del;
1367  int new_file;
1368  MYISAM_SHARE *share=info->s;
1369  char llbuff[22],llbuff2[22];
1370  SORT_INFO sort_info;
1371  MI_SORT_PARAM sort_param;
1372 
1373  memset(&sort_info, 0, sizeof(sort_info));
1374  memset(&sort_param, 0, sizeof(sort_param));
1375  start_records=info->state->records;
1376  new_header_length=(param->testflag & T_UNPACK) ? 0L :
1377  share->pack.header_length;
1378  got_error=1;
1379  new_file= -1;
1380  sort_param.sort_info=&sort_info;
1381 
1382  if (!(param->testflag & T_SILENT))
1383  {
1384  printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
1385  printf("Data records: %s\n", llstr(info->state->records,llbuff));
1386  }
1387  param->testflag|=T_REP; /* for easy checking */
1388 
1389  if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
1390  param->testflag|=T_CALC_CHECKSUM;
1391 
1392  if (!param->using_global_keycache)
1393  assert(0);
1394 
1395  if (param->read_cache.init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1396  {
1397  memset(&info->rec_cache, 0, sizeof(info->rec_cache));
1398  goto err;
1399  }
1400  if (not rep_quick)
1401  {
1402  if (info->rec_cache.init_io_cache(-1, (uint) param->write_buffer_length, WRITE_CACHE, new_header_length, 1, MYF(MY_WME | MY_WAIT_IF_FULL)))
1403  {
1404  goto err;
1405  }
1406  }
1407  info->opt_flag|=WRITE_CACHE_USED;
1408  if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
1409  !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
1410  {
1411  mi_check_print_error(param, "Not enough memory for extra record");
1412  goto err;
1413  }
1414 
1415  if (!rep_quick)
1416  {
1417  /* Get real path for data file */
1418  if ((new_file=my_create(internal::fn_format(param->temp_filename,
1419  share->data_file_name, "",
1420  DATA_TMP_EXT, 2+4),
1421  0,param->tmpfile_createflag,
1422  MYF(0))) < 0)
1423  {
1424  mi_check_print_error(param,"Can't create new tempfile: '%s'",
1425  param->temp_filename);
1426  goto err;
1427  }
1428  if (new_header_length &&
1429  filecopy(param,new_file,info->dfile,0L,new_header_length,
1430  "datafile-header"))
1431  goto err;
1432  info->s->state.dellink= HA_OFFSET_ERROR;
1433  info->rec_cache.file=new_file;
1434  if (param->testflag & T_UNPACK)
1435  {
1436  share->options&= ~HA_OPTION_COMPRESS_RECORD;
1437  mi_int2store(share->state.header.options,share->options);
1438  }
1439  }
1440  sort_info.info=info;
1441  sort_info.param = param;
1442  sort_param.read_cache=param->read_cache;
1443  sort_param.pos=sort_param.max_pos=share->pack.header_length;
1444  sort_param.filepos=new_header_length;
1445  param->read_cache.end_of_file=sort_info.filelength=
1446  lseek(info->dfile,0L,SEEK_END);
1447  sort_info.dupp=0;
1448  sort_param.fix_datafile= (bool) (! rep_quick);
1449  sort_param.master=1;
1450  sort_info.max_records= ~(ha_rows) 0;
1451 
1452  set_data_file_type(&sort_info, share);
1453  del=info->state->del;
1454  info->state->records=info->state->del=share->state.split=0;
1455  info->state->empty=0;
1456  param->glob_crc=0;
1457  if (param->testflag & T_CALC_CHECKSUM)
1458  sort_param.calc_checksum= 1;
1459 
1460  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1461 
1462  /* This function always recreates all enabled indexes. */
1463  if (param->testflag & T_CREATE_MISSING_KEYS)
1464  mi_set_all_keys_active(share->state.key_map, share->base.keys);
1465  mi_drop_all_indexes(param, info, true);
1466 
1467  lock_memory(param); /* Everything is alloced */
1468 
1469  /* Re-create all keys, which are set in key_map. */
1470  while (!(error=sort_get_next_record(&sort_param)))
1471  {
1472  if (writekeys(&sort_param))
1473  {
1474  if (errno != HA_ERR_FOUND_DUPP_KEY)
1475  goto err;
1476  mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1477  info->errkey+1,
1478  llstr(sort_param.start_recpos,llbuff),
1479  llstr(info->dupp_key_pos,llbuff2));
1480  if (param->testflag & T_VERBOSE)
1481  {
1482  _mi_make_key(info,(uint) info->errkey,info->lastkey,
1483  sort_param.record,0L);
1484  }
1485  sort_info.dupp++;
1486  if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
1487  {
1488  param->testflag|=T_RETRY_WITHOUT_QUICK;
1489  param->error_printed=1;
1490  goto err;
1491  }
1492  continue;
1493  }
1494  if (sort_write_record(&sort_param))
1495  goto err;
1496  }
1497  if (error > 0 || write_data_suffix(&sort_info, (bool)!rep_quick) ||
1498  info->rec_cache.flush() || param->read_cache.error < 0)
1499  goto err;
1500 
1501  if (param->testflag & T_WRITE_LOOP)
1502  {
1503  fputs(" \r",stdout); fflush(stdout);
1504  }
1505  if (ftruncate(share->kfile, info->state->key_file_length))
1506  {
1507  mi_check_print_warning(param,
1508  "Can't change size of indexfile, error: %d",
1509  errno);
1510  goto err;
1511  }
1512 
1513  if (rep_quick && del+sort_info.dupp != info->state->del)
1514  {
1515  mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
1516  mi_check_print_error(param,"Run recovery again without -q");
1517  got_error=1;
1518  param->retry_repair=1;
1519  param->testflag|=T_RETRY_WITHOUT_QUICK;
1520  goto err;
1521  }
1522  if (param->testflag & T_SAFE_REPAIR)
1523  {
1524  /* Don't repair if we loosed more than one row */
1525  if (info->state->records+1 < start_records)
1526  {
1527  info->state->records=start_records;
1528  got_error=1;
1529  goto err;
1530  }
1531  }
1532 
1533  if (!rep_quick)
1534  {
1535  internal::my_close(info->dfile,MYF(0));
1536  info->dfile=new_file;
1537  info->state->data_file_length=sort_param.filepos;
1538  share->state.version=(ulong) time((time_t*) 0); /* Force reopen */
1539  }
1540  else
1541  {
1542  info->state->data_file_length=sort_param.max_pos;
1543  }
1544  if (param->testflag & T_CALC_CHECKSUM)
1545  info->state->checksum=param->glob_crc;
1546 
1547  if (!(param->testflag & T_SILENT))
1548  {
1549  if (start_records != info->state->records)
1550  printf("Data records: %s\n", llstr(info->state->records,llbuff));
1551  if (sort_info.dupp)
1552  mi_check_print_warning(param,
1553  "%s records have been removed",
1554  llstr(sort_info.dupp,llbuff));
1555  }
1556 
1557  got_error=0;
1558  /* If invoked by external program that uses thr_lock */
1559  if (&share->state.state != info->state)
1560  memcpy( &share->state.state, info->state, sizeof(*info->state));
1561 
1562 err:
1563  if (!got_error)
1564  {
1565  /* Replace the actual file with the temporary file */
1566  if (new_file >= 0)
1567  {
1568  internal::my_close(new_file,MYF(0));
1569  info->dfile=new_file= -1;
1570  if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1571  DATA_TMP_EXT, share->base.raid_chunks,
1572  MYF(0)) ||
1573  mi_open_datafile(info,share,-1))
1574  got_error=1;
1575  }
1576  }
1577  if (got_error)
1578  {
1579  if (! param->error_printed)
1580  mi_check_print_error(param,"%d for record at pos %s",errno,
1581  llstr(sort_param.start_recpos,llbuff));
1582  if (new_file >= 0)
1583  {
1584  internal::my_close(new_file,MYF(0));
1585  my_delete(param->temp_filename, MYF(MY_WME));
1586  info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
1587  }
1588  mi_mark_crashed_on_repair(info);
1589  }
1590 
1591  void * rec_buff_ptr= NULL;
1592  rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
1593  free(rec_buff_ptr);
1594  rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
1595  free(rec_buff_ptr);
1596  rec_buff_ptr= NULL;
1597 
1598  free(sort_info.buff);
1599  param->read_cache.end_io_cache();
1600  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
1601  info->rec_cache.end_io_cache();
1602  if (not got_error && param->testflag & T_UNPACK)
1603  {
1604  share->state.header.options[0]&= (unsigned char) ~HA_OPTION_COMPRESS_RECORD;
1605  share->pack.header_length=0;
1606  share->data_file_type=sort_info.new_data_file_type;
1607  }
1608  share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
1609  STATE_NOT_ANALYZED);
1610  return(got_error);
1611 }
1612 
1613 
1614 /* Uppate keyfile when doing repair */
1615 
1616 static int writekeys(MI_SORT_PARAM *sort_param)
1617 {
1618  register uint32_t i;
1619  unsigned char *key;
1620  MI_INFO *info= sort_param->sort_info->info;
1621  unsigned char *buff= sort_param->record;
1622  my_off_t filepos= sort_param->filepos;
1623 
1624  key=info->lastkey+info->s->base.max_key_length;
1625  for (i=0 ; i < info->s->base.keys ; i++)
1626  {
1627  if (mi_is_key_active(info->s->state.key_map, i))
1628  {
1629  {
1630  uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
1631  if (_mi_ck_write(info,i,key,key_length))
1632  goto err;
1633  }
1634  }
1635  }
1636  return(0);
1637 
1638  err:
1639  if (errno == HA_ERR_FOUND_DUPP_KEY)
1640  {
1641  info->errkey=(int) i; /* This key was found */
1642  while ( i-- > 0 )
1643  {
1644  if (mi_is_key_active(info->s->state.key_map, i))
1645  {
1646  {
1647  uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
1648  if (_mi_ck_delete(info,i,key,key_length))
1649  break;
1650  }
1651  }
1652  }
1653  }
1654  /* Remove checksum that was added to glob_crc in sort_get_next_record */
1655  if (sort_param->calc_checksum)
1656  sort_param->sort_info->param->glob_crc-= info->checksum;
1657  return(-1);
1658 } /* writekeys */
1659 
1660 
1661  /* Change all key-pointers that points to a records */
1662 
1663 int movepoint(register MI_INFO *info, unsigned char *record, my_off_t oldpos,
1664  my_off_t newpos, uint32_t prot_key)
1665 {
1666  register uint32_t i;
1667  unsigned char *key;
1668  uint32_t key_length;
1669 
1670  key=info->lastkey+info->s->base.max_key_length;
1671  for (i=0 ; i < info->s->base.keys; i++)
1672  {
1673  if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
1674  {
1675  key_length=_mi_make_key(info,i,key,record,oldpos);
1676  if (info->s->keyinfo[i].flag & HA_NOSAME)
1677  { /* Change pointer direct */
1678  uint32_t nod_flag;
1679  MI_KEYDEF *keyinfo;
1680  keyinfo=info->s->keyinfo+i;
1681  if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
1682  (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
1683  info->s->state.key_root[i]))
1684  return(-1);
1685  nod_flag=mi_test_if_nod(info->buff);
1686  _mi_dpointer(info,info->int_keypos-nod_flag-
1687  info->s->rec_reflength,newpos);
1688  if (_mi_write_keypage(info,keyinfo,info->last_keypage,
1689  DFLT_INIT_HITS,info->buff))
1690  return(-1);
1691  }
1692  else
1693  { /* Change old key to new */
1694  if (_mi_ck_delete(info,i,key,key_length))
1695  return(-1);
1696  key_length=_mi_make_key(info,i,key,record,newpos);
1697  if (_mi_ck_write(info,i,key,key_length))
1698  return(-1);
1699  }
1700  }
1701  }
1702  return(0);
1703 } /* movepoint */
1704 
1705 
1706  /* Tell system that we want all memory for our cache */
1707 
1708 void lock_memory(MI_CHECK *)
1709 {
1710 } /* lock_memory */
1711 
1712 
1713 /* Sort index for more efficent reads */
1714 
1715 int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name)
1716 {
1717  register uint32_t key;
1718  register MI_KEYDEF *keyinfo;
1719  int new_file;
1720  my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
1721  uint32_t r_locks,w_locks;
1722  int old_lock;
1723  MYISAM_SHARE *share=info->s;
1724  MI_STATE_INFO old_state;
1725 
1726  /* cannot sort index files with R-tree indexes */
1727  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1728  key++,keyinfo++)
1729 
1730  if (!(param->testflag & T_SILENT))
1731  printf("- Sorting index for MyISAM-table '%s'\n",name);
1732 
1733  /* Get real path for index file */
1734  internal::fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
1735  if ((new_file=my_create(internal::fn_format(param->temp_filename,param->temp_filename,
1736  "", INDEX_TMP_EXT,2+4),
1737  0,param->tmpfile_createflag,MYF(0))) <= 0)
1738  {
1739  mi_check_print_error(param,"Can't create new tempfile: '%s'",
1740  param->temp_filename);
1741  return(-1);
1742  }
1743  if (filecopy(param, new_file,share->kfile,0L,
1744  (ulong) share->base.keystart, "headerblock"))
1745  goto err;
1746 
1747  param->new_file_pos=share->base.keystart;
1748  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1749  key++,keyinfo++)
1750  {
1751  if (! mi_is_key_active(info->s->state.key_map, key))
1752  continue;
1753 
1754  if (share->state.key_root[key] != HA_OFFSET_ERROR)
1755  {
1756  index_pos[key]=param->new_file_pos; /* Write first block here */
1757  if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
1758  new_file))
1759  goto err;
1760  }
1761  else
1762  index_pos[key]= HA_OFFSET_ERROR; /* No blocks */
1763  }
1764 
1765  share->state.version=(ulong) time(NULL);
1766  old_state= share->state; /* save state if not stored */
1767  r_locks= share->r_locks;
1768  w_locks= share->w_locks;
1769  old_lock= info->lock_type;
1770 
1771  /* Put same locks as old file */
1772  share->r_locks= share->w_locks= share->tot_locks= 0;
1773  (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
1774  internal::my_close(share->kfile,MYF(MY_WME));
1775  share->kfile = -1;
1776  internal::my_close(new_file,MYF(MY_WME));
1777  if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
1778  MYF(0)) ||
1779  mi_open_keyfile(share))
1780  goto err2;
1781  info->lock_type= F_UNLCK; /* Force mi_readinfo to lock */
1782  _mi_readinfo(info,F_WRLCK,0); /* Will lock the table */
1783  info->lock_type= old_lock;
1784  share->r_locks= r_locks;
1785  share->w_locks= w_locks;
1786  share->tot_locks= r_locks+w_locks;
1787  share->state= old_state; /* Restore old state */
1788 
1789  info->state->key_file_length=param->new_file_pos;
1790  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1791  for (key=0 ; key < info->s->base.keys ; key++)
1792  info->s->state.key_root[key]=index_pos[key];
1793  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
1794  info->s->state.key_del[key]= HA_OFFSET_ERROR;
1795 
1796  info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
1797  return(0);
1798 
1799 err:
1800  internal::my_close(new_file,MYF(MY_WME));
1801 err2:
1802  my_delete(param->temp_filename,MYF(MY_WME));
1803  return(-1);
1804 } /* mi_sort_index */
1805 
1806 
1807  /* Sort records recursive using one index */
1808 
1809 static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
1810  my_off_t pagepos, int new_file)
1811 {
1812  uint32_t length,nod_flag,used_length, key_length;
1813  unsigned char *buff,*keypos,*endpos;
1814  unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF];
1815  my_off_t new_page_pos,next_page;
1816  char llbuff[22];
1817 
1818  new_page_pos=param->new_file_pos;
1819  param->new_file_pos+=keyinfo->block_length;
1820 
1821  if (!(buff=(unsigned char*) malloc(keyinfo->block_length)))
1822  {
1823  mi_check_print_error(param,"Not enough memory for key block");
1824  return(-1);
1825  }
1826  if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
1827  {
1828  mi_check_print_error(param,"Can't read key block from filepos: %s",
1829  llstr(pagepos,llbuff));
1830  goto err;
1831  }
1832  if ((nod_flag=mi_test_if_nod(buff)))
1833  {
1834  used_length=mi_getint(buff);
1835  keypos=buff+2+nod_flag;
1836  endpos=buff+used_length;
1837  for ( ;; )
1838  {
1839  if (nod_flag)
1840  {
1841  next_page=_mi_kpos(nod_flag,keypos);
1842  _mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
1843  if (sort_one_index(param,info,keyinfo,next_page,new_file))
1844  {
1845  goto err;
1846  }
1847  }
1848  if (keypos >= endpos ||
1849  (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
1850  break;
1851  assert(keypos <= endpos);
1852  }
1853  }
1854 
1855  /* Fill block with zero and write it to the new index file */
1856  length=mi_getint(buff);
1857  memset(buff+length, 0, keyinfo->block_length-length);
1858  if (my_pwrite(new_file,(unsigned char*) buff,(uint) keyinfo->block_length,
1859  new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
1860  {
1861  mi_check_print_error(param,"Can't write indexblock, error: %d",errno);
1862  goto err;
1863  }
1864  free(buff);
1865  return(0);
1866 err:
1867  free(buff);
1868  return(1);
1869 } /* sort_one_index */
1870 
1871 
1872  /*
1873  Let temporary file replace old file.
1874  This assumes that the new file was created in the same
1875  directory as given by realpath(filename).
1876  This will ensure that any symlinks that are used will still work.
1877  Copy stats from old file to new file, deletes orignal and
1878  changes new file name to old file name
1879  */
1880 
1881 int change_to_newfile(const char * filename, const char * old_ext,
1882  const char * new_ext,
1883  uint32_t raid_chunks,
1884  myf MyFlags)
1885 {
1886  (void)raid_chunks;
1887  char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
1888  /* Get real path to filename */
1889  (void) internal::fn_format(old_filename,filename,"",old_ext,2+4+32);
1890  return my_redel(old_filename,
1891  internal::fn_format(new_filename,old_filename,"",new_ext,2+4),
1892  MYF(MY_WME | MY_LINK_WARNING | MyFlags));
1893 } /* change_to_newfile */
1894 
1895 
1896 
1897  /* Copy a block between two files */
1898 
1899 int filecopy(MI_CHECK *param, int to,int from,my_off_t start,
1900  my_off_t length, const char *type)
1901 {
1902  char tmp_buff[IO_SIZE],*buff;
1903  ulong buff_length;
1904 
1905  buff_length=(ulong) min(param->write_buffer_length, (size_t)length);
1906  if (!(buff=(char *)malloc(buff_length)))
1907  {
1908  buff=tmp_buff; buff_length=IO_SIZE;
1909  }
1910 
1911  lseek(from,start,SEEK_SET);
1912  while (length > buff_length)
1913  {
1914  if (my_read(from,(unsigned char*) buff,buff_length,MYF(MY_NABP)) ||
1915  my_write(to,(unsigned char*) buff,buff_length,param->myf_rw))
1916  goto err;
1917  length-= buff_length;
1918  }
1919  if (my_read(from,(unsigned char*) buff,(uint) length,MYF(MY_NABP)) ||
1920  my_write(to,(unsigned char*) buff,(uint) length,param->myf_rw))
1921  goto err;
1922  if (buff != tmp_buff)
1923  free(buff);
1924  return(0);
1925 err:
1926  if (buff != tmp_buff)
1927  free(buff);
1928  mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
1929  type,errno);
1930  return(1);
1931 }
1932 
1933 
1934 /*
1935  Repair table or given index using sorting
1936 
1937  SYNOPSIS
1938  mi_repair_by_sort()
1939  param Repair parameters
1940  info MyISAM handler to repair
1941  name Name of table (for warnings)
1942  rep_quick set to <> 0 if we should not change data file
1943 
1944  RESULT
1945  0 ok
1946  <>0 Error
1947 */
1948 
1949 int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
1950  const char * name, int rep_quick)
1951 {
1952  int got_error;
1953  uint32_t i;
1954  ulong length;
1955  ha_rows start_records;
1956  my_off_t new_header_length,del;
1957  int new_file;
1958  MI_SORT_PARAM sort_param;
1959  MYISAM_SHARE *share=info->s;
1960  HA_KEYSEG *keyseg;
1961  ulong *rec_per_key_part;
1962  char llbuff[22];
1963  SORT_INFO sort_info;
1964  uint64_t key_map= 0;
1965 
1966  start_records=info->state->records;
1967  got_error=1;
1968  new_file= -1;
1969  new_header_length=(param->testflag & T_UNPACK) ? 0 :
1970  share->pack.header_length;
1971  if (!(param->testflag & T_SILENT))
1972  {
1973  printf("- recovering (with sort) MyISAM-table '%s'\n",name);
1974  printf("Data records: %s\n", llstr(start_records,llbuff));
1975  }
1976  param->testflag|=T_REP; /* for easy checking */
1977 
1978  if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
1979  param->testflag|=T_CALC_CHECKSUM;
1980 
1981  memset(&sort_info, 0, sizeof(sort_info));
1982  memset(&sort_param, 0, sizeof(sort_param));
1983  if (!(sort_info.key_block=
1984  alloc_key_blocks(param, (uint) param->sort_key_blocks, share->base.max_key_block_length))
1985  || param->read_cache.init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME))
1986  || (! rep_quick && info->rec_cache.init_io_cache(info->dfile, (uint) param->write_buffer_length, WRITE_CACHE,new_header_length,1, MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
1987  {
1988  goto err;
1989  }
1990  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
1991  info->opt_flag|=WRITE_CACHE_USED;
1992  info->rec_cache.file=info->dfile; /* for sort_delete_record */
1993 
1994  if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
1995  !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
1996  {
1997  mi_check_print_error(param, "Not enough memory for extra record");
1998  goto err;
1999  }
2000  if (!rep_quick)
2001  {
2002  /* Get real path for data file */
2003  if ((new_file=my_create(internal::fn_format(param->temp_filename,
2004  share->data_file_name, "",
2005  DATA_TMP_EXT, 2+4),
2006  0,param->tmpfile_createflag,
2007  MYF(0))) < 0)
2008  {
2009  mi_check_print_error(param,"Can't create new tempfile: '%s'",
2010  param->temp_filename);
2011  goto err;
2012  }
2013  if (new_header_length &&
2014  filecopy(param, new_file,info->dfile,0L,new_header_length,
2015  "datafile-header"))
2016  goto err;
2017  if (param->testflag & T_UNPACK)
2018  {
2019  share->options&= ~HA_OPTION_COMPRESS_RECORD;
2020  mi_int2store(share->state.header.options,share->options);
2021  }
2022  share->state.dellink= HA_OFFSET_ERROR;
2023  info->rec_cache.file=new_file;
2024  }
2025 
2026  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2027 
2028  /* Optionally drop indexes and optionally modify the key_map. */
2029  mi_drop_all_indexes(param, info, false);
2030  key_map= share->state.key_map;
2031  if (param->testflag & T_CREATE_MISSING_KEYS)
2032  {
2033  /* Invert the copied key_map to recreate all disabled indexes. */
2034  key_map= ~key_map;
2035  }
2036 
2037  sort_info.info=info;
2038  sort_info.param = param;
2039 
2040  set_data_file_type(&sort_info, share);
2041  sort_param.filepos=new_header_length;
2042  sort_info.dupp=0;
2043  sort_info.buff=0;
2044  param->read_cache.end_of_file=sort_info.filelength=
2045  lseek(param->read_cache.file,0L,SEEK_END);
2046 
2047  sort_param.wordlist=NULL;
2048 
2049  if (share->data_file_type == DYNAMIC_RECORD)
2050  length=max(share->base.min_pack_length+1,share->base.min_block_length);
2051  else if (share->data_file_type == COMPRESSED_RECORD)
2052  length=share->base.min_block_length;
2053  else
2054  length=share->base.pack_reclength;
2055  sort_info.max_records=
2056  ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
2057  (ha_rows) (sort_info.filelength/length+1));
2058  sort_param.key_cmp=sort_key_cmp;
2059  sort_param.lock_in_memory=lock_memory;
2060  sort_param.sort_info=&sort_info;
2061  sort_param.fix_datafile= (bool) (! rep_quick);
2062  sort_param.master =1;
2063 
2064  del=info->state->del;
2065  param->glob_crc=0;
2066  if (param->testflag & T_CALC_CHECKSUM)
2067  sort_param.calc_checksum= 1;
2068 
2069  rec_per_key_part= param->rec_per_key_part;
2070  for (sort_param.key=0 ; sort_param.key < share->base.keys ;
2071  rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
2072  {
2073  sort_param.read_cache=param->read_cache;
2074  sort_param.keyinfo=share->keyinfo+sort_param.key;
2075  sort_param.seg=sort_param.keyinfo->seg;
2076  /*
2077  Skip this index if it is marked disabled in the copied
2078  (and possibly inverted) key_map.
2079  */
2080  if (! mi_is_key_active(key_map, sort_param.key))
2081  {
2082  /* Remember old statistics for key */
2083  assert(rec_per_key_part >= param->rec_per_key_part);
2084  memcpy(rec_per_key_part,
2085  (share->state.rec_per_key_part +
2086  (rec_per_key_part - param->rec_per_key_part)),
2087  sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
2088  continue;
2089  }
2090 
2091  if ((!(param->testflag & T_SILENT)))
2092  printf ("- Fixing index %d\n",sort_param.key+1);
2093  sort_param.max_pos=sort_param.pos=share->pack.header_length;
2094  keyseg=sort_param.seg;
2095  memset(sort_param.unique, 0, sizeof(sort_param.unique));
2096  sort_param.key_length=share->rec_reflength;
2097  for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
2098  {
2099  sort_param.key_length+=keyseg[i].length;
2100  if (keyseg[i].flag & HA_SPACE_PACK)
2101  sort_param.key_length+=get_pack_length(keyseg[i].length);
2102  if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2103  sort_param.key_length+=2 + test(keyseg[i].length >= 127);
2104  if (keyseg[i].flag & HA_NULL_PART)
2105  sort_param.key_length++;
2106  }
2107  info->state->records=info->state->del=share->state.split=0;
2108  info->state->empty=0;
2109 
2110  {
2111  sort_param.key_read=sort_key_read;
2112  sort_param.key_write=sort_key_write;
2113  }
2114 
2115  if (_create_index_by_sort(&sort_param,
2116  (bool) (!(param->testflag & T_VERBOSE)),
2117  (uint) param->sort_buffer_length))
2118  {
2119  param->retry_repair=1;
2120  goto err;
2121  }
2122  /* No need to calculate checksum again. */
2123  sort_param.calc_checksum= 0;
2124  sort_param.wordroot.free_root(MYF(0));
2125 
2126  /* Set for next loop */
2127  sort_info.max_records= (ha_rows) info->state->records;
2128 
2129  if (param->testflag & T_STATISTICS)
2130  update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2131  param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2132  sort_param.notnull: NULL,
2133  (uint64_t) info->state->records);
2134  /* Enable this index in the permanent (not the copied) key_map. */
2135  mi_set_key_active(share->state.key_map, sort_param.key);
2136 
2137  if (sort_param.fix_datafile)
2138  {
2139  param->read_cache.end_of_file=sort_param.filepos;
2140  if (write_data_suffix(&sort_info, 1) || info->rec_cache.end_io_cache())
2141  {
2142  goto err;
2143  }
2144  if (param->testflag & T_SAFE_REPAIR)
2145  {
2146  /* Don't repair if we loosed more than one row */
2147  if (info->state->records+1 < start_records)
2148  {
2149  info->state->records=start_records;
2150  goto err;
2151  }
2152  }
2153  share->state.state.data_file_length = info->state->data_file_length=
2154  sort_param.filepos;
2155  /* Only whole records */
2156  share->state.version=(ulong) time((time_t*) 0);
2157  internal::my_close(info->dfile,MYF(0));
2158  info->dfile=new_file;
2159  share->data_file_type=sort_info.new_data_file_type;
2160  share->pack.header_length=(ulong) new_header_length;
2161  sort_param.fix_datafile=0;
2162  }
2163  else
2164  info->state->data_file_length=sort_param.max_pos;
2165 
2166  param->read_cache.file=info->dfile; /* re-init read cache */
2167  param->read_cache.reinit_io_cache(READ_CACHE,share->pack.header_length, 1,1);
2168  }
2169 
2170  if (param->testflag & T_WRITE_LOOP)
2171  {
2172  fputs(" \r",stdout); fflush(stdout);
2173  }
2174 
2175  if (rep_quick && del+sort_info.dupp != info->state->del)
2176  {
2177  mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
2178  mi_check_print_error(param,"Run recovery again without -q");
2179  got_error=1;
2180  param->retry_repair=1;
2181  param->testflag|=T_RETRY_WITHOUT_QUICK;
2182  goto err;
2183  }
2184 
2185  if (rep_quick & T_FORCE_UNIQUENESS)
2186  {
2187  my_off_t skr=info->state->data_file_length+
2188  (share->options & HA_OPTION_COMPRESS_RECORD ?
2189  MEMMAP_EXTRA_MARGIN : 0);
2190 #ifdef USE_RELOC
2191  if (share->data_file_type == STATIC_RECORD &&
2192  skr < share->base.reloc*share->base.min_pack_length)
2193  skr=share->base.reloc*share->base.min_pack_length;
2194 #endif
2195  if (skr != sort_info.filelength && !info->s->base.raid_type)
2196  if (ftruncate(info->dfile, skr))
2197  mi_check_print_warning(param,
2198  "Can't change size of datafile, error: %d",
2199  errno);
2200  }
2201  if (param->testflag & T_CALC_CHECKSUM)
2202  info->state->checksum=param->glob_crc;
2203 
2204  if (ftruncate(share->kfile, info->state->key_file_length))
2205  mi_check_print_warning(param,
2206  "Can't change size of indexfile, error: %d",
2207  errno);
2208 
2209  if (!(param->testflag & T_SILENT))
2210  {
2211  if (start_records != info->state->records)
2212  printf("Data records: %s\n", llstr(info->state->records,llbuff));
2213  if (sort_info.dupp)
2214  mi_check_print_warning(param,
2215  "%s records have been removed",
2216  llstr(sort_info.dupp,llbuff));
2217  }
2218  got_error=0;
2219 
2220  if (&share->state.state != info->state)
2221  memcpy( &share->state.state, info->state, sizeof(*info->state));
2222 
2223 err:
2224  info->rec_cache.end_io_cache();
2225  if (!got_error)
2226  {
2227  /* Replace the actual file with the temporary file */
2228  if (new_file >= 0)
2229  {
2230  internal::my_close(new_file,MYF(0));
2231  info->dfile=new_file= -1;
2232  if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2233  DATA_TMP_EXT, share->base.raid_chunks,
2234  MYF(0)) ||
2235  mi_open_datafile(info,share,-1))
2236  got_error=1;
2237  }
2238  }
2239  if (got_error)
2240  {
2241  if (! param->error_printed)
2242  mi_check_print_error(param,"%d when fixing table",errno);
2243  if (new_file >= 0)
2244  {
2245  internal::my_close(new_file,MYF(0));
2246  my_delete(param->temp_filename, MYF(MY_WME));
2247  if (info->dfile == new_file)
2248  info->dfile= -1;
2249  }
2250  mi_mark_crashed_on_repair(info);
2251  }
2252  else if (key_map == share->state.key_map)
2253  share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
2254  share->state.changed|=STATE_NOT_SORTED_PAGES;
2255 
2256  void * rec_buff_ptr= NULL;
2257  rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
2258  free(rec_buff_ptr);
2259  rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
2260  free(rec_buff_ptr);
2261  rec_buff_ptr= NULL;
2262 
2263  free((unsigned char*) sort_info.key_block);
2264  free(sort_info.buff);
2265  param->read_cache.end_io_cache();
2266  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2267  if (!got_error && (param->testflag & T_UNPACK))
2268  {
2269  share->state.header.options[0]&= (unsigned char) ~HA_OPTION_COMPRESS_RECORD;
2270  share->pack.header_length=0;
2271  }
2272  return(got_error);
2273 }
2274 
2275  /* Read next record and return next key */
2276 
2277 int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
2278 {
2279  int error;
2280  SORT_INFO *sort_info=sort_param->sort_info;
2281  MI_INFO *info=sort_info->info;
2282 
2283  if ((error=sort_get_next_record(sort_param)))
2284  return(error);
2285  if (info->state->records == sort_info->max_records)
2286  {
2287  mi_check_print_error(sort_info->param,
2288  "Key %d - Found too many records; Can't continue",
2289  sort_param->key+1);
2290  return(1);
2291  }
2292  sort_param->real_key_length=
2293  (info->s->rec_reflength+
2294  _mi_make_key(info, sort_param->key, (unsigned char*) key,
2295  sort_param->record, sort_param->filepos));
2296 #ifdef HAVE_VALGRIND
2297  memset((unsigned char *)key+sort_param->real_key_length, 0,
2298  (sort_param->key_length-sort_param->real_key_length));
2299 #endif
2300  return(sort_write_record(sort_param));
2301 } /* sort_key_read */
2302 
2303 
2304 /*
2305  Read next record from file using parameters in sort_info.
2306 
2307  SYNOPSIS
2308  sort_get_next_record()
2309  sort_param Information about and for the sort process
2310 
2311  NOTE
2312 
2313  Dynamic Records With Non-Quick Parallel Repair
2314 
2315  For non-quick parallel repair we use a synchronized read/write
2316  cache. This means that one thread is the master who fixes the data
2317  file by reading each record from the old data file and writing it
2318  to the new data file. By doing this the records in the new data
2319  file are written contiguously. Whenever the write buffer is full,
2320  it is copied to the read buffer. The slaves read from the read
2321  buffer, which is not associated with a file. Thus read_cache.file
2322  is -1. When using _mi_read_cache(), the slaves must always set
2323  flag to READING_NEXT so that the function never tries to read from
2324  file. This is safe because the records are contiguous. There is no
2325  need to read outside the cache. This condition is evaluated in the
2326  variable 'parallel_flag' for quick reference. read_cache.file must
2327  be >= 0 in every other case.
2328 
2329  RETURN
2330  -1 end of file
2331  0 ok
2332  > 0 error
2333 */
2334 
2335 int sort_get_next_record(MI_SORT_PARAM *sort_param)
2336 {
2337  int searching;
2338  int parallel_flag;
2339  uint32_t found_record,b_type,left_length;
2340  my_off_t pos;
2341  unsigned char *to= NULL;
2342  MI_BLOCK_INFO block_info;
2343  SORT_INFO *sort_info=sort_param->sort_info;
2344  MI_CHECK *param=sort_info->param;
2345  MI_INFO *info=sort_info->info;
2346  MYISAM_SHARE *share=info->s;
2347  char llbuff[22],llbuff2[22];
2348 
2349  if (*killed_ptr(param))
2350  return(1);
2351 
2352  switch (share->data_file_type) {
2353  case STATIC_RECORD:
2354  for (;;)
2355  {
2356  if (sort_param->read_cache.read(sort_param->record, share->base.pack_reclength))
2357  {
2358  if (sort_param->read_cache.error)
2359  param->out_flag |= O_DATA_LOST;
2360  param->retry_repair=1;
2361  param->testflag|=T_RETRY_WITHOUT_QUICK;
2362  return(-1);
2363  }
2364  sort_param->start_recpos=sort_param->pos;
2365  if (!sort_param->fix_datafile)
2366  {
2367  sort_param->filepos=sort_param->pos;
2368  if (sort_param->master)
2369  share->state.split++;
2370  }
2371  sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
2372  if (*sort_param->record)
2373  {
2374  if (sort_param->calc_checksum)
2375  param->glob_crc+= (info->checksum=
2376  mi_static_checksum(info,sort_param->record));
2377  return(0);
2378  }
2379  if (!sort_param->fix_datafile && sort_param->master)
2380  {
2381  info->state->del++;
2382  info->state->empty+=share->base.pack_reclength;
2383  }
2384  }
2385  case DYNAMIC_RECORD:
2386  pos= sort_param->pos;
2387  searching= (sort_param->fix_datafile && (param->testflag & T_EXTEND));
2388  parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
2389  for (;;)
2390  {
2391  found_record=block_info.second_read= 0;
2392  left_length=1;
2393  if (searching)
2394  {
2395  pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
2396  param->testflag|=T_RETRY_WITHOUT_QUICK;
2397  sort_param->start_recpos=pos;
2398  }
2399  do
2400  {
2401  if (pos > sort_param->max_pos)
2402  sort_param->max_pos=pos;
2403  if (pos & (MI_DYN_ALIGN_SIZE-1))
2404  {
2405  if ((param->testflag & T_VERBOSE) || searching == 0)
2406  mi_check_print_info(param,"Wrong aligned block at %s",
2407  llstr(pos,llbuff));
2408  if (searching)
2409  goto try_next;
2410  }
2411  if (found_record && pos == param->search_after_block)
2412  mi_check_print_info(param,"Block: %s used by record at %s",
2413  llstr(param->search_after_block,llbuff),
2414  llstr(sort_param->start_recpos,llbuff2));
2415  if (_mi_read_cache(&sort_param->read_cache,
2416  (unsigned char*) block_info.header,pos,
2417  MI_BLOCK_INFO_HEADER_LENGTH,
2418  (! found_record ? READING_NEXT : 0) |
2419  parallel_flag | READING_HEADER))
2420  {
2421  if (found_record)
2422  {
2423  mi_check_print_info(param,
2424  "Can't read whole record at %s (errno: %d)",
2425  llstr(sort_param->start_recpos,llbuff),errno);
2426  goto try_next;
2427  }
2428  return(-1);
2429  }
2430  if (searching && ! sort_param->fix_datafile)
2431  {
2432  param->error_printed=1;
2433  param->retry_repair=1;
2434  param->testflag|=T_RETRY_WITHOUT_QUICK;
2435  return(1); /* Something wrong with data */
2436  }
2437  b_type=_mi_get_block_info(&block_info,-1,pos);
2438  if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
2439  ((b_type & BLOCK_FIRST) &&
2440  (block_info.rec_len < (uint) share->base.min_pack_length ||
2441  block_info.rec_len > (uint) share->base.max_pack_length)))
2442  {
2443  uint32_t i;
2444  if (param->testflag & T_VERBOSE || searching == 0)
2445  mi_check_print_info(param,
2446  "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
2447  block_info.header[0],block_info.header[1],
2448  block_info.header[2],llstr(pos,llbuff));
2449  if (found_record)
2450  goto try_next;
2451  block_info.second_read=0;
2452  searching=1;
2453  /* Search after block in read header string */
2454  for (i=MI_DYN_ALIGN_SIZE ;
2455  i < MI_BLOCK_INFO_HEADER_LENGTH ;
2456  i+= MI_DYN_ALIGN_SIZE)
2457  if (block_info.header[i] >= 1 &&
2458  block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
2459  break;
2460  pos+=(ulong) i;
2461  sort_param->start_recpos=pos;
2462  continue;
2463  }
2464  if (b_type & BLOCK_DELETED)
2465  {
2466  bool error=0;
2467  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
2468  share->base.min_block_length)
2469  {
2470  if (!searching)
2471  mi_check_print_info(param,
2472  "Deleted block with impossible length %u at %s",
2473  block_info.block_len,llstr(pos,llbuff));
2474  error=1;
2475  }
2476  else
2477  {
2478  if ((block_info.next_filepos != HA_OFFSET_ERROR &&
2479  block_info.next_filepos >=
2480  info->state->data_file_length) ||
2481  (block_info.prev_filepos != HA_OFFSET_ERROR &&
2482  block_info.prev_filepos >= info->state->data_file_length))
2483  {
2484  if (!searching)
2485  mi_check_print_info(param,
2486  "Delete link points outside datafile at %s",
2487  llstr(pos,llbuff));
2488  error=1;
2489  }
2490  }
2491  if (error)
2492  {
2493  if (found_record)
2494  goto try_next;
2495  searching=1;
2496  pos+= MI_DYN_ALIGN_SIZE;
2497  sort_param->start_recpos=pos;
2498  block_info.second_read=0;
2499  continue;
2500  }
2501  }
2502  else
2503  {
2504  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
2505  share->base.min_block_length ||
2506  block_info.block_len > (uint) share->base.max_pack_length+
2507  MI_SPLIT_LENGTH)
2508  {
2509  if (!searching)
2510  mi_check_print_info(param,
2511  "Found block with impossible length %u at %s; Skipped",
2512  block_info.block_len+ (uint) (block_info.filepos-pos),
2513  llstr(pos,llbuff));
2514  if (found_record)
2515  goto try_next;
2516  searching=1;
2517  pos+= MI_DYN_ALIGN_SIZE;
2518  sort_param->start_recpos=pos;
2519  block_info.second_read=0;
2520  continue;
2521  }
2522  }
2523  if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
2524  {
2525  if (!sort_param->fix_datafile && sort_param->master &&
2526  (b_type & BLOCK_DELETED))
2527  {
2528  info->state->empty+=block_info.block_len;
2529  info->state->del++;
2530  share->state.split++;
2531  }
2532  if (found_record)
2533  goto try_next;
2534  if (searching)
2535  {
2536  pos+=MI_DYN_ALIGN_SIZE;
2537  sort_param->start_recpos=pos;
2538  }
2539  else
2540  pos=block_info.filepos+block_info.block_len;
2541  block_info.second_read=0;
2542  continue;
2543  }
2544 
2545  if (!sort_param->fix_datafile && sort_param->master)
2546  share->state.split++;
2547  if (! found_record++)
2548  {
2549  sort_param->find_length=left_length=block_info.rec_len;
2550  sort_param->start_recpos=pos;
2551  if (!sort_param->fix_datafile)
2552  sort_param->filepos=sort_param->start_recpos;
2553  if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
2554  sort_param->pos=block_info.filepos+1;
2555  else
2556  sort_param->pos=block_info.filepos+block_info.block_len;
2557  if (share->base.blobs)
2558  {
2559  if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
2560  &(sort_param->rec_buff))))
2561  {
2562  if (param->max_record_length >= block_info.rec_len)
2563  {
2564  mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
2565  llstr(sort_param->start_recpos,llbuff),
2566  (ulong) block_info.rec_len);
2567  return(1);
2568  }
2569  else
2570  {
2571  mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
2572  llstr(sort_param->start_recpos,llbuff),
2573  (ulong) block_info.rec_len);
2574  goto try_next;
2575  }
2576  }
2577  }
2578  else
2579  to= sort_param->rec_buff;
2580  }
2581  if (left_length < block_info.data_len || ! block_info.data_len)
2582  {
2583  mi_check_print_info(param,
2584  "Found block with too small length at %s; Skipped",
2585  llstr(sort_param->start_recpos,llbuff));
2586  goto try_next;
2587  }
2588  if (block_info.filepos + block_info.data_len >
2589  sort_param->read_cache.end_of_file)
2590  {
2591  mi_check_print_info(param,
2592  "Found block that points outside data file at %s",
2593  llstr(sort_param->start_recpos,llbuff));
2594  goto try_next;
2595  }
2596  /*
2597  Copy information that is already read. Avoid accessing data
2598  below the cache start. This could happen if the header
2599  streched over the end of the previous buffer contents.
2600  */
2601  {
2602  uint32_t header_len= (uint) (block_info.filepos - pos);
2603  uint32_t prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
2604 
2605  if (prefetch_len > block_info.data_len)
2606  prefetch_len= block_info.data_len;
2607  if (prefetch_len)
2608  {
2609  memcpy(to, block_info.header + header_len, prefetch_len);
2610  block_info.filepos+= prefetch_len;
2611  block_info.data_len-= prefetch_len;
2612  left_length-= prefetch_len;
2613  to+= prefetch_len;
2614  }
2615  }
2616  if (block_info.data_len &&
2617  _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
2618  block_info.data_len,
2619  (found_record == 1 ? READING_NEXT : 0) |
2620  parallel_flag))
2621  {
2622  mi_check_print_info(param,
2623  "Read error for block at: %s (error: %d); Skipped",
2624  llstr(block_info.filepos,llbuff),errno);
2625  goto try_next;
2626  }
2627  left_length-=block_info.data_len;
2628  to+=block_info.data_len;
2629  pos=block_info.next_filepos;
2630  if (pos == HA_OFFSET_ERROR && left_length)
2631  {
2632  mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
2633  llstr(sort_param->start_recpos,llbuff));
2634  goto try_next;
2635  }
2636  if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
2637  {
2638  mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
2639  llstr(pos,llbuff2),
2640  llstr(sort_param->start_recpos,llbuff));
2641  goto try_next;
2642  }
2643  } while (left_length);
2644 
2645  if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
2646  sort_param->find_length) != MY_FILE_ERROR)
2647  {
2648  if (sort_param->read_cache.error < 0)
2649  return(1);
2650  if (sort_param->calc_checksum)
2651  info->checksum= mi_checksum(info, sort_param->record);
2652  if ((param->testflag & (T_EXTEND | T_REP)) || searching)
2653  {
2654  if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
2655  sort_param->find_length,
2656  (param->testflag & T_QUICK) &&
2657  sort_param->calc_checksum &&
2658  test(info->s->calc_checksum)))
2659  {
2660  mi_check_print_info(param,"Found wrong packed record at %s",
2661  llstr(sort_param->start_recpos,llbuff));
2662  goto try_next;
2663  }
2664  }
2665  if (sort_param->calc_checksum)
2666  param->glob_crc+= info->checksum;
2667  return(0);
2668  }
2669  if (!searching)
2670  mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
2671  sort_param->key+1,
2672  llstr(sort_param->start_recpos,llbuff));
2673  try_next:
2674  pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
2675  searching=1;
2676  }
2677  case COMPRESSED_RECORD:
2678  case BLOCK_RECORD:
2679  assert(0); /* Impossible */
2680  }
2681  return(1); /* Impossible */
2682 }
2683 
2684 
2685 /*
2686  Write record to new file.
2687 
2688  SYNOPSIS
2689  sort_write_record()
2690  sort_param Sort parameters.
2691 
2692  NOTE
2693  This is only called by a master thread if parallel repair is used.
2694 
2695  RETURN
2696  0 OK
2697  1 Error
2698 */
2699 
2700 int sort_write_record(MI_SORT_PARAM *sort_param)
2701 {
2702  int flag;
2703  ulong block_length,reclength;
2704  unsigned char *from;
2705  SORT_INFO *sort_info=sort_param->sort_info;
2706  MI_CHECK *param=sort_info->param;
2707  MI_INFO *info=sort_info->info;
2708  MYISAM_SHARE *share=info->s;
2709 
2710  if (sort_param->fix_datafile)
2711  {
2712  switch (sort_info->new_data_file_type) {
2713  case STATIC_RECORD:
2714  if (info->rec_cache.write(sort_param->record, share->base.pack_reclength))
2715  {
2716  mi_check_print_error(param,"%d when writing to datafile",errno);
2717  return(1);
2718  }
2719  sort_param->filepos+=share->base.pack_reclength;
2720  info->s->state.split++;
2721  /* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */
2722  break;
2723  case DYNAMIC_RECORD:
2724  if (! info->blobs)
2725  from=sort_param->rec_buff;
2726  else
2727  {
2728  /* must be sure that local buffer is big enough */
2729  reclength=info->s->base.pack_reclength+
2730  _my_calc_total_blob_length(info,sort_param->record)+
2731  ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
2732  MI_DYN_DELETE_BLOCK_HEADER;
2733  if (sort_info->buff_length < reclength)
2734  {
2735  void *tmpptr= NULL;
2736  tmpptr= realloc(sort_info->buff, reclength);
2737  if(tmpptr)
2738  {
2739  sort_info->buff_length=reclength;
2740  sort_info->buff= (unsigned char *)tmpptr;
2741  }
2742  else
2743  {
2744  mi_check_print_error(param,"Could not realloc() sort_info->buff "
2745  " to %ul bytes", reclength);
2746  return(1);
2747  }
2748  }
2749  from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
2750  }
2751  /* We can use info->checksum here as only one thread calls this. */
2752  info->checksum=mi_checksum(info,sort_param->record);
2753  reclength=_mi_rec_pack(info,from,sort_param->record);
2754  flag=0;
2755  /* sort_info->param->glob_crc+=info->checksum; */
2756 
2757  do
2758  {
2759  block_length=reclength+ 3 + test(reclength >= (65520-3));
2760  if (block_length < share->base.min_block_length)
2761  block_length=share->base.min_block_length;
2762  info->update|=HA_STATE_WRITE_AT_END;
2763  block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
2764  if (block_length > MI_MAX_BLOCK_LENGTH)
2765  block_length=MI_MAX_BLOCK_LENGTH;
2766  if (_mi_write_part_record(info,0L,block_length,
2767  sort_param->filepos+block_length,
2768  &from,&reclength,&flag))
2769  {
2770  mi_check_print_error(param,"%d when writing to datafile",errno);
2771  return(1);
2772  }
2773  sort_param->filepos+=block_length;
2774  info->s->state.split++;
2775  } while (reclength);
2776  /* sort_info->param->glob_crc+=info->checksum; */
2777  break;
2778  case COMPRESSED_RECORD:
2779  case BLOCK_RECORD:
2780  assert(0); /* Impossible */
2781  }
2782  }
2783  if (sort_param->master)
2784  {
2785  info->state->records++;
2786  if ((param->testflag & T_WRITE_LOOP) &&
2787  (info->state->records % WRITE_COUNT) == 0)
2788  {
2789  char llbuff[22];
2790  printf("%s\r", llstr(info->state->records,llbuff));
2791  fflush(stdout);
2792  }
2793  }
2794  return(0);
2795 } /* sort_write_record */
2796 
2797 
2798  /* Compare two keys from _create_index_by_sort */
2799 
2800 int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a, const void *b)
2801 {
2802  uint32_t not_used[2];
2803  return (ha_key_cmp(sort_param->seg, *((unsigned char* const *) a), *((unsigned char* const *) b),
2804  USE_WHOLE_KEY, SEARCH_SAME, not_used));
2805 } /* sort_key_cmp */
2806 
2807 
2808 int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
2809 {
2810  uint32_t diff_pos[2];
2811  char llbuff[22],llbuff2[22];
2812  SORT_INFO *sort_info=sort_param->sort_info;
2813  MI_CHECK *param= sort_info->param;
2814  int cmp;
2815 
2816  if (sort_info->key_block->inited)
2817  {
2818  cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
2819  (unsigned char*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
2820  diff_pos);
2821  if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
2822  ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
2823  (unsigned char*) a, USE_WHOLE_KEY,
2824  SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
2825  else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
2826  {
2827  diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
2828  sort_param->notnull,
2829  sort_info->key_block->lastkey,
2830  (unsigned char*)a);
2831  }
2832  sort_param->unique[diff_pos[0]-1]++;
2833  }
2834  else
2835  {
2836  cmp= -1;
2837  if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
2838  mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
2839  (unsigned char*)a);
2840  }
2841  if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
2842  {
2843  sort_info->dupp++;
2844  sort_info->info->lastpos=get_record_for_key(sort_info->info,
2845  sort_param->keyinfo,
2846  (unsigned char*) a);
2847  mi_check_print_warning(param,
2848  "Duplicate key for record at %10s against record at %10s",
2849  llstr(sort_info->info->lastpos,llbuff),
2850  llstr(get_record_for_key(sort_info->info,
2851  sort_param->keyinfo,
2852  sort_info->key_block->
2853  lastkey),
2854  llbuff2));
2855  param->testflag|=T_RETRY_WITHOUT_QUICK;
2856  return (sort_delete_record(sort_param));
2857  }
2858  return (sort_insert_key(sort_param,sort_info->key_block,
2859  (unsigned char*) a, HA_OFFSET_ERROR));
2860 } /* sort_key_write */
2861 
2862 
2863  /* get pointer to record from a key */
2864 
2865 my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
2866  unsigned char *key) {
2867  return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
2868 } /* get_record_for_key */
2869 
2870 
2871  /* Insert a key in sort-key-blocks */
2872 
2873 int sort_insert_key(MI_SORT_PARAM *sort_param,
2874  register SORT_KEY_BLOCKS *key_block, unsigned char *key,
2875  my_off_t prev_block)
2876 {
2877  uint32_t a_length,t_length,nod_flag;
2878  my_off_t filepos,key_file_length;
2879  unsigned char *anc_buff,*lastkey;
2880  MI_KEY_PARAM s_temp;
2881  MI_INFO *info;
2882  MI_KEYDEF *keyinfo=sort_param->keyinfo;
2883  SORT_INFO *sort_info= sort_param->sort_info;
2884  MI_CHECK *param=sort_info->param;
2885 
2886  anc_buff=key_block->buff;
2887  info=sort_info->info;
2888  lastkey=key_block->lastkey;
2889  nod_flag= (key_block == sort_info->key_block ? 0 :
2890  info->s->base.key_reflength);
2891 
2892  if (!key_block->inited)
2893  {
2894  key_block->inited=1;
2895  if (key_block == sort_info->key_block_end)
2896  {
2897  mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
2898  return(1);
2899  }
2900  a_length=2+nod_flag;
2901  key_block->end_pos=anc_buff+2;
2902  lastkey=0; /* No previous key in block */
2903  }
2904  else
2905  a_length=mi_getint(anc_buff);
2906 
2907  /* Save pointer to previous block */
2908  if (nod_flag)
2909  _mi_kpointer(info,key_block->end_pos,prev_block);
2910 
2911  t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
2912  (unsigned char*) 0,lastkey,lastkey,key,
2913  &s_temp);
2914  (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
2915  a_length+=t_length;
2916  mi_putint(anc_buff,a_length,nod_flag);
2917  key_block->end_pos+=t_length;
2918  if (a_length <= keyinfo->block_length)
2919  {
2920  _mi_move_key(keyinfo,key_block->lastkey,key);
2921  key_block->last_length=a_length-t_length;
2922  return(0);
2923  }
2924 
2925  /* Fill block with end-zero and write filled block */
2926  mi_putint(anc_buff,key_block->last_length,nod_flag);
2927  memset(anc_buff+key_block->last_length, 0,
2928  keyinfo->block_length - key_block->last_length);
2929  key_file_length=info->state->key_file_length;
2930  if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
2931  return(1);
2932 
2933  /* If we read the page from the key cache, we have to write it back to it */
2934  if (key_file_length == info->state->key_file_length)
2935  {
2936  if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
2937  return(1);
2938  }
2939  else if (my_pwrite(info->s->kfile,(unsigned char*) anc_buff,
2940  (uint) keyinfo->block_length,filepos, param->myf_rw))
2941  return(1);
2942 
2943  /* Write separator-key to block in next level */
2944  if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
2945  return(1);
2946 
2947  /* clear old block and write new key in it */
2948  key_block->inited=0;
2949  return(sort_insert_key(sort_param, key_block,key,prev_block));
2950 } /* sort_insert_key */
2951 
2952 
2953  /* Delete record when we found a duplicated key */
2954 
2955 int sort_delete_record(MI_SORT_PARAM *sort_param)
2956 {
2957  uint32_t i;
2958  int old_file,error;
2959  unsigned char *key;
2960  SORT_INFO *sort_info=sort_param->sort_info;
2961  MI_CHECK *param=sort_info->param;
2962  MI_INFO *info=sort_info->info;
2963 
2964  if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
2965  {
2966  mi_check_print_error(param,
2967  "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
2968  return(1);
2969  }
2970  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
2971  {
2972  mi_check_print_error(param,
2973  "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
2974  return(1);
2975  }
2976 
2977  old_file=info->dfile;
2978  info->dfile=info->rec_cache.file;
2979  if (sort_info->current_key)
2980  {
2981  key=info->lastkey+info->s->base.max_key_length;
2982  if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
2983  error != HA_ERR_RECORD_DELETED)
2984  {
2985  mi_check_print_error(param,"Can't read record to be removed");
2986  info->dfile=old_file;
2987  return(1);
2988  }
2989 
2990  for (i=0 ; i < sort_info->current_key ; i++)
2991  {
2992  uint32_t key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
2993  if (_mi_ck_delete(info,i,key,key_length))
2994  {
2995  mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
2996  info->dfile=old_file;
2997  return(1);
2998  }
2999  }
3000  if (sort_param->calc_checksum)
3001  param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
3002  }
3003  error= info->rec_cache.flush() || (*info->s->delete_record)(info);
3004  info->dfile=old_file; /* restore actual value */
3005  info->state->records--;
3006  return(error);
3007 } /* sort_delete_record */
3008 
3009  /* Fix all pending blocks and flush everything to disk */
3010 
3011 int flush_pending_blocks(MI_SORT_PARAM *sort_param)
3012 {
3013  uint32_t nod_flag,length;
3014  my_off_t filepos,key_file_length;
3015  SORT_KEY_BLOCKS *key_block;
3016  SORT_INFO *sort_info= sort_param->sort_info;
3017  myf myf_rw=sort_info->param->myf_rw;
3018  MI_INFO *info=sort_info->info;
3019  MI_KEYDEF *keyinfo=sort_param->keyinfo;
3020 
3021  filepos= HA_OFFSET_ERROR; /* if empty file */
3022  nod_flag=0;
3023  for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
3024  {
3025  key_block->inited=0;
3026  length=mi_getint(key_block->buff);
3027  if (nod_flag)
3028  _mi_kpointer(info,key_block->end_pos,filepos);
3029  key_file_length=info->state->key_file_length;
3030  memset(key_block->buff+length, 0, keyinfo->block_length-length);
3031  if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
3032  return(1);
3033 
3034  /* If we read the page from the key cache, we have to write it back */
3035  if (key_file_length == info->state->key_file_length)
3036  {
3037  if (_mi_write_keypage(info, keyinfo, filepos,
3038  DFLT_INIT_HITS, key_block->buff))
3039  return(1);
3040  }
3041  else if (my_pwrite(info->s->kfile,(unsigned char*) key_block->buff,
3042  (uint) keyinfo->block_length,filepos, myf_rw))
3043  return(1);
3044  nod_flag=1;
3045  }
3046  info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
3047  return(0);
3048 } /* flush_pending_blocks */
3049 
3050  /* alloc space and pointers for key_blocks */
3051 
3052 static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
3053  uint32_t buffer_length)
3054 {
3055  register uint32_t i;
3056  SORT_KEY_BLOCKS *block;
3057 
3058  if (!(block=(SORT_KEY_BLOCKS*) malloc((sizeof(SORT_KEY_BLOCKS)+
3059  buffer_length+IO_SIZE)*blocks)))
3060  {
3061  mi_check_print_error(param,"Not enough memory for sort-key-blocks");
3062  return(0);
3063  }
3064  for (i=0 ; i < blocks ; i++)
3065  {
3066  block[i].inited=0;
3067  block[i].buff=(unsigned char*) (block+blocks)+(buffer_length+IO_SIZE)*i;
3068  }
3069  return(block);
3070 } /* alloc_key_blocks */
3071 
3072 
3073  /* Check if file is almost full */
3074 
3075 int test_if_almost_full(MI_INFO *info)
3076 {
3077  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
3078  return 0;
3079  return (my_off_t)(lseek(info->s->kfile, 0L, SEEK_END) / 10 * 9) >
3080  (my_off_t) info->s->base.max_key_file_length ||
3081  (my_off_t)(lseek(info->dfile, 0L, SEEK_END) / 10 * 9) >
3082  (my_off_t) info->s->base.max_data_file_length;
3083 }
3084 
3085 
3086  /* write suffix to data file if neaded */
3087 
3088 int write_data_suffix(SORT_INFO *sort_info, bool fix_datafile)
3089 {
3090  MI_INFO *info=sort_info->info;
3091 
3092  if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
3093  {
3094  unsigned char buff[MEMMAP_EXTRA_MARGIN];
3095  memset(buff, 0, sizeof(buff));
3096  if (info->rec_cache.write(buff, sizeof(buff)))
3097  {
3098  mi_check_print_error(sort_info->param,
3099  "%d when writing to datafile",errno);
3100  return 1;
3101  }
3102  sort_info->param->read_cache.end_of_file+=sizeof(buff);
3103  }
3104  return 0;
3105 }
3106 
3107  /* Update state and myisamchk_time of indexfile */
3108 
3109 int update_state_info(MI_CHECK *param, MI_INFO *info,uint32_t update)
3110 {
3111  MYISAM_SHARE *share=info->s;
3112 
3113  if (update & UPDATE_OPEN_COUNT)
3114  {
3115  share->state.open_count=0;
3116  share->global_changed=0;
3117  }
3118  if (update & UPDATE_STAT)
3119  {
3120  uint32_t i, key_parts= mi_uint2korr(share->state.header.key_parts);
3121  share->state.rec_per_key_rows=info->state->records;
3122  share->state.changed&= ~STATE_NOT_ANALYZED;
3123  if (info->state->records)
3124  {
3125  for (i=0; i<key_parts; i++)
3126  {
3127  if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
3128  share->state.changed|= STATE_NOT_ANALYZED;
3129  }
3130  }
3131  }
3132  if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
3133  {
3134  if (update & UPDATE_TIME)
3135  {
3136  share->state.check_time= (long) time((time_t*) 0);
3137  if (!share->state.create_time)
3138  share->state.create_time=share->state.check_time;
3139  }
3140  /*
3141  When tables are locked we haven't synched the share state and the
3142  real state for a while so we better do it here before synching
3143  the share state to disk. Only when table is write locked is it
3144  necessary to perform this synch.
3145  */
3146  if (info->lock_type == F_WRLCK)
3147  share->state.state= *info->state;
3148  if (mi_state_info_write(share->kfile,&share->state,1+2))
3149  goto err;
3150  share->changed=0;
3151  }
3152  { /* Force update of status */
3153  int error;
3154  uint32_t r_locks=share->r_locks,w_locks=share->w_locks;
3155  share->r_locks= share->w_locks= share->tot_locks= 0;
3156  error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
3157  share->r_locks=r_locks;
3158  share->w_locks=w_locks;
3159  share->tot_locks=r_locks+w_locks;
3160  if (!error)
3161  return 0;
3162  }
3163 err:
3164  mi_check_print_error(param,"%d when updating keyfile",errno);
3165  return 1;
3166 }
3167 
3168  /*
3169  Update auto increment value for a table
3170  When setting the 'repair_only' flag we only want to change the
3171  old auto_increment value if its wrong (smaller than some given key).
3172  The reason is that we shouldn't change the auto_increment value
3173  for a table without good reason when only doing a repair; If the
3174  user have inserted and deleted rows, the auto_increment value
3175  may be bigger than the biggest current row and this is ok.
3176 
3177  If repair_only is not set, we will update the flag to the value in
3178  param->auto_increment is bigger than the biggest key.
3179  */
3180 
3181 void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
3182  bool repair_only)
3183 {
3184  unsigned char *record= 0;
3185 
3186  if (!info->s->base.auto_key ||
3187  ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
3188  {
3189  if (!(param->testflag & T_VERY_SILENT))
3190  mi_check_print_info(param,
3191  "Table: %s doesn't have an auto increment key\n",
3192  param->isam_file_name);
3193  return;
3194  }
3195  if (!(param->testflag & T_SILENT) &&
3196  !(param->testflag & T_REP))
3197  printf("Updating MyISAM file: %s\n", param->isam_file_name);
3198  /*
3199  We have to use an allocated buffer instead of info->rec_buff as
3200  _mi_put_key_in_record() may use info->rec_buff
3201  */
3202  if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
3203  {
3204  mi_check_print_error(param,"Not enough memory for extra record");
3205  return;
3206  }
3207 
3208  mi_extra(info,HA_EXTRA_KEYREAD,0);
3209  if (mi_rlast(info, record, info->s->base.auto_key-1))
3210  {
3211  if (errno != HA_ERR_END_OF_FILE)
3212  {
3213  mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
3214  free(mi_get_rec_buff_ptr(info, record));
3215  mi_check_print_error(param,"%d when reading last record",errno);
3216  return;
3217  }
3218  if (!repair_only)
3219  info->s->state.auto_increment=param->auto_increment_value;
3220  }
3221  else
3222  {
3223  uint64_t auto_increment= retrieve_auto_increment(info, record);
3224  set_if_bigger(info->s->state.auto_increment,auto_increment);
3225  if (!repair_only)
3226  set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
3227  }
3228  mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
3229  free(mi_get_rec_buff_ptr(info, record));
3230  update_state_info(param, info, UPDATE_AUTO_INC);
3231  return;
3232 }
3233 
3234 
3235 /*
3236  Update statistics for each part of an index
3237 
3238  SYNOPSIS
3239  update_key_parts()
3240  keyinfo IN Index information (only key->keysegs used)
3241  rec_per_key_part OUT Store statistics here
3242  unique IN Array of (#distinct tuples)
3243  notnull_tuples IN Array of (#tuples), or NULL
3244  records Number of records in the table
3245 
3246  DESCRIPTION
3247  This function is called produce index statistics values from unique and
3248  notnull_tuples arrays after these arrays were produced with sequential
3249  index scan (the scan is done in two places: chk_index() and
3250  sort_key_write()).
3251 
3252  This function handles all 3 index statistics collection methods.
3253 
3254  Unique is an array:
3255  unique[0]= (#different values of {keypart1}) - 1
3256  unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
3257  ...
3258 
3259  For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
3260  notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
3261  notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all
3262  keypart{i} are not NULL)
3263  ...
3264  For all other statistics collection methods notnull_tuples==NULL.
3265 
3266  Output is an array:
3267  rec_per_key_part[k] =
3268  = E(#records in the table such that keypart_1=c_1 AND ... AND
3269  keypart_k=c_k for arbitrary constants c_1 ... c_k)
3270 
3271  = {assuming that values have uniform distribution and index contains all
3272  tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
3273  index tuples}
3274 
3275  = #tuples-in-the-index / #distinct-tuples-in-the-index.
3276 
3277  The #tuples-in-the-index and #distinct-tuples-in-the-index have different
3278  meaning depending on which statistics collection method is used:
3279 
3280  MI_STATS_METHOD_* how are nulls compared? which tuples are counted?
3281  NULLS_EQUAL NULL == NULL all tuples in table
3282  NULLS_NOT_EQUAL NULL != NULL all tuples in table
3283  IGNORE_NULLS n/a tuples that don't have NULLs
3284 */
3285 
3286 void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
3287  uint64_t *unique, uint64_t *notnull,
3288  uint64_t records)
3289 {
3290  uint64_t count=0,tmp, unique_tuples;
3291  uint64_t tuples= records;
3292  uint32_t parts;
3293  for (parts=0 ; parts < keyinfo->keysegs ; parts++)
3294  {
3295  count+=unique[parts];
3296  unique_tuples= count + 1;
3297  if (notnull)
3298  {
3299  tuples= notnull[parts];
3300  /*
3301  #(unique_tuples not counting tuples with NULLs) =
3302  #(unique_tuples counting tuples with NULLs as different) -
3303  #(tuples with NULLs)
3304  */
3305  unique_tuples -= (records - notnull[parts]);
3306  }
3307 
3308  if (unique_tuples == 0)
3309  tmp= 1;
3310  else if (count == 0)
3311  tmp= tuples; /* 1 unique tuple */
3312  else
3313  tmp= (tuples + unique_tuples/2) / unique_tuples;
3314 
3315  /*
3316  for some weird keys (e.g. FULLTEXT) tmp can be <1 here.
3317  let's ensure it is not
3318  */
3319  if (tmp < 1)
3320  tmp= 1;
3321  if (tmp >= (uint64_t) ~(ulong) 0)
3322  tmp=(uint64_t) ~(ulong) 0;
3323 
3324  *rec_per_key_part=(ulong) tmp;
3325  rec_per_key_part++;
3326  }
3327 }
3328 
3329 
3330 static ha_checksum mi_byte_checksum(const unsigned char *buf, uint32_t length)
3331 {
3332  ha_checksum crc;
3333  const unsigned char *end=buf+length;
3334  for (crc=0; buf != end; buf++)
3335  crc=((crc << 1) + *((unsigned char*) buf)) +
3336  test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
3337  return crc;
3338 }
3339 
3340 static bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
3341 {
3342  uint32_t key_maxlength=key->maxlength;
3343  return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY) &&
3344  ((uint64_t) rows * key_maxlength >
3345  (uint64_t) MAX_FILE_SIZE));
3346 }
3347 
3348 /*
3349  Deactivate all not unique index that can be recreated fast
3350  These include packed keys on which sorting will use more temporary
3351  space than the max allowed file length or for which the unpacked keys
3352  will take much more space than packed keys.
3353  Note that 'rows' may be zero for the case when we don't know how many
3354  rows we will put into the file.
3355  */
3356 
3357 void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
3358 {
3359  MYISAM_SHARE *share=info->s;
3360  MI_KEYDEF *key=share->keyinfo;
3361  uint32_t i;
3362 
3363  assert(info->state->records == 0 &&
3364  (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
3365  for (i=0 ; i < share->base.keys ; i++,key++)
3366  {
3367  if (!(key->flag & (HA_NOSAME | HA_AUTO_KEY)) &&
3368  ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
3369  {
3370  mi_clear_key_active(share->state.key_map, i);
3371  info->update|= HA_STATE_CHANGED;
3372  }
3373  }
3374 }
3375 
3376 
3377 /*
3378  Return true if we can use repair by sorting
3379  One can set the force argument to force to use sorting
3380  even if the temporary file would be quite big!
3381 */
3382 
3383 bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
3384  uint64_t key_map, bool force)
3385 {
3386  MYISAM_SHARE *share=info->s;
3387  MI_KEYDEF *key=share->keyinfo;
3388  uint32_t i;
3389 
3390  /*
3391  mi_repair_by_sort only works if we have at least one key. If we don't
3392  have any keys, we should use the normal repair.
3393  */
3394  if (! mi_is_any_key_active(key_map))
3395  return false; /* Can't use sort */
3396  for (i=0 ; i < share->base.keys ; i++,key++)
3397  {
3398  if (!force && mi_too_big_key_for_sort(key,rows))
3399  return false;
3400  }
3401  return true;
3402 }
3403 
3404 
3405 static void
3406 set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
3407 {
3408  if ((sort_info->new_data_file_type=share->data_file_type) ==
3409  COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
3410  {
3411  MYISAM_SHARE tmp;
3412 
3413  if (share->options & HA_OPTION_PACK_RECORD)
3414  sort_info->new_data_file_type = DYNAMIC_RECORD;
3415  else
3416  sort_info->new_data_file_type = STATIC_RECORD;
3417 
3418  /* Set delete_function for sort_delete_record() */
3419  memcpy(&tmp, share, sizeof(*share));
3420  tmp.options= ~HA_OPTION_COMPRESS_RECORD;
3421  mi_setup_functions(&tmp);
3422  share->delete_record=tmp.delete_record;
3423  }
3424 }