43#include "myisam_priv.h"
44#include <drizzled/internal/m_string.h>
46#ifdef HAVE_SYS_VADVISE_H
47#include <sys/vadvise.h>
53#include <drizzled/util/test.h>
54#include <drizzled/error.h>
61using namespace drizzled::internal;
64#define my_off_t2double(A) ((double) (my_off_t) (A))
68static int check_k_link(MI_CHECK *param, MI_INFO *info,uint32_t nr);
69static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
70 my_off_t page,
unsigned char *buff, ha_rows *keys,
71 ha_checksum *key_checksum, uint32_t level);
72static uint32_t isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
73static ha_checksum calc_checksum(ha_rows count);
74static int writekeys(MI_SORT_PARAM *sort_param);
75static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
76 my_off_t pagepos,
int new_file);
77int sort_key_read(MI_SORT_PARAM *sort_param,
void *key);
78int sort_get_next_record(MI_SORT_PARAM *sort_param);
79int sort_key_cmp(MI_SORT_PARAM *sort_param,
const void *a,
const void *b);
80int sort_key_write(MI_SORT_PARAM *sort_param,
const void *a);
81my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
83int sort_insert_key(MI_SORT_PARAM *sort_param,
84 SORT_KEY_BLOCKS *key_block,
85 unsigned char *key, my_off_t prev_block);
86int sort_delete_record(MI_SORT_PARAM *sort_param);
89static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
90 uint32_t buffer_length);
91static ha_checksum mi_byte_checksum(
const unsigned char *buf, uint32_t length);
92static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
94void myisamchk_init(MI_CHECK *param)
96 new (param) MI_CHECK();
97 param->opt_follow_links=1;
98 param->keys_in_use= ~(uint64_t) 0;
99 param->search_after_block=HA_OFFSET_ERROR;
100 param->auto_increment_value= 0;
101 param->use_buffers=USE_BUFFER_INIT;
102 param->read_buffer_length=READ_BUFFER_INIT;
103 param->write_buffer_length=READ_BUFFER_INIT;
104 param->sort_buffer_length=SORT_BUFFER_INIT;
105 param->sort_key_blocks=BUFFERS_WHEN_SORTING;
106 param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
107 param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
108 param->start_check_pos=0;
109 param->max_record_length= INT64_MAX;
110 param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
111 param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
116int chk_status(MI_CHECK *param, MI_INFO *info)
118 MYISAM_SHARE *share=info->s;
120 if (mi_is_crashed_on_repair(info))
121 mi_check_print_warning(param,
122 "Table is marked as crashed and last repair failed");
123 else if (mi_is_crashed(info))
124 mi_check_print_warning(param,
125 "Table is marked as crashed");
126 if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
129 uint32_t save=param->warning_printed;
130 mi_check_print_warning(param,
131 share->state.open_count==1 ?
132 "%d client is using or hasn't closed the table properly" :
133 "%d clients are using or haven't closed the table properly",
134 share->state.open_count);
136 if (param->testflag & T_UPDATE_STATE)
137 param->warning_printed=save;
144int chk_del(MI_CHECK *param, MI_INFO *info, uint32_t test_flag)
146 uint32_t delete_link_length;
147 my_off_t empty, next_link, old_link= 0;
148 char buff[22],buff2[22];
150 param->record_checksum=0;
151 delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 : info->s->rec_reflength+1);
153 if (!(test_flag & T_SILENT))
154 puts(
"- check record delete-chain");
156 next_link=info->s->state.dellink;
157 if (info->state->del == 0)
159 if (test_flag & T_VERBOSE)
161 puts(
"No recordlinks");
166 if (test_flag & T_VERBOSE)
167 printf(
"Recordlinks: ");
169 ha_rows i= info->state->del;
170 for (; i > 0 && next_link != HA_OFFSET_ERROR; i--)
172 if (*killed_ptr(param))
174 if (test_flag & T_VERBOSE)
175 printf(
" %9s",llstr(next_link,buff));
176 if (next_link >= info->state->data_file_length)
178 if (my_pread(info->dfile, (
unsigned char*) buff,delete_link_length, next_link,MYF(MY_NABP)))
180 if (test_flag & T_VERBOSE) puts(
"");
181 mi_check_print_error(param,
"Can't read delete-link at filepos: %s",
182 llstr(next_link,buff));
187 if (test_flag & T_VERBOSE) puts(
"");
188 mi_check_print_error(param,
"Record at pos: %s is not remove-marked",
189 llstr(next_link,buff));
192 if (info->s->options & HA_OPTION_PACK_RECORD)
194 my_off_t prev_link=mi_sizekorr(buff+12);
195 if (empty && prev_link != old_link)
197 if (test_flag & T_VERBOSE) puts(
"");
198 mi_check_print_error(param,
"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
202 next_link=mi_sizekorr(buff+4);
203 empty+=mi_uint3korr(buff+1);
207 param->record_checksum+=(ha_checksum) next_link;
208 next_link=_mi_rec_pos(info->s,(
unsigned char*) buff+1);
209 empty+=info->s->base.pack_reclength;
212 if (test_flag & T_VERBOSE)
214 if (empty != info->state->empty)
216 mi_check_print_warning(param,
217 "Found %s deleted space in delete link chain. Should be %s",
219 llstr(info->state->empty,buff));
221 if (next_link != HA_OFFSET_ERROR)
223 mi_check_print_error(param,
224 "Found more than the expected %s deleted rows in delete link chain",
225 llstr(info->state->del, buff));
230 mi_check_print_error(param,
"Found %s deleted rows in delete link chain. Should be %s", llstr(info->state->del - i, buff2), llstr(info->state->del, buff));
237 param->testflag|=T_RETRY_WITHOUT_QUICK;
238 if (test_flag & T_VERBOSE) puts(
"");
239 mi_check_print_error(param,
"record delete-link-chain corrupted");
246static int check_k_link(MI_CHECK *param, MI_INFO *info, uint32_t nr)
249 uint32_t block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
251 char llbuff[21], llbuff2[21];
254 if (param->testflag & T_VERBOSE)
255 printf(
"block_size %4u:", block_size);
257 next_link=info->s->state.key_del[nr];
258 records= (ha_rows) (info->state->key_file_length / block_size);
259 while (next_link != HA_OFFSET_ERROR && records > 0)
261 if (*killed_ptr(param))
263 if (param->testflag & T_VERBOSE)
264 printf(
"%16s",llstr(next_link,llbuff));
267 if (next_link + block_size > info->state->key_file_length)
269 mi_check_print_error(param,
"Invalid key block position: %s "
270 "key block size: %u file_length: %s",
271 llstr(next_link, llbuff), block_size,
272 llstr(info->state->key_file_length, llbuff2));
277 if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
279 mi_check_print_error(param,
"Mis-aligned key block: %s "
280 "minimum key block length: %u",
281 llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
291 if (not pread(info->s->kfile, info->buff, MI_MIN_KEY_BLOCK_LENGTH, next_link))
293 mi_check_print_error(param,
"key cache read error for block: %s", llstr(next_link,llbuff));
297 next_link=mi_sizekorr(buff);
299 param->key_file_blocks+=block_size;
301 if (param->testflag & T_VERBOSE)
303 if (next_link != HA_OFFSET_ERROR)
304 printf(
"%16s\n",llstr(next_link,llbuff));
308 return (next_link != HA_OFFSET_ERROR);
314int chk_size(MI_CHECK *param, MI_INFO *info)
318 char buff[22],buff2[22];
320 if (!(param->testflag & T_SILENT)) puts(
"- check file-size");
322 size= lseek(info->s->kfile, 0, SEEK_END);
323 if ((skr=(my_off_t) info->state->key_file_length) != size)
326 if (skr > size && mi_is_any_key_active(info->s->state.key_map))
329 mi_check_print_error(param,
330 "Size of indexfile is: %-8s Should be: %s",
331 llstr(size,buff), llstr(skr,buff2));
334 mi_check_print_warning(param,
335 "Size of indexfile is: %-8s Should be: %s",
336 llstr(size,buff), llstr(skr,buff2));
338 if (!(param->testflag & T_VERY_SILENT) &&
339 ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
340 uint64_t2double(info->state->key_file_length) >
341 uint64_t2double(info->s->base.margin_key_file_length)*0.9)
342 mi_check_print_warning(param,
"Keyfile is almost full, %10s of %10s used",
343 llstr(info->state->key_file_length,buff),
344 llstr(info->s->base.max_key_file_length-1,buff));
346 size=lseek(info->dfile,0L,SEEK_END);
347 skr=(my_off_t) info->state->data_file_length;
348 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
349 skr+= MEMMAP_EXTRA_MARGIN;
351 if (info->data_file_type == STATIC_RECORD &&
352 skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
353 skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
357 info->state->data_file_length=size;
358 if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
361 mi_check_print_error(param,
"Size of datafile is: %-9s Should be: %s",
362 llstr(size,buff), llstr(skr,buff2));
363 param->testflag|=T_RETRY_WITHOUT_QUICK;
367 mi_check_print_warning(param,
368 "Size of datafile is: %-9s Should be: %s",
369 llstr(size,buff), llstr(skr,buff2));
372 if (!(param->testflag & T_VERY_SILENT) &&
373 !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
374 uint64_t2double(info->state->data_file_length) >
375 (uint64_t2double(info->s->base.max_data_file_length)*0.9))
376 mi_check_print_warning(param,
"Datafile is almost full, %10s of %10s used",
377 llstr(info->state->data_file_length,buff),
378 llstr(info->s->base.max_data_file_length-1,buff2));
385int chk_key(MI_CHECK *param, MI_INFO *info)
387 uint32_t key,found_keys=0,full_text_keys=0,result=0;
389 ha_checksum old_record_checksum,init_checksum;
390 my_off_t all_keydata,all_totaldata,key_totlength,length;
391 ulong *rec_per_key_part;
392 MYISAM_SHARE *share=info->s;
394 char buff[22],buff2[22];
396 if (!(param->testflag & T_SILENT))
397 puts(
"- check key delete-chain");
399 param->key_file_blocks=info->s->base.keystart;
400 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
401 if (check_k_link(param,info,key))
403 if (param->testflag & T_VERBOSE) puts(
"");
404 mi_check_print_error(param,
"key delete-link-chain corrupted");
408 if (!(param->testflag & T_SILENT)) puts(
"- check index reference");
410 all_keydata=all_totaldata=key_totlength=0;
411 old_record_checksum=0;
412 init_checksum=param->record_checksum;
413 if (!(share->options &
414 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
415 old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
416 share->base.pack_reclength;
417 rec_per_key_part= param->rec_per_key_part;
418 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
419 rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
421 param->key_crc[key]=0;
422 if (! mi_is_key_active(share->state.key_map, key))
425 assert(rec_per_key_part >= param->rec_per_key_part);
426 memcpy(rec_per_key_part,
427 (share->state.rec_per_key_part +
428 (rec_per_key_part - param->rec_per_key_part)),
429 keyinfo->keysegs*
sizeof(*rec_per_key_part));
434 param->record_checksum=init_checksum;
436 memset(¶m->unique_count, 0,
sizeof(param->unique_count));
437 memset(¶m->notnull_count, 0,
sizeof(param->notnull_count));
439 if ((!(param->testflag & T_SILENT)))
440 printf (
"- check data record references index: %d\n",key+1);
441 if (share->state.key_root[key] == HA_OFFSET_ERROR && (info->state->records == 0))
443 if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
444 DFLT_INIT_HITS,info->buff,0))
446 mi_check_print_error(param,
"Can't read indexpage from filepos: %s",
447 llstr(share->state.key_root[key],buff));
448 if (!(param->testflag & T_INFO))
453 param->key_file_blocks+=keyinfo->block_length;
455 param->keydata=param->totaldata=0;
458 if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
459 &keys, param->key_crc+key,1))
463 if (keys != info->state->records)
465 mi_check_print_error(param,
"Found %s keys of %s",llstr(keys,buff),
466 llstr(info->state->records,buff2));
467 if (!(param->testflag & T_INFO))
472 if (found_keys - full_text_keys == 1 &&
474 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
475 (param->testflag & T_DONT_CHECK_CHECKSUM)))
476 old_record_checksum=param->record_checksum;
477 else if (old_record_checksum != param->record_checksum)
480 mi_check_print_error(param,
"Key %u doesn't point at same records that key 1",
483 mi_check_print_error(param,
"Key 1 doesn't point at all records");
484 if (!(param->testflag & T_INFO))
490 if ((uint) share->base.auto_key -1 == key)
493 uint64_t auto_increment;
495 _mi_read_key_record(info, 0L, info->rec_buff);
496 auto_increment= retrieve_auto_increment(info, info->rec_buff);
497 if (auto_increment > info->s->state.auto_increment)
499 mi_check_print_warning(param,
"Auto-increment value: %s is smaller "
500 "than max used value: %s",
501 llstr(info->s->state.auto_increment,buff2),
502 llstr(auto_increment, buff));
504 if (param->testflag & T_AUTO_INC)
506 set_if_bigger(info->s->state.auto_increment,
508 set_if_bigger(info->s->state.auto_increment,
509 param->auto_increment_value);
513 mi_extra(info,HA_EXTRA_KEYREAD,0);
514 memset(info->lastkey, 0, keyinfo->seg->length);
515 if (!mi_rkey(info, info->rec_buff, key, (
const unsigned char*) info->lastkey,
516 (key_part_map)1, HA_READ_KEY_EXACT))
519 uint32_t save=param->warning_printed;
520 mi_check_print_warning(param,
"Found row where the auto_increment "
521 "column has the value 0");
522 param->warning_printed=save;
524 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
527 length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
528 if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
529 printf(
"Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n",
531 (
int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
532 (
int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
533 my_off_t2double(length)),
535 all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
538 if (param->testflag & T_STATISTICS)
539 update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
540 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
541 param->notnull_count: NULL,
542 (uint64_t)info->state->records);
544 if (param->testflag & T_INFO)
546 if (all_totaldata != 0L && found_keys > 0)
547 printf(
"Total: Keyblocks used: %3d%% Packed: %4d%%\n\n",
548 (
int) (my_off_t2double(all_keydata)*100.0/
549 my_off_t2double(all_totaldata)),
550 (
int) ((my_off_t2double(key_totlength) -
551 my_off_t2double(all_keydata))*100.0/
552 my_off_t2double(key_totlength)));
553 else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
556 if (param->key_file_blocks != info->state->key_file_length &&
557 param->keys_in_use != ~(uint64_t) 0)
558 mi_check_print_warning(param,
"Some data are unreferenced in keyfile");
559 if (found_keys != full_text_keys)
560 param->record_checksum=old_record_checksum-init_checksum;
562 param->record_checksum=0;
567static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
568 my_off_t page,
unsigned char *buff, ha_rows *keys,
569 ha_checksum *key_checksum, uint32_t level)
571 char llbuff[22],llbuff2[22];
574 if (page + keyinfo->block_length > info->state->key_file_length)
577 my_off_t max_length= lseek(info->s->kfile, 0, SEEK_END);
578 mi_check_print_error(param,
"Invalid key block position: %s "
579 "key block size: %u file_length: %s",
580 llstr(page, llbuff), keyinfo->block_length,
581 llstr(info->state->key_file_length, llbuff2));
582 if (page + keyinfo->block_length > max_length)
585 info->state->key_file_length= (max_length &
586 ~ (my_off_t) (keyinfo->block_length - 1));
590 if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
592 mi_check_print_error(param,
"Mis-aligned key block: %s "
593 "minimum key block length: %u",
594 llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
598 if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
600 mi_check_print_error(param,
"Can't read key from filepos: %s",
604 param->key_file_blocks+=keyinfo->block_length;
605 if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
631void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, uint64_t *notnull,
634 uint32_t first_null, kp;
635 first_null= ha_find_null(keyseg, key) - keyseg;
640 for (kp= 0; kp < first_null; kp++)
671int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, uint64_t *notnull,
672 unsigned char *prev_key,
unsigned char *last_key)
675 uint32_t first_null_seg, kp;
686 ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
687 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
688 seg= keyseg + diffs[0] - 1;
691 first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
692 for (kp= 0; kp < first_null_seg; kp++)
706static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
707 my_off_t page,
unsigned char *buff, ha_rows *keys,
708 ha_checksum *key_checksum, uint32_t level)
711 uint32_t used_length,comp_flag,nod_flag,key_length=0;
712 unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*endpos;
713 my_off_t next_page,record;
715 uint32_t diff_pos[2];
717 if (!(temp_buff=(
unsigned char*) malloc(keyinfo->block_length)))
719 mi_check_print_error(param,
"Not enough memory for keyblock");
723 if (keyinfo->flag & HA_NOSAME)
724 comp_flag=SEARCH_FIND | SEARCH_UPDATE;
726 comp_flag=SEARCH_SAME;
727 nod_flag=mi_test_if_nod(buff);
728 used_length=mi_getint(buff);
729 keypos=buff+2+nod_flag;
730 endpos=buff+used_length;
732 param->keydata+=used_length; param->totaldata+=keyinfo->block_length;
734 if (level > param->max_level)
735 param->max_level=level;
737 if (used_length > keyinfo->block_length)
739 mi_check_print_error(param,
"Wrong pageinfo at page: %s",
745 if (*killed_ptr(param))
747 memcpy(info->lastkey,key,key_length);
748 info->lastkey_length=key_length;
751 next_page=_mi_kpos(nod_flag,keypos);
752 if (chk_index_down(param,info,keyinfo,next_page,
753 temp_buff,keys,key_checksum,level+1))
756 if (keypos >= endpos ||
757 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
759 assert(key_length <=
sizeof(key));
762 mi_check_print_error(param,
"Wrong key block length at page: %s",llstr(page,llbuff));
766 (flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
767 comp_flag, diff_pos)) >=0)
769 if (comp_flag & SEARCH_FIND && flag == 0)
770 mi_check_print_error(param,
"Found duplicated key at page %s",llstr(page,llbuff));
772 mi_check_print_error(param,
"Key in wrong position at page %s",llstr(page,llbuff));
775 if (param->testflag & T_STATISTICS)
779 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
780 ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
781 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
783 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
785 diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg,
786 param->notnull_count,
789 param->unique_count[diff_pos[0]-1]++;
793 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
794 mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
798 (*key_checksum)+= mi_byte_checksum((
unsigned char*) key,
799 key_length- info->s->rec_reflength);
800 record= _mi_dpos(info,0,key+key_length);
801 if (record >= info->state->data_file_length)
803 mi_check_print_error(param,
"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
806 param->record_checksum+=(ha_checksum) record;
808 if (keypos != endpos)
810 mi_check_print_error(param,
"Keyblock size at page %s is not correct. Block length: %d key length: %d",
811 llstr(page,llbuff), used_length, (keypos - buff));
824static ha_checksum calc_checksum(ha_rows count)
840 return((ha_checksum) sum);
846static uint32_t isam_key_length(MI_INFO *info, MI_KEYDEF *keyinfo)
851 length= info->s->rec_reflength;
852 for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
853 length+= keyseg->length;
861int chk_data_link(MI_CHECK *param, MI_INFO *info,
int extend)
863 int error,got_error,flag;
864 uint key, left_length= 0, b_type;
865 ha_rows records, del_blocks;
866 my_off_t used, empty, pos, splits, start_recpos= 0,
867 del_length, link_used, start_block;
868 unsigned char *record= NULL, *to= NULL;
869 char llbuff[22],llbuff2[22],llbuff3[22];
870 ha_checksum intern_record_checksum;
871 ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
873 MI_BLOCK_INFO block_info;
875 if (!(param->testflag & T_SILENT))
878 puts(
"- check records and index references");
880 puts(
"- check record links");
883 if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
885 mi_check_print_error(param,
"Not enough memory for record");
888 records=del_blocks=0;
889 used=link_used=splits=del_length=0;
890 intern_record_checksum=param->glob_crc=0;
892 empty=info->s->pack.header_length;
894 pos= param->read_cache.tell();
895 memset(key_checksum, 0, info->s->base.keys *
sizeof(key_checksum[0]));
896 while (pos < info->state->data_file_length)
898 if (*killed_ptr(param))
900 switch (info->s->data_file_type) {
902 if (param->read_cache.read(record, info->s->base.pack_reclength))
905 pos+=info->s->base.pack_reclength;
910 del_length+=info->s->base.pack_reclength;
913 param->glob_crc+= mi_static_checksum(info,record);
914 used+=info->s->base.pack_reclength;
917 flag=block_info.second_read=0;
918 block_info.next_filepos=pos;
921 if (_mi_read_cache(¶m->read_cache,(
unsigned char*) block_info.header,
922 (start_block=block_info.next_filepos),
923 sizeof(block_info.header),
924 (flag ? 0 : READING_NEXT) | READING_HEADER))
926 if (start_block & (MI_DYN_ALIGN_SIZE-1))
928 mi_check_print_error(param,
"Wrong aligned block at %s",
929 llstr(start_block,llbuff));
932 b_type=_mi_get_block_info(&block_info,-1,start_block);
933 if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
936 if (b_type & BLOCK_SYNC_ERROR)
940 mi_check_print_error(param,
"Unexpected byte: %d at link: %s",
941 (
int) block_info.header[0],
942 llstr(start_block,llbuff));
945 pos=block_info.filepos+block_info.block_len;
948 if (b_type & BLOCK_DELETED)
950 if (block_info.block_len < info->s->base.min_block_length)
952 mi_check_print_error(param,
953 "Deleted block with impossible length %lu at %s",
954 block_info.block_len,llstr(pos,llbuff));
957 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
958 block_info.next_filepos >= info->state->data_file_length) ||
959 (block_info.prev_filepos != HA_OFFSET_ERROR &&
960 block_info.prev_filepos >= info->state->data_file_length))
962 mi_check_print_error(param,
"Delete link points outside datafile at %s",
967 del_length+=block_info.block_len;
968 pos=block_info.filepos+block_info.block_len;
972 mi_check_print_error(param,
"Wrong bytesec: %d-%d-%d at linkstart: %s",
973 block_info.header[0],block_info.header[1],
974 block_info.header[2],
975 llstr(start_block,llbuff));
978 if (info->state->data_file_length < block_info.filepos+
979 block_info.block_len)
981 mi_check_print_error(param,
982 "Recordlink that points outside datafile at %s",
991 pos=block_info.filepos+block_info.block_len;
992 if (block_info.rec_len > (uint) info->s->base.max_pack_length)
994 mi_check_print_error(param,
"Found too long record (%lu) at %s",
995 (ulong) block_info.rec_len,
996 llstr(start_recpos,llbuff));
1000 if (info->s->base.blobs)
1002 if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1005 mi_check_print_error(param,
1006 "Not enough memory (%lu) for blob at %s",
1007 (ulong) block_info.rec_len,
1008 llstr(start_recpos,llbuff));
1015 left_length=block_info.rec_len;
1017 if (left_length < block_info.data_len)
1019 mi_check_print_error(param,
"Found too long record (%lu) at %s",
1020 (ulong) block_info.data_len,
1021 llstr(start_recpos,llbuff));
1025 if (_mi_read_cache(¶m->read_cache,(
unsigned char*) to,block_info.filepos,
1026 (uint) block_info.data_len,
1027 flag == 1 ? READING_NEXT : 0))
1029 to+=block_info.data_len;
1030 link_used+= block_info.filepos-start_block;
1031 used+= block_info.filepos - start_block + block_info.data_len;
1032 empty+=block_info.block_len-block_info.data_len;
1033 left_length-=block_info.data_len;
1036 if (b_type & BLOCK_LAST)
1038 mi_check_print_error(param,
1039 "Wrong record length %s of %s at %s",
1040 llstr(block_info.rec_len-left_length,llbuff),
1041 llstr(block_info.rec_len, llbuff2),
1042 llstr(start_recpos,llbuff3));
1046 if (info->state->data_file_length < block_info.next_filepos)
1048 mi_check_print_error(param,
1049 "Found next-recordlink that points outside datafile at %s",
1050 llstr(block_info.filepos,llbuff));
1055 }
while (left_length);
1058 if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
1061 mi_check_print_error(param,
"Found wrong record at %s",
1062 llstr(start_recpos,llbuff));
1067 info->checksum=mi_checksum(info,record);
1068 if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
1070 if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
1071 test(info->s->calc_checksum)))
1073 mi_check_print_error(param,
"Found wrong packed record at %s",
1074 llstr(start_recpos,llbuff));
1079 param->glob_crc+= info->checksum;
1083 pos=block_info.filepos+block_info.block_len;
1085 case COMPRESSED_RECORD:
1091 intern_record_checksum+=(ha_checksum) start_recpos;
1093 if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
1095 printf(
"%s\r", llstr(records,llbuff)); fflush(stdout);
1100 for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
1103 if (mi_is_key_active(info->s->state.key_map, key))
1106 uint32_t key_length=_mi_make_key(info,key,info->lastkey,record,
1114 _mi_search(info,keyinfo,info->lastkey,key_length,
1115 SEARCH_SAME, info->s->state.key_root[key]);
1118 mi_check_print_error(param,
"Record at: %10s "
1119 "Can't find key for index: %2d",
1120 llstr(start_recpos,llbuff),key+1);
1121 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1126 key_checksum[key]+=mi_byte_checksum((
unsigned char*) info->lastkey,
1135 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1140 if (param->testflag & T_WRITE_LOOP)
1142 fputs(
" \r",stdout); fflush(stdout);
1144 if (records != info->state->records)
1146 mi_check_print_error(param,
"Record-count is not ok; is %-10s Should be: %s",
1147 llstr(records,llbuff), llstr(info->state->records,llbuff2));
1150 else if (param->record_checksum &&
1151 param->record_checksum != intern_record_checksum)
1153 mi_check_print_error(param,
1154 "Keypointers and record positions doesn't match");
1157 else if (param->glob_crc != info->state->checksum &&
1159 (HA_OPTION_COMPRESS_RECORD)))
1161 mi_check_print_warning(param,
1162 "Record checksum is not the same as checksum stored in the index file\n");
1167 for (key=0 ; key < info->s->base.keys; key++)
1169 if (key_checksum[key] != param->key_crc[key])
1171 mi_check_print_error(param,
"Checksum for key: %2d doesn't match checksum for records",
1178 if (del_length != info->state->empty)
1180 mi_check_print_warning(param,
1181 "Found %s deleted space. Should be %s",
1182 llstr(del_length,llbuff2),
1183 llstr(info->state->empty,llbuff));
1185 if (used+empty+del_length != info->state->data_file_length)
1187 mi_check_print_warning(param,
1188 "Found %s record-data and %s unused data and %s deleted-data",
1189 llstr(used,llbuff),llstr(empty,llbuff2),
1190 llstr(del_length,llbuff3));
1191 mi_check_print_warning(param,
1192 "Total %s, Should be: %s",
1193 llstr((used+empty+del_length),llbuff),
1194 llstr(info->state->data_file_length,llbuff2));
1196 if (del_blocks != info->state->del)
1198 mi_check_print_warning(param,
1199 "Found %10s deleted blocks Should be: %s",
1200 llstr(del_blocks,llbuff),
1201 llstr(info->state->del,llbuff2));
1203 if (splits != info->s->state.split)
1205 mi_check_print_warning(param,
1206 "Found %10s parts Should be: %s parts",
1207 llstr(splits,llbuff),
1208 llstr(info->s->state.split,llbuff2));
1210 if (param->testflag & T_INFO)
1212 if (param->warning_printed || param->error_printed)
1214 if (used != 0 && ! param->error_printed)
1216 printf(
"Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n",
1217 llstr(records,llbuff), (
long)((used-link_used)/records),
1218 (info->s->base.blobs ? 0.0 :
1219 (uint64_t2double((uint64_t) info->s->base.reclength*records)-
1220 my_off_t2double(used))/
1221 uint64_t2double((uint64_t) info->s->base.reclength*records)*100.0));
1222 printf(
"Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n",
1223 (uint64_t2double(used-link_used)/uint64_t2double(used-link_used+empty)*100.0),
1224 (!records ? 100 : (
int) (uint64_t2double(del_length+empty)/
1225 my_off_t2double(used)*100.0)),
1226 uint64_t2double(splits - del_blocks) / records);
1228 printf(
"Record blocks:%12s Delete blocks:%10s\n",
1229 llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
1230 printf(
"Record data: %12s Deleted data: %10s\n",
1231 llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
1232 printf(
"Lost space: %12s Linkdata: %10s\n",
1233 llstr(empty,llbuff),llstr(link_used,llbuff2));
1235 free(mi_get_rec_buff_ptr(info, record));
1238 mi_check_print_error(param,
"got error: %d when reading datafile at record: %s",errno, llstr(records,llbuff));
1240 free(mi_get_rec_buff_ptr(info, record));
1241 param->testflag|=T_RETRY_WITHOUT_QUICK;
1299static void mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info,
bool force)
1301 MYISAM_SHARE *share= info->s;
1302 MI_STATE_INFO *state= &share->state;
1312 if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
1314 for (i= 0; i < share->base.keys; i++)
1316 if ((state->key_root[i] != HA_OFFSET_ERROR) &&
1317 !mi_is_key_active(state->key_map, i))
1327 if (i >= share->base.keys)
1341 mi_clear_all_keys_active(state->key_map);
1345 for (i= 0; i < share->base.keys; i++)
1346 state->key_root[i]= HA_OFFSET_ERROR;
1349 for (i= 0; i < state->header.max_block_size_index; i++)
1350 state->key_del[i]= HA_OFFSET_ERROR;
1353 info->state->key_file_length= share->base.keystart;
1360int mi_repair(MI_CHECK *param, MI_INFO *info,
1361 char * name,
int rep_quick)
1363 int error,got_error;
1364 ha_rows start_records,new_header_length;
1367 MYISAM_SHARE *share=info->s;
1368 char llbuff[22],llbuff2[22];
1369 SORT_INFO sort_info;
1370 MI_SORT_PARAM sort_param= MI_SORT_PARAM();
1372 memset(&sort_info, 0,
sizeof(sort_info));
1373 start_records=info->state->records;
1374 new_header_length=(param->testflag & T_UNPACK) ? 0L :
1375 share->pack.header_length;
1378 sort_param.sort_info=&sort_info;
1380 if (!(param->testflag & T_SILENT))
1382 printf(
"- recovering (with keycache) MyISAM-table '%s'\n",name);
1383 printf(
"Data records: %s\n", llstr(info->state->records,llbuff));
1385 param->testflag|=T_REP;
1387 if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
1388 param->testflag|=T_CALC_CHECKSUM;
1390 if (!param->using_global_keycache)
1393 if (param->read_cache.
init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1400 if (info->rec_cache.
init_io_cache(-1, (uint) param->write_buffer_length, WRITE_CACHE, new_header_length, 1, MYF(MY_WME | MY_WAIT_IF_FULL)))
1405 info->opt_flag|=WRITE_CACHE_USED;
1406 if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
1407 !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
1409 mi_check_print_error(param,
"Not enough memory for extra record");
1416 if ((new_file=my_create(internal::fn_format(param->temp_filename,
1417 share->data_file_name,
"",
1419 0,param->tmpfile_createflag,
1422 mi_check_print_error(param,
"Can't create new tempfile: '%s'",
1423 param->temp_filename);
1426 if (new_header_length &&
1427 filecopy(param,new_file,info->dfile,0L,new_header_length,
1430 info->s->state.dellink= HA_OFFSET_ERROR;
1431 info->rec_cache.file=new_file;
1432 if (param->testflag & T_UNPACK)
1434 share->options&= ~HA_OPTION_COMPRESS_RECORD;
1435 mi_int2store(share->state.header.options,share->options);
1438 sort_info.info=info;
1439 sort_info.param = param;
1440 sort_param.read_cache=param->read_cache;
1441 sort_param.pos=sort_param.max_pos=share->pack.header_length;
1442 sort_param.filepos=new_header_length;
1443 param->read_cache.end_of_file=sort_info.filelength=
1444 lseek(info->dfile,0L,SEEK_END);
1446 sort_param.fix_datafile= (bool) (! rep_quick);
1447 sort_param.master=1;
1448 sort_info.max_records= ~(ha_rows) 0;
1450 set_data_file_type(&sort_info, share);
1451 del=info->state->del;
1452 info->state->records=info->state->del=share->state.split=0;
1453 info->state->empty=0;
1455 if (param->testflag & T_CALC_CHECKSUM)
1456 sort_param.calc_checksum= 1;
1458 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1461 if (param->testflag & T_CREATE_MISSING_KEYS)
1462 mi_set_all_keys_active(share->state.key_map, share->base.keys);
1463 mi_drop_all_indexes(param, info,
true);
1468 while (!(error=sort_get_next_record(&sort_param)))
1470 if (writekeys(&sort_param))
1472 if (errno != HA_ERR_FOUND_DUPP_KEY)
1474 mi_check_print_info(param,
"Duplicate key %2d for record at %10s against new record at %10s",
1476 llstr(sort_param.start_recpos,llbuff),
1477 llstr(info->dupp_key_pos,llbuff2));
1478 if (param->testflag & T_VERBOSE)
1480 _mi_make_key(info,(uint) info->errkey,info->lastkey,
1481 sort_param.record,0L);
1484 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
1486 param->testflag|=T_RETRY_WITHOUT_QUICK;
1487 param->error_printed=1;
1492 if (sort_write_record(&sort_param))
1495 if (error > 0 || write_data_suffix(&sort_info, (
bool)!rep_quick) ||
1496 info->rec_cache.flush() || param->read_cache.error < 0)
1499 if (param->testflag & T_WRITE_LOOP)
1501 fputs(
" \r",stdout); fflush(stdout);
1503 if (ftruncate(share->kfile, info->state->key_file_length))
1505 mi_check_print_warning(param,
1506 "Can't change size of indexfile, error: %d",
1511 if (rep_quick && del+sort_info.dupp != info->state->del)
1513 mi_check_print_error(param,
"Couldn't fix table with quick recovery: Found wrong number of deleted records");
1514 mi_check_print_error(param,
"Run recovery again without -q");
1516 param->retry_repair=1;
1517 param->testflag|=T_RETRY_WITHOUT_QUICK;
1520 if (param->testflag & T_SAFE_REPAIR)
1523 if (info->state->records+1 < start_records)
1525 info->state->records=start_records;
1533 internal::my_close(info->dfile,MYF(0));
1534 info->dfile=new_file;
1535 info->state->data_file_length=sort_param.filepos;
1536 share->state.version=(ulong) time((time_t*) 0);
1540 info->state->data_file_length=sort_param.max_pos;
1542 if (param->testflag & T_CALC_CHECKSUM)
1543 info->state->checksum=param->glob_crc;
1545 if (!(param->testflag & T_SILENT))
1547 if (start_records != info->state->records)
1548 printf(
"Data records: %s\n", llstr(info->state->records,llbuff));
1550 mi_check_print_warning(param,
1551 "%s records have been removed",
1552 llstr(sort_info.dupp,llbuff));
1557 if (&share->state.state != info->state)
1558 memcpy( &share->state.state, info->state,
sizeof(*info->state));
1566 internal::my_close(new_file,MYF(0));
1567 info->dfile=new_file= -1;
1568 if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1569 DATA_TMP_EXT, share->base.raid_chunks,
1571 mi_open_datafile(info,share,-1))
1577 if (! param->error_printed)
1578 mi_check_print_error(param,
"%d for record at pos %s",errno,
1579 llstr(sort_param.start_recpos,llbuff));
1582 internal::my_close(new_file,MYF(0));
1583 my_delete(param->temp_filename, MYF(MY_WME));
1584 info->rec_cache.file=-1;
1586 mi_mark_crashed_on_repair(info);
1589 void * rec_buff_ptr= NULL;
1590 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
1592 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
1596 free(sort_info.buff);
1598 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
1600 if (not got_error && param->testflag & T_UNPACK)
1602 share->state.header.options[0]&= (
unsigned char) ~HA_OPTION_COMPRESS_RECORD;
1603 share->pack.header_length=0;
1604 share->data_file_type=sort_info.new_data_file_type;
1606 share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
1607 STATE_NOT_ANALYZED);
1614static int writekeys(MI_SORT_PARAM *sort_param)
1618 MI_INFO *info= sort_param->sort_info->info;
1619 unsigned char *buff= sort_param->record;
1620 my_off_t filepos= sort_param->filepos;
1622 key=info->lastkey+info->s->base.max_key_length;
1623 for (i=0 ; i < info->s->base.keys ; i++)
1625 if (mi_is_key_active(info->s->state.key_map, i))
1628 uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
1629 if (_mi_ck_write(info,i,key,key_length))
1637 if (errno == HA_ERR_FOUND_DUPP_KEY)
1639 info->errkey=(int) i;
1642 if (mi_is_key_active(info->s->state.key_map, i))
1645 uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
1646 if (_mi_ck_delete(info,i,key,key_length))
1653 if (sort_param->calc_checksum)
1654 sort_param->sort_info->param->glob_crc-= info->checksum;
1661int movepoint(MI_INFO *info,
unsigned char *record, my_off_t oldpos,
1662 my_off_t newpos, uint32_t prot_key)
1666 uint32_t key_length;
1668 key=info->lastkey+info->s->base.max_key_length;
1669 for (i=0 ; i < info->s->base.keys; i++)
1671 if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
1673 key_length=_mi_make_key(info,i,key,record,oldpos);
1674 if (info->s->keyinfo[i].flag & HA_NOSAME)
1678 keyinfo=info->s->keyinfo+i;
1679 if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
1680 (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
1681 info->s->state.key_root[i]))
1683 nod_flag=mi_test_if_nod(info->buff);
1684 _mi_dpointer(info,info->int_keypos-nod_flag-
1685 info->s->rec_reflength,newpos);
1686 if (_mi_write_keypage(info,keyinfo,info->last_keypage,
1687 DFLT_INIT_HITS,info->buff))
1692 if (_mi_ck_delete(info,i,key,key_length))
1694 key_length=_mi_make_key(info,i,key,record,newpos);
1695 if (_mi_ck_write(info,i,key,key_length))
1706void lock_memory(MI_CHECK *)
1713int mi_sort_index(MI_CHECK *param, MI_INFO *info,
char * name)
1718 my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
1719 uint32_t r_locks,w_locks;
1721 MYISAM_SHARE *share=info->s;
1722 MI_STATE_INFO old_state;
1725 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1728 if (!(param->testflag & T_SILENT))
1729 printf(
"- Sorting index for MyISAM-table '%s'\n",name);
1732 internal::fn_format(param->temp_filename,name,
"", MI_NAME_IEXT,2+4+32);
1733 if ((new_file=my_create(internal::fn_format(param->temp_filename,param->temp_filename,
1734 "", INDEX_TMP_EXT,2+4),
1735 0,param->tmpfile_createflag,MYF(0))) <= 0)
1737 mi_check_print_error(param,
"Can't create new tempfile: '%s'",
1738 param->temp_filename);
1741 if (filecopy(param, new_file,share->kfile,0L,
1742 (ulong) share->base.keystart,
"headerblock"))
1745 param->new_file_pos=share->base.keystart;
1746 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1749 if (! mi_is_key_active(info->s->state.key_map, key))
1752 if (share->state.key_root[key] != HA_OFFSET_ERROR)
1754 index_pos[key]=param->new_file_pos;
1755 if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
1760 index_pos[key]= HA_OFFSET_ERROR;
1763 share->state.version=(ulong) time(NULL);
1764 old_state= share->state;
1765 r_locks= share->r_locks;
1766 w_locks= share->w_locks;
1767 old_lock= info->lock_type;
1770 share->r_locks= share->w_locks= share->tot_locks= 0;
1771 (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
1772 internal::my_close(share->kfile,MYF(MY_WME));
1774 internal::my_close(new_file,MYF(MY_WME));
1775 if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
1777 mi_open_keyfile(share))
1779 info->lock_type= F_UNLCK;
1780 _mi_readinfo(info,F_WRLCK,0);
1781 info->lock_type= old_lock;
1782 share->r_locks= r_locks;
1783 share->w_locks= w_locks;
1784 share->tot_locks= r_locks+w_locks;
1785 share->state= old_state;
1787 info->state->key_file_length=param->new_file_pos;
1788 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1789 for (key=0 ; key < info->s->base.keys ; key++)
1790 info->s->state.key_root[key]=index_pos[key];
1791 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
1792 info->s->state.key_del[key]= HA_OFFSET_ERROR;
1794 info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
1798 internal::my_close(new_file,MYF(MY_WME));
1800 my_delete(param->temp_filename,MYF(MY_WME));
1807static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
1808 my_off_t pagepos,
int new_file)
1810 uint32_t length,nod_flag,used_length, key_length;
1811 unsigned char *buff,*keypos,*endpos;
1812 unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF];
1813 my_off_t new_page_pos,next_page;
1816 new_page_pos=param->new_file_pos;
1817 param->new_file_pos+=keyinfo->block_length;
1819 if (!(buff=(
unsigned char*) malloc(keyinfo->block_length)))
1821 mi_check_print_error(param,
"Not enough memory for key block");
1824 if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
1826 mi_check_print_error(param,
"Can't read key block from filepos: %s",
1827 llstr(pagepos,llbuff));
1830 if ((nod_flag=mi_test_if_nod(buff)))
1832 used_length=mi_getint(buff);
1833 keypos=buff+2+nod_flag;
1834 endpos=buff+used_length;
1839 next_page=_mi_kpos(nod_flag,keypos);
1840 _mi_kpointer(info,keypos-nod_flag,param->new_file_pos);
1841 if (sort_one_index(param,info,keyinfo,next_page,new_file))
1846 if (keypos >= endpos ||
1847 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
1849 assert(keypos <= endpos);
1854 length=mi_getint(buff);
1855 memset(buff+length, 0, keyinfo->block_length-length);
1856 if (my_pwrite(new_file,(
unsigned char*) buff,(uint) keyinfo->block_length,
1857 new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
1859 mi_check_print_error(param,
"Can't write indexblock, error: %d",errno);
1879int change_to_newfile(
const char * filename,
const char * old_ext,
1880 const char * new_ext,
1881 uint32_t raid_chunks,
1885 char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
1887 (void) internal::fn_format(old_filename,filename,
"",old_ext,2+4+32);
1888 return my_redel(old_filename,
1889 internal::fn_format(new_filename,old_filename,
"",new_ext,2+4),
1890 MYF(MY_WME | MY_LINK_WARNING | MyFlags));
1897int filecopy(MI_CHECK *param,
int to,
int from,my_off_t start,
1898 my_off_t length,
const char *
type)
1900 char tmp_buff[IO_SIZE],*buff;
1903 buff_length=(ulong) min(param->write_buffer_length, (
size_t)length);
1904 if (!(buff=(
char *)malloc(buff_length)))
1906 buff=tmp_buff; buff_length=IO_SIZE;
1909 lseek(from,start,SEEK_SET);
1910 while (length > buff_length)
1912 if (my_read(from,(
unsigned char*) buff,buff_length,MYF(MY_NABP)) ||
1913 my_write(to,(
unsigned char*) buff,buff_length,param->myf_rw))
1915 length-= buff_length;
1917 if (my_read(from,(
unsigned char*) buff,(uint) length,MYF(MY_NABP)) ||
1918 my_write(to,(
unsigned char*) buff,(uint) length,param->myf_rw))
1920 if (buff != tmp_buff)
1924 if (buff != tmp_buff)
1926 mi_check_print_error(param,
"Can't copy %s to tempfile, error %d",
1947int mi_repair_by_sort(MI_CHECK *param, MI_INFO *info,
1948 const char * name,
int rep_quick)
1953 ha_rows start_records;
1954 my_off_t new_header_length,del;
1956 MI_SORT_PARAM sort_param= MI_SORT_PARAM();
1957 MYISAM_SHARE *share=info->s;
1959 ulong *rec_per_key_part;
1961 SORT_INFO sort_info;
1962 uint64_t key_map= 0;
1964 start_records=info->state->records;
1967 new_header_length=(param->testflag & T_UNPACK) ? 0 :
1968 share->pack.header_length;
1969 if (!(param->testflag & T_SILENT))
1971 printf(
"- recovering (with sort) MyISAM-table '%s'\n",name);
1972 printf(
"Data records: %s\n", llstr(start_records,llbuff));
1974 param->testflag|=T_REP;
1976 if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
1977 param->testflag|=T_CALC_CHECKSUM;
1979 memset(&sort_info, 0,
sizeof(sort_info));
1980 if (!(sort_info.key_block=
1981 alloc_key_blocks(param, (uint) param->sort_key_blocks, share->base.max_key_block_length))
1982 || param->read_cache.
init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME))
1983 || (! rep_quick && info->rec_cache.
init_io_cache(info->dfile, (uint) param->write_buffer_length, WRITE_CACHE,new_header_length,1, MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
1987 sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
1988 info->opt_flag|=WRITE_CACHE_USED;
1989 info->rec_cache.file=info->dfile;
1991 if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
1992 !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
1994 mi_check_print_error(param,
"Not enough memory for extra record");
2000 if ((new_file=my_create(internal::fn_format(param->temp_filename,
2001 share->data_file_name,
"",
2003 0,param->tmpfile_createflag,
2006 mi_check_print_error(param,
"Can't create new tempfile: '%s'",
2007 param->temp_filename);
2010 if (new_header_length &&
2011 filecopy(param, new_file,info->dfile,0L,new_header_length,
2014 if (param->testflag & T_UNPACK)
2016 share->options&= ~HA_OPTION_COMPRESS_RECORD;
2017 mi_int2store(share->state.header.options,share->options);
2019 share->state.dellink= HA_OFFSET_ERROR;
2020 info->rec_cache.file=new_file;
2023 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2026 mi_drop_all_indexes(param, info,
false);
2027 key_map= share->state.key_map;
2028 if (param->testflag & T_CREATE_MISSING_KEYS)
2034 sort_info.info=info;
2035 sort_info.param = param;
2037 set_data_file_type(&sort_info, share);
2038 sort_param.filepos=new_header_length;
2041 param->read_cache.end_of_file=sort_info.filelength=
2042 lseek(param->read_cache.file,0L,SEEK_END);
2044 sort_param.wordlist=NULL;
2046 if (share->data_file_type == DYNAMIC_RECORD)
2047 length=max(share->base.min_pack_length+1,share->base.min_block_length);
2048 else if (share->data_file_type == COMPRESSED_RECORD)
2049 length=share->base.min_block_length;
2051 length=share->base.pack_reclength;
2052 sort_info.max_records=
2053 ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
2054 (ha_rows) (sort_info.filelength/length+1));
2055 sort_param.key_cmp=sort_key_cmp;
2056 sort_param.lock_in_memory=lock_memory;
2057 sort_param.sort_info=&sort_info;
2058 sort_param.fix_datafile= (bool) (! rep_quick);
2059 sort_param.master =1;
2061 del=info->state->del;
2063 if (param->testflag & T_CALC_CHECKSUM)
2064 sort_param.calc_checksum= 1;
2066 rec_per_key_part= param->rec_per_key_part;
2067 for (sort_param.key=0 ; sort_param.key < share->base.keys ;
2068 rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
2070 sort_param.read_cache=param->read_cache;
2071 sort_param.keyinfo=share->keyinfo+sort_param.key;
2072 sort_param.seg=sort_param.keyinfo->seg;
2077 if (! mi_is_key_active(key_map, sort_param.key))
2080 assert(rec_per_key_part >= param->rec_per_key_part);
2081 memcpy(rec_per_key_part,
2082 (share->state.rec_per_key_part +
2083 (rec_per_key_part - param->rec_per_key_part)),
2084 sort_param.keyinfo->keysegs*
sizeof(*rec_per_key_part));
2088 if ((!(param->testflag & T_SILENT)))
2089 printf (
"- Fixing index %d\n",sort_param.key+1);
2090 sort_param.max_pos=sort_param.pos=share->pack.header_length;
2091 keyseg=sort_param.seg;
2092 memset(sort_param.unique, 0,
sizeof(sort_param.unique));
2093 sort_param.key_length=share->rec_reflength;
2094 for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
2096 sort_param.key_length+=keyseg[i].length;
2097 if (keyseg[i].flag & HA_SPACE_PACK)
2098 sort_param.key_length+=get_pack_length(keyseg[i].length);
2099 if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2100 sort_param.key_length+=2 + test(keyseg[i].length >= 127);
2101 if (keyseg[i].flag & HA_NULL_PART)
2102 sort_param.key_length++;
2104 info->state->records=info->state->del=share->state.split=0;
2105 info->state->empty=0;
2108 sort_param.key_read=sort_key_read;
2109 sort_param.key_write=sort_key_write;
2112 if (_create_index_by_sort(&sort_param,
2113 (
bool) (!(param->testflag & T_VERBOSE)),
2114 (uint) param->sort_buffer_length))
2116 param->retry_repair=1;
2120 sort_param.calc_checksum= 0;
2124 sort_info.max_records= (ha_rows) info->state->records;
2126 if (param->testflag & T_STATISTICS)
2127 update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2128 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2129 sort_param.notnull: NULL,
2130 (uint64_t) info->state->records);
2132 mi_set_key_active(share->state.key_map, sort_param.key);
2134 if (sort_param.fix_datafile)
2136 param->read_cache.end_of_file=sort_param.filepos;
2137 if (write_data_suffix(&sort_info, 1) || info->rec_cache.
end_io_cache())
2141 if (param->testflag & T_SAFE_REPAIR)
2144 if (info->state->records+1 < start_records)
2146 info->state->records=start_records;
2150 share->state.state.data_file_length = info->state->data_file_length=
2153 share->state.version=(ulong) time((time_t*) 0);
2154 internal::my_close(info->dfile,MYF(0));
2155 info->dfile=new_file;
2156 share->data_file_type=sort_info.new_data_file_type;
2157 share->pack.header_length=(ulong) new_header_length;
2158 sort_param.fix_datafile=0;
2161 info->state->data_file_length=sort_param.max_pos;
2163 param->read_cache.file=info->dfile;
2164 param->read_cache.
reinit_io_cache(READ_CACHE,share->pack.header_length, 1,1);
2167 if (param->testflag & T_WRITE_LOOP)
2169 fputs(
" \r",stdout); fflush(stdout);
2172 if (rep_quick && del+sort_info.dupp != info->state->del)
2174 mi_check_print_error(param,
"Couldn't fix table with quick recovery: Found wrong number of deleted records");
2175 mi_check_print_error(param,
"Run recovery again without -q");
2177 param->retry_repair=1;
2178 param->testflag|=T_RETRY_WITHOUT_QUICK;
2182 if (rep_quick & T_FORCE_UNIQUENESS)
2184 my_off_t skr=info->state->data_file_length+
2185 (share->options & HA_OPTION_COMPRESS_RECORD ?
2186 MEMMAP_EXTRA_MARGIN : 0);
2188 if (share->data_file_type == STATIC_RECORD &&
2189 skr < share->base.reloc*share->base.min_pack_length)
2190 skr=share->base.reloc*share->base.min_pack_length;
2192 if (skr != sort_info.filelength && !info->s->base.raid_type)
2193 if (ftruncate(info->dfile, skr))
2194 mi_check_print_warning(param,
2195 "Can't change size of datafile, error: %d",
2198 if (param->testflag & T_CALC_CHECKSUM)
2199 info->state->checksum=param->glob_crc;
2201 if (ftruncate(share->kfile, info->state->key_file_length))
2202 mi_check_print_warning(param,
2203 "Can't change size of indexfile, error: %d",
2206 if (!(param->testflag & T_SILENT))
2208 if (start_records != info->state->records)
2209 printf(
"Data records: %s\n", llstr(info->state->records,llbuff));
2211 mi_check_print_warning(param,
2212 "%s records have been removed",
2213 llstr(sort_info.dupp,llbuff));
2217 if (&share->state.state != info->state)
2218 memcpy( &share->state.state, info->state,
sizeof(*info->state));
2227 internal::my_close(new_file,MYF(0));
2228 info->dfile=new_file= -1;
2229 if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2230 DATA_TMP_EXT, share->base.raid_chunks,
2232 mi_open_datafile(info,share,-1))
2238 if (! param->error_printed)
2239 mi_check_print_error(param,
"%d when fixing table",errno);
2242 internal::my_close(new_file,MYF(0));
2243 my_delete(param->temp_filename, MYF(MY_WME));
2244 if (info->dfile == new_file)
2247 mi_mark_crashed_on_repair(info);
2249 else if (key_map == share->state.key_map)
2250 share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
2251 share->state.changed|=STATE_NOT_SORTED_PAGES;
2253 void * rec_buff_ptr= NULL;
2254 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
2256 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
2260 free((
unsigned char*) sort_info.key_block);
2261 free(sort_info.buff);
2263 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2264 if (!got_error && (param->testflag & T_UNPACK))
2266 share->state.header.options[0]&= (
unsigned char) ~HA_OPTION_COMPRESS_RECORD;
2267 share->pack.header_length=0;
2274int sort_key_read(MI_SORT_PARAM *sort_param,
void *key)
2277 SORT_INFO *sort_info=sort_param->sort_info;
2278 MI_INFO *info=sort_info->info;
2280 if ((error=sort_get_next_record(sort_param)))
2282 if (info->state->records == sort_info->max_records)
2284 mi_check_print_error(sort_info->param,
2285 "Key %d - Found too many records; Can't continue",
2289 sort_param->real_key_length=
2290 (info->s->rec_reflength+
2291 _mi_make_key(info, sort_param->key, (
unsigned char*) key,
2292 sort_param->record, sort_param->filepos));
2294 memset((
unsigned char *)key+sort_param->real_key_length, 0,
2295 (sort_param->key_length-sort_param->real_key_length));
2297 return(sort_write_record(sort_param));
2332int sort_get_next_record(MI_SORT_PARAM *sort_param)
2336 uint32_t found_record,b_type,left_length;
2338 unsigned char *to= NULL;
2339 MI_BLOCK_INFO block_info;
2340 SORT_INFO *sort_info=sort_param->sort_info;
2341 MI_CHECK *param=sort_info->param;
2342 MI_INFO *info=sort_info->info;
2343 MYISAM_SHARE *share=info->s;
2344 char llbuff[22],llbuff2[22];
2346 if (*killed_ptr(param))
2349 switch (share->data_file_type) {
2353 if (sort_param->read_cache.read(sort_param->record, share->base.pack_reclength))
2355 if (sort_param->read_cache.error)
2356 param->out_flag |= O_DATA_LOST;
2357 param->retry_repair=1;
2358 param->testflag|=T_RETRY_WITHOUT_QUICK;
2361 sort_param->start_recpos=sort_param->pos;
2362 if (!sort_param->fix_datafile)
2364 sort_param->filepos=sort_param->pos;
2365 if (sort_param->master)
2366 share->state.split++;
2368 sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
2369 if (*sort_param->record)
2371 if (sort_param->calc_checksum)
2372 param->glob_crc+= (info->checksum=
2373 mi_static_checksum(info,sort_param->record));
2376 if (!sort_param->fix_datafile && sort_param->master)
2379 info->state->empty+=share->base.pack_reclength;
2382 case DYNAMIC_RECORD:
2383 pos= sort_param->pos;
2384 searching= (sort_param->fix_datafile && (param->testflag & T_EXTEND));
2385 parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
2388 found_record=block_info.second_read= 0;
2392 pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
2393 param->testflag|=T_RETRY_WITHOUT_QUICK;
2394 sort_param->start_recpos=pos;
2398 if (pos > sort_param->max_pos)
2399 sort_param->max_pos=pos;
2400 if (pos & (MI_DYN_ALIGN_SIZE-1))
2402 if ((param->testflag & T_VERBOSE) || searching == 0)
2403 mi_check_print_info(param,
"Wrong aligned block at %s",
2408 if (found_record && pos == param->search_after_block)
2409 mi_check_print_info(param,
"Block: %s used by record at %s",
2410 llstr(param->search_after_block,llbuff),
2411 llstr(sort_param->start_recpos,llbuff2));
2412 if (_mi_read_cache(&sort_param->read_cache,
2413 (
unsigned char*) block_info.header,pos,
2414 MI_BLOCK_INFO_HEADER_LENGTH,
2415 (! found_record ? READING_NEXT : 0) |
2416 parallel_flag | READING_HEADER))
2420 mi_check_print_info(param,
2421 "Can't read whole record at %s (errno: %d)",
2422 llstr(sort_param->start_recpos,llbuff),errno);
2427 if (searching && ! sort_param->fix_datafile)
2429 param->error_printed=1;
2430 param->retry_repair=1;
2431 param->testflag|=T_RETRY_WITHOUT_QUICK;
2434 b_type=_mi_get_block_info(&block_info,-1,pos);
2435 if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
2436 ((b_type & BLOCK_FIRST) &&
2437 (block_info.rec_len < (uint) share->base.min_pack_length ||
2438 block_info.rec_len > (uint) share->base.max_pack_length)))
2441 if (param->testflag & T_VERBOSE || searching == 0)
2442 mi_check_print_info(param,
2443 "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
2444 block_info.header[0],block_info.header[1],
2445 block_info.header[2],llstr(pos,llbuff));
2448 block_info.second_read=0;
2451 for (i=MI_DYN_ALIGN_SIZE ;
2452 i < MI_BLOCK_INFO_HEADER_LENGTH ;
2453 i+= MI_DYN_ALIGN_SIZE)
2454 if (block_info.header[i] >= 1 &&
2455 block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
2458 sort_param->start_recpos=pos;
2461 if (b_type & BLOCK_DELETED)
2464 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
2465 share->base.min_block_length)
2468 mi_check_print_info(param,
2469 "Deleted block with impossible length %u at %s",
2470 block_info.block_len,llstr(pos,llbuff));
2475 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
2476 block_info.next_filepos >=
2477 info->state->data_file_length) ||
2478 (block_info.prev_filepos != HA_OFFSET_ERROR &&
2479 block_info.prev_filepos >= info->state->data_file_length))
2482 mi_check_print_info(param,
2483 "Delete link points outside datafile at %s",
2493 pos+= MI_DYN_ALIGN_SIZE;
2494 sort_param->start_recpos=pos;
2495 block_info.second_read=0;
2501 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
2502 share->base.min_block_length ||
2503 block_info.block_len > (uint) share->base.max_pack_length+
2507 mi_check_print_info(param,
2508 "Found block with impossible length %u at %s; Skipped",
2509 block_info.block_len+ (uint) (block_info.filepos-pos),
2514 pos+= MI_DYN_ALIGN_SIZE;
2515 sort_param->start_recpos=pos;
2516 block_info.second_read=0;
2520 if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
2522 if (!sort_param->fix_datafile && sort_param->master &&
2523 (b_type & BLOCK_DELETED))
2525 info->state->empty+=block_info.block_len;
2527 share->state.split++;
2533 pos+=MI_DYN_ALIGN_SIZE;
2534 sort_param->start_recpos=pos;
2537 pos=block_info.filepos+block_info.block_len;
2538 block_info.second_read=0;
2542 if (!sort_param->fix_datafile && sort_param->master)
2543 share->state.split++;
2544 if (! found_record++)
2546 sort_param->find_length=left_length=block_info.rec_len;
2547 sort_param->start_recpos=pos;
2548 if (!sort_param->fix_datafile)
2549 sort_param->filepos=sort_param->start_recpos;
2550 if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
2551 sort_param->pos=block_info.filepos+1;
2553 sort_param->pos=block_info.filepos+block_info.block_len;
2554 if (share->base.blobs)
2556 if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
2557 &(sort_param->rec_buff))))
2559 if (param->max_record_length >= block_info.rec_len)
2561 mi_check_print_error(param,
"Not enough memory for blob at %s (need %lu)",
2562 llstr(sort_param->start_recpos,llbuff),
2563 (ulong) block_info.rec_len);
2568 mi_check_print_info(param,
"Not enough memory for blob at %s (need %lu); Row skipped",
2569 llstr(sort_param->start_recpos,llbuff),
2570 (ulong) block_info.rec_len);
2576 to= sort_param->rec_buff;
2578 if (left_length < block_info.data_len || ! block_info.data_len)
2580 mi_check_print_info(param,
2581 "Found block with too small length at %s; Skipped",
2582 llstr(sort_param->start_recpos,llbuff));
2585 if (block_info.filepos + block_info.data_len >
2586 sort_param->read_cache.end_of_file)
2588 mi_check_print_info(param,
2589 "Found block that points outside data file at %s",
2590 llstr(sort_param->start_recpos,llbuff));
2599 uint32_t header_len= (uint) (block_info.filepos - pos);
2600 uint32_t prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
2602 if (prefetch_len > block_info.data_len)
2603 prefetch_len= block_info.data_len;
2606 memcpy(to, block_info.header + header_len, prefetch_len);
2607 block_info.filepos+= prefetch_len;
2608 block_info.data_len-= prefetch_len;
2609 left_length-= prefetch_len;
2613 if (block_info.data_len &&
2614 _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
2615 block_info.data_len,
2616 (found_record == 1 ? READING_NEXT : 0) |
2619 mi_check_print_info(param,
2620 "Read error for block at: %s (error: %d); Skipped",
2621 llstr(block_info.filepos,llbuff),errno);
2624 left_length-=block_info.data_len;
2625 to+=block_info.data_len;
2626 pos=block_info.next_filepos;
2627 if (pos == HA_OFFSET_ERROR && left_length)
2629 mi_check_print_info(param,
"Wrong block with wrong total length starting at %s",
2630 llstr(sort_param->start_recpos,llbuff));
2633 if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
2635 mi_check_print_info(param,
"Found link that points at %s (outside data file) at %s",
2637 llstr(sort_param->start_recpos,llbuff));
2640 }
while (left_length);
2642 if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
2643 sort_param->find_length) != MY_FILE_ERROR)
2645 if (sort_param->read_cache.error < 0)
2647 if (sort_param->calc_checksum)
2648 info->checksum= mi_checksum(info, sort_param->record);
2649 if ((param->testflag & (T_EXTEND | T_REP)) || searching)
2651 if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
2652 sort_param->find_length,
2653 (param->testflag & T_QUICK) &&
2654 sort_param->calc_checksum &&
2655 test(info->s->calc_checksum)))
2657 mi_check_print_info(param,
"Found wrong packed record at %s",
2658 llstr(sort_param->start_recpos,llbuff));
2662 if (sort_param->calc_checksum)
2663 param->glob_crc+= info->checksum;
2667 mi_check_print_info(param,
"Key %d - Found wrong stored record at %s",
2669 llstr(sort_param->start_recpos,llbuff));
2671 pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
2674 case COMPRESSED_RECORD:
2697int sort_write_record(MI_SORT_PARAM *sort_param)
2700 ulong block_length,reclength;
2701 unsigned char *from;
2702 SORT_INFO *sort_info=sort_param->sort_info;
2703 MI_CHECK *param=sort_info->param;
2704 MI_INFO *info=sort_info->info;
2705 MYISAM_SHARE *share=info->s;
2707 if (sort_param->fix_datafile)
2709 switch (sort_info->new_data_file_type) {
2711 if (info->rec_cache.write(sort_param->record, share->base.pack_reclength))
2713 mi_check_print_error(param,
"%d when writing to datafile",errno);
2716 sort_param->filepos+=share->base.pack_reclength;
2717 info->s->state.split++;
2720 case DYNAMIC_RECORD:
2722 from=sort_param->rec_buff;
2726 reclength=info->s->base.pack_reclength+
2727 _my_calc_total_blob_length(info,sort_param->record)+
2728 ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
2729 MI_DYN_DELETE_BLOCK_HEADER;
2730 if (sort_info->buff_length < reclength)
2733 tmpptr= realloc(sort_info->buff, reclength);
2736 sort_info->buff_length=reclength;
2737 sort_info->buff= (
unsigned char *)tmpptr;
2741 mi_check_print_error(param,
"Could not realloc() sort_info->buff "
2742 " to %ul bytes", reclength);
2746 from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
2749 info->checksum=mi_checksum(info,sort_param->record);
2750 reclength=_mi_rec_pack(info,from,sort_param->record);
2756 block_length=reclength+ 3 + test(reclength >= (65520-3));
2757 if (block_length < share->base.min_block_length)
2758 block_length=share->base.min_block_length;
2759 info->update|=HA_STATE_WRITE_AT_END;
2760 block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
2761 if (block_length > MI_MAX_BLOCK_LENGTH)
2762 block_length=MI_MAX_BLOCK_LENGTH;
2763 if (_mi_write_part_record(info,0L,block_length,
2764 sort_param->filepos+block_length,
2765 &from,&reclength,&flag))
2767 mi_check_print_error(param,
"%d when writing to datafile",errno);
2770 sort_param->filepos+=block_length;
2771 info->s->state.split++;
2772 }
while (reclength);
2775 case COMPRESSED_RECORD:
2780 if (sort_param->master)
2782 info->state->records++;
2783 if ((param->testflag & T_WRITE_LOOP) &&
2784 (info->state->records % WRITE_COUNT) == 0)
2787 printf(
"%s\r", llstr(info->state->records,llbuff));
2797int sort_key_cmp(MI_SORT_PARAM *sort_param,
const void *a,
const void *b)
2799 uint32_t not_used[2];
2800 return (ha_key_cmp(sort_param->seg, *((
unsigned char*
const *) a), *((
unsigned char*
const *) b),
2801 USE_WHOLE_KEY, SEARCH_SAME, not_used));
2805int sort_key_write(MI_SORT_PARAM *sort_param,
const void *a)
2807 uint32_t diff_pos[2];
2808 char llbuff[22],llbuff2[22];
2809 SORT_INFO *sort_info=sort_param->sort_info;
2810 MI_CHECK *param= sort_info->param;
2813 if (sort_info->key_block->inited)
2815 cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
2816 (
unsigned char*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
2818 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
2819 ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
2820 (
unsigned char*) a, USE_WHOLE_KEY,
2821 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
2822 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
2824 diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
2825 sort_param->notnull,
2826 sort_info->key_block->lastkey,
2829 sort_param->unique[diff_pos[0]-1]++;
2834 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
2835 mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
2838 if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
2841 sort_info->info->lastpos=get_record_for_key(sort_info->info,
2842 sort_param->keyinfo,
2843 (
unsigned char*) a);
2844 mi_check_print_warning(param,
2845 "Duplicate key for record at %10s against record at %10s",
2846 llstr(sort_info->info->lastpos,llbuff),
2847 llstr(get_record_for_key(sort_info->info,
2848 sort_param->keyinfo,
2849 sort_info->key_block->
2852 param->testflag|=T_RETRY_WITHOUT_QUICK;
2853 return (sort_delete_record(sort_param));
2855 return (sort_insert_key(sort_param,sort_info->key_block,
2856 (
unsigned char*) a, HA_OFFSET_ERROR));
2862my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
2863 unsigned char *key) {
2864 return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
2870int sort_insert_key(MI_SORT_PARAM *sort_param,
2871 SORT_KEY_BLOCKS *key_block,
unsigned char *key,
2872 my_off_t prev_block)
2874 uint32_t a_length,t_length,nod_flag;
2875 my_off_t filepos,key_file_length;
2876 unsigned char *anc_buff,*lastkey;
2877 MI_KEY_PARAM s_temp;
2879 MI_KEYDEF *keyinfo=sort_param->keyinfo;
2880 SORT_INFO *sort_info= sort_param->sort_info;
2881 MI_CHECK *param=sort_info->param;
2883 anc_buff=key_block->buff;
2884 info=sort_info->info;
2885 lastkey=key_block->lastkey;
2886 nod_flag= (key_block == sort_info->key_block ? 0 :
2887 info->s->base.key_reflength);
2889 if (!key_block->inited)
2891 key_block->inited=1;
2892 if (key_block == sort_info->key_block_end)
2894 mi_check_print_error(param,
"To many key-block-levels; Try increasing sort_key_blocks");
2897 a_length=2+nod_flag;
2898 key_block->end_pos=anc_buff+2;
2902 a_length=mi_getint(anc_buff);
2906 _mi_kpointer(info,key_block->end_pos,prev_block);
2908 t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
2909 (
unsigned char*) 0,lastkey,lastkey,key,
2911 (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
2913 mi_putint(anc_buff,a_length,nod_flag);
2914 key_block->end_pos+=t_length;
2915 if (a_length <= keyinfo->block_length)
2917 _mi_move_key(keyinfo,key_block->lastkey,key);
2918 key_block->last_length=a_length-t_length;
2923 mi_putint(anc_buff,key_block->last_length,nod_flag);
2924 memset(anc_buff+key_block->last_length, 0,
2925 keyinfo->block_length - key_block->last_length);
2926 key_file_length=info->state->key_file_length;
2927 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
2931 if (key_file_length == info->state->key_file_length)
2933 if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
2936 else if (my_pwrite(info->s->kfile,(
unsigned char*) anc_buff,
2937 (uint) keyinfo->block_length,filepos, param->myf_rw))
2941 if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
2945 key_block->inited=0;
2946 return(sort_insert_key(sort_param, key_block,key,prev_block));
2952int sort_delete_record(MI_SORT_PARAM *sort_param)
2957 SORT_INFO *sort_info=sort_param->sort_info;
2958 MI_CHECK *param=sort_info->param;
2959 MI_INFO *info=sort_info->info;
2961 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
2963 mi_check_print_error(param,
2964 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
2967 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
2969 mi_check_print_error(param,
2970 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
2974 old_file=info->dfile;
2975 info->dfile=info->rec_cache.file;
2976 if (sort_info->current_key)
2978 key=info->lastkey+info->s->base.max_key_length;
2979 if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
2980 error != HA_ERR_RECORD_DELETED)
2982 mi_check_print_error(param,
"Can't read record to be removed");
2983 info->dfile=old_file;
2987 for (i=0 ; i < sort_info->current_key ; i++)
2989 uint32_t key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
2990 if (_mi_ck_delete(info,i,key,key_length))
2992 mi_check_print_error(param,
"Can't delete key %d from record to be removed",i+1);
2993 info->dfile=old_file;
2997 if (sort_param->calc_checksum)
2998 param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
3000 error= info->rec_cache.flush() || (*info->s->delete_record)(info);
3001 info->dfile=old_file;
3002 info->state->records--;
3008int flush_pending_blocks(MI_SORT_PARAM *sort_param)
3010 uint32_t nod_flag,length;
3011 my_off_t filepos,key_file_length;
3012 SORT_KEY_BLOCKS *key_block;
3013 SORT_INFO *sort_info= sort_param->sort_info;
3014 myf myf_rw=sort_info->param->myf_rw;
3015 MI_INFO *info=sort_info->info;
3016 MI_KEYDEF *keyinfo=sort_param->keyinfo;
3018 filepos= HA_OFFSET_ERROR;
3020 for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
3022 key_block->inited=0;
3023 length=mi_getint(key_block->buff);
3025 _mi_kpointer(info,key_block->end_pos,filepos);
3026 key_file_length=info->state->key_file_length;
3027 memset(key_block->buff+length, 0, keyinfo->block_length-length);
3028 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
3032 if (key_file_length == info->state->key_file_length)
3034 if (_mi_write_keypage(info, keyinfo, filepos,
3035 DFLT_INIT_HITS, key_block->buff))
3038 else if (my_pwrite(info->s->kfile,(
unsigned char*) key_block->buff,
3039 (uint) keyinfo->block_length,filepos, myf_rw))
3043 info->s->state.key_root[sort_param->key]=filepos;
3049static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
3050 uint32_t buffer_length)
3053 SORT_KEY_BLOCKS *block;
3055 if (!(block=(SORT_KEY_BLOCKS*) malloc((
sizeof(SORT_KEY_BLOCKS)+
3056 buffer_length+IO_SIZE)*blocks)))
3058 mi_check_print_error(param,
"Not enough memory for sort-key-blocks");
3061 for (i=0 ; i < blocks ; i++)
3064 block[i].buff=(
unsigned char*) (block+blocks)+(buffer_length+IO_SIZE)*i;
3072int test_if_almost_full(MI_INFO *info)
3074 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
3076 return (my_off_t)(lseek(info->s->kfile, 0L, SEEK_END) / 10 * 9) >
3077 (my_off_t) info->s->base.max_key_file_length ||
3078 (my_off_t)(lseek(info->dfile, 0L, SEEK_END) / 10 * 9) >
3079 (my_off_t) info->s->base.max_data_file_length;
3085int write_data_suffix(SORT_INFO *sort_info,
bool fix_datafile)
3087 MI_INFO *info=sort_info->info;
3089 if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
3091 unsigned char buff[MEMMAP_EXTRA_MARGIN];
3092 memset(buff, 0,
sizeof(buff));
3093 if (info->rec_cache.write(buff,
sizeof(buff)))
3095 mi_check_print_error(sort_info->param,
3096 "%d when writing to datafile",errno);
3099 sort_info->param->read_cache.end_of_file+=
sizeof(buff);
3106int update_state_info(MI_CHECK *param, MI_INFO *info,uint32_t update)
3108 MYISAM_SHARE *share=info->s;
3110 if (update & UPDATE_OPEN_COUNT)
3112 share->state.open_count=0;
3113 share->global_changed=0;
3115 if (update & UPDATE_STAT)
3117 uint32_t i, key_parts= mi_uint2korr(share->state.header.key_parts);
3118 share->state.rec_per_key_rows=info->state->records;
3119 share->state.changed&= ~STATE_NOT_ANALYZED;
3120 if (info->state->records)
3122 for (i=0; i<key_parts; i++)
3124 if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
3125 share->state.changed|= STATE_NOT_ANALYZED;
3129 if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
3131 if (update & UPDATE_TIME)
3133 share->state.check_time= (long) time((time_t*) 0);
3134 if (!share->state.create_time)
3135 share->state.create_time=share->state.check_time;
3143 if (info->lock_type == F_WRLCK)
3144 share->state.state= *info->state;
3145 if (mi_state_info_write(share->kfile,&share->state,1+2))
3151 uint32_t r_locks=share->r_locks,w_locks=share->w_locks;
3152 share->r_locks= share->w_locks= share->tot_locks= 0;
3153 error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
3154 share->r_locks=r_locks;
3155 share->w_locks=w_locks;
3156 share->tot_locks=r_locks+w_locks;
3161 mi_check_print_error(param,
"%d when updating keyfile",errno);
3178void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
3181 unsigned char *record= 0;
3183 if (!info->s->base.auto_key ||
3184 ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
3186 if (!(param->testflag & T_VERY_SILENT))
3187 mi_check_print_info(param,
3188 "Table: %s doesn't have an auto increment key\n",
3189 param->isam_file_name);
3192 if (!(param->testflag & T_SILENT) &&
3193 !(param->testflag & T_REP))
3194 printf(
"Updating MyISAM file: %s\n", param->isam_file_name);
3199 if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
3201 mi_check_print_error(param,
"Not enough memory for extra record");
3205 mi_extra(info,HA_EXTRA_KEYREAD,0);
3206 if (mi_rlast(info, record, info->s->base.auto_key-1))
3208 if (errno != HA_ERR_END_OF_FILE)
3210 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
3211 free(mi_get_rec_buff_ptr(info, record));
3212 mi_check_print_error(param,
"%d when reading last record",errno);
3216 info->s->state.auto_increment=param->auto_increment_value;
3220 uint64_t auto_increment= retrieve_auto_increment(info, record);
3221 set_if_bigger(info->s->state.auto_increment,auto_increment);
3223 set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
3225 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
3226 free(mi_get_rec_buff_ptr(info, record));
3227 update_state_info(param, info, UPDATE_AUTO_INC);
3283void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
3284 uint64_t *unique, uint64_t *notnull,
3287 uint64_t count=0,tmp, unique_tuples;
3288 uint64_t tuples= records;
3290 for (parts=0 ; parts < keyinfo->keysegs ; parts++)
3292 count+=unique[parts];
3293 unique_tuples= count + 1;
3296 tuples= notnull[parts];
3302 unique_tuples -= (records - notnull[parts]);
3305 if (unique_tuples == 0)
3307 else if (count == 0)
3310 tmp= (tuples + unique_tuples/2) / unique_tuples;
3318 if (tmp >= (uint64_t) ~(ulong) 0)
3319 tmp=(uint64_t) ~(ulong) 0;
3321 *rec_per_key_part=(ulong) tmp;
3327static ha_checksum mi_byte_checksum(
const unsigned char *buf, uint32_t length)
3330 const unsigned char *end=buf+length;
3331 for (crc=0; buf != end; buf++)
3332 crc=((crc << 1) + *((
unsigned char*) buf)) +
3333 test(crc & (((ha_checksum) 1) << (8*
sizeof(ha_checksum)-1)));
3337static bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
3339 uint32_t key_maxlength=key->maxlength;
3340 return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY) &&
3341 ((uint64_t) rows * key_maxlength >
3342 (uint64_t) MAX_FILE_SIZE));
3354void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
3356 MYISAM_SHARE *share=info->s;
3357 MI_KEYDEF *key=share->keyinfo;
3360 assert(info->state->records == 0 &&
3361 (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
3362 for (i=0 ; i < share->base.keys ; i++,key++)
3364 if (!(key->flag & (HA_NOSAME | HA_AUTO_KEY)) &&
3365 ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
3367 mi_clear_key_active(share->state.key_map, i);
3368 info->update|= HA_STATE_CHANGED;
3380bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
3381 uint64_t key_map,
bool force)
3383 MYISAM_SHARE *share=info->s;
3384 MI_KEYDEF *key=share->keyinfo;
3391 if (! mi_is_any_key_active(key_map))
3393 for (i=0 ; i < share->base.keys ; i++,key++)
3395 if (!force && mi_too_big_key_for_sort(key,rows))
3403set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
3405 if ((sort_info->new_data_file_type=share->data_file_type) ==
3406 COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
3410 if (share->options & HA_OPTION_PACK_RECORD)
3411 sort_info->new_data_file_type = DYNAMIC_RECORD;
3413 sort_info->new_data_file_type = STATIC_RECORD;
3416 memcpy(&tmp, share,
sizeof(*share));
3417 tmp.options= ~HA_OPTION_COMPRESS_RECORD;
3418 mi_setup_functions(&tmp);
3419 share->delete_record=tmp.delete_record;
bool reinit_io_cache(cache_type type_arg, my_off_t seek_offset, bool use_async_io, bool clear_cache)
Reset the cache.
int end_io_cache()
Free an io_cache_st object.
int init_io_cache(int file, size_t cachesize, cache_type type, my_off_t seek_offset, bool use_async_io, myf cache_myflags)
Initialize an io_cache_st object.
void free_root(myf MyFLAGS)
Deallocate everything used by alloc_root or just move used blocks to free list if called with MY_USED...
Convert from decimal to binary representation.
TODO: Rename this file - func.h is stupid.