drizzle
C++ MySQL-protocol-compatible database server
Loading...
Searching...
No Matches
mi_check.cc
1/* Copyright (C) 2000-2006 MySQL AB
2
3 This program is free software; you can redistribute it and/or modify
4 it under the terms of the GNU General Public License as published by
5 the Free Software Foundation; version 2 of the License.
6
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 GNU General Public License for more details.
11
12 You should have received a copy of the GNU General Public License
13 along with this program; if not, write to the Free Software
14 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
15
16/* Describe, check and repair of MyISAM tables */
17
18/*
19 About checksum calculation.
20
21 There are two types of checksums. Table checksum and row checksum.
22
23 Row checksum is an additional byte at the end of dynamic length
24 records. It must be calculated if the table is configured for them.
25 Otherwise they must not be used. The variable
26 MYISAM_SHARE::calc_checksum determines if row checksums are used.
27 MI_INFO::checksum is used as temporary storage during row handling.
28 For parallel repair we must assure that only one thread can use this
29 variable. There is no problem on the write side as this is done by one
30 thread only. But when checking a record after read this could go
31 wrong. But since all threads read through a common read buffer, it is
32 sufficient if only one thread checks it.
33
34 Table checksum is an eight byte value in the header of the index file.
35 It can be calculated even if row checksums are not used. The variable
36 MI_CHECK::glob_crc is calculated over all records.
37 MI_SORT_PARAM::calc_checksum determines if this should be done. This
38 variable is not part of MI_CHECK because it must be set per thread for
39 parallel repair. The global glob_crc must be changed by one thread
40 only. And it is sufficient to calculate the checksum once only.
41*/
42
43#include "myisam_priv.h"
44#include <drizzled/internal/m_string.h>
45#include <stdarg.h>
46#ifdef HAVE_SYS_VADVISE_H
47#include <sys/vadvise.h>
48#endif
49#ifdef HAVE_SYS_TYPES
50#include <sys/types.h>
51#endif
52#include <sys/mman.h>
53#include <drizzled/util/test.h>
54#include <drizzled/error.h>
55
56#include <algorithm>
57#include <new>
58
59using namespace std;
60using namespace drizzled;
61using namespace drizzled::internal;
62
63
64#define my_off_t2double(A) ((double) (my_off_t) (A))
65
66/* Functions defined in this file */
67
68static int check_k_link(MI_CHECK *param, MI_INFO *info,uint32_t nr);
69static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
70 my_off_t page, unsigned char *buff, ha_rows *keys,
71 ha_checksum *key_checksum, uint32_t level);
72static uint32_t isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
73static ha_checksum calc_checksum(ha_rows count);
74static int writekeys(MI_SORT_PARAM *sort_param);
75static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
76 my_off_t pagepos, int new_file);
77int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
78int sort_get_next_record(MI_SORT_PARAM *sort_param);
79int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
80int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
81my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
82 unsigned char *key);
83int sort_insert_key(MI_SORT_PARAM *sort_param,
84 SORT_KEY_BLOCKS *key_block,
85 unsigned char *key, my_off_t prev_block);
86int sort_delete_record(MI_SORT_PARAM *sort_param);
87
88/*static int flush_pending_blocks(MI_CHECK *param);*/
89static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
90 uint32_t buffer_length);
91static ha_checksum mi_byte_checksum(const unsigned char *buf, uint32_t length);
92static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
93
94void myisamchk_init(MI_CHECK *param)
95{
96 new (param) MI_CHECK();
97 param->opt_follow_links=1;
98 param->keys_in_use= ~(uint64_t) 0;
99 param->search_after_block=HA_OFFSET_ERROR;
100 param->auto_increment_value= 0;
101 param->use_buffers=USE_BUFFER_INIT;
102 param->read_buffer_length=READ_BUFFER_INIT;
103 param->write_buffer_length=READ_BUFFER_INIT;
104 param->sort_buffer_length=SORT_BUFFER_INIT;
105 param->sort_key_blocks=BUFFERS_WHEN_SORTING;
106 param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
107 param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
108 param->start_check_pos=0;
109 param->max_record_length= INT64_MAX;
110 param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
111 param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
112}
113
114 /* Check the status flags for the table */
115
116int chk_status(MI_CHECK *param, MI_INFO *info)
117{
118 MYISAM_SHARE *share=info->s;
119
120 if (mi_is_crashed_on_repair(info))
121 mi_check_print_warning(param,
122 "Table is marked as crashed and last repair failed");
123 else if (mi_is_crashed(info))
124 mi_check_print_warning(param,
125 "Table is marked as crashed");
126 if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
127 {
128 /* Don't count this as a real warning, as check can correct this ! */
129 uint32_t save=param->warning_printed;
130 mi_check_print_warning(param,
131 share->state.open_count==1 ?
132 "%d client is using or hasn't closed the table properly" :
133 "%d clients are using or haven't closed the table properly",
134 share->state.open_count);
135 /* If this will be fixed by the check, forget the warning */
136 if (param->testflag & T_UPDATE_STATE)
137 param->warning_printed=save;
138 }
139 return 0;
140}
141
142 /* Check delete links */
143
144int chk_del(MI_CHECK *param, MI_INFO *info, uint32_t test_flag)
145{
146 uint32_t delete_link_length;
147 my_off_t empty, next_link, old_link= 0;
148 char buff[22],buff2[22];
149
150 param->record_checksum=0;
151 delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 : info->s->rec_reflength+1);
152
153 if (!(test_flag & T_SILENT))
154 puts("- check record delete-chain");
155
156 next_link=info->s->state.dellink;
157 if (info->state->del == 0)
158 {
159 if (test_flag & T_VERBOSE)
160 {
161 puts("No recordlinks");
162 }
163 }
164 else
165 {
166 if (test_flag & T_VERBOSE)
167 printf("Recordlinks: ");
168 empty=0;
169 ha_rows i= info->state->del;
170 for (; i > 0 && next_link != HA_OFFSET_ERROR; i--)
171 {
172 if (*killed_ptr(param))
173 return(1);
174 if (test_flag & T_VERBOSE)
175 printf(" %9s",llstr(next_link,buff));
176 if (next_link >= info->state->data_file_length)
177 goto wrong;
178 if (my_pread(info->dfile, (unsigned char*) buff,delete_link_length, next_link,MYF(MY_NABP)))
179 {
180 if (test_flag & T_VERBOSE) puts("");
181 mi_check_print_error(param,"Can't read delete-link at filepos: %s",
182 llstr(next_link,buff));
183 return(1);
184 }
185 if (*buff != '\0')
186 {
187 if (test_flag & T_VERBOSE) puts("");
188 mi_check_print_error(param,"Record at pos: %s is not remove-marked",
189 llstr(next_link,buff));
190 goto wrong;
191 }
192 if (info->s->options & HA_OPTION_PACK_RECORD)
193 {
194 my_off_t prev_link=mi_sizekorr(buff+12);
195 if (empty && prev_link != old_link)
196 {
197 if (test_flag & T_VERBOSE) puts("");
198 mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
199 goto wrong;
200 }
201 old_link=next_link;
202 next_link=mi_sizekorr(buff+4);
203 empty+=mi_uint3korr(buff+1);
204 }
205 else
206 {
207 param->record_checksum+=(ha_checksum) next_link;
208 next_link=_mi_rec_pos(info->s,(unsigned char*) buff+1);
209 empty+=info->s->base.pack_reclength;
210 }
211 }
212 if (test_flag & T_VERBOSE)
213 puts("\n");
214 if (empty != info->state->empty)
215 {
216 mi_check_print_warning(param,
217 "Found %s deleted space in delete link chain. Should be %s",
218 llstr(empty,buff2),
219 llstr(info->state->empty,buff));
220 }
221 if (next_link != HA_OFFSET_ERROR)
222 {
223 mi_check_print_error(param,
224 "Found more than the expected %s deleted rows in delete link chain",
225 llstr(info->state->del, buff));
226 goto wrong;
227 }
228 if (i != 0)
229 {
230 mi_check_print_error(param, "Found %s deleted rows in delete link chain. Should be %s", llstr(info->state->del - i, buff2), llstr(info->state->del, buff));
231 goto wrong;
232 }
233 }
234 return 0;
235
236wrong:
237 param->testflag|=T_RETRY_WITHOUT_QUICK;
238 if (test_flag & T_VERBOSE) puts("");
239 mi_check_print_error(param,"record delete-link-chain corrupted");
240 return 1;
241} /* chk_del */
242
243
244 /* Check delete links in index file */
245
246static int check_k_link(MI_CHECK *param, MI_INFO *info, uint32_t nr)
247{
248 my_off_t next_link;
249 uint32_t block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
250 ha_rows records;
251 char llbuff[21], llbuff2[21];
252 unsigned char *buff;
253
254 if (param->testflag & T_VERBOSE)
255 printf("block_size %4u:", block_size);
256
257 next_link=info->s->state.key_del[nr];
258 records= (ha_rows) (info->state->key_file_length / block_size);
259 while (next_link != HA_OFFSET_ERROR && records > 0)
260 {
261 if (*killed_ptr(param))
262 return(1);
263 if (param->testflag & T_VERBOSE)
264 printf("%16s",llstr(next_link,llbuff));
265
266 /* Key blocks must lay within the key file length entirely. */
267 if (next_link + block_size > info->state->key_file_length)
268 {
269 mi_check_print_error(param, "Invalid key block position: %s "
270 "key block size: %u file_length: %s",
271 llstr(next_link, llbuff), block_size,
272 llstr(info->state->key_file_length, llbuff2));
273 return(1);
274 }
275
276 /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
277 if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
278 {
279 mi_check_print_error(param, "Mis-aligned key block: %s "
280 "minimum key block length: %u",
281 llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
282 return(1);
283 }
284
285 /*
286 Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
287 If the key cache block size is smaller than block_size, we can so
288 avoid unecessary eviction of cache block.
289 */
290
291 if (not pread(info->s->kfile, info->buff, MI_MIN_KEY_BLOCK_LENGTH, next_link))
292 {
293 mi_check_print_error(param, "key cache read error for block: %s", llstr(next_link,llbuff));
294 return(1);
295 }
296 buff= info->buff;
297 next_link=mi_sizekorr(buff);
298 records--;
299 param->key_file_blocks+=block_size;
300 }
301 if (param->testflag & T_VERBOSE)
302 {
303 if (next_link != HA_OFFSET_ERROR)
304 printf("%16s\n",llstr(next_link,llbuff));
305 else
306 puts("");
307 }
308 return (next_link != HA_OFFSET_ERROR);
309} /* check_k_link */
310
311
312 /* Check sizes of files */
313
314int chk_size(MI_CHECK *param, MI_INFO *info)
315{
316 int error=0;
317 my_off_t skr,size;
318 char buff[22],buff2[22];
319
320 if (!(param->testflag & T_SILENT)) puts("- check file-size");
321
322 size= lseek(info->s->kfile, 0, SEEK_END);
323 if ((skr=(my_off_t) info->state->key_file_length) != size)
324 {
325 /* Don't give error if file generated by myisampack */
326 if (skr > size && mi_is_any_key_active(info->s->state.key_map))
327 {
328 error=1;
329 mi_check_print_error(param,
330 "Size of indexfile is: %-8s Should be: %s",
331 llstr(size,buff), llstr(skr,buff2));
332 }
333 else
334 mi_check_print_warning(param,
335 "Size of indexfile is: %-8s Should be: %s",
336 llstr(size,buff), llstr(skr,buff2));
337 }
338 if (!(param->testflag & T_VERY_SILENT) &&
339 ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
340 uint64_t2double(info->state->key_file_length) >
341 uint64_t2double(info->s->base.margin_key_file_length)*0.9)
342 mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
343 llstr(info->state->key_file_length,buff),
344 llstr(info->s->base.max_key_file_length-1,buff));
345
346 size=lseek(info->dfile,0L,SEEK_END);
347 skr=(my_off_t) info->state->data_file_length;
348 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
349 skr+= MEMMAP_EXTRA_MARGIN;
350#ifdef USE_RELOC
351 if (info->data_file_type == STATIC_RECORD &&
352 skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
353 skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
354#endif
355 if (skr != size)
356 {
357 info->state->data_file_length=size; /* Skip other errors */
358 if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
359 {
360 error=1;
361 mi_check_print_error(param,"Size of datafile is: %-9s Should be: %s",
362 llstr(size,buff), llstr(skr,buff2));
363 param->testflag|=T_RETRY_WITHOUT_QUICK;
364 }
365 else
366 {
367 mi_check_print_warning(param,
368 "Size of datafile is: %-9s Should be: %s",
369 llstr(size,buff), llstr(skr,buff2));
370 }
371 }
372 if (!(param->testflag & T_VERY_SILENT) &&
373 !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
374 uint64_t2double(info->state->data_file_length) >
375 (uint64_t2double(info->s->base.max_data_file_length)*0.9))
376 mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
377 llstr(info->state->data_file_length,buff),
378 llstr(info->s->base.max_data_file_length-1,buff2));
379 return(error);
380} /* chk_size */
381
382
383 /* Check keys */
384
385int chk_key(MI_CHECK *param, MI_INFO *info)
386{
387 uint32_t key,found_keys=0,full_text_keys=0,result=0;
388 ha_rows keys;
389 ha_checksum old_record_checksum,init_checksum;
390 my_off_t all_keydata,all_totaldata,key_totlength,length;
391 ulong *rec_per_key_part;
392 MYISAM_SHARE *share=info->s;
393 MI_KEYDEF *keyinfo;
394 char buff[22],buff2[22];
395
396 if (!(param->testflag & T_SILENT))
397 puts("- check key delete-chain");
398
399 param->key_file_blocks=info->s->base.keystart;
400 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
401 if (check_k_link(param,info,key))
402 {
403 if (param->testflag & T_VERBOSE) puts("");
404 mi_check_print_error(param,"key delete-link-chain corrupted");
405 return(-1);
406 }
407
408 if (!(param->testflag & T_SILENT)) puts("- check index reference");
409
410 all_keydata=all_totaldata=key_totlength=0;
411 old_record_checksum=0;
412 init_checksum=param->record_checksum;
413 if (!(share->options &
414 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
415 old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
416 share->base.pack_reclength;
417 rec_per_key_part= param->rec_per_key_part;
418 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
419 rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
420 {
421 param->key_crc[key]=0;
422 if (! mi_is_key_active(share->state.key_map, key))
423 {
424 /* Remember old statistics for key */
425 assert(rec_per_key_part >= param->rec_per_key_part);
426 memcpy(rec_per_key_part,
427 (share->state.rec_per_key_part +
428 (rec_per_key_part - param->rec_per_key_part)),
429 keyinfo->keysegs*sizeof(*rec_per_key_part));
430 continue;
431 }
432 found_keys++;
433
434 param->record_checksum=init_checksum;
435
436 memset(&param->unique_count, 0, sizeof(param->unique_count));
437 memset(&param->notnull_count, 0, sizeof(param->notnull_count));
438
439 if ((!(param->testflag & T_SILENT)))
440 printf ("- check data record references index: %d\n",key+1);
441 if (share->state.key_root[key] == HA_OFFSET_ERROR && (info->state->records == 0))
442 goto do_stat;
443 if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
444 DFLT_INIT_HITS,info->buff,0))
445 {
446 mi_check_print_error(param,"Can't read indexpage from filepos: %s",
447 llstr(share->state.key_root[key],buff));
448 if (!(param->testflag & T_INFO))
449 return(-1);
450 result= UINT32_MAX;
451 continue;
452 }
453 param->key_file_blocks+=keyinfo->block_length;
454 keys=0;
455 param->keydata=param->totaldata=0;
456 param->key_blocks=0;
457 param->max_level=0;
458 if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
459 &keys, param->key_crc+key,1))
460 return(-1);
461 if(!(0))
462 {
463 if (keys != info->state->records)
464 {
465 mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
466 llstr(info->state->records,buff2));
467 if (!(param->testflag & T_INFO))
468 return(-1);
469 result= UINT32_MAX;
470 continue;
471 }
472 if (found_keys - full_text_keys == 1 &&
473 ((share->options &
474 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
475 (param->testflag & T_DONT_CHECK_CHECKSUM)))
476 old_record_checksum=param->record_checksum;
477 else if (old_record_checksum != param->record_checksum)
478 {
479 if (key)
480 mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
481 key+1);
482 else
483 mi_check_print_error(param,"Key 1 doesn't point at all records");
484 if (!(param->testflag & T_INFO))
485 return(-1);
486 result= UINT32_MAX;
487 continue;
488 }
489 }
490 if ((uint) share->base.auto_key -1 == key)
491 {
492 /* Check that auto_increment key is bigger than max key value */
493 uint64_t auto_increment;
494 info->lastinx=key;
495 _mi_read_key_record(info, 0L, info->rec_buff);
496 auto_increment= retrieve_auto_increment(info, info->rec_buff);
497 if (auto_increment > info->s->state.auto_increment)
498 {
499 mi_check_print_warning(param, "Auto-increment value: %s is smaller "
500 "than max used value: %s",
501 llstr(info->s->state.auto_increment,buff2),
502 llstr(auto_increment, buff));
503 }
504 if (param->testflag & T_AUTO_INC)
505 {
506 set_if_bigger(info->s->state.auto_increment,
507 auto_increment);
508 set_if_bigger(info->s->state.auto_increment,
509 param->auto_increment_value);
510 }
511
512 /* Check that there isn't a row with auto_increment = 0 in the table */
513 mi_extra(info,HA_EXTRA_KEYREAD,0);
514 memset(info->lastkey, 0, keyinfo->seg->length);
515 if (!mi_rkey(info, info->rec_buff, key, (const unsigned char*) info->lastkey,
516 (key_part_map)1, HA_READ_KEY_EXACT))
517 {
518 /* Don't count this as a real warning, as myisamchk can't correct it */
519 uint32_t save=param->warning_printed;
520 mi_check_print_warning(param, "Found row where the auto_increment "
521 "column has the value 0");
522 param->warning_printed=save;
523 }
524 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
525 }
526
527 length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
528 if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
529 printf("Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n",
530 key+1,
531 (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
532 (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
533 my_off_t2double(length)),
534 param->max_level);
535 all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
536
537do_stat:
538 if (param->testflag & T_STATISTICS)
539 update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
540 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
541 param->notnull_count: NULL,
542 (uint64_t)info->state->records);
543 }
544 if (param->testflag & T_INFO)
545 {
546 if (all_totaldata != 0L && found_keys > 0)
547 printf("Total: Keyblocks used: %3d%% Packed: %4d%%\n\n",
548 (int) (my_off_t2double(all_keydata)*100.0/
549 my_off_t2double(all_totaldata)),
550 (int) ((my_off_t2double(key_totlength) -
551 my_off_t2double(all_keydata))*100.0/
552 my_off_t2double(key_totlength)));
553 else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
554 puts("");
555 }
556 if (param->key_file_blocks != info->state->key_file_length &&
557 param->keys_in_use != ~(uint64_t) 0)
558 mi_check_print_warning(param, "Some data are unreferenced in keyfile");
559 if (found_keys != full_text_keys)
560 param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */
561 else
562 param->record_checksum=0;
563 return(result);
564} /* chk_key */
565
566
567static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
568 my_off_t page, unsigned char *buff, ha_rows *keys,
569 ha_checksum *key_checksum, uint32_t level)
570{
571 char llbuff[22],llbuff2[22];
572
573 /* Key blocks must lay within the key file length entirely. */
574 if (page + keyinfo->block_length > info->state->key_file_length)
575 {
576 /* Give it a chance to fit in the real file size. */
577 my_off_t max_length= lseek(info->s->kfile, 0, SEEK_END);
578 mi_check_print_error(param, "Invalid key block position: %s "
579 "key block size: %u file_length: %s",
580 llstr(page, llbuff), keyinfo->block_length,
581 llstr(info->state->key_file_length, llbuff2));
582 if (page + keyinfo->block_length > max_length)
583 goto err;
584 /* Fix the remebered key file length. */
585 info->state->key_file_length= (max_length &
586 ~ (my_off_t) (keyinfo->block_length - 1));
587 }
588
589 /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
590 if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
591 {
592 mi_check_print_error(param, "Mis-aligned key block: %s "
593 "minimum key block length: %u",
594 llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
595 goto err;
596 }
597
598 if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
599 {
600 mi_check_print_error(param,"Can't read key from filepos: %s",
601 llstr(page,llbuff));
602 goto err;
603 }
604 param->key_file_blocks+=keyinfo->block_length;
605 if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
606 goto err;
607
608 return(0);
609
610err:
611 return(1);
612}
613
614
615/*
616 "Ignore NULLs" statistics collection method: process first index tuple.
617
618 SYNOPSIS
619 mi_collect_stats_nonulls_first()
620 keyseg IN Array of key part descriptions
621 notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
622 tuples that don't contain NULLs)
623 key IN Key values tuple
624
625 DESCRIPTION
626 Process the first index tuple - find out which prefix tuples don't
627 contain NULLs, and update the array of notnull counters accordingly.
628*/
629
630static
631void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, uint64_t *notnull,
632 unsigned char *key)
633{
634 uint32_t first_null, kp;
635 first_null= ha_find_null(keyseg, key) - keyseg;
636 /*
637 All prefix tuples that don't include keypart_{first_null} are not-null
638 tuples (and all others aren't), increment counters for them.
639 */
640 for (kp= 0; kp < first_null; kp++)
641 notnull[kp]++;
642}
643
644
645/*
646 "Ignore NULLs" statistics collection method: process next index tuple.
647
648 SYNOPSIS
649 mi_collect_stats_nonulls_next()
650 keyseg IN Array of key part descriptions
651 notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
652 tuples that don't contain NULLs)
653 prev_key IN Previous key values tuple
654 last_key IN Next key values tuple
655
656 DESCRIPTION
657 Process the next index tuple:
658 1. Find out which prefix tuples of last_key don't contain NULLs, and
659 update the array of notnull counters accordingly.
660 2. Find the first keypart number where the prev_key and last_key tuples
661 are different(A), or last_key has NULL value(B), and return it, so the
662 caller can count number of unique tuples for each key prefix. We don't
663 need (B) to be counted, and that is compensated back in
664 update_key_parts().
665
666 RETURN
667 1 + number of first keypart where values differ or last_key tuple has NULL
668*/
669
670static
671int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, uint64_t *notnull,
672 unsigned char *prev_key, unsigned char *last_key)
673{
674 uint32_t diffs[2];
675 uint32_t first_null_seg, kp;
676 HA_KEYSEG *seg;
677
678 /*
679 Find the first keypart where values are different or either of them is
680 NULL. We get results in diffs array:
681 diffs[0]= 1 + number of first different keypart
682 diffs[1]=offset: (last_key + diffs[1]) points to first value in
683 last_key that is NULL or different from corresponding
684 value in prev_key.
685 */
686 ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
687 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
688 seg= keyseg + diffs[0] - 1;
689
690 /* Find first NULL in last_key */
691 first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
692 for (kp= 0; kp < first_null_seg; kp++)
693 notnull[kp]++;
694
695 /*
696 Return 1+ number of first key part where values differ. Don't care if
697 these were NULLs and not .... We compensate for that in
698 update_key_parts.
699 */
700 return diffs[0];
701}
702
703
704 /* Check if index is ok */
705
706static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
707 my_off_t page, unsigned char *buff, ha_rows *keys,
708 ha_checksum *key_checksum, uint32_t level)
709{
710 int flag;
711 uint32_t used_length,comp_flag,nod_flag,key_length=0;
712 unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*endpos;
713 my_off_t next_page,record;
714 char llbuff[22];
715 uint32_t diff_pos[2];
716
717 if (!(temp_buff=(unsigned char*) malloc(keyinfo->block_length)))
718 {
719 mi_check_print_error(param,"Not enough memory for keyblock");
720 return(-1);
721 }
722
723 if (keyinfo->flag & HA_NOSAME)
724 comp_flag=SEARCH_FIND | SEARCH_UPDATE; /* Not real duplicates */
725 else
726 comp_flag=SEARCH_SAME; /* Keys in positionorder */
727 nod_flag=mi_test_if_nod(buff);
728 used_length=mi_getint(buff);
729 keypos=buff+2+nod_flag;
730 endpos=buff+used_length;
731
732 param->keydata+=used_length; param->totaldata+=keyinfo->block_length; /* INFO */
733 param->key_blocks++;
734 if (level > param->max_level)
735 param->max_level=level;
736
737 if (used_length > keyinfo->block_length)
738 {
739 mi_check_print_error(param,"Wrong pageinfo at page: %s",
740 llstr(page,llbuff));
741 goto err;
742 }
743 for ( ;; )
744 {
745 if (*killed_ptr(param))
746 goto err;
747 memcpy(info->lastkey,key,key_length);
748 info->lastkey_length=key_length;
749 if (nod_flag)
750 {
751 next_page=_mi_kpos(nod_flag,keypos);
752 if (chk_index_down(param,info,keyinfo,next_page,
753 temp_buff,keys,key_checksum,level+1))
754 goto err;
755 }
756 if (keypos >= endpos ||
757 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
758 break;
759 assert(key_length <= sizeof(key));
760 if (keypos > endpos)
761 {
762 mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
763 goto err;
764 }
765 if ((*keys)++ &&
766 (flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
767 comp_flag, diff_pos)) >=0)
768 {
769 if (comp_flag & SEARCH_FIND && flag == 0)
770 mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
771 else
772 mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
773 goto err;
774 }
775 if (param->testflag & T_STATISTICS)
776 {
777 if (*keys != 1L) /* not first_key */
778 {
779 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
780 ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
781 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
782 diff_pos);
783 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
784 {
785 diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg,
786 param->notnull_count,
787 info->lastkey, key);
788 }
789 param->unique_count[diff_pos[0]-1]++;
790 }
791 else
792 {
793 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
794 mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
795 key);
796 }
797 }
798 (*key_checksum)+= mi_byte_checksum((unsigned char*) key,
799 key_length- info->s->rec_reflength);
800 record= _mi_dpos(info,0,key+key_length);
801 if (record >= info->state->data_file_length)
802 {
803 mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
804 goto err;
805 }
806 param->record_checksum+=(ha_checksum) record;
807 }
808 if (keypos != endpos)
809 {
810 mi_check_print_error(param,"Keyblock size at page %s is not correct. Block length: %d key length: %d",
811 llstr(page,llbuff), used_length, (keypos - buff));
812 goto err;
813 }
814 free(temp_buff);
815 return(0);
816 err:
817 free(temp_buff);
818 return(1);
819} /* chk_index */
820
821
822 /* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */
823
824static ha_checksum calc_checksum(ha_rows count)
825{
826 uint64_t sum,a,b;
827
828 sum=0;
829 a=count; b=count+1;
830 if (a & 1)
831 b>>=1;
832 else
833 a>>=1;
834 while (b)
835 {
836 if (b & 1)
837 sum+=a;
838 a<<=1; b>>=1;
839 }
840 return((ha_checksum) sum);
841} /* calc_checksum */
842
843
844 /* Calc length of key in normal isam */
845
846static uint32_t isam_key_length(MI_INFO *info, MI_KEYDEF *keyinfo)
847{
848 uint32_t length;
849 HA_KEYSEG *keyseg;
850
851 length= info->s->rec_reflength;
852 for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
853 length+= keyseg->length;
854
855 return(length);
856} /* key_length */
857
858
859 /* Check that record-link is ok */
860
861int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
862{
863 int error,got_error,flag;
864 uint key, left_length= 0, b_type;
865 ha_rows records, del_blocks;
866 my_off_t used, empty, pos, splits, start_recpos= 0,
867 del_length, link_used, start_block;
868 unsigned char *record= NULL, *to= NULL;
869 char llbuff[22],llbuff2[22],llbuff3[22];
870 ha_checksum intern_record_checksum;
871 ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
872 MI_KEYDEF *keyinfo;
873 MI_BLOCK_INFO block_info;
874
875 if (!(param->testflag & T_SILENT))
876 {
877 if (extend)
878 puts("- check records and index references");
879 else
880 puts("- check record links");
881 }
882
883 if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
884 {
885 mi_check_print_error(param,"Not enough memory for record");
886 return(-1);
887 }
888 records=del_blocks=0;
889 used=link_used=splits=del_length=0;
890 intern_record_checksum=param->glob_crc=0;
891 got_error=error=0;
892 empty=info->s->pack.header_length;
893
894 pos= param->read_cache.tell();
895 memset(key_checksum, 0, info->s->base.keys * sizeof(key_checksum[0]));
896 while (pos < info->state->data_file_length)
897 {
898 if (*killed_ptr(param))
899 goto err2;
900 switch (info->s->data_file_type) {
901 case STATIC_RECORD:
902 if (param->read_cache.read(record, info->s->base.pack_reclength))
903 goto err;
904 start_recpos=pos;
905 pos+=info->s->base.pack_reclength;
906 splits++;
907 if (*record == '\0')
908 {
909 del_blocks++;
910 del_length+=info->s->base.pack_reclength;
911 continue; /* Record removed */
912 }
913 param->glob_crc+= mi_static_checksum(info,record);
914 used+=info->s->base.pack_reclength;
915 break;
916 case DYNAMIC_RECORD:
917 flag=block_info.second_read=0;
918 block_info.next_filepos=pos;
919 do
920 {
921 if (_mi_read_cache(&param->read_cache,(unsigned char*) block_info.header,
922 (start_block=block_info.next_filepos),
923 sizeof(block_info.header),
924 (flag ? 0 : READING_NEXT) | READING_HEADER))
925 goto err;
926 if (start_block & (MI_DYN_ALIGN_SIZE-1))
927 {
928 mi_check_print_error(param,"Wrong aligned block at %s",
929 llstr(start_block,llbuff));
930 goto err2;
931 }
932 b_type=_mi_get_block_info(&block_info,-1,start_block);
933 if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
934 BLOCK_FATAL_ERROR))
935 {
936 if (b_type & BLOCK_SYNC_ERROR)
937 {
938 if (flag)
939 {
940 mi_check_print_error(param,"Unexpected byte: %d at link: %s",
941 (int) block_info.header[0],
942 llstr(start_block,llbuff));
943 goto err2;
944 }
945 pos=block_info.filepos+block_info.block_len;
946 goto next;
947 }
948 if (b_type & BLOCK_DELETED)
949 {
950 if (block_info.block_len < info->s->base.min_block_length)
951 {
952 mi_check_print_error(param,
953 "Deleted block with impossible length %lu at %s",
954 block_info.block_len,llstr(pos,llbuff));
955 goto err2;
956 }
957 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
958 block_info.next_filepos >= info->state->data_file_length) ||
959 (block_info.prev_filepos != HA_OFFSET_ERROR &&
960 block_info.prev_filepos >= info->state->data_file_length))
961 {
962 mi_check_print_error(param,"Delete link points outside datafile at %s",
963 llstr(pos,llbuff));
964 goto err2;
965 }
966 del_blocks++;
967 del_length+=block_info.block_len;
968 pos=block_info.filepos+block_info.block_len;
969 splits++;
970 goto next;
971 }
972 mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
973 block_info.header[0],block_info.header[1],
974 block_info.header[2],
975 llstr(start_block,llbuff));
976 goto err2;
977 }
978 if (info->state->data_file_length < block_info.filepos+
979 block_info.block_len)
980 {
981 mi_check_print_error(param,
982 "Recordlink that points outside datafile at %s",
983 llstr(pos,llbuff));
984 got_error=1;
985 break;
986 }
987 splits++;
988 if (!flag++) /* First block */
989 {
990 start_recpos=pos;
991 pos=block_info.filepos+block_info.block_len;
992 if (block_info.rec_len > (uint) info->s->base.max_pack_length)
993 {
994 mi_check_print_error(param,"Found too long record (%lu) at %s",
995 (ulong) block_info.rec_len,
996 llstr(start_recpos,llbuff));
997 got_error=1;
998 break;
999 }
1000 if (info->s->base.blobs)
1001 {
1002 if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1003 &info->rec_buff)))
1004 {
1005 mi_check_print_error(param,
1006 "Not enough memory (%lu) for blob at %s",
1007 (ulong) block_info.rec_len,
1008 llstr(start_recpos,llbuff));
1009 got_error=1;
1010 break;
1011 }
1012 }
1013 else
1014 to= info->rec_buff;
1015 left_length=block_info.rec_len;
1016 }
1017 if (left_length < block_info.data_len)
1018 {
1019 mi_check_print_error(param,"Found too long record (%lu) at %s",
1020 (ulong) block_info.data_len,
1021 llstr(start_recpos,llbuff));
1022 got_error=1;
1023 break;
1024 }
1025 if (_mi_read_cache(&param->read_cache,(unsigned char*) to,block_info.filepos,
1026 (uint) block_info.data_len,
1027 flag == 1 ? READING_NEXT : 0))
1028 goto err;
1029 to+=block_info.data_len;
1030 link_used+= block_info.filepos-start_block;
1031 used+= block_info.filepos - start_block + block_info.data_len;
1032 empty+=block_info.block_len-block_info.data_len;
1033 left_length-=block_info.data_len;
1034 if (left_length)
1035 {
1036 if (b_type & BLOCK_LAST)
1037 {
1038 mi_check_print_error(param,
1039 "Wrong record length %s of %s at %s",
1040 llstr(block_info.rec_len-left_length,llbuff),
1041 llstr(block_info.rec_len, llbuff2),
1042 llstr(start_recpos,llbuff3));
1043 got_error=1;
1044 break;
1045 }
1046 if (info->state->data_file_length < block_info.next_filepos)
1047 {
1048 mi_check_print_error(param,
1049 "Found next-recordlink that points outside datafile at %s",
1050 llstr(block_info.filepos,llbuff));
1051 got_error=1;
1052 break;
1053 }
1054 }
1055 } while (left_length);
1056 if (! got_error)
1057 {
1058 if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
1059 MY_FILE_ERROR)
1060 {
1061 mi_check_print_error(param,"Found wrong record at %s",
1062 llstr(start_recpos,llbuff));
1063 got_error=1;
1064 }
1065 else
1066 {
1067 info->checksum=mi_checksum(info,record);
1068 if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
1069 {
1070 if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
1071 test(info->s->calc_checksum)))
1072 {
1073 mi_check_print_error(param,"Found wrong packed record at %s",
1074 llstr(start_recpos,llbuff));
1075 got_error=1;
1076 }
1077 }
1078 if (!got_error)
1079 param->glob_crc+= info->checksum;
1080 }
1081 }
1082 else if (!flag)
1083 pos=block_info.filepos+block_info.block_len;
1084 break;
1085 case COMPRESSED_RECORD:
1086 case BLOCK_RECORD:
1087 assert(0); /* Impossible */
1088 } /* switch */
1089 if (! got_error)
1090 {
1091 intern_record_checksum+=(ha_checksum) start_recpos;
1092 records++;
1093 if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
1094 {
1095 printf("%s\r", llstr(records,llbuff)); fflush(stdout);
1096 }
1097
1098 /* Check if keys match the record */
1099
1100 for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
1101 key++,keyinfo++)
1102 {
1103 if (mi_is_key_active(info->s->state.key_map, key))
1104 {
1105 {
1106 uint32_t key_length=_mi_make_key(info,key,info->lastkey,record,
1107 start_recpos);
1108 if (extend)
1109 {
1110 /* We don't need to lock the key tree here as we don't allow
1111 concurrent threads when running myisamchk
1112 */
1113 int search_result=
1114 _mi_search(info,keyinfo,info->lastkey,key_length,
1115 SEARCH_SAME, info->s->state.key_root[key]);
1116 if (search_result)
1117 {
1118 mi_check_print_error(param,"Record at: %10s "
1119 "Can't find key for index: %2d",
1120 llstr(start_recpos,llbuff),key+1);
1121 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1122 goto err2;
1123 }
1124 }
1125 else
1126 key_checksum[key]+=mi_byte_checksum((unsigned char*) info->lastkey,
1127 key_length);
1128 }
1129 }
1130 }
1131 }
1132 else
1133 {
1134 got_error=0;
1135 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1136 goto err2;
1137 }
1138 next:; /* Next record */
1139 }
1140 if (param->testflag & T_WRITE_LOOP)
1141 {
1142 fputs(" \r",stdout); fflush(stdout);
1143 }
1144 if (records != info->state->records)
1145 {
1146 mi_check_print_error(param,"Record-count is not ok; is %-10s Should be: %s",
1147 llstr(records,llbuff), llstr(info->state->records,llbuff2));
1148 error=1;
1149 }
1150 else if (param->record_checksum &&
1151 param->record_checksum != intern_record_checksum)
1152 {
1153 mi_check_print_error(param,
1154 "Keypointers and record positions doesn't match");
1155 error=1;
1156 }
1157 else if (param->glob_crc != info->state->checksum &&
1158 (info->s->options &
1159 (HA_OPTION_COMPRESS_RECORD)))
1160 {
1161 mi_check_print_warning(param,
1162 "Record checksum is not the same as checksum stored in the index file\n");
1163 error=1;
1164 }
1165 else if (!extend)
1166 {
1167 for (key=0 ; key < info->s->base.keys; key++)
1168 {
1169 if (key_checksum[key] != param->key_crc[key])
1170 {
1171 mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
1172 key+1);
1173 error=1;
1174 }
1175 }
1176 }
1177
1178 if (del_length != info->state->empty)
1179 {
1180 mi_check_print_warning(param,
1181 "Found %s deleted space. Should be %s",
1182 llstr(del_length,llbuff2),
1183 llstr(info->state->empty,llbuff));
1184 }
1185 if (used+empty+del_length != info->state->data_file_length)
1186 {
1187 mi_check_print_warning(param,
1188 "Found %s record-data and %s unused data and %s deleted-data",
1189 llstr(used,llbuff),llstr(empty,llbuff2),
1190 llstr(del_length,llbuff3));
1191 mi_check_print_warning(param,
1192 "Total %s, Should be: %s",
1193 llstr((used+empty+del_length),llbuff),
1194 llstr(info->state->data_file_length,llbuff2));
1195 }
1196 if (del_blocks != info->state->del)
1197 {
1198 mi_check_print_warning(param,
1199 "Found %10s deleted blocks Should be: %s",
1200 llstr(del_blocks,llbuff),
1201 llstr(info->state->del,llbuff2));
1202 }
1203 if (splits != info->s->state.split)
1204 {
1205 mi_check_print_warning(param,
1206 "Found %10s parts Should be: %s parts",
1207 llstr(splits,llbuff),
1208 llstr(info->s->state.split,llbuff2));
1209 }
1210 if (param->testflag & T_INFO)
1211 {
1212 if (param->warning_printed || param->error_printed)
1213 puts("");
1214 if (used != 0 && ! param->error_printed)
1215 {
1216 printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n",
1217 llstr(records,llbuff), (long)((used-link_used)/records),
1218 (info->s->base.blobs ? 0.0 :
1219 (uint64_t2double((uint64_t) info->s->base.reclength*records)-
1220 my_off_t2double(used))/
1221 uint64_t2double((uint64_t) info->s->base.reclength*records)*100.0));
1222 printf("Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n",
1223 (uint64_t2double(used-link_used)/uint64_t2double(used-link_used+empty)*100.0),
1224 (!records ? 100 : (int) (uint64_t2double(del_length+empty)/
1225 my_off_t2double(used)*100.0)),
1226 uint64_t2double(splits - del_blocks) / records);
1227 }
1228 printf("Record blocks:%12s Delete blocks:%10s\n",
1229 llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
1230 printf("Record data: %12s Deleted data: %10s\n",
1231 llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
1232 printf("Lost space: %12s Linkdata: %10s\n",
1233 llstr(empty,llbuff),llstr(link_used,llbuff2));
1234 }
1235 free(mi_get_rec_buff_ptr(info, record));
1236 return (error);
1237 err:
1238 mi_check_print_error(param,"got error: %d when reading datafile at record: %s",errno, llstr(records,llbuff));
1239 err2:
1240 free(mi_get_rec_buff_ptr(info, record));
1241 param->testflag|=T_RETRY_WITHOUT_QUICK;
1242 return(1);
1243} /* chk_data_link */
1244
1245
1298
1299static void mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, bool force)
1300{
1301 MYISAM_SHARE *share= info->s;
1302 MI_STATE_INFO *state= &share->state;
1303 uint32_t i;
1304
1305 /*
1306 If any of the disabled indexes has a key block assigned, we must
1307 drop and recreate all indexes to avoid losing index blocks.
1308
1309 If we want to recreate disabled indexes only _and_ all of these
1310 indexes are empty, we don't need to recreate the existing indexes.
1311 */
1312 if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
1313 {
1314 for (i= 0; i < share->base.keys; i++)
1315 {
1316 if ((state->key_root[i] != HA_OFFSET_ERROR) &&
1317 !mi_is_key_active(state->key_map, i))
1318 {
1319 /*
1320 This index has at least one key block and it is disabled.
1321 We would lose its block(s) if would just recreate it.
1322 So we need to drop and recreate all indexes.
1323 */
1324 break;
1325 }
1326 }
1327 if (i >= share->base.keys)
1328 {
1329 /*
1330 All of the disabled indexes are empty. We can just recreate them.
1331 Flush dirty blocks of this index file from key cache and remove
1332 all blocks of this index file from key cache.
1333 */
1334 return;
1335 }
1336 /*
1337 We do now drop all indexes and declare them disabled. With the
1338 T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
1339 disabled indexes and enable them.
1340 */
1341 mi_clear_all_keys_active(state->key_map);
1342 }
1343
1344 /* Clear index root block pointers. */
1345 for (i= 0; i < share->base.keys; i++)
1346 state->key_root[i]= HA_OFFSET_ERROR;
1347
1348 /* Clear the delete chains. */
1349 for (i= 0; i < state->header.max_block_size_index; i++)
1350 state->key_del[i]= HA_OFFSET_ERROR;
1351
1352 /* Reset index file length to end of index file header. */
1353 info->state->key_file_length= share->base.keystart;
1354}
1355
1356
1357 /* Recover old table by reading each record and writing all keys */
1358 /* Save new datafile-name in temp_filename */
1359
1360int mi_repair(MI_CHECK *param, MI_INFO *info,
1361 char * name, int rep_quick)
1362{
1363 int error,got_error;
1364 ha_rows start_records,new_header_length;
1365 my_off_t del;
1366 int new_file;
1367 MYISAM_SHARE *share=info->s;
1368 char llbuff[22],llbuff2[22];
1369 SORT_INFO sort_info;
1370 MI_SORT_PARAM sort_param= MI_SORT_PARAM();
1371
1372 memset(&sort_info, 0, sizeof(sort_info));
1373 start_records=info->state->records;
1374 new_header_length=(param->testflag & T_UNPACK) ? 0L :
1375 share->pack.header_length;
1376 got_error=1;
1377 new_file= -1;
1378 sort_param.sort_info=&sort_info;
1379
1380 if (!(param->testflag & T_SILENT))
1381 {
1382 printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
1383 printf("Data records: %s\n", llstr(info->state->records,llbuff));
1384 }
1385 param->testflag|=T_REP; /* for easy checking */
1386
1387 if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
1388 param->testflag|=T_CALC_CHECKSUM;
1389
1390 if (!param->using_global_keycache)
1391 assert(0);
1392
1393 if (param->read_cache.init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1394 {
1395 info->rec_cache= drizzled::internal::io_cache_st();
1396 goto err;
1397 }
1398 if (not rep_quick)
1399 {
1400 if (info->rec_cache.init_io_cache(-1, (uint) param->write_buffer_length, WRITE_CACHE, new_header_length, 1, MYF(MY_WME | MY_WAIT_IF_FULL)))
1401 {
1402 goto err;
1403 }
1404 }
1405 info->opt_flag|=WRITE_CACHE_USED;
1406 if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
1407 !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
1408 {
1409 mi_check_print_error(param, "Not enough memory for extra record");
1410 goto err;
1411 }
1412
1413 if (!rep_quick)
1414 {
1415 /* Get real path for data file */
1416 if ((new_file=my_create(internal::fn_format(param->temp_filename,
1417 share->data_file_name, "",
1418 DATA_TMP_EXT, 2+4),
1419 0,param->tmpfile_createflag,
1420 MYF(0))) < 0)
1421 {
1422 mi_check_print_error(param,"Can't create new tempfile: '%s'",
1423 param->temp_filename);
1424 goto err;
1425 }
1426 if (new_header_length &&
1427 filecopy(param,new_file,info->dfile,0L,new_header_length,
1428 "datafile-header"))
1429 goto err;
1430 info->s->state.dellink= HA_OFFSET_ERROR;
1431 info->rec_cache.file=new_file;
1432 if (param->testflag & T_UNPACK)
1433 {
1434 share->options&= ~HA_OPTION_COMPRESS_RECORD;
1435 mi_int2store(share->state.header.options,share->options);
1436 }
1437 }
1438 sort_info.info=info;
1439 sort_info.param = param;
1440 sort_param.read_cache=param->read_cache;
1441 sort_param.pos=sort_param.max_pos=share->pack.header_length;
1442 sort_param.filepos=new_header_length;
1443 param->read_cache.end_of_file=sort_info.filelength=
1444 lseek(info->dfile,0L,SEEK_END);
1445 sort_info.dupp=0;
1446 sort_param.fix_datafile= (bool) (! rep_quick);
1447 sort_param.master=1;
1448 sort_info.max_records= ~(ha_rows) 0;
1449
1450 set_data_file_type(&sort_info, share);
1451 del=info->state->del;
1452 info->state->records=info->state->del=share->state.split=0;
1453 info->state->empty=0;
1454 param->glob_crc=0;
1455 if (param->testflag & T_CALC_CHECKSUM)
1456 sort_param.calc_checksum= 1;
1457
1458 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1459
1460 /* This function always recreates all enabled indexes. */
1461 if (param->testflag & T_CREATE_MISSING_KEYS)
1462 mi_set_all_keys_active(share->state.key_map, share->base.keys);
1463 mi_drop_all_indexes(param, info, true);
1464
1465 lock_memory(param); /* Everything is alloced */
1466
1467 /* Re-create all keys, which are set in key_map. */
1468 while (!(error=sort_get_next_record(&sort_param)))
1469 {
1470 if (writekeys(&sort_param))
1471 {
1472 if (errno != HA_ERR_FOUND_DUPP_KEY)
1473 goto err;
1474 mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1475 info->errkey+1,
1476 llstr(sort_param.start_recpos,llbuff),
1477 llstr(info->dupp_key_pos,llbuff2));
1478 if (param->testflag & T_VERBOSE)
1479 {
1480 _mi_make_key(info,(uint) info->errkey,info->lastkey,
1481 sort_param.record,0L);
1482 }
1483 sort_info.dupp++;
1484 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
1485 {
1486 param->testflag|=T_RETRY_WITHOUT_QUICK;
1487 param->error_printed=1;
1488 goto err;
1489 }
1490 continue;
1491 }
1492 if (sort_write_record(&sort_param))
1493 goto err;
1494 }
1495 if (error > 0 || write_data_suffix(&sort_info, (bool)!rep_quick) ||
1496 info->rec_cache.flush() || param->read_cache.error < 0)
1497 goto err;
1498
1499 if (param->testflag & T_WRITE_LOOP)
1500 {
1501 fputs(" \r",stdout); fflush(stdout);
1502 }
1503 if (ftruncate(share->kfile, info->state->key_file_length))
1504 {
1505 mi_check_print_warning(param,
1506 "Can't change size of indexfile, error: %d",
1507 errno);
1508 goto err;
1509 }
1510
1511 if (rep_quick && del+sort_info.dupp != info->state->del)
1512 {
1513 mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
1514 mi_check_print_error(param,"Run recovery again without -q");
1515 got_error=1;
1516 param->retry_repair=1;
1517 param->testflag|=T_RETRY_WITHOUT_QUICK;
1518 goto err;
1519 }
1520 if (param->testflag & T_SAFE_REPAIR)
1521 {
1522 /* Don't repair if we loosed more than one row */
1523 if (info->state->records+1 < start_records)
1524 {
1525 info->state->records=start_records;
1526 got_error=1;
1527 goto err;
1528 }
1529 }
1530
1531 if (!rep_quick)
1532 {
1533 internal::my_close(info->dfile,MYF(0));
1534 info->dfile=new_file;
1535 info->state->data_file_length=sort_param.filepos;
1536 share->state.version=(ulong) time((time_t*) 0); /* Force reopen */
1537 }
1538 else
1539 {
1540 info->state->data_file_length=sort_param.max_pos;
1541 }
1542 if (param->testflag & T_CALC_CHECKSUM)
1543 info->state->checksum=param->glob_crc;
1544
1545 if (!(param->testflag & T_SILENT))
1546 {
1547 if (start_records != info->state->records)
1548 printf("Data records: %s\n", llstr(info->state->records,llbuff));
1549 if (sort_info.dupp)
1550 mi_check_print_warning(param,
1551 "%s records have been removed",
1552 llstr(sort_info.dupp,llbuff));
1553 }
1554
1555 got_error=0;
1556 /* If invoked by external program that uses thr_lock */
1557 if (&share->state.state != info->state)
1558 memcpy( &share->state.state, info->state, sizeof(*info->state));
1559
1560err:
1561 if (!got_error)
1562 {
1563 /* Replace the actual file with the temporary file */
1564 if (new_file >= 0)
1565 {
1566 internal::my_close(new_file,MYF(0));
1567 info->dfile=new_file= -1;
1568 if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1569 DATA_TMP_EXT, share->base.raid_chunks,
1570 MYF(0)) ||
1571 mi_open_datafile(info,share,-1))
1572 got_error=1;
1573 }
1574 }
1575 if (got_error)
1576 {
1577 if (! param->error_printed)
1578 mi_check_print_error(param,"%d for record at pos %s",errno,
1579 llstr(sort_param.start_recpos,llbuff));
1580 if (new_file >= 0)
1581 {
1582 internal::my_close(new_file,MYF(0));
1583 my_delete(param->temp_filename, MYF(MY_WME));
1584 info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
1585 }
1586 mi_mark_crashed_on_repair(info);
1587 }
1588
1589 void * rec_buff_ptr= NULL;
1590 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
1591 free(rec_buff_ptr);
1592 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
1593 free(rec_buff_ptr);
1594 rec_buff_ptr= NULL;
1595
1596 free(sort_info.buff);
1597 param->read_cache.end_io_cache();
1598 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
1599 info->rec_cache.end_io_cache();
1600 if (not got_error && param->testflag & T_UNPACK)
1601 {
1602 share->state.header.options[0]&= (unsigned char) ~HA_OPTION_COMPRESS_RECORD;
1603 share->pack.header_length=0;
1604 share->data_file_type=sort_info.new_data_file_type;
1605 }
1606 share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
1607 STATE_NOT_ANALYZED);
1608 return(got_error);
1609}
1610
1611
1612/* Uppate keyfile when doing repair */
1613
1614static int writekeys(MI_SORT_PARAM *sort_param)
1615{
1616 uint32_t i;
1617 unsigned char *key;
1618 MI_INFO *info= sort_param->sort_info->info;
1619 unsigned char *buff= sort_param->record;
1620 my_off_t filepos= sort_param->filepos;
1621
1622 key=info->lastkey+info->s->base.max_key_length;
1623 for (i=0 ; i < info->s->base.keys ; i++)
1624 {
1625 if (mi_is_key_active(info->s->state.key_map, i))
1626 {
1627 {
1628 uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
1629 if (_mi_ck_write(info,i,key,key_length))
1630 goto err;
1631 }
1632 }
1633 }
1634 return(0);
1635
1636 err:
1637 if (errno == HA_ERR_FOUND_DUPP_KEY)
1638 {
1639 info->errkey=(int) i; /* This key was found */
1640 while ( i-- > 0 )
1641 {
1642 if (mi_is_key_active(info->s->state.key_map, i))
1643 {
1644 {
1645 uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
1646 if (_mi_ck_delete(info,i,key,key_length))
1647 break;
1648 }
1649 }
1650 }
1651 }
1652 /* Remove checksum that was added to glob_crc in sort_get_next_record */
1653 if (sort_param->calc_checksum)
1654 sort_param->sort_info->param->glob_crc-= info->checksum;
1655 return(-1);
1656} /* writekeys */
1657
1658
1659 /* Change all key-pointers that points to a records */
1660
1661int movepoint(MI_INFO *info, unsigned char *record, my_off_t oldpos,
1662 my_off_t newpos, uint32_t prot_key)
1663{
1664 uint32_t i;
1665 unsigned char *key;
1666 uint32_t key_length;
1667
1668 key=info->lastkey+info->s->base.max_key_length;
1669 for (i=0 ; i < info->s->base.keys; i++)
1670 {
1671 if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
1672 {
1673 key_length=_mi_make_key(info,i,key,record,oldpos);
1674 if (info->s->keyinfo[i].flag & HA_NOSAME)
1675 { /* Change pointer direct */
1676 uint32_t nod_flag;
1677 MI_KEYDEF *keyinfo;
1678 keyinfo=info->s->keyinfo+i;
1679 if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
1680 (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
1681 info->s->state.key_root[i]))
1682 return(-1);
1683 nod_flag=mi_test_if_nod(info->buff);
1684 _mi_dpointer(info,info->int_keypos-nod_flag-
1685 info->s->rec_reflength,newpos);
1686 if (_mi_write_keypage(info,keyinfo,info->last_keypage,
1687 DFLT_INIT_HITS,info->buff))
1688 return(-1);
1689 }
1690 else
1691 { /* Change old key to new */
1692 if (_mi_ck_delete(info,i,key,key_length))
1693 return(-1);
1694 key_length=_mi_make_key(info,i,key,record,newpos);
1695 if (_mi_ck_write(info,i,key,key_length))
1696 return(-1);
1697 }
1698 }
1699 }
1700 return(0);
1701} /* movepoint */
1702
1703
1704 /* Tell system that we want all memory for our cache */
1705
1706void lock_memory(MI_CHECK *)
1707{
1708} /* lock_memory */
1709
1710
1711/* Sort index for more efficent reads */
1712
1713int mi_sort_index(MI_CHECK *param, MI_INFO *info, char * name)
1714{
1715 uint32_t key;
1716 MI_KEYDEF *keyinfo;
1717 int new_file;
1718 my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
1719 uint32_t r_locks,w_locks;
1720 int old_lock;
1721 MYISAM_SHARE *share=info->s;
1722 MI_STATE_INFO old_state;
1723
1724 /* cannot sort index files with R-tree indexes */
1725 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1726 key++,keyinfo++)
1727
1728 if (!(param->testflag & T_SILENT))
1729 printf("- Sorting index for MyISAM-table '%s'\n",name);
1730
1731 /* Get real path for index file */
1732 internal::fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
1733 if ((new_file=my_create(internal::fn_format(param->temp_filename,param->temp_filename,
1734 "", INDEX_TMP_EXT,2+4),
1735 0,param->tmpfile_createflag,MYF(0))) <= 0)
1736 {
1737 mi_check_print_error(param,"Can't create new tempfile: '%s'",
1738 param->temp_filename);
1739 return(-1);
1740 }
1741 if (filecopy(param, new_file,share->kfile,0L,
1742 (ulong) share->base.keystart, "headerblock"))
1743 goto err;
1744
1745 param->new_file_pos=share->base.keystart;
1746 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1747 key++,keyinfo++)
1748 {
1749 if (! mi_is_key_active(info->s->state.key_map, key))
1750 continue;
1751
1752 if (share->state.key_root[key] != HA_OFFSET_ERROR)
1753 {
1754 index_pos[key]=param->new_file_pos; /* Write first block here */
1755 if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
1756 new_file))
1757 goto err;
1758 }
1759 else
1760 index_pos[key]= HA_OFFSET_ERROR; /* No blocks */
1761 }
1762
1763 share->state.version=(ulong) time(NULL);
1764 old_state= share->state; /* save state if not stored */
1765 r_locks= share->r_locks;
1766 w_locks= share->w_locks;
1767 old_lock= info->lock_type;
1768
1769 /* Put same locks as old file */
1770 share->r_locks= share->w_locks= share->tot_locks= 0;
1771 (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
1772 internal::my_close(share->kfile,MYF(MY_WME));
1773 share->kfile = -1;
1774 internal::my_close(new_file,MYF(MY_WME));
1775 if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
1776 MYF(0)) ||
1777 mi_open_keyfile(share))
1778 goto err2;
1779 info->lock_type= F_UNLCK; /* Force mi_readinfo to lock */
1780 _mi_readinfo(info,F_WRLCK,0); /* Will lock the table */
1781 info->lock_type= old_lock;
1782 share->r_locks= r_locks;
1783 share->w_locks= w_locks;
1784 share->tot_locks= r_locks+w_locks;
1785 share->state= old_state; /* Restore old state */
1786
1787 info->state->key_file_length=param->new_file_pos;
1788 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1789 for (key=0 ; key < info->s->base.keys ; key++)
1790 info->s->state.key_root[key]=index_pos[key];
1791 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
1792 info->s->state.key_del[key]= HA_OFFSET_ERROR;
1793
1794 info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
1795 return(0);
1796
1797err:
1798 internal::my_close(new_file,MYF(MY_WME));
1799err2:
1800 my_delete(param->temp_filename,MYF(MY_WME));
1801 return(-1);
1802} /* mi_sort_index */
1803
1804
1805 /* Sort records recursive using one index */
1806
1807static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
1808 my_off_t pagepos, int new_file)
1809{
1810 uint32_t length,nod_flag,used_length, key_length;
1811 unsigned char *buff,*keypos,*endpos;
1812 unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF];
1813 my_off_t new_page_pos,next_page;
1814 char llbuff[22];
1815
1816 new_page_pos=param->new_file_pos;
1817 param->new_file_pos+=keyinfo->block_length;
1818
1819 if (!(buff=(unsigned char*) malloc(keyinfo->block_length)))
1820 {
1821 mi_check_print_error(param,"Not enough memory for key block");
1822 return(-1);
1823 }
1824 if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
1825 {
1826 mi_check_print_error(param,"Can't read key block from filepos: %s",
1827 llstr(pagepos,llbuff));
1828 goto err;
1829 }
1830 if ((nod_flag=mi_test_if_nod(buff)))
1831 {
1832 used_length=mi_getint(buff);
1833 keypos=buff+2+nod_flag;
1834 endpos=buff+used_length;
1835 for ( ;; )
1836 {
1837 if (nod_flag)
1838 {
1839 next_page=_mi_kpos(nod_flag,keypos);
1840 _mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
1841 if (sort_one_index(param,info,keyinfo,next_page,new_file))
1842 {
1843 goto err;
1844 }
1845 }
1846 if (keypos >= endpos ||
1847 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
1848 break;
1849 assert(keypos <= endpos);
1850 }
1851 }
1852
1853 /* Fill block with zero and write it to the new index file */
1854 length=mi_getint(buff);
1855 memset(buff+length, 0, keyinfo->block_length-length);
1856 if (my_pwrite(new_file,(unsigned char*) buff,(uint) keyinfo->block_length,
1857 new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
1858 {
1859 mi_check_print_error(param,"Can't write indexblock, error: %d",errno);
1860 goto err;
1861 }
1862 free(buff);
1863 return(0);
1864err:
1865 free(buff);
1866 return(1);
1867} /* sort_one_index */
1868
1869
1870 /*
1871 Let temporary file replace old file.
1872 This assumes that the new file was created in the same
1873 directory as given by realpath(filename).
1874 This will ensure that any symlinks that are used will still work.
1875 Copy stats from old file to new file, deletes orignal and
1876 changes new file name to old file name
1877 */
1878
1879int change_to_newfile(const char * filename, const char * old_ext,
1880 const char * new_ext,
1881 uint32_t raid_chunks,
1882 myf MyFlags)
1883{
1884 (void)raid_chunks;
1885 char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
1886 /* Get real path to filename */
1887 (void) internal::fn_format(old_filename,filename,"",old_ext,2+4+32);
1888 return my_redel(old_filename,
1889 internal::fn_format(new_filename,old_filename,"",new_ext,2+4),
1890 MYF(MY_WME | MY_LINK_WARNING | MyFlags));
1891} /* change_to_newfile */
1892
1893
1894
1895 /* Copy a block between two files */
1896
1897int filecopy(MI_CHECK *param, int to,int from,my_off_t start,
1898 my_off_t length, const char *type)
1899{
1900 char tmp_buff[IO_SIZE],*buff;
1901 ulong buff_length;
1902
1903 buff_length=(ulong) min(param->write_buffer_length, (size_t)length);
1904 if (!(buff=(char *)malloc(buff_length)))
1905 {
1906 buff=tmp_buff; buff_length=IO_SIZE;
1907 }
1908
1909 lseek(from,start,SEEK_SET);
1910 while (length > buff_length)
1911 {
1912 if (my_read(from,(unsigned char*) buff,buff_length,MYF(MY_NABP)) ||
1913 my_write(to,(unsigned char*) buff,buff_length,param->myf_rw))
1914 goto err;
1915 length-= buff_length;
1916 }
1917 if (my_read(from,(unsigned char*) buff,(uint) length,MYF(MY_NABP)) ||
1918 my_write(to,(unsigned char*) buff,(uint) length,param->myf_rw))
1919 goto err;
1920 if (buff != tmp_buff)
1921 free(buff);
1922 return(0);
1923err:
1924 if (buff != tmp_buff)
1925 free(buff);
1926 mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
1927 type,errno);
1928 return(1);
1929}
1930
1931
1932/*
1933 Repair table or given index using sorting
1934
1935 SYNOPSIS
1936 mi_repair_by_sort()
1937 param Repair parameters
1938 info MyISAM handler to repair
1939 name Name of table (for warnings)
1940 rep_quick set to <> 0 if we should not change data file
1941
1942 RESULT
1943 0 ok
1944 <>0 Error
1945*/
1946
1947int mi_repair_by_sort(MI_CHECK *param, MI_INFO *info,
1948 const char * name, int rep_quick)
1949{
1950 int got_error;
1951 uint32_t i;
1952 ulong length;
1953 ha_rows start_records;
1954 my_off_t new_header_length,del;
1955 int new_file;
1956 MI_SORT_PARAM sort_param= MI_SORT_PARAM();
1957 MYISAM_SHARE *share=info->s;
1958 HA_KEYSEG *keyseg;
1959 ulong *rec_per_key_part;
1960 char llbuff[22];
1961 SORT_INFO sort_info;
1962 uint64_t key_map= 0;
1963
1964 start_records=info->state->records;
1965 got_error=1;
1966 new_file= -1;
1967 new_header_length=(param->testflag & T_UNPACK) ? 0 :
1968 share->pack.header_length;
1969 if (!(param->testflag & T_SILENT))
1970 {
1971 printf("- recovering (with sort) MyISAM-table '%s'\n",name);
1972 printf("Data records: %s\n", llstr(start_records,llbuff));
1973 }
1974 param->testflag|=T_REP; /* for easy checking */
1975
1976 if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
1977 param->testflag|=T_CALC_CHECKSUM;
1978
1979 memset(&sort_info, 0, sizeof(sort_info));
1980 if (!(sort_info.key_block=
1981 alloc_key_blocks(param, (uint) param->sort_key_blocks, share->base.max_key_block_length))
1982 || param->read_cache.init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME))
1983 || (! rep_quick && info->rec_cache.init_io_cache(info->dfile, (uint) param->write_buffer_length, WRITE_CACHE,new_header_length,1, MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
1984 {
1985 goto err;
1986 }
1987 sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
1988 info->opt_flag|=WRITE_CACHE_USED;
1989 info->rec_cache.file=info->dfile; /* for sort_delete_record */
1990
1991 if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
1992 !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
1993 {
1994 mi_check_print_error(param, "Not enough memory for extra record");
1995 goto err;
1996 }
1997 if (!rep_quick)
1998 {
1999 /* Get real path for data file */
2000 if ((new_file=my_create(internal::fn_format(param->temp_filename,
2001 share->data_file_name, "",
2002 DATA_TMP_EXT, 2+4),
2003 0,param->tmpfile_createflag,
2004 MYF(0))) < 0)
2005 {
2006 mi_check_print_error(param,"Can't create new tempfile: '%s'",
2007 param->temp_filename);
2008 goto err;
2009 }
2010 if (new_header_length &&
2011 filecopy(param, new_file,info->dfile,0L,new_header_length,
2012 "datafile-header"))
2013 goto err;
2014 if (param->testflag & T_UNPACK)
2015 {
2016 share->options&= ~HA_OPTION_COMPRESS_RECORD;
2017 mi_int2store(share->state.header.options,share->options);
2018 }
2019 share->state.dellink= HA_OFFSET_ERROR;
2020 info->rec_cache.file=new_file;
2021 }
2022
2023 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2024
2025 /* Optionally drop indexes and optionally modify the key_map. */
2026 mi_drop_all_indexes(param, info, false);
2027 key_map= share->state.key_map;
2028 if (param->testflag & T_CREATE_MISSING_KEYS)
2029 {
2030 /* Invert the copied key_map to recreate all disabled indexes. */
2031 key_map= ~key_map;
2032 }
2033
2034 sort_info.info=info;
2035 sort_info.param = param;
2036
2037 set_data_file_type(&sort_info, share);
2038 sort_param.filepos=new_header_length;
2039 sort_info.dupp=0;
2040 sort_info.buff=0;
2041 param->read_cache.end_of_file=sort_info.filelength=
2042 lseek(param->read_cache.file,0L,SEEK_END);
2043
2044 sort_param.wordlist=NULL;
2045
2046 if (share->data_file_type == DYNAMIC_RECORD)
2047 length=max(share->base.min_pack_length+1,share->base.min_block_length);
2048 else if (share->data_file_type == COMPRESSED_RECORD)
2049 length=share->base.min_block_length;
2050 else
2051 length=share->base.pack_reclength;
2052 sort_info.max_records=
2053 ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
2054 (ha_rows) (sort_info.filelength/length+1));
2055 sort_param.key_cmp=sort_key_cmp;
2056 sort_param.lock_in_memory=lock_memory;
2057 sort_param.sort_info=&sort_info;
2058 sort_param.fix_datafile= (bool) (! rep_quick);
2059 sort_param.master =1;
2060
2061 del=info->state->del;
2062 param->glob_crc=0;
2063 if (param->testflag & T_CALC_CHECKSUM)
2064 sort_param.calc_checksum= 1;
2065
2066 rec_per_key_part= param->rec_per_key_part;
2067 for (sort_param.key=0 ; sort_param.key < share->base.keys ;
2068 rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
2069 {
2070 sort_param.read_cache=param->read_cache;
2071 sort_param.keyinfo=share->keyinfo+sort_param.key;
2072 sort_param.seg=sort_param.keyinfo->seg;
2073 /*
2074 Skip this index if it is marked disabled in the copied
2075 (and possibly inverted) key_map.
2076 */
2077 if (! mi_is_key_active(key_map, sort_param.key))
2078 {
2079 /* Remember old statistics for key */
2080 assert(rec_per_key_part >= param->rec_per_key_part);
2081 memcpy(rec_per_key_part,
2082 (share->state.rec_per_key_part +
2083 (rec_per_key_part - param->rec_per_key_part)),
2084 sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
2085 continue;
2086 }
2087
2088 if ((!(param->testflag & T_SILENT)))
2089 printf ("- Fixing index %d\n",sort_param.key+1);
2090 sort_param.max_pos=sort_param.pos=share->pack.header_length;
2091 keyseg=sort_param.seg;
2092 memset(sort_param.unique, 0, sizeof(sort_param.unique));
2093 sort_param.key_length=share->rec_reflength;
2094 for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
2095 {
2096 sort_param.key_length+=keyseg[i].length;
2097 if (keyseg[i].flag & HA_SPACE_PACK)
2098 sort_param.key_length+=get_pack_length(keyseg[i].length);
2099 if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2100 sort_param.key_length+=2 + test(keyseg[i].length >= 127);
2101 if (keyseg[i].flag & HA_NULL_PART)
2102 sort_param.key_length++;
2103 }
2104 info->state->records=info->state->del=share->state.split=0;
2105 info->state->empty=0;
2106
2107 {
2108 sort_param.key_read=sort_key_read;
2109 sort_param.key_write=sort_key_write;
2110 }
2111
2112 if (_create_index_by_sort(&sort_param,
2113 (bool) (!(param->testflag & T_VERBOSE)),
2114 (uint) param->sort_buffer_length))
2115 {
2116 param->retry_repair=1;
2117 goto err;
2118 }
2119 /* No need to calculate checksum again. */
2120 sort_param.calc_checksum= 0;
2121 sort_param.wordroot.free_root(MYF(0));
2122
2123 /* Set for next loop */
2124 sort_info.max_records= (ha_rows) info->state->records;
2125
2126 if (param->testflag & T_STATISTICS)
2127 update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2128 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2129 sort_param.notnull: NULL,
2130 (uint64_t) info->state->records);
2131 /* Enable this index in the permanent (not the copied) key_map. */
2132 mi_set_key_active(share->state.key_map, sort_param.key);
2133
2134 if (sort_param.fix_datafile)
2135 {
2136 param->read_cache.end_of_file=sort_param.filepos;
2137 if (write_data_suffix(&sort_info, 1) || info->rec_cache.end_io_cache())
2138 {
2139 goto err;
2140 }
2141 if (param->testflag & T_SAFE_REPAIR)
2142 {
2143 /* Don't repair if we loosed more than one row */
2144 if (info->state->records+1 < start_records)
2145 {
2146 info->state->records=start_records;
2147 goto err;
2148 }
2149 }
2150 share->state.state.data_file_length = info->state->data_file_length=
2151 sort_param.filepos;
2152 /* Only whole records */
2153 share->state.version=(ulong) time((time_t*) 0);
2154 internal::my_close(info->dfile,MYF(0));
2155 info->dfile=new_file;
2156 share->data_file_type=sort_info.new_data_file_type;
2157 share->pack.header_length=(ulong) new_header_length;
2158 sort_param.fix_datafile=0;
2159 }
2160 else
2161 info->state->data_file_length=sort_param.max_pos;
2162
2163 param->read_cache.file=info->dfile; /* re-init read cache */
2164 param->read_cache.reinit_io_cache(READ_CACHE,share->pack.header_length, 1,1);
2165 }
2166
2167 if (param->testflag & T_WRITE_LOOP)
2168 {
2169 fputs(" \r",stdout); fflush(stdout);
2170 }
2171
2172 if (rep_quick && del+sort_info.dupp != info->state->del)
2173 {
2174 mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
2175 mi_check_print_error(param,"Run recovery again without -q");
2176 got_error=1;
2177 param->retry_repair=1;
2178 param->testflag|=T_RETRY_WITHOUT_QUICK;
2179 goto err;
2180 }
2181
2182 if (rep_quick & T_FORCE_UNIQUENESS)
2183 {
2184 my_off_t skr=info->state->data_file_length+
2185 (share->options & HA_OPTION_COMPRESS_RECORD ?
2186 MEMMAP_EXTRA_MARGIN : 0);
2187#ifdef USE_RELOC
2188 if (share->data_file_type == STATIC_RECORD &&
2189 skr < share->base.reloc*share->base.min_pack_length)
2190 skr=share->base.reloc*share->base.min_pack_length;
2191#endif
2192 if (skr != sort_info.filelength && !info->s->base.raid_type)
2193 if (ftruncate(info->dfile, skr))
2194 mi_check_print_warning(param,
2195 "Can't change size of datafile, error: %d",
2196 errno);
2197 }
2198 if (param->testflag & T_CALC_CHECKSUM)
2199 info->state->checksum=param->glob_crc;
2200
2201 if (ftruncate(share->kfile, info->state->key_file_length))
2202 mi_check_print_warning(param,
2203 "Can't change size of indexfile, error: %d",
2204 errno);
2205
2206 if (!(param->testflag & T_SILENT))
2207 {
2208 if (start_records != info->state->records)
2209 printf("Data records: %s\n", llstr(info->state->records,llbuff));
2210 if (sort_info.dupp)
2211 mi_check_print_warning(param,
2212 "%s records have been removed",
2213 llstr(sort_info.dupp,llbuff));
2214 }
2215 got_error=0;
2216
2217 if (&share->state.state != info->state)
2218 memcpy( &share->state.state, info->state, sizeof(*info->state));
2219
2220err:
2221 info->rec_cache.end_io_cache();
2222 if (!got_error)
2223 {
2224 /* Replace the actual file with the temporary file */
2225 if (new_file >= 0)
2226 {
2227 internal::my_close(new_file,MYF(0));
2228 info->dfile=new_file= -1;
2229 if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2230 DATA_TMP_EXT, share->base.raid_chunks,
2231 MYF(0)) ||
2232 mi_open_datafile(info,share,-1))
2233 got_error=1;
2234 }
2235 }
2236 if (got_error)
2237 {
2238 if (! param->error_printed)
2239 mi_check_print_error(param,"%d when fixing table",errno);
2240 if (new_file >= 0)
2241 {
2242 internal::my_close(new_file,MYF(0));
2243 my_delete(param->temp_filename, MYF(MY_WME));
2244 if (info->dfile == new_file)
2245 info->dfile= -1;
2246 }
2247 mi_mark_crashed_on_repair(info);
2248 }
2249 else if (key_map == share->state.key_map)
2250 share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
2251 share->state.changed|=STATE_NOT_SORTED_PAGES;
2252
2253 void * rec_buff_ptr= NULL;
2254 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
2255 free(rec_buff_ptr);
2256 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
2257 free(rec_buff_ptr);
2258 rec_buff_ptr= NULL;
2259
2260 free((unsigned char*) sort_info.key_block);
2261 free(sort_info.buff);
2262 param->read_cache.end_io_cache();
2263 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2264 if (!got_error && (param->testflag & T_UNPACK))
2265 {
2266 share->state.header.options[0]&= (unsigned char) ~HA_OPTION_COMPRESS_RECORD;
2267 share->pack.header_length=0;
2268 }
2269 return(got_error);
2270}
2271
2272 /* Read next record and return next key */
2273
2274int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
2275{
2276 int error;
2277 SORT_INFO *sort_info=sort_param->sort_info;
2278 MI_INFO *info=sort_info->info;
2279
2280 if ((error=sort_get_next_record(sort_param)))
2281 return(error);
2282 if (info->state->records == sort_info->max_records)
2283 {
2284 mi_check_print_error(sort_info->param,
2285 "Key %d - Found too many records; Can't continue",
2286 sort_param->key+1);
2287 return(1);
2288 }
2289 sort_param->real_key_length=
2290 (info->s->rec_reflength+
2291 _mi_make_key(info, sort_param->key, (unsigned char*) key,
2292 sort_param->record, sort_param->filepos));
2293#ifdef HAVE_VALGRIND
2294 memset((unsigned char *)key+sort_param->real_key_length, 0,
2295 (sort_param->key_length-sort_param->real_key_length));
2296#endif
2297 return(sort_write_record(sort_param));
2298} /* sort_key_read */
2299
2300
2301/*
2302 Read next record from file using parameters in sort_info.
2303
2304 SYNOPSIS
2305 sort_get_next_record()
2306 sort_param Information about and for the sort process
2307
2308 NOTE
2309
2310 Dynamic Records With Non-Quick Parallel Repair
2311
2312 For non-quick parallel repair we use a synchronized read/write
2313 cache. This means that one thread is the master who fixes the data
2314 file by reading each record from the old data file and writing it
2315 to the new data file. By doing this the records in the new data
2316 file are written contiguously. Whenever the write buffer is full,
2317 it is copied to the read buffer. The slaves read from the read
2318 buffer, which is not associated with a file. Thus read_cache.file
2319 is -1. When using _mi_read_cache(), the slaves must always set
2320 flag to READING_NEXT so that the function never tries to read from
2321 file. This is safe because the records are contiguous. There is no
2322 need to read outside the cache. This condition is evaluated in the
2323 variable 'parallel_flag' for quick reference. read_cache.file must
2324 be >= 0 in every other case.
2325
2326 RETURN
2327 -1 end of file
2328 0 ok
2329 > 0 error
2330*/
2331
2332int sort_get_next_record(MI_SORT_PARAM *sort_param)
2333{
2334 int searching;
2335 int parallel_flag;
2336 uint32_t found_record,b_type,left_length;
2337 my_off_t pos;
2338 unsigned char *to= NULL;
2339 MI_BLOCK_INFO block_info;
2340 SORT_INFO *sort_info=sort_param->sort_info;
2341 MI_CHECK *param=sort_info->param;
2342 MI_INFO *info=sort_info->info;
2343 MYISAM_SHARE *share=info->s;
2344 char llbuff[22],llbuff2[22];
2345
2346 if (*killed_ptr(param))
2347 return(1);
2348
2349 switch (share->data_file_type) {
2350 case STATIC_RECORD:
2351 for (;;)
2352 {
2353 if (sort_param->read_cache.read(sort_param->record, share->base.pack_reclength))
2354 {
2355 if (sort_param->read_cache.error)
2356 param->out_flag |= O_DATA_LOST;
2357 param->retry_repair=1;
2358 param->testflag|=T_RETRY_WITHOUT_QUICK;
2359 return(-1);
2360 }
2361 sort_param->start_recpos=sort_param->pos;
2362 if (!sort_param->fix_datafile)
2363 {
2364 sort_param->filepos=sort_param->pos;
2365 if (sort_param->master)
2366 share->state.split++;
2367 }
2368 sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
2369 if (*sort_param->record)
2370 {
2371 if (sort_param->calc_checksum)
2372 param->glob_crc+= (info->checksum=
2373 mi_static_checksum(info,sort_param->record));
2374 return(0);
2375 }
2376 if (!sort_param->fix_datafile && sort_param->master)
2377 {
2378 info->state->del++;
2379 info->state->empty+=share->base.pack_reclength;
2380 }
2381 }
2382 case DYNAMIC_RECORD:
2383 pos= sort_param->pos;
2384 searching= (sort_param->fix_datafile && (param->testflag & T_EXTEND));
2385 parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
2386 for (;;)
2387 {
2388 found_record=block_info.second_read= 0;
2389 left_length=1;
2390 if (searching)
2391 {
2392 pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
2393 param->testflag|=T_RETRY_WITHOUT_QUICK;
2394 sort_param->start_recpos=pos;
2395 }
2396 do
2397 {
2398 if (pos > sort_param->max_pos)
2399 sort_param->max_pos=pos;
2400 if (pos & (MI_DYN_ALIGN_SIZE-1))
2401 {
2402 if ((param->testflag & T_VERBOSE) || searching == 0)
2403 mi_check_print_info(param,"Wrong aligned block at %s",
2404 llstr(pos,llbuff));
2405 if (searching)
2406 goto try_next;
2407 }
2408 if (found_record && pos == param->search_after_block)
2409 mi_check_print_info(param,"Block: %s used by record at %s",
2410 llstr(param->search_after_block,llbuff),
2411 llstr(sort_param->start_recpos,llbuff2));
2412 if (_mi_read_cache(&sort_param->read_cache,
2413 (unsigned char*) block_info.header,pos,
2414 MI_BLOCK_INFO_HEADER_LENGTH,
2415 (! found_record ? READING_NEXT : 0) |
2416 parallel_flag | READING_HEADER))
2417 {
2418 if (found_record)
2419 {
2420 mi_check_print_info(param,
2421 "Can't read whole record at %s (errno: %d)",
2422 llstr(sort_param->start_recpos,llbuff),errno);
2423 goto try_next;
2424 }
2425 return(-1);
2426 }
2427 if (searching && ! sort_param->fix_datafile)
2428 {
2429 param->error_printed=1;
2430 param->retry_repair=1;
2431 param->testflag|=T_RETRY_WITHOUT_QUICK;
2432 return(1); /* Something wrong with data */
2433 }
2434 b_type=_mi_get_block_info(&block_info,-1,pos);
2435 if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
2436 ((b_type & BLOCK_FIRST) &&
2437 (block_info.rec_len < (uint) share->base.min_pack_length ||
2438 block_info.rec_len > (uint) share->base.max_pack_length)))
2439 {
2440 uint32_t i;
2441 if (param->testflag & T_VERBOSE || searching == 0)
2442 mi_check_print_info(param,
2443 "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
2444 block_info.header[0],block_info.header[1],
2445 block_info.header[2],llstr(pos,llbuff));
2446 if (found_record)
2447 goto try_next;
2448 block_info.second_read=0;
2449 searching=1;
2450 /* Search after block in read header string */
2451 for (i=MI_DYN_ALIGN_SIZE ;
2452 i < MI_BLOCK_INFO_HEADER_LENGTH ;
2453 i+= MI_DYN_ALIGN_SIZE)
2454 if (block_info.header[i] >= 1 &&
2455 block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
2456 break;
2457 pos+=(ulong) i;
2458 sort_param->start_recpos=pos;
2459 continue;
2460 }
2461 if (b_type & BLOCK_DELETED)
2462 {
2463 bool error=0;
2464 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
2465 share->base.min_block_length)
2466 {
2467 if (!searching)
2468 mi_check_print_info(param,
2469 "Deleted block with impossible length %u at %s",
2470 block_info.block_len,llstr(pos,llbuff));
2471 error=1;
2472 }
2473 else
2474 {
2475 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
2476 block_info.next_filepos >=
2477 info->state->data_file_length) ||
2478 (block_info.prev_filepos != HA_OFFSET_ERROR &&
2479 block_info.prev_filepos >= info->state->data_file_length))
2480 {
2481 if (!searching)
2482 mi_check_print_info(param,
2483 "Delete link points outside datafile at %s",
2484 llstr(pos,llbuff));
2485 error=1;
2486 }
2487 }
2488 if (error)
2489 {
2490 if (found_record)
2491 goto try_next;
2492 searching=1;
2493 pos+= MI_DYN_ALIGN_SIZE;
2494 sort_param->start_recpos=pos;
2495 block_info.second_read=0;
2496 continue;
2497 }
2498 }
2499 else
2500 {
2501 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
2502 share->base.min_block_length ||
2503 block_info.block_len > (uint) share->base.max_pack_length+
2504 MI_SPLIT_LENGTH)
2505 {
2506 if (!searching)
2507 mi_check_print_info(param,
2508 "Found block with impossible length %u at %s; Skipped",
2509 block_info.block_len+ (uint) (block_info.filepos-pos),
2510 llstr(pos,llbuff));
2511 if (found_record)
2512 goto try_next;
2513 searching=1;
2514 pos+= MI_DYN_ALIGN_SIZE;
2515 sort_param->start_recpos=pos;
2516 block_info.second_read=0;
2517 continue;
2518 }
2519 }
2520 if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
2521 {
2522 if (!sort_param->fix_datafile && sort_param->master &&
2523 (b_type & BLOCK_DELETED))
2524 {
2525 info->state->empty+=block_info.block_len;
2526 info->state->del++;
2527 share->state.split++;
2528 }
2529 if (found_record)
2530 goto try_next;
2531 if (searching)
2532 {
2533 pos+=MI_DYN_ALIGN_SIZE;
2534 sort_param->start_recpos=pos;
2535 }
2536 else
2537 pos=block_info.filepos+block_info.block_len;
2538 block_info.second_read=0;
2539 continue;
2540 }
2541
2542 if (!sort_param->fix_datafile && sort_param->master)
2543 share->state.split++;
2544 if (! found_record++)
2545 {
2546 sort_param->find_length=left_length=block_info.rec_len;
2547 sort_param->start_recpos=pos;
2548 if (!sort_param->fix_datafile)
2549 sort_param->filepos=sort_param->start_recpos;
2550 if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
2551 sort_param->pos=block_info.filepos+1;
2552 else
2553 sort_param->pos=block_info.filepos+block_info.block_len;
2554 if (share->base.blobs)
2555 {
2556 if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
2557 &(sort_param->rec_buff))))
2558 {
2559 if (param->max_record_length >= block_info.rec_len)
2560 {
2561 mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
2562 llstr(sort_param->start_recpos,llbuff),
2563 (ulong) block_info.rec_len);
2564 return(1);
2565 }
2566 else
2567 {
2568 mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
2569 llstr(sort_param->start_recpos,llbuff),
2570 (ulong) block_info.rec_len);
2571 goto try_next;
2572 }
2573 }
2574 }
2575 else
2576 to= sort_param->rec_buff;
2577 }
2578 if (left_length < block_info.data_len || ! block_info.data_len)
2579 {
2580 mi_check_print_info(param,
2581 "Found block with too small length at %s; Skipped",
2582 llstr(sort_param->start_recpos,llbuff));
2583 goto try_next;
2584 }
2585 if (block_info.filepos + block_info.data_len >
2586 sort_param->read_cache.end_of_file)
2587 {
2588 mi_check_print_info(param,
2589 "Found block that points outside data file at %s",
2590 llstr(sort_param->start_recpos,llbuff));
2591 goto try_next;
2592 }
2593 /*
2594 Copy information that is already read. Avoid accessing data
2595 below the cache start. This could happen if the header
2596 streched over the end of the previous buffer contents.
2597 */
2598 {
2599 uint32_t header_len= (uint) (block_info.filepos - pos);
2600 uint32_t prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
2601
2602 if (prefetch_len > block_info.data_len)
2603 prefetch_len= block_info.data_len;
2604 if (prefetch_len)
2605 {
2606 memcpy(to, block_info.header + header_len, prefetch_len);
2607 block_info.filepos+= prefetch_len;
2608 block_info.data_len-= prefetch_len;
2609 left_length-= prefetch_len;
2610 to+= prefetch_len;
2611 }
2612 }
2613 if (block_info.data_len &&
2614 _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
2615 block_info.data_len,
2616 (found_record == 1 ? READING_NEXT : 0) |
2617 parallel_flag))
2618 {
2619 mi_check_print_info(param,
2620 "Read error for block at: %s (error: %d); Skipped",
2621 llstr(block_info.filepos,llbuff),errno);
2622 goto try_next;
2623 }
2624 left_length-=block_info.data_len;
2625 to+=block_info.data_len;
2626 pos=block_info.next_filepos;
2627 if (pos == HA_OFFSET_ERROR && left_length)
2628 {
2629 mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
2630 llstr(sort_param->start_recpos,llbuff));
2631 goto try_next;
2632 }
2633 if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
2634 {
2635 mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
2636 llstr(pos,llbuff2),
2637 llstr(sort_param->start_recpos,llbuff));
2638 goto try_next;
2639 }
2640 } while (left_length);
2641
2642 if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
2643 sort_param->find_length) != MY_FILE_ERROR)
2644 {
2645 if (sort_param->read_cache.error < 0)
2646 return(1);
2647 if (sort_param->calc_checksum)
2648 info->checksum= mi_checksum(info, sort_param->record);
2649 if ((param->testflag & (T_EXTEND | T_REP)) || searching)
2650 {
2651 if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
2652 sort_param->find_length,
2653 (param->testflag & T_QUICK) &&
2654 sort_param->calc_checksum &&
2655 test(info->s->calc_checksum)))
2656 {
2657 mi_check_print_info(param,"Found wrong packed record at %s",
2658 llstr(sort_param->start_recpos,llbuff));
2659 goto try_next;
2660 }
2661 }
2662 if (sort_param->calc_checksum)
2663 param->glob_crc+= info->checksum;
2664 return(0);
2665 }
2666 if (!searching)
2667 mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
2668 sort_param->key+1,
2669 llstr(sort_param->start_recpos,llbuff));
2670 try_next:
2671 pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
2672 searching=1;
2673 }
2674 case COMPRESSED_RECORD:
2675 case BLOCK_RECORD:
2676 assert(0); /* Impossible */
2677 }
2678 return(1); /* Impossible */
2679}
2680
2681
2682/*
2683 Write record to new file.
2684
2685 SYNOPSIS
2686 sort_write_record()
2687 sort_param Sort parameters.
2688
2689 NOTE
2690 This is only called by a master thread if parallel repair is used.
2691
2692 RETURN
2693 0 OK
2694 1 Error
2695*/
2696
2697int sort_write_record(MI_SORT_PARAM *sort_param)
2698{
2699 int flag;
2700 ulong block_length,reclength;
2701 unsigned char *from;
2702 SORT_INFO *sort_info=sort_param->sort_info;
2703 MI_CHECK *param=sort_info->param;
2704 MI_INFO *info=sort_info->info;
2705 MYISAM_SHARE *share=info->s;
2706
2707 if (sort_param->fix_datafile)
2708 {
2709 switch (sort_info->new_data_file_type) {
2710 case STATIC_RECORD:
2711 if (info->rec_cache.write(sort_param->record, share->base.pack_reclength))
2712 {
2713 mi_check_print_error(param,"%d when writing to datafile",errno);
2714 return(1);
2715 }
2716 sort_param->filepos+=share->base.pack_reclength;
2717 info->s->state.split++;
2718 /* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */
2719 break;
2720 case DYNAMIC_RECORD:
2721 if (! info->blobs)
2722 from=sort_param->rec_buff;
2723 else
2724 {
2725 /* must be sure that local buffer is big enough */
2726 reclength=info->s->base.pack_reclength+
2727 _my_calc_total_blob_length(info,sort_param->record)+
2728 ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
2729 MI_DYN_DELETE_BLOCK_HEADER;
2730 if (sort_info->buff_length < reclength)
2731 {
2732 void *tmpptr= NULL;
2733 tmpptr= realloc(sort_info->buff, reclength);
2734 if(tmpptr)
2735 {
2736 sort_info->buff_length=reclength;
2737 sort_info->buff= (unsigned char *)tmpptr;
2738 }
2739 else
2740 {
2741 mi_check_print_error(param,"Could not realloc() sort_info->buff "
2742 " to %ul bytes", reclength);
2743 return(1);
2744 }
2745 }
2746 from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
2747 }
2748 /* We can use info->checksum here as only one thread calls this. */
2749 info->checksum=mi_checksum(info,sort_param->record);
2750 reclength=_mi_rec_pack(info,from,sort_param->record);
2751 flag=0;
2752 /* sort_info->param->glob_crc+=info->checksum; */
2753
2754 do
2755 {
2756 block_length=reclength+ 3 + test(reclength >= (65520-3));
2757 if (block_length < share->base.min_block_length)
2758 block_length=share->base.min_block_length;
2759 info->update|=HA_STATE_WRITE_AT_END;
2760 block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
2761 if (block_length > MI_MAX_BLOCK_LENGTH)
2762 block_length=MI_MAX_BLOCK_LENGTH;
2763 if (_mi_write_part_record(info,0L,block_length,
2764 sort_param->filepos+block_length,
2765 &from,&reclength,&flag))
2766 {
2767 mi_check_print_error(param,"%d when writing to datafile",errno);
2768 return(1);
2769 }
2770 sort_param->filepos+=block_length;
2771 info->s->state.split++;
2772 } while (reclength);
2773 /* sort_info->param->glob_crc+=info->checksum; */
2774 break;
2775 case COMPRESSED_RECORD:
2776 case BLOCK_RECORD:
2777 assert(0); /* Impossible */
2778 }
2779 }
2780 if (sort_param->master)
2781 {
2782 info->state->records++;
2783 if ((param->testflag & T_WRITE_LOOP) &&
2784 (info->state->records % WRITE_COUNT) == 0)
2785 {
2786 char llbuff[22];
2787 printf("%s\r", llstr(info->state->records,llbuff));
2788 fflush(stdout);
2789 }
2790 }
2791 return(0);
2792} /* sort_write_record */
2793
2794
2795 /* Compare two keys from _create_index_by_sort */
2796
2797int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a, const void *b)
2798{
2799 uint32_t not_used[2];
2800 return (ha_key_cmp(sort_param->seg, *((unsigned char* const *) a), *((unsigned char* const *) b),
2801 USE_WHOLE_KEY, SEARCH_SAME, not_used));
2802} /* sort_key_cmp */
2803
2804
2805int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
2806{
2807 uint32_t diff_pos[2];
2808 char llbuff[22],llbuff2[22];
2809 SORT_INFO *sort_info=sort_param->sort_info;
2810 MI_CHECK *param= sort_info->param;
2811 int cmp;
2812
2813 if (sort_info->key_block->inited)
2814 {
2815 cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
2816 (unsigned char*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
2817 diff_pos);
2818 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
2819 ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
2820 (unsigned char*) a, USE_WHOLE_KEY,
2821 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
2822 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
2823 {
2824 diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
2825 sort_param->notnull,
2826 sort_info->key_block->lastkey,
2827 (unsigned char*)a);
2828 }
2829 sort_param->unique[diff_pos[0]-1]++;
2830 }
2831 else
2832 {
2833 cmp= -1;
2834 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
2835 mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
2836 (unsigned char*)a);
2837 }
2838 if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
2839 {
2840 sort_info->dupp++;
2841 sort_info->info->lastpos=get_record_for_key(sort_info->info,
2842 sort_param->keyinfo,
2843 (unsigned char*) a);
2844 mi_check_print_warning(param,
2845 "Duplicate key for record at %10s against record at %10s",
2846 llstr(sort_info->info->lastpos,llbuff),
2847 llstr(get_record_for_key(sort_info->info,
2848 sort_param->keyinfo,
2849 sort_info->key_block->
2850 lastkey),
2851 llbuff2));
2852 param->testflag|=T_RETRY_WITHOUT_QUICK;
2853 return (sort_delete_record(sort_param));
2854 }
2855 return (sort_insert_key(sort_param,sort_info->key_block,
2856 (unsigned char*) a, HA_OFFSET_ERROR));
2857} /* sort_key_write */
2858
2859
2860 /* get pointer to record from a key */
2861
2862my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
2863 unsigned char *key) {
2864 return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
2865} /* get_record_for_key */
2866
2867
2868 /* Insert a key in sort-key-blocks */
2869
2870int sort_insert_key(MI_SORT_PARAM *sort_param,
2871 SORT_KEY_BLOCKS *key_block, unsigned char *key,
2872 my_off_t prev_block)
2873{
2874 uint32_t a_length,t_length,nod_flag;
2875 my_off_t filepos,key_file_length;
2876 unsigned char *anc_buff,*lastkey;
2877 MI_KEY_PARAM s_temp;
2878 MI_INFO *info;
2879 MI_KEYDEF *keyinfo=sort_param->keyinfo;
2880 SORT_INFO *sort_info= sort_param->sort_info;
2881 MI_CHECK *param=sort_info->param;
2882
2883 anc_buff=key_block->buff;
2884 info=sort_info->info;
2885 lastkey=key_block->lastkey;
2886 nod_flag= (key_block == sort_info->key_block ? 0 :
2887 info->s->base.key_reflength);
2888
2889 if (!key_block->inited)
2890 {
2891 key_block->inited=1;
2892 if (key_block == sort_info->key_block_end)
2893 {
2894 mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
2895 return(1);
2896 }
2897 a_length=2+nod_flag;
2898 key_block->end_pos=anc_buff+2;
2899 lastkey=0; /* No previous key in block */
2900 }
2901 else
2902 a_length=mi_getint(anc_buff);
2903
2904 /* Save pointer to previous block */
2905 if (nod_flag)
2906 _mi_kpointer(info,key_block->end_pos,prev_block);
2907
2908 t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
2909 (unsigned char*) 0,lastkey,lastkey,key,
2910 &s_temp);
2911 (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
2912 a_length+=t_length;
2913 mi_putint(anc_buff,a_length,nod_flag);
2914 key_block->end_pos+=t_length;
2915 if (a_length <= keyinfo->block_length)
2916 {
2917 _mi_move_key(keyinfo,key_block->lastkey,key);
2918 key_block->last_length=a_length-t_length;
2919 return(0);
2920 }
2921
2922 /* Fill block with end-zero and write filled block */
2923 mi_putint(anc_buff,key_block->last_length,nod_flag);
2924 memset(anc_buff+key_block->last_length, 0,
2925 keyinfo->block_length - key_block->last_length);
2926 key_file_length=info->state->key_file_length;
2927 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
2928 return(1);
2929
2930 /* If we read the page from the key cache, we have to write it back to it */
2931 if (key_file_length == info->state->key_file_length)
2932 {
2933 if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
2934 return(1);
2935 }
2936 else if (my_pwrite(info->s->kfile,(unsigned char*) anc_buff,
2937 (uint) keyinfo->block_length,filepos, param->myf_rw))
2938 return(1);
2939
2940 /* Write separator-key to block in next level */
2941 if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
2942 return(1);
2943
2944 /* clear old block and write new key in it */
2945 key_block->inited=0;
2946 return(sort_insert_key(sort_param, key_block,key,prev_block));
2947} /* sort_insert_key */
2948
2949
2950 /* Delete record when we found a duplicated key */
2951
2952int sort_delete_record(MI_SORT_PARAM *sort_param)
2953{
2954 uint32_t i;
2955 int old_file,error;
2956 unsigned char *key;
2957 SORT_INFO *sort_info=sort_param->sort_info;
2958 MI_CHECK *param=sort_info->param;
2959 MI_INFO *info=sort_info->info;
2960
2961 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
2962 {
2963 mi_check_print_error(param,
2964 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
2965 return(1);
2966 }
2967 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
2968 {
2969 mi_check_print_error(param,
2970 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
2971 return(1);
2972 }
2973
2974 old_file=info->dfile;
2975 info->dfile=info->rec_cache.file;
2976 if (sort_info->current_key)
2977 {
2978 key=info->lastkey+info->s->base.max_key_length;
2979 if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
2980 error != HA_ERR_RECORD_DELETED)
2981 {
2982 mi_check_print_error(param,"Can't read record to be removed");
2983 info->dfile=old_file;
2984 return(1);
2985 }
2986
2987 for (i=0 ; i < sort_info->current_key ; i++)
2988 {
2989 uint32_t key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
2990 if (_mi_ck_delete(info,i,key,key_length))
2991 {
2992 mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
2993 info->dfile=old_file;
2994 return(1);
2995 }
2996 }
2997 if (sort_param->calc_checksum)
2998 param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
2999 }
3000 error= info->rec_cache.flush() || (*info->s->delete_record)(info);
3001 info->dfile=old_file; /* restore actual value */
3002 info->state->records--;
3003 return(error);
3004} /* sort_delete_record */
3005
3006 /* Fix all pending blocks and flush everything to disk */
3007
3008int flush_pending_blocks(MI_SORT_PARAM *sort_param)
3009{
3010 uint32_t nod_flag,length;
3011 my_off_t filepos,key_file_length;
3012 SORT_KEY_BLOCKS *key_block;
3013 SORT_INFO *sort_info= sort_param->sort_info;
3014 myf myf_rw=sort_info->param->myf_rw;
3015 MI_INFO *info=sort_info->info;
3016 MI_KEYDEF *keyinfo=sort_param->keyinfo;
3017
3018 filepos= HA_OFFSET_ERROR; /* if empty file */
3019 nod_flag=0;
3020 for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
3021 {
3022 key_block->inited=0;
3023 length=mi_getint(key_block->buff);
3024 if (nod_flag)
3025 _mi_kpointer(info,key_block->end_pos,filepos);
3026 key_file_length=info->state->key_file_length;
3027 memset(key_block->buff+length, 0, keyinfo->block_length-length);
3028 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
3029 return(1);
3030
3031 /* If we read the page from the key cache, we have to write it back */
3032 if (key_file_length == info->state->key_file_length)
3033 {
3034 if (_mi_write_keypage(info, keyinfo, filepos,
3035 DFLT_INIT_HITS, key_block->buff))
3036 return(1);
3037 }
3038 else if (my_pwrite(info->s->kfile,(unsigned char*) key_block->buff,
3039 (uint) keyinfo->block_length,filepos, myf_rw))
3040 return(1);
3041 nod_flag=1;
3042 }
3043 info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
3044 return(0);
3045} /* flush_pending_blocks */
3046
3047 /* alloc space and pointers for key_blocks */
3048
3049static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
3050 uint32_t buffer_length)
3051{
3052 uint32_t i;
3053 SORT_KEY_BLOCKS *block;
3054
3055 if (!(block=(SORT_KEY_BLOCKS*) malloc((sizeof(SORT_KEY_BLOCKS)+
3056 buffer_length+IO_SIZE)*blocks)))
3057 {
3058 mi_check_print_error(param,"Not enough memory for sort-key-blocks");
3059 return(0);
3060 }
3061 for (i=0 ; i < blocks ; i++)
3062 {
3063 block[i].inited=0;
3064 block[i].buff=(unsigned char*) (block+blocks)+(buffer_length+IO_SIZE)*i;
3065 }
3066 return(block);
3067} /* alloc_key_blocks */
3068
3069
3070 /* Check if file is almost full */
3071
3072int test_if_almost_full(MI_INFO *info)
3073{
3074 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
3075 return 0;
3076 return (my_off_t)(lseek(info->s->kfile, 0L, SEEK_END) / 10 * 9) >
3077 (my_off_t) info->s->base.max_key_file_length ||
3078 (my_off_t)(lseek(info->dfile, 0L, SEEK_END) / 10 * 9) >
3079 (my_off_t) info->s->base.max_data_file_length;
3080}
3081
3082
3083 /* write suffix to data file if neaded */
3084
3085int write_data_suffix(SORT_INFO *sort_info, bool fix_datafile)
3086{
3087 MI_INFO *info=sort_info->info;
3088
3089 if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
3090 {
3091 unsigned char buff[MEMMAP_EXTRA_MARGIN];
3092 memset(buff, 0, sizeof(buff));
3093 if (info->rec_cache.write(buff, sizeof(buff)))
3094 {
3095 mi_check_print_error(sort_info->param,
3096 "%d when writing to datafile",errno);
3097 return 1;
3098 }
3099 sort_info->param->read_cache.end_of_file+=sizeof(buff);
3100 }
3101 return 0;
3102}
3103
3104 /* Update state and myisamchk_time of indexfile */
3105
3106int update_state_info(MI_CHECK *param, MI_INFO *info,uint32_t update)
3107{
3108 MYISAM_SHARE *share=info->s;
3109
3110 if (update & UPDATE_OPEN_COUNT)
3111 {
3112 share->state.open_count=0;
3113 share->global_changed=0;
3114 }
3115 if (update & UPDATE_STAT)
3116 {
3117 uint32_t i, key_parts= mi_uint2korr(share->state.header.key_parts);
3118 share->state.rec_per_key_rows=info->state->records;
3119 share->state.changed&= ~STATE_NOT_ANALYZED;
3120 if (info->state->records)
3121 {
3122 for (i=0; i<key_parts; i++)
3123 {
3124 if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
3125 share->state.changed|= STATE_NOT_ANALYZED;
3126 }
3127 }
3128 }
3129 if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
3130 {
3131 if (update & UPDATE_TIME)
3132 {
3133 share->state.check_time= (long) time((time_t*) 0);
3134 if (!share->state.create_time)
3135 share->state.create_time=share->state.check_time;
3136 }
3137 /*
3138 When tables are locked we haven't synched the share state and the
3139 real state for a while so we better do it here before synching
3140 the share state to disk. Only when table is write locked is it
3141 necessary to perform this synch.
3142 */
3143 if (info->lock_type == F_WRLCK)
3144 share->state.state= *info->state;
3145 if (mi_state_info_write(share->kfile,&share->state,1+2))
3146 goto err;
3147 share->changed=0;
3148 }
3149 { /* Force update of status */
3150 int error;
3151 uint32_t r_locks=share->r_locks,w_locks=share->w_locks;
3152 share->r_locks= share->w_locks= share->tot_locks= 0;
3153 error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
3154 share->r_locks=r_locks;
3155 share->w_locks=w_locks;
3156 share->tot_locks=r_locks+w_locks;
3157 if (!error)
3158 return 0;
3159 }
3160err:
3161 mi_check_print_error(param,"%d when updating keyfile",errno);
3162 return 1;
3163}
3164
3165 /*
3166 Update auto increment value for a table
3167 When setting the 'repair_only' flag we only want to change the
3168 old auto_increment value if its wrong (smaller than some given key).
3169 The reason is that we shouldn't change the auto_increment value
3170 for a table without good reason when only doing a repair; If the
3171 user have inserted and deleted rows, the auto_increment value
3172 may be bigger than the biggest current row and this is ok.
3173
3174 If repair_only is not set, we will update the flag to the value in
3175 param->auto_increment is bigger than the biggest key.
3176 */
3177
3178void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
3179 bool repair_only)
3180{
3181 unsigned char *record= 0;
3182
3183 if (!info->s->base.auto_key ||
3184 ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
3185 {
3186 if (!(param->testflag & T_VERY_SILENT))
3187 mi_check_print_info(param,
3188 "Table: %s doesn't have an auto increment key\n",
3189 param->isam_file_name);
3190 return;
3191 }
3192 if (!(param->testflag & T_SILENT) &&
3193 !(param->testflag & T_REP))
3194 printf("Updating MyISAM file: %s\n", param->isam_file_name);
3195 /*
3196 We have to use an allocated buffer instead of info->rec_buff as
3197 _mi_put_key_in_record() may use info->rec_buff
3198 */
3199 if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
3200 {
3201 mi_check_print_error(param,"Not enough memory for extra record");
3202 return;
3203 }
3204
3205 mi_extra(info,HA_EXTRA_KEYREAD,0);
3206 if (mi_rlast(info, record, info->s->base.auto_key-1))
3207 {
3208 if (errno != HA_ERR_END_OF_FILE)
3209 {
3210 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
3211 free(mi_get_rec_buff_ptr(info, record));
3212 mi_check_print_error(param,"%d when reading last record",errno);
3213 return;
3214 }
3215 if (!repair_only)
3216 info->s->state.auto_increment=param->auto_increment_value;
3217 }
3218 else
3219 {
3220 uint64_t auto_increment= retrieve_auto_increment(info, record);
3221 set_if_bigger(info->s->state.auto_increment,auto_increment);
3222 if (!repair_only)
3223 set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
3224 }
3225 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
3226 free(mi_get_rec_buff_ptr(info, record));
3227 update_state_info(param, info, UPDATE_AUTO_INC);
3228 return;
3229}
3230
3231
3232/*
3233 Update statistics for each part of an index
3234
3235 SYNOPSIS
3236 update_key_parts()
3237 keyinfo IN Index information (only key->keysegs used)
3238 rec_per_key_part OUT Store statistics here
3239 unique IN Array of (#distinct tuples)
3240 notnull_tuples IN Array of (#tuples), or NULL
3241 records Number of records in the table
3242
3243 DESCRIPTION
3244 This function is called produce index statistics values from unique and
3245 notnull_tuples arrays after these arrays were produced with sequential
3246 index scan (the scan is done in two places: chk_index() and
3247 sort_key_write()).
3248
3249 This function handles all 3 index statistics collection methods.
3250
3251 Unique is an array:
3252 unique[0]= (#different values of {keypart1}) - 1
3253 unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
3254 ...
3255
3256 For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
3257 notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
3258 notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all
3259 keypart{i} are not NULL)
3260 ...
3261 For all other statistics collection methods notnull_tuples==NULL.
3262
3263 Output is an array:
3264 rec_per_key_part[k] =
3265 = E(#records in the table such that keypart_1=c_1 AND ... AND
3266 keypart_k=c_k for arbitrary constants c_1 ... c_k)
3267
3268 = {assuming that values have uniform distribution and index contains all
3269 tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
3270 index tuples}
3271
3272 = #tuples-in-the-index / #distinct-tuples-in-the-index.
3273
3274 The #tuples-in-the-index and #distinct-tuples-in-the-index have different
3275 meaning depending on which statistics collection method is used:
3276
3277 MI_STATS_METHOD_* how are nulls compared? which tuples are counted?
3278 NULLS_EQUAL NULL == NULL all tuples in table
3279 NULLS_NOT_EQUAL NULL != NULL all tuples in table
3280 IGNORE_NULLS n/a tuples that don't have NULLs
3281*/
3282
3283void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
3284 uint64_t *unique, uint64_t *notnull,
3285 uint64_t records)
3286{
3287 uint64_t count=0,tmp, unique_tuples;
3288 uint64_t tuples= records;
3289 uint32_t parts;
3290 for (parts=0 ; parts < keyinfo->keysegs ; parts++)
3291 {
3292 count+=unique[parts];
3293 unique_tuples= count + 1;
3294 if (notnull)
3295 {
3296 tuples= notnull[parts];
3297 /*
3298 #(unique_tuples not counting tuples with NULLs) =
3299 #(unique_tuples counting tuples with NULLs as different) -
3300 #(tuples with NULLs)
3301 */
3302 unique_tuples -= (records - notnull[parts]);
3303 }
3304
3305 if (unique_tuples == 0)
3306 tmp= 1;
3307 else if (count == 0)
3308 tmp= tuples; /* 1 unique tuple */
3309 else
3310 tmp= (tuples + unique_tuples/2) / unique_tuples;
3311
3312 /*
3313 for some weird keys (e.g. FULLTEXT) tmp can be <1 here.
3314 let's ensure it is not
3315 */
3316 if (tmp < 1)
3317 tmp= 1;
3318 if (tmp >= (uint64_t) ~(ulong) 0)
3319 tmp=(uint64_t) ~(ulong) 0;
3320
3321 *rec_per_key_part=(ulong) tmp;
3322 rec_per_key_part++;
3323 }
3324}
3325
3326
3327static ha_checksum mi_byte_checksum(const unsigned char *buf, uint32_t length)
3328{
3329 ha_checksum crc;
3330 const unsigned char *end=buf+length;
3331 for (crc=0; buf != end; buf++)
3332 crc=((crc << 1) + *((unsigned char*) buf)) +
3333 test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
3334 return crc;
3335}
3336
3337static bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
3338{
3339 uint32_t key_maxlength=key->maxlength;
3340 return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY) &&
3341 ((uint64_t) rows * key_maxlength >
3342 (uint64_t) MAX_FILE_SIZE));
3343}
3344
3345/*
3346 Deactivate all not unique index that can be recreated fast
3347 These include packed keys on which sorting will use more temporary
3348 space than the max allowed file length or for which the unpacked keys
3349 will take much more space than packed keys.
3350 Note that 'rows' may be zero for the case when we don't know how many
3351 rows we will put into the file.
3352 */
3353
3354void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
3355{
3356 MYISAM_SHARE *share=info->s;
3357 MI_KEYDEF *key=share->keyinfo;
3358 uint32_t i;
3359
3360 assert(info->state->records == 0 &&
3361 (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
3362 for (i=0 ; i < share->base.keys ; i++,key++)
3363 {
3364 if (!(key->flag & (HA_NOSAME | HA_AUTO_KEY)) &&
3365 ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
3366 {
3367 mi_clear_key_active(share->state.key_map, i);
3368 info->update|= HA_STATE_CHANGED;
3369 }
3370 }
3371}
3372
3373
3374/*
3375 Return true if we can use repair by sorting
3376 One can set the force argument to force to use sorting
3377 even if the temporary file would be quite big!
3378*/
3379
3380bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
3381 uint64_t key_map, bool force)
3382{
3383 MYISAM_SHARE *share=info->s;
3384 MI_KEYDEF *key=share->keyinfo;
3385 uint32_t i;
3386
3387 /*
3388 mi_repair_by_sort only works if we have at least one key. If we don't
3389 have any keys, we should use the normal repair.
3390 */
3391 if (! mi_is_any_key_active(key_map))
3392 return false; /* Can't use sort */
3393 for (i=0 ; i < share->base.keys ; i++,key++)
3394 {
3395 if (!force && mi_too_big_key_for_sort(key,rows))
3396 return false;
3397 }
3398 return true;
3399}
3400
3401
3402static void
3403set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
3404{
3405 if ((sort_info->new_data_file_type=share->data_file_type) ==
3406 COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
3407 {
3408 MYISAM_SHARE tmp;
3409
3410 if (share->options & HA_OPTION_PACK_RECORD)
3411 sort_info->new_data_file_type = DYNAMIC_RECORD;
3412 else
3413 sort_info->new_data_file_type = STATIC_RECORD;
3414
3415 /* Set delete_function for sort_delete_record() */
3416 memcpy(&tmp, share, sizeof(*share));
3417 tmp.options= ~HA_OPTION_COMPRESS_RECORD;
3418 mi_setup_functions(&tmp);
3419 share->delete_record=tmp.delete_record;
3420 }
3421}
bool reinit_io_cache(cache_type type_arg, my_off_t seek_offset, bool use_async_io, bool clear_cache)
Reset the cache.
int end_io_cache()
Free an io_cache_st object.
int init_io_cache(int file, size_t cachesize, cache_type type, my_off_t seek_offset, bool use_async_io, myf cache_myflags)
Initialize an io_cache_st object.
void free_root(myf MyFLAGS)
Deallocate everything used by alloc_root or just move used blocks to free list if called with MY_USED...
Definition root.cc:287
Convert from decimal to binary representation.
Definition common_fwd.h:215
TODO: Rename this file - func.h is stupid.
STL namespace.