src/book.c

   1 /**
   2
   3 Copyright (c) Scott Gasch
   4
   5 Module Name:
   6
   7     book.c
   8
   9 Abstract:
  10
  11     Opening book routines: everything from probing the opening book to
  12     building a new one to editing the current book.  The structure of
  13     a book entry is in book.h.  The book is kept on disk and sorted by
  14     position signature.  There may be more than one entry per
  15     signature -- there will be one entry per book move from that
  16     positions so for example the initial position will have quite a
  17     few entries.
  18
  19     Probing the book (BookMove, called from Think in brain.c) entails
  20     binary searching the on disk book file for a matching signature,
  21     reading all entries that match, computing a "weight" for each one,
  22     and then randomly choosing one.
  23
  24     Building a new book is done with a temporary "membook" buffer in
  25     memory.  It requires quite a bit of memory so before the operation
  26     takes place the main hash table and pawn hash table structures are
  27     freed (see movelist.c).  A PGN file is read one game at a time and
  28     the moves are made on a POSITION struct.  Once the game is over
  29     the signature-move pairs are used to create new entries in the
  30     membook.  Finally the membook is sorted and written to disk.  The
  31     process can take a while, especially for a large PGN file or on a
  32     machine with limited memory resources.
  33
  34 Author:
  35
  36     Scott Gasch ([email protected]) 27 Jun 2004
  37
  38 Revision History:
  39
  40     $Id: book.c 355 2008-07-01 15:46:43Z scott $
  41
  42 **/
  43
  44 #include "chess.h"
  45
  46 static int g_fdBook = -1;                     // file descriptor for open book
  47 static ULONG g_uMemBookCount = 0;             // count of entries in membook
  48 static BOOK_ENTRY *g_pMemBook = NULL;         // allocated membook struct
  49 static BOOK_ENTRY *g_pMemBookHead = NULL;     // ptr to 1st entry in membook
  50
  51 //
  52 // Globals exported
  53 //
  54 FLAG g_fTournamentMode = FALSE;               // are we in tournament mode
  55 ULONG g_uBookProbeFailures = 0;               // how many times have we missed
  56 ULONG g_uMemBookSize = 0x1000000;
  57 OPENING_NAME_MAPPING *g_pOpeningNames = NULL; // opening names file
  58
  59 static FLAG
  60 _VerifyBook(CHAR *szName)
  61 /**
  62
  63 Routine description:
  64
  65 Parameters:
  66
  67     CHAR *szName
  68
  69 Return value:
  70
  71     static FLAG
  72
  73 **/
  74 {
  75     int fd = -1;
  76     int iRet;
  77     ULONG uPos;
  78     ULONG uNum;
  79     ULONG u;
  80     struct stat s;
  81     UINT64 u64LastSig;
  82     ULONG uLastMove;
  83     BOOK_ENTRY be;
  84     FLAG fMyRetVal = FALSE;
  85
  86     fd = open(szName, O_RDONLY | O_BINARY);
  87     if (fd < 0)
  88     {
  89         Trace("VerifyBook: Can't open %s for read/write.\n", szName);
  90         goto end;
  91     }
  92
  93     if (0 != stat(szName, &s))
  94     {
  95         Trace("VerifyBook: Can't stat %s.\n", szName);
  96         goto end;
  97     }
  98
  99     if ((s.st_size % sizeof(BOOK_ENTRY)) != 0)
 100     {
 101         Trace("VerifyBook: Size of book is not a multiple of entry size.\n");
 102         goto end;
 103     }
 104
 105     //
 106     // Foreach book entry...
 107     //
 108     uNum = (s.st_size / sizeof(BOOK_ENTRY));
 109     u64LastSig = 0;
 110     uLastMove = 0;
 111     for (u = 0; u < uNum; u++)
 112     {
 113         //
 114         // Seek to and read the entry.
 115         //
 116         uPos = u * sizeof(BOOK_ENTRY);
 117         if (uPos != lseek(fd, uPos, SEEK_SET))
 118         {
 119             Trace("VerifyBook: Can't seek to entry %u (offset %u) in book.\n",
 120                   u, uPos);
 121             goto end;
 122         }
 123         iRet = read(fd, &be, sizeof(BOOK_ENTRY));
 124         if (sizeof(BOOK_ENTRY) != iRet)
 125         {
 126             Trace("VerifyBook: Can't read entry number %u (offset %u) in the "
 127                   "book. Read returned %u but I expected %u.\n",
 128                   u, uPos, iRet, sizeof(BOOK_ENTRY));
 129             goto end;
 130         }
 131
 132         //
 133         // Make sure the book is sorted by signature
 134         //
 135         if (be.u64Sig < u64LastSig)
 136         {
 137             Trace("VerifyBook: Book signatures out-of-order at index %u "
 138                   "(offset %u).\n", u, uPos);
 139             goto end;
 140         }
 141
 142         //
 143         // If our sig is equal to the last entry's then the move here must
 144         // be greater than or equal to the last entry's move.
 145         //
 146         else if (be.u64Sig == u64LastSig)
 147         {
 148             if (be.mvNext.uMove < uLastMove)
 149             {
 150                 Trace("VerifyBook: Book moves out-of-order at index %u "
 151                       "(offset %u).\n", u, uPos);
 152                 goto end;
 153             }
 154         }
 155
 156         u64LastSig = be.u64Sig;
 157         uLastMove = be.mvNext.uMove;
 158     }
 159     fMyRetVal = TRUE;
 160
 161  end:
 162     if (-1 != fd)
 163     {
 164         close(fd);
 165     }
 166     return(fMyRetVal);
 167 }
 168
 169
 170 static CHAR *
 171 _BookLineToString(UINT64 u64Line)
 172 /**
 173
 174 Routine description:
 175
 176     Converts a position signature into an opening name, if known.
 177
 178 Parameters:
 179
 180     UINT64 u64Line
 181
 182 Return value:
 183
 184     static CHAR
 185
 186 **/
 187 {
 188     ULONG u = 0;
 189
 190     if (NULL == g_pOpeningNames) return(NULL);
 191     while(g_pOpeningNames[u].szString != NULL)
 192     {
 193         if (g_pOpeningNames[u].u64Sig == u64Line)
 194         {
 195             return(g_pOpeningNames[u].szString);
 196         }
 197         u++;
 198     }
 199     return(NULL);
 200 }
 201
 202
 203 static void
 204 _DumpUserOpeningList(void)
 205 /**
 206
 207 Routine description:
 208
 209     Called in response to a user "book dump openings" command, this
 210     routine simply shows every named opening in the opening names
 211     file.  This file is not the opening book, it's an auxillary disk
 212     file containing a list of moves and a name for the opening.  It's
 213     only used to announce the opening line played in a game and
 214     annotate the PGN for a game stored in monsoon's logfile.
 215
 216 Parameters:
 217
 218     void
 219
 220 Return value:
 221
 222     static void
 223
 224 **/
 225 {
 226     int fd = open(OPENING_LEARNING_FILENAME,
 227                   O_RDWR | O_CREAT | O_BINARY | O_RANDOM,
 228                   _S_IREAD | _S_IWRITE);
 229     ULONG uNumRecords;
 230     ULONG u;
 231     OPENING_LEARNING_ENTRY ole;
 232     struct stat s;
 233     CHAR *p;
 234
 235     //
 236     // If we failed to open it, abort.
 237     //
 238     if (fd < 0) goto end;
 239
 240     //
 241     // Stat it to see how many entries there are.
 242     //
 243     if (0 != stat(OPENING_LEARNING_FILENAME, &s))
 244     {
 245         ASSERT(FALSE);
 246         Trace("UpdateUserOpeningList: Corrupt %s file.\n",
 247               OPENING_LEARNING_FILENAME);
 248         goto end;
 249     }
 250     if ((s.st_size % sizeof(OPENING_LEARNING_ENTRY)) != 0)
 251     {
 252         ASSERT(FALSE);
 253         Trace("UpdateUserOpeningList: Corrupt %s file.\n",
 254               OPENING_LEARNING_FILENAME);
 255         goto end;
 256     }
 257     uNumRecords = s.st_size / sizeof(OPENING_LEARNING_ENTRY);
 258     Trace("Learned data on %u openings:\n\n", uNumRecords);
 259
 260     //
 261     // Read each entry looking for a match.
 262     //
 263     for (u = 0; u < uNumRecords; u++)
 264     {
 265         if (sizeof(OPENING_LEARNING_ENTRY) !=
 266             read(fd, &ole, sizeof(OPENING_LEARNING_ENTRY)))
 267         {
 268             Trace("UpdateUserOpeningList: Read error.\n");
 269             goto end;
 270         }
 271
 272         p = _BookLineToString(ole.u64Sig);
 273         if (NULL != p)
 274         {
 275             Trace("%-45s ww=%u, dr=%u, bw=%u\n",
 276                   p, ole.uWhiteWins, ole.uDraws, ole.uBlackWins);
 277         }
 278     }
 279
 280  end:
 281     if (fd >= 0)
 282     {
 283         close(fd);
 284     }
 285 }
 286
 287 FLAG
 288 InitializeOpeningBook(void)
 289 /**
 290
 291 Routine description:
 292
 293 Parameters:
 294
 295     void
 296
 297 Return value:
 298
 299     FLAG
 300
 301 **/
 302 {
 303     return(TRUE);
 304 }
 305
 306 void
 307 ResetOpeningBook(void)
 308 /**
 309
 310 Routine description:
 311
 312 Parameters:
 313
 314     void
 315
 316 Return value:
 317
 318     void
 319
 320 **/
 321 {
 322     g_uBookProbeFailures = 0;
 323 }
 324
 325 void
 326 CleanupOpeningBook(void)
 327 /**
 328
 329 Routine description:
 330
 331 Parameters:
 332
 333     void
 334
 335 Return value:
 336
 337     void
 338
 339 **/
 340 {
 341     ULONG x = 0;
 342
 343     //
 344     // If we have read an "opening names" list and created a dynamic memory
 345     // structure for it, free that now.
 346     //
 347     if (NULL != g_pOpeningNames)
 348     {
 349         while (g_pOpeningNames[x].szString != NULL)
 350         {
 351             SystemFreeMemory(g_pOpeningNames[x].szString);
 352             g_pOpeningNames[x].szString = NULL;
 353             g_pOpeningNames[x].u64Sig = 0;
 354             x++;
 355         }
 356         SystemFreeMemory(g_pOpeningNames);
 357         g_pOpeningNames = NULL;
 358     }
 359
 360     //
 361     // Cleanup membook if they quit in the middle of book building somehow
 362     //
 363     if (NULL != g_pMemBook)
 364     {
 365         ASSERT(FALSE);
 366         SystemFreeMemory(g_pMemBook);
 367     }
 368 }
 369
 370
 371 static FLAG
 372 _BookOpen(void)
 373 /**
 374
 375 Routine description:
 376
 377 Parameters:
 378
 379     void
 380
 381 Return value:
 382
 383     static FLAG
 384
 385 **/
 386 {
 387     //
 388     // Need to have set the book name before calling this.
 389     //
 390     if (!strlen(g_Options.szBookName))
 391     {
 392         return(FALSE);
 393     }
 394
 395     //
 396     // Open it.
 397     //
 398     ASSERT(g_fdBook == -1);
 399     g_fdBook = open(g_Options.szBookName,
 400                     O_RDWR | O_CREAT |
 401                     O_RANDOM | O_BINARY | _S_IREAD | _S_IWRITE);
 402     if (g_fdBook < 0)
 403     {
 404         Bug("_BookOpen: Failed to open book \"%s\".\n", g_Options.szBookName);
 405         return(FALSE);
 406     }
 407
 408     return((FLAG)(g_fdBook >= 0));
 409 }
 410
 411
 412 static void
 413 _BookClose(void)
 414 /**
 415
 416 Routine description:
 417
 418 Parameters:
 419
 420     void
 421
 422 Return value:
 423
 424     static void
 425
 426 **/
 427 {
 428     if (-1 != g_fdBook)
 429     {
 430         (void)close(g_fdBook);
 431     }
 432     g_fdBook = -1;
 433 }
 434
 435
 436 static FLAG
 437 _BookSeek(ULONG n)
 438 /**
 439
 440 Routine description:
 441
 442 Parameters:
 443
 444     ULONG n
 445
 446 Return value:
 447
 448     static FLAG
 449
 450 **/
 451 {
 452     ULONG uPos;
 453
 454     //
 455     // During normal book operations we have an open file handle and we
 456     // simply calculate and set its file pointer.
 457     //
 458     // However when in the process of building the opening book from a
 459     // file full of PGN games all book commands access a large buffer in
 460     // memory... so instead of setting a file pointer to a byte offset in
 461     // the book file on disk, we set a normal pointer to an offset in
 462     // this large book building buffer.
 463     //
 464     if (g_pMemBook == NULL)
 465     {
 466         uPos = n * sizeof(BOOK_ENTRY);
 467         ASSERT(g_fdBook != -1);
 468
 469         if (uPos != lseek(g_fdBook, uPos, SEEK_SET))
 470         {
 471             Trace("_BookSeek: Cannot seek to byte offset %u\n", uPos);
 472             return(FALSE);
 473         }
 474     }
 475     else
 476     {
 477         ASSERT(g_uMemBookSize >= 0);
 478         ASSERT(n < g_uMemBookSize);
 479         g_pMemBookHead = g_pMemBook + n;
 480     }
 481     return(TRUE);
 482 }
 483
 484 #if 0
 485 static FLAG
 486 _BookWrite(BOOK_ENTRY *x)
 487 /**
 488
 489 Routine description:
 490
 491 Parameters:
 492
 493     BOOK_ENTRY *x
 494
 495 Return value:
 496
 497     static FLAG
 498
 499 **/
 500 {
 501     int iNum;
 502
 503     //
 504     // If we are building the book, operate on memory
 505     // buffer... otherwise operate on the book file.
 506     //
 507     if (g_pMemBook != NULL)
 508     {
 509         memcpy(g_pMemBookHead, x, sizeof(BOOK_ENTRY));
 510     }
 511     else
 512     {
 513         ASSERT(g_fdBook != -1);
 514
 515         iNum = write(g_fdBook, x, sizeof(BOOK_ENTRY));
 516         if (sizeof(BOOK_ENTRY) != iNum)
 517         {
 518             Trace("_BookWrite: I/O error on fd=%d, got %d and expected %d\n",
 519                   g_fdBook, iNum, sizeof(BOOK_ENTRY));
 520             ASSERT(FALSE);
 521             return(FALSE);
 522         }
 523     }
 524     return(TRUE);
 525 }
 526 #endif
 527
 528 static FLAG
 529 _BookRead(BOOK_ENTRY *x)
 530 /**
 531
 532 Routine description:
 533
 534 Parameters:
 535
 536     BOOK_ENTRY *x
 537
 538 Return value:
 539
 540     static FLAG
 541
 542 **/
 543 {
 544     int iNum;
 545
 546     if (g_pMemBook != NULL)
 547     {
 548         memcpy(x, g_pMemBookHead, sizeof(BOOK_ENTRY));
 549     }
 550     else
 551     {
 552         ASSERT(g_fdBook != -1);
 553
 554         iNum = read(g_fdBook, x, sizeof(BOOK_ENTRY));
 555         if (sizeof(BOOK_ENTRY) != iNum)
 556         {
 557             Trace("_BookRead: I/O error on fd=%d, got %d and expected %d\n",
 558                   g_fdBook, iNum, sizeof(BOOK_ENTRY));
 559             perror("_BookRead");
 560             ASSERT(FALSE);
 561             return(FALSE);
 562         }
 563     }
 564     return(TRUE);
 565 }
 566
 567
 568 static ULONG
 569 _BookCount(void)
 570 /**
 571
 572 Routine description:
 573
 574 Parameters:
 575
 576     void
 577
 578 Return value:
 579
 580     static ULONG
 581
 582 **/
 583 {
 584     struct stat s;
 585
 586     if (NULL != g_pMemBook)
 587     {
 588         return(g_uMemBookCount);
 589     }
 590     else
 591     {
 592         if (!strlen(g_Options.szBookName))
 593         {
 594             return(0);
 595         }
 596
 597         if (0 != stat(g_Options.szBookName, &s))
 598         {
 599             Trace("_BookCount: Stat failed.\n");
 600             ASSERT(FALSE);
 601             return(0);
 602         }
 603         else
 604         {
 605             if ((s.st_size % sizeof(BOOK_ENTRY)) != 0)
 606             {
 607                 Trace("_BookCount: Size is %u... not a mult of %u!\n",
 608                       s.st_size, sizeof(BOOK_ENTRY));
 609                 return(0);
 610             }
 611             return(s.st_size / sizeof(BOOK_ENTRY));
 612         }
 613     }
 614 }
 615
 616 #if 0
 617 static FLAG
 618 _BookAppend(BOOK_ENTRY *x)
 619 /**
 620
 621 Routine description:
 622
 623 Parameters:
 624
 625     BOOK_ENTRY *x
 626
 627 Return value:
 628
 629     static FLAG
 630
 631 **/
 632 {
 633     if (g_pMemBook != NULL)
 634     {
 635         g_pMemBookHead = g_pMemBook + g_uMemBookCount;
 636         memcpy(g_pMemBookHead, x, sizeof(BOOK_ENTRY));
 637         g_uMemBookCount++;
 638         return(TRUE);
 639     }
 640     else
 641     {
 642         ASSERT(g_fdBook != -1);
 643
 644         if (-1 == lseek(g_fdBook, 0, SEEK_END))
 645         {
 646             ASSERT(FALSE);
 647             return(FALSE);
 648         }
 649         return((FLAG)(sizeof(BOOK_ENTRY) ==
 650                       write(g_fdBook, x, sizeof(BOOK_ENTRY))));
 651     }
 652 }
 653 #endif
 654
 655 static ULONG
 656 _BookFindSig(UINT64 u64Sig, ULONG *puLow, ULONG *puHigh)
 657 /**
 658
 659 Routine description:
 660
 661 Parameters:
 662
 663     UINT64 u64Sig,
 664     ULONG *puLow,
 665     ULONG *puHigh
 666
 667 Return value:
 668
 669     static ULONG
 670
 671 **/
 672 {
 673     ULONG uLow = 0;
 674     ULONG uHigh = 0;
 675     ULONG uCurrent;
 676     ULONG uIndex;
 677     ULONG uTotalEntries = _BookCount();
 678     BOOK_ENTRY entry;
 679     ULONG uRetVal = 0;
 680     FLAG fFound = FALSE;
 681
 682     //
 683     // Open the book.
 684     //
 685     if (0 == uTotalEntries)
 686     {
 687         Trace("BookFindSig: Opening book \"%s\" is empty.\n",
 688               g_Options.szBookName);
 689         goto end;
 690     }
 691     if (g_fdBook < 0)
 692     {
 693         Trace("BookFindSig: No book is opened!\n");
 694         goto end;
 695     }
 696
 697     //
 698     // Perform a a binary search looking for an entry with a signature
 699     // that matches this position's.
 700     //
 701     uLow = 0;
 702     uHigh = uTotalEntries - 1;
 703
 704     while ((uLow <= uHigh) &&
 705            (FALSE == fFound))
 706     {
 707         uCurrent = (uLow + uHigh) / 2;
 708         _BookSeek(uCurrent);
 709         _BookRead(&entry);
 710
 711         if (u64Sig == entry.u64Sig)
 712         {
 713             fFound = TRUE;
 714             break;
 715         }
 716         else if (u64Sig < entry.u64Sig)
 717         {
 718             uHigh = uCurrent - 1;
 719         }
 720         else
 721         {
 722             ASSERT(u64Sig > entry.u64Sig);
 723             uLow = uCurrent + 1;
 724         }
 725     }
 726     if (FALSE == fFound)
 727     {
 728         goto end;                             // not found
 729     }
 730     ASSERT(uCurrent >= 0);
 731     ASSERT(uCurrent < uTotalEntries);
 732     uRetVal = uCurrent;
 733
 734     //
 735     // Ok, we matched a signature at current.  Since there can be many
 736     // entries with the same checksum (many moves from the same board
 737     // position), seek backwards until the first one matching the
 738     // checksum.
 739     //
 740     uIndex = uCurrent - 1;
 741     while (uIndex != (ULONG)-1)
 742     {
 743         _BookSeek(uIndex);
 744         _BookRead(&entry);
 745
 746         if (entry.u64Sig != u64Sig)
 747         {
 748             uIndex++;
 749 #ifdef DEBUG
 750             ASSERT(uIndex <= uCurrent);
 751             _BookSeek(uIndex);
 752             _BookRead(&entry);
 753             ASSERT(entry.u64Sig == u64Sig);
 754 #endif
 755             break;
 756         }
 757         uIndex--;
 758     }
 759     uLow = uIndex;
 760
 761     //
 762     // Walk forward until the last book entry with a matching signature.
 763     //
 764     uIndex = uCurrent + 1;
 765     while (uIndex < uTotalEntries)
 766     {
 767         _BookSeek(uIndex);
 768         _BookRead(&entry);
 769         if (entry.u64Sig != u64Sig)
 770         {
 771             uIndex--;
 772 #ifdef DEBUG
 773             ASSERT(uIndex >= uCurrent);
 774             _BookSeek(uIndex);
 775             _BookRead(&entry);
 776             ASSERT(entry.u64Sig == u64Sig);
 777 #endif
 778             break;
 779         }
 780         uIndex++;
 781     }
 782     uHigh = uIndex;
 783
 784  end:
 785     *puLow = uLow;
 786     *puHigh = uHigh;
 787     return(uRetVal);
 788 }
 789
 790
 791 static ULONG _BookComputeWeight(BOOK_ENTRY *e)
 792 /**
 793
 794 Routine description:
 795
 796
 797     Given a book entry, compute its weight.  The higher the more
 798     likely it will be chosen and played.  Used in BookMove and
 799     BookEdit.  The formula is:
 800
 801         weight(mv) = winningness_coefficient * popularity_factor
 802
 803                                        wins
 804         winningness_coefficient = ---------------
 805                                   (wins + losses)
 806
 807         popularity_factor = wins + losses + draws
 808
 809 Parameters:
 810
 811     BOOK_ENTRY *e
 812
 813 Return value:
 814
 815     static ULONG
 816
 817 **/
 818 {
 819     ULONG uRet;                               // computed return value
 820     double dWinning = 0.0;                    // winningness coefficient
 821     double dPopular;                          // popularity factor
 822
 823     //
 824     // dWinning is 0.0 now.  The only way it gets a value is if we've
 825     // seen this line win at least once.
 826     //
 827     if (e->uWins > 0)
 828     {
 829         dWinning = e->uWins;
 830         dWinning /= (double)(e->uWins + e->uLosses);
 831
 832         //
 833         // dWinning is now the ratio of wins to deterministic games
 834         // (games that were not draws).  If this is less than 20% then
 835         // ignore the line (give the move a zero weight).
 836         //
 837         if (dWinning < 0.20) return(0);
 838     }
 839
 840     //
 841     // If we get here dWinning, the winningness coefficient, has been
 842     // computed and is non-zero.  Compute the rest of the weight term
 843     // which is based on the popularity of the move (how many times it
 844     // has been played).
 845     //
 846     dPopular = (double)(e->uWins + e->uLosses + e->uDraws);
 847     uRet = (ULONG)(dPopular * dWinning);
 848     return(uRet);
 849 }
 850
 851
 852 static void
 853 _QuickSortBook(ULONG uLeft, ULONG uRight)
 854 /**
 855
 856 Routine description:
 857
 858 Parameters:
 859
 860     ULONG uLeft,
 861     ULONG uRight
 862
 863 Return value:
 864
 865     static void
 866
 867 **/
 868 {
 869     UINT64 u64Pivot;
 870     ULONG uPivotSec;
 871     ULONG uL, uR, uMid;
 872     BOOK_ENTRY temp;
 873
 874     ASSERT(uLeft >= 0);
 875     ASSERT(uLeft < g_uMemBookSize);
 876     ASSERT(uRight >= 0);
 877     ASSERT(uRight < g_uMemBookSize);
 878     if (uLeft < uRight)
 879     {
 880         //
 881         // Select a pivot point.
 882         //
 883         u64Pivot = g_pMemBook[uLeft].u64Sig;
 884         uPivotSec = g_pMemBook[uLeft].mvNext.uMove;
 885
 886         //
 887         // Partition the space based on the pivot.  Everything left of
 888         // it is less, everything right is greater.
 889         //
 890         uL = uLeft;
 891         uR = uRight;
 892         while (uL < uR)
 893         {
 894             while (((g_pMemBook[uL].u64Sig < u64Pivot) ||
 895                     ((g_pMemBook[uL].u64Sig == u64Pivot) &&
 896                      (g_pMemBook[uL].mvNext.uMove <= uPivotSec))) &&
 897                    (uL <= uRight))
 898             {
 899                 uL++;
 900             }
 901
 902             while (((g_pMemBook[uR].u64Sig > u64Pivot) ||
 903                     ((g_pMemBook[uR].u64Sig == u64Pivot) &&
 904                      (g_pMemBook[uR].mvNext.uMove > uPivotSec))) &&
 905                    (uR >= uLeft))
 906             {
 907                 uR--;
 908             }
 909
 910             if (uL < uR)
 911             {
 912                 temp = g_pMemBook[uL];
 913                 g_pMemBook[uL] = g_pMemBook[uR];
 914                 g_pMemBook[uR] = temp;
 915             }
 916         }
 917         uMid = uR;
 918
 919         temp = g_pMemBook[uLeft];
 920         g_pMemBook[uLeft] = g_pMemBook[uMid];
 921         g_pMemBook[uMid] = temp;
 922
 923         //
 924         // Recurse on the two halves
 925         //
 926         if (uLeft != uMid)
 927         {
 928             _QuickSortBook(uLeft, uMid - 1);
 929         }
 930         if (uMid != uRight)
 931         {
 932             _QuickSortBook(uMid + 1, uRight);
 933         }
 934     }
 935 }
 936
 937
 938 static void
 939 _CompactMemBook(void)
 940 /**
 941
 942 Routine description:
 943
 944 Parameters:
 945
 946     void
 947
 948 Return value:
 949
 950     static void
 951
 952 **/
 953 {
 954     ULONG uCount = 0;
 955     ULONG i;
 956     ULONG uOccurances;
 957
 958     Trace("Compacting the opening book... one moment.\n");
 959
 960     //
 961     // Lose the blank signatures to reduce the working set.
 962     //
 963     for (i = 0;
 964          i < g_uMemBookSize;
 965          i++)
 966     {
 967         if (0 != g_pMemBook[i].u64Sig)
 968         {
 969             uOccurances = (g_pMemBook[i].uWins +
 970                            g_pMemBook[i].uDraws +
 971                            g_pMemBook[i].uLosses);
 972
 973             if (0 != uOccurances)
 974             {
 975                 memcpy(&(g_pMemBook[uCount]),
 976                        &(g_pMemBook[i]),
 977                        sizeof(BOOK_ENTRY));
 978                 uCount++;
 979             }
 980         }
 981     }
 982
 983     g_uMemBookCount = 0;
 984     if (uCount > 0)
 985     {
 986         g_uMemBookCount = uCount - 1;
 987     }
 988 }
 989
 990
 991 static void
 992 _StrainMemBook(ULONG uLimit)
 993 /**
 994
 995 Routine description:
 996
 997 Parameters:
 998
 999     ULONG uLimit
1000
1001 Return value:
1002
1003     static void
1004
1005 **/
1006 {
1007     ULONG i;
1008     ULONG uOccurances;
1009
1010     Trace("Straining out unpopular positions to compact buffer...\n");
1011
1012     for (i = 0;
1013          i < g_uMemBookSize;
1014          i++)
1015     {
1016         if (0 != g_pMemBook[i].u64Sig)
1017         {
1018             uOccurances = (g_pMemBook[i].uWins +
1019                            g_pMemBook[i].uDraws +
1020                            g_pMemBook[i].uLosses);
1021
1022             if (uOccurances <= uLimit)
1023             {
1024                 memset(&(g_pMemBook[i]),
1025                        0,
1026                        sizeof(BOOK_ENTRY));
1027                 g_uMemBookCount--;
1028             }
1029         }
1030     }
1031 }
1032
1033
1034 //+----------------------------------------------------------------------------
1035 //
1036 // Function:  _BookHashFind
1037 //
1038 // Arguments: BOOK_ENTRY *p - pointer to a book entry
1039 //
1040 // Returns:   BOOK_ENTRY * - pointer to a membook location to store in
1041 //
1042 //+----------------------------------------------------------------------------
1043
1044
1045
1046
1047
1048 BOOK_ENTRY *_BookHashFind(BOOK_ENTRY *p)
1049 /**
1050
1051 Routine description:
1052
1053     This routine returns a pointer to the membook entry that should be
1054     used to store data about a position-move.  It's called while
1055     building a new opening book (BookBuild, see below).  It uses a
1056     simple hash-lookup algorthm with linear probing in the event of a
1057     collision.
1058
1059 Parameters:
1060
1061     BOOK_ENTRY *p
1062
1063 Return value:
1064
1065     BOOK_ENTRY
1066
1067 **/
1068 {
1069     ULONG uKey = (ULONG)((p->u64Sig >> 32) & (g_uMemBookSize - 1));
1070     BOOK_ENTRY *pBe = g_pMemBook + uKey;
1071 #ifdef DEBUG
1072     ULONG uInitialKey = uKey;
1073
1074     ASSERT(g_pMemBook != NULL);
1075     ASSERT(p->u64Sig);
1076 #endif
1077
1078     while(1)
1079     {
1080         //
1081         // If this is an empty book entry, use it.
1082         //
1083         if (pBe->u64Sig == 0) break;
1084
1085         //
1086         // If this book entry is the same position and move as the one
1087         // we are adding, return it.
1088         //
1089         if ((pBe->u64Sig == p->u64Sig) &&
1090             (pBe->mvNext.uMove == p->mvNext.uMove))
1091         {
1092             break;
1093         }
1094
1095         //
1096         // Otherwise check the next book entry... if we go off the end
1097         // then loop around.
1098         //
1099         uKey++;
1100         if (uKey >= g_uMemBookSize) uKey = 0;
1101         pBe = g_pMemBook + uKey;
1102
1103         //
1104         // This should never happen -- it means a totally full book.
1105         // Before we let the membook fill up we should have already
1106         // called the strain routine to drop unpopular position-moves.
1107         //
1108         ASSERT(uKey != uInitialKey);
1109     }
1110     return(pBe);
1111 }
1112
1113
1114
1115 MOVE
1116 BookMove(POSITION *pos, BITV bvFlags)
1117 /**
1118
1119 Routine description:
1120
1121     This is a bit confusing.
1122
1123     The primary purpose of this function is, given a position pointer,
1124     to find and return a book move to play.
1125
1126     This behavior can be achieved by calling with a good pos pointer
1127     and bvFlags containing BOOKMOVE_SELECT_MOVE.
1128
1129     If bvFlags doesn't contain this flag then no move will be
1130     returned.
1131
1132     If bvFlags contains BOOKMOVE_DUMP this function will trace a bunch
1133     of info about move weights.  Hence, something like BookDump can be
1134     achieved via:
1135
1136         (void)BookMove(pos, BOOKMOVE_DUMP);
1137
1138     This is done so that the code to find all moves from a position
1139     and show them is not duplicated more than once.
1140
1141 Parameters:
1142
1143     POSITION *pos,
1144     BITV bvFlags
1145
1146 Return value:
1147
1148     MOVE
1149
1150 */
1151 {
1152     LIGHTWEIGHT_SEARCHER_CONTEXT *ctx = NULL;
1153     UINT64 u64Sig;
1154     ULONG uTotalWeight = 0;
1155     static ULONG uMoveWeights[32];
1156     static CHAR *szNames[32];
1157     ULONG uMoveNum;
1158     static BOOK_ENTRY entry;
1159     ULONG uBestWeight = 0;
1160     ULONG uBestIndex = (ULONG)-1;
1161     ULONG uLow, uHigh;
1162     ULONG uIndex;
1163     MOVE mvBook = {0};
1164     ULONG uRandom;
1165     ULONG uCounter;
1166     ULONG x;
1167     double d;
1168     CHAR *szName = NULL;
1169     FLAG fOldValue = g_Options.fShouldAnnounceOpening;
1170
1171     g_Options.fShouldAnnounceOpening = FALSE;
1172 #ifdef DEBUG
1173     memset(uMoveWeights, 0, sizeof(uMoveWeights));
1174 #endif
1175     ctx = SystemAllocateMemory(sizeof(LIGHTWEIGHT_SEARCHER_CONTEXT));
1176     if (NULL == ctx)
1177     {
1178         Trace("Out of memory.\n");
1179         goto end;
1180     }
1181
1182     //
1183     // We cannot probe the opening book if membook is non-NULL (which
1184     // indicates the opening book is currently in-memory and still
1185     // being built.
1186     //
1187     if (g_pMemBook != NULL)
1188     {
1189         Trace("Cannot probe the book while its still being built!\n");
1190         ASSERT(FALSE);
1191         goto end;
1192     }
1193     if (FALSE == _BookOpen())
1194     {
1195         Trace("BookMove: Could not open the book file.\n");
1196         goto end;
1197     }
1198
1199     //
1200     // Binary search for sig in the book, this sets iLow and iHigh.
1201     //
1202     u64Sig = pos->u64PawnSig ^ pos->u64NonPawnSig;
1203     if (0 == _BookFindSig(u64Sig, &uLow, &uHigh))
1204     {
1205         Trace("BookMove: Signature not in book.\n", u64Sig);
1206         goto end;
1207     }
1208     ASSERT((uHigh - uLow) < 32);
1209     ASSERT(uLow <= uHigh);
1210
1211     //
1212     // Consider each entry from iLow to iHigh and determine the chances
1213     // of playing it.
1214     //
1215     for (uIndex = uLow, uMoveNum = 0;
1216          uIndex <= uHigh;
1217          uIndex++, uMoveNum++)
1218     {
1219         _BookSeek(uIndex);
1220         _BookRead(&entry);
1221 #ifdef DEBUG
1222         u64Sig = pos->u64PawnSig ^ pos->u64NonPawnSig;
1223         ASSERT(entry.u64Sig == u64Sig);
1224 #endif
1225         uMoveWeights[uMoveNum] = 0;
1226         szNames[uMoveNum] = NULL;
1227
1228         //
1229         // Don't consider the move if its flagged deleted or disabled.
1230         //
1231         if ((entry.bvFlags & FLAG_DISABLED) ||
1232             (entry.bvFlags & FLAG_DELETED))
1233         {
1234             continue;
1235         }
1236
1237         //
1238         // Make sure it's legal and doesn't draw... also get the name
1239         // of this opening.
1240         //
1241         InitializeLightweightSearcherContext(pos, ctx);
1242         if (FALSE == MakeMove((SEARCHER_THREAD_CONTEXT *)ctx, entry.mvNext))
1243         {
1244             ASSERT(FALSE);
1245             continue;
1246         }
1247         else
1248         {
1249             if (TRUE == IsDraw((SEARCHER_THREAD_CONTEXT *)ctx))
1250             {
1251                 UnmakeMove((SEARCHER_THREAD_CONTEXT *)ctx, entry.mvNext);
1252                 continue;
1253             }
1254
1255             u64Sig = (ctx->sPosition.u64PawnSig ^
1256                       ctx->sPosition.u64NonPawnSig);
1257             szNames[uMoveNum] = _BookLineToString(u64Sig);
1258             UnmakeMove((SEARCHER_THREAD_CONTEXT *)ctx, entry.mvNext);
1259         }
1260
1261         //
1262         // If one is marked always, just return it.
1263         //
1264         if ((entry.bvFlags & FLAG_ALWAYSPLAY) &&
1265             (bvFlags & BOOKMOVE_SELECT_MOVE))
1266         {
1267             mvBook = entry.mvNext;
1268             szName = szNames[uMoveNum];
1269             goto end;
1270         }
1271
1272         //
1273         // Set this move's weight.
1274         //
1275         uMoveWeights[uMoveNum] = _BookComputeWeight(&entry);
1276         uTotalWeight += uMoveWeights[uMoveNum];
1277
1278         //
1279         // If we are in tournament mode we will play the top weighted
1280         // move; look for it now.
1281         //
1282         if (uMoveWeights[uMoveNum] > uBestWeight)
1283         {
1284             uBestWeight = uMoveWeights[uMoveNum];
1285             uBestIndex = uIndex;
1286         }
1287
1288     } // next book move
1289
1290     //
1291     // If we threw every move out (there is no total weight) then we
1292     // don't have a book move here.
1293     //
1294     if (0 == uTotalWeight)
1295     {
1296         ASSERT(mvBook.uMove == 0);
1297         goto end;
1298     }
1299
1300     //
1301     // We are done assigning weights to moves.  If we are in tournament
1302     // mode we should have picked a best one.  If so, return it.
1303     //
1304     if ((TRUE == g_fTournamentMode) && (bvFlags & BOOKMOVE_SELECT_MOVE))
1305     {
1306         if (uBestIndex != (ULONG)-1)
1307         {
1308             ASSERT(uBestWeight);
1309             _BookSeek(uBestIndex);
1310             _BookRead(&entry);
1311             mvBook = entry.mvNext;
1312             ASSERT(uBestIndex >= uLow);
1313             szName = szNames[uBestIndex - uLow];
1314             goto end;
1315         }
1316     }
1317
1318     //
1319     // We didn't throw everything out... so pick a move randomly based
1320     // on the weights assigned.
1321     //
1322     uRandom = (randomMT() % uTotalWeight) + 1;
1323     if (bvFlags & BOOKMOVE_DUMP)
1324     {
1325         u64Sig = pos->u64PawnSig ^ pos->u64NonPawnSig;
1326         Trace(" BookMove: List of moves in position 0x%"
1327               COMPILER_LONGLONG_HEX_FORMAT ":\n",
1328               u64Sig);
1329         for (x = 0; x < uMoveNum; x++)
1330         {
1331             _BookSeek(uLow + x);
1332             _BookRead(&entry);
1333             ASSERT(entry.u64Sig == u64Sig);
1334
1335             d = 0.0;
1336             if (uMoveWeights[x] != 0)
1337             {
1338                 d = (double)uMoveWeights[x] / (double)uTotalWeight;
1339                 d *= 100.0;
1340             }
1341             Trace("%02u. %-4s [+%u =%u -%u] \"%s\" @ %5.3f\n",
1342                   x,
1343                   MoveToSan(entry.mvNext, pos),
1344                   entry.uWins, entry.uDraws, entry.uLosses,
1345                   szNames[x],
1346                   d);
1347         }
1348         Trace(" Total weight was %u.\n\n", uTotalWeight);
1349         if (!(bvFlags & BOOKMOVE_SELECT_MOVE))
1350         {
1351             mvBook.uMove = 0;
1352             goto end;
1353         }
1354     }
1355
1356     //
1357     // Based on the weighted list and the random number we
1358     // selected pick a move.
1359     //
1360     for (uIndex = 0, uCounter = 0;
1361          uIndex < uMoveNum;
1362          uIndex++)
1363     {
1364         uCounter += uMoveWeights[uIndex];
1365
1366         if (uCounter >= uRandom)
1367         {
1368             _BookSeek(uLow + uIndex);
1369             _BookRead(&entry);
1370             mvBook = entry.mvNext;
1371             szName = szNames[uIndex];
1372             goto end;
1373         }
1374     }
1375 #ifdef DEBUG
1376     UtilPanic(SHOULD_NOT_GET_HERE,
1377               NULL, NULL, NULL, NULL,
1378               __FILE__, __LINE__);
1379 #endif
1380
1381  end:
1382     g_Options.fShouldAnnounceOpening = fOldValue;
1383     _BookClose();
1384
1385     if (mvBook.uMove)
1386     {
1387         ASSERT(bvFlags & BOOKMOVE_SELECT_MOVE);
1388
1389         //
1390         // We could be probing the book after a predicted ponder move (in
1391         // which case we will have forced fAnnounceOpening to FALSE because
1392         // we don't want to whisper two book moves in a row).
1393         //
1394         if (FALSE != g_Options.fShouldAnnounceOpening) {
1395             Trace("tellothers book move %s [+%u =%u -%u]\n",
1396                   MoveToSan(entry.mvNext, pos),
1397                   entry.uWins, entry.uDraws, entry.uLosses);
1398         }
1399     }
1400
1401     if (NULL != ctx)
1402     {
1403         SystemFreeMemory(ctx);
1404     }
1405     return(mvBook);
1406 }
1407
1408
1409 #if 0
1410 //+---------------------------------------------------------------------------
1411 //
1412 // Function:  BookEdit
1413 //
1414 // Synopsis:  Giant hack to edit the book.
1415 //
1416 // Arguments: POSITION *pos
1417 //
1418 // Returns:   void
1419 //
1420 //+---------------------------------------------------------------------------
1421 void
1422 BookEdit(POSITION *pos)
1423 {
1424     int iTotalEntries = _BookCount();
1425     int iNumMoves;
1426     int iLow, iHigh;
1427     FILE *pfLearn = NULL;
1428     FLAG fFound = FALSE;
1429     static BOOK_ENTRY entry;
1430     int iIndex = 0;
1431     int iCurrent = 0;
1432     int x;
1433     char *pCh, *pArg;
1434     int arg;
1435     int iSum;
1436     int iWeights[64];
1437     float fl;
1438     FLAG fOldValue = g_Options.fAnnounceOpening;
1439
1440     g_Options.fAnnounceOpening = FALSE;
1441
1442     //
1443     // We cannot probe the opening book if membook is non-NULL (which
1444     // indicates the opening book is currently in-memory and still being
1445     // built.
1446     //
1447     if (g_pMemBook != NULL)
1448     {
1449         ASSERT(FALSE);
1450         goto end;
1451     }
1452
1453     //
1454     // Open the book.
1455     //
1456     iTotalEntries = _BookCount();
1457     if (0 == iTotalEntries)
1458     {
1459         Trace("BookDump: Opening book \"%s\" is empty.\n",
1460               g_Options.szBookName);
1461         goto end;
1462     }
1463
1464     ASSERT(g_fdBook == -1);
1465     if (!_BookOpen())
1466     {
1467         Trace("BookDump: Could not open the book.\n");
1468         goto end;
1469     }
1470
1471     //
1472     // Open the learning recorder
1473     //
1474     pfLearn = fopen(BOOK_EDITING_RECORD, "a+b");
1475     if (NULL == pfLearn)
1476     {
1477         Trace("BookEdit: Warning: can't open %s to save learning.\n",
1478               BOOK_EDITING_RECORD);
1479     }
1480
1481     if (!_BookFindSig(pos->u64Sig, &iLow, &iHigh))
1482     {
1483         Trace("BookEdit: Position %I64x is not in the book.\n");
1484         goto end;
1485     }
1486
1487     iNumMoves = iHigh - iLow + 1;
1488     iCurrent = 0;
1489     do
1490     {
1491         printf("\e[H\e[J\n");
1492         Trace("--EDITING OPENING BOOK--\n\n");
1493         Trace("There are %u positions in the opening book \"%s\".\n",
1494               iTotalEntries, g_Options.szBookName);
1495         Trace("There are %u book moves in this position, %u to %u:\n",
1496               iNumMoves, iLow, iHigh);
1497
1498         for (x = 0, iSum = 0;
1499              x < iNumMoves;
1500              x++)
1501         {
1502             _BookSeek(iLow + x);
1503             _BookRead(&entry);
1504
1505             //
1506             // If the disabled flag is asserted, do not allow this
1507             // move to be played.  If the disabled flag is not
1508             // asserted, calculate a weight for this move.
1509             //
1510             if ((entry.bvFlags & FLAG_DISABLED) ||
1511                 (entry.bvFlags & FLAG_DELETED))
1512             {
1513                 iWeights[x] = 0;
1514             }
1515             else
1516             {
1517                 iWeights[x] = _BookComputeWeight(&entry);
1518             }
1519             iSum += iWeights[x];
1520         }
1521
1522         for (x = 0;
1523              x < iNumMoves;
1524              x++)
1525         {
1526             _BookSeek(iLow + x);
1527             _BookRead(&entry);
1528
1529             fl = (float)iWeights[x];
1530             if (iSum)
1531             {
1532                 fl /= (float)iSum;
1533                 fl *= 100.0;
1534             }
1535
1536             if (x == iCurrent)
1537             {
1538                 Trace(">> ");
1539             }
1540             else
1541             {
1542                 Trace("   ");
1543             }
1544
1545             Trace("%02u. %6u..%6u = %4s [+%6u =%6u -%6u] @ ~%6.3f%% | %s%s",
1546                   x + 1,
1547                   0,
1548                   iWeights[x],
1549                   MoveToSAN(entry.mvNext, pos),
1550                   entry.iWins,
1551                   entry.iDraws,
1552                   entry.iLosses,
1553                   fl,
1554                   (entry.bvFlags & FLAG_DISABLED) ? "NEV" : "",
1555                   (entry.bvFlags & FLAG_ALWAYSPLAY) ? "ALW" : "");
1556
1557             if (x == iCurrent)
1558             {
1559                 Trace("<<\n");
1560             }
1561             else
1562             {
1563                 Trace("\n");
1564             }
1565         }
1566
1567         _BookSeek(iLow + iCurrent);
1568         _BookRead(&entry);
1569         Trace("Command (q, p, n, +, =, -, h, l, e, a, d, ?): ");
1570         pCh = BlockingRead();
1571
1572         switch(tolower(*pCh))
1573         {
1574         case 'q':
1575             Trace("--DONE EDITING OPENING BOOK--\n\n");
1576             goto end;
1577         case 'p':
1578             iCurrent--;
1579             if (iCurrent < 0) iCurrent = iNumMoves - 1;
1580             break;
1581         case 'n':
1582             iCurrent++;
1583             if (iCurrent >= iNumMoves) iCurrent = 0;
1584             break;
1585         case '+':
1586             Trace("Enter new wincount: ");
1587             pArg = BlockingRead();
1588             arg = atoi(pArg);
1589             if (arg)
1590             {
1591                 entry.iWins = arg;
1592                 _BookSeek(iLow + iCurrent);
1593                 _BookWrite(&entry);
1594                 if (NULL != pfLearn)
1595                 {
1596                     fprintf(pfLearn, "%I64x (%x) + %u // %s\n",
1597                             entry.u64Sig,
1598                             entry.mvNext.move,
1599                             arg,
1600                             BoardToFEN(pos));
1601                 }
1602             }
1603             SystemFreeMemory(pArg);
1604             break;
1605         case '=':
1606             Trace("Enter new drawcount: ");
1607             pArg = BlockingRead();
1608             arg = atoi(pArg);
1609             if (arg)
1610             {
1611                 entry.iDraws = arg;
1612                 _BookSeek(iLow + iCurrent);
1613                 _BookWrite(&entry);
1614                 if (NULL != pfLearn)
1615                 {
1616                     fprintf(pfLearn, "%I64x (%x) = %u // %s\n",
1617                             entry.u64Sig,
1618                             entry.mvNext.move,
1619                             arg,
1620                             BoardToFEN(pos));
1621                 }
1622             }
1623             SystemFreeMemory(pArg);
1624             break;
1625         case '-':
1626             Trace("Enter new losscount: ");
1627             pArg = BlockingRead();
1628             arg = atoi(pArg);
1629             if (arg)
1630             {
1631                 entry.iLosses = arg;
1632                 _BookSeek(iLow + iCurrent);
1633                 _BookWrite(&entry);
1634                 if (NULL != pfLearn)
1635                 {
1636                     fprintf(pfLearn, "%I64x (%x) - %u // %s\n",
1637                             entry.u64Sig,
1638                             entry.mvNext.move,
1639                             arg,
1640                             BoardToFEN(pos));
1641                 }
1642             }
1643             SystemFreeMemory(pArg);
1644             break;
1645         case 'e':
1646             if (entry.bvFlags & FLAG_DISABLED)
1647             {
1648                 entry.bvFlags &= ~FLAG_DISABLED;
1649             }
1650             else
1651             {
1652                 entry.bvFlags |= FLAG_DISABLED;
1653                 entry.bvFlags &= ~FLAG_ALWAYSPLAY;
1654             }
1655             _BookSeek(iLow + iCurrent);
1656             _BookWrite(&entry);
1657             if (NULL != pfLearn)
1658             {
1659                 fprintf(pfLearn, "%I64x (%x) FLAGS %x // %s\n",
1660                         entry.u64Sig,
1661                         entry.mvNext.move,
1662                         entry.bvFlags,
1663                         BoardToFEN(pos));
1664             }
1665             break;
1666         case 'a':
1667             if (entry.bvFlags & FLAG_ALWAYSPLAY)
1668             {
1669                 entry.bvFlags &= ~FLAG_ALWAYSPLAY;
1670             }
1671             else
1672             {
1673                 entry.bvFlags |= FLAG_ALWAYSPLAY;
1674                 entry.bvFlags &= ~FLAG_DISABLED;
1675             }
1676             _BookSeek(iLow + iCurrent);
1677             _BookWrite(&entry);
1678             if (NULL != pfLearn)
1679             {
1680                 fprintf(pfLearn, "%I64x (%x) FLAGS %x // %s\n",
1681                         entry.u64Sig,
1682                         entry.mvNext.move,
1683                         entry.bvFlags,
1684                         BoardToFEN(pos));
1685             }
1686             break;
1687         case 'h':
1688             PreferPosition(pos, pfLearn, pos->u64Sig, -1);
1689             Trace("I hate this position.\n");
1690             break;
1691         case 'i':
1692             PreferPosition(pos, pfLearn, pos->u64Sig, 0);
1693             Trace("I'm indifferent towards this position.\n");
1694             break;
1695         case 'l':
1696             PreferPosition(pos, pfLearn, pos->u64Sig, 1);
1697             Trace("I love this position!\n");
1698             break;
1699         case '?':
1700             Trace(
1701 "Commands: \n"
1702 "           +) adjust wins     =) adjust draws   -) adjust losses\n"
1703 "           h) hate this pos   i) indifferent    l) love this pos\n\n"
1704
1705 "           e) enable/disable\n"
1706 "           a) always/sometimes play\n\n"
1707
1708 "           p) previous move   n) next move      q) quit editing\n\n");
1709             break;
1710         default:
1711             break;
1712         }
1713         SystemFreeMemory(pCh);
1714     }
1715     while(1);
1716
1717  end:
1718     g_Options.fAnnounceOpening = fOldValue;
1719
1720     ASSERT(g_fdBook != -1);
1721     _BookClose();
1722
1723     if (NULL != pfLearn)
1724     {
1725         fclose(pfLearn);
1726     }
1727 }
1728 #endif
1729
1730
1731
1732
1733
1734 static FLAG
1735 _BookBuild(void)
1736 /**
1737
1738 Routine description:
1739
1740     Builds a new opening book.  This routine is called by
1741     LearnPGNOpenings in movelist.c after it has played through one PGN
1742     game.  g_MoveList is loaded with position signatures and move
1743     structs representing this game.  Our goal is to parse these and
1744     create/update the membook entries to reflect the game played and
1745     its result.
1746
1747 Parameters:
1748
1749     void
1750
1751 Return value:
1752
1753     static FLAG
1754
1755 **/
1756 {
1757     BOOK_ENTRY entry;
1758     BOOK_ENTRY *pEntry;
1759     ULONG n;
1760     GAME_MOVE *q;
1761     ULONG u;
1762     double dFull;
1763     GAME_RESULT result = GetGameResult();
1764
1765     if (result.eResult != RESULT_WHITE_WON &&
1766         result.eResult != RESULT_BLACK_WON &&
1767         result.eResult != RESULT_DRAW) {
1768         return(FALSE);
1769     }
1770
1771     //
1772     // If the membook has not yet been allocated, do it now.
1773     //
1774     if (g_pMemBook == NULL)
1775     {
1776         ASSERT((g_uMemBookSize & (g_uMemBookSize - 1)) == 0);
1777         g_uMemBookCount = 0;
1778         g_pMemBookHead = NULL;
1779         g_pMemBook = (BOOK_ENTRY *)
1780             SystemAllocateMemory(g_uMemBookSize * sizeof(BOOK_ENTRY));
1781         g_pMemBookHead = g_pMemBook;
1782         ASSERT(g_pMemBook != NULL);
1783     }
1784
1785     //
1786     // Consider each position in the game we have played up until now.
1787     //
1788     u = 0;
1789     q = GetNthOfficialMoveRecord(u);
1790     while((q != NULL) && (q->mv.uMove != 0))
1791     {
1792         entry.u64Sig = q->u64PositionSigBeforeMove;
1793         entry.mvNext = q->mv;
1794         entry.u64NextSig = q->u64PositionSigAfterMove;
1795
1796         //
1797         // Have we ever seen this position before?
1798         //
1799         pEntry = _BookHashFind(&entry);
1800         if (0 != pEntry->u64Sig)
1801         {
1802             //
1803             // Yes we have seen this signature-move before.  Update the
1804             // win/draw/loss counter at this entry in the membook.
1805             //
1806             ASSERT(pEntry->u64Sig == entry.u64Sig);
1807             ASSERT(pEntry->mvNext.uMove == entry.mvNext.uMove);
1808             if (((result.eResult == RESULT_BLACK_WON) &&
1809                  (GET_COLOR(entry.mvNext.pMoved) == BLACK)) ||
1810                 ((result.eResult == RESULT_WHITE_WON) &&
1811                  (GET_COLOR(entry.mvNext.pMoved) == WHITE)))
1812             {
1813                 pEntry->uWins++;
1814             }
1815             else
1816             {
1817                 if (RESULT_DRAW == result.eResult)
1818                 {
1819                     pEntry->uDraws++;
1820                 }
1821                 else
1822                 {
1823                     pEntry->uLosses++;
1824                 }
1825             }
1826         }
1827
1828         //
1829         // Nope never seen this signature-move entry before, create a new
1830         // entry for it and add it to the membook.
1831         //
1832         else
1833         {
1834             memset(pEntry, 0, sizeof(*pEntry));
1835             pEntry->u64Sig = entry.u64Sig;
1836             pEntry->mvNext = entry.mvNext;
1837             pEntry->u64NextSig = entry.u64NextSig;
1838
1839             if (((result.eResult == RESULT_BLACK_WON) &&
1840                  (GET_COLOR(entry.mvNext.pMoved) == BLACK)) ||
1841                 ((result.eResult == RESULT_WHITE_WON) &&
1842                  (GET_COLOR(entry.mvNext.pMoved) == WHITE)) )
1843             {
1844                 pEntry->uWins = 1;
1845             }
1846             else
1847             {
1848                 if (RESULT_DRAW == result.eResult)
1849                 {
1850                     pEntry->uDraws = 1;
1851                 }
1852                 else
1853                 {
1854                     pEntry->uLosses = 1;
1855                 }
1856             }
1857             g_uMemBookCount++;
1858
1859             //
1860             // If the in memory buffer is getting too full, throw out
1861             // all positions that occur only once in it in order to
1862             // compact it.  We should try to limit this because it can
1863             // weaken the book.
1864             //
1865             n = 1;
1866             dFull = (double)g_uMemBookCount;
1867             dFull /= (double)g_uMemBookSize;
1868             dFull *= 100.0;
1869             while (dFull > 97.0)
1870             {
1871                 _StrainMemBook(n);
1872                 n++;
1873
1874                 dFull = (double)g_uMemBookCount;
1875                 dFull /= (double)g_uMemBookSize;
1876                 dFull *= 100.0;
1877             }
1878         }
1879         u++;
1880         q = GetNthOfficialMoveRecord(u);
1881     }
1882     return(TRUE);
1883 }
1884
1885
1886 static FLAG
1887 _BookToDisk(ULONG uStrainLimit)
1888 /**
1889
1890 Routine description:
1891
1892 Parameters:
1893
1894     ULONG uStrainLimit
1895
1896 Return value:
1897
1898     static FLAG
1899
1900 **/
1901 {
1902     ULONG i;
1903     int ifd;
1904     CHAR *szTempBook = "tempbook.bin";
1905     CHAR szCmd[SMALL_STRING_LEN_CHAR];
1906     BOOK_ENTRY sBe;
1907     ULONG uOldBook = 0;
1908     ULONG uNewBook = 0;
1909     ULONG uOld_BookCount;
1910     ULONG uNew_BookCount;
1911     BOOK_ENTRY *pOldBook = NULL;
1912     BOOK_ENTRY *pNewBook = NULL;
1913     FLAG fMyRetVal = FALSE;
1914     struct stat s;
1915
1916     if (!strlen(g_Options.szBookName))
1917     {
1918         return(FALSE);
1919     }
1920     i = g_uMemBookCount;
1921
1922     //
1923     // Drops all book entries with less than limit occurrances
1924     //
1925     _StrainMemBook(uStrainLimit);
1926
1927     //
1928     // Drops all unused entries
1929     //
1930     Trace("Stage 2: Compacting the book.\n");
1931     _CompactMemBook();
1932
1933     //
1934     // Sorts the by signature.
1935     //
1936     Trace("Stage 3: Sorting the book.\n");
1937     _QuickSortBook(0, g_uMemBookCount);
1938     uNew_BookCount = g_uMemBookCount;
1939
1940     //
1941     // Create the temporary book on the disk.
1942     //
1943     (void)unlink(szTempBook);
1944     if (SystemDoesFileExist(szTempBook))
1945     {
1946         Trace("_BookToDisk: Can't create a temporary book!\n");
1947         return(FALSE);
1948     }
1949
1950     ifd = open(szTempBook,
1951                O_RDWR | O_CREAT | O_BINARY,
1952                _S_IREAD | _S_IWRITE);
1953     if (ifd < 0)
1954     {
1955         Trace("_BookToDisk: Can't create a temporary book!\n");
1956         return(FALSE);
1957     }
1958 #ifndef _MSC_VER
1959     fchmod(ifd, 0644);
1960 #endif
1961
1962     //
1963     // Count the entries in the old book on disk.
1964     //
1965     uOld_BookCount = 0;
1966     if ((TRUE == _VerifyBook(g_Options.szBookName)) && (TRUE == _BookOpen()))
1967     {
1968         if (0 == stat(g_Options.szBookName, &s))
1969         {
1970             uOld_BookCount = (s.st_size / sizeof(BOOK_ENTRY));
1971         }
1972     }
1973
1974     //
1975     // Merge the old book (on disk) with the new (in memory) in the
1976     // temp book.
1977     //
1978     Trace("Merging book lines and writing to disk...\n");
1979     do
1980     {
1981         pOldBook = pNewBook = NULL;
1982
1983         //
1984         // Read the next old book entry.
1985         //
1986  retry:
1987         if (uOldBook < uOld_BookCount)
1988         {
1989             if ((int)(uOldBook * sizeof(BOOK_ENTRY)) !=
1990                 lseek(g_fdBook, (uOldBook * sizeof(BOOK_ENTRY)), SEEK_SET))
1991             {
1992                 Trace("_BookToDisk: Cannot seek to byte offset %u in old "
1993                       "book.\n", uOldBook * sizeof(BOOK_ENTRY));
1994                 ASSERT(FALSE);
1995                 goto end;
1996             }
1997
1998             if (sizeof(BOOK_ENTRY) != read(g_fdBook,
1999                                            &sBe,
2000                                            sizeof(BOOK_ENTRY)))
2001             {
2002                 Trace("_BookToDisk: Can't read error in old book, trying "
2003                       "to recover...\n");
2004                 uOldBook++;
2005                 goto retry;
2006             }
2007             pOldBook = &sBe;
2008         }
2009
2010         //
2011         // Get the next new book entry
2012         //
2013         if (uNewBook < uNew_BookCount)
2014         {
2015             ASSERT(0 != g_pMemBook[uNewBook].u64Sig);
2016             pNewBook = &(g_pMemBook[uNewBook]);
2017         }
2018
2019         //
2020         // Decide which to write into the merged (temporary) book we
2021         // are constructing next...
2022         //
2023         if ((NULL == pNewBook) && (NULL == pOldBook))
2024         {
2025             break;
2026         }
2027         else if ((NULL == pNewBook) && (NULL != pOldBook))
2028         {
2029             goto writeoldbook;
2030         }
2031         else if ((NULL != pNewBook) && (NULL == pOldBook))
2032         {
2033             goto writenewbook;
2034         }
2035         else
2036         {
2037             if (pOldBook->u64Sig < pNewBook->u64Sig)
2038             {
2039                 goto writeoldbook;
2040             }
2041             else if (pOldBook->u64Sig > pNewBook->u64Sig)
2042             {
2043                 goto writenewbook;
2044             }
2045             else
2046             {
2047                 ASSERT(pOldBook->u64Sig == pNewBook->u64Sig);
2048                 if (pOldBook->mvNext.uMove < pNewBook->mvNext.uMove)
2049                 {
2050                     goto writeoldbook;
2051                 }
2052                 else if (pOldBook->mvNext.uMove > pNewBook->mvNext.uMove)
2053                 {
2054                     goto writenewbook;
2055                 }
2056                 else
2057                 {
2058                     //
2059                     // Same position and move...
2060                     //
2061                     pNewBook->uWins += pOldBook->uWins;
2062                     pNewBook->uDraws += pOldBook->uDraws;
2063                     pNewBook->uLosses += pOldBook->uLosses;
2064                     pNewBook->bvFlags |= pOldBook->bvFlags;
2065                     uOldBook++;
2066                     goto writenewbook;
2067                 }
2068             }
2069         }
2070 #ifdef DEBUG
2071         UtilPanic(SHOULD_NOT_GET_HERE,
2072                   NULL, NULL, NULL, NULL,
2073                   __FILE__, __LINE__);
2074 #endif
2075
2076  writeoldbook:
2077         if (-1 == lseek(ifd, 0, SEEK_END))
2078         {
2079             ASSERT(FALSE);
2080         }
2081         if (sizeof(BOOK_ENTRY) != write(ifd, pOldBook, sizeof(BOOK_ENTRY)))
2082         {
2083             Trace("_BookToDisk: Error writing new book, aborting procedure.\n");
2084             goto end;
2085         }
2086         uOldBook++;
2087         continue;
2088
2089  writenewbook:
2090         if (-1 == lseek(ifd, 0, SEEK_END))
2091         {
2092             ASSERT(FALSE);
2093         }
2094         if (sizeof(BOOK_ENTRY) != write(ifd, pNewBook, sizeof(BOOK_ENTRY)))
2095         {
2096             Trace("_BookToDisk: Error writing new book, aborting procedure."
2097                   "\n");
2098             goto end;
2099         }
2100         uNewBook++;
2101         continue;
2102     }
2103     while(1);
2104     fMyRetVal = TRUE;
2105
2106  end:
2107     close(ifd);
2108     SystemFreeMemory(g_pMemBook);
2109     g_pMemBook = NULL;
2110     g_pMemBookHead = NULL;
2111     g_uMemBookCount = 0;
2112     _BookClose();
2113
2114     //
2115     // Verify the consistency of the new book before klobbering the old one
2116     //
2117     if (FALSE == _VerifyBook(szTempBook))
2118     {
2119         Trace("_BookToDisk: The book I just wrote is corrupt!!!\n");
2120         fMyRetVal = FALSE;
2121         ASSERT(FALSE);
2122     }
2123     else
2124     {
2125         //
2126         // HACKHACK: put this in system, use CopyFile in win32
2127         //
2128 #ifdef _MSC_VER
2129         _snprintf(szCmd, 255, "move %s %s%c", szTempBook,
2130                   g_Options.szBookName, 0);
2131 #else
2132         snprintf(szCmd, 255, "/bin/mv %s %s%c", szTempBook,
2133                  g_Options.szBookName, 0);
2134 #endif
2135         system(szCmd);
2136     }
2137
2138     if (TRUE == fMyRetVal)
2139     {
2140         Trace("Book successfully built.\n");
2141     }
2142     return(fMyRetVal);
2143 }
2144
2145
2146 static FLAG
2147 _LearnPgnOpenings(CHAR *szFilename, ULONG uStrainLimit)
2148 /**
2149
2150 Routine description:
2151
2152 Parameters:
2153
2154     CHAR *szFilename,
2155     ULONG uStrainLimit
2156
2157 Return value:
2158
2159     static FLAG
2160
2161 **/
2162 {
2163     CHAR *p;
2164     FILE *pf = NULL;
2165     struct stat s;
2166     double d;
2167
2168     if (FALSE == SystemDoesFileExist(szFilename))
2169     {
2170         Trace("LearnPgnOpenings: file \"%s\" doesn't exist.\n",
2171               szFilename);
2172         return(FALSE);
2173     }
2174
2175     if (0 != stat(szFilename, &s))
2176     {
2177         Trace("LearnPgnOpenings: Can't read file \"%s\".\n",
2178               szFilename);
2179         return(FALSE);
2180     }
2181
2182     pf = fopen(szFilename, "rb");
2183     if (NULL == pf)
2184     {
2185         Trace("LearnPgnOpenings: error reading file %s.\n",
2186               szFilename);
2187         return(FALSE);
2188     }
2189
2190     CleanupHashSystem();
2191
2192     Trace("Stage 1: reading and parsing PGN\n");
2193     while((p = ReadNextGameFromPgnFile(pf)) != NULL)
2194     {
2195         if (TRUE == LoadPgn(p))
2196         {
2197             _BookBuild();
2198             d = (double)ftell(pf);
2199             d /= (double)s.st_size;
2200             d *= 100.0;
2201             printf("%u/%u (%5.2f%% done) ",
2202                    (unsigned)ftell(pf), (unsigned)s.st_size, d);
2203             d = (double)g_uMemBookCount;
2204             d /= (double)g_uMemBookSize;
2205             d *= 100.0;
2206             printf("[membook %5.2f%% full]\r", d);
2207         }
2208         SystemFreeMemory(p);
2209     }
2210     fclose(pf);
2211
2212     _BookToDisk(uStrainLimit);
2213
2214     InitializeHashSystem();
2215
2216     return(TRUE);
2217 }
2218
2219
2220 #if 0
2221 static void
2222 _UpdateUserOpeningList(INT iResult)
2223 {
2224     int fd = open(OPENING_LEARNING_FILENAME,
2225                   O_RDWR | O_CREAT | O_BINARY | O_RANDOM,
2226                   _S_IREAD | _S_IWRITE);
2227     ULONG uNumRecords;
2228     ULONG u;
2229     OPENING_LEARNING_ENTRY ole;
2230     ULONG uPos;
2231     struct stat s;
2232
2233     //
2234     // If we failed to open it, abort.
2235     //
2236     if (fd < 0) goto end;
2237
2238     //
2239     // Stat it to see how many entries there are.
2240     //
2241     if (0 != stat(OPENING_LEARNING_FILENAME, &s))
2242     {
2243         ASSERT(FALSE);
2244         Trace("UpdateUserOpeningList: Corrupt %s file.\n",
2245               OPENING_LEARNING_FILENAME);
2246         goto end;
2247     }
2248     if ((s.st_size % sizeof(OPENING_LEARNING_ENTRY)) != 0)
2249     {
2250         ASSERT(FALSE);
2251         Trace("UpdateUserOpeningList: Corrupt %s file.\n",
2252               OPENING_LEARNING_FILENAME);
2253         goto end;
2254     }
2255     uNumRecords = s.st_size / sizeof(OPENING_LEARNING_ENTRY);
2256
2257     //
2258     // Read each entry looking for a match.
2259     //
2260     for (u = 0; u < uNumRecords; u++)
2261     {
2262         if (sizeof(OPENING_LEARNING_ENTRY) !=
2263             read(fd, &ole, sizeof(OPENING_LEARNING_ENTRY)))
2264         {
2265             Trace("UpdateUserOpeningList: Read error.\n");
2266             goto end;
2267         }
2268
2269         if (ole.u64Sig == g_Options.u64OpeningSig)
2270         {
2271             //
2272             // Found it, so update and write back in place.
2273             //
2274             switch (iResult)
2275             {
2276                 case -1:
2277                     ole.uBlackWins++;
2278                     break;
2279                 case 0:
2280                     ole.uDraws++;
2281                     break;
2282                 case 1:
2283                     ole.uWhiteWins++;
2284                     break;
2285 #ifdef DEBUG
2286                 default:
2287                     ASSERT(FALSE);
2288                     break;
2289 #endif
2290             }
2291
2292             uPos = u * sizeof(OPENING_LEARNING_ENTRY);
2293             if (uPos != lseek(fd, uPos, SEEK_SET))
2294             {
2295                 Trace("UpdateUserOpeningList: Seek error.\n");
2296                 goto end;
2297             }
2298             if (sizeof(OPENING_LEARNING_ENTRY) !=
2299                 write(fd, &ole, sizeof(OPENING_LEARNING_ENTRY)))
2300             {
2301                 Trace("UpdateUserOpeningList: Write error.\n");
2302             }
2303             goto end;
2304         }
2305     }
2306
2307     //
2308     // We never found it, so add one at the end.
2309     //
2310     memset(&ole, 0, sizeof(ole));
2311     ole.u64Sig = g_Options.u64OpeningSig;
2312     switch(iResult)
2313     {
2314         case -1:
2315             ole.uBlackWins++;
2316             break;
2317         case 0:
2318             ole.uDraws++;
2319             break;
2320         case 1:
2321             ole.uWhiteWins++;
2322             break;
2323 #ifdef DEBUG
2324         default:
2325             ASSERT(FALSE);
2326             break;
2327 #endif
2328     }
2329
2330     if (-1 == lseek(fd, 0, SEEK_END))
2331     {
2332         Trace("UpdateUserOpeningList: Seek error.\n");
2333         goto end;
2334     }
2335
2336     if (sizeof(OPENING_LEARNING_ENTRY) !=
2337         write(fd, &ole, sizeof(OPENING_LEARNING_ENTRY)))
2338     {
2339         Trace("UpdateUserOpeningList: Append error.\n");
2340     }
2341
2342  end:
2343     if (fd >= 0)
2344     {
2345         close(fd);
2346     }
2347 }
2348 #endif
2349
2350
2351 #if 0
2352 //+----------------------------------------------------------------------------
2353 //
2354 // Function:  BookLearn
2355 //
2356 // Synopsis:  Handles book learning at the end of a game based on the
2357 //            result of the game and the calibre of the opponent.
2358 //
2359 // Arguments: int iResult
2360 //
2361 // Returns:   void
2362 //
2363 //+----------------------------------------------------------------------------
2364 void
2365 BookLearn(INT iResult)
2366 {
2367     ULONG iIndex, x;
2368     unsigned int iChanged = 0;
2369     int iTotalEntries = _BookCount();
2370     int iLow, iHigh, iCurrent = 0;
2371     BOOK_ENTRY entry;
2372     FLAG fFound = FALSE;
2373
2374 #ifdef DEBUG
2375     if (g_Options.ePlayMode == I_PLAY_BLACK)
2376     {
2377         Trace("BookLearn: I was playing black\n");
2378     }
2379     else if (g_Options.ePlayMode == I_PLAY_WHITE)
2380     {
2381         Trace("BookLearn: I was playing white\n");
2382     }
2383
2384     if (iResult == -1)
2385     {
2386         Trace("BookLearn: Black won.\n");
2387     }
2388     else if (iResult == 1)
2389     {
2390         Trace("BookLearn: White won.\n");
2391     }
2392     else
2393     {
2394         Trace("BookLearn: The game was a draw.\n");
2395     }
2396 #endif
2397
2398     //
2399     // Should we learn anything?
2400     //
2401
2402     //
2403     // Not if the game was bullet
2404     //
2405     if ((g_Options.eGameType == GAME_BULLET) ||
2406         (g_Options.eGameType == GAME_UNKNOWN))
2407     {
2408         Trace("BookLearn: Game was too fast or unknown speed, don't learn.\n");
2409         goto end;
2410     }
2411
2412     //
2413     // Not if the game wasn't rated.
2414     //
2415     if ((g_Options.fGameIsRated == FALSE))
2416     {
2417         Trace("BookLearn: Game was unrated, don't learn.\n");
2418         goto end;
2419     }
2420
2421     //
2422     // Not if someone lost on time or because of forfeit.
2423     //
2424     if ((NULL != strstr(g_Options.szDescription, "forfeit")) ||
2425         (NULL != strstr(g_Options.szDescription, "time")) ||
2426         (NULL != strstr(g_Options.szDescription, "abort")))
2427     {
2428         Trace("BookLearn: Loss by forfeit/abort, don't learn.\n");
2429         goto end;
2430     }
2431
2432     //
2433     // Not if we won but the opponent was weak.
2434     //
2435     if (((g_Options.ePlayMode == I_PLAY_WHITE) && (iResult == 1)) ||
2436         ((g_Options.ePlayMode == I_PLAY_BLACK) && (iResult == -1)))
2437     {
2438 #ifdef DEBUG
2439         Trace("BookLearn: I won, my opponent's rating was %u, "
2440               "my rating is %u.\n",
2441               g_Options.iOpponentsRating,
2442               g_Options.iMyRating);
2443 #endif
2444
2445         if (g_Options.iOpponentsRating < g_Options.iMyRating - 300)
2446         {
2447             Trace("BookLearn: Opponent sucks, not learning.\n");
2448             goto end;
2449         }
2450     }
2451
2452     //
2453     // Not if we lost but the opponent was strong.
2454     //
2455     if (((g_Options.ePlayMode == I_PLAY_BLACK) && (iResult == 1)) ||
2456         ((g_Options.ePlayMode == I_PLAY_WHITE) && (iResult == -1)))
2457     {
2458         Trace("BookLearn: I lost, my opponent's rating was %u, "
2459               "my rating is %u.\n",
2460               g_Options.iOpponentsRating,
2461               g_Options.iMyRating);
2462
2463         if (g_Options.iOpponentsRating > g_Options.iMyRating + 300)
2464         {
2465             Trace("BookLearn: Opponent kicks ass, not learning.\n");
2466             goto end;
2467         }
2468         ToXboard("tellics say Nice game!\n");
2469     }
2470
2471     //
2472     // Yes, we have decided to learn.
2473     //
2474     Trace("BookLearn: Revising opening book line...\n");
2475
2476     //
2477     // 1. Save the learning on a user-friendly list...
2478     //
2479     UpdateUserOpeningList(iResult);
2480
2481     //
2482     // 2. Save the learning in the real opening book...
2483     //
2484     if ((0 == iTotalEntries) ||
2485         (FALSE == _BookOpen()))
2486     {
2487         Trace("BookLearn: Could not open the book.\n");
2488         goto end;
2489     }
2490
2491     //
2492     // Look at the moves in the move list.
2493     //
2494     for (iIndex = 0;
2495          iIndex < g_iMoveNum - 1;
2496          iIndex++)
2497     {
2498         //
2499         // Only learn on the first 30 moves.
2500         //
2501         if (iIndex > 60) break;
2502
2503         Trace("BookLearn: Seeking %I64x with move %s (%x) in the book.\n",
2504             g_MoveList[iIndex].u64Sig,
2505             MoveToICS(g_MoveList[iIndex + 1].mv),
2506             g_MoveList[iIndex + 1].mv.move);
2507
2508         //
2509         // Binary search for sig in the book, this sets iLow and iHigh.
2510         //
2511         if (0 != _BookFindSig(g_MoveList[iIndex].u64Sig, &iLow, &iHigh))
2512         {
2513             Trace("BookLearn: Found a (P) : %I64x at %u.\n",
2514                 g_MoveList[iIndex].u64Sig, iCurrent);
2515
2516             //
2517             // Current now points to the first book entry matching
2518             // the sig... look for the entry matching the move too.
2519             //
2520             for (x = iLow; x <= iHigh; x++)
2521             {
2522                 _BookSeek(x);
2523                 _BookRead(&entry);
2524
2525                 ASSERT(entry.u64Sig ==
2526                        g_MoveList[iIndex].u64Sig);
2527
2528                 //
2529                 // If we match the move and signature, update the entry
2530                 // based on this game's results.
2531                 //
2532                 if (entry.mvNext.move == g_MoveList[iIndex + 1].mv.move)
2533                 {
2534                     Trace("BookLearn: Matched mv : %x at %u.\n",
2535                         g_MoveList[iIndex + 1].mv.move, x);
2536                     iChanged++;
2537
2538                     //
2539                     // Did the side making the move win in the end?
2540                     //
2541                     if (((iResult == -1) &&
2542                          (IS_BLACK(g_MoveList[iIndex + 1].mv.data.pMoved))) ||
2543                         ((iResult == 1) &&
2544                          (IS_WHITE(g_MoveList[iIndex + 1].mv.data.pMoved))))
2545                     {
2546                         Trace("BookLearn: Side moving won, enforce this "
2547                               "book entry.\n");
2548                         entry.iWins++;
2549                     }
2550                     else
2551                     {
2552                         //
2553                         // No so it drew or lost.
2554                         //
2555                         if (0 == iResult)
2556                         {
2557                             Trace("BookLearn: Game was a draw, increase "
2558                                   "drawishness of this book entry.\n");
2559                             entry.iDraws++;
2560                         }
2561                         else
2562                         {
2563                             Trace("BookLearn: Side moving lost, discourage "
2564                                   "this book entry.\n");
2565                             entry.iLosses++;
2566                         }
2567                     }
2568
2569                     //
2570                     // Write it back into the book.
2571                     //
2572                     _BookSeek(x);
2573                     _BookWrite(&entry);
2574
2575                     //
2576                     // Done looking for this move.
2577                     //
2578                     break;
2579
2580                 } // entry.move matches
2581
2582             } // loop to look at all moves on entries w/ matching sig
2583
2584         } // was sig found in book?
2585
2586     } // main game move loop
2587
2588  end:
2589     Trace("BookLearn: Changed %u book positions.\n", iChanged);
2590     _BookClose();
2591 }
2592
2593
2594 //+----------------------------------------------------------------------------
2595 //
2596 // Function:  PreferPosition
2597 //
2598 // Synopsis:  Given a position, love, hate or be neutral towards it.
2599 //
2600 //            If the position is to be loved, all other book entries
2601 //            that lead to it will be flagged "always play".
2602 //
2603 //            If the position is to be hated, all other book entries
2604 //            that lead to it will be flagged "never play".
2605 //
2606 //            A neutral setting will cause all other book entries that
2607 //            lead to have their flags cleared.
2608 //
2609 // Arguments: POSITION pos  - the pos, needed to make a FEN
2610 //            FILE *fd      - file to record permanently in
2611 //            INT64 i64Targ - the signature to love/hate/neuter
2612 //            int i         : -1 means hate
2613 //                          : 0 means neuter
2614 //                          : 1 means love
2615 //
2616 // Returns:   void
2617 //
2618 //+----------------------------------------------------------------------------
2619 void
2620 PreferPosition(POSITION *pos, FILE *fd, INT64 i64Targ, int i)
2621 {
2622     int iTotalEntries = _BookCount();
2623     int x;
2624     int iChanged = 0;
2625     static BOOK_ENTRY entry;
2626
2627     ASSERT(pos->u64Sig == i64Targ);
2628
2629     if (g_pMemBook != NULL)
2630     {
2631         ASSERT(FALSE);
2632         return;
2633     }
2634
2635     iTotalEntries = _BookCount();
2636
2637     for (x = 0;
2638          x < iTotalEntries - 1;
2639          x++)
2640     {
2641         if ((x % 10000) == 0)
2642         {
2643             printf("%u / %u\r", x, iTotalEntries);
2644         }
2645         _BookSeek(x);
2646         _BookRead(&entry);
2647
2648         if (entry.i64NextSig == i64Targ)
2649         {
2650             iChanged++;
2651
2652             switch(i)
2653             {
2654             case -1:
2655                 _BookSeek(x);
2656                 entry.bvFlags = FLAG_DISABLED;
2657                 _BookWrite(&entry);
2658                 if (NULL != fd)
2659                 {
2660                     fprintf(fd, "%I64x (%x) FLAGS %x // %s\n",
2661                             entry.u64Sig,
2662                             entry.mvNext.move,
2663                             entry.bvFlags,
2664                             BoardToFEN(pos));
2665                 }
2666                 break;
2667             case 0:
2668                 _BookSeek(x);
2669                 entry.bvFlags = 0;
2670                 _BookWrite(&entry);
2671                 if (NULL != fd)
2672                 {
2673                     fprintf(fd, "%I64x (%x) FLAGS %x // %s\n",
2674                             entry.u64Sig,
2675                             entry.mvNext.move,
2676                             entry.bvFlags,
2677                             BoardToFEN(pos));
2678                 }
2679                 break;
2680             case 1:
2681                 _BookSeek(x);
2682                 entry.bvFlags = FLAG_ALWAYSPLAY;
2683                 _BookWrite(&entry);
2684                 if (NULL != fd)
2685                 {
2686                     fprintf(fd, "%I64x (%x) FLAGS %x // %s\n",
2687                             entry.u64Sig,
2688                             entry.mvNext.move,
2689                             entry.bvFlags,
2690                             BoardToFEN(pos));
2691                 }
2692                 break;
2693             }
2694         }
2695     }
2696
2697     printf("\n\n");
2698     Trace("PreferPosition: Changed %u lines\n", iChanged);
2699 }
2700
2701 //+----------------------------------------------------------------------------
2702 //
2703 // Function:  ImportEditing
2704 //
2705 // Synopsis:  Affect the opening book file based on an old editing record
2706 //            (.EDT file).  Apply editing / learning record to the new book
2707 //            positions.
2708 //
2709 // Arguments: char *szFile
2710 //
2711 // Returns:   FLAG
2712 //
2713 //+----------------------------------------------------------------------------
2714 FLAG
2715 ImportEditing(char *szFile)
2716 {
2717     FILE *p = fopen(szFile, "r");
2718     FLAG fMyRetVal = FALSE;
2719     static char szLine[512];
2720     char szOpcode[512];
2721     int iArg;
2722     int iTotalEntries;
2723     INT64 u64Sig;
2724     MOVE mv;
2725     int iLow, iHigh, iCurrent = 0;
2726     int iIndex;
2727     BOOK_ENTRY entry;
2728     BOOK_ENTRY klobber;
2729
2730     //
2731     // We cannot probe the opening book if membook is non-NULL (which
2732     // indicates the opening book is currently in-memory and still being
2733     // built.
2734     //
2735     if (g_pMemBook != NULL)
2736     {
2737         goto end;
2738     }
2739
2740     //
2741     // Open the book.
2742     //
2743     iTotalEntries = _BookCount();
2744     if (0 == iTotalEntries)
2745     {
2746         Trace("BookMove: Opening book \"%s\" is empty.\n",
2747               g_Options.szBookName);
2748         goto end;
2749     }
2750
2751     if (FALSE == _BookOpen())
2752     {
2753         Trace("BookMove: Could not open the book.\n");
2754         goto end;
2755     }
2756
2757     if (NULL != p)
2758     {
2759         while(fgets(szLine, 511, p))
2760         {
2761             if (_snscanf(szLine, 511, "%I64x (%x) %s",
2762                          &u64Sig,
2763                          &mv.move,
2764                          &szOpcode) != 3)
2765             {
2766                 continue;
2767             }
2768
2769 #ifdef DEBUG
2770             Trace("Searching for %I64x move %x\n",
2771                   u64Sig, mv.move);
2772 #endif
2773
2774             if (_BookFindSig(u64Sig, &iLow, &iHigh))
2775             {
2776                 //
2777                 // Walk forward looking for the right move.
2778                 //
2779                 memset(&entry, 0, sizeof(entry));
2780                 for (iIndex = iLow;
2781                      iIndex <= iHigh;
2782                      iIndex++)
2783                 {
2784                     _BookSeek(iIndex);
2785                     _BookRead(&entry);
2786                     ASSERT(entry.u64Sig == u64Sig);
2787
2788                     if (entry.mvNext.move == mv.move) break;
2789                 }
2790
2791                 //
2792                 // Did we find the move?
2793                 //
2794                 if (entry.mvNext.move != mv.move)
2795                 {
2796                     Trace("Move %x not present in book at position %I64x\n",
2797                           mv.move, u64Sig);
2798                     continue;
2799                 }
2800
2801                 //
2802                 // Yes.  What should we do with it?
2803                 //
2804                 if (!STRNCMP(szOpcode, "FLAGS", 5))
2805                 {
2806                     if (_snscanf(szLine, 511, "%I64x (%x) %s %x",
2807                                  &u64Sig,
2808                                  &mv.move,
2809                                  &szOpcode,
2810                                  &iArg) == 4)
2811                     {
2812                         if (iArg >= 0) entry.bvFlags = iArg;
2813                     }
2814                     else
2815                     {
2816                         Trace("Error\n");
2817                     }
2818                 }
2819                 else if (!STRNCMP(szOpcode, "+", 1))
2820                 {
2821                     if (_snscanf(szLine, 511, "%I64x (%x) %s %u",
2822                                  &u64Sig,
2823                                  &mv.move,
2824                                  &szOpcode,
2825                                  &iArg) == 4)
2826                     {
2827                         if (iArg >= 0) entry.iWins = iArg;
2828                     }
2829                     else
2830                     {
2831                         Trace("Error\n");
2832                     }
2833                 }
2834                 else if (!STRNCMP(szOpcode, "=", 1))
2835                 {
2836                     if (_snscanf(szLine, 511, "%I64x (%x) %s %u",
2837                                  &u64Sig,
2838                                  &mv.move,
2839                                  &szOpcode,
2840                                  &iArg) == 4)
2841                     {
2842                         if (iArg >= 0) entry.iDraws = iArg;
2843                     }
2844                     else
2845                     {
2846                         Trace("Error\n");
2847                     }
2848                 }
2849                 else if (!STRNCMP(szOpcode, "-", 1))
2850                 {
2851                     if (_snscanf(szLine, 511, "%I64x (%x) %s %u",
2852                                  &u64Sig,
2853                                  &mv.move,
2854                                  &szOpcode,
2855                                  &iArg) == 4)
2856                     {
2857                         if (iArg >= 0) entry.iLosses = iArg;
2858                     }
2859                     else
2860                     {
2861                         Trace("Error\n");
2862                     }
2863                 }
2864                 else
2865                 {
2866                     Trace("Error\n");
2867                 }
2868
2869                 //
2870                 // Write it back to the book
2871                 //
2872                 _BookSeek(iIndex);
2873                 _BookRead(&klobber);
2874
2875                 if ((entry.u64Sig ==
2876                      klobber.u64Sig) &&
2877                     (entry.mvNext.move ==
2878                      klobber.mvNext.move))
2879                 {
2880                     _BookWrite(&entry);
2881                 }
2882             }
2883         }
2884     }
2885
2886     fMyRetVal = TRUE;
2887  end:
2888     _BookClose();
2889     return(fMyRetVal);
2890 }
2891 #endif
2892
2893
2894 static FLAG
2895 _FindTerminalPositions(CHAR *szFilename,
2896                        ULONG uPlyBeforeMate)
2897 /**
2898
2899 Routine description:
2900
2901 Parameters:
2902
2903     CHAR *szFilename,
2904     ULONG uMovesBeforeMate
2905
2906 Return value:
2907
2908     static FLAG
2909
2910 **/
2911 {
2912     CHAR *p;
2913     FILE *pf = NULL;
2914     struct stat s;
2915     POSITION *pos;
2916     SEARCHER_THREAD_CONTEXT *ctx;
2917     FLAG fInCheck;
2918     MOVE mv;
2919     ULONG u;
2920     FLAG fRet = FALSE;
2921
2922     ctx = SystemAllocateMemory(sizeof(SEARCHER_THREAD_CONTEXT));
2923     if (NULL == ctx)
2924     {
2925         Trace("Out of memory.\n");
2926         goto end;
2927     }
2928
2929     if (FALSE == SystemDoesFileExist(szFilename))
2930     {
2931         Trace("FindTerminalPositions: file \"%s\" doesn't exist.\n",
2932               szFilename);
2933         goto end;
2934     }
2935
2936     if (0 != stat(szFilename, &s))
2937     {
2938         Trace("FindTerminalPositions: Can't read file \"%s\".\n",
2939               szFilename);
2940         goto end;
2941     }
2942
2943     pf = fopen(szFilename, "rb");
2944     if (NULL == pf)
2945     {
2946         Trace("FindTerminalPositions: error reading file %s.\n",
2947               szFilename);
2948         goto end;
2949     }
2950
2951     mv.uMove = 0;
2952     while((p = ReadNextGameFromPgnFile(pf)) != NULL)
2953     {
2954         if (TRUE == LoadPgn(p))
2955         {
2956             pos = GetRootPosition();
2957
2958             fInCheck = InCheck(pos, pos->uToMove);
2959             if (TRUE == fInCheck)
2960             {
2961                 InitializeSearcherContext(pos, ctx);
2962                 GenerateMoves(ctx, mv, GENERATE_ESCAPES);
2963                 if (0 == MOVE_COUNT(ctx, 0))
2964                 {
2965                     u = uPlyBeforeMate;
2966                     while(u)
2967                     {
2968                         if (FALSE == OfficiallyTakebackMove())
2969                         {
2970                             break;
2971                         }
2972                         u--;
2973                     }
2974                     pos = GetRootPosition();
2975                     InitializeSearcherContext(pos, ctx);
2976                     DumpPosition(pos);
2977                     (void)Eval(ctx, -INFINITY, +INFINITY);
2978                     Trace("King safeties: W=%d, B=%d\n",
2979                           ctx->sPlyInfo[ctx->uPly].iKingScore[WHITE],
2980                           ctx->sPlyInfo[ctx->uPly].iKingScore[BLACK]);
2981                 }
2982             }
2983         }
2984         SystemFreeMemory(p);
2985     }
2986     fclose(pf);
2987     fRet = TRUE;
2988
2989  end:
2990     if (NULL != ctx)
2991     {
2992         SystemFreeMemory(ctx);
2993     }
2994     return(fRet);
2995 }
2996
2997
2998
2999 COMMAND(BookCommand)
3000 {
3001     CHAR *szOpcode;
3002     CHAR *pDot;
3003
3004     ASSERT(!STRNCMPI(argv[0], "BOOK", 4));
3005     if (argc < 2)
3006     {
3007         Trace("Available opening book commands:\n\n"
3008               "    book name          : set opening book file\n"
3009               "    book import        : import PGN or EDT data to book file\n"
3010               "    book dump learning : show book learning\n"
3011               "    book dump moves    : show book moves in current position\n"
3012               "    book edit          : edit book moves in current position\n"
3013               "    book tourn         : view / toggle tournament mode\n"
3014               "    book openings      : read opening line names PGN file\n"
3015               "    book hackhack      : read PGN files looking for mate\n"
3016               "\n");
3017         return;
3018     }
3019
3020     szOpcode = argv[1];
3021     if (!STRNCMPI(szOpcode, "NAME", 4))
3022     {
3023         //
3024         // book name <filename> : set opening book to use
3025         //
3026         if (argc == 2)
3027         {
3028             Trace("Usage: book name <required-filename>\n");
3029             return;
3030         }
3031         strncpy(g_Options.szBookName, argv[2], SMALL_STRING_LEN_CHAR);
3032         Trace("Opening book name set to \"%s\"\n", argv[2]);
3033     }
3034     else if (!STRNCMPI(argv[1], "DUMP", 4))
3035     {
3036         //
3037         // book dump : show book dump help
3038         //
3039         if (argc == 2)
3040         {
3041             Trace("Available book dump commands:\n"
3042                   "    book dump learning : show book learning\n"
3043                   "    book dump moves    : show book moves in current"
3044                   "position\n"
3045                   "\n");
3046             return;
3047         }
3048         ASSERT(argc > 2);
3049
3050         if (!STRNCMPI(argv[2], "LEARNING", 5))
3051         {
3052             //
3053             // book dump learning : show book learning on stdout
3054             //
3055             _DumpUserOpeningList();
3056         }
3057         else
3058         {
3059             //
3060             // book dump moves : show book moves in this position
3061             //
3062             BookMove(pos, BOOKMOVE_DUMP);
3063         }
3064     }
3065     else if (!STRNCMPI(argv[1], "TOURNAMENTMODE", 5))
3066     {
3067         if (argc == 2)
3068         {
3069             //
3070             // book tourn : show current tournament mode setting
3071             //
3072             if (g_fTournamentMode == TRUE)
3073             {
3074                 Trace("Tournament mode is currently TRUE.\n");
3075             }
3076             else
3077             {
3078                 Trace("Tournament mode is currently FALSE.\n");
3079             }
3080         }
3081         else
3082         {
3083             //
3084             // book tourn <t|f> : set tournament mode on or off
3085             //
3086             if (tolower(*argv[2]) == 't')
3087             {
3088                 g_fTournamentMode = TRUE;
3089                 Trace("Tournament mode set to TRUE.\n");
3090             }
3091             else if (tolower(*argv[2]) == 'f')
3092             {
3093                 g_fTournamentMode = FALSE;
3094                 Trace("Tournament mode set to FALSE.\n");
3095             }
3096             else
3097             {
3098                 Trace("Usage: book tournamentmode [<true | false>]\n");
3099             }
3100         }
3101     }
3102
3103 #if 0
3104     else if (!STRNCMPI(argv[1], "EDIT", 4))
3105     {
3106         //
3107         // book edit : edit book moves in this position
3108         //
3109         BookEdit(pos);
3110         return(TRUE);
3111     }
3112 #endif
3113
3114     else if (!STRNCMPI(argv[1], "HACKHACK", 8))
3115     {
3116         if (argc == 2)
3117         {
3118             Trace("Usage: book hackhack <required-filename>\n");
3119             return;
3120         }
3121         _FindTerminalPositions(argv[2], 4);
3122     }
3123
3124     else if (!STRNCMPI(argv[1], "OPENINGS", 8))
3125     {
3126         //
3127         // book openings : read book opening names file
3128         //
3129         if (argc == 2)
3130         {
3131             Trace("Usage: book openings <required-filename>\n");
3132             return;
3133         }
3134
3135         if (FALSE == SystemDoesFileExist(argv[2]))
3136         {
3137             Trace("Error (file doesn't exist): %s\n", argv[2]);
3138             return;
3139         }
3140
3141 //        Trace("%u opening lines named.\n", LearnOpeningNames(argv[2]));
3142     }
3143
3144     else if (!STRNCMPI(argv[1], "IMPORT", 6))
3145     {
3146         //
3147         // book import : create new book file from PGN -or-
3148         //               apply editing / learning to a book file
3149         //
3150         if (argc == 2)
3151         {
3152             Trace("Usage: book import <required-filename>\n");
3153             return;
3154         }
3155
3156         if (FALSE == SystemDoesFileExist(argv[2]))
3157         {
3158             Trace("Error (file doesn't exist): %s\n", argv[2]);
3159             return;
3160         }
3161
3162         pDot = strrchr(argv[2], '.');
3163         if (NULL == pDot)
3164         {
3165             Trace("Error (unknown filetype): %s\n", argv[2]);
3166             return;
3167         }
3168
3169         if (!STRNCMPI(pDot, ".pgn", 4))
3170         {
3171             if (TRUE == SystemDoesFileExist(g_Options.szBookName))
3172             {
3173                 Trace("The opening book %s already exists, "
3174                       "importing new lines\n", g_Options.szBookName);
3175                 _LearnPgnOpenings(argv[2], 0);
3176             }
3177             else
3178             {
3179                 Trace("The opening book %s does not exists, "
3180                       "creating new book\n", g_Options.szBookName);
3181                 _LearnPgnOpenings(argv[2], 1);
3182             }
3183         }
3184 #if 0
3185         else if (!STRNCMPI(pDot, ".edt", 4))
3186         {
3187             if (FALSE == ImportEditing(argv[2]))
3188             {
3189                 Trace("An error occurred while importing .edt file.\n");
3190             }
3191         }
3192 #endif
3193         else
3194         {
3195             Trace("Error (unknown filetype): %s\n", argv[2]);
3196         }
3197     }
3198 }