Add transform minimize_binpack for minimizing existing .binpack datasets. (#4447)

Takes advantage of the sample skipping rules that are used during training (capture, check, or VALUE_NONE).
Adds positions to keep continuity, which improves compression.

.github/ISSUE_TEMPLATE/BUG-REPORT.yml

@@ -1,65 +0,0 @@
-name: Report issue
-description: Create a report to help us fix issues with the engine
-body:
-- type: textarea
-  attributes:
-    label: Describe the issue
-    description: A clear and concise description of what you're experiencing.
-  validations:
-    required: true
-
-- type: textarea
-  attributes:
-    label: Expected behavior
-    description: A clear and concise description of what you expected to happen.
-  validations:
-    required: true
-
-- type: textarea
-  attributes:
-    label: Steps to reproduce
-    description: |
-      Steps to reproduce the behavior.
-      You can also use this section to paste the command line output.
-    placeholder: |
-      ```
-      position startpos moves g2g4 e7e5 f2f3
-      go mate 1
-      info string NNUE evaluation using nn-6877cd24400e.nnue enabled
-      info depth 1 seldepth 1 multipv 1 score mate 1 nodes 33 nps 11000 tbhits 0 time 3 pv d8h4
-      bestmove d8h4
-      ```
-  validations:
-    required: true
-
-- type: textarea
-  attributes:
-    label: Anything else?
-    description: |
-      Anything that will give us more context about the issue you are encountering.
-      You can also use this section to propose ideas on how to solve the issue. 
-  validations:
-    required: false
-
-- type: dropdown
-  attributes:
-    label: Operating system
-    options:
-      - All
-      - Windows
-      - Linux
-      - MacOS
-      - Android
-      - Other or N/A
-  validations:
-    required: true
-
-- type: input
-  attributes:
-    label: Stockfish version
-    description: |
-      This can be found by running the engine.
-      You can also use the commit ID.
-    placeholder: Stockfish 15 / e6e324e
-  validations:
-    required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Discord server
-    url: https://discord.gg/GWDRS3kU6R
-    about: Feel free to ask for support or have a chat with us in our Discord server!
-  - name: Discussions, Q&A, ideas, show us something...
-    url: https://github.com/official-stockfish/Stockfish/discussions/new
-    about: Do you have an idea for Stockfish? Do you want to show something that you made? Please open a discussion about it!

.github/workflows/stockfish.yml

@@ -1,51 +1,260 @@
 name: Stockfish
 on:
   push:
-    tags:        
-      - '*' 
     branches:
       - master
       - tools
       - github_ci
+      - github_ci_armv7
   pull_request:
     branches:
       - master
       - tools
 jobs:
-  Prerelease:
-    if: github.ref == 'refs/heads/master'
-    runs-on: ubuntu-latest
+  Stockfish:
+    name: ${{ matrix.config.name }}
+    runs-on: ${{ matrix.config.os }}
+    env:
+      COMPILER: ${{ matrix.config.compiler }}
+      COMP: ${{ matrix.config.comp }}
+    strategy:
+      matrix:
+        config:
+          # set the variable for the required tests:
+          # run_expensive_tests: true
+          # run_32bit_tests: true
+          # run_64bit_tests: true
+          - {
+              name: "Ubuntu 20.04 GCC",
+              os: ubuntu-20.04,
+              compiler: g++,
+              comp: gcc,
+              run_expensive_tests: true,
+              run_64bit_tests: true,
+              shell: 'bash {0}'
+            }
+          - {
+              name: "Ubuntu 20.04 Clang",
+              os: ubuntu-20.04,
+              compiler: clang++,
+              comp: clang,
+              run_64bit_tests: true,
+              shell: 'bash {0}'
+            }
+          - {
+              name: "MacOS 10.15 Apple Clang",
+              os: macos-10.15,
+              compiler: clang++,
+              comp: clang,
+              run_64bit_tests: true,
+              shell: 'bash {0}'
+            }
+          - {
+              name: "MacOS 10.15 GCC 10",
+              os: macos-10.15,
+              compiler: g++-10,
+              comp: gcc,
+              run_64bit_tests: true,
+              shell: 'bash {0}'
+            }
+          - {
+              name: "Windows 2022 Mingw-w64 GCC x86_64",
+              os: windows-2022,
+              compiler: g++,
+              comp: mingw,
+              run_64bit_tests: true,
+              msys_sys: 'mingw64',
+              msys_env: 'x86_64-gcc',
+              shell: 'msys2 {0}'
+            }
+
+    defaults:
+      run:
+        working-directory: src
+        shell: ${{ matrix.config.shell }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
-      # returns null if no pre-release exists
-      - name: Get Commit SHA of Latest Pre-release
+      - name: Download required linux packages
+        if: runner.os == 'Linux'
         run: |
-          # Install required packages
-          sudo apt-get update
-          sudo apt-get install -y curl jq
+          sudo apt update
+          sudo apt install expect valgrind g++-multilib
 
-          echo "COMMIT_SHA=$(jq -r 'map(select(.prerelease)) | first | .tag_name' <<< $(curl -s https://api.github.com/repos/${{ github.repository_owner }}/Stockfish/releases))" >> $GITHUB_ENV
-
-      # delete old previous pre-release and tag
-      - uses: dev-drprasad/delete-tag-and-release@v0.2.1
-        if: env.COMMIT_SHA != 'null'
+      - name: Setup msys and install required packages
+        if: runner.os == 'Windows'
+        uses: msys2/setup-msys2@v2
         with:
-          tag_name: ${{ env.COMMIT_SHA }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          msystem: ${{matrix.config.msys_sys}}
+          install: mingw-w64-${{matrix.config.msys_env}} make git expect
 
-  Sanitizers:
-    uses: ./.github/workflows/stockfish_sanitizers.yml
-  Tests:
-    uses: ./.github/workflows/stockfish_test.yml
-  Compiles:
-    uses: ./.github/workflows/stockfish_compile_test.yml
-  Binaries:
-    if: github.ref == 'refs/heads/master' || (startsWith(github.ref_name, 'sf_') && github.ref_type == 'tag')
-    uses: ./.github/workflows/stockfish_binaries.yml
-  ARM_Binaries:
-    if: github.ref == 'refs/heads/master' || (startsWith(github.ref_name, 'sf_') && github.ref_type == 'tag')
-    uses: ./.github/workflows/stockfish_arm_binaries.yml
+      - name: Download the used network from the fishtest framework
+        run: |
+          make net
+
+      - name: Extract the bench number from the commit history
+        run: |
+          git log HEAD | grep "\b[Bb]ench[ :]\+[0-9]\{7\}" | head -n 1 | sed "s/[^0-9]*\([0-9]*\).*/\1/g" > git_sig
+          [ -s git_sig ] && echo "benchref=$(cat git_sig)" >> $GITHUB_ENV && echo "Reference bench:" $(cat git_sig) || echo "No bench found"
+
+      - name: Check compiler
+        run: |
+          $COMPILER -v
+
+      - name: Test help target
+        run: |
+          make help
+
+      # x86-32 tests
+
+      - name: Test debug x86-32 build
+        if: ${{ matrix.config.run_32bit_tests }}
+        run: |
+          export CXXFLAGS="-D_GLIBCXX_DEBUG"
+          make clean
+          make -j2 ARCH=x86-32 optimize=no debug=yes build
+          ../tests/signature.sh $benchref
+
+      - name: Test x86-32 build
+        if: ${{ matrix.config.run_32bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-32 build
+          ../tests/signature.sh $benchref
+
+      - name: Test x86-32-sse41-popcnt build
+        if: ${{ matrix.config.run_32bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-32-sse41-popcnt build
+          ../tests/signature.sh $benchref
+
+      - name: Test x86-32-sse2 build
+        if: ${{ matrix.config.run_32bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-32-sse2 build
+          ../tests/signature.sh $benchref
+
+      - name: Test general-32 build
+        if: ${{ matrix.config.run_32bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=general-32 build
+          ../tests/signature.sh $benchref
+
+      # x86-64 tests
+
+      - name: Test debug x86-64-modern build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          export CXXFLAGS="-D_GLIBCXX_DEBUG"
+          make clean
+          make -j2 ARCH=x86-64-modern optimize=no debug=yes build
+          ../tests/signature.sh $benchref
+
+      - name: Test x86-64-modern build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64-modern build
+          ../tests/signature.sh $benchref
+
+      - name: Test x86-64-ssse3 build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64-ssse3 build
+          ../tests/signature.sh $benchref
+
+      - name: Test x86-64-sse3-popcnt build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64-sse3-popcnt build
+          ../tests/signature.sh $benchref
+
+      - name: Test x86-64 build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64 build
+          ../tests/signature.sh $benchref
+
+      - name: Test general-64 build
+        if: matrix.config.run_64bit_tests
+        run: |
+          make clean
+          make -j2 ARCH=general-64 build
+          ../tests/signature.sh $benchref
+
+      # x86-64 with newer extensions tests
+
+      - name: Compile x86-64-avx2 build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64-avx2 build
+
+      - name: Compile x86-64-bmi2 build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64-bmi2 build
+
+      - name: Compile x86-64-avx512 build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64-avx512 build
+
+      - name: Compile x86-64-vnni512 build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64-vnni512 build
+
+      - name: Compile x86-64-vnni256 build
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64-vnni256 build
+
+      # Other tests
+
+      - name: Check perft and search reproducibility
+        if: ${{ matrix.config.run_64bit_tests }}
+        run: |
+          make clean
+          make -j2 ARCH=x86-64-modern build
+          ../tests/perft.sh
+          ../tests/reprosearch.sh
+
+      # Sanitizers
+
+      - name: Run under valgrind
+        if: ${{ matrix.config.run_expensive_tests }}
+        run: |
+          export CXXFLAGS="-O1 -fno-inline"
+          make clean
+          make -j2 ARCH=x86-64-modern debug=yes optimize=no build > /dev/null
+          ../tests/instrumented.sh --valgrind
+          ../tests/instrumented.sh --valgrind-thread
+
+      - name: Run with UB sanitizer
+        if: ${{ matrix.config.run_expensive_tests }}
+        run: |
+          export CXXFLAGS="-O1 -fno-inline"
+          make clean
+          make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes build > /dev/null
+          ../tests/instrumented.sh --sanitizer-undefined
+
+      - name: Run with thread sanitizer
+        if: ${{ matrix.config.run_expensive_tests }}
+        run: |
+          export CXXFLAGS="-O1 -fno-inline"
+          make clean
+          make -j2 ARCH=x86-64-modern sanitize=thread optimize=no debug=yes build > /dev/null
+          ../tests/instrumented.sh --sanitizer-thread

.github/workflows/stockfish_arm_binaries.yml

@@ -1,158 +0,0 @@
-name: Stockfish
-on:
-  workflow_call:
-jobs:
-  Stockfish:
-    name: ${{ matrix.config.name }} ${{ matrix.binaries }}
-    runs-on: ${{ matrix.config.os }}
-    env:
-      COMPILER: ${{ matrix.config.compiler }}
-      COMP: ${{ matrix.config.comp }}
-      EMU: ${{ matrix.config.emu }}
-      EXT: ${{ matrix.config.ext }}
-      OS: ${{ matrix.config.os }}
-      BINARY: ${{ matrix.binaries }}
-    strategy:
-      matrix:
-        config:
-          - name: Android NDK aarch64
-            os: ubuntu-22.04
-            compiler: aarch64-linux-android21-clang++
-            emu: qemu-aarch64
-            comp: ndk
-            shell: bash {0}
-          - name: Android NDK arm
-            os: ubuntu-22.04
-            compiler: armv7a-linux-androideabi21-clang++
-            emu: qemu-arm
-            comp: ndk
-            shell: bash {0}
-        binaries:
-          - armv8
-          - armv7
-          - armv7-neon
-        exclude:
-          - binaries: armv8
-            config: {compiler: armv7a-linux-androideabi21-clang++}
-          - binaries: armv7
-            config: {compiler: aarch64-linux-android21-clang++}
-          - binaries: armv7-neon
-            config: {compiler: aarch64-linux-android21-clang++}
-    defaults:
-      run:
-        working-directory: src
-        shell: ${{ matrix.config.shell }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Download required linux packages
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt update
-          sudo apt install qemu-user
-
-      - name: Install NDK
-        if: runner.os == 'Linux'
-        run: |
-          if [ $COMP == ndk ]; then
-            NDKV="21.4.7075529"
-            ANDROID_ROOT=/usr/local/lib/android
-            ANDROID_SDK_ROOT=$ANDROID_ROOT/sdk
-            SDKMANAGER=$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager
-            echo "y" | $SDKMANAGER "ndk;$NDKV"
-            ANDROID_NDK_ROOT=$ANDROID_SDK_ROOT/ndk/$NDKV
-            ANDROID_NDK_BIN=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin
-            echo "ANDROID_NDK_BIN=$ANDROID_NDK_BIN" >> $GITHUB_ENV
-          fi
-
-      - name: Download the used network from the fishtest framework
-        run: make net
-
-      - name: Check compiler
-        run: |
-          if [ $COMP == ndk ]; then
-            export PATH=${{ env.ANDROID_NDK_BIN }}:$PATH
-          fi
-          $COMPILER -v
-
-      - name: Test help target
-        run: make help
-
-      - name: Check git
-        run: git --version
-
-      # Compile profile guided builds
-
-      - name: Compile ${{ matrix.binaries }} build
-        run: |
-          if [ $COMP == ndk ]; then
-            export PATH=${{ env.ANDROID_NDK_BIN }}:$PATH
-            export LDFLAGS="-static -Wno-unused-command-line-argument"
-          fi
-          make clean
-          make -j2 profile-build ARCH=$BINARY COMP=$COMP WINE_PATH=$EMU
-          make strip ARCH=$BINARY COMP=$COMP
-          mv ./stockfish$EXT ../stockfish-android-$BINARY$EXT
-
-      - name: Remove non src files
-        run: rm -f *.o .depend *.nnue
-
-      - name: Download wiki
-        run: |
-          git clone https://github.com/official-stockfish/Stockfish.wiki.git ../wiki
-          cd ../wiki
-          rm -rf .git
-
-      - name: Create tar archive.
-        run: |
-          cd ..
-          mkdir stockfish
-          cp -r wiki stockfish/
-          cp -r src stockfish/
-          cp stockfish-android-$BINARY$EXT stockfish/
-          cp "Top CPU Contributors.txt" stockfish/
-          cp Copying.txt stockfish/
-          cp AUTHORS stockfish/
-          cp CITATION.cff stockfish/
-          cp README.md stockfish/
-          tar -cvf stockfish-android-$BINARY.tar stockfish
-
-      - name: Upload binaries
-        uses: actions/upload-artifact@v3
-        with:
-          name: stockfish-android-${{ matrix.binaries }}
-          path: stockfish-android-${{ matrix.binaries }}.tar
-
-      - name: Release
-        if: startsWith(github.ref_name, 'sf_') && github.ref_type == 'tag'
-        uses: softprops/action-gh-release@v1
-        with:
-          files: stockfish-android-${{ matrix.binaries }}.tar
-
-      - name: Get last commit sha
-        id: last_commit
-        run: echo "COMMIT_SHA=$(git rev-parse HEAD | cut -c 1-8)" >> $GITHUB_ENV
-
-      - name: Get commit date
-        id: commit_date
-        run: echo "COMMIT_DATE=$(git show -s --date=format:'%Y%m%d' --format=%cd HEAD)" >> $GITHUB_ENV
-
-      # Make sure that an old ci which still runs on master doesn't recreate a prerelease
-      - name: Check Pullable Commits
-        id: check_commits
-        run: |
-          git fetch
-          CHANGES=$(git rev-list HEAD..origin/master --count)
-          echo "CHANGES=$CHANGES" >> $GITHUB_ENV
-
-      - name: Prerelease
-        if: github.ref_name == 'master' && env.CHANGES == '0'
-        continue-on-error: true
-        uses: softprops/action-gh-release@v1
-        with:
-          name: Stockfish dev-${{ env.COMMIT_DATE }}-${{ env.COMMIT_SHA }}
-          tag_name: stockfish-dev-${{ env.COMMIT_DATE }}-${{ env.COMMIT_SHA }}
-          prerelease: true
-          files: stockfish-android-${{ matrix.binaries }}.tar

.github/workflows/stockfish_binaries.yml

@@ -1,168 +0,0 @@
-name: Stockfish
-on:
-  workflow_call:
-jobs:
-  Stockfish:
-    name: ${{ matrix.config.name }} ${{ matrix.binaries }}
-    runs-on: ${{ matrix.config.os }}
-    env:
-      COMPILER: ${{ matrix.config.compiler }}
-      COMP: ${{ matrix.config.comp }}
-      EXT: ${{ matrix.config.ext }}
-      NAME: ${{ matrix.config.simple_name }}
-      BINARY: ${{ matrix.binaries }}
-    strategy:
-      matrix:
-        config:
-          - name: Ubuntu 20.04 GCC
-            os: ubuntu-20.04
-            simple_name: ubuntu
-            compiler: g++
-            comp: gcc
-            shell: bash {0}
-            archive_ext: tar
-          - name: MacOS 12 Apple Clang
-            os: macos-12
-            simple_name: macos
-            compiler: clang++
-            comp: clang
-            shell: bash {0}
-            archive_ext: tar
-          - name: Windows 2022 Mingw-w64 GCC x86_64
-            os: windows-2022
-            simple_name: windows
-            compiler: g++
-            comp: mingw
-            msys_sys: mingw64
-            msys_env: x86_64-gcc
-            shell: msys2 {0}
-            ext: .exe
-            archive_ext: zip
-        binaries:
-          - x86-64
-          - x86-64-modern
-          - x86-64-avx2
-        exclude:
-          - binaries: x86-64-avx2
-            config: {os: macos-12}
-    defaults:
-      run:
-        working-directory: src
-        shell: ${{ matrix.config.shell }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Download required linux packages
-        if: runner.os == 'Linux'
-        run: sudo apt update
-
-      - name: Setup msys and install required packages
-        if: runner.os == 'Windows'
-        uses: msys2/setup-msys2@v2
-        with:
-          msystem: ${{ matrix.config.msys_sys }}
-          install: mingw-w64-${{ matrix.config.msys_env }} make git zip
-
-      - name: Download the used network from the fishtest framework
-        run: make net
-
-      - name: Check compiler
-        run: $COMPILER -v
-
-      - name: Test help target
-        run: make help
-
-      - name: Check git
-        run: git --version
-
-      # Compile profile guided builds
-
-      - name: Compile ${{ matrix.binaries }} build
-        run: |
-          make -j2 profile-build ARCH=$BINARY COMP=$COMP
-          make strip ARCH=$BINARY COMP=$COMP
-          mv ./stockfish$EXT ../stockfish-$NAME-$BINARY$EXT
-
-      - name: Remove non src files
-        run: git clean -fx
-
-      - name: Download wiki
-        run: |
-          git clone https://github.com/official-stockfish/Stockfish.wiki.git ../wiki
-          rm -rf ../wiki/.git
-
-      - name: Create directory.
-        run: |
-          cd ..
-          mkdir stockfish
-          cp -r wiki stockfish/
-          cp -r src stockfish/
-          cp stockfish-$NAME-$BINARY$EXT stockfish/
-          cp "Top CPU Contributors.txt" stockfish/
-          cp Copying.txt stockfish/
-          cp AUTHORS stockfish/
-          cp CITATION.cff stockfish/
-          cp README.md stockfish/
-
-      - name: Create tar
-        if: runner.os != 'Windows'
-        run: |
-          cd ..
-          tar -cvf stockfish-$NAME-$BINARY.tar stockfish
-
-      - name: Create zip
-        if: runner.os == 'Windows'
-        run: |
-          cd ..
-          zip -r stockfish-$NAME-$BINARY.zip stockfish
-
-      - name: Upload binaries
-        if: runner.os != 'Windows'
-        uses: actions/upload-artifact@v3
-        with:
-          name: stockfish-${{ matrix.config.os }}-${{ matrix.binaries }}
-          path: stockfish-${{ matrix.config.simple_name }}-${{ matrix.binaries }}.tar
-
-      # Artifacts automatically get zipped
-      # to avoid double zipping, we use the unzipped directory
-      - name: Upload binaries
-        if: runner.os == 'Windows'
-        uses: actions/upload-artifact@v3
-        with:
-          name: stockfish-${{ matrix.config.os }}-${{ matrix.binaries }}
-          path: stockfish
-
-      - name: Release
-        if: startsWith(github.ref_name, 'sf_') && github.ref_type == 'tag'
-        uses: softprops/action-gh-release@v1
-        with:
-          files: stockfish-${{ matrix.config.simple_name }}-${{ matrix.binaries }}.${{ matrix.config.archive_ext }}
-
-      - name: Get last commit sha
-        id: last_commit
-        run: echo "COMMIT_SHA=$(git rev-parse HEAD | cut -c 1-8)" >> $GITHUB_ENV
-
-      - name: Get commit date
-        id: commit_date
-        run: echo "COMMIT_DATE=$(git show -s --date=format:'%Y%m%d' --format=%cd HEAD)" >> $GITHUB_ENV
-
-      # Make sure that an old ci which still runs on master doesn't recreate a prerelease
-      - name: Check Pullable Commits
-        id: check_commits
-        run: |
-          git fetch
-          CHANGES=$(git rev-list HEAD..origin/master --count)
-          echo "CHANGES=$CHANGES" >> $GITHUB_ENV
-
-      - name: Prerelease
-        if: github.ref_name == 'master' && env.CHANGES == '0'
-        continue-on-error: true
-        uses: softprops/action-gh-release@v1
-        with:
-          name: Stockfish dev-${{ env.COMMIT_DATE }}-${{ env.COMMIT_SHA }}
-          tag_name: stockfish-dev-${{ env.COMMIT_DATE }}-${{ env.COMMIT_SHA }}
-          prerelease: true
-          files: stockfish-${{ matrix.config.simple_name }}-${{ matrix.binaries }}.${{ matrix.config.archive_ext }}
-

.github/workflows/stockfish_compile_test.yml

@@ -1,102 +0,0 @@
-name: Stockfish
-on:
-  workflow_call:
-jobs:
-  Stockfish:
-    name: ${{ matrix.config.name }}
-    runs-on: ${{ matrix.config.os }}
-    env:
-      COMPILER: ${{ matrix.config.compiler }}
-      COMP: ${{ matrix.config.comp }}
-    strategy:
-      matrix:
-        config:
-          - name: Ubuntu 20.04 GCC
-            os: ubuntu-20.04
-            compiler: g++
-            comp: gcc
-            shell: bash {0}
-          - name: Ubuntu 20.04 Clang
-            os: ubuntu-20.04
-            compiler: clang++
-            comp: clang
-            shell: bash {0}
-          - name: MacOS 12 Apple Clang
-            os: macos-12
-            compiler: clang++
-            comp: clang
-            shell: bash {0}
-          - name: MacOS 12 GCC 11
-            os: macos-12
-            compiler: g++-11
-            comp: gcc
-            shell: bash {0}
-          - name: Windows 2022 Mingw-w64 GCC x86_64
-            os: windows-2022
-            compiler: g++
-            comp: mingw
-            msys_sys: mingw64
-            msys_env: x86_64-gcc
-            shell: msys2 {0}
-          - name: Windows 2022 Mingw-w64 Clang x86_64
-            os: windows-2022
-            compiler: clang++
-            comp: clang
-            msys_sys: clang64
-            msys_env: clang-x86_64-clang
-            shell: msys2 {0}
-
-    defaults:
-      run:
-        working-directory: src
-        shell: ${{ matrix.config.shell }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Setup msys and install required packages
-        if: runner.os == 'Windows'
-        uses: msys2/setup-msys2@v2
-        with:
-          msystem: ${{matrix.config.msys_sys}}
-          install: mingw-w64-${{matrix.config.msys_env}} make git
-
-      - name: Download the used network from the fishtest framework
-        run: make net
-
-      - name: Check compiler
-        run: $COMPILER -v
-
-      - name: Test help target
-        run: make help
-
-      - name: Check git
-        run: git --version
-
-      # x86-64 with newer extensions tests
-
-      - name: Compile x86-64-avx2 build
-        run: |
-          make clean
-          make -j2 ARCH=x86-64-avx2 build
-
-      - name: Compile x86-64-bmi2 build
-        run: |
-          make clean
-          make -j2 ARCH=x86-64-bmi2 build
-
-      - name: Compile x86-64-avx512 build
-        run: |
-          make clean
-          make -j2 ARCH=x86-64-avx512 build
-
-      - name: Compile x86-64-vnni512 build
-        run: |
-          make clean
-          make -j2 ARCH=x86-64-vnni512 build
-
-      - name: Compile x86-64-vnni256 build
-        run: |
-          make clean
-          make -j2 ARCH=x86-64-vnni256 build

.github/workflows/stockfish_sanitizers.yml

@@ -1,66 +0,0 @@
-name: Stockfish
-on:
-  workflow_call:
-jobs:
-  Stockfish:
-    name: ${{ matrix.sanitizers.name }}
-    runs-on: ${{ matrix.config.os }}
-    env:
-      COMPILER: ${{ matrix.config.compiler }}
-      COMP: ${{ matrix.config.comp }}
-      CXXFLAGS: "-Werror"
-    strategy:
-      matrix:
-        config:
-          - name: Ubuntu 20.04 GCC
-            os: ubuntu-20.04
-            compiler: g++
-            comp: gcc
-            shell: bash {0}
-        sanitizers:
-          - name: Run with thread sanitizer
-            make_option: sanitize=thread
-            instrumented_option: sanitizer-thread
-          - name: Run with UB sanitizer
-            make_option: sanitize=undefined
-            instrumented_option: sanitizer-undefined
-          - name: Run under valgrind
-            make_option: ""
-            instrumented_option: valgrind
-          - name: Run under valgrind-thread
-            make_option: ""
-            instrumented_option: valgrind-thread
-    defaults:
-      run:
-        working-directory: src
-        shell: ${{ matrix.config.shell }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Download required linux packages
-        run: |
-          sudo apt update
-          sudo apt install expect valgrind g++-multilib
-
-      - name: Download the used network from the fishtest framework
-        run: make net
-
-      - name: Check compiler
-        run: $COMPILER -v
-
-      - name: Test help target
-        run: make help
-
-      - name: Check git
-        run: git --version
-
-      # Sanitizers
-
-      - name: ${{ matrix.sanitizers.name }}
-        run: |
-          export CXXFLAGS="-O1 -fno-inline"
-          make clean
-          make -j2 ARCH=x86-64-modern ${{ matrix.sanitizers.make_option }} debug=yes optimize=no build > /dev/null
-          ../tests/instrumented.sh --${{ matrix.sanitizers.instrumented_option }}

.github/workflows/stockfish_test.yml

@@ -1,256 +0,0 @@
-name: Stockfish
-on:
-  workflow_call:
-jobs:
-  Stockfish:
-    name: ${{ matrix.config.name }}
-    runs-on: ${{ matrix.config.os }}
-    env:
-      COMPILER: ${{ matrix.config.compiler }}
-      COMP: ${{ matrix.config.comp }}
-      CXXFLAGS: "-Werror"
-    strategy:
-      matrix:
-        config:
-          - name: Ubuntu 20.04 GCC
-            os: ubuntu-20.04
-            compiler: g++
-            comp: gcc
-            run_32bit_tests: true
-            run_64bit_tests: true
-            shell: bash {0}
-          - name: Ubuntu 20.04 Clang
-            os: ubuntu-20.04
-            compiler: clang++
-            comp: clang
-            run_32bit_tests: true
-            run_64bit_tests: true
-            shell: bash {0}
-          - name: Android NDK aarch64
-            os: ubuntu-22.04
-            compiler: aarch64-linux-android21-clang++
-            comp: ndk
-            run_armv8_tests: true
-            shell: bash {0}
-          - name: Android NDK arm
-            os: ubuntu-22.04
-            compiler: armv7a-linux-androideabi21-clang++
-            comp: ndk
-            run_armv7_tests: true
-            shell: bash {0}
-          - name: MacOS 12 Apple Clang
-            os: macos-12
-            compiler: clang++
-            comp: clang
-            run_64bit_tests: true
-            shell: bash {0}
-          - name: MacOS 12 GCC 11
-            os: macos-12
-            compiler: g++-11
-            comp: gcc
-            run_64bit_tests: true
-            shell: bash {0}
-          - name: Windows 2022 Mingw-w64 GCC x86_64
-            os: windows-2022
-            compiler: g++
-            comp: mingw
-            run_64bit_tests: true
-            msys_sys: mingw64
-            msys_env: x86_64-gcc
-            shell: msys2 {0}
-          - name: Windows 2022 Mingw-w64 GCC i686
-            os: windows-2022
-            compiler: g++
-            comp: mingw
-            run_32bit_tests: true
-            msys_sys: mingw32
-            msys_env: i686-gcc
-            shell: msys2 {0}
-          - name: Windows 2022 Mingw-w64 Clang x86_64
-            os: windows-2022
-            compiler: clang++
-            comp: clang
-            run_64bit_tests: true
-            msys_sys: clang64
-            msys_env: clang-x86_64-clang
-            shell: msys2 {0}
-    defaults:
-      run:
-        working-directory: src
-        shell: ${{ matrix.config.shell }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Download required linux packages
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt update
-          sudo apt install expect valgrind g++-multilib qemu-user
-
-      - name: Install NDK
-        if: runner.os == 'Linux'
-        run: |
-          if [ $COMP == ndk ]; then
-            NDKV="21.4.7075529"
-            ANDROID_ROOT=/usr/local/lib/android
-            ANDROID_SDK_ROOT=$ANDROID_ROOT/sdk
-            SDKMANAGER=$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager
-            echo "y" | $SDKMANAGER "ndk;$NDKV"
-            ANDROID_NDK_ROOT=$ANDROID_SDK_ROOT/ndk/$NDKV
-            ANDROID_NDK_BIN=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin
-            echo "ANDROID_NDK_BIN=$ANDROID_NDK_BIN" >> $GITHUB_ENV
-          fi
-
-      - name: Setup msys and install required packages
-        if: runner.os == 'Windows'
-        uses: msys2/setup-msys2@v2
-        with:
-          msystem: ${{ matrix.config.msys_sys }}
-          install: mingw-w64-${{ matrix.config.msys_env }} make git expect
-
-      - name: Download the used network from the fishtest framework
-        run: make net
-
-      - name: Extract the bench number from the commit history
-        run: |
-          git log HEAD | grep -o "\b[Bb]ench[ :]\+[1-9][0-9]\{5,9\}\b" | head -n 1 | sed "s/[^0-9]//g" > git_sig
-          [ -s git_sig ] && echo "benchref=$(cat git_sig)" >> $GITHUB_ENV && echo "Reference bench:" $(cat git_sig) || echo "No bench found"
-
-      - name: Check compiler
-        run: |
-          if [ $COMP == ndk ]; then
-            export PATH=${{ env.ANDROID_NDK_BIN }}:$PATH
-          fi
-          $COMPILER -v
-
-      - name: Test help target
-        run: make help
-
-      - name: Check git
-        run: git --version
-
-      # x86-32 tests
-
-      - name: Test debug x86-32 build
-        if: ${{ matrix.config.run_32bit_tests }}
-        run: |
-          export CXXFLAGS="-Werror -D_GLIBCXX_DEBUG"
-          make clean
-          make -j2 ARCH=x86-32 optimize=no debug=yes build
-          ../tests/signature.sh $benchref
-
-      - name: Test x86-32 build
-        if: ${{ matrix.config.run_32bit_tests }}
-        run: |
-          make clean
-          make -j2 ARCH=x86-32 build
-          ../tests/signature.sh $benchref
-
-      - name: Test x86-32-sse41-popcnt build
-        if: ${{ matrix.config.run_32bit_tests }}
-        run: |
-          make clean
-          make -j2 ARCH=x86-32-sse41-popcnt build
-          ../tests/signature.sh $benchref
-
-      - name: Test x86-32-sse2 build
-        if: ${{ matrix.config.run_32bit_tests }}
-        run: |
-          make clean
-          make -j2 ARCH=x86-32-sse2 build
-          ../tests/signature.sh $benchref
-
-      - name: Test general-32 build
-        if: ${{ matrix.config.run_32bit_tests }}
-        run: |
-          make clean
-          make -j2 ARCH=general-32 build
-          ../tests/signature.sh $benchref
-
-      # x86-64 tests
-
-      - name: Test debug x86-64-modern build
-        if: ${{ matrix.config.run_64bit_tests }}
-        run: |
-          export CXXFLAGS="-Werror -D_GLIBCXX_DEBUG"
-          make clean
-          make -j2 ARCH=x86-64-modern optimize=no debug=yes build
-          ../tests/signature.sh $benchref
-
-      - name: Test x86-64-modern build
-        if: ${{ matrix.config.run_64bit_tests }}
-        run: |
-          make clean
-          make -j2 ARCH=x86-64-modern build
-          ../tests/signature.sh $benchref
-
-      - name: Test x86-64-ssse3 build
-        if: ${{ matrix.config.run_64bit_tests }}
-        run: |
-          make clean
-          make -j2 ARCH=x86-64-ssse3 build
-          ../tests/signature.sh $benchref
-
-      - name: Test x86-64-sse3-popcnt build
-        if: ${{ matrix.config.run_64bit_tests }}
-        run: |
-          make clean
-          make -j2 ARCH=x86-64-sse3-popcnt build
-          ../tests/signature.sh $benchref
-
-      - name: Test x86-64 build
-        if: ${{ matrix.config.run_64bit_tests }}
-        run: |
-          make clean
-          make -j2 ARCH=x86-64 build
-          ../tests/signature.sh $benchref
-
-      - name: Test general-64 build
-        if: matrix.config.run_64bit_tests
-        run: |
-          make clean
-          make -j2 ARCH=general-64 build
-          ../tests/signature.sh $benchref
-
-      # armv8 tests
-
-      - name: Test armv8 build
-        if: ${{ matrix.config.run_armv8_tests }}
-        run: |
-          export PATH=${{ env.ANDROID_NDK_BIN }}:$PATH
-          export LDFLAGS="-static -Wno-unused-command-line-argument"
-          make clean
-          make -j2 ARCH=armv8 build
-          ../tests/signature.sh $benchref
-
-      # armv7 tests
-
-      - name: Test armv7 build
-        if: ${{ matrix.config.run_armv7_tests }}
-        run: |
-          export PATH=${{ env.ANDROID_NDK_BIN }}:$PATH
-          export LDFLAGS="-static -Wno-unused-command-line-argument"
-          make clean
-          make -j2 ARCH=armv7 build
-          ../tests/signature.sh $benchref
-
-      - name: Test armv7-neon build
-        if: ${{ matrix.config.run_armv7_tests }}
-        run: |
-          export PATH=${{ env.ANDROID_NDK_BIN }}:$PATH
-          export LDFLAGS="-static -Wno-unused-command-line-argument"
-          make clean
-          make -j2 ARCH=armv7-neon build
-          ../tests/signature.sh $benchref
-
-      # Other tests
-
-      - name: Check perft and search reproducibility
-        if: ${{ matrix.config.run_64bit_tests }}
-        run: |
-          make clean
-          make -j2 ARCH=x86-64-modern build
-          ../tests/perft.sh
-          ../tests/reprosearch.sh

AUTHORS

@@ -1,15 +1,17 @@
-# Founders of the Stockfish project and Fishtest infrastructure
+# List of authors for Stockfish
+
+# Founders of the Stockfish project and fishtest infrastructure
 Tord Romstad (romstad)
 Marco Costalba (mcostalba)
 Joona Kiiski (zamar)
 Gary Linscott (glinscott)
 
-# Authors and inventors of NNUE, training, and NNUE port
+# Authors and inventors of NNUE, training, NNUE port
 Yu Nasu (ynasu87)
 Motohiro Isozaki (yaneurao)
 Hisayori Noda (nodchip)
 
-# All other authors of Stockfish code (in alphabetical order)
+# all other authors of the code in alphabetical order
 Aditya (absimaldata)
 Adrian Petrescu (apetresc)
 Ajith Chandy Jose (ajithcj)
@@ -19,7 +21,6 @@ Alexander Kure
 Alexander Pagel (Lolligerhans)
 Alfredo Menezes (lonfom169)
 Ali AlZhrani (Cooffe)
-Andreas Matthies (Matthies)
 Andrei Vetrov (proukornew)
 Andrew Grant (AndyGrant)
 Andrey Neporada (nepal)
@@ -34,23 +35,18 @@ Ben Chaney (Chaneybenjamini)
 Ben Koshy (BKSpurgeon)
 Bill Henry (VoyagerOne)
 Bojun Guo (noobpwnftw, Nooby)
-borg323
-Boštjan Mejak (PedanticHacker)
 braich
 Brian Sheppard (SapphireBrand, briansheppard-toast)
 Bruno de Melo Costa (BM123499)
-Bruno Pellanda (pellanda)
 Bryan Cross (crossbr)
 candirufish
 Chess13234
 Chris Cain (ceebo)
-clefrks
 Dale Weiler (graphitemaster)
+Dan Schmidt (dfannius)
 Daniel Axtens (daxtens)
 Daniel Dugovic (ddugovic)
-Dan Schmidt (dfannius)
 Dariusz Orzechowski (dorzechowski)
-David (dav1312)
 David Zar
 Daylen Yang (daylen)
 Deshawn Mohan-Smith (GoldenRare)
@@ -65,6 +61,7 @@ Eelco de Groot (KingDefender)
 Elvin Liu (solarlight2)
 erbsenzaehler
 Ernesto Gatti
+Linmiao Xu (linrock)
 Fabian Beuke (madnight)
 Fabian Fichter (ianfab)
 Fanael Linithien (Fanael)
@@ -77,35 +74,32 @@ George Sobala (gsobala)
 gguliash
 Giacomo Lorenzetti (G-Lorenz)
 Gian-Carlo Pascutto (gcp)
-Goh CJ (cj5716)
 Gontran Lemaire (gonlem)
 Goodkov Vasiliy Aleksandrovich (goodkov)
 Gregor Cramer
 GuardianRM
-Guy Vreuls (gvreuls)
 Günther Demetz (pb00067, pb00068)
+Guy Vreuls (gvreuls)
 Henri Wiechers
 Hiraoka Takuya (HiraokaTakuya)
 homoSapiensSapiens
 Hongzhi Cheng
 Ivan Ivec (IIvec)
 Jacques B. (Timshel)
-Jake Senne (w1wwwwww)
 Jan Ondruš (hxim)
-Jared Kish (Kurtbusch, kurt22i)
+Jared Kish (Kurtbusch)
 Jarrod Torriero (DU-jdto)
-Jean-Francois Romang (jromang)
 Jean Gauthier (OuaisBla)
+Jean-Francois Romang (jromang)
 Jekaa
 Jerry Donald Watson (jerrydonaldwatson)
 jjoshua2
-Jonathan Buladas Dumale (SFisGOD)
 Jonathan Calovski (Mysseno)
-Jonathan McDermid (jonathanmcdermid)
+Jonathan Buladas Dumale (SFisGOD)
 Joost VandeVondele (vondele)
+Jörg Oster (joergoster)
 Joseph Ellis (jhellis3)
 Joseph R. Prostko
-Jörg Oster (joergoster)
 Julian Willemer (NightlyKing)
 jundery
 Justin Blanchard (UncombedCoconut)
@@ -119,7 +113,6 @@ Krystian Kuzniarek (kuzkry)
 Leonardo Ljubičić (ICCF World Champion)
 Leonid Pechenik (lp--)
 Liam Keegan (lkeegan)
-Linmiao Xu (linrock)
 Linus Arver (listx)
 loco-loco
 Lub van den Berg (ElbertoOne)
@@ -134,7 +127,6 @@ Matt Ginsberg (mattginsberg)
 Matthew Lai (matthewlai)
 Matthew Sullivan (Matt14916)
 Max A. (Disservin)
-Maxim Masiutin (maximmasiutin)
 Maxim Molchanov (Maxim)
 Michael An (man)
 Michael Byrne (MichaelB7)
@@ -149,39 +141,35 @@ Mira
 Miroslav Fontán (Hexik)
 Moez Jellouli (MJZ1977)
 Mohammed Li (tthsqe12)
-Muzhen J (XInTheDark)
 Nathan Rugg (nmrugg)
-Nguyen Pham (nguyenpham)
-Nicklas Persson (NicklasPersson)
 Nick Pelling (nickpelling)
+Nicklas Persson (NicklasPersson)
 Niklas Fiekas (niklasf)
 Nikolay Kostov (NikolayIT)
+Nguyen Pham (nguyenpham)
 Norman Schmidt (FireFather)
 notruck
 Ofek Shochat (OfekShochat, ghostway)
 Ondrej Mosnáček (WOnder93)
-Ondřej Mišina (AndrovT)
 Oskar Werkelin Ahlin
 Pablo Vazquez
 Panthee
 Pascal Romaret
 Pasquale Pigazzini (ppigazzini)
 Patrick Jansen (mibere)
+pellanda
 Peter Schneider (pschneider1968)
 Peter Zsifkovits (CoffeeOne)
-PikaCat
 Praveen Kumar Tummala (praveentml)
-Prokop Randáček (ProkopRandacek)
 Rahul Dsilva (silversolver1)
 Ralph Stößer (Ralph Stoesser)
 Raminder Singh
 renouve
-Reuven Peleg (R-Peleg)
-Richard Lloyd (Richard-Lloyd)
+Reuven Peleg
+Richard Lloyd
 Rodrigo Exterckötter Tjäder
-Rodrigo Roim (roim)
-Ronald de Man (syzygy1, syzygy)
 Ron Britvich (Britvich)
+Ronald de Man (syzygy1, syzygy)
 rqs
 Rui Coelho (ruicoelhopedro)
 Ryan Schmitt
@@ -192,22 +180,20 @@ Sergei Antonov (saproj)
 Sergei Ivanov (svivanov72)
 Sergio Vieri (sergiovieri)
 sf-x
-Shahin M. Shahin (peregrine)
 Shane Booth (shane31)
 Shawn Varghese (xXH4CKST3RXx)
 Siad Daboul (Topologist)
 Stefan Geschwentner (locutus2)
 Stefano Cardanobile (Stefano80)
-Stefano Di Martino (StefanoD)
 Steinar Gunderson (sesse)
 Stéphane Nicolet (snicolet)
-Syine Mineta (MinetaS)
+Prokop Randáček (ProkopRandacek)
 Thanar2
 thaspel
 theo77186
-Tomasz Sobczyk (Sopel97)
 Tom Truscott
 Tom Vijlbrief (tomtor)
+Tomasz Sobczyk (Sopel97)
 Torsten Franz (torfranz, tfranzer)
 Torsten Hellwig (Torom)
 Tracey Emery (basepr1me)
@@ -215,12 +201,11 @@ tttak
 Unai Corzo (unaiic)
 Uri Blass (uriblass)
 Vince Negri (cuddlestmonkey)
-Viren
-windfishballad
 xefoci7612
 zz4032
 
+
 # Additionally, we acknowledge the authors and maintainers of fishtest,
-# an amazing and essential framework for Stockfish development!
+# an amazing and essential framework for the development of Stockfish!
 #
 # https://github.com/glinscott/fishtest/blob/master/AUTHORS

CITATION.cff

@@ -1,23 +0,0 @@
-# This CITATION.cff file was generated with cffinit.
-# Visit https://bit.ly/cffinit to generate yours today!
-
-cff-version: 1.2.0
-title: Stockfish
-message: >-
-  Please cite this software using the metadata from this
-  file.
-type: software
-authors:
-  - name: The Stockfish developers (see AUTHORS file)
-repository-code: 'https://github.com/official-stockfish/Stockfish'
-url: 'https://stockfishchess.org/'
-repository-artifact: 'https://stockfishchess.org/download/'
-abstract: Stockfish is a free and strong UCI chess engine.
-keywords:
-  - chess
-  - artificial intelligence (AI)
-  - tree search
-  - alpha-beta search
-  - neural networks (NN)
-  - efficiently updatable neural networks (NNUE)
-license: GPL-3.0

README.md

@@ -1,164 +1,371 @@
-<div align="center">
-
-  [![Stockfish][stockfish128-logo]][website-link]
-
-  <h3>Stockfish</h3>
-
-  A free and strong UCI chess engine.
-  <br>
-  <strong>[Explore Stockfish docs »][wiki-link]</strong>
-  <br>
-  <br>
-  [Report bug][issue-link]
-  ·
-  [Open a discussion][discussions-link]
-  ·
-  [Discord][discord-link]
-  ·
-  [Blog][website-blog-link]
-
-  [![Build][build-badge]][build-link]
-  [![License][license-badge]][license-link]
-  <br>
-  [![Release][release-badge]][release-link]
-  [![Commits][commits-badge]][commits-link]
-  <br>
-  [![Website][website-badge]][website-link]
-  [![Fishtest][fishtest-badge]][fishtest-link]
-  [![Discord][discord-badge]][discord-link]
-
-</div>
-
 ## Overview
 
-[Stockfish][website-link] is a **free and strong UCI chess engine** derived from
-Glaurung 2.1 that analyzes chess positions and computes the optimal moves.
+[![Build Status](https://github.com/official-stockfish/Stockfish/actions/workflows/stockfish.yml/badge.svg)](https://github.com/official-stockfish/Stockfish/actions)
 
-Stockfish **does not include a graphical user interface** (GUI) that is required
-to display a chessboard and to make it easy to input moves. These GUIs are
-developed independently from Stockfish and are available online. **Read the
-documentation for your GUI** of choice for information about how to use
-Stockfish with it.
+[Stockfish](https://stockfishchess.org) is a free, powerful UCI chess engine
+derived from Glaurung 2.1. Stockfish is not a complete chess program and requires a
+UCI-compatible graphical user interface (GUI) (e.g. XBoard with PolyGlot, Scid,
+Cute Chess, eboard, Arena, Sigma Chess, Shredder, Chess Partner or Fritz) in order
+to be used comfortably. Read the documentation for your GUI of choice for information
+about how to use Stockfish with it.
 
-See also the Stockfish [documentation][wiki-usage-link] for further usage help.
+The Stockfish engine features two evaluation functions for chess. The efficiently 
+updatable neural network (NNUE) based evaluation is the default and by far the strongest.
+The classical evaluation based on handcrafted terms remains available. The strongest 
+network is integrated in the binary and downloaded automatically during the build process.
+The NNUE evaluation benefits from the vector intrinsics available on most CPUs (sse2, 
+avx2, neon, or similar).
 
 ## Files
 
 This distribution of Stockfish consists of the following files:
 
-  * [README.md][readme-link], the file you are currently reading.
+  * [README.md](https://github.com/official-stockfish/Stockfish/blob/master/README.md),
+    the file you are currently reading.
 
-  * [Copying.txt][license-link], a text file containing the GNU General Public
-    License version 3.
+  * [Copying.txt](https://github.com/official-stockfish/Stockfish/blob/master/Copying.txt),
+    a text file containing the GNU General Public License version 3.
 
-  * [AUTHORS][authors-link], a text file with the list of authors for the project.
+  * [AUTHORS](https://github.com/official-stockfish/Stockfish/blob/master/AUTHORS), 
+    a text file with the list of authors for the project
 
-  * [src][src-link], a subdirectory containing the full source code, including a
-    Makefile that can be used to compile Stockfish on Unix-like systems.
+  * [src](https://github.com/official-stockfish/Stockfish/tree/master/src),
+    a subdirectory containing the full source code, including a Makefile
+    that can be used to compile Stockfish on Unix-like systems.
 
   * a file with the .nnue extension, storing the neural network for the NNUE
     evaluation. Binary distributions will have this file embedded.
 
-## The UCI protocol
+## The UCI protocol and available options
 
-The [Universal Chess Interface][uci-link] (UCI) is a standard text-based protocol
-used to communicate with a chess engine and is the recommended way to do so for
-typical graphical user interfaces (GUI) or chess tools. Stockfish implements the
-majority of its options.
+The Universal Chess Interface (UCI) is a standard protocol used to communicate with
+a chess engine, and is the recommended way to do so for typical graphical user interfaces
+(GUI) or chess tools. Stockfish implements the majority of its options as described
+in [the UCI protocol](https://www.shredderchess.com/download/div/uci.zip).
 
-Developers can see the default values for the UCI options available in Stockfish
-by typing `./stockfish uci` in a terminal, but most users should typically use a
-chess GUI to interact with Stockfish.
+Developers can see the default values for UCI options available in Stockfish by typing
+`./stockfish uci` in a terminal, but the majority of users will typically see them and
+change them via a chess GUI. This is a list of available UCI options in Stockfish:
 
-For more information on UCI or debug commands, see our [documentation][wiki-commands-link].
+  * #### Threads
+    The number of CPU threads used for searching a position. For best performance, set
+    this equal to the number of CPU cores available.
 
-## Compiling Stockfish
+  * #### Hash
+    The size of the hash table in MB. It is recommended to set Hash after setting Threads.
 
-Stockfish has support for 32 or 64-bit CPUs, certain hardware instructions,
-big-endian machines such as Power PC, and other platforms.
+  * #### Clear Hash
+    Clear the hash table.
 
-On Unix-like systems, it should be easy to compile Stockfish directly from the
-source code with the included Makefile in the folder `src`. In general, it is
-recommended to run `make help` to see a list of make targets with corresponding
-descriptions.
+  * #### Ponder
+    Let Stockfish ponder its next move while the opponent is thinking.
+
+  * #### MultiPV
+    Output the N best lines (principal variations, PVs) when searching.
+    Leave at 1 for best performance.
+
+  * #### Use NNUE
+    Toggle between the NNUE and classical evaluation functions. If set to "true",
+    the network parameters must be available to load from file (see also EvalFile),
+    if they are not embedded in the binary.
+
+  * #### EvalFile
+    The name of the file of the NNUE evaluation parameters. Depending on the GUI the
+    filename might have to include the full path to the folder/directory that contains
+    the file. Other locations, such as the directory that contains the binary and the
+    working directory, are also searched.
+
+  * #### UCI_AnalyseMode
+    An option handled by your GUI.
+
+  * #### UCI_Chess960
+    An option handled by your GUI. If true, Stockfish will play Chess960.
+
+  * #### UCI_ShowWDL
+    If enabled, show approximate WDL statistics as part of the engine output.
+    These WDL numbers model expected game outcomes for a given evaluation and
+    game ply for engine self-play at fishtest LTC conditions (60+0.6s per game).
+
+  * #### UCI_LimitStrength
+    Enable weaker play aiming for an Elo rating as set by UCI_Elo. This option overrides Skill Level.
+
+  * #### UCI_Elo
+    If enabled by UCI_LimitStrength, aim for an engine strength of the given Elo.
+    This Elo rating has been calibrated at a time control of 60s+0.6s and anchored to CCRL 40/4.
+
+  * #### Skill Level
+    Lower the Skill Level in order to make Stockfish play weaker (see also UCI_LimitStrength).
+    Internally, MultiPV is enabled, and with a certain probability depending on the Skill Level a
+    weaker move will be played.
+
+  * #### SyzygyPath
+    Path to the folders/directories storing the Syzygy tablebase files. Multiple
+    directories are to be separated by ";" on Windows and by ":" on Unix-based
+    operating systems. Do not use spaces around the ";" or ":".
+
+    Example: `C:\tablebases\wdl345;C:\tablebases\wdl6;D:\tablebases\dtz345;D:\tablebases\dtz6`
+
+    It is recommended to store .rtbw files on an SSD. There is no loss in storing
+    the .rtbz files on a regular HDD. It is recommended to verify all md5 checksums
+    of the downloaded tablebase files (`md5sum -c checksum.md5`) as corruption will
+    lead to engine crashes.
+
+  * #### SyzygyProbeDepth
+    Minimum remaining search depth for which a position is probed. Set this option
+    to a higher value to probe less aggressively if you experience too much slowdown
+    (in terms of nps) due to tablebase probing.
+
+  * #### Syzygy50MoveRule
+    Disable to let fifty-move rule draws detected by Syzygy tablebase probes count
+    as wins or losses. This is useful for ICCF correspondence games.
+
+  * #### SyzygyProbeLimit
+    Limit Syzygy tablebase probing to positions with at most this many pieces left
+    (including kings and pawns).
+
+  * #### Move Overhead
+    Assume a time delay of x ms due to network and GUI overheads. This is useful to
+    avoid losses on time in those cases.
+
+  * #### Slow Mover
+    Lower values will make Stockfish take less time in games, higher values will
+    make it think longer.
+
+  * #### nodestime
+    Tells the engine to use nodes searched instead of wall time to account for
+    elapsed time. Useful for engine testing.
+
+  * #### Debug Log File
+    Write all communication to and from the engine into a text file.
+
+For developers the following non-standard commands might be of interest, mainly useful for debugging:
+
+  * #### bench *ttSize threads limit fenFile limitType evalType*
+    Performs a standard benchmark using various options. The signature of a version 
+    (standard node count) is obtained using all defaults. `bench` is currently 
+    `bench 16 1 13 default depth mixed`.
+
+  * #### compiler
+    Give information about the compiler and environment used for building a binary.
+
+  * #### d
+    Display the current position, with ascii art and fen.
+
+  * #### eval
+    Return the evaluation of the current position.
+
+  * #### export_net [filename]
+    Exports the currently loaded network to a file.
+    If the currently loaded network is the embedded network and the filename
+    is not specified then the network is saved to the file matching the name
+    of the embedded network, as defined in evaluate.h.
+    If the currently loaded network is not the embedded network (some net set
+    through the UCI setoption) then the filename parameter is required and the
+    network is saved into that file.
+
+  * #### flip
+    Flips the side to move.
+
+### Generating Training Data
+
+To generate training data from the classic eval, use the generate_training_data command with the setting "Use NNUE" set to "false". The given example is generation in its simplest form. There are more commands.
 
 ```
-cd src
-make -j build ARCH=x86-64-modern
+uci
+setoption name PruneAtShallowDepth value false
+setoption name Use NNUE value false
+setoption name Threads value X
+setoption name Hash value Y
+setoption name SyzygyPath value path
+isready
+generate_training_data depth A count B keep_draws 1 eval_limit 32000
 ```
 
-Detailed compilation instructions for all platforms can be found in our
-[documentation][wiki-compile-link].
+- `A` is the searched depth per move, or how far the engine looks forward. This value is an integer.
+- `B` is the amount of positions generated. This value is also an integer.
 
-## Contributing
+Specify how many threads and how much memory you would like to use with the `x` and `y` values. The option SyzygyPath is not necessary, but if you would like to use it, you must first have Syzygy endgame tablebases on your computer, which you can find [here](http://oics.olympuschess.com/tracker/index.php). You will need to have a torrent client to download these tablebases, as that is probably the fastest way to obtain them. The `path` is the path to the folder containing those tablebases. It does not have to be surrounded in quotes.
+
+This will create a file named "training_data.binpack" in the same folder as the binary containing the generated training data. Once generation is done, you can rename the file to something like "1billiondepth12.binpack" to remember the depth and quantity of the positions and move it to a folder named "trainingdata" in the same directory as the binaries.
+
+You will also need validation data that is used for loss calculation and accuracy computation. Validation data is generated in the same way as training data, but generally at most 1 million positions should be used as there's no need for more and it would just slow the learning process down. It may also be better to slightly increase the depth for validation data. After generation you can rename the validation data file to "val.binpack" and drop it in a folder named "validationdata" in the same directory to make it easier.
+
+## Training data formats.
+
+Currently there are 3 training data formats. Two of them are supported directly.
+
+- `.bin` - the original training data format. Uses 40 bytes per entry. Is supported directly by the `generate_training_data` command.
+- `.plain` - a human readable training data format. This one is not supported directly by the `generate_training_data` command. It should not be used for data exchange because it's less compact than other formats. It is mostly useful for inspection of the data.
+- `.binpack` - a compact binary training data format that exploits positions chains to further reduce size. It uses on average between 2 to 3 bytes per entry when generating data with `generate_training_data`. It is supported directly by `generate_training_data` command. It is currently the default for the `generate_training_data` command. A more in depth description can be found [here](docs/binpack.md)
+
+### Conversion between formats.
+
+There is a builting converted that support all 3 formats described above. Any of them can be converted to any other. For more information and usage guide see [here](docs/convert.md).
+
+## A note on classical evaluation versus NNUE evaluation
+
+Both approaches assign a value to a position that is used in alpha-beta (PVS) search
+to find the best move. The classical evaluation computes this value as a function
+of various chess concepts, handcrafted by experts, tested and tuned using fishtest.
+The NNUE evaluation computes this value with a neural network based on basic
+inputs (e.g. piece positions only). The network is optimized and trained
+on the evaluations of millions of positions at moderate search depth.
+
+The NNUE evaluation was first introduced in shogi, and ported to Stockfish afterward.
+It can be evaluated efficiently on CPUs, and exploits the fact that only parts
+of the neural network need to be updated after a typical chess move.
+[The nodchip repository](https://github.com/nodchip/Stockfish) provided the first 
+version of the needed tools to train and develop the NNUE networks. Today, more 
+advanced training tools are available in 
+[the nnue-pytorch repository](https://github.com/glinscott/nnue-pytorch/), 
+while data generation tools are available in 
+[a dedicated branch](https://github.com/official-stockfish/Stockfish/tree/tools).
+
+On CPUs supporting modern vector instructions (avx2 and similar), the NNUE evaluation
+results in much stronger playing strength, even if the nodes per second computed by
+the engine is somewhat lower (roughly 80% of nps is typical).
+
+Notes:
+
+1) the NNUE evaluation depends on the Stockfish binary and the network parameter file 
+(see the EvalFile UCI option). Not every parameter file is compatible with a given
+Stockfish binary, but the default value of the EvalFile UCI option is the name of a 
+network that is guaranteed to be compatible with that binary.
+
+2) to use the NNUE evaluation, the additional data file with neural network parameters
+needs to be available. Normally, this file is already embedded in the binary or it can
+be downloaded. The filename for the default (recommended) net can be found as the default
+value of the `EvalFile` UCI option, with the format `nn-[SHA256 first 12 digits].nnue`
+(for instance, `nn-c157e0a5755b.nnue`). This file can be downloaded from
+```
+https://tests.stockfishchess.org/api/nn/[filename]
+```
+replacing `[filename]` as needed.
+
+## What to expect from the Syzygy tablebases?
+
+If the engine is searching a position that is not in the tablebases (e.g.
+a position with 8 pieces), it will access the tablebases during the search.
+If the engine reports a very large score (typically 153.xx), this means
+it has found a winning line into a tablebase position.
+
+If the engine is given a position to search that is in the tablebases, it
+will use the tablebases at the beginning of the search to preselect all
+good moves, i.e. all moves that preserve the win or preserve the draw while
+taking into account the 50-move rule.
+It will then perform a search only on those moves. **The engine will not move
+immediately**, unless there is only a single good move. **The engine likely
+will not report a mate score, even if the position is known to be won.**
+
+It is therefore clear that this behaviour is not identical to what one might
+be used to with Nalimov tablebases. There are technical reasons for this
+difference, the main technical reason being that Nalimov tablebases use the
+DTM metric (distance-to-mate), while the Syzygy tablebases use a variation of the
+DTZ metric (distance-to-zero, zero meaning any move that resets the 50-move
+counter). This special metric is one of the reasons that the Syzygy tablebases are
+more compact than Nalimov tablebases, while still storing all information
+needed for optimal play and in addition being able to take into account
+the 50-move rule.
+
+## Large Pages
+
+Stockfish supports large pages on Linux and Windows. Large pages make
+the hash access more efficient, improving the engine speed, especially
+on large hash sizes. Typical increases are 5..10% in terms of nodes per
+second, but speed increases up to 30% have been measured. The support is
+automatic. Stockfish attempts to use large pages when available and
+will fall back to regular memory allocation when this is not the case.
+
+### Support on Linux
+
+Large page support on Linux is obtained by the Linux kernel
+transparent huge pages functionality. Typically, transparent huge pages
+are already enabled, and no configuration is needed.
+
+### Support on Windows
+
+The use of large pages requires "Lock Pages in Memory" privilege. See
+[Enable the Lock Pages in Memory Option (Windows)](https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows)
+on how to enable this privilege, then run [RAMMap](https://docs.microsoft.com/en-us/sysinternals/downloads/rammap)
+to double-check that large pages are used. We suggest that you reboot
+your computer after you have enabled large pages, because long Windows
+sessions suffer from memory fragmentation, which may prevent Stockfish
+from getting large pages: a fresh session is better in this regard.
+
+## Compiling Stockfish yourself from the sources
+
+Stockfish has support for 32 or 64-bit CPUs, certain hardware
+instructions, big-endian machines such as Power PC, and other platforms.
+
+On Unix-like systems, it should be easy to compile Stockfish
+directly from the source code with the included Makefile in the folder
+`src`. In general it is recommended to run `make help` to see a list of make
+targets with corresponding descriptions.
+
+```
+    cd src
+    make help
+    make net
+    make build ARCH=x86-64-modern
+```
+
+When not using the Makefile to compile (for instance, with Microsoft MSVC) you
+need to manually set/unset some switches in the compiler command line; see
+file *types.h* for a quick reference.
+
+When reporting an issue or a bug, please tell us which Stockfish version
+and which compiler you used to create your executable. This information
+can be found by typing the following command in a console:
+
+```
+    ./stockfish compiler
+```
+
+## Understanding the code base and participating in the project
+
+Stockfish's improvement over the last decade has been a great community
+effort. There are a few ways to help contribute to its growth.
 
 ### Donating hardware
 
-Improving Stockfish requires a massive amount of testing. You can donate your
-hardware resources by installing the [Fishtest Worker][worker-link] and viewing
-the current tests on [Fishtest][fishtest-link].
+Improving Stockfish requires a massive amount of testing. You can donate
+your hardware resources by installing the [Fishtest Worker](https://github.com/glinscott/fishtest/wiki/Running-the-worker:-overview)
+and view the current tests on [Fishtest](https://tests.stockfishchess.org/tests).
 
 ### Improving the code
 
-In the [chessprogramming wiki][programming-link], many techniques used in
-Stockfish are explained with a lot of background information.
-The [section on Stockfish][programmingsf-link] describes many features
-and techniques used by Stockfish. However, it is generic rather than
-focused on Stockfish's precise implementation.
+If you want to help improve the code, there are several valuable resources:
 
-The engine testing is done on [Fishtest][fishtest-link].
-If you want to help improve Stockfish, please read this [guideline][guideline-link]
+* [In this wiki,](https://www.chessprogramming.org) many techniques used in
+Stockfish are explained with a lot of background information.
+
+* [The section on Stockfish](https://www.chessprogramming.org/Stockfish)
+describes many features and techniques used by Stockfish. However, it is
+generic rather than being focused on Stockfish's precise implementation.
+Nevertheless, a helpful resource.
+
+* The latest source can always be found on [GitHub](https://github.com/official-stockfish/Stockfish).
+Discussions about Stockfish take place these days mainly in the [FishCooking](https://groups.google.com/forum/#!forum/fishcooking)
+group and on the [Stockfish Discord channel](https://discord.gg/nv8gDtt).
+The engine testing is done on [Fishtest](https://tests.stockfishchess.org/tests).
+If you want to help improve Stockfish, please read this [guideline](https://github.com/glinscott/fishtest/wiki/Creating-my-first-test)
 first, where the basics of Stockfish development are explained.
 
-Discussions about Stockfish take place these days mainly in the Stockfish
-[Discord server][discord-link]. This is also the best place to ask questions
-about the codebase and how to improve it.
 
 ## Terms of use
 
-Stockfish is free and distributed under the
-[**GNU General Public License version 3**][license-link] (GPL v3). Essentially,
-this means you are free to do almost exactly what you want with the program,
-including distributing it among your friends, making it available for download
-from your website, selling it (either by itself or as part of some bigger
-software package), or using it as the starting point for a software project of
-your own.
+Stockfish is free, and distributed under the **GNU General Public License version 3**
+(GPL v3). Essentially, this means you are free to do almost exactly
+what you want with the program, including distributing it among your
+friends, making it available for download from your website, selling
+it (either by itself or as part of some bigger software package), or
+using it as the starting point for a software project of your own.
 
-The only real limitation is that whenever you distribute Stockfish in some way,
-you MUST always include the license and the full source code (or a pointer to
-where the source code can be found) to generate the exact binary you are
-distributing. If you make any changes to the source code, these changes must
-also be made available under GPL v3.
+The only real limitation is that whenever you distribute Stockfish in
+some way, you MUST always include the license and the full source code
+(or a pointer to where the source code can be found) to generate the 
+exact binary you are distributing. If you make any changes to the
+source code, these changes must also be made available under the GPL v3.
 
-
-[authors-link]:       https://github.com/official-stockfish/Stockfish/blob/master/AUTHORS
-[build-link]:         https://github.com/official-stockfish/Stockfish/actions/workflows/stockfish.yml
-[commits-link]:       https://github.com/official-stockfish/Stockfish/commits/master
-[discord-link]:       https://discord.gg/GWDRS3kU6R
-[issue-link]:         https://github.com/official-stockfish/Stockfish/issues/new?assignees=&labels=&template=BUG-REPORT.yml
-[discussions-link]:   https://github.com/official-stockfish/Stockfish/discussions/new
-[fishtest-link]:      https://tests.stockfishchess.org/tests
-[guideline-link]:     https://github.com/glinscott/fishtest/wiki/Creating-my-first-test
-[license-link]:       https://github.com/official-stockfish/Stockfish/blob/master/Copying.txt
-[programming-link]:   https://www.chessprogramming.org/Main_Page
-[programmingsf-link]: https://www.chessprogramming.org/Stockfish
-[readme-link]:        https://github.com/official-stockfish/Stockfish/blob/master/README.md
-[release-link]:       https://github.com/official-stockfish/Stockfish/releases/latest
-[src-link]:           https://github.com/official-stockfish/Stockfish/tree/master/src
-[stockfish128-logo]:  https://stockfishchess.org/images/logo/icon_128x128.png
-[uci-link]:           https://backscattering.de/chess/uci/
-[website-link]:       https://stockfishchess.org
-[website-blog-link]:  https://stockfishchess.org/blog/
-[wiki-link]:          https://github.com/official-stockfish/Stockfish/wiki
-[wiki-usage-link]:    https://github.com/official-stockfish/Stockfish/wiki/Download-and-usage
-[wiki-compile-link]:  https://github.com/official-stockfish/Stockfish/wiki/Compiling-from-source
-[wiki-commands-link]: https://github.com/official-stockfish/Stockfish/wiki/Commands
-[worker-link]:        https://github.com/glinscott/fishtest/wiki/Running-the-worker
-
-[build-badge]:        https://img.shields.io/github/actions/workflow/status/official-stockfish/Stockfish/stockfish.yml?branch=master&style=for-the-badge&label=stockfish&logo=github
-[commits-badge]:      https://img.shields.io/github/commits-since/official-stockfish/Stockfish/latest?style=for-the-badge
-[discord-badge]:      https://img.shields.io/discord/435943710472011776?style=for-the-badge&label=discord&logo=Discord
-[fishtest-badge]:     https://img.shields.io/website?style=for-the-badge&down_color=red&down_message=Offline&label=Fishtest&up_color=success&up_message=Online&url=https%3A%2F%2Ftests.stockfishchess.org%2Ftests%2Ffinished
-[license-badge]:      https://img.shields.io/github/license/official-stockfish/Stockfish?style=for-the-badge&label=license&color=success
-[release-badge]:      https://img.shields.io/github/v/release/official-stockfish/Stockfish?style=for-the-badge&label=official%20release
-[website-badge]:      https://img.shields.io/website?style=for-the-badge&down_color=red&down_message=Offline&label=website&up_color=success&up_message=Online&url=https%3A%2F%2Fstockfishchess.org
+For full details, read the copy of the GPL v3 found in the file named
+[*Copying.txt*](https://github.com/official-stockfish/Stockfish/blob/master/Copying.txt).

Top CPU Contributors.txt

@@ -1,267 +1,235 @@
-Contributors to Fishtest with >10,000 CPU hours, as of 2023-06-20.
+Contributors to Fishtest with >10,000 CPU hours, as of 2022-04-14.
 Thank you!
 
 Username                                CPU Hours     Games played
 ------------------------------------------------------------------
-noobpwnftw                               37457426       2850540907
-technologov                              14135647        742892808
-linrock                                   4423514        303254809
-mlang                                     3026000        200065824
-dew                                       1689162        100033738
-okrout                                    1578136        148855886
-pemo                                      1508508         48814305
-grandphish2                               1461406         91540343
-TueRens                                   1194790         70400852
-JojoM                                      947612         61773190
-tvijlbrief                                 796125         51897690
-sebastronomy                               742434         38218524
+noobpwnftw                               31714850       2267266129
+mlang                                     2954099        198421098
+technologov                               2324150        102449398
+dew                                       1670874         99276012
+grandphish2                               1134273         68070459
+okrout                                     901194         77738874
+TueRens                                    821388         50207666
+tvijlbrief                                 795993         51894442
+pemo                                       744463         32486677
+JojoM                                      724378         43660674
 mibere                                     703840         46867607
-gvreuls                                    651026         42988582
-oz                                         543438         39314736
-cw                                         517858         34869755
-fastgm                                     503862         30260818
-leszek                                     467278         33514883
-CSU_Dynasty                                464940         31177118
-ctoks                                      434416         28506889
+linrock                                    626939         17408017
+gvreuls                                    534079         34352532
+cw                                         507221         34006775
+fastgm                                     489749         29344518
 crunchy                                    427035         27344275
-maximmasiutin                              424795         26577722
-bcross                                     415722         29060963
-olafm                                      395922         32268020
-rpngn                                      348378         24560289
-velislav                                   342567         22138992
+CSU_Dynasty                                424643         28525220
+ctoks                                      415771         27364603
+oz                                         369200         27017658
+bcross                                     342642         23671289
 Fisherman                                  327231         21829379
-mgrabiak                                   300612         20608380
-Dantist                                    296386         18031762
-nordlandia                                 246201         16189678
-robal                                      241300         15656382
-marrco                                     234581         17714473
-ncfish1                                    227517         15233777
-glinscott                                  208125         13277240
-drabel                                     204167         13930674
-mhoram                                     202894         12601997
+velislav                                   325670         20911076
+leszek                                     321295         19874113
+Dantist                                    274747         16910258
+mgrabiak                                   237604         15418700
+robal                                      217959         13840386
+glinscott                                  217799         13780820
+nordlandia                                 211692         13484886
+drabel                                     201967         13798360
 bking_US                                   198894         11876016
+mhoram                                     194862         12261809
 Thanar                                     179852         12365359
 vdv                                        175544          9904472
 spams                                      157128         10319326
+rpngn                                      154081          9652139
+marrco                                     150300          9402229
 sqrt2                                      147963          9724586
-DesolatedDodo                              146350          9536172
-Calis007                                   143165          9478764
-vdbergh                                    138650          9064413
+vdbergh                                    137430          8955097
 CoffeeOne                                  137100          5024116
-armo9494                                   136191          9460264
 malala                                     136182          8002293
 xoto                                       133759          9159372
-davar                                      129023          8376525
-DMBK                                       122960          8980062
+davar                                      125240          8117121
 dsmith                                     122059          7570238
-amicic                                     119661          7938029
+amicic                                     119659          7937885
 Data                                       113305          8220352
 BrunoBanani                                112960          7436849
-CypressChess                               108331          7759788
-skiminki                                   107583          7218170
-jcAEie                                     105675          8238962
+CypressChess                               108321          7759588
+DesolatedDodo                              106811          6776980
 MaZePallas                                 102823          6633619
 sterni1971                                 100532          5880772
 sunu                                       100167          7040199
-zeryl                                       99331          6221261
-thirdlife                                   99124          2242380
 ElbertoOne                                  99028          7023771
-cuistot                                     98853          6069816
-bigpen0r                                    94809          6529203
+skiminki                                    98123          6478402
 brabos                                      92118          6186135
-Wolfgang                                    91939          6105872
+cuistot                                     90358          5351004
 psk                                         89957          5984901
-sschnee                                     88235          5268000
-racerschmacer                               85805          6122790
-Fifis                                       85722          5709729
-Dubslow                                     84986          6042456
+racerschmacer                               85712          6119648
 Vizvezdenec                                 83761          5344740
+zeryl                                       83680          5250995
+sschnee                                     83003          4840890
 0x3C33                                      82614          5271253
 BRAVONE                                     81239          5054681
 nssy                                        76497          5259388
-jromang                                     76106          5236025
 teddybaer                                   75125          5407666
-tolkki963                                   74762          5149662
-megaman7de                                  74351          4940352
-Wencey                                      74181          4711488
+jromang                                     74796          5175825
 Pking_cda                                   73776          5293873
-yurikvelo                                   73150          5004382
-markkulix                                   72607          5304642
-Bobo1239                                    70579          4794999
+Calis007                                    72477          4088576
 solarlight                                  70517          5028306
 dv8silencer                                 70287          3883992
+Bobo1239                                    68515          4652287
 manap                                       66273          4121774
+yurikvelo                                   65716          4457300
 tinker                                      64333          4268790
+Wolfgang                                    62644          3817410
 qurashee                                    61208          3429862
-Mineta                                      59357          4418202
-Spprtr                                      58723          3911011
-AGI                                         58147          4325994
 robnjr                                      57262          4053117
 Freja                                       56938          3733019
-MaxKlaxxMiner                               56879          3423958
-MarcusTullius                               56746          3762951
 ttruscott                                   56010          3680085
 rkl                                         55132          4164467
 renouve                                     53811          3501516
-javran                                      53785          4627608
+megaman7de                                  52434          3243016
+MaxKlaxxMiner                               51977          3153032
 finfish                                     51360          3370515
 eva42                                       51272          3599691
-eastorwest                                  51117          3454811
+eastorwest                                  51058          3451555
 rap                                         49985          3219146
-pb00067                                     49733          3298934
-OuaisBla                                    48626          3445134
+pb00067                                     49727          3298270
+Spprtr                                      48920          3161711
+bigpen0r                                    47667          3336927
 ronaldjerum                                 47654          3240695
 biffhero                                    46564          3111352
+Fifis                                       45843          3088497
 VoyagerOne                                  45476          3452465
-jmdana                                      44893          3065205
-maposora                                    44597          4039578
-oryx                                        44570          3454238
 speedycpu                                   43842          3003273
 jbwiebe                                     43305          2805433
-GPUex                                       42378          3133332
 Antihistamine                               41788          2761312
 mhunt                                       41735          2691355
 homyur                                      39893          2850481
 gri                                         39871          2515779
-Garf                                        37741          2999686
+armo9494                                    39064          2832326
+oryx                                        38867          2976992
 SC                                          37299          2731694
+Garf                                        37213          2986270
+tolkki963                                   37059          2154330
 csnodgrass                                  36207          2688994
+jmdana                                      36157          2210661
 strelock                                    34716          2074055
-szupaw                                      34102          2880346
+DMBK                                        34010          2482916
 EthanOConnor                                33370          2090311
 slakovv                                     32915          2021889
-Gelma                                       31771          1551204
-gopeto                                      31671          2060990
-kdave                                       31157          2198362
+gopeto                                      30993          2028106
 manapbk                                     30987          1810399
 Prcuvu                                      30377          2170122
 anst                                        30301          2190091
 jkiiski                                     30136          1904470
-spcc                                        29925          1901692
 hyperbolic.tom                              29840          2017394
 chuckstablers                               29659          2093438
 Pyafue                                      29650          1902349
-belzedar94                                  28846          1811530
+ncfish1                                     29105          1704011
+belzedar94                                  27935          1789106
+OuaisBla                                    27636          1578800
 chriswk                                     26902          1868317
-xwziegtm                                    26897          2124586
 achambord                                   26582          1767323
 Patrick_G                                   26276          1801617
 yorkman                                     26193          1992080
-Ulysses                                     25288          1689730
 SFTUser                                     25182          1675689
 nabildanial                                 24942          1519409
 Sharaf_DG                                   24765          1786697
-Maxim                                       24705          1502062
-rodneyc                                     24376          1416402
+rodneyc                                     24275          1410450
 agg177                                      23890          1395014
-Goatminola                                  23763          1956036
-Ente                                        23639          1671638
-Jopo12321                                   23467          1483172
 JanErik                                     23408          1703875
 Isidor                                      23388          1680691
-Norabor                                     23371          1603244
-cisco2015                                   22920          1763301
-jsys14                                      22824          1591906
+Norabor                                     23339          1602636
+Ente                                        23270          1651432
+cisco2015                                   22897          1762669
+MarcusTullius                               22688          1274821
 Zirie                                       22542          1472937
 team-oh                                     22272          1636708
-Roady                                       22220          1465606
 MazeOfGalious                               21978          1629593
-sg4032                                      21947          1643353
+sg4032                                      21947          1643265
 ianh2105                                    21725          1632562
 xor12                                       21628          1680365
 dex                                         21612          1467203
 nesoneg                                     21494          1463031
-user213718                                  21454          1404128
+Roady                                       21323          1433822
 sphinx                                      21211          1384728
-AndreasKrug                                 21097          1634811
+user213718                                  21196          1397710
+spcc                                        21065          1311338
 jjoshua2                                    21001          1423089
-Zake9298                                    20938          1565848
 horst.prack                                 20878          1465656
 0xB00B1ES                                   20590          1208666
 j3corre                                     20405           941444
+kdave                                       20364          1389254
 Adrian.Schmidt123                           20316          1281436
+Ulysses                                     20217          1351500
+markkulix                                   19976          1115258
 wei                                         19973          1745989
-notchris                                    19958          1800128
-Serpensin                                   19840          1697528
-Gaster319                                   19712          1677310
-fishtester                                  19617          1257388
 rstoesser                                   19569          1293588
 eudhan                                      19274          1283717
-votoanthuan                                 19108          1609992
+fishtester                                  18995          1238686
 vulcan                                      18871          1729392
-Karpovbot                                   18766          1053178
-qoo_charly_cai                              18543          1284937
 jundery                                     18445          1115855
+iisiraider                                  18247          1101015
 ville                                       17883          1384026
 chris                                       17698          1487385
 purplefishies                               17595          1092533
-dju                                         17414           981289
-iisiraider                                  17275          1049015
+dju                                         17353           978595
+Wencey                                      17125           805964
 DragonLord                                  17014          1162790
-redstone59                                  16842          1461780
-Alb11747                                    16787          1213926
+thirdlife                                   16996           447356
 IgorLeMasson                                16064          1147232
-Karby                                       15982           979610
-scuzzi                                      15757           968735
 ako027ako                                   15671          1173203
+AndreasKrug                                 15550          1194497
 Nikolay.IT                                  15154          1068349
 Andrew Grant                                15114           895539
-Naven94                                     15054           834762
+scuzzi                                      14928           953313
 OssumOpossum                                14857          1007129
-ZacHFX                                      14783          1021842
+Karby                                       14808           867120
+jsys14                                      14652           855642
 enedene                                     14476           905279
-bpfliegel                                   14233           882523
+bpfliegel                                   14298           884523
 mpx86                                       14019           759568
 jpulman                                     13982           870599
-Skiff84                                     13826           721996
 crocogoat                                   13803          1117422
+joster                                      13794           950160
 Nesa92                                      13786          1114691
-joster                                      13710           946160
 mbeier                                      13650          1044928
 Hjax                                        13535           915487
-Nullvalue                                   13468          1140498
 Dark_wizzie                                 13422          1007152
+Jopo12321                                   13367           678852
 Rudolphous                                  13244           883140
-pirt                                        13100          1009897
 Machariel                                   13010           863104
-infinigon                                   12991           943216
 mabichito                                   12903           749391
 thijsk                                      12886           722107
 AdrianSA                                    12860           804972
+infinigon                                   12807           937332
 Flopzee                                     12698           894821
-korposzczur                                 12606           838168
 fatmurphy                                   12547           853210
 SapphireBrand                               12416           969604
-Oakwen                                      12399           844109
-deflectooor                                 12386           579392
 modolief                                    12386           896470
 Farseer                                     12249           694108
-Jackfish                                    12213           805008
 pgontarz                                    12151           848794
-dbernier                                    12103           860824
-getraideBFF                                 12072          1024966
+pirt                                        12008           923149
 stocky                                      11954           699440
 mschmidt                                    11941           803401
-MooTheCow                                   11870           773598
-FormazChar                                  11766           885707
-whelanh                                     11557           245188
-3cho                                        11494          1031076
+dbernier                                    11609           818636
+Maxim                                       11543           836024
 infinity                                    11470           727027
-aga                                         11412           695127
+aga                                         11409           695071
 torbjo                                      11395           729145
 Thomas A. Anderson                          11372           732094
 savage84                                    11358           670860
+FormazChar                                  11349           850327
 d64                                         11263           789184
-ali-al-zhrani                               11245           779246
+MooTheCow                                   11237           720174
 snicolet                                    11106           869170
-dapper                                      11032           771402
-ols                                         10947           624903
-Karmatron                                   10828           677458
+ali-al-zhrani                               11098           768494
+whelanh                                     11067           235676
+Jackfish                                    10978           720078
+deflectooor                                 10886           520116
 basepi                                      10637           744851
 Cubox                                       10621           826448
 michaelrpg                                  10509           739239
 OIVAS7572                                   10420           995586
-jojo2357                                    10419           929708
-WoodMan777                                  10380           873720
-Garruk                                      10365           706465
 dzjp                                        10343           732529
+Garruk                                      10334           704065
+ols                                         10259           570669
+lbraesch                                    10252           647825
+qoo_charly_cai                              10212           620407
+Naven94                                     10069           503192

docs/binpack.md

@@ -0,0 +1,42 @@
+# Binpack
+
+Binpack is a binary training data storage format designed to take advantage of position chains differing by a single move. Therefore it is very good at compactly storing data generated from real games (as opposed to random positions for example sourced from an opening book).
+
+It is currently implemented through a single header library in `extra/nnue_data_binpack_format.h`.
+
+Below follows a rough description of the format in a BNF-like notation.
+
+```
+[[nodiscard]] std::uint16_t signedToUnsigned(std::int16_t a) {
+    std::uint16_t r;
+    std::memcpy(&r, &a, sizeof(std::uint16_t));
+    if (r & 0x8000) r ^= 0x7FFF; // flip value bits if negative
+    r = (r << 1) | (r >> 15); // store sign bit at bit 0
+    return r;
+}
+
+file := <block>*
+block := BINP<chain>*
+chain := <stem><movetext>
+stem := <pos><move><score><ply_and_result><rule50> (32 bytes)
+pos := https://github.com/Sopel97/nnue_data_compress/blob/master/src/chess/Position.h#L1166 (24 bytes)
+move := https://github.com/Sopel97/nnue_data_compress/blob/master/src/chess/Chess.h#L1044 (2 bytes)
+score := signedToUnsigned(score) (2 bytes, big endian)
+ply_and_result := ply bitwise_or (signedToUnsigned(result) << 14) (2 bytes, big endian)
+rule50 := rule_50_counter (2 bytes, big endian)
+    // this is a small defect from old version,
+    I didn't want to break backwards compatibility. Effectively means that there's
+    one byte left for something else in the future because rule50 always fits in one byte.
+
+movetext := <count><move_and_score>*
+count := number of plies in the movetext (2 bytes, big endian). Can be 0.
+move_and_score := <encoded_move><encoded_score> (~2 bytes)
+encoded_move := oof this one is complicated to explain.
+    https://github.com/Sopel97/nnue_data_compress/blob/master/src/compress_file.cpp#L827.
+    https://github.com/Sopel97/chess_pos_db/blob/master/docs/bcgn/variable_length.md
+
+encoded_score := https://en.wikipedia.org/wiki/Variable-width_encoding
+    with block size of 4 bits + 1 bit for extension bit.
+    Encoded value is signedToUnsigned(-prev_score - current_score)
+    (scores are always seen from the perspective of side to move in <pos>, that's why the '-' before prev_score)
+```

docs/convert.md

@@ -0,0 +1,18 @@
+# Convert
+
+`convert` allows conversion of training data between any of `.plain`, `.bin`, and `.binpack`.
+
+As all commands in stockfish `convert` can be invoked either from command line (as `stockfish.exe convert ...`) or in the interactive prompt.
+
+The syntax of this command is as follows:
+```
+convert from_path to_path [append] [validate]
+```
+
+`from_path` is the path to the file to convert from. The type of the data is deduced based on its extension (one of `.plain`, `.bin`, `.binpack`).
+`to_path` is the path to an output file. The type of the data is deduced from its extension. If the file does not exist it is created.
+
+`append` and `validate` can come in any order and are optional.
+If `append` not specified then the output file will be truncated prior to any writes. If `append` is specified then the converted training data will be appended to the end of the output file.
+
+If `validate` is specified then the conversion will stop on the first illegal move found and a diagnostic will be shown.

docs/generate_training_data.md

@@ -0,0 +1,67 @@
+# generate_training_data
+
+`generate_training_data` command allows generation of training data from self-play in a manner that suits training better than traditional games. It introduces random moves to diversify openings, and fixed depth evaluation.
+
+As all commands in stockfish `generate_training_data` can be invoked either from command line (as `stockfish.exe generate_training_data ...`, but this is not recommended because it's not possible to specify UCI options before `generate_training_data` executes) or in the interactive prompt.
+
+It is recommended to set the `PruneAtShallowDepth` UCI option to `false` as it will increase the quality of fixed depth searches.
+
+It is recommended to keep the `EnableTranspositionTable` UCI option at the default `true` value as it will make the generation process faster without noticably harming the uniformity of the data.
+
+`generate_training_data` takes named parameters in the form of `generate_training_data param_1_name param_1_value param_2_name param_2_value ...`.
+
+Currently the following options are available:
+
+`set_recommended_uci_options` - this is a modifier not a parameter, no value follows it. If specified then some UCI options are set to recommended values.
+
+`depth` - sets minimum and maximum depth of evaluation of each position. Default: 3.
+
+`mindepth` - minimum depth of evaluation of each position. If not specified then the same as `depth`.
+
+`maxdepth` - minimum depth of evaluation of each position. If not specified then the same as `depth`.
+
+`nodes` - the number of nodes to use for evaluation of each position. This number is multiplied by the number of PVs of the current search. This does NOT override the `depth` and `depth2` options. If specified then whichever of depth or nodes limit is reached first applies.
+
+`count` - the number of training data entries to generate. 1 entry == 1 position. If both `count` and `max_time_*` are specified the data generation process will end when any of conditions is fullfilled. Default: 8000000000 (8B).
+
+`max_time_seconds`, `max_time_minutes`, `max_time_hours` - specifies the maximum runtime for the data generation. The data generation will NOT be interrupted while a self-play game is in progress. If both `count` and `max_time_*` are specified the data generation process will end when any of conditions is fullfilled. Default: \~250 years.
+
+`output_file_name` - the name of the file to output to. If the extension is not present or doesn't match the selected training data format the right extension will be appened. Default: generated_kifu
+
+`eval_limit` - evaluations with higher absolute value than this will not be written and will terminate a self-play game. Should not exceed 10000 which is VALUE_KNOWN_WIN, but is only hardcapped at mate in 2 (\~30000). Default: 3000
+
+`random_move_min_ply` - the minimal ply at which a random move may be executed instead of a move chosen by search. Default: 1.
+
+`random_move_max_ply` - the maximal ply at which a random move may be executed instead of a move chosen by search. Default: 24.
+
+`random_move_count` - maximum number of random moves in a single self-play game. Default: 5.
+
+`random_move_like_apery` - either 0 or 1. If 1 then random king moves will be followed by a random king move from the opponent whenever possible with 50% probability. Default: 0.
+
+`random_multi_pv` - the number of PVs used for determining the random move. If not specified then a truly random move will be chosen. If specified then a multiPV search will be performed the random move will be one of the moves chosen by the search.
+
+`random_multi_pv_diff` - Makes the multiPV random move selection consider only moves that are at most `random_multi_pv_diff` worse than the next best move. Default: 30000 (all multiPV moves).
+
+`random_multi_pv_depth` - the depth to use for multiPV search for random move. Default: `depth2`.
+
+`random_multi_pv_nodes` - the maximum number of nodes for a multiPV search for random move. Default: `nodes`.
+
+`write_min_ply` - minimum ply for which the training data entry will be emitted. Default: 16.
+
+`write_max_ply` - maximum ply for which the training data entry will be emitted. Default: 400.
+
+`book` - a path to an opening book to use for the starting positions. Currently only .epd format is supported. If not specified then the starting position is always the standard chess starting position.
+
+`save_every` - the number of training data entries per file. If not specified then there will be always one file. If specified there may be more than one file generated (each having at most `save_every` training data entries) and each file will have a unique number attached.
+
+`random_file_name` - if specified then the output filename will be chosen randomly. Overrides `output_file_name`.
+
+`keep_draws` - either 0 or 1. If 1 then training data from drawn games will be emitted too. Default: 1.
+
+`adjudicate_draws_by_score` - either 0 or 1. If 1 then drawn games will be adjudicated when the score remains 0 for at least 8 plies after ply 80. Default: 1.
+
+`adjudicate_draws_by_insufficient_mating_material` - either 0 or 1. If 1 then position with insufficient material will be adjudicated as draws. Default: 1.
+
+`data_format` - format of the training data to use. Either `bin` or `binpack`. Default: `binpack`.
+
+`seed` - seed for the PRNG. Can be either a number or a string. If it's a string then its hash will be used. If not specified then the current time will be used.

docs/generate_training_data_nonpv.md

@@ -0,0 +1,41 @@
+# generate_training_data_nonpv
+
+`generate_training_data_nonpv` command allows generation of training data from self-play in a manner that suits training better than traditional games. It plays fixed nodes self play games for exploration and records [some of] the evaluated positions. Then rescores them with fixed depth search.
+
+As all commands in stockfish `generate_training_data_nonpv` can be invoked either from command line (as `stockfish.exe generate_training_data_nonpv ...`, but this is not recommended because it's not possible to specify UCI options before `generate_training_data_nonpv` executes) or in the interactive prompt.
+
+It is recommended to set the `PruneAtShallowDepth` UCI option to `false` as it will increase the quality of fixed depth searches.
+
+It is recommended to keep the `EnableTranspositionTable` UCI option at the default `true` value as it will make the generation process faster without noticably harming the uniformity of the data.
+
+`generate_training_data_nonpv` takes named parameters in the form of `generate_training_data_nonpv param_1_name param_1_value param_2_name param_2_value ...`.
+
+Currently the following options are available:
+
+`depth` - the search depth to use for rescoring. Default: 3.
+
+`count` - the number of training data entries to generate. 1 entry == 1 position. Default: 1000000 (1M).
+
+`exploration_min_nodes` - the min number of nodes to use for exploraton during selfplay. Default: 5000.
+
+`exploration_max_nodes` - the max number of nodes to use for exploraton during selfplay. The number of nodes is chosen from a uniform distribution between min and max. Default: 15000.
+
+`exploration_save_rate` - the ratio of positions seen during exploration self play games that are saved for later rescoring. Default: 0.01 (meaning 1 in 100 positions seen during search get saved for rescoring).
+
+`output_file` - the name of the file to output to. If the extension is not present or doesn't match the selected training data format the right extension will be appened. Default: generated_gensfen_nonpv
+
+`eval_limit` - evaluations with higher absolute value than this will not be written and will terminate a self-play game. Should not exceed 10000 which is VALUE_KNOWN_WIN, but is only hardcapped at mate in 2 (\~30000). Default: 4000
+
+`exploration_eval_limit` - same as `eval_limit` but used during exploration with a value from fixed depth search.
+
+`exploration_min_pieces` - the min number of pieces in the self play games to start the fixed depth search. Note that even if there's N pieces on the board the fixed nodes search usually reaches positions with less pieces and they are saved too. Default: 8.
+
+`exploration_max_ply` the max ply for the exploration self play. Default: 200.
+
+`smart_fen_skipping` - this is a flag option. When specified some position that are not good candidates for teaching are removed from the output. This includes positions where the best move is a capture or promotion, and position where a king is in check.
+
+`book` - a path to an opening book to use for the starting positions. Currently only .epd format is supported. If not specified then the starting position is always the standard chess starting position.
+
+`data_format` - format of the training data to use. Either `bin` or `binpack`. Default: `binpack`.
+
+`seed` - seed for the PRNG. Can be either a number or a string. If it's a string then its hash will be used. If not specified then the current time will be used.

docs/stats.md

@@ -0,0 +1,41 @@
+# Stats
+
+`gather_statistics` command allows gathering various statistics from a .bin or a .binpack file. The syntax is `gather_statistics (GROUP)* input_file FILENAME`. There can be many groups specified. Any statistic gatherer that belongs to at least one of the specified groups will be used.
+
+Simplest usage: `stockfish.exe gather_statistics all input_file a.binpack`
+
+Any name that doesn't designate an argument name or is not an argument will be interpreted as a group name.
+
+## Parameters
+
+`input_file` - the path to the .bin or .binpack input file to read
+
+`output_file` - optional path to the output file to write the results too. Results are always written on the console, so if this is specified the results will be written in both places.
+
+`max_count` - the maximum number of positions to process. Default: no limit.
+
+## Groups
+
+`all` - a special group designating all statistics gatherers available.
+
+`position_count` - the total number of positions in the file.
+
+`king`, `king_square_count` - the number of times a king was on each square. Output is layed out as a chessboard, with the 8th rank being the topmost. Separate values for white and black kings.
+
+`move`, `move_from_count` - same as `king_square_count` but for from_sq(move)
+
+`move`, `move_to_count` - same as `king_square_count` but for to_sq(move)
+
+`move`, `move_type` - the number of moves with each type. Includes normal, captures, castling, promotions, enpassant. The groups are not disjoint.
+
+`move`, `moved_piece_type` - the number of times a piece of each type was moved
+
+`piece_count` - the histogram of the number of pieces on the board
+
+`ply_discontinuities` - the number of times the ply jumped by a value different than 1 between two consecutive positions. Usually the number of games.
+
+`material_imbalance` - the histogram of imbalances, with values computed using "simple eval", that is pawn=1, bishop=knight=3, rook=5, queen=9
+
+`results` - the distribution of game results
+
+`endgames_6man` - distribution of endgame configurations for <=6 pieces (including kings)

docs/transform.md

@@ -0,0 +1,42 @@
+# Transform
+
+`transform` command exposes subcommands that perform some specific transformation over data. The call syntax is `transform <subcommand>`. Currently implemented subcommands are listed and described below.
+
+## `nudged_static`
+
+`transform nudged_static` takes named parameters in the form of `transform nudged_static param_1_name param_1_value param_2_name param_2_value ...` and flag parameters which don't require values.
+
+This command goes through positions in the input files and replaces the scores with new ones - generated from static eval - but slightly adjusted based on the scores in the original input file.
+
+Currently the following options are available:
+
+`input_file` - path to the input file. Supports bin and binpack formats. Default: in.binpack.
+
+`output_file` - path to the output file. Supports bin and binpack formats. Default: out.binpack.
+
+`absolute` - states that the adjustment should be bounded by an absolute value. After this token follows the maximum absolute adjustment. Values are always adjusted towards scores in the input file. This is the default mode. Default maximum adjustement: 5.
+
+`relative` - states that the adjustment should be bounded by a value relative in magnitude to the static eval value. After this token follows the maximum relative change - a floating point value greater than 0. For example a value of 0.1 only allows changing the static eval by at most 10% towards the score from the input file.
+
+`interpolate` states that the output score should be a value interpolated between static eval and the score from the input file. After this token follows the interpolation constant `t`. `t` of 0 means that only static eval is used. `t` of 1 means that only score from the input file is used. `t` of 0.5 means that the static eval and input score are averaged. It accepts values outside of range `<0, 1>`, but the usefulness is questionable.
+
+## `rescore`
+
+`transform rescore` takes named parameters in the form of `transform rescore param_1_name param_1_value param_2_name param_2_value ...` and flag parameters which don't require values.
+
+This tool respects the UCI option `Threads` and uses all available threads.
+
+This command takes a path to the input file that is either a .epd file which contains one FEN per line or a .bin or .binpack file and outputs a .bin or .binpack file with these positions rescored with specified depth search.
+
+Currently the following options are available:
+
+`input_file` - path to the input file. Default: in.binpack.
+
+`output_file` - path to the output .bin or .binpack file. The file is opened in append mode. Default: out.binpack.
+
+`depth` - the search depth to use for rescoring. Default: 3.
+
+`keep_moves` - whether to keep moves from the input file if available. Allows to keep compression in .binpack. Default: 1.
+
+`research_count` - number of additional searches of depth N done on the same position before using the eval. Default: 0.
+

docs/validate_training_data.md

@@ -0,0 +1,12 @@
+# validate_training_data
+
+`validate_training_data` allows validation of training data of types `.plain`, `.bin`, and `.binpack`.
+
+As all commands in stockfish `validate_training_data` can be invoked either from command line (as `stockfish.exe validate_training_data ...`) or in the interactive prompt.
+
+The syntax of this command is as follows:
+```
+validate_training_data in_path
+```
+
+`in_path` is the path to the file to validate. The type of the data is deduced based on its extension (one of `.plain`, `.bin`, `.binpack`).

script/README.md

@@ -0,0 +1,52 @@
+# `pgn_to_plain`
+This script converts pgn files into text file to apply `learn convert_bin` command. You need to import [python-chess](https://pypi.org/project/python-chess/) to use this script.
+
+
+    pip install python-chess
+	
+
+# Example of Qhapaq's finetune using `pgn_to_plain`
+
+## Download data
+You can download data from [here](http://rebel13.nl/index.html)
+
+## Convert pgn files
+
+**Important : convert text will be superheavy (approx 200 byte / position)**
+
+    python pgn_to_plain.py --pgn "pgn/*.pgn" --start_ply 1 --output converted_pgn.txt
+
+
+`--pgn` option supports wildcard. When you use pgn files with elo >= 3300, You will get 1.7 GB text file.
+	
+	
+## Convert into training data
+
+
+### Example build command
+
+    make nnue-learn ARCH=x86-64
+
+See `src/Makefile` for detail.
+
+
+### Convert
+
+    ./stockfish
+    learn convert_bin converted_pgn.txt output_file_name pgn_bin.bin
+	learn shuffle pgn_bin.bin
+	
+You also need to prepare validation data for training like following.
+	
+	python pgn_to_plain.py --pgn "pgn/ccrl-40-15-3400.pgn" --start_ply 1 --output ccrl-40-15-3400.txt
+	./stockfish
+    learn convert_bin ccrl-40-15-3400.txt ccrl-40-15-3400_plain.bin
+	
+	
+### Learn
+
+    ./stockfish
+	setoption name Threads value 8
+    learn shuffled_sfen.bin newbob_decay 0.5  validation_set_file_name ccrl-40-15-3400_plain.bin  nn_batch_size 50000 batchsize 1000000 eval_save_interval 8000000 eta 0.05 lambda 0.0 eval_limit 3000 mirror_percentage 0 use_draw_in_training 1
+
+

script/extract_bin.py

@@ -0,0 +1,42 @@
+import sys
+
+ENTRY_SIZE = 40
+NUM_ENTRIES_IN_CHUNK = 1024*1024
+
+def copy(infile, outfile, count, times):
+    if times > 1:
+        outfile.write(infile.read(count*ENTRY_SIZE)*times)
+    else:
+        offset = 0
+        while offset < count:
+            to_read = NUM_ENTRIES_IN_CHUNK if offset + NUM_ENTRIES_IN_CHUNK <= count else count - offset
+
+            outfile.write(infile.read(to_read*ENTRY_SIZE))
+
+            offset += NUM_ENTRIES_IN_CHUNK
+
+def work():
+    filename = sys.argv[1]
+    offset = int(sys.argv[2])
+    count = int(sys.argv[3])
+    times = int(sys.argv[4]) if len(sys.argv) >= 5 else 1
+
+    with open(filename, 'rb') as infile:
+        infile.seek(offset * ENTRY_SIZE)
+        filename_parts = filename.split('.')
+        out_path = '.'.join(filename_parts[:-1]) + '_' + str(offset) + '_' + str(count) + '_' + str(times) + '.' + filename_parts[-1]
+        with open(out_path, 'wb') as outfile:
+            copy(infile, outfile, count, times)
+
+def show_help():
+    print('Usage: python extract_bin.py filename offset count [times]')
+    print('filename - the path to the .bin file to process')
+    print('offset - the number of sfens to skip')
+    print('count - the number of sfens to extract')
+    print('times - the number of times to repeat the extracted sfens. Default = 1')
+    print('The result is saved in a new file named `filename.stem`_`offset`_`count`_`times`.bin')
+
+if len(sys.argv) < 4:
+    show_help()
+else:
+    work()

script/interleave_binpacks.py

@@ -0,0 +1,86 @@
+import struct
+import sys
+import os
+import random
+from pathlib import Path
+
+
+def copy_next_chunk(in_file, out_file):
+    chunk_header = in_file.read(8)
+    assert chunk_header[0:4] == b"BINP"
+    size = struct.unpack("<I", chunk_header[4:])[0]
+
+    out_file.write(chunk_header)
+    data = in_file.read(size)
+    out_file.write(data)
+
+    return size + 8
+
+
+def main():
+    if len(sys.argv) < 4:
+        print("Usage: python interleave_binpacks.py infile1 ... infileN outfile")
+        print("       The output binpack, will contain all data from the input files.")
+        print("       Data is read sequentially from the input, randomly alternating between files.")
+        return
+
+    # open last arg as output file name
+    out_filename = sys.argv[-1]
+    print("outfile: ", out_filename)
+
+    if Path(out_filename).exists():
+        print(
+            "Output path {} already exists. Please specify a path to a file that does not exist.".format(
+                out_filename
+            )
+        )
+        return
+
+    out_file = open(out_filename, "wb")
+
+    # open other args as input file names, and get their sizes
+    in_filenames = []
+    for i in range(1, len(sys.argv) - 1):
+        in_filenames.append(sys.argv[i])
+    print("infiles: ", in_filenames)
+
+    in_files = []
+    in_files_remaining = []
+    for in_filename in in_filenames:
+        in_file = open(in_filename, "rb")
+        in_files.append(in_file)
+        file_size = os.path.getsize(in_filename)
+        in_files_remaining.append(file_size)
+
+    # randomly pick a file, with a probability related to their sizes.
+    # copy from the front and keep track of remaining sizes
+    total_remaining = sum(in_files_remaining)
+    print("Merging {} bytes ".format(total_remaining))
+
+    total_size = 0
+    report_every = 100
+    prev_mib = -report_every
+
+    while total_remaining > 0:
+        where = random.randrange(total_remaining)
+        i = 0
+        while where >= in_files_remaining[i]:
+            where -= in_files_remaining[i]
+            i += 1
+        size = copy_next_chunk(in_files[i], out_file)
+        in_files_remaining[i] -= size
+        total_remaining -= size
+        total_size += size
+        mib = total_size // 1024 // 1024
+        if mib // 100 != prev_mib // 100:
+            print("Copied {} MiB".format(mib))
+            prev_mib = mib
+
+    out_file.close()
+    for in_file in in_files:
+        in_file.close()
+
+    print("Merged  {} bytes".format(total_size))
+
+
+main()

script/pgn_to_plain.py

@@ -0,0 +1,110 @@
+import chess.pgn
+import argparse
+import glob
+import re
+from typing import List
+
+# todo close in c++ tools using pgn-extract
+# https://www.cs.kent.ac.uk/people/staff/djb/pgn-extract/help.html#-w
+
+commentRe = re.compile("([+-]*M*[0-9.]*)/([0-9]*)")
+mateRe = re.compile("([+-])M([0-9]*)")
+flip_black = False
+
+def parse_result(result_str:str, board:chess.Board) -> int:
+    if result_str == "1/2-1/2":
+        return 0
+    if result_str == "0-1":
+        if board.turn == chess.WHITE:
+            return -1
+        else:
+            return 1
+    elif result_str == "1-0":
+        if board.turn == chess.WHITE:
+            return 1
+        else:
+            return -1
+    else:
+        print("illegal result", result_str)
+        raise ValueError
+
+def game_sanity_check(game: chess.pgn.Game) -> bool:
+    if not game.headers["Result"] in ["1/2-1/2", "0-1", "1-0"]:
+        print("invalid result", game.headers["Result"])
+        return False
+    return True
+
+def parse_comment_for_score(comment_str: str, board: chess.Board) -> int:
+    global commentRe
+    global mateRe
+    global flip_black
+
+    try:
+      m = commentRe.search(comment_str)
+      if m:
+         score = m.group(1)
+         # depth = int(m.group(2))
+         m = mateRe.search(score)
+         if m:
+            if m.group(1) == "+":
+               score =  32000 - int(m.group(2))
+            else:
+               score = -32000 + int(m.group(2))
+         else:
+            score = int(float(score) * 208) # pawn to SF PawnValueEg
+
+         if flip_black and board.turn == chess.BLACK:
+            score = -score
+      else:
+         score = 0
+    except:
+      score = 0
+
+    return score
+
+def parse_game(game: chess.pgn.Game, writer, start_play: int=1)->None:
+    board: chess.Board = game.board()
+    if not game_sanity_check(game):
+        return
+
+    result: str = game.headers["Result"]
+    ply = 0
+    for node in game.mainline():
+        move = node.move
+        if ply >= start_play:
+            comment: str = node.comment
+            writer.write("fen " + board.fen() + "\n")
+            writer.write("move " + str(move) + "\n")
+            writer.write("score " + str(parse_comment_for_score(comment, board)) + "\n")
+            writer.write("ply " + str(ply)+"\n")
+            writer.write("result " + str(parse_result(result, board)) +"\n")
+            writer.write("e\n")
+        ply += 1
+        board.push(move)
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--pgn", type=str, required=True)
+    parser.add_argument("--start_ply", type=int, default=1)
+    parser.add_argument("--output", type=str, default="plain.txt")
+    parser.add_argument("--flip_black_score", action='store_true', dest='flip_black_score', help="Flip black score. Default: False")
+    args = parser.parse_args()
+
+    global flip_black
+    flip_black = args.flip_black_score
+
+    pgn_files: List[str] = glob.glob(args.pgn)
+    pgn_files = sorted(pgn_files, key=lambda x:float(re.findall("-(\d+).pgn",x)[0] if re.findall("-(\d+).pgn",x) else 0.0))
+    f = open(args.output, 'w')
+    for pgn_file in pgn_files:
+        print("parse", pgn_file)
+        pgn_loader = open(pgn_file)
+        while True:
+            game = chess.pgn.read_game(pgn_loader)
+            if game is None:
+                break
+            parse_game(game, f, args.start_ply)
+    f.close()
+
+if __name__=="__main__":
+    main()

script/shuffle_binpack.py

@@ -0,0 +1,89 @@
+import struct
+import sys
+import os
+import random
+from pathlib import Path
+
+def index_binpack(file):
+    print('Indexing...')
+    index = []
+    offset = 0
+    report_every = 100
+    prev_mib = -report_every
+    while file.peek():
+        chunk_header = file.read(8)
+        assert chunk_header[0:4] == b'BINP'
+        size = struct.unpack('<I', chunk_header[4:])[0]
+        file.seek(size, os.SEEK_CUR)
+        index.append((offset, size + 8))
+        offset += size + 8
+
+        mib = offset // 1024 // 1024
+        if mib // 100 != prev_mib // 100:
+            print('Indexed {} MiB'.format(mib))
+            prev_mib = mib
+
+    return index
+
+def copy_binpack_indexed(in_file, index, out_files):
+    print('Copying...')
+    total_size = 0
+    report_every = 100
+    prev_mib = -report_every
+    nextfile = 0
+    for offset, size in index:
+        in_file.seek(offset, os.SEEK_SET)
+        data = in_file.read(size)
+        assert len(data) == size
+        out_files[nextfile].write(data)
+        nextfile = (nextfile + 1) % len(out_files)
+
+        total_size += size
+        mib = total_size // 1024 // 1024
+        if mib // 100 != prev_mib // 100:
+            print('Copied {} MiB'.format(mib))
+            prev_mib = mib
+
+def main():
+    if len(sys.argv) < 3:
+        print('Usage: python shuffle_binpack.py infile outfile [split_count]')
+        return
+
+    in_filename = sys.argv[1]
+
+    if len(sys.argv) > 3:
+       # split the infile in split_count pieces, creating new outfile names based on the provided name
+       basefile = sys.argv[2]
+       split_count = int(sys.argv[3])
+       base=os.path.splitext(basefile)[0]
+       ext=os.path.splitext(basefile)[1]
+       out_filenames = []
+       for i in range(split_count):
+           out_filenames.append(base+"_{}".format(i)+ext)
+    else:
+       out_filenames = [sys.argv[2]]
+
+    for out_filename in out_filenames:
+      if (Path(out_filename).exists()):
+          print('Output path {} already exists. Please specify a path to a file that does not exist.'.format(out_filename))
+          return
+
+    print(out_filenames)
+
+    in_file = open(in_filename, 'rb')
+    index = index_binpack(in_file)
+
+    print('Shuffling...')
+    random.shuffle(index)
+
+    out_files = []
+    for out_filename in out_filenames:
+        out_files.append(open(out_filename, 'wb'))
+
+    copy_binpack_indexed(in_file, index, out_files)
+
+    in_file.close()
+    for out_file in out_files:
+        out_file.close()
+
+main()

src/Makefile

@@ -1,5 +1,5 @@
 # Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-# Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+# Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 #
 # Stockfish is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -44,6 +44,12 @@ else
 	EXE = stockfish
 endif
 
+### Establish the operating system name
+KERNEL = $(shell uname -s)
+ifeq ($(KERNEL),Linux)
+	OS = $(shell uname -o)
+endif
+
 ### Installation dir definitions
 PREFIX = /usr/local
 BINDIR = $(PREFIX)/bin
@@ -51,51 +57,61 @@ BINDIR = $(PREFIX)/bin
 ### Built-in benchmark for pgo-builds
 ifeq ($(SDE_PATH),)
 	PGOBENCH = $(WINE_PATH) ./$(EXE) bench
+	PGOGENSFEN = $(WINE_PATH) ./$(EXE) gensfen depth 3 loop 1000 sfen_format bin output_file_name $(PGO_TRAINING_DATA_FILE)
 else
 	PGOBENCH = $(SDE_PATH) -- $(WINE_PATH) ./$(EXE) bench
+	PGOGENSFEN = $(SDE_PATH) -- $(WINE_PATH) ./$(EXE) gensfen depth 3 loop 1000 sfen_format bin output_file_name $(PGO_TRAINING_DATA_FILE)
 endif
 
 ### Source and object files
 SRCS = benchmark.cpp bitbase.cpp bitboard.cpp endgame.cpp evaluate.cpp main.cpp \
 	material.cpp misc.cpp movegen.cpp movepick.cpp pawns.cpp position.cpp psqt.cpp \
 	search.cpp thread.cpp timeman.cpp tt.cpp uci.cpp ucioption.cpp tune.cpp syzygy/tbprobe.cpp \
-	nnue/evaluate_nnue.cpp nnue/features/half_ka_v2_hm.cpp
+	nnue/evaluate_nnue.cpp \
+	nnue/features/half_ka_v2_hm.cpp \
+	tools/validate_training_data.cpp \
+	tools/sfen_packer.cpp \
+	tools/training_data_generator.cpp \
+	tools/training_data_generator_nonpv.cpp \
+	tools/opening_book.cpp \
+	tools/convert.cpp \
+	tools/transform.cpp \
+	tools/stats.cpp
 
 OBJS = $(notdir $(SRCS:.cpp=.o))
 
-VPATH = syzygy:nnue:nnue/features
+VPATH = syzygy:nnue:nnue/features:eval:extra:tools
 
 ### ==========================================================================
 ### Section 2. High-level Configuration
 ### ==========================================================================
 #
-# flag                --- Comp switch        --- Description
+# flag                --- Comp switch      --- Description
 # ----------------------------------------------------------------------------
 #
-# debug = yes/no      --- -DNDEBUG           --- Enable/Disable debug mode
+# debug = yes/no      --- -DNDEBUG         --- Enable/Disable debug mode
 # sanitize = none/<sanitizer> ... (-fsanitize )
-#                     --- ( undefined )      --- enable undefined behavior checks
-#                     --- ( thread    )      --- enable threading error checks
-#                     --- ( address   )      --- enable memory access checks
-#                     --- ...etc...          --- see compiler documentation for supported sanitizers
-# optimize = yes/no   --- (-O3/-fast etc.)   --- Enable/Disable optimizations
-# arch = (name)       --- (-arch)            --- Target architecture
-# bits = 64/32        --- -DIS_64BIT         --- 64-/32-bit operating system
-# prefetch = yes/no   --- -DUSE_PREFETCH     --- Use prefetch asm-instruction
-# popcnt = yes/no     --- -DUSE_POPCNT       --- Use popcnt asm-instruction
-# pext = yes/no       --- -DUSE_PEXT         --- Use pext x86_64 asm-instruction
-# sse = yes/no        --- -msse              --- Use Intel Streaming SIMD Extensions
-# mmx = yes/no        --- -mmmx              --- Use Intel MMX instructions
-# sse2 = yes/no       --- -msse2             --- Use Intel Streaming SIMD Extensions 2
-# ssse3 = yes/no      --- -mssse3            --- Use Intel Supplemental Streaming SIMD Extensions 3
-# sse41 = yes/no      --- -msse4.1           --- Use Intel Streaming SIMD Extensions 4.1
-# avx2 = yes/no       --- -mavx2             --- Use Intel Advanced Vector Extensions 2
-# avxvnni = yes/no    --- -mavxvnni          --- Use Intel Vector Neural Network Instructions AVX
-# avx512 = yes/no     --- -mavx512bw         --- Use Intel Advanced Vector Extensions 512
-# vnni256 = yes/no    --- -mavx256vnni       --- Use Intel Vector Neural Network Instructions 512 with 256bit operands
-# vnni512 = yes/no    --- -mavx512vnni       --- Use Intel Vector Neural Network Instructions 512
-# neon = yes/no       --- -DUSE_NEON         --- Use ARM SIMD architecture
-# dotprod = yes/no    --- -DUSE_NEON_DOTPROD --- Use ARM advanced SIMD Int8 dot product instructions
+#                     --- ( undefined )    --- enable undefined behavior checks
+#                     --- ( thread    )    --- enable threading error checks
+#                     --- ( address   )    --- enable memory access checks
+#                     --- ...etc...        --- see compiler documentation for supported sanitizers
+# optimize = yes/no   --- (-O3/-fast etc.) --- Enable/Disable optimizations
+# arch = (name)       --- (-arch)          --- Target architecture
+# bits = 64/32        --- -DIS_64BIT       --- 64-/32-bit operating system
+# prefetch = yes/no   --- -DUSE_PREFETCH   --- Use prefetch asm-instruction
+# popcnt = yes/no     --- -DUSE_POPCNT     --- Use popcnt asm-instruction
+# pext = yes/no       --- -DUSE_PEXT       --- Use pext x86_64 asm-instruction
+# sse = yes/no        --- -msse            --- Use Intel Streaming SIMD Extensions
+# mmx = yes/no        --- -mmmx            --- Use Intel MMX instructions
+# sse2 = yes/no       --- -msse2           --- Use Intel Streaming SIMD Extensions 2
+# ssse3 = yes/no      --- -mssse3          --- Use Intel Supplemental Streaming SIMD Extensions 3
+# sse41 = yes/no      --- -msse4.1         --- Use Intel Streaming SIMD Extensions 4.1
+# avx2 = yes/no       --- -mavx2           --- Use Intel Advanced Vector Extensions 2
+# avxvnni = yes/no    --- -mavxvnni        --- Use Intel Vector Neural Network Instructions AVX
+# avx512 = yes/no     --- -mavx512bw       --- Use Intel Advanced Vector Extensions 512
+# vnni256 = yes/no    --- -mavx512vnni     --- Use Intel Vector Neural Network Instructions 256
+# vnni512 = yes/no    --- -mavx512vnni     --- Use Intel Vector Neural Network Instructions 512
+# neon = yes/no       --- -DUSE_NEON       --- Use ARM SIMD architecture
 #
 # Note that Makefile is space sensitive, so when adding new architectures
 # or modifying existing flags, you have to make sure there are no extra spaces
@@ -117,7 +133,7 @@ ifeq ($(ARCH), $(filter $(ARCH), \
                  x86-64-vnni512 x86-64-vnni256 x86-64-avx512 x86-64-avxvnni x86-64-bmi2 \
                  x86-64-avx2 x86-64-sse41-popcnt x86-64-modern x86-64-ssse3 x86-64-sse3-popcnt \
                  x86-64 x86-32-sse41-popcnt x86-32-sse2 x86-32 ppc-64 ppc-32 e2k \
-                 armv7 armv7-neon armv8 armv8-dotprod apple-silicon general-64 general-32 riscv64))
+                 armv7 armv7-neon armv8 apple-silicon general-64 general-32))
    SUPPORTED_ARCH=true
 else
    SUPPORTED_ARCH=false
@@ -141,7 +157,6 @@ avx512 = no
 vnni256 = no
 vnni512 = no
 neon = no
-dotprod = no
 arm_version = 0
 STRIP = strip
 
@@ -310,21 +325,11 @@ ifeq ($(ARCH),armv8)
 	arm_version = 8
 endif
 
-ifeq ($(ARCH),armv8-dotprod)
-	arch = armv8
-	prefetch = yes
-	popcnt = yes
-	neon = yes
-	dotprod = yes
-	arm_version = 8
-endif
-
 ifeq ($(ARCH),apple-silicon)
 	arch = arm64
 	prefetch = yes
 	popcnt = yes
 	neon = yes
-	dotprod = yes
 	arm_version = 8
 endif
 
@@ -350,11 +355,7 @@ ifeq ($(findstring e2k,$(ARCH)),e2k)
 	popcnt = yes
 endif
 
-ifeq ($(ARCH),riscv64)
-	arch = riscv64
 endif
-endif
-
 
 ### ==========================================================================
 ### Section 3. Low-level Configuration
@@ -367,8 +368,9 @@ ifeq ($(MAKELEVEL),0)
        export ENV_LDFLAGS := $(LDFLAGS)
 endif
 
-CXXFLAGS = $(ENV_CXXFLAGS) -Wall -Wcast-qual -fno-exceptions -std=c++17 $(EXTRACXXFLAGS)
-DEPENDFLAGS = $(ENV_DEPENDFLAGS) -std=c++17
+ADDITIONAL_INCLUDE_DIRECTORIES = -I.
+CXXFLAGS = $(ENV_CXXFLAGS) -Wall -Wcast-qual -fno-exceptions -std=c++17 $(ADDITIONAL_INCLUDE_DIRECTORIES) $(EXTRACXXFLAGS)
+DEPENDFLAGS = $(ENV_DEPENDFLAGS) -std=c++17 $(ADDITIONAL_INCLUDE_DIRECTORIES)
 LDFLAGS = $(ENV_LDFLAGS) $(EXTRALDFLAGS)
 
 ifeq ($(COMP),)
@@ -378,16 +380,13 @@ endif
 ifeq ($(COMP),gcc)
 	comp=gcc
 	CXX=g++
-	CXXFLAGS += -pedantic -Wextra -Wshadow -Wmissing-declarations
+	CXXFLAGS += -pedantic -Wextra -Wshadow -lstdc++fs
 
-	ifeq ($(arch),$(filter $(arch),armv7 armv8 riscv64))
+	ifeq ($(arch),$(filter $(arch),armv7 armv8))
 		ifeq ($(OS),Android)
 			CXXFLAGS += -m$(bits)
 			LDFLAGS += -m$(bits)
 		endif
-		ifeq ($(ARCH),riscv64)
-			CXXFLAGS += -latomic
-		endif
 	else
 		CXXFLAGS += -m$(bits)
 		LDFLAGS += -m$(bits)
@@ -422,14 +421,13 @@ ifeq ($(COMP),mingw)
 			CXX=i686-w64-mingw32-c++-posix
 		endif
 	endif
-	CXXFLAGS += -pedantic -Wextra -Wshadow -Wmissing-declarations
+	CXXFLAGS += -pedantic -Wextra -Wshadow
 endif
 
-ifeq ($(COMP),icx)
-	comp=icx
-	CXX=icpx
-	CXXFLAGS += --intel -pedantic -Wextra -Wshadow -Wmissing-prototypes \
-		-Wconditional-uninitialized -Wabi -Wdeprecated
+ifeq ($(COMP),icc)
+	comp=icc
+	CXX=icpc
+	CXXFLAGS += -diag-disable 1476,10120 -Wcheck -Wabi -Wdeprecated -strict-ansi
 endif
 
 ifeq ($(COMP),clang)
@@ -439,8 +437,7 @@ ifeq ($(COMP),clang)
 		CXX=x86_64-w64-mingw32-clang++
 	endif
 
-	CXXFLAGS += -pedantic -Wextra -Wshadow -Wmissing-prototypes \
-	            -Wconditional-uninitialized
+	CXXFLAGS += -pedantic -Wextra -Wshadow
 
 	ifeq ($(filter $(KERNEL),Darwin OpenBSD FreeBSD),)
 	ifeq ($(target_windows),)
@@ -450,14 +447,11 @@ ifeq ($(COMP),clang)
 	endif
 	endif
 
-	ifeq ($(arch),$(filter $(arch),armv7 armv8 riscv64))
+	ifeq ($(arch),$(filter $(arch),armv7 armv8))
 		ifeq ($(OS),Android)
 			CXXFLAGS += -m$(bits)
 			LDFLAGS += -m$(bits)
 		endif
-		ifeq ($(ARCH),riscv64)
-			CXXFLAGS += -latomic
-		endif
 	else
 		CXXFLAGS += -m$(bits)
 		LDFLAGS += -m$(bits)
@@ -465,8 +459,8 @@ ifeq ($(COMP),clang)
 endif
 
 ifeq ($(KERNEL),Darwin)
-	CXXFLAGS += -mmacosx-version-min=10.14
-	LDFLAGS += -mmacosx-version-min=10.14
+	CXXFLAGS += -mmacosx-version-min=10.15
+	LDFLAGS += -mmacosx-version-min=10.15
 	ifneq ($(arch),any)
 		CXXFLAGS += -arch $(arch)
 		LDFLAGS += -arch $(arch)
@@ -500,9 +494,9 @@ ifeq ($(COMP),ndk)
 	LDFLAGS += -static-libstdc++ -pie -lm -latomic
 endif
 
-ifeq ($(comp),icx)
-	profile_make = icx-profile-make
-	profile_use = icx-profile-use
+ifeq ($(comp),icc)
+	profile_make = icc-profile-make
+	profile_use = icc-profile-use
 else ifeq ($(comp),clang)
 	profile_make = clang-profile-make
 	profile_use = clang-profile-use
@@ -526,7 +520,7 @@ endif
 
 ### Sometimes gcc is really clang
 ifeq ($(COMP),gcc)
-	gccversion = $(shell $(CXX) --version 2>/dev/null)
+	gccversion = $(shell $(CXX) --version)
 	gccisclang = $(findstring clang,$(gccversion))
 	ifneq ($(gccisclang),)
 		profile_make = clang-profile-make
@@ -542,13 +536,13 @@ ifneq ($(comp),mingw)
 		# Haiku has pthreads in its libroot, so only link it in on other platforms
 		ifneq ($(KERNEL),Haiku)
 			ifneq ($(COMP),ndk)
-				LDFLAGS += -lpthread
-			endif
+			LDFLAGS += -lpthread
 		endif
 	endif
 endif
+endif
 
-### 3.2.1 Debugging
+### 3.2.2 Debugging
 ifeq ($(debug),no)
 	CXXFLAGS += -DNDEBUG
 else
@@ -573,7 +567,7 @@ ifeq ($(optimize),yes)
 	endif
 
 	ifeq ($(KERNEL),Darwin)
-		ifeq ($(comp),$(filter $(comp),clang icx))
+		ifeq ($(comp),$(filter $(comp),clang icc))
 			CXXFLAGS += -mdynamic-no-pic
 		endif
 
@@ -585,10 +579,7 @@ ifeq ($(optimize),yes)
 	endif
 
 	ifeq ($(comp),clang)
-		clangmajorversion = $(shell $(CXX) -dumpversion 2>/dev/null | cut -f1 -d.)
-		ifeq ($(shell expr $(clangmajorversion) \< 16),1)
-			CXXFLAGS += -fexperimental-new-pass-manager
-		endif
+		CXXFLAGS += -fexperimental-new-pass-manager
 	endif
 endif
 
@@ -609,6 +600,8 @@ endif
 ifeq ($(popcnt),yes)
 	ifeq ($(arch),$(filter $(arch),ppc64 armv7 armv8 arm64))
 		CXXFLAGS += -DUSE_POPCNT
+	else ifeq ($(comp),icc)
+		CXXFLAGS += -msse3 -DUSE_POPCNT
 	else
 		CXXFLAGS += -msse3 -mpopcnt -DUSE_POPCNT
 	endif
@@ -617,63 +610,63 @@ endif
 ### 3.6 SIMD architectures
 ifeq ($(avx2),yes)
 	CXXFLAGS += -DUSE_AVX2
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
-		CXXFLAGS += -mavx2 -mbmi
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
+		CXXFLAGS += -mavx2
 	endif
 endif
 
 ifeq ($(avxvnni),yes)
 	CXXFLAGS += -DUSE_VNNI -DUSE_AVXVNNI
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
 		CXXFLAGS += -mavxvnni
 	endif
 endif
 
 ifeq ($(avx512),yes)
 	CXXFLAGS += -DUSE_AVX512
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
 		CXXFLAGS += -mavx512f -mavx512bw
 	endif
 endif
 
 ifeq ($(vnni256),yes)
 	CXXFLAGS += -DUSE_VNNI
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
 		CXXFLAGS += -mavx512f -mavx512bw -mavx512vnni -mavx512dq -mavx512vl -mprefer-vector-width=256
 	endif
 endif
 
 ifeq ($(vnni512),yes)
 	CXXFLAGS += -DUSE_VNNI
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
-		CXXFLAGS += -mavx512f -mavx512bw -mavx512vnni -mavx512dq -mavx512vl -mprefer-vector-width=512
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
+		CXXFLAGS += -mavx512vnni -mavx512dq -mavx512vl
 	endif
 endif
 
 ifeq ($(sse41),yes)
 	CXXFLAGS += -DUSE_SSE41
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
 		CXXFLAGS += -msse4.1
 	endif
 endif
 
 ifeq ($(ssse3),yes)
 	CXXFLAGS += -DUSE_SSSE3
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
 		CXXFLAGS += -mssse3
 	endif
 endif
 
 ifeq ($(sse2),yes)
 	CXXFLAGS += -DUSE_SSE2
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
 		CXXFLAGS += -msse2
 	endif
 endif
 
 ifeq ($(mmx),yes)
 	CXXFLAGS += -DUSE_MMX
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
 		CXXFLAGS += -mmmx
 	endif
 endif
@@ -686,43 +679,24 @@ ifeq ($(neon),yes)
 		CXXFLAGS += -mfpu=neon
 	endif
 	endif
-	endif
 endif
-
-ifeq ($(dotprod),yes)
-	CXXFLAGS += -march=armv8.2-a+dotprod -DUSE_NEON_DOTPROD
 endif
 
 ### 3.7 pext
 ifeq ($(pext),yes)
 	CXXFLAGS += -DUSE_PEXT
-	ifeq ($(comp),$(filter $(comp),gcc clang mingw icx))
+	ifeq ($(comp),$(filter $(comp),gcc clang mingw))
 		CXXFLAGS += -mbmi2
 	endif
 endif
 
-### 3.7.1 Try to include git commit sha for versioning
-GIT_SHA = $(shell git rev-parse HEAD 2>/dev/null | cut -c 1-8)
-ifneq ($(GIT_SHA), )
-	CXXFLAGS += -DGIT_SHA=$(GIT_SHA)
-endif
-
-### 3.7.2 Try to include git commit date for versioning
-GIT_DATE = $(shell git show -s --date=format:'%Y%m%d' --format=%cd HEAD 2>/dev/null)
-ifneq ($(GIT_DATE), )
-	CXXFLAGS += -DGIT_DATE=$(GIT_DATE)
-endif
-
 ### 3.8 Link Time Optimization
 ### This is a mix of compile and link time options because the lto link phase
 ### needs access to the optimization flags.
 ifeq ($(optimize),yes)
 ifeq ($(debug), no)
-	ifeq ($(comp),$(filter $(comp),clang icx))
-		CXXFLAGS += -flto=full
-		ifeq ($(comp),icx)
-			CXXFLAGS += -fwhole-program-vtables
-                endif
+	ifeq ($(comp),clang)
+		CXXFLAGS += -flto
 		ifeq ($(target_windows),yes)
 			CXXFLAGS += -fuse-ld=lld
 		endif
@@ -732,19 +706,21 @@ ifeq ($(debug), no)
 # GCC on some systems.
 	else ifeq ($(comp),gcc)
 	ifeq ($(gccisclang),)
-		CXXFLAGS += -flto -flto-partition=one
+		CXXFLAGS += -flto
 		LDFLAGS += $(CXXFLAGS) -flto=jobserver
 	else
-		CXXFLAGS += -flto=full
+		CXXFLAGS += -flto
 		LDFLAGS += $(CXXFLAGS)
 	endif
 
 # To use LTO and static linking on Windows,
 # the tool chain requires gcc version 10.1 or later.
 	else ifeq ($(comp),mingw)
-		CXXFLAGS += -flto -flto-partition=one
+	ifneq ($(arch),i386)
+		CXXFLAGS += -flto
 		LDFLAGS += $(CXXFLAGS) -save-temps
 	endif
+	endif
 endif
 endif
 
@@ -769,19 +745,19 @@ help:
 	@echo "Supported targets:"
 	@echo ""
 	@echo "help                    > Display architecture details"
-	@echo "profile-build           > standard build with profile-guided optimization"
-	@echo "build                   > skip profile-guided optimization"
+	@echo "build                   > Standard build"
 	@echo "net                     > Download the default nnue net"
+	@echo "profile-build           > Faster build (with profile-guided optimization)"
 	@echo "strip                   > Strip executable"
 	@echo "install                 > Install executable"
 	@echo "clean                   > Clean up"
 	@echo ""
 	@echo "Supported archs:"
 	@echo ""
-	@echo "x86-64-vnni512          > x86 64-bit with vnni 512bit support"
-	@echo "x86-64-vnni256          > x86 64-bit with vnni 512bit support, limit operands to 256bit wide"
+	@echo "x86-64-vnni512          > x86 64-bit with vnni support 512bit wide"
+	@echo "x86-64-vnni256          > x86 64-bit with vnni support 256bit wide"
 	@echo "x86-64-avx512           > x86 64-bit with avx512 support"
-	@echo "x86-64-avxvnni          > x86 64-bit with vnni 256bit support"
+	@echo "x86-64-avxvnni          > x86 64-bit with avxvnni support"
 	@echo "x86-64-bmi2             > x86 64-bit with bmi2 support"
 	@echo "x86-64-avx2             > x86 64-bit with avx2 support"
 	@echo "x86-64-sse41-popcnt     > x86 64-bit with sse41 and popcnt support"
@@ -797,30 +773,27 @@ help:
 	@echo "armv7                   > ARMv7 32-bit"
 	@echo "armv7-neon              > ARMv7 32-bit with popcnt and neon"
 	@echo "armv8                   > ARMv8 64-bit with popcnt and neon"
-	@echo "armv8-dotprod           > ARMv8 64-bit with popcnt, neon and dot product support"
 	@echo "e2k                     > Elbrus 2000"
 	@echo "apple-silicon           > Apple silicon ARM64"
 	@echo "general-64              > unspecified 64-bit"
 	@echo "general-32              > unspecified 32-bit"
-	@echo "riscv64                 > RISC-V 64-bit"
 	@echo ""
 	@echo "Supported compilers:"
 	@echo ""
 	@echo "gcc                     > Gnu compiler (default)"
 	@echo "mingw                   > Gnu compiler with MinGW under Windows"
 	@echo "clang                   > LLVM Clang compiler"
-	@echo "icx                     > Intel oneAPI DPC++/C++ Compiler"
+	@echo "icc                     > Intel compiler"
 	@echo "ndk                     > Google NDK to cross-compile for Android"
 	@echo ""
-	@echo "Simple examples. If you don't know what to do, you likely want to run one of: "
+	@echo "Simple examples. If you don't know what to do, you likely want to run: "
 	@echo ""
-	@echo "make -j profile-build ARCH=x86-64-avx2    # typically a fast compile for common systems "
-	@echo "make -j profile-build ARCH=x86-64-modern  # A more portable compile for 64-bit systems "
-	@echo "make -j profile-build ARCH=x86-64         # A portable compile for 64-bit systems "
+	@echo "make -j build ARCH=x86-64  (A portable, slow compile for 64-bit systems)"
+	@echo "make -j build ARCH=x86-32  (A portable, slow compile for 32-bit systems)"
 	@echo ""
-	@echo "Advanced examples, for experienced users: "
+	@echo "Advanced examples, for experienced users looking for performance: "
 	@echo ""
-	@echo "make -j profile-build ARCH=x86-64-bmi2"
+	@echo "make    help  ARCH=x86-64-bmi2"
 	@echo "make -j profile-build ARCH=x86-64-bmi2 COMP=gcc COMPCXX=g++-9.0"
 	@echo "make -j build ARCH=x86-64-ssse3 COMP=clang"
 	@echo ""
@@ -835,10 +808,8 @@ endif
 
 
 .PHONY: help build profile-build strip install clean net objclean profileclean \
-	config-sanity \
-	icx-profile-use icx-profile-make \
-	gcc-profile-use gcc-profile-make \
-	clang-profile-use clang-profile-make FORCE
+        config-sanity icc-profile-use icc-profile-make gcc-profile-use gcc-profile-make \
+        clang-profile-use clang-profile-make
 
 build: net config-sanity
 	$(MAKE) ARCH=$(ARCH) COMP=$(COMP) all
@@ -850,6 +821,7 @@ profile-build: net config-sanity objclean profileclean
 	@echo ""
 	@echo "Step 2/4. Running benchmark for pgo-build ..."
 	$(PGOBENCH) 2>&1 | tail -n 4
+	$(PGOGENSFEN) 2>&1 | tail -n 4
 	@echo ""
 	@echo "Step 3/4. Building optimized executable ..."
 	$(MAKE) ARCH=$(ARCH) COMP=$(COMP) objclean
@@ -874,46 +846,34 @@ clean: objclean profileclean
 net:
 	$(eval nnuenet := $(shell grep EvalFileDefaultName evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/'))
 	@echo "Default net: $(nnuenet)"
-	$(eval nnuedownloadurl1 := https://tests.stockfishchess.org/api/nn/$(nnuenet))
-	$(eval nnuedownloadurl2 := https://github.com/official-stockfish/networks/raw/master/$(nnuenet))
+	$(eval nnuedownloadurl := https://tests.stockfishchess.org/api/nn/$(nnuenet))
 	$(eval curl_or_wget := $(shell if hash curl 2>/dev/null; then echo "curl -skL"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi))
-	@if [ "x$(curl_or_wget)" = "x" ]; then \
-	    echo "Neither curl nor wget is installed. Install one of these tools unless the net has been downloaded manually"; \
-        fi
+	@if test -f "$(nnuenet)"; then \
+		echo "Already available."; \
+	else \
+		if [ "x$(curl_or_wget)" = "x" ]; then \
+			echo "Automatic download failed: neither curl nor wget is installed. Install one of these tools or download the net manually"; exit 1; \
+		else \
+			echo "Downloading $(nnuedownloadurl)"; $(curl_or_wget) $(nnuedownloadurl) > $(nnuenet);\
+		fi; \
+	fi;
 	$(eval shasum_command := $(shell if hash shasum 2>/dev/null; then echo "shasum -a 256 "; elif hash sha256sum 2>/dev/null; then echo "sha256sum "; fi))
-	@if [ "x$(shasum_command)" = "x" ]; then \
-            echo "shasum / sha256sum not found, skipping net validation"; \
-        fi
-	@for nnuedownloadurl in "$(nnuedownloadurl1)" "$(nnuedownloadurl2)"; do \
-	   if test -f "$(nnuenet)"; then \
-	      echo "$(nnuenet) available."; \
-	   else \
-	      if [ "x$(curl_or_wget)" != "x" ]; then \
-                 echo "Downloading $${nnuedownloadurl}"; $(curl_or_wget) $${nnuedownloadurl} > $(nnuenet);\
-              else \
-                 echo "No net found and download not possible"; exit 1;\
-	      fi; \
-	   fi; \
-	   if [ "x$(shasum_command)" != "x" ]; then \
-	      if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
-	         echo "Removing failed download"; rm -f $(nnuenet); \
-	      else \
-	         echo "Network validated"; break; \
-	      fi; \
-	   fi; \
-	done
-	@if ! test -f "$(nnuenet)"; then \
-	    echo "Failed to download $(nnuenet)."; \
+	@if [ "x$(shasum_command)" != "x" ]; then \
+		if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
+			echo "Failed download or $(nnuenet) corrupted, please delete!"; exit 1; \
+		fi \
+	else \
+		echo "shasum / sha256sum not found, skipping net validation"; \
 	fi
 
 # clean binaries and objects
 objclean:
-	@rm -f stockfish stockfish.exe *.o ./syzygy/*.o ./nnue/*.o ./nnue/features/*.o
+	@rm -f $(EXE) *.o ./syzygy/*.o ./nnue/*.o ./nnue/features/*.o ./tools/*.o ./extra/*.o ./eval/*.o
 
 # clean auxiliary profiling files
 profileclean:
 	@rm -rf profdir
-	@rm -f bench.txt *.gcda *.gcno ./syzygy/*.gcda ./nnue/*.gcda ./nnue/features/*.gcda *.s
+	@rm -f bench.txt *.gcda *.gcno ./syzygy/*.gcda ./nnue/*.gcda ./nnue/features/*.gcda *.s ./tools/*.gcda ./extra/*.gcda ./eval/*.gcda
 	@rm -f stockfish.profdata *.profraw
 	@rm -f stockfish.*args*
 	@rm -f stockfish.*lt*
@@ -953,9 +913,7 @@ config-sanity: net
 	@echo "vnni256: '$(vnni256)'"
 	@echo "vnni512: '$(vnni512)'"
 	@echo "neon: '$(neon)'"
-	@echo "dotprod: '$(dotprod)'"
 	@echo "arm_version: '$(arm_version)'"
-	@echo "target_windows: '$(target_windows)'"
 	@echo ""
 	@echo "Flags:"
 	@echo "CXX: $(CXX)"
@@ -969,7 +927,7 @@ config-sanity: net
 	@test "$(SUPPORTED_ARCH)" = "true"
 	@test "$(arch)" = "any" || test "$(arch)" = "x86_64" || test "$(arch)" = "i386" || \
 	 test "$(arch)" = "ppc64" || test "$(arch)" = "ppc" || test "$(arch)" = "e2k" || \
-	 test "$(arch)" = "armv7" || test "$(arch)" = "armv8" || test "$(arch)" = "arm64" || test "$(arch)" = "riscv64"
+	 test "$(arch)" = "armv7" || test "$(arch)" = "armv8" || test "$(arch)" = "arm64"
 	@test "$(bits)" = "32" || test "$(bits)" = "64"
 	@test "$(prefetch)" = "yes" || test "$(prefetch)" = "no"
 	@test "$(popcnt)" = "yes" || test "$(popcnt)" = "no"
@@ -984,16 +942,12 @@ config-sanity: net
 	@test "$(vnni256)" = "yes" || test "$(vnni256)" = "no"
 	@test "$(vnni512)" = "yes" || test "$(vnni512)" = "no"
 	@test "$(neon)" = "yes" || test "$(neon)" = "no"
-	@test "$(comp)" = "gcc" || test "$(comp)" = "icx" || test "$(comp)" = "mingw" || test "$(comp)" = "clang" \
+	@test "$(comp)" = "gcc" || test "$(comp)" = "icc" || test "$(comp)" = "mingw" || test "$(comp)" = "clang" \
 	|| test "$(comp)" = "armv7a-linux-androideabi16-clang"  || test "$(comp)" = "aarch64-linux-android21-clang"
 
 $(EXE): $(OBJS)
 	+$(CXX) -o $@ $(OBJS) $(LDFLAGS)
 
-# Force recompilation to ensure version info is up-to-date
-misc.o: FORCE
-FORCE:
-
 clang-profile-make:
 	$(MAKE) ARCH=$(ARCH) COMP=$(COMP) \
 	EXTRACXXFLAGS='-fprofile-instr-generate ' \
@@ -1022,22 +976,18 @@ gcc-profile-use:
 	EXTRALDFLAGS='-lgcov' \
 	all
 
-icx-profile-make:
+icc-profile-make:
+	@mkdir -p profdir
 	$(MAKE) ARCH=$(ARCH) COMP=$(COMP) \
-	EXTRACXXFLAGS='-fprofile-instr-generate ' \
-	EXTRALDFLAGS=' -fprofile-instr-generate' \
+	EXTRACXXFLAGS='-prof-gen=srcpos -prof_dir ./profdir' \
 	all
 
-icx-profile-use:
-	$(XCRUN) llvm-profdata merge -output=stockfish.profdata *.profraw
+icc-profile-use:
 	$(MAKE) ARCH=$(ARCH) COMP=$(COMP) \
-	EXTRACXXFLAGS='-fprofile-instr-use=stockfish.profdata' \
-	EXTRALDFLAGS='-fprofile-use ' \
+	EXTRACXXFLAGS='-prof_use -prof_dir ./profdir' \
 	all
 
 .depend: $(SRCS)
 	-@$(CXX) $(DEPENDFLAGS) -MM $(SRCS) > $@ 2> /dev/null
 
-ifeq (, $(filter $(MAKECMDGOALS), help strip install clean net objclean profileclean config-sanity))
 -include .depend
-endif

src/benchmark.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -16,8 +16,6 @@
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "benchmark.h"
-
 #include <fstream>
 #include <iostream>
 #include <istream>

src/benchmark.h

@@ -1,34 +0,0 @@
-/*
-  Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
-
-  Stockfish is free software: you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation, either version 3 of the License, or
-  (at your option) any later version.
-
-  Stockfish is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef BENCHMARK_H_INCLUDED
-#define BENCHMARK_H_INCLUDED
-
-#include <iosfwd>
-#include <string>
-#include <vector>
-
-namespace Stockfish {
-
-class Position;
-
-std::vector<std::string> setup_bench(const Position&, std::istream&);
-
-} // namespace Stockfish
-
-#endif // #ifndef BENCHMARK_H_INCLUDED

src/bitbase.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/bitboard.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -27,6 +27,7 @@ namespace Stockfish {
 uint8_t PopCnt16[1 << 16];
 uint8_t SquareDistance[SQUARE_NB][SQUARE_NB];
 
+Bitboard SquareBB[SQUARE_NB];
 Bitboard LineBB[SQUARE_NB][SQUARE_NB];
 Bitboard BetweenBB[SQUARE_NB][SQUARE_NB];
 Bitboard PseudoAttacks[PIECE_TYPE_NB][SQUARE_NB];
@@ -81,6 +82,9 @@ void Bitboards::init() {
   for (unsigned i = 0; i < (1 << 16); ++i)
       PopCnt16[i] = uint8_t(std::bitset<16>(i).count());
 
+  for (Square s = SQ_A1; s <= SQ_H8; ++s)
+      SquareBB[s] = (1ULL << s);
+
   for (Square s1 = SQ_A1; s1 <= SQ_H8; ++s1)
       for (Square s2 = SQ_A1; s2 <= SQ_H8; ++s2)
           SquareDistance[s1][s2] = std::max(distance<File>(s1, s2), distance<Rank>(s1, s2));

src/bitboard.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -74,6 +74,7 @@ constexpr Bitboard KingFlank[FILE_NB] = {
 extern uint8_t PopCnt16[1 << 16];
 extern uint8_t SquareDistance[SQUARE_NB][SQUARE_NB];
 
+extern Bitboard SquareBB[SQUARE_NB];
 extern Bitboard BetweenBB[SQUARE_NB][SQUARE_NB];
 extern Bitboard LineBB[SQUARE_NB][SQUARE_NB];
 extern Bitboard PseudoAttacks[PIECE_TYPE_NB][SQUARE_NB];
@@ -107,7 +108,7 @@ extern Magic BishopMagics[SQUARE_NB];
 
 inline Bitboard square_bb(Square s) {
   assert(is_ok(s));
-  return (1ULL << s);
+  return SquareBB[s];
 }
 
 

src/endgame.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/endgame.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/evaluate.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -27,6 +27,8 @@
 #include <streambuf>
 #include <vector>
 
+#include "nnue/evaluate_nnue.h"
+
 #include "bitboard.h"
 #include "evaluate.h"
 #include "material.h"
@@ -36,7 +38,6 @@
 #include "timeman.h"
 #include "uci.h"
 #include "incbin/incbin.h"
-#include "nnue/evaluate_nnue.h"
 
 // Macro to embed the default efficiently updatable neural network (NNUE) file
 // data in the engine binary (using incbin.h, by Dale Weiler).
@@ -53,15 +54,28 @@
   const unsigned int         gEmbeddedNNUESize = 1;
 #endif
 
-
 using namespace std;
 
 namespace Stockfish {
 
 namespace Eval {
 
-  bool useNNUE;
-  string currentEvalFileName = "None";
+  namespace NNUE {
+    string currentEvalFileName = "None";
+    UseNNUEMode useNNUE;
+
+    static UseNNUEMode nnue_mode_from_option(const UCI::Option& mode)
+    {
+      if (mode == "false")
+        return UseNNUEMode::False;
+      else if (mode == "true")
+         return UseNNUEMode::True;
+      else if (mode == "pure")
+        return UseNNUEMode::Pure;
+
+      return UseNNUEMode::False;
+    }
+  }
 
   /// NNUE::init() tries to load a NNUE network at startup time, or when the engine
   /// receives a UCI command "setoption name EvalFile value nn-[a-z0-9]{12}.nnue"
@@ -73,8 +87,8 @@ namespace Eval {
 
   void NNUE::init() {
 
-    useNNUE = Options["Use NNUE"];
-    if (!useNNUE)
+    useNNUE = nnue_mode_from_option(Options["Use NNUE"]);
+    if (useNNUE == UseNNUEMode::False)
         return;
 
     string eval_file = string(Options["EvalFile"]);
@@ -82,18 +96,20 @@ namespace Eval {
         eval_file = EvalFileDefaultName;
 
     #if defined(DEFAULT_NNUE_DIRECTORY)
+    #define stringify2(x) #x
+    #define stringify(x) stringify2(x)
     vector<string> dirs = { "<internal>" , "" , CommandLine::binaryDirectory , stringify(DEFAULT_NNUE_DIRECTORY) };
     #else
     vector<string> dirs = { "<internal>" , "" , CommandLine::binaryDirectory };
     #endif
 
-    for (const string& directory : dirs)
+    for (string directory : dirs)
         if (currentEvalFileName != eval_file)
         {
             if (directory != "<internal>")
             {
                 ifstream stream(directory + eval_file, ios::binary);
-                if (NNUE::load_eval(eval_file, stream))
+                if (load_eval(eval_file, stream))
                     currentEvalFileName = eval_file;
             }
 
@@ -109,7 +125,7 @@ namespace Eval {
                 (void) gEmbeddedNNUEEnd; // Silence warning on unused variable
 
                 istream stream(&buffer);
-                if (NNUE::load_eval(eval_file, stream))
+                if (load_eval(eval_file, stream))
                     currentEvalFileName = eval_file;
             }
         }
@@ -122,7 +138,7 @@ namespace Eval {
     if (eval_file.empty())
         eval_file = EvalFileDefaultName;
 
-    if (useNNUE && currentEvalFileName != eval_file)
+    if (useNNUE != UseNNUEMode::False && currentEvalFileName != eval_file)
     {
 
         string msg1 = "If the UCI option \"Use NNUE\" is set to true, network evaluation parameters compatible with the engine must be available.";
@@ -140,7 +156,7 @@ namespace Eval {
         exit(EXIT_FAILURE);
     }
 
-    if (useNNUE)
+    if (useNNUE != UseNNUEMode::False)
         sync_cout << "info string NNUE evaluation using " << eval_file << " enabled" << sync_endl;
     else
         sync_cout << "info string classical evaluation enabled" << sync_endl;
@@ -157,24 +173,24 @@ namespace Trace {
 
   Score scores[TERM_NB][COLOR_NB];
 
-  static double to_cp(Value v) { return double(v) / UCI::NormalizeToPawnValue; }
+  double to_cp(Value v) { return double(v) / PawnValueEg; }
 
-  static void add(int idx, Color c, Score s) {
+  void add(int idx, Color c, Score s) {
     scores[idx][c] = s;
   }
 
-  static void add(int idx, Score w, Score b = SCORE_ZERO) {
+  void add(int idx, Score w, Score b = SCORE_ZERO) {
     scores[idx][WHITE] = w;
     scores[idx][BLACK] = b;
   }
 
-  static std::ostream& operator<<(std::ostream& os, Score s) {
+  std::ostream& operator<<(std::ostream& os, Score s) {
     os << std::setw(5) << to_cp(mg_value(s)) << " "
        << std::setw(5) << to_cp(eg_value(s));
     return os;
   }
 
-  static std::ostream& operator<<(std::ostream& os, Term t) {
+  std::ostream& operator<<(std::ostream& os, Term t) {
 
     if (t == MATERIAL || t == IMBALANCE || t == WINNABLE || t == TOTAL)
         os << " ----  ----"    << " | " << " ----  ----";
@@ -191,8 +207,8 @@ using namespace Trace;
 namespace {
 
   // Threshold for lazy and space evaluation
-  constexpr Value LazyThreshold1    =  Value(3622);
-  constexpr Value LazyThreshold2    =  Value(1962);
+  constexpr Value LazyThreshold1    =  Value(3631);
+  constexpr Value LazyThreshold2    =  Value(2084);
   constexpr Value SpaceThreshold    =  Value(11551);
 
   // KingAttackWeights[PieceType] contains king attack weights by piece type
@@ -386,10 +402,10 @@ namespace {
   template<Tracing T> template<Color Us, PieceType Pt>
   Score Evaluation<T>::pieces() {
 
-    constexpr Color Them = ~Us;
-    [[maybe_unused]] constexpr Direction Down = -pawn_push(Us);
-    [[maybe_unused]] constexpr Bitboard OutpostRanks = (Us == WHITE ? Rank4BB | Rank5BB | Rank6BB
-                                                                    : Rank5BB | Rank4BB | Rank3BB);
+    constexpr Color     Them = ~Us;
+    constexpr Direction Down = -pawn_push(Us);
+    constexpr Bitboard OutpostRanks = (Us == WHITE ? Rank4BB | Rank5BB | Rank6BB
+                                                   : Rank5BB | Rank4BB | Rank3BB);
     Bitboard b1 = pos.pieces(Us, Pt);
     Bitboard b, bb;
     Score score = SCORE_ZERO;
@@ -428,7 +444,7 @@ namespace {
         int mob = popcount(b & mobilityArea[Us]);
         mobility[Us] += MobilityBonus[Pt - 2][mob];
 
-        if constexpr (Pt == BISHOP || Pt == KNIGHT)
+        if (Pt == BISHOP || Pt == KNIGHT)
         {
             // Bonus if the piece is on an outpost square or can reach one
             // Bonus for knights (UncontestedOutpost) if few relevant targets
@@ -979,7 +995,7 @@ namespace {
     // Initialize score by reading the incrementally updated scores included in
     // the position object (material + piece square tables) and the material
     // imbalance. Score is computed internally from the white point of view.
-    Score score = pos.psq_score() + me->imbalance();
+    Score score = pos.psq_score() + me->imbalance() + pos.this_thread()->trend;
 
     // Probe the pawn hash table
     pe = Pawns::probe(pos);
@@ -1040,6 +1056,38 @@ make_v:
     return v;
   }
 
+
+  /// Fisher Random Chess: correction for cornered bishops, to fix chess960 play with NNUE
+
+  Value fix_FRC(const Position& pos) {
+
+    constexpr Bitboard Corners =  1ULL << SQ_A1 | 1ULL << SQ_H1 | 1ULL << SQ_A8 | 1ULL << SQ_H8;
+
+    if (!(pos.pieces(BISHOP) & Corners))
+        return VALUE_ZERO;
+
+    int correction = 0;
+
+    if (   pos.piece_on(SQ_A1) == W_BISHOP
+        && pos.piece_on(SQ_B2) == W_PAWN)
+        correction -= CorneredBishop;
+
+    if (   pos.piece_on(SQ_H1) == W_BISHOP
+        && pos.piece_on(SQ_G2) == W_PAWN)
+        correction -= CorneredBishop;
+
+    if (   pos.piece_on(SQ_A8) == B_BISHOP
+        && pos.piece_on(SQ_B7) == B_PAWN)
+        correction += CorneredBishop;
+
+    if (   pos.piece_on(SQ_H8) == B_BISHOP
+        && pos.piece_on(SQ_G7) == B_PAWN)
+        correction += CorneredBishop;
+
+    return pos.side_to_move() == WHITE ?  Value(3 * correction)
+                                       : -Value(3 * correction);
+  }
+
 } // namespace Eval
 
 
@@ -1048,35 +1096,51 @@ make_v:
 
 Value Eval::evaluate(const Position& pos) {
 
-  assert(!pos.checkers());
+  pos.this_thread()->on_eval();
 
   Value v;
-  Value psq = pos.psq_eg_stm();
 
-  // We use the much less accurate but faster Classical eval when the NNUE
-  // option is set to false. Otherwise we use the NNUE eval unless the
-  // PSQ advantage is decisive. (~4 Elo at STC, 1 Elo at LTC)
-  bool useClassical = !useNNUE || abs(psq) > 2048;
+  if (NNUE::useNNUE == NNUE::UseNNUEMode::Pure) {
+      v = NNUE::evaluate(pos);
 
-  if (useClassical)
-      v = Evaluation<NO_TRACE>(pos).value();
-  else
+      // Guarantee evaluation does not hit the tablebase range
+      v = std::clamp(v, VALUE_TB_LOSS_IN_MAX_PLY + 1, VALUE_TB_WIN_IN_MAX_PLY - 1);
+
+      return v;
+  }
+
+  // Deciding between classical and NNUE eval (~10 Elo): for high PSQ imbalance we use classical,
+  // but we switch to NNUE during long shuffling or with high material on the board.
+  bool useClassical = (pos.this_thread()->depth > 9 || pos.count<ALL_PIECES>() > 7) &&
+          abs(eg_value(pos.psq_score())) * 5 > (856 + pos.non_pawn_material() / 64) * (10 + pos.rule50_count());
+
+  // Deciding between classical and NNUE eval (~10 Elo): for high PSQ imbalance we use classical,
+  // but we switch to NNUE during long shuffling or with high material on the board.
+  if (NNUE::useNNUE == NNUE::UseNNUEMode::False || useClassical)
   {
-      int nnueComplexity;
-      int npm = pos.non_pawn_material() / 64;
+      v = Evaluation<NO_TRACE>(pos).value();          // classical
+      useClassical = abs(v) >= 297;
+  }
 
-      Color stm = pos.side_to_move();
-      Value optimism = pos.this_thread()->optimism[stm];
+  // If result of a classical evaluation is much lower than threshold fall back to NNUE
+  if (NNUE::useNNUE != NNUE::UseNNUEMode::False && !useClassical)
+  {
+       Value nnue     = NNUE::evaluate(pos, true);     // NNUE
+       int scale      = 1080 + 110 * pos.non_pawn_material() / 5120;
+       Color stm      = pos.side_to_move();
+       Value optimism = pos.this_thread()->optimism[stm];
+       Value psq      = (stm == WHITE ? 1 : -1) * eg_value(pos.psq_score());
+       int complexity = (278 * abs(nnue - psq)) / 256;
 
-      Value nnue = NNUE::evaluate(pos, true, &nnueComplexity);
+       optimism = optimism * (251 + complexity) / 256;
+       v = (nnue * scale + optimism * (scale - 852)) / 1024;
 
-      // Blend optimism with nnue complexity and (semi)classical complexity
-      optimism += optimism * (nnueComplexity + abs(psq - nnue)) / 512;
-      v = (nnue * (945 + npm) + optimism * (150 + npm)) / 1024;
+       if (pos.is_chess960())
+           v += fix_FRC(pos);
   }
 
   // Damp down the evaluation linearly when shuffling
-  v = v * (200 - pos.rule50_count()) / 214;
+  v = v * (195 - pos.rule50_count()) / 211;
 
   // Guarantee evaluation does not hit the tablebase range
   v = std::clamp(v, VALUE_TB_LOSS_IN_MAX_PLY + 1, VALUE_TB_WIN_IN_MAX_PLY - 1);
@@ -1102,6 +1166,8 @@ std::string Eval::trace(Position& pos) {
   std::memset(scores, 0, sizeof(scores));
 
   // Reset any global variable used in eval
+  pos.this_thread()->depth           = 0;
+  pos.this_thread()->trend           = SCORE_ZERO;
   pos.this_thread()->bestValue       = VALUE_ZERO;
   pos.this_thread()->optimism[WHITE] = VALUE_ZERO;
   pos.this_thread()->optimism[BLACK] = VALUE_ZERO;
@@ -1131,14 +1197,14 @@ std::string Eval::trace(Position& pos) {
      << "|      Total | " << Term(TOTAL)
      << "+------------+-------------+-------------+-------------+\n";
 
-  if (Eval::useNNUE)
+  if (NNUE::useNNUE != NNUE::UseNNUEMode::False)
       ss << '\n' << NNUE::trace(pos) << '\n';
 
   ss << std::showpoint << std::showpos << std::fixed << std::setprecision(2) << std::setw(15);
 
   v = pos.side_to_move() == WHITE ? v : -v;
   ss << "\nClassical evaluation   " << to_cp(v) << " (white side)\n";
-  if (Eval::useNNUE)
+  if (NNUE::useNNUE != NNUE::UseNNUEMode::False)
   {
       v = NNUE::evaluate(pos, false);
       v = pos.side_to_move() == WHITE ? v : -v;
@@ -1148,7 +1214,7 @@ std::string Eval::trace(Position& pos) {
   v = evaluate(pos);
   v = pos.side_to_move() == WHITE ? v : -v;
   ss << "Final evaluation       " << to_cp(v) << " (white side)";
-  if (Eval::useNNUE)
+  if (NNUE::useNNUE != NNUE::UseNNUEMode::False)
      ss << " [with scaled NNUE, hybrid, ...]";
   ss << "\n";
 

src/evaluate.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -33,19 +33,32 @@ namespace Eval {
   std::string trace(Position& pos);
   Value evaluate(const Position& pos);
 
-  extern bool useNNUE;
-  extern std::string currentEvalFileName;
-
   // The default net name MUST follow the format nn-[SHA256 first 12 digits].nnue
   // for the build process (profile-build and fishtest) to work. Do not change the
   // name of the macro, as it is used in the Makefile.
-  #define EvalFileDefaultName   "nn-5af11540bbfe.nnue"
+  #define EvalFileDefaultName   "nn-3c0aa92af1da.nnue"
 
   namespace NNUE {
+    enum struct UseNNUEMode
+    {
+      False,
+      True,
+      Pure
+    };
+
+    extern UseNNUEMode useNNUE;
+    extern std::string currentEvalFileName;
+
+    std::string trace(Position& pos);
+    Value evaluate(const Position& pos, bool adjusted = false);
 
     void init();
     void verify();
 
+    bool load_eval(std::string name, std::istream& stream);
+    bool save_eval(std::ostream& stream);
+    bool save_eval(const std::optional<std::string>& filename);
+
   } // namespace NNUE
 
 } // namespace Eval

src/main.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -18,6 +18,8 @@
 
 #include <iostream>
 
+#include "nnue/evaluate_nnue.h"
+
 #include "bitboard.h"
 #include "endgame.h"
 #include "position.h"

src/material.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/material.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -28,7 +28,7 @@ namespace Stockfish::Material {
 
 /// Material::Entry contains various information about a material configuration.
 /// It contains a material imbalance evaluation, a function pointer to a special
-/// endgame evaluation function (which in most cases is nullptr, meaning that the
+/// endgame evaluation function (which in most cases is NULL, meaning that the
 /// standard evaluation function will be used), and scale factors.
 ///
 /// The scale factors are used to scale the evaluation score up or down. For
@@ -62,7 +62,7 @@ struct Entry {
   uint8_t factor[COLOR_NB];
 };
 
-using Table = HashTable<Entry, 8192>;
+typedef HashTable<Entry, 8192> Table;
 
 Entry* probe(const Position& pos);
 

src/misc.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -32,26 +32,21 @@
 // the calls at compile time), try to load them at runtime. To do this we need
 // first to define the corresponding function pointers.
 extern "C" {
-using fun1_t = bool(*)(LOGICAL_PROCESSOR_RELATIONSHIP,
-                       PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX, PDWORD);
-using fun2_t = bool(*)(USHORT, PGROUP_AFFINITY);
-using fun3_t = bool(*)(HANDLE, CONST GROUP_AFFINITY*, PGROUP_AFFINITY);
-using fun4_t = bool(*)(USHORT, PGROUP_AFFINITY, USHORT, PUSHORT);
-using fun5_t = WORD(*)();
-using fun6_t = bool(*)(HANDLE, DWORD, PHANDLE);
-using fun7_t = bool(*)(LPCSTR, LPCSTR, PLUID);
-using fun8_t = bool(*)(HANDLE, BOOL, PTOKEN_PRIVILEGES, DWORD, PTOKEN_PRIVILEGES, PDWORD);
+typedef bool(*fun1_t)(LOGICAL_PROCESSOR_RELATIONSHIP,
+                      PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX, PDWORD);
+typedef bool(*fun2_t)(USHORT, PGROUP_AFFINITY);
+typedef bool(*fun3_t)(HANDLE, CONST GROUP_AFFINITY*, PGROUP_AFFINITY);
+typedef bool(*fun4_t)(USHORT, PGROUP_AFFINITY, USHORT, PUSHORT);
+typedef WORD(*fun5_t)();
 }
 #endif
 
-#include <cmath>
-#include <cstdlib>
 #include <fstream>
 #include <iomanip>
 #include <iostream>
 #include <sstream>
-#include <string_view>
 #include <vector>
+#include <cstdlib>
 
 #if defined(__linux__) && !defined(__ANDROID__)
 #include <stdlib.h>
@@ -70,10 +65,13 @@ using namespace std;
 
 namespace Stockfish {
 
+SynchronizedRegionLogger sync_region_cout(std::cout);
+
 namespace {
 
-/// Version number or dev.
-constexpr string_view version = "16";
+/// Version number. If Version is left empty, then compile date in the format
+/// DD-MM-YY and show in engine_info.
+const string Version = "";
 
 /// Our fancy logging facility. The trick here is to replace cin.rdbuf() and
 /// cout.rdbuf() with two Tie objects that tie cin and cout to a file stream. We
@@ -142,41 +140,23 @@ public:
 } // namespace
 
 
-/// engine_info() returns the full name of the current Stockfish version.
-/// For local dev compiles we try to append the commit sha and commit date
-/// from git if that fails only the local compilation date is set and "nogit" is specified:
-/// Stockfish dev-YYYYMMDD-SHA
-/// or
-/// Stockfish dev-YYYYMMDD-nogit
-///
-/// For releases (non dev builds) we only include the version number:
-/// Stockfish version
+/// engine_info() returns the full name of the current Stockfish version. This
+/// will be either "Stockfish <Tag> DD-MM-YY" (where DD-MM-YY is the date when
+/// the program was compiled) or "Stockfish <Version>", depending on whether
+/// Version is empty.
 
 string engine_info(bool to_uci) {
-  stringstream ss;
-  ss << "Stockfish " << version << setfill('0');
 
-  if constexpr (version == "dev")
+  const string months("Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec");
+  string month, day, year;
+  stringstream ss, date(__DATE__); // From compiler, format is "Sep 21 2008"
+
+  ss << "Stockfish " << Version << setfill('0');
+
+  if (Version.empty())
   {
-      ss << "-";
-      #ifdef GIT_DATE
-      ss << stringify(GIT_DATE);
-      #else
-      constexpr string_view months("Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec");
-      string month, day, year;
-      stringstream date(__DATE__); // From compiler, format is "Sep 21 2008"
-
       date >> month >> day >> year;
-      ss << year << setw(2) << setfill('0') << (1 + months.find(month) / 4) << setw(2) << setfill('0') << day;
-      #endif
-
-      ss << "-";
-
-      #ifdef GIT_SHA
-      ss << stringify(GIT_SHA);
-      #else
-      ss << "nogit";
-      #endif
+      ss << setw(2) << day << setw(2) << (1 + months.find(month) / 4) << year.substr(2);
   }
 
   ss << (to_uci  ? "\nid author ": " by ")
@@ -190,6 +170,8 @@ string engine_info(bool to_uci) {
 
 std::string compiler_info() {
 
+  #define stringify2(x) #x
+  #define stringify(x) stringify2(x)
   #define make_version_string(major, minor, patch) stringify(major) "." stringify(minor) "." stringify(patch)
 
 /// Predefined macros hell:
@@ -301,94 +283,21 @@ std::string compiler_info() {
 
 
 /// Debug functions used mainly to collect run-time statistics
-constexpr int MaxDebugSlots = 32;
+static std::atomic<int64_t> hits[2], means[2];
 
-namespace {
-
-template<size_t N>
-struct DebugInfo {
-    std::atomic<int64_t> data[N] = { 0 };
-
-    constexpr inline std::atomic<int64_t>& operator[](int index) { return data[index]; }
-};
-
-DebugInfo<2> hit[MaxDebugSlots];
-DebugInfo<2> mean[MaxDebugSlots];
-DebugInfo<3> stdev[MaxDebugSlots];
-DebugInfo<6> correl[MaxDebugSlots];
-
-}  // namespace
-
-void dbg_hit_on(bool cond, int slot) {
-
-    ++hit[slot][0];
-    if (cond)
-        ++hit[slot][1];
-}
-
-void dbg_mean_of(int64_t value, int slot) {
-
-    ++mean[slot][0];
-    mean[slot][1] += value;
-}
-
-void dbg_stdev_of(int64_t value, int slot) {
-
-    ++stdev[slot][0];
-    stdev[slot][1] += value;
-    stdev[slot][2] += value * value;
-}
-
-void dbg_correl_of(int64_t value1, int64_t value2, int slot) {
-
-    ++correl[slot][0];
-    correl[slot][1] += value1;
-    correl[slot][2] += value1 * value1;
-    correl[slot][3] += value2;
-    correl[slot][4] += value2 * value2;
-    correl[slot][5] += value1 * value2;
-}
+void dbg_hit_on(bool b) { ++hits[0]; if (b) ++hits[1]; }
+void dbg_hit_on(bool c, bool b) { if (c) dbg_hit_on(b); }
+void dbg_mean_of(int v) { ++means[0]; means[1] += v; }
 
 void dbg_print() {
 
-    int64_t n;
-    auto E   = [&n](int64_t x) { return double(x) / n; };
-    auto sqr = [](double x) { return x * x; };
+  if (hits[0])
+      cerr << "Total " << hits[0] << " Hits " << hits[1]
+           << " hit rate (%) " << 100 * hits[1] / hits[0] << endl;
 
-    for (int i = 0; i < MaxDebugSlots; ++i)
-        if ((n = hit[i][0]))
-            std::cerr << "Hit #" << i
-                      << ": Total " << n << " Hits " << hit[i][1]
-                      << " Hit Rate (%) " << 100.0 * E(hit[i][1])
-                      << std::endl;
-
-    for (int i = 0; i < MaxDebugSlots; ++i)
-        if ((n = mean[i][0]))
-        {
-            std::cerr << "Mean #" << i
-                      << ": Total " << n << " Mean " << E(mean[i][1])
-                      << std::endl;
-        }
-
-    for (int i = 0; i < MaxDebugSlots; ++i)
-        if ((n = stdev[i][0]))
-        {
-            double r = sqrtl(E(stdev[i][2]) - sqr(E(stdev[i][1])));
-            std::cerr << "Stdev #" << i
-                      << ": Total " << n << " Stdev " << r
-                      << std::endl;
-        }
-
-    for (int i = 0; i < MaxDebugSlots; ++i)
-        if ((n = correl[i][0]))
-        {
-            double r = (E(correl[i][5]) - E(correl[i][1]) * E(correl[i][3]))
-                       / (  sqrtl(E(correl[i][2]) - sqr(E(correl[i][1])))
-                          * sqrtl(E(correl[i][4]) - sqr(E(correl[i][3]))));
-            std::cerr << "Correl. #" << i
-                      << ": Total " << n << " Coefficient " << r
-                      << std::endl;
-        }
+  if (means[0])
+      cerr << "Total " << means[0] << " Mean "
+           << (double)means[1] / means[0] << endl;
 }
 
 
@@ -449,10 +358,8 @@ void* std_aligned_alloc(size_t alignment, size_t size) {
 #if defined(POSIXALIGNEDALLOC)
   void *mem;
   return posix_memalign(&mem, alignment, size) ? nullptr : mem;
-#elif defined(_WIN32) && !defined(_M_ARM) && !defined(_M_ARM64)
-  return _mm_malloc(size, alignment);
 #elif defined(_WIN32)
-  return _aligned_malloc(size, alignment);
+  return _mm_malloc(size, alignment);
 #else
   return std::aligned_alloc(alignment, size);
 #endif
@@ -462,10 +369,8 @@ void std_aligned_free(void* ptr) {
 
 #if defined(POSIXALIGNEDALLOC)
   free(ptr);
-#elif defined(_WIN32) && !defined(_M_ARM) && !defined(_M_ARM64)
-  _mm_free(ptr);
 #elif defined(_WIN32)
-  _aligned_free(ptr);
+  _mm_free(ptr);
 #else
   free(ptr);
 #endif
@@ -475,9 +380,10 @@ void std_aligned_free(void* ptr) {
 
 #if defined(_WIN32)
 
-static void* aligned_large_pages_alloc_windows([[maybe_unused]] size_t allocSize) {
+static void* aligned_large_pages_alloc_windows(size_t allocSize) {
 
   #if !defined(_WIN64)
+    (void)allocSize; // suppress unused-parameter compiler warning
     return nullptr;
   #else
 
@@ -489,30 +395,11 @@ static void* aligned_large_pages_alloc_windows([[maybe_unused]] size_t allocSize
   if (!largePageSize)
       return nullptr;
 
-  // Dynamically link OpenProcessToken, LookupPrivilegeValue and AdjustTokenPrivileges
-
-  HMODULE hAdvapi32 = GetModuleHandle(TEXT("advapi32.dll"));
-
-  if (!hAdvapi32)
-      hAdvapi32 = LoadLibrary(TEXT("advapi32.dll"));
-
-  auto fun6 = (fun6_t)(void(*)())GetProcAddress(hAdvapi32, "OpenProcessToken");
-  if (!fun6)
-      return nullptr;
-  auto fun7 = (fun7_t)(void(*)())GetProcAddress(hAdvapi32, "LookupPrivilegeValueA");
-  if (!fun7)
-      return nullptr;
-  auto fun8 = (fun8_t)(void(*)())GetProcAddress(hAdvapi32, "AdjustTokenPrivileges");
-  if (!fun8)
-      return nullptr;
-
   // We need SeLockMemoryPrivilege, so try to enable it for the process
-  if (!fun6( // OpenProcessToken()
-      GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hProcessToken))
-          return nullptr;
+  if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hProcessToken))
+      return nullptr;
 
-  if (fun7( // LookupPrivilegeValue(nullptr, SE_LOCK_MEMORY_NAME, &luid)
-      nullptr, "SeLockMemoryPrivilege", &luid))
+  if (LookupPrivilegeValue(NULL, SE_LOCK_MEMORY_NAME, &luid))
   {
       TOKEN_PRIVILEGES tp { };
       TOKEN_PRIVILEGES prevTp { };
@@ -524,18 +411,17 @@ static void* aligned_large_pages_alloc_windows([[maybe_unused]] size_t allocSize
 
       // Try to enable SeLockMemoryPrivilege. Note that even if AdjustTokenPrivileges() succeeds,
       // we still need to query GetLastError() to ensure that the privileges were actually obtained.
-      if (fun8( // AdjustTokenPrivileges()
+      if (AdjustTokenPrivileges(
               hProcessToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), &prevTp, &prevTpLen) &&
           GetLastError() == ERROR_SUCCESS)
       {
           // Round up size to full pages and allocate
           allocSize = (allocSize + largePageSize - 1) & ~size_t(largePageSize - 1);
           mem = VirtualAlloc(
-              nullptr, allocSize, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
+              NULL, allocSize, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
 
           // Privilege no longer needed, restore previous state
-          fun8( // AdjustTokenPrivileges ()
-              hProcessToken, FALSE, &prevTp, 0, nullptr, nullptr);
+          AdjustTokenPrivileges(hProcessToken, FALSE, &prevTp, 0, NULL, NULL);
       }
   }
 
@@ -553,7 +439,7 @@ void* aligned_large_pages_alloc(size_t allocSize) {
 
   // Fall back to regular, page aligned, allocation if necessary
   if (!mem)
-      mem = VirtualAlloc(nullptr, allocSize, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+      mem = VirtualAlloc(NULL, allocSize, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
 
   return mem;
 }
@@ -617,7 +503,7 @@ void bindThisThread(size_t) {}
 /// API and returns the best node id for the thread with index idx. Original
 /// code from Texel by Peter Österlund.
 
-static int best_node(size_t idx) {
+int best_node(size_t idx) {
 
   int threads = 0;
   int nodes = 0;
@@ -626,7 +512,7 @@ static int best_node(size_t idx) {
   DWORD byteOffset = 0;
 
   // Early exit if the needed API is not available at runtime
-  HMODULE k32 = GetModuleHandle(TEXT("Kernel32.dll"));
+  HMODULE k32 = GetModuleHandle("Kernel32.dll");
   auto fun1 = (fun1_t)(void(*)())GetProcAddress(k32, "GetLogicalProcessorInformationEx");
   if (!fun1)
       return -1;
@@ -696,7 +582,7 @@ void bindThisThread(size_t idx) {
       return;
 
   // Early exit if the needed API are not available at runtime
-  HMODULE k32 = GetModuleHandle(TEXT("Kernel32.dll"));
+  HMODULE k32 = GetModuleHandle("Kernel32.dll");
   auto fun2 = (fun2_t)(void(*)())GetProcAddress(k32, "GetNumaNodeProcessorMaskEx");
   auto fun3 = (fun3_t)(void(*)())GetProcAddress(k32, "SetThreadGroupAffinity");
   auto fun4 = (fun4_t)(void(*)())GetProcAddress(k32, "GetNumaNodeProcessorMask2");
@@ -742,7 +628,8 @@ string argv0;            // path+name of the executable binary, as given by argv
 string binaryDirectory;  // path of the executable directory
 string workingDirectory; // path of the working directory
 
-void init([[maybe_unused]] int argc, char* argv[]) {
+void init(int argc, char* argv[]) {
+    (void)argc;
     string pathSeparator;
 
     // extract the path+name of the executable binary
@@ -784,4 +671,30 @@ void init([[maybe_unused]] int argc, char* argv[]) {
 
 } // namespace CommandLine
 
+// Returns a string that represents the current time. (Used when learning evaluation functions)
+std::string now_string()
+{
+    // Using std::ctime(), localtime() gives a warning that MSVC is not secure.
+    // This shouldn't happen in the C++ standard, but...
+
+#if defined(_MSC_VER)
+  // C4996 : 'ctime' : This function or variable may be unsafe.Consider using ctime_s instead.
+#pragma warning(disable : 4996)
+#endif
+
+    auto now = std::chrono::system_clock::now();
+    auto tp = std::chrono::system_clock::to_time_t(now);
+    auto result = string(std::ctime(&tp));
+
+    // remove line endings if they are included at the end
+    while (*result.rbegin() == '\n' || (*result.rbegin() == '\r'))
+        result.pop_back();
+    return result;
+}
+
+void sleep(int ms)
+{
+    std::this_thread::sleep_for(std::chrono::milliseconds(ms));
+}
+
 } // namespace Stockfish

src/misc.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -19,18 +19,24 @@
 #ifndef MISC_H_INCLUDED
 #define MISC_H_INCLUDED
 
+#include <algorithm>
 #include <cassert>
 #include <chrono>
+#include <functional>
+#include <mutex>
 #include <ostream>
 #include <string>
 #include <vector>
+#include <iostream>
+
 #include <cstdint>
+#include <cmath>
+#include <cctype>
+#include <sstream>
+#include <deque>
 
 #include "types.h"
 
-#define stringify2(x) #x
-#define stringify(x) stringify2(x)
-
 namespace Stockfish {
 
 std::string engine_info(bool to_uci = false);
@@ -42,13 +48,26 @@ void std_aligned_free(void* ptr);
 void* aligned_large_pages_alloc(size_t size); // memory aligned by page size, min alignment: 4096 bytes
 void aligned_large_pages_free(void* mem); // nop if mem == nullptr
 
-void dbg_hit_on(bool cond, int slot = 0);
-void dbg_mean_of(int64_t value, int slot = 0);
-void dbg_stdev_of(int64_t value, int slot = 0);
-void dbg_correl_of(int64_t value1, int64_t value2, int slot = 0);
+void dbg_hit_on(bool b);
+void dbg_hit_on(bool c, bool b);
+void dbg_mean_of(int v);
 void dbg_print();
 
-using TimePoint = std::chrono::milliseconds::rep; // A value in milliseconds
+/// Debug macro to write to std::err if NDEBUG flag is set, and do nothing otherwise
+#if defined(NDEBUG)
+#define debug 1 && std::cerr
+#else
+#define debug 0 && std::cerr
+#endif
+
+inline void hit_any_key() {
+#ifndef NDEBUG
+  debug << "Hit any key to continue..." << std::endl << std::flush;
+  system("read");   // on Windows, should be system("pause");
+#endif
+}
+
+typedef std::chrono::milliseconds::rep TimePoint; // A value in milliseconds
 static_assert(sizeof(TimePoint) == sizeof(int64_t), "TimePoint should be 64 bits");
 inline TimePoint now() {
   return std::chrono::duration_cast<std::chrono::milliseconds>
@@ -89,20 +108,299 @@ static inline const union { uint32_t i; char c[4]; } Le = { 0x01020304 };
 static inline const bool IsLittleEndian = (Le.c[0] == 4);
 
 
+// RunningAverage : a class to calculate a running average of a series of values.
+// For efficiency, all computations are done with integers.
+class RunningAverage {
+  public:
+
+      // Reset the running average to rational value p / q
+      void set(int64_t p, int64_t q)
+        { average = p * PERIOD * RESOLUTION / q; }
+
+      // Update average with value v
+      void update(int64_t v)
+        { average = RESOLUTION * v + (PERIOD - 1) * average / PERIOD; }
+
+      // Test if average is strictly greater than rational a / b
+      bool is_greater(int64_t a, int64_t b) const
+        { return b * average > a * (PERIOD * RESOLUTION); }
+
+      int64_t value() const
+        { return average / (PERIOD * RESOLUTION); }
+
+  private :
+      static constexpr int64_t PERIOD     = 4096;
+      static constexpr int64_t RESOLUTION = 1024;
+      int64_t average;
+};
+
 template <typename T, std::size_t MaxSize>
 class ValueList {
 
 public:
   std::size_t size() const { return size_; }
+  void resize(std::size_t newSize) { size_ = newSize; }
   void push_back(const T& value) { values_[size_++] = value; }
+  T& operator[](std::size_t index) { return values_[index]; }
+  T* begin() { return values_; }
+  T* end() { return values_ + size_; }
+  const T& operator[](std::size_t index) const { return values_[index]; }
   const T* begin() const { return values_; }
   const T* end() const { return values_ + size_; }
 
+  void swap(ValueList& other) {
+    const std::size_t maxSize = std::max(size_, other.size_);
+    for (std::size_t i = 0; i < maxSize; ++i) {
+      std::swap(values_[i], other.values_[i]);
+    }
+    std::swap(size_, other.size_);
+  }
+
 private:
   T values_[MaxSize];
   std::size_t size_ = 0;
 };
 
+// This logger allows printing many parts in a region atomically
+// but doesn't block the threads trying to append to other regions.
+// Instead if some region tries to pring while other region holds
+// the lock the messages are queued to be printed as soon as the
+// current region releases the lock.
+struct SynchronizedRegionLogger
+{
+  using RegionId = std::uint64_t;
+
+  struct Region
+  {
+    friend struct SynchronizedRegionLogger;
+
+    Region() :
+      logger(nullptr), region_id(0), is_held(false)
+    {
+    }
+
+    Region(const Region&) = delete;
+    Region& operator=(const Region&) = delete;
+
+    Region(Region&& other) :
+      logger(other.logger), region_id(other.region_id), is_held(other.is_held)
+    {
+      other.logger = nullptr;
+      other.is_held = false;
+    }
+
+    Region& operator=(Region&& other) {
+      if (is_held && logger != nullptr)
+      {
+        logger->release_region(region_id);
+      }
+
+      logger = other.logger;
+      region_id = other.region_id;
+      is_held = other.is_held;
+
+      other.is_held = false;
+
+      return *this;
+    }
+
+    ~Region() { unlock(); }
+
+    void unlock() {
+      if (is_held) {
+        is_held = false;
+
+        if (logger != nullptr)
+          logger->release_region(region_id);
+      }
+    }
+
+    Region& operator << (std::ostream&(*pManip)(std::ostream&)) {
+      if (logger != nullptr)
+        logger->write(region_id, pManip);
+
+      return *this;
+    }
+
+    template <typename T>
+    Region& operator << (const T& value) {
+      if (logger != nullptr)
+        logger->write(region_id, value);
+
+      return *this;
+    }
+
+  private:
+    SynchronizedRegionLogger* logger;
+    RegionId region_id;
+    bool is_held;
+
+    Region(SynchronizedRegionLogger& log, RegionId id) :
+      logger(&log), region_id(id), is_held(true)
+    {
+    }
+  };
+
+private:
+  struct RegionBookkeeping
+  {
+    RegionBookkeeping(RegionId rid) : id(rid), is_held(true) {}
+
+    std::vector<std::string> pending_parts;
+    RegionId id;
+    bool is_held;
+  };
+
+  RegionId init_next_region()
+  {
+    static RegionId next_id = 0;
+
+    std::lock_guard lock(mutex);
+
+    const auto id = next_id++;
+    regions.emplace_back(id);
+
+    return id;
+  }
+
+  void write(RegionId id, std::ostream&(*pManip)(std::ostream&)) {
+    std::lock_guard lock(mutex);
+
+    if (regions.empty())
+      return;
+
+    if (id == regions.front().id) {
+      // We can just directly print to the output because
+      // we are at the front of the region queue.
+      out << *pManip;
+    } else {
+      // We have to schedule the print until previous regions are
+      // processed
+      auto* region = find_region_nolock(id);
+      if (region == nullptr)
+        return;
+
+      std::stringstream ss;
+      ss << *pManip;
+      region->pending_parts.emplace_back(std::move(ss).str());
+    }
+  }
+
+  template <typename T>
+  void write(RegionId id, const T& value) {
+    std::lock_guard lock(mutex);
+
+    if (regions.empty())
+      return;
+
+    if (id == regions.front().id) {
+      // We can just directly print to the output because
+      // we are at the front of the region queue.
+      out << value;
+    } else {
+      // We have to schedule the print until previous regions are
+      // processed
+      auto* region = find_region_nolock(id);
+      if (region == nullptr)
+        return;
+
+      std::stringstream ss;
+      ss << value;
+      region->pending_parts.emplace_back(std::move(ss).str());
+    }
+  }
+
+  std::ostream& out;
+
+  std::deque<RegionBookkeeping> regions;
+
+  std::mutex mutex;
+
+  RegionBookkeeping* find_region_nolock(RegionId id) {
+    // Linear search because the amount of concurrent regions should be small.
+    auto it = std::find_if(
+      regions.begin(),
+      regions.end(),
+      [id](const RegionBookkeeping& r) { return r.id == id; });
+
+    if (it == regions.end())
+      return nullptr;
+    else
+      return &*it;
+  }
+
+  void release_region(RegionId id) {
+    std::lock_guard lock(mutex);
+
+    auto* region = find_region_nolock(id);
+    if (region == nullptr)
+      return;
+
+    region->is_held = false;
+
+    process_backlog_nolock();
+  }
+
+  void process_backlog_nolock()
+  {
+    while(!regions.empty()) {
+      auto& region = regions.front();
+
+      for(auto& part : region.pending_parts) {
+        out << part;
+      }
+
+      // If the region is still held then we don't
+      // want to start printing stuff from the next region.
+      if (region.is_held)
+        break;
+
+      regions.pop_front();
+    }
+  }
+
+public:
+
+  SynchronizedRegionLogger(std::ostream& s) :
+    out(s)
+  {
+  }
+
+  [[nodiscard]] Region new_region() {
+    const auto id = init_next_region();
+    return Region(*this, id);
+  }
+
+};
+
+extern SynchronizedRegionLogger sync_region_cout;
+
+/// sigmoid(t, x0, y0, C, P, Q) implements a sigmoid-like function using only integers,
+/// with the following properties:
+///
+///  -  sigmoid is centered in (x0, y0)
+///  -  sigmoid has amplitude [-P/Q , P/Q] instead of [-1 , +1]
+///  -  limit is (y0 - P/Q) when t tends to -infinity
+///  -  limit is (y0 + P/Q) when t tends to +infinity
+///  -  the slope can be adjusted using C > 0, smaller C giving a steeper sigmoid
+///  -  the slope of the sigmoid when t = x0 is P/(Q*C)
+///  -  sigmoid is increasing with t when P > 0 and Q > 0
+///  -  to get a decreasing sigmoid, change sign of P
+///  -  mean value of the sigmoid is y0
+///
+/// Use <https://www.desmos.com/calculator/jhh83sqq92> to draw the sigmoid
+
+inline int64_t sigmoid(int64_t t, int64_t x0,
+                                  int64_t y0,
+                                  int64_t  C,
+                                  int64_t  P,
+                                  int64_t  Q)
+{
+   assert(C > 0);
+   assert(Q != 0);
+   return y0 + P * (t-x0) / (Q * (std::abs(t-x0) + C)) ;
+}
+
 
 /// xorshift64star Pseudo-Random Number Generator
 /// This class is based on original code written and dedicated
@@ -119,6 +417,19 @@ private:
 /// For further analysis see
 ///   <http://vigna.di.unimi.it/ftp/papers/xorshift.pdf>
 
+static uint64_t string_hash(const std::string& str)
+{
+  uint64_t h = 525201411107845655ull;
+
+  for (auto c : str) {
+    h ^= static_cast<uint64_t>(c);
+    h *= 0x5bd1e9955bd1e995ull;
+    h ^= h >> 47;
+  }
+
+  return h;
+}
+
 class PRNG {
 
   uint64_t s;
@@ -130,7 +441,9 @@ class PRNG {
   }
 
 public:
+  PRNG() { set_seed_from_time(); }
   PRNG(uint64_t seed) : s(seed) { assert(seed); }
+  PRNG(const std::string& seed) { set_seed(seed); }
 
   template<typename T> T rand() { return T(rand64()); }
 
@@ -138,11 +451,57 @@ public:
   /// Output values only have 1/8th of their bits set on average.
   template<typename T> T sparse_rand()
   { return T(rand64() & rand64() & rand64()); }
+  // Returns a random number from 0 to n-1. (Not uniform distribution, but this is enough in reality)
+  uint64_t rand(uint64_t n) { return rand<uint64_t>() % n; }
+
+  // Return the random seed used internally.
+  uint64_t get_seed() const { return s; }
+
+  void set_seed(uint64_t seed) { s = seed; }
+
+  uint64_t next_random_seed()
+  {
+    uint64_t seed = 0;
+    for(int i = 0; i < 64; ++i)
+    {
+      const auto off = rand64() % 64;
+      seed |= (rand64() & (uint64_t(1) << off)) >> off;
+      seed <<= 1;
+    }
+    return seed;
+  }
+
+  void set_seed_from_time()
+  {
+      set_seed(std::chrono::system_clock::now().time_since_epoch().count());
+  }
+
+  void set_seed(const std::string& str)
+  {
+    if (str.empty())
+    {
+      set_seed_from_time();
+    }
+    else if (std::all_of(str.begin(), str.end(), [](char c) { return std::isdigit(c);} )) {
+      set_seed(std::stoull(str));
+    }
+    else
+    {
+      set_seed(string_hash(str));
+    }
+  }
 };
 
+// Display a random seed. (For debugging)
+inline std::ostream& operator<<(std::ostream& os, PRNG& prng)
+{
+  os << "PRNG::seed = " << std::hex << prng.get_seed() << std::dec;
+  return os;
+}
+
 inline uint64_t mul_hi64(uint64_t a, uint64_t b) {
 #if defined(__GNUC__) && defined(IS_64BIT)
-    __extension__ using uint128 = unsigned __int128;
+    __extension__ typedef unsigned __int128 uint128;
     return ((uint128)a * (uint128)b) >> 64;
 #else
     uint64_t aL = (uint32_t)a, aH = a >> 32;
@@ -154,6 +513,74 @@ inline uint64_t mul_hi64(uint64_t a, uint64_t b) {
 #endif
 }
 
+// This bitset can be accessed concurrently, provided
+// the concurrent accesses are performed on distinct
+// instances of underlying type. That means the cuncurrent
+// accesses need to be spaced by at least
+// bits_per_bucket bits.
+// But at least best_concurrent_access_stride bits
+// is recommended to prevent false sharing.
+template <uint64_t N>
+struct LargeBitset
+{
+private:
+    constexpr static uint64_t cache_line_size = 64;
+
+public:
+    using UnderlyingType = uint64_t;
+
+    constexpr static uint64_t num_bits = N;
+    constexpr static uint64_t bits_per_bucket = 8 * sizeof(uint64_t);
+    constexpr static uint64_t num_buckets = (num_bits + bits_per_bucket - 1) / bits_per_bucket;
+    constexpr static uint64_t best_concurrent_access_stride = 8 * cache_line_size;
+
+    LargeBitset()
+    {
+        std::fill(std::begin(bits), std::end(bits), 0);
+    }
+
+    void set(uint64_t idx)
+    {
+        const uint64_t bucket = idx / bits_per_bucket;
+        const uint64_t bit = uint64_t(1) << (idx % bits_per_bucket);
+        bits[bucket] |= bit;
+    }
+
+    bool test(uint64_t idx) const
+    {
+        const uint64_t bucket = idx / bits_per_bucket;
+        const uint64_t bit = uint64_t(1) << (idx % bits_per_bucket);
+        return bits[bucket] & bit;
+    }
+
+    uint64_t count() const
+    {
+        uint64_t c = 0;
+        uint64_t i = 0;
+
+        for (; i < num_buckets - 3; i += 4)
+        {
+            uint64_t c0 = popcount(bits[i+0]);
+            uint64_t c1 = popcount(bits[i+1]);
+            uint64_t c2 = popcount(bits[i+2]);
+            uint64_t c3 = popcount(bits[i+3]);
+            c0 += c1;
+            c2 += c3;
+            c += c0 + c2;
+        }
+
+        for (; i < num_buckets; ++i)
+        {
+            c += popcount(bits[i]);
+        }
+
+        return c;
+    }
+
+private:
+    alignas(cache_line_size) UnderlyingType bits[num_buckets];
+};
+
 /// Under Windows it is not possible for a process to run on more than one
 /// logical processor group. This usually means to be limited to use max 64
 /// cores. To overcome this, some special platform specific API should be
@@ -164,6 +591,111 @@ namespace WinProcGroup {
   void bindThisThread(size_t idx);
 }
 
+// Returns a string that represents the current time. (Used for log output when learning evaluation function)
+std::string now_string();
+void sleep(int ms);
+
+namespace Algo {
+    // Fisher-Yates
+    template <typename Rng, typename T>
+    void shuffle(std::vector<T>& buf, Rng&& prng)
+    {
+        const auto size = buf.size();
+        for (uint64_t i = 0; i < size; ++i)
+            std::swap(buf[i], buf[prng.rand(size - i) + i]);
+    }
+
+    // split the string
+    inline std::vector<std::string> split(const std::string& input, char delimiter) {
+        std::istringstream stream(input);
+        std::string field;
+        std::vector<std::string> fields;
+
+        while (std::getline(stream, field, delimiter)) {
+            fields.push_back(field);
+        }
+
+        return fields;
+    }
+}
+
+// --------------------
+//       Path
+// --------------------
+
+// Something like Path class in C#. File name manipulation.
+// Match with the C# method name.
+struct Path
+{
+	// Combine the path name and file name and return it.
+	// If the folder name is not an empty string, append it if there is no'/' or'\\' at the end.
+	static std::string combine(const std::string& folder, const std::string& filename)
+	{
+		if (folder.length() >= 1 && *folder.rbegin() != '/' && *folder.rbegin() != '\\')
+			return folder + "/" + filename;
+
+		return folder + filename;
+	}
+
+	// Get the file name part (excluding the folder name) from the full path expression.
+	static std::string get_file_name(const std::string& path)
+	{
+		// I don't know which "\" or "/" is used.
+		auto path_index1 = path.find_last_of("\\") + 1;
+		auto path_index2 = path.find_last_of("/") + 1;
+		auto path_index = std::max(path_index1, path_index2);
+
+		return path.substr(path_index);
+	}
+};
+
+// It is ignored when new even though alignas is specified & because it is ignored when the STL container allocates memory,
+// A custom allocator used for that.
+template <typename T>
+class AlignedAllocator {
+public:
+  using value_type = T;
+
+  AlignedAllocator() {}
+  AlignedAllocator(const AlignedAllocator&) {}
+  AlignedAllocator(AlignedAllocator&&) {}
+
+  template <typename U> AlignedAllocator(const AlignedAllocator<U>&) {}
+
+  T* allocate(std::size_t n) { return (T*)std_aligned_alloc(alignof(T), n * sizeof(T)); }
+  void deallocate(T* p, std::size_t ) { std_aligned_free(p); }
+};
+
+template <typename T>
+class CacheLineAlignedAllocator {
+public:
+    using value_type = T;
+
+    constexpr static uint64_t cache_line_size = 64;
+
+    CacheLineAlignedAllocator() {}
+    CacheLineAlignedAllocator(const CacheLineAlignedAllocator&) {}
+    CacheLineAlignedAllocator(CacheLineAlignedAllocator&&) {}
+
+    template <typename U> CacheLineAlignedAllocator(const CacheLineAlignedAllocator<U>&) {}
+
+    T* allocate(std::size_t n) { return (T*)std_aligned_alloc(cache_line_size, n * sizeof(T)); }
+    void deallocate(T* p, std::size_t) { std_aligned_free(p); }
+};
+
+// --------------------
+//  Dependency Wrapper
+// --------------------
+
+namespace Dependency
+{
+  // In the Linux environment, if you getline() the text file is'\r\n'
+  // Since'\r' remains at the end, write a wrapper to remove this'\r'.
+  // So when calling getline() on fstream,
+  // just write getline() instead of std::getline() and use this function.
+  extern bool getline(std::ifstream& fs, std::string& s);
+}
+
 namespace CommandLine {
   void init(int argc, char* argv[]);
 

src/movegen.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -25,21 +25,13 @@ namespace Stockfish {
 
 namespace {
 
-  template<GenType Type, Direction D, bool Enemy>
-  ExtMove* make_promotions(ExtMove* moveList, [[maybe_unused]] Square to) {
+  template<GenType Type, Direction D>
+  ExtMove* make_promotions(ExtMove* moveList, Square to) {
 
-    if constexpr (Type == CAPTURES || Type == EVASIONS || Type == NON_EVASIONS)
-    {
+    if (Type == CAPTURES || Type == EVASIONS || Type == NON_EVASIONS)
         *moveList++ = make<PROMOTION>(to - D, to, QUEEN);
-        if constexpr (Enemy && Type == CAPTURES)
-        {
-            *moveList++ = make<PROMOTION>(to - D, to, ROOK);
-            *moveList++ = make<PROMOTION>(to - D, to, BISHOP);
-            *moveList++ = make<PROMOTION>(to - D, to, KNIGHT);
-        }
-    }
 
-    if constexpr ((Type == QUIETS && !Enemy) || Type == EVASIONS || Type == NON_EVASIONS)
+    if (Type == QUIETS || Type == EVASIONS || Type == NON_EVASIONS)
     {
         *moveList++ = make<PROMOTION>(to - D, to, ROOK);
         *moveList++ = make<PROMOTION>(to - D, to, BISHOP);
@@ -68,18 +60,18 @@ namespace {
     Bitboard pawnsNotOn7 = pos.pieces(Us, PAWN) & ~TRank7BB;
 
     // Single and double pawn pushes, no promotions
-    if constexpr (Type != CAPTURES)
+    if (Type != CAPTURES)
     {
         Bitboard b1 = shift<Up>(pawnsNotOn7)   & emptySquares;
         Bitboard b2 = shift<Up>(b1 & TRank3BB) & emptySquares;
 
-        if constexpr (Type == EVASIONS) // Consider only blocking squares
+        if (Type == EVASIONS) // Consider only blocking squares
         {
             b1 &= target;
             b2 &= target;
         }
 
-        if constexpr (Type == QUIET_CHECKS)
+        if (Type == QUIET_CHECKS)
         {
             // To make a quiet check, you either make a direct check by pushing a pawn
             // or push a blocker pawn that is not on the same file as the enemy king.
@@ -110,21 +102,21 @@ namespace {
         Bitboard b2 = shift<UpLeft >(pawnsOn7) & enemies;
         Bitboard b3 = shift<Up     >(pawnsOn7) & emptySquares;
 
-        if constexpr (Type == EVASIONS)
+        if (Type == EVASIONS)
             b3 &= target;
 
         while (b1)
-            moveList = make_promotions<Type, UpRight, true>(moveList, pop_lsb(b1));
+            moveList = make_promotions<Type, UpRight>(moveList, pop_lsb(b1));
 
         while (b2)
-            moveList = make_promotions<Type, UpLeft, true>(moveList, pop_lsb(b2));
+            moveList = make_promotions<Type, UpLeft >(moveList, pop_lsb(b2));
 
         while (b3)
-            moveList = make_promotions<Type, Up,    false>(moveList, pop_lsb(b3));
+            moveList = make_promotions<Type, Up     >(moveList, pop_lsb(b3));
     }
 
     // Standard and en passant captures
-    if constexpr (Type == CAPTURES || Type == EVASIONS || Type == NON_EVASIONS)
+    if (Type == CAPTURES || Type == EVASIONS || Type == NON_EVASIONS)
     {
         Bitboard b1 = shift<UpRight>(pawnsNotOn7) & enemies;
         Bitboard b2 = shift<UpLeft >(pawnsNotOn7) & enemies;
@@ -272,7 +264,7 @@ ExtMove* generate<LEGAL>(const Position& pos, ExtMove* moveList) {
   moveList = pos.checkers() ? generate<EVASIONS    >(pos, moveList)
                             : generate<NON_EVASIONS>(pos, moveList);
   while (cur != moveList)
-      if (  ((pinned & from_sq(*cur)) || from_sq(*cur) == ksq || type_of(*cur) == EN_PASSANT)
+      if (  ((pinned && pinned & from_sq(*cur)) || from_sq(*cur) == ksq || type_of(*cur) == EN_PASSANT)
           && !pos.legal(*cur))
           *cur = (--moveList)->move;
       else

src/movegen.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -68,6 +68,9 @@ struct MoveList {
     return std::find(begin(), end(), move) != end();
   }
 
+  // returns the i th element
+  const ExtMove at(size_t i) const { assert(0 <= i && i < size()); return begin()[i]; }
+
 private:
   ExtMove moveList[MAX_MOVES], *last;
 };

src/movepick.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -82,71 +82,82 @@ MovePicker::MovePicker(const Position& p, Move ttm, Depth d, const ButterflyHist
 
   stage = (pos.checkers() ? EVASION_TT : QSEARCH_TT) +
           !(   ttm
+            && (pos.checkers() || depth > DEPTH_QS_RECAPTURES || to_sq(ttm) == recaptureSquare)
             && pos.pseudo_legal(ttm));
 }
 
 /// MovePicker constructor for ProbCut: we generate captures with SEE greater
 /// than or equal to the given threshold.
-MovePicker::MovePicker(const Position& p, Move ttm, Value th, const CapturePieceToHistory* cph)
-           : pos(p), captureHistory(cph), ttMove(ttm), threshold(th)
+MovePicker::MovePicker(const Position& p, Move ttm, Value th, Depth d, const CapturePieceToHistory* cph)
+           : pos(p), captureHistory(cph), ttMove(ttm), threshold(th), depth(d)
 {
   assert(!pos.checkers());
 
-  stage = PROBCUT_TT + !(ttm && pos.capture_stage(ttm)
+  stage = PROBCUT_TT + !(ttm && pos.capture(ttm)
                              && pos.pseudo_legal(ttm)
                              && pos.see_ge(ttm, threshold));
 }
 
 /// MovePicker::score() assigns a numerical value to each move in a list, used
 /// for sorting. Captures are ordered by Most Valuable Victim (MVV), preferring
-/// captures with a good history. Quiets moves are ordered using the history tables.
+/// captures with a good history. Quiets moves are ordered using the histories.
 template<GenType Type>
 void MovePicker::score() {
 
   static_assert(Type == CAPTURES || Type == QUIETS || Type == EVASIONS, "Wrong type");
 
-  [[maybe_unused]] Bitboard threatenedByPawn, threatenedByMinor, threatenedByRook, threatenedPieces;
+  Bitboard threatened, threatenedByPawn, threatenedByMinor, threatenedByRook;
   if constexpr (Type == QUIETS)
   {
       Color us = pos.side_to_move();
-
+      // squares threatened by pawns
       threatenedByPawn  = pos.attacks_by<PAWN>(~us);
+      // squares threatened by minors or pawns
       threatenedByMinor = pos.attacks_by<KNIGHT>(~us) | pos.attacks_by<BISHOP>(~us) | threatenedByPawn;
+      // squares threatened by rooks, minors or pawns
       threatenedByRook  = pos.attacks_by<ROOK>(~us) | threatenedByMinor;
 
-      // Pieces threatened by pieces of lesser material value
-      threatenedPieces = (pos.pieces(us, QUEEN) & threatenedByRook)
-                       | (pos.pieces(us, ROOK)  & threatenedByMinor)
-                       | (pos.pieces(us, KNIGHT, BISHOP) & threatenedByPawn);
+      // pieces threatened by pieces of lesser material value
+      threatened =  (pos.pieces(us, QUEEN) & threatenedByRook)
+                  | (pos.pieces(us, ROOK)  & threatenedByMinor)
+                  | (pos.pieces(us, KNIGHT, BISHOP) & threatenedByPawn);
+  }
+  else
+  {
+      // Silence unused variable warnings
+      (void) threatened;
+      (void) threatenedByPawn;
+      (void) threatenedByMinor;
+      (void) threatenedByRook;
   }
 
   for (auto& m : *this)
       if constexpr (Type == CAPTURES)
-          m.value =  (7 * int(PieceValue[MG][pos.piece_on(to_sq(m))])
-                   +     (*captureHistory)[pos.moved_piece(m)][to_sq(m)][type_of(pos.piece_on(to_sq(m)))]) / 16;
+          m.value =  6 * int(PieceValue[MG][pos.piece_on(to_sq(m))])
+                   +     (*captureHistory)[pos.moved_piece(m)][to_sq(m)][type_of(pos.piece_on(to_sq(m)))];
 
       else if constexpr (Type == QUIETS)
-          m.value =  2 * (*mainHistory)[pos.side_to_move()][from_to(m)]
+          m.value =      (*mainHistory)[pos.side_to_move()][from_to(m)]
                    + 2 * (*continuationHistory[0])[pos.moved_piece(m)][to_sq(m)]
                    +     (*continuationHistory[1])[pos.moved_piece(m)][to_sq(m)]
                    +     (*continuationHistory[3])[pos.moved_piece(m)][to_sq(m)]
                    +     (*continuationHistory[5])[pos.moved_piece(m)][to_sq(m)]
-                   +     (threatenedPieces & from_sq(m) ?
+                   +     (threatened & from_sq(m) ?
                            (type_of(pos.moved_piece(m)) == QUEEN && !(to_sq(m) & threatenedByRook)  ? 50000
                           : type_of(pos.moved_piece(m)) == ROOK  && !(to_sq(m) & threatenedByMinor) ? 25000
                           :                                         !(to_sq(m) & threatenedByPawn)  ? 15000
                           :                                                                           0)
-                          :                                                                           0)
-                   +     bool(pos.check_squares(type_of(pos.moved_piece(m))) & to_sq(m)) * 16384;
+                          :                                                                           0);
+
       else // Type == EVASIONS
       {
-          if (pos.capture_stage(m))
+          if (pos.capture(m))
               m.value =  PieceValue[MG][pos.piece_on(to_sq(m))]
-                       - Value(type_of(pos.moved_piece(m)))
-                       + (1 << 28);
+                       - Value(type_of(pos.moved_piece(m)));
           else
-              m.value =  (*mainHistory)[pos.side_to_move()][from_to(m)]
-                       + (*continuationHistory[0])[pos.moved_piece(m)][to_sq(m)];
+              m.value =      (*mainHistory)[pos.side_to_move()][from_to(m)]
+                       + 2 * (*continuationHistory[0])[pos.moved_piece(m)][to_sq(m)]
+                       - (1 << 28);
       }
 }
 
@@ -157,7 +168,7 @@ Move MovePicker::select(Pred filter) {
 
   while (cur < endMoves)
   {
-      if constexpr (T == Best)
+      if (T == Best)
           std::swap(*cur, *std::max_element(cur, endMoves));
 
       if (*cur != ttMove && filter())
@@ -190,13 +201,13 @@ top:
       endMoves = generate<CAPTURES>(pos, cur);
 
       score<CAPTURES>();
-      partial_insertion_sort(cur, endMoves, std::numeric_limits<int>::min());
+      partial_insertion_sort(cur, endMoves, -3000 * depth);
       ++stage;
       goto top;
 
   case GOOD_CAPTURE:
       if (select<Next>([&](){
-                       return pos.see_ge(*cur, Value(-cur->value)) ?
+                       return pos.see_ge(*cur, Value(-69 * cur->value / 1024)) ?
                               // Move losing capture to endBadCaptures to be tried later
                               true : (*endBadCaptures++ = *cur, false); }))
           return *(cur - 1);
@@ -215,7 +226,7 @@ top:
 
   case REFUTATION:
       if (select<Next>([&](){ return    *cur != MOVE_NONE
-                                    && !pos.capture_stage(*cur)
+                                    && !pos.capture(*cur)
                                     &&  pos.pseudo_legal(*cur); }))
           return *(cur - 1);
       ++stage;

src/movepick.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -62,14 +62,14 @@ public:
 template <typename T, int D, int Size, int... Sizes>
 struct Stats : public std::array<Stats<T, D, Sizes...>, Size>
 {
-  using stats = Stats<T, D, Size, Sizes...>;
+  typedef Stats<T, D, Size, Sizes...> stats;
 
   void fill(const T& v) {
 
     // For standard-layout 'this' points to first struct member
     assert(std::is_standard_layout<stats>::value);
 
-    using entry = StatsEntry<T, D>;
+    typedef StatsEntry<T, D> entry;
     entry* p = reinterpret_cast<entry*>(this);
     std::fill(p, p + sizeof(*this) / sizeof(entry), v);
   }
@@ -86,24 +86,22 @@ enum StatsType { NoCaptures, Captures };
 /// unsuccessful during the current search, and is used for reduction and move
 /// ordering decisions. It uses 2 tables (one for each color) indexed by
 /// the move's from and to squares, see www.chessprogramming.org/Butterfly_Boards
-/// (~11 elo)
-using ButterflyHistory = Stats<int16_t, 7183, COLOR_NB, int(SQUARE_NB) * int(SQUARE_NB)>;
+typedef Stats<int16_t, 14365, COLOR_NB, int(SQUARE_NB) * int(SQUARE_NB)> ButterflyHistory;
 
 /// CounterMoveHistory stores counter moves indexed by [piece][to] of the previous
 /// move, see www.chessprogramming.org/Countermove_Heuristic
-using CounterMoveHistory = Stats<Move, NOT_USED, PIECE_NB, SQUARE_NB>;
+typedef Stats<Move, NOT_USED, PIECE_NB, SQUARE_NB> CounterMoveHistory;
 
 /// CapturePieceToHistory is addressed by a move's [piece][to][captured piece type]
-using CapturePieceToHistory = Stats<int16_t, 10692, PIECE_NB, SQUARE_NB, PIECE_TYPE_NB>;
+typedef Stats<int16_t, 10692, PIECE_NB, SQUARE_NB, PIECE_TYPE_NB> CapturePieceToHistory;
 
 /// PieceToHistory is like ButterflyHistory but is addressed by a move's [piece][to]
-using PieceToHistory = Stats<int16_t, 29952, PIECE_NB, SQUARE_NB>;
+typedef Stats<int16_t, 29952, PIECE_NB, SQUARE_NB> PieceToHistory;
 
 /// ContinuationHistory is the combined history of a given pair of moves, usually
 /// the current one given a previous one. The nested history table is based on
 /// PieceToHistory instead of ButterflyBoards.
-/// (~63 elo)
-using ContinuationHistory = Stats<PieceToHistory, NOT_USED, PIECE_NB, SQUARE_NB>;
+typedef Stats<PieceToHistory, NOT_USED, PIECE_NB, SQUARE_NB> ContinuationHistory;
 
 
 /// MovePicker class is used to pick one pseudo-legal move at a time from the
@@ -128,7 +126,7 @@ public:
                                            const CapturePieceToHistory*,
                                            const PieceToHistory**,
                                            Square);
-  MovePicker(const Position&, Move, Value, const CapturePieceToHistory*);
+  MovePicker(const Position&, Move, Value, Depth, const CapturePieceToHistory*);
   Move next_move(bool skipQuiets = false);
 
 private:

src/nnue/evaluate_nnue.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -18,15 +18,15 @@
 
 // Code for calculating NNUE evaluation function
 
-#include <fstream>
-#include <iomanip>
 #include <iostream>
 #include <set>
 #include <sstream>
-#include <string_view>
+#include <iomanip>
+#include <fstream>
 
 #include "../evaluate.h"
 #include "../position.h"
+#include "../misc.h"
 #include "../uci.h"
 #include "../types.h"
 
@@ -83,7 +83,7 @@ namespace Stockfish::Eval::NNUE {
   }  // namespace Detail
 
   // Initialize the evaluation function parameters
-  static void initialize() {
+  void initialize() {
 
     Detail::initialize(featureTransformer);
     for (std::size_t i = 0; i < LayerStacks; ++i)
@@ -91,7 +91,7 @@ namespace Stockfish::Eval::NNUE {
   }
 
   // Read network header
-  static bool read_header(std::istream& stream, std::uint32_t* hashValue, std::string* desc)
+  bool read_header(std::istream& stream, std::uint32_t* hashValue, std::string* desc)
   {
     std::uint32_t version, size;
 
@@ -105,7 +105,7 @@ namespace Stockfish::Eval::NNUE {
   }
 
   // Write network header
-  static bool write_header(std::ostream& stream, std::uint32_t hashValue, const std::string& desc)
+  bool write_header(std::ostream& stream, std::uint32_t hashValue, const std::string& desc)
   {
     write_little_endian<std::uint32_t>(stream, Version);
     write_little_endian<std::uint32_t>(stream, hashValue);
@@ -115,7 +115,7 @@ namespace Stockfish::Eval::NNUE {
   }
 
   // Read network parameters
-  static bool read_parameters(std::istream& stream) {
+  bool read_parameters(std::istream& stream) {
 
     std::uint32_t hashValue;
     if (!read_header(stream, &hashValue, &netDescription)) return false;
@@ -127,7 +127,7 @@ namespace Stockfish::Eval::NNUE {
   }
 
   // Write network parameters
-  static bool write_parameters(std::ostream& stream) {
+  bool write_parameters(std::ostream& stream) {
 
     if (!write_header(stream, HashValue, netDescription)) return false;
     if (!Detail::write_parameters(stream, *featureTransformer)) return false;
@@ -136,19 +136,14 @@ namespace Stockfish::Eval::NNUE {
     return (bool)stream;
   }
 
-  void hint_common_parent_position(const Position& pos) {
-    if (Eval::useNNUE)
-        featureTransformer->hint_common_access(pos);
-  }
-
   // Evaluation function. Perform differential calculation.
-  Value evaluate(const Position& pos, bool adjusted, int* complexity) {
+  Value evaluate(const Position& pos, bool adjusted) {
 
     // We manually align the arrays on the stack because with gcc < 9.3
     // overaligning stack variables with alignas() doesn't work correctly.
 
     constexpr uint64_t alignment = CacheLineSize;
-    constexpr int delta = 24;
+    int delta = 10 - pos.non_pawn_material() / 1515;
 
 #if defined(ALIGNAS_ON_STACK_VARIABLES_BROKEN)
     TransformedFeatureType transformedFeaturesUnaligned[
@@ -166,12 +161,9 @@ namespace Stockfish::Eval::NNUE {
     const auto psqt = featureTransformer->transform(pos, transformedFeatures, bucket);
     const auto positional = network[bucket]->propagate(transformedFeatures);
 
-    if (complexity)
-        *complexity = abs(psqt - positional) / OutputScale;
-
     // Give more value to positional evaluation when adjusted flag is set
     if (adjusted)
-        return static_cast<Value>(((1024 - delta) * psqt + (1024 + delta) * positional) / (1024 * OutputScale));
+        return static_cast<Value>(((128 - delta) * psqt + (128 + delta) * positional) / 128 / OutputScale);
     else
         return static_cast<Value>((psqt + positional) / OutputScale);
   }
@@ -216,7 +208,7 @@ namespace Stockfish::Eval::NNUE {
     return t;
   }
 
-  constexpr std::string_view PieceToChar(" PNBRQK  pnbrqk");
+  static const std::string PieceToChar(" PNBRQK  pnbrqk");
 
 
   // format_cp_compact() converts a Value into (centi)pawns and writes it in a buffer.
@@ -225,7 +217,7 @@ namespace Stockfish::Eval::NNUE {
 
     buffer[0] = (v < 0 ? '-' : v > 0 ? '+' : ' ');
 
-    int cp = std::abs(100 * v / UCI::NormalizeToPawnValue);
+    int cp = std::abs(100 * v / PawnValueEg);
     if (cp >= 10000)
     {
         buffer[1] = '0' + cp / 10000; cp %= 10000;
@@ -250,15 +242,14 @@ namespace Stockfish::Eval::NNUE {
   }
 
 
-  // format_cp_aligned_dot() converts a Value into (centi)pawns, always keeping two decimals.
-  static void format_cp_aligned_dot(Value v, std::stringstream &stream) {
-    const double cp = 1.0 * std::abs(int(v)) / UCI::NormalizeToPawnValue;
+  // format_cp_aligned_dot() converts a Value into (centi)pawns and writes it in a buffer,
+  // always keeping two decimals. The buffer must have capacity for at least 7 chars.
+  static void format_cp_aligned_dot(Value v, char* buffer) {
 
-    stream << (v < 0 ? '-' : v > 0 ? '+' : ' ')
-           << std::setiosflags(std::ios::fixed)
-           << std::setw(6)
-           << std::setprecision(2)
-           << cp;
+    buffer[0] = (v < 0 ? '-' : v > 0 ? '+' : ' ');
+
+    double cp = 1.0 * std::abs(int(v)) / PawnValueEg;
+    sprintf(&buffer[1], "%6.2f", cp);
   }
 
 
@@ -338,10 +329,17 @@ namespace Stockfish::Eval::NNUE {
 
     for (std::size_t bucket = 0; bucket < LayerStacks; ++bucket)
     {
-      ss <<  "|  " << bucket    << "        ";
-      ss << " |  "; format_cp_aligned_dot(t.psqt[bucket], ss); ss << "  "
-         << " |  "; format_cp_aligned_dot(t.positional[bucket], ss); ss << "  "
-         << " |  "; format_cp_aligned_dot(t.psqt[bucket] + t.positional[bucket], ss); ss << "  "
+      char buffer[3][8];
+      std::memset(buffer, '\0', sizeof(buffer));
+
+      format_cp_aligned_dot(t.psqt[bucket], buffer[0]);
+      format_cp_aligned_dot(t.positional[bucket], buffer[1]);
+      format_cp_aligned_dot(t.psqt[bucket] + t.positional[bucket], buffer[2]);
+
+      ss <<  "|  " << bucket    << "        "
+         << " |  " << buffer[0] << "  "
+         << " |  " << buffer[1] << "  "
+         << " |  " << buffer[2] << "  "
          << " |";
       if (bucket == t.correctBucket)
           ss << " <-- this bucket is used";

src/nnue/evaluate_nnue.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -31,7 +31,6 @@ namespace Stockfish::Eval::NNUE {
   constexpr std::uint32_t HashValue =
       FeatureTransformer::get_hash_value() ^ Network::get_hash_value();
 
-
   // Deleter for automating release of memory area
   template <typename T>
   struct AlignedDeleter {
@@ -55,14 +54,6 @@ namespace Stockfish::Eval::NNUE {
   template <typename T>
   using LargePagePtr = std::unique_ptr<T, LargePageDeleter<T>>;
 
-  std::string trace(Position& pos);
-  Value evaluate(const Position& pos, bool adjusted = false, int* complexity = nullptr);
-  void hint_common_parent_position(const Position& pos);
-
-  bool load_eval(std::string name, std::istream& stream);
-  bool save_eval(std::ostream& stream);
-  bool save_eval(const std::optional<std::string>& filename);
-
 }  // namespace Stockfish::Eval::NNUE
 
 #endif // #ifndef NNUE_EVALUATE_NNUE_H_INCLUDED

src/nnue/features/half_ka_v2_hm.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -24,51 +24,50 @@
 
 namespace Stockfish::Eval::NNUE::Features {
 
+  // Orient a square according to perspective (rotates by 180 for black)
+  inline Square HalfKAv2_hm::orient(Color perspective, Square s, Square ksq) {
+    return Square(int(s) ^ (bool(perspective) * SQ_A8) ^ ((file_of(ksq) < FILE_E) * SQ_H1));
+  }
+
   // Index of a feature for a given king position and another piece on some square
-  template<Color Perspective>
-  inline IndexType HalfKAv2_hm::make_index(Square s, Piece pc, Square ksq) {
-    return IndexType((int(s) ^ OrientTBL[Perspective][ksq]) + PieceSquareIndex[Perspective][pc] + KingBuckets[Perspective][ksq]);
+  inline IndexType HalfKAv2_hm::make_index(Color perspective, Square s, Piece pc, Square ksq) {
+    Square o_ksq = orient(perspective, ksq, ksq);
+    return IndexType(orient(perspective, s, ksq) + PieceSquareIndex[perspective][pc] + PS_NB * KingBuckets[o_ksq]);
   }
 
   // Get a list of indices for active features
-  template<Color Perspective>
   void HalfKAv2_hm::append_active_indices(
     const Position& pos,
+    Color perspective,
     IndexList& active
   ) {
-    Square ksq = pos.square<KING>(Perspective);
+    Square ksq = pos.square<KING>(perspective);
     Bitboard bb = pos.pieces();
     while (bb)
     {
       Square s = pop_lsb(bb);
-      active.push_back(make_index<Perspective>(s, pos.piece_on(s), ksq));
+      active.push_back(make_index(perspective, s, pos.piece_on(s), ksq));
     }
   }
 
-  // Explicit template instantiations
-  template void HalfKAv2_hm::append_active_indices<WHITE>(const Position& pos, IndexList& active);
-  template void HalfKAv2_hm::append_active_indices<BLACK>(const Position& pos, IndexList& active);
 
   // append_changed_indices() : get a list of indices for recently changed features
-  template<Color Perspective>
+
   void HalfKAv2_hm::append_changed_indices(
     Square ksq,
     const DirtyPiece& dp,
+    Color perspective,
     IndexList& removed,
     IndexList& added
   ) {
     for (int i = 0; i < dp.dirty_num; ++i) {
       if (dp.from[i] != SQ_NONE)
-        removed.push_back(make_index<Perspective>(dp.from[i], dp.piece[i], ksq));
+        removed.push_back(make_index(perspective, dp.from[i], dp.piece[i], ksq));
       if (dp.to[i] != SQ_NONE)
-        added.push_back(make_index<Perspective>(dp.to[i], dp.piece[i], ksq));
+        added.push_back(make_index(perspective, dp.to[i], dp.piece[i], ksq));
     }
   }
 
-  // Explicit template instantiations
-  template void HalfKAv2_hm::append_changed_indices<WHITE>(Square ksq, const DirtyPiece& dp, IndexList& removed, IndexList& added);
-  template void HalfKAv2_hm::append_changed_indices<BLACK>(Square ksq, const DirtyPiece& dp, IndexList& removed, IndexList& added);
-
   int HalfKAv2_hm::update_cost(const StateInfo* st) {
     return st->dirtyPiece.dirty_num;
   }

src/nnue/features/half_ka_v2_hm.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -49,8 +49,8 @@ namespace Stockfish::Eval::NNUE::Features {
       PS_B_ROOK   =  7 * SQUARE_NB,
       PS_W_QUEEN  =  8 * SQUARE_NB,
       PS_B_QUEEN  =  9 * SQUARE_NB,
-      PS_KING     = 10 * SQUARE_NB,
-      PS_NB       = 11 * SQUARE_NB
+      PS_KING     =  10 * SQUARE_NB,
+      PS_NB       =  11 * SQUARE_NB
     };
 
     static constexpr IndexType PieceSquareIndex[COLOR_NB][PIECE_NB] = {
@@ -62,9 +62,11 @@ namespace Stockfish::Eval::NNUE::Features {
         PS_NONE, PS_W_PAWN, PS_W_KNIGHT, PS_W_BISHOP, PS_W_ROOK, PS_W_QUEEN, PS_KING, PS_NONE }
     };
 
+    // Orient a square according to perspective (rotates by 180 for black)
+    static Square orient(Color perspective, Square s, Square ksq);
+
     // Index of a feature for a given king position and another piece on some square
-    template<Color Perspective>
-    static IndexType make_index(Square s, Piece pc, Square ksq);
+    static IndexType make_index(Color perspective, Square s, Piece pc, Square ksq);
 
    public:
     // Feature name
@@ -77,45 +79,15 @@ namespace Stockfish::Eval::NNUE::Features {
     static constexpr IndexType Dimensions =
         static_cast<IndexType>(SQUARE_NB) * static_cast<IndexType>(PS_NB) / 2;
 
-#define B(v) (v * PS_NB)
-    static constexpr int KingBuckets[COLOR_NB][SQUARE_NB] = {
-      { B(28), B(29), B(30), B(31), B(31), B(30), B(29), B(28),
-        B(24), B(25), B(26), B(27), B(27), B(26), B(25), B(24),
-        B(20), B(21), B(22), B(23), B(23), B(22), B(21), B(20),
-        B(16), B(17), B(18), B(19), B(19), B(18), B(17), B(16),
-        B(12), B(13), B(14), B(15), B(15), B(14), B(13), B(12),
-        B( 8), B( 9), B(10), B(11), B(11), B(10), B( 9), B( 8),
-        B( 4), B( 5), B( 6), B( 7), B( 7), B( 6), B( 5), B( 4),
-        B( 0), B( 1), B( 2), B( 3), B( 3), B( 2), B( 1), B( 0) },
-      { B( 0), B( 1), B( 2), B( 3), B( 3), B( 2), B( 1), B( 0),
-        B( 4), B( 5), B( 6), B( 7), B( 7), B( 6), B( 5), B( 4),
-        B( 8), B( 9), B(10), B(11), B(11), B(10), B( 9), B( 8),
-        B(12), B(13), B(14), B(15), B(15), B(14), B(13), B(12),
-        B(16), B(17), B(18), B(19), B(19), B(18), B(17), B(16),
-        B(20), B(21), B(22), B(23), B(23), B(22), B(21), B(20),
-        B(24), B(25), B(26), B(27), B(27), B(26), B(25), B(24),
-        B(28), B(29), B(30), B(31), B(31), B(30), B(29), B(28) }
-    };
-#undef B
-
-    // Orient a square according to perspective (rotates by 180 for black)
-    static constexpr int OrientTBL[COLOR_NB][SQUARE_NB] = {
-      { SQ_H1, SQ_H1, SQ_H1, SQ_H1, SQ_A1, SQ_A1, SQ_A1, SQ_A1,
-        SQ_H1, SQ_H1, SQ_H1, SQ_H1, SQ_A1, SQ_A1, SQ_A1, SQ_A1,
-        SQ_H1, SQ_H1, SQ_H1, SQ_H1, SQ_A1, SQ_A1, SQ_A1, SQ_A1,
-        SQ_H1, SQ_H1, SQ_H1, SQ_H1, SQ_A1, SQ_A1, SQ_A1, SQ_A1,
-        SQ_H1, SQ_H1, SQ_H1, SQ_H1, SQ_A1, SQ_A1, SQ_A1, SQ_A1,
-        SQ_H1, SQ_H1, SQ_H1, SQ_H1, SQ_A1, SQ_A1, SQ_A1, SQ_A1,
-        SQ_H1, SQ_H1, SQ_H1, SQ_H1, SQ_A1, SQ_A1, SQ_A1, SQ_A1,
-        SQ_H1, SQ_H1, SQ_H1, SQ_H1, SQ_A1, SQ_A1, SQ_A1, SQ_A1 },
-      { SQ_H8, SQ_H8, SQ_H8, SQ_H8, SQ_A8, SQ_A8, SQ_A8, SQ_A8,
-        SQ_H8, SQ_H8, SQ_H8, SQ_H8, SQ_A8, SQ_A8, SQ_A8, SQ_A8,
-        SQ_H8, SQ_H8, SQ_H8, SQ_H8, SQ_A8, SQ_A8, SQ_A8, SQ_A8,
-        SQ_H8, SQ_H8, SQ_H8, SQ_H8, SQ_A8, SQ_A8, SQ_A8, SQ_A8,
-        SQ_H8, SQ_H8, SQ_H8, SQ_H8, SQ_A8, SQ_A8, SQ_A8, SQ_A8,
-        SQ_H8, SQ_H8, SQ_H8, SQ_H8, SQ_A8, SQ_A8, SQ_A8, SQ_A8,
-        SQ_H8, SQ_H8, SQ_H8, SQ_H8, SQ_A8, SQ_A8, SQ_A8, SQ_A8,
-        SQ_H8, SQ_H8, SQ_H8, SQ_H8, SQ_A8, SQ_A8, SQ_A8, SQ_A8 }
+    static constexpr int KingBuckets[64] = {
+      -1, -1, -1, -1, 31, 30, 29, 28,
+      -1, -1, -1, -1, 27, 26, 25, 24,
+      -1, -1, -1, -1, 23, 22, 21, 20,
+      -1, -1, -1, -1, 19, 18, 17, 16,
+      -1, -1, -1, -1, 15, 14, 13, 12,
+      -1, -1, -1, -1, 11, 10,  9,  8,
+      -1, -1, -1, -1,  7,  6,  5,  4,
+      -1, -1, -1, -1,  3,  2,  1,  0
     };
 
     // Maximum number of simultaneously active features.
@@ -123,16 +95,16 @@ namespace Stockfish::Eval::NNUE::Features {
     using IndexList = ValueList<IndexType, MaxActiveDimensions>;
 
     // Get a list of indices for active features
-    template<Color Perspective>
     static void append_active_indices(
       const Position& pos,
+      Color perspective,
       IndexList& active);
 
     // Get a list of indices for recently changed features
-    template<Color Perspective>
     static void append_changed_indices(
       Square ksq,
       const DirtyPiece& dp,
+      Color perspective,
       IndexList& removed,
       IndexList& added
     );

src/nnue/layers/affine_transform.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -25,13 +25,13 @@
 #include <algorithm>
 #include <type_traits>
 #include "../nnue_common.h"
-#include "simd.h"
+#include "../../simd.h"
 
 /*
   This file contains the definition for a fully connected layer (aka affine transform).
   Two approaches are employed, depending on the sizes of the transform.
 
-  Approach 1 (a specialization for large inputs):
+  Approach 1:
     - used when the PaddedInputDimensions >= 128
     - uses AVX512 if possible
     - processes inputs in batches of 2*InputSimdWidth
@@ -42,8 +42,9 @@
       depends on the architecture (the amount of registers)
     - accumulate + hadd is used
 
-  Approach 2 (a specialization for small inputs):
+  Approach 2:
     - used when the PaddedInputDimensions < 128
+    - does not use AVX512
     - expected use-case is for when PaddedInputDimensions == 32 and InputDimensions <= 32.
       - that's why AVX512 is hard to implement
     - expected use-case is small layers
@@ -71,10 +72,6 @@ namespace Stockfish::Eval::NNUE::Layers {
     const __m64 Zeros = _mm_setzero_si64();
     const auto inputVector = reinterpret_cast<const __m64*>(input);
 
-# elif defined(USE_NEON_DOTPROD)
-    constexpr IndexType NumChunks = ceil_to_multiple<IndexType>(InputDimensions, 16) / 16;
-    const auto inputVector = reinterpret_cast<const int8x16_t*>(input);
-
 # elif defined(USE_NEON)
     constexpr IndexType NumChunks = ceil_to_multiple<IndexType>(InputDimensions, 16) / 16;
     const auto inputVector = reinterpret_cast<const int8x8_t*>(input);
@@ -126,14 +123,6 @@ namespace Stockfish::Eval::NNUE::Layers {
       sum = _mm_add_pi32(sum, _mm_unpackhi_pi32(sum, sum));
       output[i] = _mm_cvtsi64_si32(sum);
 
-# elif defined(USE_NEON_DOTPROD)
-      int32x4_t sum = {biases[i]};
-      const auto row = reinterpret_cast<const int8x16_t*>(&weights[offset]);
-      for (IndexType j = 0; j < NumChunks; ++j) {
-        sum = vdotq_s32(sum, inputVector[j], row[j]);
-      }
-      output[i] = vaddvq_s32(sum);
-
 # elif defined(USE_NEON)
       int32x4_t sum = {biases[i]};
       const auto row = reinterpret_cast<const int8x8_t*>(&weights[offset]);
@@ -162,15 +151,9 @@ namespace Stockfish::Eval::NNUE::Layers {
   template <IndexType InDims, IndexType OutDims, typename Enabled = void>
   class AffineTransform;
 
-#if defined (USE_AVX512)
-  constexpr IndexType LargeInputSize = 2 * 64;
-#else
-  constexpr IndexType LargeInputSize = std::numeric_limits<IndexType>::max();
-#endif
-
-  // A specialization for large inputs
+  // A specialization for large inputs.
   template <IndexType InDims, IndexType OutDims>
-  class AffineTransform<InDims, OutDims, std::enable_if_t<(ceil_to_multiple<IndexType>(InDims, MaxSimdWidth) >= LargeInputSize)>> {
+  class AffineTransform<InDims, OutDims, std::enable_if_t<(ceil_to_multiple<IndexType>(InDims, MaxSimdWidth) >= 2*64)>> {
    public:
     // Input/output type
     using InputType = std::uint8_t;
@@ -187,39 +170,36 @@ namespace Stockfish::Eval::NNUE::Layers {
 
     using OutputBuffer = OutputType[PaddedOutputDimensions];
 
-    static_assert(PaddedInputDimensions >= LargeInputSize, "Something went wrong. This specialization (for large inputs) should not have been chosen.");
+    static_assert(PaddedInputDimensions >= 128, "Something went wrong. This specialization should not have been chosen.");
 
 #if defined (USE_AVX512)
-    static constexpr IndexType InputSimdWidth = 64;
-    static constexpr IndexType MaxNumOutputRegs = 16;
+    static constexpr const IndexType InputSimdWidth = 64;
+    static constexpr const IndexType MaxNumOutputRegs = 16;
 #elif defined (USE_AVX2)
-    static constexpr IndexType InputSimdWidth = 32;
-    static constexpr IndexType MaxNumOutputRegs = 8;
+    static constexpr const IndexType InputSimdWidth = 32;
+    static constexpr const IndexType MaxNumOutputRegs = 8;
 #elif defined (USE_SSSE3)
-    static constexpr IndexType InputSimdWidth = 16;
-    static constexpr IndexType MaxNumOutputRegs = 8;
-#elif defined (USE_NEON_DOTPROD)
-    static constexpr IndexType InputSimdWidth = 16;
-    static constexpr IndexType MaxNumOutputRegs = 8;
+    static constexpr const IndexType InputSimdWidth = 16;
+    static constexpr const IndexType MaxNumOutputRegs = 8;
 #elif defined (USE_NEON)
-    static constexpr IndexType InputSimdWidth = 8;
-    static constexpr IndexType MaxNumOutputRegs = 8;
+    static constexpr const IndexType InputSimdWidth = 8;
+    static constexpr const IndexType MaxNumOutputRegs = 8;
 #else
     // The fallback implementation will not have permuted weights.
     // We define these to avoid a lot of ifdefs later.
-    static constexpr IndexType InputSimdWidth = 1;
-    static constexpr IndexType MaxNumOutputRegs = 1;
+    static constexpr const IndexType InputSimdWidth = 1;
+    static constexpr const IndexType MaxNumOutputRegs = 1;
 #endif
 
     // A big block is a region in the weight matrix of the size [PaddedInputDimensions, NumOutputRegs].
     // A small block is a region of size [InputSimdWidth, 1]
 
-    static constexpr IndexType NumOutputRegs = std::min(MaxNumOutputRegs, OutputDimensions);
-    static constexpr IndexType SmallBlockSize = InputSimdWidth;
-    static constexpr IndexType BigBlockSize = NumOutputRegs * PaddedInputDimensions;
-    static constexpr IndexType NumSmallBlocksInBigBlock = BigBlockSize / SmallBlockSize;
-    static constexpr IndexType NumSmallBlocksPerOutput = PaddedInputDimensions / SmallBlockSize;
-    static constexpr IndexType NumBigBlocks = OutputDimensions / NumOutputRegs;
+    static constexpr const IndexType NumOutputRegs = std::min(MaxNumOutputRegs, OutputDimensions);
+    static constexpr const IndexType SmallBlockSize = InputSimdWidth;
+    static constexpr const IndexType BigBlockSize = NumOutputRegs * PaddedInputDimensions;
+    static constexpr const IndexType NumSmallBlocksInBigBlock = BigBlockSize / SmallBlockSize;
+    static constexpr const IndexType NumSmallBlocksPerOutput = PaddedInputDimensions / SmallBlockSize;
+    static constexpr const IndexType NumBigBlocks = OutputDimensions / NumOutputRegs;
 
     static_assert(OutputDimensions % NumOutputRegs == 0);
 
@@ -255,7 +235,8 @@ namespace Stockfish::Eval::NNUE::Layers {
 
     // Read network parameters
     bool read_parameters(std::istream& stream) {
-      read_little_endian<BiasType>(stream, biases, OutputDimensions);
+      for (IndexType i = 0; i < OutputDimensions; ++i)
+        biases[i] = read_little_endian<BiasType>(stream);
 
       for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i)
         weights[get_weight_index(i)] = read_little_endian<WeightType>(stream);
@@ -265,7 +246,8 @@ namespace Stockfish::Eval::NNUE::Layers {
 
     // Write network parameters
     bool write_parameters(std::ostream& stream) const {
-      write_little_endian<BiasType>(stream, biases, OutputDimensions);
+      for (IndexType i = 0; i < OutputDimensions; ++i)
+          write_little_endian<BiasType>(stream, biases[i]);
 
       for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i)
         write_little_endian<WeightType>(stream, weights[get_weight_index(i)]);
@@ -304,15 +286,6 @@ namespace Stockfish::Eval::NNUE::Layers {
       #define vec_add_dpbusd_32x2 Simd::m128_add_dpbusd_epi32x2
       #define vec_hadd Simd::m128_hadd
       #define vec_haddx4 Simd::m128_haddx4
-#elif defined (USE_NEON_DOTPROD)
-      using acc_vec_t = int32x4_t;
-      using bias_vec_t = int32x4_t;
-      using weight_vec_t = int8x16_t;
-      using in_vec_t = int8x16_t;
-      #define vec_zero {0}
-      #define vec_add_dpbusd_32x2 Simd::dotprod_m128_add_dpbusd_epi32x2
-      #define vec_hadd Simd::neon_m128_hadd
-      #define vec_haddx4 Simd::neon_m128_haddx4
 #elif defined (USE_NEON)
       using acc_vec_t = int32x4_t;
       using bias_vec_t = int32x4_t;
@@ -395,9 +368,8 @@ namespace Stockfish::Eval::NNUE::Layers {
     alignas(CacheLineSize) WeightType weights[OutputDimensions * PaddedInputDimensions];
   };
 
-  // A specialization for small inputs
   template <IndexType InDims, IndexType OutDims>
-  class AffineTransform<InDims, OutDims, std::enable_if_t<(ceil_to_multiple<IndexType>(InDims, MaxSimdWidth) < LargeInputSize)>> {
+  class AffineTransform<InDims, OutDims, std::enable_if_t<(ceil_to_multiple<IndexType>(InDims, MaxSimdWidth) < 2*64)>> {
    public:
     // Input/output type
     // Input/output type
@@ -415,7 +387,12 @@ namespace Stockfish::Eval::NNUE::Layers {
 
     using OutputBuffer = OutputType[PaddedOutputDimensions];
 
-    static_assert(PaddedInputDimensions < LargeInputSize, "Something went wrong. This specialization (for small inputs) should not have been chosen.");
+    static_assert(PaddedInputDimensions < 128, "Something went wrong. This specialization should not have been chosen.");
+
+#if defined (USE_SSSE3)
+    static constexpr const IndexType OutputSimdWidth = SimdWidth / 4;
+    static constexpr const IndexType InputSimdWidth = SimdWidth;
+#endif
 
     // Hash value embedded in the evaluation file
     static constexpr std::uint32_t get_hash_value(std::uint32_t prevHash) {
@@ -445,7 +422,8 @@ namespace Stockfish::Eval::NNUE::Layers {
 
     // Read network parameters
     bool read_parameters(std::istream& stream) {
-      read_little_endian<BiasType>(stream, biases, OutputDimensions);
+      for (IndexType i = 0; i < OutputDimensions; ++i)
+        biases[i] = read_little_endian<BiasType>(stream);
       for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i)
         weights[get_weight_index(i)] = read_little_endian<WeightType>(stream);
 
@@ -454,7 +432,8 @@ namespace Stockfish::Eval::NNUE::Layers {
 
     // Write network parameters
     bool write_parameters(std::ostream& stream) const {
-      write_little_endian<BiasType>(stream, biases, OutputDimensions);
+      for (IndexType i = 0; i < OutputDimensions; ++i)
+        write_little_endian<BiasType>(stream, biases[i]);
 
       for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i)
         write_little_endian<WeightType>(stream, weights[get_weight_index(i)]);
@@ -465,34 +444,29 @@ namespace Stockfish::Eval::NNUE::Layers {
     const OutputType* propagate(
         const InputType* input, OutputType* output) const {
 
-#if defined (USE_AVX512)
-      using vec_t = __m512i;
-      #define vec_setzero _mm512_setzero_si512
-      #define vec_set_32 _mm512_set1_epi32
-      #define vec_add_dpbusd_32 Simd::m512_add_dpbusd_epi32
-      #define vec_add_dpbusd_32x2 Simd::m512_add_dpbusd_epi32x2
-      #define vec_hadd Simd::m512_hadd
-#elif defined (USE_AVX2)
+#if defined (USE_AVX2)
       using vec_t = __m256i;
       #define vec_setzero _mm256_setzero_si256
       #define vec_set_32 _mm256_set1_epi32
       #define vec_add_dpbusd_32 Simd::m256_add_dpbusd_epi32
       #define vec_add_dpbusd_32x2 Simd::m256_add_dpbusd_epi32x2
+      #define vec_add_dpbusd_32x4 Simd::m256_add_dpbusd_epi32x4
       #define vec_hadd Simd::m256_hadd
+      #define vec_haddx4 Simd::m256_haddx4
 #elif defined (USE_SSSE3)
       using vec_t = __m128i;
       #define vec_setzero _mm_setzero_si128
       #define vec_set_32 _mm_set1_epi32
       #define vec_add_dpbusd_32 Simd::m128_add_dpbusd_epi32
       #define vec_add_dpbusd_32x2 Simd::m128_add_dpbusd_epi32x2
+      #define vec_add_dpbusd_32x4 Simd::m128_add_dpbusd_epi32x4
       #define vec_hadd Simd::m128_hadd
+      #define vec_haddx4 Simd::m128_haddx4
 #endif
 
 #if defined (USE_SSSE3)
       const auto inputVector = reinterpret_cast<const vec_t*>(input);
 
-      static constexpr IndexType OutputSimdWidth = sizeof(vec_t) / sizeof(OutputType);
-
       static_assert(OutputDimensions % OutputSimdWidth == 0 || OutputDimensions == 1);
 
       if constexpr (OutputDimensions % OutputSimdWidth == 0)
@@ -538,7 +512,9 @@ namespace Stockfish::Eval::NNUE::Layers {
 # undef vec_set_32
 # undef vec_add_dpbusd_32
 # undef vec_add_dpbusd_32x2
+# undef vec_add_dpbusd_32x4
 # undef vec_hadd
+# undef vec_haddx4
 #else
       // Use old implementation for the other architectures.
       affine_transform_non_ssse3<

src/nnue/layers/affine_transform_sparse_input.h

@@ -1,286 +0,0 @@
-/*
-  Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
-
-  Stockfish is free software: you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation, either version 3 of the License, or
-  (at your option) any later version.
-
-  Stockfish is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-// Definition of layer AffineTransformSparseInput of NNUE evaluation function
-
-#ifndef NNUE_LAYERS_AFFINE_TRANSFORM_SPARSE_INPUT_H_INCLUDED
-#define NNUE_LAYERS_AFFINE_TRANSFORM_SPARSE_INPUT_H_INCLUDED
-
-#include <iostream>
-#include <algorithm>
-#include <array>
-#include <type_traits>
-#include "../nnue_common.h"
-#include "affine_transform.h"
-#include "simd.h"
-
-/*
-  This file contains the definition for a fully connected layer (aka affine transform) with block sparse input.
-*/
-
-namespace Stockfish::Eval::NNUE::Layers {
-#if defined(__GNUC__)  // GCC, Clang, ICC
-
-  static inline IndexType lsb_(std::uint32_t b) {
-    assert(b);
-    return IndexType(__builtin_ctzl(b));
-  }
-
-#elif defined(_MSC_VER)  // MSVC
-
-  static inline IndexType lsb_(std::uint32_t b) {
-    assert(b);
-    unsigned long idx;
-    _BitScanForward(&idx, b);
-    return (IndexType) idx;
-  }
-
-#else  // Compiler is neither GCC nor MSVC compatible
-
-#error "Compiler not supported."
-
-#endif
-
-
-#if defined(USE_SSSE3)
-  alignas(CacheLineSize) static inline const std::array<std::array<std::uint16_t, 8>, 256> lookup_indices = [](){
-    std::array<std::array<std::uint16_t, 8>, 256> v{};
-    for (int i = 0; i < 256; ++i)
-    {
-      int j = i;
-      int k = 0;
-      while(j)
-      {
-        const IndexType lsbIndex = lsb_(std::uint32_t(j));
-        j &= j - 1;
-        v[i][k] = lsbIndex;
-        ++k;
-      }
-    }
-    return v;
-  }();
-  alignas(CacheLineSize) static inline const std::array<unsigned, 256> lookup_count = [](){
-    std::array<unsigned, 256> v;
-    for (int i = 0; i < 256; ++i)
-    {
-      int j = i;
-      int k = 0;
-      while(j)
-      {
-        j &= j - 1;
-        ++k;
-      }
-      v[i] = k;
-    }
-    return v;
-  }();
-
-  // Find indices of nonzero numbers in an int32_t array
-  template<const IndexType InputDimensions>
-  void find_nnz(const std::int32_t* input, std::uint16_t* out, IndexType& count_out) {
-#if defined (USE_AVX512)
-    using vec_t = __m512i;
-    #define vec_nnz(a) _mm512_cmpgt_epi32_mask(a, _mm512_setzero_si512())
-#elif defined (USE_AVX2)
-    using vec_t = __m256i;
-    #define vec_nnz(a) _mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(a, _mm256_setzero_si256())))
-#elif defined (USE_SSSE3)
-    using vec_t = __m128i;
-    #define vec_nnz(a) _mm_movemask_ps(_mm_castsi128_ps(_mm_cmpgt_epi32(a, _mm_setzero_si128())))
-#endif
-    constexpr IndexType InputSimdWidth = sizeof(vec_t) / sizeof(std::int32_t);
-    // Inputs are processed InputSimdWidth at a time and outputs are processed 8 at a time so we process in chunks of max(InputSimdWidth, 8)
-    constexpr IndexType ChunkSize = std::max<IndexType>(InputSimdWidth, 8);
-    constexpr IndexType NumChunks = InputDimensions / ChunkSize;
-    constexpr IndexType InputsPerChunk = ChunkSize / InputSimdWidth;
-    constexpr IndexType OutputsPerChunk = ChunkSize / 8;
-
-    const auto inputVector = reinterpret_cast<const vec_t*>(input);
-    IndexType count = 0;
-    __m128i base = _mm_set1_epi16(0);
-    __m128i increment = _mm_set1_epi16(8);
-    for (IndexType i = 0; i < NumChunks; ++i)
-    {
-      // bitmask of nonzero values in this chunk
-      unsigned nnz = 0;
-      for (IndexType j = 0; j < InputsPerChunk; ++j)
-      {
-        const vec_t inputChunk = inputVector[i * InputsPerChunk + j];
-        nnz |= (unsigned)vec_nnz(inputChunk) << (j * InputSimdWidth);
-      }
-      for (IndexType j = 0; j < OutputsPerChunk; ++j)
-      {
-        const auto lookup = (nnz >> (j * 8)) & 0xFF;
-        const auto offsets = _mm_loadu_si128(reinterpret_cast<const __m128i*>(&lookup_indices[lookup]));
-        _mm_storeu_si128(reinterpret_cast<__m128i*>(out + count), _mm_add_epi16(base, offsets));
-        count += lookup_count[lookup];
-        base = _mm_add_epi16(base, increment);
-      }
-    }
-    count_out = count;
-  }
-# undef vec_nnz
-#endif
-
-  // Sparse input implementation
-  template <IndexType InDims, IndexType OutDims>
-  class AffineTransformSparseInput {
-   public:
-    // Input/output type
-    // Input/output type
-    using InputType = std::uint8_t;
-    using OutputType = std::int32_t;
-
-    // Number of input/output dimensions
-    static constexpr IndexType InputDimensions = InDims;
-    static constexpr IndexType OutputDimensions = OutDims;
-
-    static_assert(OutputDimensions % 16 == 0, "Only implemented for OutputDimensions divisible by 16.");
-
-    static constexpr IndexType PaddedInputDimensions =
-      ceil_to_multiple<IndexType>(InputDimensions, MaxSimdWidth);
-    static constexpr IndexType PaddedOutputDimensions =
-      ceil_to_multiple<IndexType>(OutputDimensions, MaxSimdWidth);
-
-#if defined (USE_SSSE3)
-    static constexpr IndexType ChunkSize = 4;
-#else
-    static constexpr IndexType ChunkSize = 1;
-#endif
-
-    using OutputBuffer = OutputType[PaddedOutputDimensions];
-
-    // Hash value embedded in the evaluation file
-    static constexpr std::uint32_t get_hash_value(std::uint32_t prevHash) {
-      std::uint32_t hashValue = 0xCC03DAE4u;
-      hashValue += OutputDimensions;
-      hashValue ^= prevHash >> 1;
-      hashValue ^= prevHash << 31;
-      return hashValue;
-    }
-
-    static IndexType get_weight_index_scrambled(IndexType i)
-    {
-      return
-        (i / ChunkSize) % (PaddedInputDimensions / ChunkSize) * OutputDimensions * ChunkSize +
-        i / PaddedInputDimensions * ChunkSize +
-        i % ChunkSize;
-    }
-
-    static IndexType get_weight_index(IndexType i)
-    {
-#if defined (USE_SSSE3)
-      return get_weight_index_scrambled(i);
-#else
-      return i;
-#endif
-    }
-
-    // Read network parameters
-    bool read_parameters(std::istream& stream) {
-      read_little_endian<BiasType>(stream, biases, OutputDimensions);
-      for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i)
-        weights[get_weight_index(i)] = read_little_endian<WeightType>(stream);
-
-      return !stream.fail();
-    }
-
-    // Write network parameters
-    bool write_parameters(std::ostream& stream) const {
-      write_little_endian<BiasType>(stream, biases, OutputDimensions);
-
-      for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i)
-        write_little_endian<WeightType>(stream, weights[get_weight_index(i)]);
-
-      return !stream.fail();
-    }
-    // Forward propagation
-    const OutputType* propagate(
-        const InputType* input, OutputType* output) const {
-
-#if defined (USE_SSSE3)
-#if defined (USE_AVX512)
-      using vec_t = __m512i;
-      #define vec_setzero _mm512_setzero_si512
-      #define vec_set_32 _mm512_set1_epi32
-      #define vec_add_dpbusd_32 Simd::m512_add_dpbusd_epi32
-#elif defined (USE_AVX2)
-      using vec_t = __m256i;
-      #define vec_setzero _mm256_setzero_si256
-      #define vec_set_32 _mm256_set1_epi32
-      #define vec_add_dpbusd_32 Simd::m256_add_dpbusd_epi32
-#elif defined (USE_SSSE3)
-      using vec_t = __m128i;
-      #define vec_setzero _mm_setzero_si128
-      #define vec_set_32 _mm_set1_epi32
-      #define vec_add_dpbusd_32 Simd::m128_add_dpbusd_epi32
-#endif
-      static constexpr IndexType OutputSimdWidth = sizeof(vec_t) / sizeof(OutputType);
-
-      constexpr IndexType NumChunks = ceil_to_multiple<IndexType>(InputDimensions, 8) / ChunkSize;
-      constexpr IndexType NumRegs = OutputDimensions / OutputSimdWidth;
-      std::uint16_t nnz[NumChunks];
-      IndexType count;
-
-      const auto input32 = reinterpret_cast<const std::int32_t*>(input);
-
-      // Find indices of nonzero 32bit blocks
-      find_nnz<NumChunks>(input32, nnz, count);
-
-      const vec_t* biasvec = reinterpret_cast<const vec_t*>(biases);
-      vec_t acc[NumRegs];
-      for (IndexType k = 0; k < NumRegs; ++k)
-        acc[k] = biasvec[k];
-
-      for (IndexType j = 0; j < count; ++j)
-      {
-        const auto i = nnz[j];
-        const vec_t in = vec_set_32(input32[i]);
-        const auto col = reinterpret_cast<const vec_t*>(&weights[i * OutputDimensions * ChunkSize]);
-        for (IndexType k = 0; k < NumRegs; ++k)
-          vec_add_dpbusd_32(acc[k], in, col[k]);
-      }
-
-      vec_t* outptr = reinterpret_cast<vec_t*>(output);
-      for (IndexType k = 0; k < NumRegs; ++k)
-        outptr[k] = acc[k];
-# undef vec_setzero
-# undef vec_set_32
-# undef vec_add_dpbusd_32
-#else
-      // Use dense implementation for the other architectures.
-      affine_transform_non_ssse3<
-        InputDimensions,
-        PaddedInputDimensions,
-        OutputDimensions>(output, weights, biases, input);
-#endif
-
-      return output;
-    }
-
-   private:
-    using BiasType = OutputType;
-    using WeightType = std::int8_t;
-
-    alignas(CacheLineSize) BiasType biases[OutputDimensions];
-    alignas(CacheLineSize) WeightType weights[OutputDimensions * PaddedInputDimensions];
-  };
-
-}  // namespace Stockfish::Eval::NNUE::Layers
-
-#endif // #ifndef NNUE_LAYERS_AFFINE_TRANSFORM_SPARSE_INPUT_H_INCLUDED

src/nnue/layers/clipped_relu.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/nnue/layers/sqr_clipped_relu.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -106,7 +106,7 @@ namespace Stockfish::Eval::NNUE::Layers {
 
       for (IndexType i = Start; i < InputDimensions; ++i) {
         output[i] = static_cast<OutputType>(
-            // really should be /127 but we need to make it fast
+            // realy should be /127 but we need to make it fast
             // needs to be accounted for in the trainer
             std::max(0ll, std::min(127ll, (((long long)input[i] * input[i]) >> (2 * WeightScaleBits)) / 128)));
       }

src/nnue/nnue_accumulator.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/nnue/nnue_architecture.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -27,7 +27,6 @@
 
 #include "features/half_ka_v2_hm.h"
 
-#include "layers/affine_transform_sparse_input.h"
 #include "layers/affine_transform.h"
 #include "layers/clipped_relu.h"
 #include "layers/sqr_clipped_relu.h"
@@ -40,7 +39,7 @@ namespace Stockfish::Eval::NNUE {
 using FeatureSet = Features::HalfKAv2_hm;
 
 // Number of input feature dimensions after conversion
-constexpr IndexType TransformedFeatureDimensions = 1536;
+constexpr IndexType TransformedFeatureDimensions = 1024;
 constexpr IndexType PSQTBuckets = 8;
 constexpr IndexType LayerStacks = 8;
 
@@ -49,7 +48,7 @@ struct Network
   static constexpr int FC_0_OUTPUTS = 15;
   static constexpr int FC_1_OUTPUTS = 32;
 
-  Layers::AffineTransformSparseInput<TransformedFeatureDimensions, FC_0_OUTPUTS + 1> fc_0;
+  Layers::AffineTransform<TransformedFeatureDimensions, FC_0_OUTPUTS + 1> fc_0;
   Layers::SqrClippedReLU<FC_0_OUTPUTS + 1> ac_sqr_0;
   Layers::ClippedReLU<FC_0_OUTPUTS + 1> ac_0;
   Layers::AffineTransform<FC_0_OUTPUTS * 2, FC_1_OUTPUTS> fc_1;
@@ -73,20 +72,22 @@ struct Network
 
   // Read network parameters
   bool read_parameters(std::istream& stream) {
-    return   fc_0.read_parameters(stream)
-          && ac_0.read_parameters(stream)
-          && fc_1.read_parameters(stream)
-          && ac_1.read_parameters(stream)
-          && fc_2.read_parameters(stream);
+    if (!fc_0.read_parameters(stream)) return false;
+    if (!ac_0.read_parameters(stream)) return false;
+    if (!fc_1.read_parameters(stream)) return false;
+    if (!ac_1.read_parameters(stream)) return false;
+    if (!fc_2.read_parameters(stream)) return false;
+    return true;
   }
 
-  // Write network parameters
+  // Read network parameters
   bool write_parameters(std::ostream& stream) const {
-    return   fc_0.write_parameters(stream)
-          && ac_0.write_parameters(stream)
-          && fc_1.write_parameters(stream)
-          && ac_1.write_parameters(stream)
-          && fc_2.write_parameters(stream);
+    if (!fc_0.write_parameters(stream)) return false;
+    if (!ac_0.write_parameters(stream)) return false;
+    if (!fc_1.write_parameters(stream)) return false;
+    if (!ac_1.write_parameters(stream)) return false;
+    if (!fc_2.write_parameters(stream)) return false;
+    return true;
   }
 
   std::int32_t propagate(const TransformedFeatureType* transformedFeatures)

src/nnue/nnue_common.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -21,6 +21,8 @@
 #ifndef NNUE_COMMON_H_INCLUDED
 #define NNUE_COMMON_H_INCLUDED
 
+#include "../types.h"
+
 #include <cstring>
 #include <iostream>
 
@@ -57,9 +59,6 @@ namespace Stockfish::Eval::NNUE {
   // Size of cache line (in bytes)
   constexpr std::size_t CacheLineSize = 64;
 
-  constexpr const char Leb128MagicString[] = "COMPRESSED_LEB128";
-  constexpr const std::size_t Leb128MagicStringSize = sizeof(Leb128MagicString) - 1;
-
   // SIMD width (in bytes)
   #if defined(USE_AVX2)
   constexpr std::size_t SimdWidth = 32;
@@ -162,80 +161,6 @@ namespace Stockfish::Eval::NNUE {
               write_little_endian<IntType>(stream, values[i]);
   }
 
-  template <typename IntType>
-  inline void read_leb_128(std::istream& stream, IntType* out, std::size_t count) {
-      static_assert(std::is_signed_v<IntType>, "Not implemented for unsigned types");
-      char leb128MagicString[Leb128MagicStringSize];
-      stream.read(leb128MagicString, Leb128MagicStringSize);
-      assert(strncmp(Leb128MagicString, leb128MagicString, Leb128MagicStringSize) == 0);
-      const std::uint32_t BUF_SIZE = 4096;
-      std::uint8_t buf[BUF_SIZE];
-      auto bytes_left = read_little_endian<std::uint32_t>(stream);
-      std::uint32_t buf_pos = BUF_SIZE;
-      for (std::size_t i = 0; i < count; ++i) {
-          IntType result = 0;
-          size_t shift = 0;
-          do {
-              if (buf_pos == BUF_SIZE) {
-                  stream.read(reinterpret_cast<char*>(buf), std::min(bytes_left, BUF_SIZE));
-                  buf_pos = 0;
-              }
-              std::uint8_t byte = buf[buf_pos++];
-              --bytes_left;
-              result |= (byte & 0x7f) << shift;
-              shift += 7;
-              if ((byte & 0x80) == 0) {
-                  out[i] = sizeof(IntType) * 8 <= shift || (byte & 0x40) == 0 ? result : result | ~((1 << shift) - 1);
-                  break;
-              }
-          } while (shift < sizeof(IntType) * 8);
-      }
-      assert(bytes_left == 0);
-  }
-
-  template <typename IntType>
-  inline void write_leb_128(std::ostream& stream, const IntType* values, std::size_t count) {
-      static_assert(std::is_signed_v<IntType>, "Not implemented for unsigned types");
-      stream.write(Leb128MagicString, Leb128MagicStringSize);
-      std::uint32_t byte_count = 0;
-      for (std::size_t i = 0; i < count; ++i) {
-          IntType value = values[i];
-          std::uint8_t byte;
-          do {
-              byte = value & 0x7f;
-              value >>= 7;
-              ++byte_count;
-          } while ((byte & 0x40) == 0 ? value != 0 : value != -1);
-      }
-      write_little_endian(stream, byte_count);
-      const std::uint32_t BUF_SIZE = 4096;
-      std::uint8_t buf[BUF_SIZE];
-      std::uint32_t buf_pos = 0;
-      auto flush = [&]() {
-          if (buf_pos > 0) {
-              stream.write(reinterpret_cast<char*>(buf), buf_pos);
-              buf_pos = 0;
-          }
-      };
-      auto write = [&](std::uint8_t byte) {
-          buf[buf_pos++] = byte;
-          if (buf_pos == BUF_SIZE) flush();
-      };
-      for (std::size_t i = 0; i < count; ++i) {
-          IntType value = values[i];
-          while (true) {
-              std::uint8_t byte = value & 0x7f;
-              value >>= 7;
-              if ((byte & 0x40) == 0 ? value == 0 : value == -1) {
-                  write(byte);
-                  break;
-              }
-              write(byte | 0x80);
-          }
-      }
-      flush();
-  }
-
 }  // namespace Stockfish::Eval::NNUE
 
 #endif // #ifndef NNUE_COMMON_H_INCLUDED

src/nnue/nnue_feature_transformer.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -24,8 +24,10 @@
 #include "nnue_common.h"
 #include "nnue_architecture.h"
 
+#include "../misc.h"
+#include "../position.h"
+
 #include <cstring> // std::memset()
-#include <utility> // std::pair
 
 namespace Stockfish::Eval::NNUE {
 
@@ -42,8 +44,8 @@ namespace Stockfish::Eval::NNUE {
     "Per feature PSQT values cannot be processed at granularity lower than 8 at a time.");
 
   #ifdef USE_AVX512
-  using vec_t = __m512i;
-  using psqt_vec_t = __m256i;
+  typedef __m512i vec_t;
+  typedef __m256i psqt_vec_t;
   #define vec_load(a) _mm512_load_si512(a)
   #define vec_store(a,b) _mm512_store_si512(a,b)
   #define vec_add_16(a,b) _mm512_add_epi16(a,b)
@@ -66,8 +68,8 @@ namespace Stockfish::Eval::NNUE {
   #define MaxChunkSize 64
 
   #elif USE_AVX2
-  using vec_t = __m256i;
-  using psqt_vec_t = __m256i;
+  typedef __m256i vec_t;
+  typedef __m256i psqt_vec_t;
   #define vec_load(a) _mm256_load_si256(a)
   #define vec_store(a,b) _mm256_store_si256(a,b)
   #define vec_add_16(a,b) _mm256_add_epi16(a,b)
@@ -90,8 +92,8 @@ namespace Stockfish::Eval::NNUE {
   #define MaxChunkSize 32
 
   #elif USE_SSE2
-  using vec_t = __m128i;
-  using psqt_vec_t = __m128i;
+  typedef __m128i vec_t;
+  typedef __m128i psqt_vec_t;
   #define vec_load(a) (*(a))
   #define vec_store(a,b) *(a)=(b)
   #define vec_add_16(a,b) _mm_add_epi16(a,b)
@@ -111,8 +113,8 @@ namespace Stockfish::Eval::NNUE {
   #define MaxChunkSize 16
 
   #elif USE_MMX
-  using vec_t = __m64;
-  using psqt_vec_t = __m64;
+  typedef __m64 vec_t;
+  typedef __m64 psqt_vec_t;
   #define vec_load(a) (*(a))
   #define vec_store(a,b) *(a)=(b)
   #define vec_add_16(a,b) _mm_add_pi16(a,b)
@@ -139,8 +141,8 @@ namespace Stockfish::Eval::NNUE {
   #define MaxChunkSize 8
 
   #elif USE_NEON
-  using vec_t = int16x8_t;
-  using psqt_vec_t = int32x4_t;
+  typedef int16x8_t vec_t;
+  typedef int32x4_t psqt_vec_t;
   #define vec_load(a) (*(a))
   #define vec_store(a,b) *(a)=(b)
   #define vec_add_16(a,b) vaddq_s16(a,b)
@@ -253,9 +255,9 @@ namespace Stockfish::Eval::NNUE {
     // Read network parameters
     bool read_parameters(std::istream& stream) {
 
-      read_leb_128<BiasType      >(stream, biases     , HalfDimensions                  );
-      read_leb_128<WeightType    >(stream, weights    , HalfDimensions * InputDimensions);
-      read_leb_128<PSQTWeightType>(stream, psqtWeights, PSQTBuckets    * InputDimensions);
+      read_little_endian<BiasType      >(stream, biases     , HalfDimensions                  );
+      read_little_endian<WeightType    >(stream, weights    , HalfDimensions * InputDimensions);
+      read_little_endian<PSQTWeightType>(stream, psqtWeights, PSQTBuckets    * InputDimensions);
 
       return !stream.fail();
     }
@@ -263,17 +265,17 @@ namespace Stockfish::Eval::NNUE {
     // Write network parameters
     bool write_parameters(std::ostream& stream) const {
 
-      write_leb_128<BiasType      >(stream, biases     , HalfDimensions                  );
-      write_leb_128<WeightType    >(stream, weights    , HalfDimensions * InputDimensions);
-      write_leb_128<PSQTWeightType>(stream, psqtWeights, PSQTBuckets    * InputDimensions);
+      write_little_endian<BiasType      >(stream, biases     , HalfDimensions                  );
+      write_little_endian<WeightType    >(stream, weights    , HalfDimensions * InputDimensions);
+      write_little_endian<PSQTWeightType>(stream, psqtWeights, PSQTBuckets    * InputDimensions);
 
       return !stream.fail();
     }
 
     // Convert input features
     std::int32_t transform(const Position& pos, OutputType* output, int bucket) const {
-      update_accumulator<WHITE>(pos);
-      update_accumulator<BLACK>(pos);
+      update_accumulator(pos, WHITE);
+      update_accumulator(pos, BLACK);
 
       const Color perspectives[2] = {pos.side_to_move(), ~pos.side_to_move()};
       const auto& accumulation = pos.state()->accumulator.accumulation;
@@ -333,203 +335,19 @@ namespace Stockfish::Eval::NNUE {
 #endif
 
       return psqt;
-    } // end of function transform()
 
-    void hint_common_access(const Position& pos) const {
-      hint_common_access_for_perspective<WHITE>(pos);
-      hint_common_access_for_perspective<BLACK>(pos);
-    }
+   } // end of function transform()
+
+
 
    private:
-    template<Color Perspective>
-    [[nodiscard]] std::pair<StateInfo*, StateInfo*> try_find_computed_accumulator(const Position& pos) const {
-      // Look for a usable accumulator of an earlier position. We keep track
-      // of the estimated gain in terms of features to be added/subtracted.
-      StateInfo *st = pos.state(), *next = nullptr;
-      int gain = FeatureSet::refresh_cost(pos);
-      while (st->previous && !st->accumulator.computed[Perspective])
-      {
-        // This governs when a full feature refresh is needed and how many
-        // updates are better than just one full refresh.
-        if (   FeatureSet::requires_refresh(st, Perspective)
-            || (gain -= FeatureSet::update_cost(st) + 1) < 0)
-          break;
-        next = st;
-        st = st->previous;
-      }
-      return { st, next };
-    }
-
-    // NOTE: The parameter states_to_update is an array of position states, ending with nullptr.
-    //       All states must be sequential, that is states_to_update[i] must either be reachable
-    //       by repeatedly applying ->previous from states_to_update[i+1] or states_to_update[i] == nullptr.
-    //       computed_st must be reachable by repeatedly applying ->previous on states_to_update[0], if not nullptr.
-    template<Color Perspective, size_t N>
-    void update_accumulator_incremental(const Position& pos, StateInfo* computed_st, StateInfo* states_to_update[N]) const {
-      static_assert(N > 0);
-      assert(states_to_update[N-1] == nullptr);
-
-  #ifdef VECTOR
-      // Gcc-10.2 unnecessarily spills AVX2 registers if this array
-      // is defined in the VECTOR code below, once in each branch
-      vec_t acc[NumRegs];
-      psqt_vec_t psqt[NumPsqtRegs];
-  #endif
-
-      if (states_to_update[0] == nullptr)
-        return;
-
-      // Update incrementally going back through states_to_update.
-
-      // Gather all features to be updated.
-      const Square ksq = pos.square<KING>(Perspective);
+    void update_accumulator(const Position& pos, const Color perspective) const {
 
       // The size must be enough to contain the largest possible update.
       // That might depend on the feature set and generally relies on the
       // feature set's update cost calculation to be correct and never
       // allow updates with more added/removed features than MaxActiveDimensions.
-      FeatureSet::IndexList removed[N-1], added[N-1];
 
-      {
-        int i = N-2; // last potential state to update. Skip last element because it must be nullptr.
-        while (states_to_update[i] == nullptr)
-          --i;
-
-        StateInfo *st2 = states_to_update[i];
-
-        for (; i >= 0; --i)
-        {
-          states_to_update[i]->accumulator.computed[Perspective] = true;
-
-          StateInfo* end_state = i == 0 ? computed_st : states_to_update[i - 1];
-
-          for (; st2 != end_state; st2 = st2->previous)
-            FeatureSet::append_changed_indices<Perspective>(
-              ksq, st2->dirtyPiece, removed[i], added[i]);
-        }
-      }
-
-      StateInfo* st = computed_st;
-
-      // Now update the accumulators listed in states_to_update[], where the last element is a sentinel.
-#ifdef VECTOR
-      for (IndexType j = 0; j < HalfDimensions / TileHeight; ++j)
-      {
-        // Load accumulator
-        auto accTile = reinterpret_cast<vec_t*>(
-          &st->accumulator.accumulation[Perspective][j * TileHeight]);
-        for (IndexType k = 0; k < NumRegs; ++k)
-          acc[k] = vec_load(&accTile[k]);
-
-        for (IndexType i = 0; states_to_update[i]; ++i)
-        {
-          // Difference calculation for the deactivated features
-          for (const auto index : removed[i])
-          {
-            const IndexType offset = HalfDimensions * index + j * TileHeight;
-            auto column = reinterpret_cast<const vec_t*>(&weights[offset]);
-            for (IndexType k = 0; k < NumRegs; ++k)
-              acc[k] = vec_sub_16(acc[k], column[k]);
-          }
-
-          // Difference calculation for the activated features
-          for (const auto index : added[i])
-          {
-            const IndexType offset = HalfDimensions * index + j * TileHeight;
-            auto column = reinterpret_cast<const vec_t*>(&weights[offset]);
-            for (IndexType k = 0; k < NumRegs; ++k)
-              acc[k] = vec_add_16(acc[k], column[k]);
-          }
-
-          // Store accumulator
-          accTile = reinterpret_cast<vec_t*>(
-            &states_to_update[i]->accumulator.accumulation[Perspective][j * TileHeight]);
-          for (IndexType k = 0; k < NumRegs; ++k)
-            vec_store(&accTile[k], acc[k]);
-        }
-      }
-
-      for (IndexType j = 0; j < PSQTBuckets / PsqtTileHeight; ++j)
-      {
-        // Load accumulator
-        auto accTilePsqt = reinterpret_cast<psqt_vec_t*>(
-          &st->accumulator.psqtAccumulation[Perspective][j * PsqtTileHeight]);
-        for (std::size_t k = 0; k < NumPsqtRegs; ++k)
-          psqt[k] = vec_load_psqt(&accTilePsqt[k]);
-
-        for (IndexType i = 0; states_to_update[i]; ++i)
-        {
-          // Difference calculation for the deactivated features
-          for (const auto index : removed[i])
-          {
-            const IndexType offset = PSQTBuckets * index + j * PsqtTileHeight;
-            auto columnPsqt = reinterpret_cast<const psqt_vec_t*>(&psqtWeights[offset]);
-            for (std::size_t k = 0; k < NumPsqtRegs; ++k)
-              psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]);
-          }
-
-          // Difference calculation for the activated features
-          for (const auto index : added[i])
-          {
-            const IndexType offset = PSQTBuckets * index + j * PsqtTileHeight;
-            auto columnPsqt = reinterpret_cast<const psqt_vec_t*>(&psqtWeights[offset]);
-            for (std::size_t k = 0; k < NumPsqtRegs; ++k)
-              psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]);
-          }
-
-          // Store accumulator
-          accTilePsqt = reinterpret_cast<psqt_vec_t*>(
-            &states_to_update[i]->accumulator.psqtAccumulation[Perspective][j * PsqtTileHeight]);
-          for (std::size_t k = 0; k < NumPsqtRegs; ++k)
-            vec_store_psqt(&accTilePsqt[k], psqt[k]);
-        }
-      }
-
-#else
-      for (IndexType i = 0; states_to_update[i]; ++i)
-      {
-        std::memcpy(states_to_update[i]->accumulator.accumulation[Perspective],
-            st->accumulator.accumulation[Perspective],
-            HalfDimensions * sizeof(BiasType));
-
-        for (std::size_t k = 0; k < PSQTBuckets; ++k)
-          states_to_update[i]->accumulator.psqtAccumulation[Perspective][k] = st->accumulator.psqtAccumulation[Perspective][k];
-
-        st = states_to_update[i];
-
-        // Difference calculation for the deactivated features
-        for (const auto index : removed[i])
-        {
-          const IndexType offset = HalfDimensions * index;
-
-          for (IndexType j = 0; j < HalfDimensions; ++j)
-            st->accumulator.accumulation[Perspective][j] -= weights[offset + j];
-
-          for (std::size_t k = 0; k < PSQTBuckets; ++k)
-            st->accumulator.psqtAccumulation[Perspective][k] -= psqtWeights[index * PSQTBuckets + k];
-        }
-
-        // Difference calculation for the activated features
-        for (const auto index : added[i])
-        {
-          const IndexType offset = HalfDimensions * index;
-
-          for (IndexType j = 0; j < HalfDimensions; ++j)
-            st->accumulator.accumulation[Perspective][j] += weights[offset + j];
-
-          for (std::size_t k = 0; k < PSQTBuckets; ++k)
-            st->accumulator.psqtAccumulation[Perspective][k] += psqtWeights[index * PSQTBuckets + k];
-        }
-      }
-#endif
-
-  #if defined(USE_MMX)
-      _mm_empty();
-  #endif
-    }
-
-    template<Color Perspective>
-    void update_accumulator_refresh(const Position& pos) const {
   #ifdef VECTOR
       // Gcc-10.2 unnecessarily spills AVX2 registers if this array
       // is defined in the VECTOR code below, once in each branch
@@ -537,131 +355,230 @@ namespace Stockfish::Eval::NNUE {
       psqt_vec_t psqt[NumPsqtRegs];
   #endif
 
-      // Refresh the accumulator
-      // Could be extracted to a separate function because it's done in 2 places,
-      // but it's unclear if compilers would correctly handle register allocation.
-      auto& accumulator = pos.state()->accumulator;
-      accumulator.computed[Perspective] = true;
-      FeatureSet::IndexList active;
-      FeatureSet::append_active_indices<Perspective>(pos, active);
-
-#ifdef VECTOR
-      for (IndexType j = 0; j < HalfDimensions / TileHeight; ++j)
-      {
-        auto biasesTile = reinterpret_cast<const vec_t*>(
-            &biases[j * TileHeight]);
-        for (IndexType k = 0; k < NumRegs; ++k)
-          acc[k] = biasesTile[k];
-
-        for (const auto index : active)
-        {
-          const IndexType offset = HalfDimensions * index + j * TileHeight;
-          auto column = reinterpret_cast<const vec_t*>(&weights[offset]);
-
-          for (unsigned k = 0; k < NumRegs; ++k)
-            acc[k] = vec_add_16(acc[k], column[k]);
-        }
-
-        auto accTile = reinterpret_cast<vec_t*>(
-            &accumulator.accumulation[Perspective][j * TileHeight]);
-        for (unsigned k = 0; k < NumRegs; k++)
-          vec_store(&accTile[k], acc[k]);
-      }
-
-      for (IndexType j = 0; j < PSQTBuckets / PsqtTileHeight; ++j)
-      {
-        for (std::size_t k = 0; k < NumPsqtRegs; ++k)
-          psqt[k] = vec_zero_psqt();
-
-        for (const auto index : active)
-        {
-          const IndexType offset = PSQTBuckets * index + j * PsqtTileHeight;
-          auto columnPsqt = reinterpret_cast<const psqt_vec_t*>(&psqtWeights[offset]);
-
-          for (std::size_t k = 0; k < NumPsqtRegs; ++k)
-            psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]);
-        }
-
-        auto accTilePsqt = reinterpret_cast<psqt_vec_t*>(
-          &accumulator.psqtAccumulation[Perspective][j * PsqtTileHeight]);
-        for (std::size_t k = 0; k < NumPsqtRegs; ++k)
-          vec_store_psqt(&accTilePsqt[k], psqt[k]);
-      }
-
-#else
-      std::memcpy(accumulator.accumulation[Perspective], biases,
-          HalfDimensions * sizeof(BiasType));
-
-      for (std::size_t k = 0; k < PSQTBuckets; ++k)
-        accumulator.psqtAccumulation[Perspective][k] = 0;
-
-      for (const auto index : active)
-      {
-        const IndexType offset = HalfDimensions * index;
-
-        for (IndexType j = 0; j < HalfDimensions; ++j)
-          accumulator.accumulation[Perspective][j] += weights[offset + j];
-
-        for (std::size_t k = 0; k < PSQTBuckets; ++k)
-          accumulator.psqtAccumulation[Perspective][k] += psqtWeights[index * PSQTBuckets + k];
-      }
-#endif
-
-  #if defined(USE_MMX)
-      _mm_empty();
-  #endif
-    }
-
-    template<Color Perspective>
-    void hint_common_access_for_perspective(const Position& pos) const {
-
-      // Works like update_accumulator, but performs less work.
-      // Updates ONLY the accumulator for pos.
-
       // Look for a usable accumulator of an earlier position. We keep track
       // of the estimated gain in terms of features to be added/subtracted.
-      // Fast early exit.
-      if (pos.state()->accumulator.computed[Perspective])
-        return;
-
-      auto [oldest_st, _] = try_find_computed_accumulator<Perspective>(pos);
-
-      if (oldest_st->accumulator.computed[Perspective])
+      StateInfo *st = pos.state(), *next = nullptr;
+      int gain = FeatureSet::refresh_cost(pos);
+      while (st->previous && !st->accumulator.computed[perspective])
       {
-        // Only update current position accumulator to minimize work.
-        StateInfo* states_to_update[2] = { pos.state(), nullptr };
-        update_accumulator_incremental<Perspective, 2>(pos, oldest_st, states_to_update);
+        // This governs when a full feature refresh is needed and how many
+        // updates are better than just one full refresh.
+        if (   FeatureSet::requires_refresh(st, perspective)
+            || (gain -= FeatureSet::update_cost(st) + 1) < 0)
+          break;
+        next = st;
+        st = st->previous;
       }
-      else
-      {
-        update_accumulator_refresh<Perspective>(pos);
-      }
-    }
 
-    template<Color Perspective>
-    void update_accumulator(const Position& pos) const {
-
-      auto [oldest_st, next] = try_find_computed_accumulator<Perspective>(pos);
-
-      if (oldest_st->accumulator.computed[Perspective])
+      if (st->accumulator.computed[perspective])
       {
         if (next == nullptr)
           return;
 
+        // Update incrementally in two steps. First, we update the "next"
+        // accumulator. Then, we update the current accumulator (pos.state()).
+
+        // Gather all features to be updated.
+        const Square ksq = pos.square<KING>(perspective);
+        FeatureSet::IndexList removed[2], added[2];
+        FeatureSet::append_changed_indices(
+          ksq, next->dirtyPiece, perspective, removed[0], added[0]);
+        for (StateInfo *st2 = pos.state(); st2 != next; st2 = st2->previous)
+          FeatureSet::append_changed_indices(
+            ksq, st2->dirtyPiece, perspective, removed[1], added[1]);
+
+        // Mark the accumulators as computed.
+        next->accumulator.computed[perspective] = true;
+        pos.state()->accumulator.computed[perspective] = true;
+
         // Now update the accumulators listed in states_to_update[], where the last element is a sentinel.
-        // Currently we update 2 accumulators.
-        //     1. for the current position
-        //     2. the next accumulator after the computed one
-        // The heuristic may change in the future.
         StateInfo *states_to_update[3] =
           { next, next == pos.state() ? nullptr : pos.state(), nullptr };
+  #ifdef VECTOR
+        for (IndexType j = 0; j < HalfDimensions / TileHeight; ++j)
+        {
+          // Load accumulator
+          auto accTile = reinterpret_cast<vec_t*>(
+            &st->accumulator.accumulation[perspective][j * TileHeight]);
+          for (IndexType k = 0; k < NumRegs; ++k)
+            acc[k] = vec_load(&accTile[k]);
 
-        update_accumulator_incremental<Perspective, 3>(pos, oldest_st, states_to_update);
+          for (IndexType i = 0; states_to_update[i]; ++i)
+          {
+            // Difference calculation for the deactivated features
+            for (const auto index : removed[i])
+            {
+              const IndexType offset = HalfDimensions * index + j * TileHeight;
+              auto column = reinterpret_cast<const vec_t*>(&weights[offset]);
+              for (IndexType k = 0; k < NumRegs; ++k)
+                acc[k] = vec_sub_16(acc[k], column[k]);
+            }
+
+            // Difference calculation for the activated features
+            for (const auto index : added[i])
+            {
+              const IndexType offset = HalfDimensions * index + j * TileHeight;
+              auto column = reinterpret_cast<const vec_t*>(&weights[offset]);
+              for (IndexType k = 0; k < NumRegs; ++k)
+                acc[k] = vec_add_16(acc[k], column[k]);
+            }
+
+            // Store accumulator
+            accTile = reinterpret_cast<vec_t*>(
+              &states_to_update[i]->accumulator.accumulation[perspective][j * TileHeight]);
+            for (IndexType k = 0; k < NumRegs; ++k)
+              vec_store(&accTile[k], acc[k]);
+          }
+        }
+
+        for (IndexType j = 0; j < PSQTBuckets / PsqtTileHeight; ++j)
+        {
+          // Load accumulator
+          auto accTilePsqt = reinterpret_cast<psqt_vec_t*>(
+            &st->accumulator.psqtAccumulation[perspective][j * PsqtTileHeight]);
+          for (std::size_t k = 0; k < NumPsqtRegs; ++k)
+            psqt[k] = vec_load_psqt(&accTilePsqt[k]);
+
+          for (IndexType i = 0; states_to_update[i]; ++i)
+          {
+            // Difference calculation for the deactivated features
+            for (const auto index : removed[i])
+            {
+              const IndexType offset = PSQTBuckets * index + j * PsqtTileHeight;
+              auto columnPsqt = reinterpret_cast<const psqt_vec_t*>(&psqtWeights[offset]);
+              for (std::size_t k = 0; k < NumPsqtRegs; ++k)
+                psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]);
+            }
+
+            // Difference calculation for the activated features
+            for (const auto index : added[i])
+            {
+              const IndexType offset = PSQTBuckets * index + j * PsqtTileHeight;
+              auto columnPsqt = reinterpret_cast<const psqt_vec_t*>(&psqtWeights[offset]);
+              for (std::size_t k = 0; k < NumPsqtRegs; ++k)
+                psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]);
+            }
+
+            // Store accumulator
+            accTilePsqt = reinterpret_cast<psqt_vec_t*>(
+              &states_to_update[i]->accumulator.psqtAccumulation[perspective][j * PsqtTileHeight]);
+            for (std::size_t k = 0; k < NumPsqtRegs; ++k)
+              vec_store_psqt(&accTilePsqt[k], psqt[k]);
+          }
+        }
+
+  #else
+        for (IndexType i = 0; states_to_update[i]; ++i)
+        {
+          std::memcpy(states_to_update[i]->accumulator.accumulation[perspective],
+              st->accumulator.accumulation[perspective],
+              HalfDimensions * sizeof(BiasType));
+
+          for (std::size_t k = 0; k < PSQTBuckets; ++k)
+            states_to_update[i]->accumulator.psqtAccumulation[perspective][k] = st->accumulator.psqtAccumulation[perspective][k];
+
+          st = states_to_update[i];
+
+          // Difference calculation for the deactivated features
+          for (const auto index : removed[i])
+          {
+            const IndexType offset = HalfDimensions * index;
+
+            for (IndexType j = 0; j < HalfDimensions; ++j)
+              st->accumulator.accumulation[perspective][j] -= weights[offset + j];
+
+            for (std::size_t k = 0; k < PSQTBuckets; ++k)
+              st->accumulator.psqtAccumulation[perspective][k] -= psqtWeights[index * PSQTBuckets + k];
+          }
+
+          // Difference calculation for the activated features
+          for (const auto index : added[i])
+          {
+            const IndexType offset = HalfDimensions * index;
+
+            for (IndexType j = 0; j < HalfDimensions; ++j)
+              st->accumulator.accumulation[perspective][j] += weights[offset + j];
+
+            for (std::size_t k = 0; k < PSQTBuckets; ++k)
+              st->accumulator.psqtAccumulation[perspective][k] += psqtWeights[index * PSQTBuckets + k];
+          }
+        }
+  #endif
       }
       else
       {
-        update_accumulator_refresh<Perspective>(pos);
+        // Refresh the accumulator
+        auto& accumulator = pos.state()->accumulator;
+        accumulator.computed[perspective] = true;
+        FeatureSet::IndexList active;
+        FeatureSet::append_active_indices(pos, perspective, active);
+
+  #ifdef VECTOR
+        for (IndexType j = 0; j < HalfDimensions / TileHeight; ++j)
+        {
+          auto biasesTile = reinterpret_cast<const vec_t*>(
+              &biases[j * TileHeight]);
+          for (IndexType k = 0; k < NumRegs; ++k)
+            acc[k] = biasesTile[k];
+
+          for (const auto index : active)
+          {
+            const IndexType offset = HalfDimensions * index + j * TileHeight;
+            auto column = reinterpret_cast<const vec_t*>(&weights[offset]);
+
+            for (unsigned k = 0; k < NumRegs; ++k)
+              acc[k] = vec_add_16(acc[k], column[k]);
+          }
+
+          auto accTile = reinterpret_cast<vec_t*>(
+              &accumulator.accumulation[perspective][j * TileHeight]);
+          for (unsigned k = 0; k < NumRegs; k++)
+            vec_store(&accTile[k], acc[k]);
+        }
+
+        for (IndexType j = 0; j < PSQTBuckets / PsqtTileHeight; ++j)
+        {
+          for (std::size_t k = 0; k < NumPsqtRegs; ++k)
+            psqt[k] = vec_zero_psqt();
+
+          for (const auto index : active)
+          {
+            const IndexType offset = PSQTBuckets * index + j * PsqtTileHeight;
+            auto columnPsqt = reinterpret_cast<const psqt_vec_t*>(&psqtWeights[offset]);
+
+            for (std::size_t k = 0; k < NumPsqtRegs; ++k)
+              psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]);
+          }
+
+          auto accTilePsqt = reinterpret_cast<psqt_vec_t*>(
+            &accumulator.psqtAccumulation[perspective][j * PsqtTileHeight]);
+          for (std::size_t k = 0; k < NumPsqtRegs; ++k)
+            vec_store_psqt(&accTilePsqt[k], psqt[k]);
+        }
+
+  #else
+        std::memcpy(accumulator.accumulation[perspective], biases,
+            HalfDimensions * sizeof(BiasType));
+
+        for (std::size_t k = 0; k < PSQTBuckets; ++k)
+          accumulator.psqtAccumulation[perspective][k] = 0;
+
+        for (const auto index : active)
+        {
+          const IndexType offset = HalfDimensions * index;
+
+          for (IndexType j = 0; j < HalfDimensions; ++j)
+            accumulator.accumulation[perspective][j] += weights[offset + j];
+
+          for (std::size_t k = 0; k < PSQTBuckets; ++k)
+            accumulator.psqtAccumulation[perspective][k] += psqtWeights[index * PSQTBuckets + k];
+        }
+  #endif
       }
+
+  #if defined(USE_MMX)
+      _mm_empty();
+  #endif
     }
 
     alignas(CacheLineSize) BiasType biases[HalfDimensions];

src/pawns.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/pawns.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -61,7 +61,7 @@ struct Entry {
   int blockedCount;
 };
 
-using Table = HashTable<Entry, 131072>;
+typedef HashTable<Entry, 131072> Table;
 
 Entry* probe(const Position& pos);
 

src/position.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -22,7 +22,8 @@
 #include <cstring> // For std::memset, std::memcmp
 #include <iomanip>
 #include <sstream>
-#include <string_view>
+
+#include "nnue/evaluate_nnue.h"
 
 #include "bitboard.h"
 #include "misc.h"
@@ -33,6 +34,9 @@
 #include "uci.h"
 #include "syzygy/tbprobe.h"
 
+#include "tools/packed_sfen.h"
+#include "tools/sfen_packer.h"
+
 using std::string;
 
 namespace Stockfish {
@@ -47,7 +51,7 @@ namespace Zobrist {
 
 namespace {
 
-constexpr std::string_view PieceToChar(" PNBRQK  pnbrqk");
+const string PieceToChar(" PNBRQK  pnbrqk");
 
 constexpr Piece Pieces[] = { W_PAWN, W_KNIGHT, W_BISHOP, W_ROOK, W_QUEEN, W_KING,
                              B_PAWN, B_KNIGHT, B_BISHOP, B_ROOK, B_QUEEN, B_KING };
@@ -97,7 +101,7 @@ std::ostream& operator<<(std::ostream& os, const Position& pos) {
 
 // Marcel van Kervinck's cuckoo algorithm for fast detection of "upcoming repetition"
 // situations. Description of the algorithm in the following paper:
-// http://web.archive.org/web/20201107002606/https://marcelk.net/2013-04-06/paper/upcoming-rep-v2.pdf
+// https://marcelk.net/2013-04-06/paper/upcoming-rep-v2.pdf
 
 // First and second hash functions for indexing the cuckoo tables
 inline int H1(Key h) { return h & 0x1fff; }
@@ -130,7 +134,7 @@ void Position::init() {
   // Prepare the cuckoo tables
   std::memset(cuckoo, 0, sizeof(cuckoo));
   std::memset(cuckooMove, 0, sizeof(cuckooMove));
-  [[maybe_unused]] int count = 0;
+  int count = 0;
   for (Piece pc : Pieces)
       for (Square s1 = SQ_A1; s1 <= SQ_H8; ++s1)
           for (Square s2 = Square(s1 + 1); s2 <= SQ_H8; ++s2)
@@ -282,7 +286,7 @@ Position& Position::set(const string& fenStr, bool isChess960, StateInfo* si, Th
 
   chess960 = isChess960;
   thisThread = th;
-  set_state();
+  set_state(st);
 
   assert(pos_is_ok());
 
@@ -313,59 +317,60 @@ void Position::set_castling_right(Color c, Square rfrom) {
 
 /// Position::set_check_info() sets king attacks to detect if a move gives check
 
-void Position::set_check_info() const {
+void Position::set_check_info(StateInfo* si) const {
 
-  st->blockersForKing[WHITE] = slider_blockers(pieces(BLACK), square<KING>(WHITE), st->pinners[BLACK]);
-  st->blockersForKing[BLACK] = slider_blockers(pieces(WHITE), square<KING>(BLACK), st->pinners[WHITE]);
+  si->blockersForKing[WHITE] = slider_blockers(pieces(BLACK), square<KING>(WHITE), si->pinners[BLACK]);
+  si->blockersForKing[BLACK] = slider_blockers(pieces(WHITE), square<KING>(BLACK), si->pinners[WHITE]);
 
   Square ksq = square<KING>(~sideToMove);
 
-  st->checkSquares[PAWN]   = pawn_attacks_bb(~sideToMove, ksq);
-  st->checkSquares[KNIGHT] = attacks_bb<KNIGHT>(ksq);
-  st->checkSquares[BISHOP] = attacks_bb<BISHOP>(ksq, pieces());
-  st->checkSquares[ROOK]   = attacks_bb<ROOK>(ksq, pieces());
-  st->checkSquares[QUEEN]  = st->checkSquares[BISHOP] | st->checkSquares[ROOK];
-  st->checkSquares[KING]   = 0;
+  si->checkSquares[PAWN]   = pawn_attacks_bb(~sideToMove, ksq);
+  si->checkSquares[KNIGHT] = attacks_bb<KNIGHT>(ksq);
+  si->checkSquares[BISHOP] = attacks_bb<BISHOP>(ksq, pieces());
+  si->checkSquares[ROOK]   = attacks_bb<ROOK>(ksq, pieces());
+  si->checkSquares[QUEEN]  = si->checkSquares[BISHOP] | si->checkSquares[ROOK];
+  si->checkSquares[KING]   = 0;
 }
 
 
 /// Position::set_state() computes the hash keys of the position, and other
 /// data that once computed is updated incrementally as moves are made.
-/// The function is only used when a new position is set up
+/// The function is only used when a new position is set up, and to verify
+/// the correctness of the StateInfo data when running in debug mode.
 
-void Position::set_state() const {
+void Position::set_state(StateInfo* si) const {
 
-  st->key = st->materialKey = 0;
-  st->pawnKey = Zobrist::noPawns;
-  st->nonPawnMaterial[WHITE] = st->nonPawnMaterial[BLACK] = VALUE_ZERO;
-  st->checkersBB = attackers_to(square<KING>(sideToMove)) & pieces(~sideToMove);
+  si->key = si->materialKey = 0;
+  si->pawnKey = Zobrist::noPawns;
+  si->nonPawnMaterial[WHITE] = si->nonPawnMaterial[BLACK] = VALUE_ZERO;
+  si->checkersBB = attackers_to(square<KING>(sideToMove)) & pieces(~sideToMove);
 
-  set_check_info();
+  set_check_info(si);
 
   for (Bitboard b = pieces(); b; )
   {
       Square s = pop_lsb(b);
       Piece pc = piece_on(s);
-      st->key ^= Zobrist::psq[pc][s];
+      si->key ^= Zobrist::psq[pc][s];
 
       if (type_of(pc) == PAWN)
-          st->pawnKey ^= Zobrist::psq[pc][s];
+          si->pawnKey ^= Zobrist::psq[pc][s];
 
       else if (type_of(pc) != KING)
-          st->nonPawnMaterial[color_of(pc)] += PieceValue[MG][pc];
+          si->nonPawnMaterial[color_of(pc)] += PieceValue[MG][pc];
   }
 
-  if (st->epSquare != SQ_NONE)
-      st->key ^= Zobrist::enpassant[file_of(st->epSquare)];
+  if (si->epSquare != SQ_NONE)
+      si->key ^= Zobrist::enpassant[file_of(si->epSquare)];
 
   if (sideToMove == BLACK)
-      st->key ^= Zobrist::side;
+      si->key ^= Zobrist::side;
 
-  st->key ^= Zobrist::castling[st->castlingRights];
+  si->key ^= Zobrist::castling[si->castlingRights];
 
   for (Piece pc : Pieces)
       for (int cnt = 0; cnt < pieceCount[pc]; ++cnt)
-          st->materialKey ^= Zobrist::psq[pc][cnt];
+          si->materialKey ^= Zobrist::psq[pc][cnt];
 }
 
 
@@ -568,7 +573,8 @@ bool Position::pseudo_legal(const Move m) const {
                         : MoveList<NON_EVASIONS>(*this).contains(m);
 
   // Is not a promotion, so promotion piece must be empty
-  assert(promotion_type(m) - KNIGHT == NO_PIECE_TYPE);
+  if (promotion_type(m) - KNIGHT != NO_PIECE_TYPE)
+      return false;
 
   // If the 'from' square is not occupied by a piece belonging to the side to
   // move, the move is obviously not legal.
@@ -753,7 +759,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) {
       else
           st->nonPawnMaterial[them] -= PieceValue[MG][captured];
 
-      if (Eval::useNNUE)
+      if (Eval::NNUE::useNNUE != Eval::NNUE::UseNNUEMode::False)
       {
           dp.dirty_num = 2;  // 1 piece moved, 1 piece captured
           dp.piece[1] = captured;
@@ -764,6 +770,9 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) {
       // Update board and piece lists
       remove_piece(capsq);
 
+      if (type_of(m) == EN_PASSANT)
+          board[capsq] = NO_PIECE;
+
       // Update material hash key and prefetch access to materialTable
       k ^= Zobrist::psq[captured][capsq];
       st->materialKey ^= Zobrist::psq[captured][pieceCount[captured]];
@@ -794,7 +803,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) {
   // Move the piece. The tricky Chess960 castling is handled earlier
   if (type_of(m) != CASTLING)
   {
-      if (Eval::useNNUE)
+      if (Eval::NNUE::useNNUE != Eval::NNUE::UseNNUEMode::False)
       {
           dp.piece[0] = pc;
           dp.from[0] = from;
@@ -825,7 +834,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) {
           remove_piece(to);
           put_piece(promotion, to);
 
-          if (Eval::useNNUE)
+          if (Eval::NNUE::useNNUE != Eval::NNUE::UseNNUEMode::False)
           {
               // Promoting pawn to SQ_NONE, promoted piece from SQ_NONE
               dp.to[0] = SQ_NONE;
@@ -864,7 +873,7 @@ void Position::do_move(Move m, StateInfo& newSt, bool givesCheck) {
   sideToMove = ~sideToMove;
 
   // Update king attacks used for fast check detection
-  set_check_info();
+  set_check_info(st);
 
   // Calculate the repetition info. It is the ply distance from the previous
   // occurrence of the same position, negative in the 3-fold case, or zero
@@ -963,7 +972,7 @@ void Position::do_castling(Color us, Square from, Square& to, Square& rfrom, Squ
   rto = relative_square(us, kingSide ? SQ_F1 : SQ_D1);
   to = relative_square(us, kingSide ? SQ_G1 : SQ_C1);
 
-  if (Do && Eval::useNNUE)
+  if (Do && Eval::NNUE::useNNUE != Eval::NNUE::UseNNUEMode::False)
   {
       auto& dp = st->dirtyPiece;
       dp.piece[0] = make_piece(us, KING);
@@ -997,6 +1006,7 @@ void Position::do_null_move(StateInfo& newSt) {
   newSt.previous = st;
   st = &newSt;
 
+  // Used by NNUE
   st->dirtyPiece.dirty_num = 0;
   st->dirtyPiece.piece[0] = NO_PIECE; // Avoid checks in UpdateAccumulator()
   st->accumulator.computed[WHITE] = false;
@@ -1016,7 +1026,7 @@ void Position::do_null_move(StateInfo& newSt) {
 
   sideToMove = ~sideToMove;
 
-  set_check_info();
+  set_check_info(st);
 
   st->repetition = 0;
 
@@ -1050,10 +1060,7 @@ Key Position::key_after(Move m) const {
   if (captured)
       k ^= Zobrist::psq[captured][to];
 
-  k ^= Zobrist::psq[pc][to] ^ Zobrist::psq[pc][from];
-
-  return (captured || type_of(pc) == PAWN)
-      ? k : adjust_key50<true>(k);
+  return k ^ Zobrist::psq[pc][to] ^ Zobrist::psq[pc][from];
 }
 
 
@@ -1061,7 +1068,7 @@ Key Position::key_after(Move m) const {
 /// SEE value of move is greater or equal to the given threshold. We'll use an
 /// algorithm similar to alpha-beta pruning with a null window.
 
-bool Position::see_ge(Move m, Bitboard& occupied, Value threshold) const {
+bool Position::see_ge(Move m, Value threshold) const {
 
   assert(is_ok(m));
 
@@ -1080,7 +1087,7 @@ bool Position::see_ge(Move m, Bitboard& occupied, Value threshold) const {
       return true;
 
   assert(color_of(piece_on(from)) == sideToMove);
-  occupied = pieces() ^ from ^ to; // xoring to is important for pinned piece logic
+  Bitboard occupied = pieces() ^ from ^ to;
   Color stm = sideToMove;
   Bitboard attackers = attackers_to(to, occupied);
   Bitboard stmAttackers, bb;
@@ -1098,12 +1105,10 @@ bool Position::see_ge(Move m, Bitboard& occupied, Value threshold) const {
       // Don't allow pinned pieces to attack as long as there are
       // pinners on their original square.
       if (pinners(~stm) & occupied)
-      {
           stmAttackers &= ~blockers_for_king(stm);
 
-          if (!stmAttackers)
-              break;
-      }
+      if (!stmAttackers)
+          break;
 
       res ^= 1;
 
@@ -1111,44 +1116,45 @@ bool Position::see_ge(Move m, Bitboard& occupied, Value threshold) const {
       // the bitboard 'attackers' any X-ray attackers behind it.
       if ((bb = stmAttackers & pieces(PAWN)))
       {
-          occupied ^= least_significant_square_bb(bb);
           if ((swap = PawnValueMg - swap) < res)
               break;
 
+          occupied ^= least_significant_square_bb(bb);
           attackers |= attacks_bb<BISHOP>(to, occupied) & pieces(BISHOP, QUEEN);
       }
 
       else if ((bb = stmAttackers & pieces(KNIGHT)))
       {
-          occupied ^= least_significant_square_bb(bb);
           if ((swap = KnightValueMg - swap) < res)
               break;
+
+          occupied ^= least_significant_square_bb(bb);
       }
 
       else if ((bb = stmAttackers & pieces(BISHOP)))
       {
-          occupied ^= least_significant_square_bb(bb);
           if ((swap = BishopValueMg - swap) < res)
               break;
 
+          occupied ^= least_significant_square_bb(bb);
           attackers |= attacks_bb<BISHOP>(to, occupied) & pieces(BISHOP, QUEEN);
       }
 
       else if ((bb = stmAttackers & pieces(ROOK)))
       {
-          occupied ^= least_significant_square_bb(bb);
           if ((swap = RookValueMg - swap) < res)
               break;
 
+          occupied ^= least_significant_square_bb(bb);
           attackers |= attacks_bb<ROOK>(to, occupied) & pieces(ROOK, QUEEN);
       }
 
       else if ((bb = stmAttackers & pieces(QUEEN)))
       {
-          occupied ^= least_significant_square_bb(bb);
           if ((swap = QueenValueMg - swap) < res)
               break;
 
+          occupied ^= least_significant_square_bb(bb);
           attackers |=  (attacks_bb<BISHOP>(to, occupied) & pieces(BISHOP, QUEEN))
                       | (attacks_bb<ROOK  >(to, occupied) & pieces(ROOK  , QUEEN));
       }
@@ -1162,11 +1168,6 @@ bool Position::see_ge(Move m, Bitboard& occupied, Value threshold) const {
   return bool(res);
 }
 
-bool Position::see_ge(Move m, Value threshold) const {
-    Bitboard occupied;
-    return see_ge(m, occupied, threshold);
-}
-
 
 /// Position::is_draw() tests whether the position is drawn by 50-move rule
 /// or by repetition. It does not detect stalemates.
@@ -1182,6 +1183,22 @@ bool Position::is_draw(int ply) const {
 }
 
 
+/// Position::is_fifty_move_draw() returns true if a game can be claimed
+/// by a fifty-move draw rule.
+
+bool Position::is_fifty_move_draw() const {
+
+  return (st->rule50 > 99 && (!checkers() || MoveList<LEGAL>(*this).size()));
+}
+
+
+/// Position::is_three_fold_repetition() returns true if there is 3-fold repetition.
+bool Position::is_three_fold_repetition() const {
+
+  return st->repetition < 0;
+}
+
+
 // Position::has_repeated() tests whether there has been at least one repetition
 // of positions since the last capture or pawn move.
 
@@ -1323,6 +1340,12 @@ bool Position::pos_is_ok() const {
           if (p1 != p2 && (pieces(p1) & pieces(p2)))
               assert(0 && "pos_is_ok: Bitboards");
 
+  StateInfo si = *st;
+  ASSERT_ALIGNED(&si, Eval::NNUE::CacheLineSize);
+
+  set_state(&si);
+  if (std::memcmp(&si, st, sizeof(StateInfo)))
+      assert(0 && "pos_is_ok: State");
 
   for (Piece pc : Pieces)
       if (   pieceCount[pc] != popcount(pieces(color_of(pc), type_of(pc)))
@@ -1344,4 +1367,18 @@ bool Position::pos_is_ok() const {
   return true;
 }
 
+// Add a function that directly unpacks for speed. It's pretty tough.
+// Write it by combining packer::unpack() and Position::set().
+// If there is a problem with the passed phase and there is an error, non-zero is returned.
+int Position::set_from_packed_sfen(const Tools::PackedSfen& sfen , StateInfo* si, Thread* th, bool frc)
+{
+  return Tools::set_from_packed_sfen(*this, sfen, si, th, frc);
+}
+
+// Get the packed sfen. Returns to the buffer specified in the argument.
+void Position::sfen_pack(Tools::PackedSfen& sfen, bool resetCastlingRights)
+{
+  sfen = Tools::sfen_pack(*this, resetCastlingRights);
+}
+
 } // namespace Stockfish

src/position.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -31,6 +31,9 @@
 
 #include "nnue/nnue_accumulator.h"
 
+#include "tools/packed_sfen.h"
+#include "tools/sfen_packer.h"
+
 namespace Stockfish {
 
 /// StateInfo struct stores information needed to restore a Position object to
@@ -68,7 +71,7 @@ struct StateInfo {
 /// start position to the position just before the search starts). Needed by
 /// 'draw by repetition' detection. Use a std::deque because pointers to
 /// elements are not invalidated upon list resizing.
-using StateListPtr = std::unique_ptr<std::deque<StateInfo>>;
+typedef std::unique_ptr<std::deque<StateInfo>> StateListPtr;
 
 
 /// Position class stores information regarding the board representation as
@@ -92,9 +95,10 @@ public:
 
   // Position representation
   Bitboard pieces(PieceType pt) const;
-  template<typename ...PieceTypes> Bitboard pieces(PieceType pt, PieceTypes... pts) const;
+  Bitboard pieces(PieceType pt1, PieceType pt2) const;
   Bitboard pieces(Color c) const;
-  template<typename ...PieceTypes> Bitboard pieces(Color c, PieceTypes... pts) const;
+  Bitboard pieces(Color c, PieceType pt) const;
+  Bitboard pieces(Color c, PieceType pt1, PieceType pt2) const;
   Piece piece_on(Square s) const;
   Square ep_square() const;
   bool empty(Square s) const;
@@ -125,7 +129,7 @@ public:
   bool legal(Move m) const;
   bool pseudo_legal(const Move m) const;
   bool capture(Move m) const;
-  bool capture_stage(Move m) const;
+  bool capture_or_promotion(Move m) const;
   bool gives_check(Move m) const;
   Piece moved_piece(Move m) const;
   Piece captured_piece() const;
@@ -144,7 +148,6 @@ public:
 
   // Static Exchange Evaluation
   bool see_ge(Move m, Value threshold = VALUE_ZERO) const;
-  bool see_ge(Move m, Bitboard& occupied, Value threshold = VALUE_ZERO) const;
 
   // Accessing hash keys
   Key key() const;
@@ -158,11 +161,12 @@ public:
   bool is_chess960() const;
   Thread* this_thread() const;
   bool is_draw(int ply) const;
+  bool is_fifty_move_draw() const;
+  bool is_three_fold_repetition() const;
   bool has_game_cycle(int ply) const;
   bool has_repeated() const;
   int rule50_count() const;
   Score psq_score() const;
-  Value psq_eg_stm() const;
   Value non_pawn_material(Color c) const;
   Value non_pawn_material() const;
 
@@ -173,21 +177,41 @@ public:
   // Used by NNUE
   StateInfo* state() const;
 
+  // --sfenization helper
+
+  friend int Tools::set_from_packed_sfen(Position& pos, const Tools::PackedSfen& sfen, StateInfo* si, Thread* th, bool frc);
+
+  // Get the packed sfen. Returns to the buffer specified in the argument.
+  // Do not include gamePly in pack.
+  void sfen_pack(Tools::PackedSfen& sfen, bool resetCastlingRights);
+
+  // It is slow to go through sfen, so I made a function to set packed sfen directly.
+  // Equivalent to pos.set(sfen_unpack(data),si,th);.
+  // If there is a problem with the passed phase and there is an error, non-zero is returned.
+  // PackedSfen does not include gamePly so it cannot be restored. If you want to set it, specify it with an argument.
+  int set_from_packed_sfen(const Tools::PackedSfen& sfen, StateInfo* si, Thread* th, bool frc);
+
+  void clear() { std::memset(this, 0, sizeof(Position)); }
+
+  // Give the board, hand piece, and turn, and return the sfen.
+  //static std::string sfen_from_rawdata(Piece board[81], Hand hands[2], Color turn, int gamePly);
+
+  // Returns the position of the ball on the c side.
+  Square king_square(Color c) const { return lsb(pieces(c, KING)); }
+  
   void put_piece(Piece pc, Square s);
   void remove_piece(Square s);
 
 private:
   // Initialization helpers (used while setting up a position)
   void set_castling_right(Color c, Square rfrom);
-  void set_state() const;
-  void set_check_info() const;
+  void set_state(StateInfo* si) const;
+  void set_check_info(StateInfo* si) const;
 
   // Other helpers
   void move_piece(Square from, Square to);
   template<bool Do>
   void do_castling(Color us, Square from, Square& to, Square& rfrom, Square& rto);
-  template<bool AfterMove>
-  Key adjust_key50(Key k) const;
 
   // Data members
   Piece board[SQUARE_NB];
@@ -205,7 +229,7 @@ private:
   bool chess960;
 };
 
-std::ostream& operator<<(std::ostream& os, const Position& pos);
+extern std::ostream& operator<<(std::ostream& os, const Position& pos);
 
 inline Color Position::side_to_move() const {
   return sideToMove;
@@ -228,18 +252,20 @@ inline Bitboard Position::pieces(PieceType pt = ALL_PIECES) const {
   return byTypeBB[pt];
 }
 
-template<typename ...PieceTypes>
-inline Bitboard Position::pieces(PieceType pt, PieceTypes... pts) const {
-  return pieces(pt) | pieces(pts...);
+inline Bitboard Position::pieces(PieceType pt1, PieceType pt2) const {
+  return pieces(pt1) | pieces(pt2);
 }
 
 inline Bitboard Position::pieces(Color c) const {
   return byColorBB[c];
 }
 
-template<typename ...PieceTypes>
-inline Bitboard Position::pieces(Color c, PieceTypes... pts) const {
-  return pieces(c) & pieces(pts...);
+inline Bitboard Position::pieces(Color c, PieceType pt) const {
+  return pieces(c) & pieces(pt);
+}
+
+inline Bitboard Position::pieces(Color c, PieceType pt1, PieceType pt2) const {
+  return pieces(c) & (pieces(pt1) | pieces(pt2));
 }
 
 template<PieceType Pt> inline int Position::count(Color c) const {
@@ -328,14 +354,8 @@ inline int Position::pawns_on_same_color_squares(Color c, Square s) const {
 }
 
 inline Key Position::key() const {
-  return adjust_key50<false>(st->key);
-}
-
-template<bool AfterMove>
-inline Key Position::adjust_key50(Key k) const
-{
-  return st->rule50 < 14 - AfterMove
-      ? k : k ^ make_key((st->rule50 - (14 - AfterMove)) / 8);
+  return st->rule50 < 14 ? st->key
+                         : st->key ^ make_key((st->rule50 - 14) / 8);
 }
 
 inline Key Position::pawn_key() const {
@@ -350,10 +370,6 @@ inline Score Position::psq_score() const {
   return psq;
 }
 
-inline Value Position::psq_eg_stm() const {
-  return (sideToMove == WHITE ? 1 : -1) * eg_value(psq);
-}
-
 inline Value Position::non_pawn_material(Color c) const {
   return st->nonPawnMaterial[c];
 }
@@ -380,18 +396,15 @@ inline bool Position::is_chess960() const {
   return chess960;
 }
 
-inline bool Position::capture(Move m) const {
+inline bool Position::capture_or_promotion(Move m) const {
   assert(is_ok(m));
-  return     (!empty(to_sq(m)) && type_of(m) != CASTLING)
-          ||  type_of(m) == EN_PASSANT;
+  return type_of(m) != NORMAL ? type_of(m) != CASTLING : !empty(to_sq(m));
 }
 
-// returns true if a move is generated from the capture stage
-// having also queen promotions covered, i.e. consistency with the capture stage move generation
-// is needed to avoid the generation of duplicate moves.
-inline bool Position::capture_stage(Move m) const {
+inline bool Position::capture(Move m) const {
   assert(is_ok(m));
-  return  capture(m) || promotion_type(m) == QUEEN;
+  // Castling is encoded as "king captures rook"
+  return (!empty(to_sq(m)) && type_of(m) != CASTLING) || type_of(m) == EN_PASSANT;
 }
 
 inline Piece Position::captured_piece() const {
@@ -445,6 +458,8 @@ inline StateInfo* Position::state() const {
   return st;
 }
 
+static const char* const StartFEN = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1";
+
 } // namespace Stockfish
 
 #endif // #ifndef POSITION_H_INCLUDED

src/psqt.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/psqt.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -30,7 +30,7 @@ namespace Stockfish::PSQT
 extern Score psq[PIECE_NB][SQUARE_NB];
 
 // Fill psqt array from a set of internally linked parameters
-void init();
+extern void init();
 
 } // namespace Stockfish::PSQT
 

src/search.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
 #include "misc.h"
 #include "movepick.h"
 #include "types.h"
+#include "uci.h"
 
 namespace Stockfish {
 
@@ -31,6 +32,10 @@ class Position;
 
 namespace Search {
 
+/// Threshold used for countermoves based pruning
+constexpr int CounterMovePruneThreshold = 0;
+
+extern bool prune_at_shallow_depth;
 
 /// Stack struct keeps track of the information we need to remember from nodes
 /// shallower and deeper in the tree during the search. Each search thread has
@@ -71,16 +76,13 @@ struct RootMove {
   Value score = -VALUE_INFINITE;
   Value previousScore = -VALUE_INFINITE;
   Value averageScore = -VALUE_INFINITE;
-  Value uciScore = -VALUE_INFINITE;
-  bool scoreLowerbound = false;
-  bool scoreUpperbound = false;
   int selDepth = 0;
   int tbRank = 0;
   Value tbScore;
   std::vector<Move> pv;
 };
 
-using RootMoves = std::vector<RootMove>;
+typedef std::vector<RootMove> RootMoves;
 
 
 /// LimitsType struct stores information sent by GUI about available time to
@@ -92,6 +94,7 @@ struct LimitsType {
     time[WHITE] = time[BLACK] = inc[WHITE] = inc[BLACK] = npmsec = movetime = TimePoint(0);
     movestogo = depth = mate = perft = infinite = 0;
     nodes = 0;
+    silent = false;
   }
 
   bool use_time_management() const {
@@ -102,6 +105,9 @@ struct LimitsType {
   TimePoint time[COLOR_NB], inc[COLOR_NB], npmsec, movetime, startTime;
   int movestogo, depth, mate, perft, infinite;
   int64_t nodes;
+  // Silent mode that does not output to the screen (for continuous self-play in process)
+  // Do not output PV at this time.
+  bool silent;
 };
 
 extern LimitsType Limits;
@@ -109,7 +115,35 @@ extern LimitsType Limits;
 void init();
 void clear();
 
-} // namespace Search
+// A pair of reader and evaluation value. Returned by Tools::search(),Tools::qsearch().
+using ValueAndPV = std::pair<Value, std::vector<Move>>;
+
+ValueAndPV qsearch(Position& pos);
+ValueAndPV search(Position& pos, int depth_, size_t multiPV = 1, uint64_t nodesLimit = 0);
+
+namespace MCTS {
+
+  struct MctsContinuation {
+    std::uint64_t numVisits;
+    Value value;
+    float actionValue;
+    std::vector<Move> pv;
+  };
+
+  ValueAndPV search_mcts(
+    Position& pos,
+    std::uint64_t nodes,
+    Depth leafDepth,
+    float explorationFactor);
+
+  std::vector<MctsContinuation> search_mcts_multipv(
+    Position& pos,
+    std::uint64_t numPlayouts,
+    Depth leafDepth,
+    float explorationFactor);
+}
+
+}
 
 } // namespace Stockfish
 

src/simd.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -153,7 +153,7 @@ namespace Stockfish::Simd {
       asm(
         "vpdpbusd %[b0], %[a0], %[acc]\n\t"
         "vpdpbusd %[b1], %[a1], %[acc]\n\t"
-        : [acc]"+&v"(acc)
+        : [acc]"+v"(acc)
         : [a0]"v"(a0), [b0]"vm"(b0), [a1]"v"(a1), [b1]"vm"(b1)
       );
 #   else
@@ -165,19 +165,18 @@ namespace Stockfish::Simd {
       __m512i tmp0 = _mm512_maddubs_epi16(a0, b0);
       __m512i tmp1 = _mm512_maddubs_epi16(a1, b1);
       asm(
+          "vpaddsw     %[tmp0], %[tmp1], %[tmp0]\n\t"
           "vpmaddwd    %[tmp0], %[ones], %[tmp0]\n\t"
-          "vpmaddwd    %[tmp1], %[ones], %[tmp1]\n\t"
-          "vpaddd      %[tmp0], %[tmp1], %[tmp0]\n\t"
           "vpaddd      %[acc], %[tmp0], %[acc]\n\t"
-          : [acc]"+v"(acc), [tmp0]"+&v"(tmp0), [tmp1]"+&v"(tmp1)
-          : [ones]"v"(_mm512_set1_epi16(1))
+          : [acc]"+v"(acc), [tmp0]"+&v"(tmp0)
+          : [tmp1]"v"(tmp1), [ones]"v"(_mm512_set1_epi16(1))
       );
 #   else
       __m512i product0 = _mm512_maddubs_epi16(a0, b0);
       __m512i product1 = _mm512_maddubs_epi16(a1, b1);
+      product0 = _mm512_adds_epi16(product0, product1);
       product0 = _mm512_madd_epi16(product0, _mm512_set1_epi16(1));
-      product1 = _mm512_madd_epi16(product1, _mm512_set1_epi16(1));
-      acc = _mm512_add_epi32(acc, _mm512_add_epi32(product0, product1));
+      acc = _mm512_add_epi32(acc, product0);
 #   endif
 # endif
     }
@@ -250,7 +249,7 @@ namespace Stockfish::Simd {
       asm(
         VNNI_PREFIX "vpdpbusd %[b0], %[a0], %[acc]\n\t"
         VNNI_PREFIX "vpdpbusd %[b1], %[a1], %[acc]\n\t"
-        : [acc]"+&v"(acc)
+        : [acc]"+v"(acc)
         : [a0]"v"(a0), [b0]"vm"(b0), [a1]"v"(a1), [b1]"vm"(b1)
       );
 #   else
@@ -262,19 +261,18 @@ namespace Stockfish::Simd {
       __m256i tmp0 = _mm256_maddubs_epi16(a0, b0);
       __m256i tmp1 = _mm256_maddubs_epi16(a1, b1);
       asm(
+          "vpaddsw     %[tmp0], %[tmp1], %[tmp0]\n\t"
           "vpmaddwd    %[tmp0], %[ones], %[tmp0]\n\t"
-          "vpmaddwd    %[tmp1], %[ones], %[tmp1]\n\t"
-          "vpaddd      %[tmp0], %[tmp1], %[tmp0]\n\t"
           "vpaddd      %[acc], %[tmp0], %[acc]\n\t"
-          : [acc]"+v"(acc), [tmp0]"+&v"(tmp0), [tmp1]"+&v"(tmp1)
-          : [ones]"v"(_mm256_set1_epi16(1))
+          : [acc]"+v"(acc), [tmp0]"+&v"(tmp0)
+          : [tmp1]"v"(tmp1), [ones]"v"(_mm256_set1_epi16(1))
       );
 #   else
       __m256i product0 = _mm256_maddubs_epi16(a0, b0);
       __m256i product1 = _mm256_maddubs_epi16(a1, b1);
+      product0 = _mm256_adds_epi16(product0, product1);
       product0 = _mm256_madd_epi16(product0, _mm256_set1_epi16(1));
-      product1 = _mm256_madd_epi16(product1, _mm256_set1_epi16(1));
-      acc = _mm256_add_epi32(acc, _mm256_add_epi32(product0, product1));
+      acc = _mm256_add_epi32(acc, product0);
 #   endif
 # endif
     }
@@ -328,37 +326,23 @@ namespace Stockfish::Simd {
       __m128i tmp0 = _mm_maddubs_epi16(a0, b0);
       __m128i tmp1 = _mm_maddubs_epi16(a1, b1);
       asm(
+          "paddsw     %[tmp1], %[tmp0]\n\t"
           "pmaddwd    %[ones], %[tmp0]\n\t"
-          "pmaddwd    %[ones], %[tmp1]\n\t"
-          "paddd      %[tmp1], %[tmp0]\n\t"
           "paddd      %[tmp0], %[acc]\n\t"
-          : [acc]"+v"(acc), [tmp0]"+&v"(tmp0), [tmp1]"+&v"(tmp1)
-          : [ones]"v"(_mm_set1_epi16(1))
+          : [acc]"+v"(acc), [tmp0]"+&v"(tmp0)
+          : [tmp1]"v"(tmp1), [ones]"v"(_mm_set1_epi16(1))
       );
 #   else
       __m128i product0 = _mm_maddubs_epi16(a0, b0);
       __m128i product1 = _mm_maddubs_epi16(a1, b1);
+      product0 = _mm_adds_epi16(product0, product1);
       product0 = _mm_madd_epi16(product0, _mm_set1_epi16(1));
-      product1 = _mm_madd_epi16(product1, _mm_set1_epi16(1));
-      acc = _mm_add_epi32(acc, _mm_add_epi32(product0, product1));
+      acc = _mm_add_epi32(acc, product0);
 #   endif
     }
 
 #endif
 
-#if defined (USE_NEON_DOTPROD)
-
-    [[maybe_unused]] static void dotprod_m128_add_dpbusd_epi32x2(
-        int32x4_t& acc,
-        int8x16_t a0, int8x16_t b0,
-        int8x16_t a1, int8x16_t b1) {
-
-        acc = vdotq_s32(acc, a0, b0);
-        acc = vdotq_s32(acc, a1, b1);
-    }
-
-#endif
-
 #if defined (USE_NEON)
 
     [[maybe_unused]] static int neon_m128_reduce_add_epi32(int32x4_t s) {

src/syzygy/tbprobe.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -24,10 +24,9 @@
 #include <fstream>
 #include <iostream>
 #include <list>
-#include <mutex>
 #include <sstream>
-#include <string_view>
 #include <type_traits>
+#include <mutex>
 
 #include "../bitboard.h"
 #include "../movegen.h"
@@ -53,14 +52,13 @@
 
 using namespace Stockfish::Tablebases;
 
-int Stockfish::Tablebases::MaxCardinality;
+int Stockfish::Tablebases::MaxCardinality = 0;
 
 namespace Stockfish {
 
 namespace {
 
 constexpr int TBPIECES = 7; // Max number of supported pieces
-constexpr int MAX_DTZ = 1 << 18; // Max DTZ supported, large enough to deal with the syzygy TB limit.
 
 enum { BigEndian, LittleEndian };
 enum TBType { WDL, DTZ }; // Used as template parameter
@@ -71,7 +69,7 @@ enum TBFlag { STM = 1, Mapped = 2, WinPlies = 4, LossPlies = 8, Wide = 16, Singl
 inline WDLScore operator-(WDLScore d) { return WDLScore(-int(d)); }
 inline Square operator^(Square s, int i) { return Square(int(s) ^ i); }
 
-constexpr std::string_view PieceToChar = " PNBRQK  pnbrqk";
+const std::string PieceToChar = " PNBRQK  pnbrqk";
 
 int MapPawns[SQUARE_NB];
 int MapB1H1H7[SQUARE_NB];
@@ -142,7 +140,7 @@ struct SparseEntry {
 
 static_assert(sizeof(SparseEntry) == 6, "SparseEntry must be 6 bytes");
 
-using Sym = uint16_t; // Huffman symbol
+typedef uint16_t Sym; // Huffman symbol
 
 struct LR {
     enum Side { Left, Right };
@@ -200,10 +198,13 @@ public:
         }
     }
 
-    // Memory map the file and check it.
+    // Memory map the file and check it. File should be already open and will be
+    // closed after mapping.
     uint8_t* map(void** baseAddress, uint64_t* mapping, TBType type) {
-        if (is_open())
-            close(); // Need to re-open to get native file descriptor
+
+        assert(is_open());
+
+        close(); // Need to re-open to get native file descriptor
 
 #ifndef _WIN32
         struct stat statbuf;
@@ -234,7 +235,7 @@ public:
         }
 #else
         // Note FILE_FLAG_RANDOM_ACCESS is only a hint to Windows and as such may get ignored.
-        HANDLE fd = CreateFileA(fname.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr,
+        HANDLE fd = CreateFile(fname.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr,
                                OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, nullptr);
 
         if (fd == INVALID_HANDLE_VALUE)
@@ -327,7 +328,7 @@ struct PairsData {
 // first access, when the corresponding file is memory mapped.
 template<TBType Type>
 struct TBTable {
-    using Ret = typename std::conditional<Type == WDL, WDLScore, int>::type;
+    typedef typename std::conditional<Type == WDL, WDLScore, int>::type Ret;
 
     static constexpr int Sides = Type == WDL ? 2 : 1;
 
@@ -1512,7 +1513,7 @@ int Tablebases::probe_dtz(Position& pos, ProbeState* result) {
 // A return value false indicates that not all probes were successful.
 bool Tablebases::root_probe(Position& pos, Search::RootMoves& rootMoves) {
 
-    ProbeState result = OK;
+    ProbeState result;
     StateInfo st;
 
     // Obtain 50-move counter for the root position
@@ -1521,7 +1522,7 @@ bool Tablebases::root_probe(Position& pos, Search::RootMoves& rootMoves) {
     // Check whether a position was repeated since the last zeroing move.
     bool rep = pos.has_repeated();
 
-    int dtz, bound = Options["Syzygy50MoveRule"] ? (MAX_DTZ - 100) : 1;
+    int dtz, bound = Options["Syzygy50MoveRule"] ? 900 : 1;
 
     // Probe and rank each move
     for (auto& m : rootMoves)
@@ -1564,8 +1565,8 @@ bool Tablebases::root_probe(Position& pos, Search::RootMoves& rootMoves) {
 
         // Better moves are ranked higher. Certain wins are ranked equally.
         // Losing moves are ranked equally unless a 50-move draw is in sight.
-        int r =  dtz > 0 ? (dtz + cnt50 <= 99 && !rep ? MAX_DTZ : MAX_DTZ - (dtz + cnt50))
-               : dtz < 0 ? (-dtz * 2 + cnt50 < 100 ? -MAX_DTZ : -MAX_DTZ + (-dtz + cnt50))
+        int r =  dtz > 0 ? (dtz + cnt50 <= 99 && !rep ? 1000 : 1000 - (dtz + cnt50))
+               : dtz < 0 ? (-dtz * 2 + cnt50 < 100 ? -1000 : -1000 + (-dtz + cnt50))
                : 0;
         m.tbRank = r;
 
@@ -1573,9 +1574,9 @@ bool Tablebases::root_probe(Position& pos, Search::RootMoves& rootMoves) {
         // 1 cp to cursed wins and let it grow to 49 cp as the positions gets
         // closer to a real win.
         m.tbScore =  r >= bound ? VALUE_MATE - MAX_PLY - 1
-                   : r >  0     ? Value((std::max( 3, r - (MAX_DTZ - 200)) * int(PawnValueEg)) / 200)
+                   : r >  0     ? Value((std::max( 3, r - 800) * int(PawnValueEg)) / 200)
                    : r == 0     ? VALUE_DRAW
-                   : r > -bound ? Value((std::min(-3, r + (MAX_DTZ - 200)) * int(PawnValueEg)) / 200)
+                   : r > -bound ? Value((std::min(-3, r + 800) * int(PawnValueEg)) / 200)
                    :             -VALUE_MATE + MAX_PLY + 1;
     }
 
@@ -1589,9 +1590,9 @@ bool Tablebases::root_probe(Position& pos, Search::RootMoves& rootMoves) {
 // A return value false indicates that not all probes were successful.
 bool Tablebases::root_probe_wdl(Position& pos, Search::RootMoves& rootMoves) {
 
-    static const int WDL_to_rank[] = { -MAX_DTZ, -MAX_DTZ + 101, 0, MAX_DTZ - 101, MAX_DTZ };
+    static const int WDL_to_rank[] = { -1000, -899, 0, 899, 1000 };
 
-    ProbeState result = OK;
+    ProbeState result;
     StateInfo st;
     WDLScore wdl;
 

src/syzygy/tbprobe.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -31,6 +31,8 @@ enum WDLScore {
     WDLDraw        =  0, // Draw
     WDLCursedWin   =  1, // Win, but draw under 50-move rule
     WDLWin         =  2, // Win
+
+    WDLScoreNone  = -1000
 };
 
 // Possible states after a probing operation

src/thread.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -34,9 +34,10 @@ ThreadPool Threads; // Global object
 /// Thread constructor launches the thread and waits until it goes to sleep
 /// in idle_loop(). Note that 'searching' and 'exit' should be already set.
 
-Thread::Thread(size_t n) : idx(n), stdThread(&Thread::idle_loop, this) {
+Thread::Thread(size_t n) : idx(n), stdThread(&Thread::idle_loop, this), maxNodes(0) {
 
   wait_for_search_finished();
+  wait_for_worker_finished();
 }
 
 
@@ -60,21 +61,33 @@ void Thread::clear() {
   counterMoves.fill(MOVE_NONE);
   mainHistory.fill(0);
   captureHistory.fill(0);
-
+  previousDepth = 0;
+  
   for (bool inCheck : { false, true })
       for (StatsType c : { NoCaptures, Captures })
+      {
           for (auto& to : continuationHistory[inCheck][c])
-              for (auto& h : to)
-                  h->fill(-71);
+                for (auto& h : to)
+                      h->fill(-71);
+          continuationHistory[inCheck][c][NO_PIECE][0]->fill(Search::CounterMovePruneThreshold - 1);
+      }
 }
 
 
 /// Thread::start_searching() wakes up the thread that will start the search
 
 void Thread::start_searching() {
-  mutex.lock();
+
+  std::lock_guard<std::mutex> lk(mutex);
+  searching = true;
+  cv.notify_one(); // Wake up the thread in idle_loop()
+}
+
+void Thread::execute_with_worker(std::function<void(Thread&)> t)
+{
+  std::lock_guard<std::mutex> lk(mutex);
+  worker = std::move(t);
   searching = true;
-  mutex.unlock(); // Unlock before notifying saves a few CPU-cycles
   cv.notify_one(); // Wake up the thread in idle_loop()
 }
 
@@ -89,6 +102,12 @@ void Thread::wait_for_search_finished() {
 }
 
 
+void Thread::wait_for_worker_finished() {
+
+  std::unique_lock<std::mutex> lk(mutex);
+  cv.wait(lk, [&]{ return !searching; });
+}
+
 /// Thread::idle_loop() is where the thread is parked, blocked on the
 /// condition variable, when it has no work to do.
 
@@ -106,15 +125,25 @@ void Thread::idle_loop() {
   {
       std::unique_lock<std::mutex> lk(mutex);
       searching = false;
+      worker = nullptr;
       cv.notify_one(); // Wake up anyone waiting for search finished
       cv.wait(lk, [&]{ return searching; });
 
       if (exit)
           return;
 
+      auto wrk = std::move(worker);
+
       lk.unlock();
 
-      search();
+      if (wrk)
+      {
+        wrk(*this);
+      }
+      else
+      {
+        search();
+      }
   }
 }
 
@@ -124,20 +153,20 @@ void Thread::idle_loop() {
 
 void ThreadPool::set(size_t requested) {
 
-  if (threads.size() > 0)   // destroy any existing thread(s)
+  if (size() > 0)   // destroy any existing thread(s)
   {
       main()->wait_for_search_finished();
 
-      while (threads.size() > 0)
-          delete threads.back(), threads.pop_back();
+      while (size() > 0)
+          delete back(), pop_back();
   }
 
   if (requested > 0)   // create new thread(s)
   {
-      threads.push_back(new MainThread(0));
+      push_back(new MainThread(0));
 
-      while (threads.size() < requested)
-          threads.push_back(new Thread(threads.size()));
+      while (size() < requested)
+          push_back(new Thread(size()));
       clear();
 
       // Reallocate the hash with the new threadpool size
@@ -153,7 +182,7 @@ void ThreadPool::set(size_t requested) {
 
 void ThreadPool::clear() {
 
-  for (Thread* th : threads)
+  for (Thread* th : *this)
       th->clear();
 
   main()->callsCnt = 0;
@@ -162,6 +191,13 @@ void ThreadPool::clear() {
   main()->previousTimeReduction = 1.0;
 }
 
+void ThreadPool::execute_with_workers(const std::function<void(Thread&)>& worker)
+{
+  for(Thread* th : *this)
+  {
+    th->execute_with_worker(worker);
+  }
+}
 
 /// ThreadPool::start_thinking() wakes up main thread waiting in idle_loop() and
 /// returns immediately. Main thread will wake up other threads and start the search.
@@ -182,11 +218,8 @@ void ThreadPool::start_thinking(Position& pos, StateListPtr& states,
           || std::count(limits.searchmoves.begin(), limits.searchmoves.end(), m))
           rootMoves.emplace_back(m);
 
-  if (!rootMoves.empty())
-      Tablebases::rank_root_moves(pos, rootMoves);
-
   // After ownership transfer 'states' becomes empty, so if we stop the search
-  // and call 'go' again without setting a new position states.get() == nullptr.
+  // and call 'go' again without setting a new position states.get() == NULL.
   assert(states.get() || setupStates.get());
 
   if (states.get())
@@ -197,7 +230,7 @@ void ThreadPool::start_thinking(Position& pos, StateListPtr& states,
   // be deduced from a fen string, so set() clears them and they are set from
   // setupStates->back() later. The rootState is per thread, earlier states are shared
   // since they are read-only.
-  for (Thread* th : threads)
+  for (Thread* th : *this)
   {
       th->nodes = th->tbHits = th->nmpMinPly = th->bestMoveChanges = 0;
       th->rootDepth = th->completedDepth = 0;
@@ -211,23 +244,20 @@ void ThreadPool::start_thinking(Position& pos, StateListPtr& states,
 
 Thread* ThreadPool::get_best_thread() const {
 
-    Thread* bestThread = threads.front();
+    Thread* bestThread = front();
     std::map<Move, int64_t> votes;
     Value minScore = VALUE_NONE;
 
     // Find minimum score of all threads
-    for (Thread* th: threads)
+    for (Thread* th: *this)
         minScore = std::min(minScore, th->rootMoves[0].score);
 
     // Vote according to score and depth, and select the best thread
-    auto thread_value = [minScore](Thread* th) {
-            return (th->rootMoves[0].score - minScore + 14) * int(th->completedDepth);
-        };
+    for (Thread* th : *this)
+    {
+        votes[th->rootMoves[0].pv[0]] +=
+            (th->rootMoves[0].score - minScore + 14) * int(th->completedDepth);
 
-    for (Thread* th : threads)
-        votes[th->rootMoves[0].pv[0]] += thread_value(th);
-
-    for (Thread* th : threads)
         if (abs(bestThread->rootMoves[0].score) >= VALUE_TB_WIN_IN_MAX_PLY)
         {
             // Make sure we pick the shortest mate / TB conversion or stave off mate the longest
@@ -236,11 +266,9 @@ Thread* ThreadPool::get_best_thread() const {
         }
         else if (   th->rootMoves[0].score >= VALUE_TB_WIN_IN_MAX_PLY
                  || (   th->rootMoves[0].score > VALUE_TB_LOSS_IN_MAX_PLY
-                     && (   votes[th->rootMoves[0].pv[0]] > votes[bestThread->rootMoves[0].pv[0]]
-                         || (   votes[th->rootMoves[0].pv[0]] == votes[bestThread->rootMoves[0].pv[0]]
-                             &&   thread_value(th) * int(th->rootMoves[0].pv.size() > 2)
-                                > thread_value(bestThread) * int(bestThread->rootMoves[0].pv.size() > 2)))))
+                     && votes[th->rootMoves[0].pv[0]] > votes[bestThread->rootMoves[0].pv[0]]))
             bestThread = th;
+    }
 
     return bestThread;
 }
@@ -250,8 +278,8 @@ Thread* ThreadPool::get_best_thread() const {
 
 void ThreadPool::start_searching() {
 
-    for (Thread* th : threads)
-        if (th != threads.front())
+    for (Thread* th : *this)
+        if (th != front())
             th->start_searching();
 }
 
@@ -260,9 +288,16 @@ void ThreadPool::start_searching() {
 
 void ThreadPool::wait_for_search_finished() const {
 
-    for (Thread* th : threads)
-        if (th != threads.front())
+    for (Thread* th : *this)
+        if (th != front())
             th->wait_for_search_finished();
 }
 
+
+void ThreadPool::wait_for_workers_finished() const {
+
+    for (Thread* th : *this)
+        th->wait_for_worker_finished();
+}
+
 } // namespace Stockfish

src/thread.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
 #include <mutex>
 #include <thread>
 #include <vector>
+#include <functional>
 
 #include "material.h"
 #include "movepick.h"
@@ -39,40 +40,76 @@ namespace Stockfish {
 /// pointer to an entry its life time is unlimited and we don't have
 /// to care about someone changing the entry under our feet.
 
+namespace Detail {
+
+  template <typename T>
+  struct TypeIdentity {
+    using Type = T;
+  };
+
+}
+
 class Thread {
 
   std::mutex mutex;
   std::condition_variable cv;
   size_t idx;
   bool exit = false, searching = true; // Set before starting std::thread
+  std::function<void(Thread&)> worker;
+  std::function<void(Position&)> on_eval_callback;
   NativeThread stdThread;
 
 public:
   explicit Thread(size_t);
   virtual ~Thread();
   virtual void search();
+
+  // The function object to be executed is taken by value to remove
+  // the need for separate lvalue and rvalue overloads.
+  // The worker thread needs to have ownership of the task
+  // to be executed because otherwise there's no way to manage its lifetime.
+  virtual void execute_with_worker(std::function<void(Thread&)> t);
+
   void clear();
   void idle_loop();
   void start_searching();
   void wait_for_search_finished();
   size_t id() const { return idx; }
 
+  void wait_for_worker_finished();
+
+  template <typename FuncT>
+  void set_eval_callback(FuncT&& f) { on_eval_callback = std::forward<FuncT>(f); }
+
+  void clear_eval_callback() { on_eval_callback = nullptr; }
+
+  void on_eval() { if (on_eval_callback) on_eval_callback(rootPos); }
+
   Pawns::Table pawnsTable;
   Material::Table materialTable;
   size_t pvIdx, pvLast;
+  RunningAverage complexityAverage;
   std::atomic<uint64_t> nodes, tbHits, bestMoveChanges;
   int selDepth, nmpMinPly;
+  Color nmpColor;
   Value bestValue, optimism[COLOR_NB];
+  uint64_t maxNodes;
 
   Position rootPos;
   StateInfo rootState;
   Search::RootMoves rootMoves;
-  Depth rootDepth, completedDepth;
+  Depth rootDepth, completedDepth, depth, previousDepth;
   Value rootDelta;
   CounterMoveHistory counterMoves;
   ButterflyHistory mainHistory;
   CapturePieceToHistory captureHistory;
   ContinuationHistory continuationHistory[2][2];
+  Score trend;
+  int failedHighCnt;
+  bool rootInTB;
+  int Cardinality;
+  bool UseRule50;
+  Depth ProbeDepth;
 };
 
 
@@ -99,36 +136,84 @@ struct MainThread : public Thread {
 /// parking and, most importantly, launching a thread. All the access to threads
 /// is done through this class.
 
-struct ThreadPool {
+struct ThreadPool : public std::vector<Thread*> {
+
+  // Each thread gets its own copy of the `worker` function object.
+  // This means that each worker thread will have exclusive access
+  // to the state of the `worker` function object.
+  void execute_with_workers(const std::function<void(Thread&)>& worker);
+
+  template <typename IndexT, typename FuncT>
+  void for_each_index_with_workers(
+    IndexT begin,
+    typename Detail::TypeIdentity<IndexT>::Type end,
+    FuncT func)
+  {
+    // This value must outlive the function call.
+    // It's fairly safe if we make it static
+    // because for_each_index_with_workers
+    // is not reentrant nor thread safe.
+    static std::atomic<IndexT> i_atomic;
+    i_atomic.store(begin);
+
+    execute_with_workers(
+      [end, func](Thread& th) mutable {
+        for(;;) {
+          const auto i = i_atomic.fetch_add(1);
+          if (i >= end)
+            break;
+
+          func(th, i);
+        }
+      });
+  }
+
+  template <typename IndexT, typename FuncT>
+  void for_each_index_chunk_with_workers(
+    IndexT begin,
+    typename Detail::TypeIdentity<IndexT>::Type end,
+    FuncT func)
+  {
+    // This value must outlive the function call.
+    // It's fairly safe if we make it static
+    // because for_each_index_with_workers
+    // is not reentrant nor thread safe.
+    const IndexT size = end - begin;
+    const IndexT chunk_size = (size + this->size()) / this->size();
+
+    execute_with_workers(
+      [chunk_size, end, func](Thread& th) mutable {
+        const IndexT thread_id = th.id();
+        const IndexT offset = chunk_size * thread_id;
+        if (offset >= end)
+          return;
+
+        const IndexT count = offset + chunk_size > end ? end - offset : chunk_size;
+        func(th, offset, count);
+      });
+  }
 
   void start_thinking(Position&, StateListPtr&, const Search::LimitsType&, bool = false);
   void clear();
   void set(size_t);
 
-  MainThread* main()        const { return static_cast<MainThread*>(threads.front()); }
+  MainThread* main()        const { return static_cast<MainThread*>(front()); }
   uint64_t nodes_searched() const { return accumulate(&Thread::nodes); }
   uint64_t tb_hits()        const { return accumulate(&Thread::tbHits); }
   Thread* get_best_thread() const;
   void start_searching();
   void wait_for_search_finished() const;
+  void wait_for_workers_finished() const;
 
   std::atomic_bool stop, increaseDepth;
 
-  auto cbegin() const noexcept { return threads.cbegin(); }
-  auto begin() noexcept { return threads.begin(); }
-  auto end() noexcept { return threads.end(); }
-  auto cend() const noexcept { return threads.cend(); }
-  auto size() const noexcept { return threads.size(); }
-  auto empty() const noexcept { return threads.empty(); }
-
 private:
   StateListPtr setupStates;
-  std::vector<Thread*> threads;
 
   uint64_t accumulate(std::atomic<uint64_t> Thread::* member) const {
 
     uint64_t sum = 0;
-    for (Thread* th : threads)
+    for (Thread* th : *this)
         sum += (th->*member).load(std::memory_order_relaxed);
     return sum;
   }

src/thread_win32_osx.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -41,7 +41,7 @@ void* start_routine(void* ptr)
    P* p = reinterpret_cast<P*>(ptr);
    (p->first->*(p->second))(); // Call member function pointer
    delete p;
-   return nullptr;
+   return NULL;
 }
 
 class NativeThread {
@@ -56,7 +56,7 @@ public:
     pthread_attr_setstacksize(attr, TH_STACK_SIZE);
     pthread_create(&thread, attr, start_routine<T>, new P(obj, fun));
   }
-  void join() { pthread_join(thread, nullptr); }
+  void join() { pthread_join(thread, NULL); }
 };
 
 } // namespace Stockfish
@@ -65,7 +65,7 @@ public:
 
 namespace Stockfish {
 
-using NativeThread = std::thread;
+typedef std::thread NativeThread;
 
 } // namespace Stockfish
 

src/timeman.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -36,12 +36,6 @@ TimeManagement Time; // Our global time management object
 
 void TimeManagement::init(Search::LimitsType& limits, Color us, int ply) {
 
-  // if we have no time, no need to initialize TM, except for the start time,
-  // which is used by movetime.
-  startTime = limits.startTime;
-  if (limits.time[us] == 0)
-      return;
-
   TimePoint moveOverhead    = TimePoint(Options["Move Overhead"]);
   TimePoint slowMover       = TimePoint(Options["Slow Mover"]);
   TimePoint npmsec          = TimePoint(Options["nodestime"]);
@@ -65,6 +59,8 @@ void TimeManagement::init(Search::LimitsType& limits, Color us, int ply) {
       limits.npmsec = npmsec;
   }
 
+  startTime = limits.startTime;
+
   // Maximum move horizon of 50 moves
   int mtg = limits.movestogo ? std::min(limits.movestogo, 50) : 50;
 
@@ -84,7 +80,7 @@ void TimeManagement::init(Search::LimitsType& limits, Color us, int ply) {
   // game time for the current move, so also cap to 20% of available game time.
   if (limits.movestogo == 0)
   {
-      optScale = std::min(0.0120 + std::pow(ply + 3.0, 0.45) * 0.0039,
+      optScale = std::min(0.0084 + std::pow(ply + 3.0, 0.5) * 0.0042,
                            0.2 * limits.time[us] / double(timeLeft))
                  * optExtra;
       maxScale = std::min(7.0, 4.0 + ply / 12.0);

src/timeman.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/tools/convert.cpp

@@ -0,0 +1,815 @@
+#include "convert.h"
+
+#include "uci.h"
+#include "misc.h"
+#include "thread.h"
+#include "position.h"
+#include "tt.h"
+
+#include "extra/nnue_data_binpack_format.h"
+
+#include "nnue/evaluate_nnue.h"
+
+#include "syzygy/tbprobe.h"
+
+#include <sstream>
+#include <fstream>
+#include <unordered_set>
+#include <iomanip>
+#include <list>
+#include <cmath>    // std::exp(),std::pow(),std::log()
+#include <cstring>  // memcpy()
+#include <memory>
+#include <limits>
+#include <optional>
+#include <chrono>
+#include <random>
+#include <regex>
+#include <filesystem>
+
+using namespace std;
+namespace sys = std::filesystem;
+
+namespace Stockfish::Tools
+{
+    bool fen_is_ok(Position& pos, std::string input_fen) {
+        std::string pos_fen = pos.fen();
+        std::istringstream ss_input(input_fen);
+        std::istringstream ss_pos(pos_fen);
+
+        // example : "2r4r/4kpp1/nb1np3/p2p3p/B2P1BP1/PP6/4NPKP/2R1R3 w - h6 0 24"
+        //       --> "2r4r/4kpp1/nb1np3/p2p3p/B2P1BP1/PP6/4NPKP/2R1R3"
+        std::string str_input, str_pos;
+        ss_input >> str_input;
+        ss_pos >> str_pos;
+
+        // Only compare "Piece placement field" between input_fen and pos.fen().
+        return str_input == str_pos;
+    }
+
+    void convert_bin(
+        const vector<string>& filenames,
+        const string& output_file_name,
+        const int ply_minimum,
+        const int ply_maximum,
+        const int interpolate_eval,
+        const int src_score_min_value,
+        const int src_score_max_value,
+        const int dest_score_min_value,
+        const int dest_score_max_value,
+        const bool check_invalid_fen,
+        const bool check_illegal_move)
+    {
+        std::cout << "check_invalid_fen=" << check_invalid_fen << std::endl;
+        std::cout << "check_illegal_move=" << check_illegal_move << std::endl;
+
+        std::fstream fs;
+        uint64_t data_size = 0;
+        uint64_t filtered_size = 0;
+        uint64_t filtered_size_fen = 0;
+        uint64_t filtered_size_move = 0;
+        uint64_t filtered_size_ply = 0;
+        auto th = Threads.main();
+        auto& tpos = th->rootPos;
+        // convert plain rag to packed sfenvalue for Yaneura king
+        fs.open(output_file_name, ios::app | ios::binary);
+        StateListPtr states;
+        for (auto filename : filenames) {
+            std::cout << "convert " << filename << " ... ";
+            std::string line;
+            ifstream ifs;
+            ifs.open(filename);
+            PackedSfenValue p;
+            data_size = 0;
+            filtered_size = 0;
+            filtered_size_fen = 0;
+            filtered_size_move = 0;
+            filtered_size_ply = 0;
+            p.gamePly = 1; // Not included in apery format. Should be initialized
+            bool ignore_flag_fen = false;
+            bool ignore_flag_move = false;
+            bool ignore_flag_ply = false;
+            while (std::getline(ifs, line)) {
+                std::stringstream ss(line);
+                std::string token;
+                std::string value;
+                ss >> token;
+                if (token == "fen") {
+                    states = StateListPtr(new std::deque<StateInfo>(1)); // Drop old and create a new one
+                    std::string input_fen = line.substr(4);
+                    tpos.set(input_fen, false, &states->back(), Threads.main());
+                    if (check_invalid_fen && !fen_is_ok(tpos, input_fen)) {
+                        ignore_flag_fen = true;
+                        filtered_size_fen++;
+                    }
+                    else {
+                        tpos.sfen_pack(p.sfen, false);
+                    }
+                }
+                else if (token == "move") {
+                    ss >> value;
+                    Move move = UCI::to_move(tpos, value);
+                    if (check_illegal_move && move == MOVE_NONE) {
+                        ignore_flag_move = true;
+                        filtered_size_move++;
+                    }
+                    else {
+                        p.move = move;
+                    }
+                }
+                else if (token == "score") {
+                    double score;
+                    ss >> score;
+                    // Training Formula ?Issue #71 ?nodchip/Stockfish https://github.com/nodchip/Stockfish/issues/71
+                    // Normalize to [0.0, 1.0].
+                    score = (score - src_score_min_value) / (src_score_max_value - src_score_min_value);
+                    // Scale to [dest_score_min_value, dest_score_max_value].
+                    score = score * (dest_score_max_value - dest_score_min_value) + dest_score_min_value;
+                    p.score = std::clamp((int32_t)std::round(score), -(int32_t)VALUE_MATE, (int32_t)VALUE_MATE);
+                }
+                else if (token == "ply") {
+                    int temp;
+                    ss >> temp;
+                    if (temp < ply_minimum || temp > ply_maximum) {
+                        ignore_flag_ply = true;
+                        filtered_size_ply++;
+                    }
+                    p.gamePly = uint16_t(temp); // No cast here?
+                    if (interpolate_eval != 0) {
+                        p.score = min(3000, interpolate_eval * temp);
+                    }
+                }
+                else if (token == "result") {
+                    int temp;
+                    ss >> temp;
+                    p.game_result = int8_t(temp); // Do you need a cast here?
+                    if (interpolate_eval) {
+                        p.score = p.score * p.game_result;
+                    }
+                }
+                else if (token == "e") {
+                    if (!(ignore_flag_fen || ignore_flag_move || ignore_flag_ply)) {
+                        fs.write((char*)&p, sizeof(PackedSfenValue));
+                        data_size += 1;
+                        // debug
+                        // std::cout<<tpos<<std::endl;
+                        // std::cout<<p.score<<","<<int(p.gamePly)<<","<<int(p.game_result)<<std::endl;
+                    }
+                    else {
+                        filtered_size++;
+                    }
+                    ignore_flag_fen = false;
+                    ignore_flag_move = false;
+                    ignore_flag_ply = false;
+                }
+            }
+            std::cout << "done " << data_size << " parsed " << filtered_size << " is filtered"
+                << " (invalid fen:" << filtered_size_fen << ", illegal move:" << filtered_size_move << ", invalid ply:" << filtered_size_ply << ")" << std::endl;
+            ifs.close();
+        }
+        std::cout << "all done" << std::endl;
+        fs.close();
+    }
+
+    static inline void ltrim(std::string& s) {
+        s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) {
+            return !std::isspace(ch);
+            }));
+    }
+
+    static inline void rtrim(std::string& s) {
+        s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) {
+            return !std::isspace(ch);
+            }).base(), s.end());
+    }
+
+    static inline void trim(std::string& s) {
+        ltrim(s);
+        rtrim(s);
+    }
+
+    int parse_game_result_from_pgn_extract(std::string result) {
+        // White Win
+        if (result == "\"1-0\"") {
+            return 1;
+        }
+        // Black Win
+        else if (result == "\"0-1\"") {
+            return -1;
+        }
+        // Draw
+        else {
+            return 0;
+        }
+    }
+
+    // 0.25 -->  0.25 * PawnValueEg
+    // #-4  --> -mate_in(4)
+    // #3   -->  mate_in(3)
+    // -M4  --> -mate_in(4)
+    // +M3  -->  mate_in(3)
+    Value parse_score_from_pgn_extract(std::string eval, bool& success) {
+        success = true;
+
+        if (eval.substr(0, 1) == "#") {
+            if (eval.substr(1, 1) == "-") {
+                return -mate_in(stoi(eval.substr(2, eval.length() - 2)));
+            }
+            else {
+                return mate_in(stoi(eval.substr(1, eval.length() - 1)));
+            }
+        }
+        else if (eval.substr(0, 2) == "-M") {
+            //std::cout << "eval=" << eval << std::endl;
+            return -mate_in(stoi(eval.substr(2, eval.length() - 2)));
+        }
+        else if (eval.substr(0, 2) == "+M") {
+            //std::cout << "eval=" << eval << std::endl;
+            return mate_in(stoi(eval.substr(2, eval.length() - 2)));
+        }
+        else {
+            char* endptr;
+            double value = strtod(eval.c_str(), &endptr);
+
+            if (*endptr != '\0') {
+                success = false;
+                return VALUE_ZERO;
+            }
+            else {
+                return Value(value * static_cast<double>(PawnValueEg));
+            }
+        }
+    }
+
+    // for Debug
+    //#define DEBUG_CONVERT_BIN_FROM_PGN_EXTRACT
+
+    bool is_like_fen(std::string fen) {
+        int count_space = std::count(fen.cbegin(), fen.cend(), ' ');
+        int count_slash = std::count(fen.cbegin(), fen.cend(), '/');
+
+#if defined(DEBUG_CONVERT_BIN_FROM_PGN_EXTRACT)
+        //std::cout << "count_space=" << count_space << std::endl;
+        //std::cout << "count_slash=" << count_slash << std::endl;
+#endif
+
+        return count_space == 5 && count_slash == 7;
+    }
+
+    void convert_bin_from_pgn_extract(
+        const vector<string>& filenames,
+        const string& output_file_name,
+        const bool pgn_eval_side_to_move,
+        const bool convert_no_eval_fens_as_score_zero)
+    {
+        std::cout << "pgn_eval_side_to_move=" << pgn_eval_side_to_move << std::endl;
+        std::cout << "convert_no_eval_fens_as_score_zero=" << convert_no_eval_fens_as_score_zero << std::endl;
+
+        auto th = Threads.main();
+        auto& pos = th->rootPos;
+
+        std::fstream ofs;
+        ofs.open(output_file_name, ios::out | ios::binary);
+
+        int game_count = 0;
+        int fen_count = 0;
+
+        for (auto filename : filenames) {
+            std::cout << now_string() << " convert " << filename << std::endl;
+            ifstream ifs;
+            ifs.open(filename);
+
+            int game_result = 0;
+
+            std::string line;
+            while (std::getline(ifs, line)) {
+
+                if (line.empty()) {
+                    continue;
+                }
+
+                else if (line.substr(0, 1) == "[") {
+                    std::regex pattern_result(R"(\[Result (.+?)\])");
+                    std::smatch match;
+
+                    // example: [Result "1-0"]
+                    if (std::regex_search(line, match, pattern_result)) {
+                        game_result = parse_game_result_from_pgn_extract(match.str(1));
+#if defined(DEBUG_CONVERT_BIN_FROM_PGN_EXTRACT)
+                        std::cout << "game_result=" << game_result << std::endl;
+#endif
+                        game_count++;
+                        if (game_count % 10000 == 0) {
+                            std::cout << now_string() << " game_count=" << game_count << ", fen_count=" << fen_count << std::endl;
+                        }
+                    }
+
+                    continue;
+                }
+
+                else {
+                    int gamePly = 1;
+                    auto itr = line.cbegin();
+
+                    while (true) {
+                        gamePly++;
+
+                        PackedSfenValue psv;
+                        memset((char*)&psv, 0, sizeof(PackedSfenValue));
+
+                        // fen
+                        {
+                            bool fen_found = false;
+
+                            while (!fen_found) {
+                                std::regex pattern_bracket(R"(\{(.+?)\})");
+                                std::smatch match;
+                                if (!std::regex_search(itr, line.cend(), match, pattern_bracket)) {
+                                    break;
+                                }
+
+                                itr += match.position(0) + match.length(0) - 1;
+                                std::string str_fen = match.str(1);
+                                trim(str_fen);
+
+                                if (is_like_fen(str_fen)) {
+                                    fen_found = true;
+
+                                    StateInfo si;
+                                    pos.set(str_fen, false, &si, th);
+                                    pos.sfen_pack(psv.sfen, false);
+                                }
+
+#if defined(DEBUG_CONVERT_BIN_FROM_PGN_EXTRACT)
+                                std::cout << "str_fen=" << str_fen << std::endl;
+                                std::cout << "fen_found=" << fen_found << std::endl;
+#endif
+                            }
+
+                            if (!fen_found) {
+                                break;
+                            }
+                        }
+
+                        // move
+                        {
+                            std::regex pattern_move(R"(\}(.+?)\{)");
+                            std::smatch match;
+                            if (!std::regex_search(itr, line.cend(), match, pattern_move)) {
+                                break;
+                            }
+
+                            itr += match.position(0) + match.length(0) - 1;
+                            std::string str_move = match.str(1);
+                            trim(str_move);
+#if defined(DEBUG_CONVERT_BIN_FROM_PGN_EXTRACT)
+                            std::cout << "str_move=" << str_move << std::endl;
+#endif
+                            psv.move = UCI::to_move(pos, str_move);
+                        }
+
+                        // eval
+                        bool eval_found = false;
+                        {
+                            std::regex pattern_bracket(R"(\{(.+?)\})");
+                            std::smatch match;
+                            if (!std::regex_search(itr, line.cend(), match, pattern_bracket)) {
+                                break;
+                            }
+
+                            std::string str_eval_clk = match.str(1);
+                            trim(str_eval_clk);
+#if defined(DEBUG_CONVERT_BIN_FROM_PGN_EXTRACT)
+                            std::cout << "str_eval_clk=" << str_eval_clk << std::endl;
+#endif
+
+                            // example: { [%eval 0.25] [%clk 0:10:00] }
+                            // example: { [%eval #-4] [%clk 0:10:00] }
+                            // example: { [%eval #3] [%clk 0:10:00] }
+                            // example: { +0.71/22 1.2s }
+                            // example: { -M4/7 0.003s }
+                            // example: { M3/245 0.017s }
+                            // example: { +M1/245 0.010s, White mates }
+                            // example: { 0.60 }
+                            // example: { book }
+                            // example: { rnbqkb1r/pp3ppp/2p1pn2/3p4/2PP4/2N2N2/PP2PPPP/R1BQKB1R w KQkq - 0 5 }
+
+                            // Considering the absence of eval
+                            if (!is_like_fen(str_eval_clk)) {
+                                itr += match.position(0) + match.length(0) - 1;
+
+                                if (str_eval_clk != "book") {
+                                    std::regex pattern_eval1(R"(\[\%eval (.+?)\])");
+                                    std::regex pattern_eval2(R"((.+?)\/)");
+
+                                    std::string str_eval;
+                                    if (std::regex_search(str_eval_clk, match, pattern_eval1) ||
+                                        std::regex_search(str_eval_clk, match, pattern_eval2)) {
+                                        str_eval = match.str(1);
+                                        trim(str_eval);
+                                    }
+                                    else {
+                                        str_eval = str_eval_clk;
+                                    }
+
+                                    bool success = false;
+                                    Value value = parse_score_from_pgn_extract(str_eval, success);
+                                    if (success) {
+                                        eval_found = true;
+                                        psv.score = std::clamp(value, -VALUE_MATE, VALUE_MATE);
+                                    }
+
+#if defined(DEBUG_CONVERT_BIN_FROM_PGN_EXTRACT)
+                                    std::cout << "str_eval=" << str_eval << std::endl;
+                                    std::cout << "success=" << success << ", psv.score=" << psv.score << std::endl;
+#endif
+                                }
+                            }
+                        }
+
+                        // write
+                        if (eval_found || convert_no_eval_fens_as_score_zero) {
+                            if (!eval_found && convert_no_eval_fens_as_score_zero) {
+                                psv.score = 0;
+                            }
+
+                            psv.gamePly = gamePly;
+                            psv.game_result = game_result;
+
+                            if (pos.side_to_move() == BLACK) {
+                                if (!pgn_eval_side_to_move) {
+                                    psv.score *= -1;
+                                }
+                                psv.game_result *= -1;
+                            }
+
+                            ofs.write((char*)&psv, sizeof(PackedSfenValue));
+
+                            fen_count++;
+                        }
+                    }
+
+                    game_result = 0;
+                }
+            }
+        }
+
+        std::cout << now_string() << " game_count=" << game_count << ", fen_count=" << fen_count << std::endl;
+        std::cout << now_string() << " all done" << std::endl;
+        ofs.close();
+    }
+
+    void convert_plain(
+        const vector<string>& filenames,
+        const string& output_file_name)
+    {
+        Position tpos;
+        std::ofstream ofs;
+        ofs.open(output_file_name, ios::app);
+        auto th = Threads.main();
+        for (auto filename : filenames) {
+            std::cout << "convert " << filename << " ... ";
+
+            // Just convert packedsfenvalue to text
+            std::fstream fs;
+            fs.open(filename, ios::in | ios::binary);
+            PackedSfenValue p;
+            while (true)
+            {
+                if (fs.read((char*)&p, sizeof(PackedSfenValue))) {
+                    StateInfo si;
+                    tpos.set_from_packed_sfen(p.sfen, &si, th, false);
+
+                    // write as plain text
+                    ofs << "fen " << tpos.fen() << std::endl;
+                    ofs << "move " << UCI::move(Move(p.move), false) << std::endl;
+                    ofs << "score " << p.score << std::endl;
+                    ofs << "ply " << int(p.gamePly) << std::endl;
+                    ofs << "result " << int(p.game_result) << std::endl;
+                    ofs << "e" << std::endl;
+                }
+                else {
+                    break;
+                }
+            }
+            fs.close();
+            std::cout << "done" << std::endl;
+        }
+        ofs.close();
+        std::cout << "all done" << std::endl;
+    }
+
+    static inline const std::string plain_extension = ".plain";
+    static inline const std::string bin_extension = ".bin";
+    static inline const std::string binpack_extension = ".binpack";
+
+    static bool file_exists(const std::string& name)
+    {
+        std::ifstream f(name);
+        return f.good();
+    }
+
+    static bool ends_with(const std::string& lhs, const std::string& end)
+    {
+        if (end.size() > lhs.size()) return false;
+
+        return std::equal(end.rbegin(), end.rend(), lhs.rbegin());
+    }
+
+    static bool is_convert_of_type(
+        const std::string& input_path,
+        const std::string& output_path,
+        const std::string& expected_input_extension,
+        const std::string& expected_output_extension)
+    {
+        return ends_with(input_path, expected_input_extension)
+            && ends_with(output_path, expected_output_extension);
+    }
+
+    using ConvertFunctionType = void(std::string inputPath, std::string outputPath, std::ios_base::openmode om, bool validate);
+
+    static ConvertFunctionType* get_convert_function(const std::string& input_path, const std::string& output_path)
+    {
+        if (is_convert_of_type(input_path, output_path, plain_extension, bin_extension))
+            return binpack::convertPlainToBin;
+        if (is_convert_of_type(input_path, output_path, plain_extension, binpack_extension))
+            return binpack::convertPlainToBinpack;
+
+        if (is_convert_of_type(input_path, output_path, bin_extension, plain_extension))
+            return binpack::convertBinToPlain;
+        if (is_convert_of_type(input_path, output_path, bin_extension, binpack_extension))
+            return binpack::convertBinToBinpack;
+
+        if (is_convert_of_type(input_path, output_path, binpack_extension, plain_extension))
+            return binpack::convertBinpackToPlain;
+        if (is_convert_of_type(input_path, output_path, binpack_extension, bin_extension))
+            return binpack::convertBinpackToBin;
+
+        return nullptr;
+    }
+
+    static void convert(const std::string& input_path, const std::string& output_path, std::ios_base::openmode om, bool validate)
+    {
+        if(!file_exists(input_path))
+        {
+            std::cerr << "Input file does not exist.\n";
+            return;
+        }
+
+        auto func = get_convert_function(input_path, output_path);
+        if (func != nullptr)
+        {
+            func(input_path, output_path, om, validate);
+        }
+        else
+        {
+            std::cerr << "Conversion between files of these types is not supported.\n";
+        }
+    }
+
+    static void convert(const std::vector<std::string>& args)
+    {
+        if (args.size() < 2 || args.size() > 4)
+        {
+            std::cerr << "Invalid arguments.\n";
+            std::cerr << "Usage: convert from_path to_path [append] [validate]\n";
+            return;
+        }
+
+        const bool append = std::find(args.begin() + 2, args.end(), "append") != args.end();
+        const bool validate = std::find(args.begin() + 2, args.end(), "validate") != args.end();
+
+        const std::ios_base::openmode openmode =
+            append
+            ? std::ios_base::app
+            : std::ios_base::trunc;
+
+        convert(args[0], args[1], openmode, validate);
+    }
+
+    void convert(istringstream& is)
+    {
+        std::vector<std::string> args;
+
+        while (true)
+        {
+            std::string token = "";
+            is >> token;
+            if (token == "")
+                break;
+
+            args.push_back(token);
+        }
+
+        convert(args);
+    }
+
+    static void append_files_from_dir(
+        std::vector<std::string>& filenames,
+        const std::string& base_dir,
+        const std::string& target_dir)
+    {
+        string kif_base_dir = Path::combine(base_dir, target_dir);
+
+        sys::path p(kif_base_dir); // Origin of enumeration
+        std::for_each(sys::directory_iterator(p), sys::directory_iterator(),
+            [&](const sys::path& path) {
+                if (sys::is_regular_file(path))
+                    filenames.push_back(Path::combine(target_dir, path.filename().generic_string()));
+            });
+    }
+
+    static void rebase_files(
+        std::vector<std::string>& filenames,
+        const std::string& base_dir)
+    {
+        for (auto& file : filenames)
+        {
+            file = Path::combine(base_dir, file);
+        }
+    }
+
+    void convert_bin_from_pgn_extract(std::istringstream& is)
+    {
+        std::vector<std::string> filenames;
+
+        string base_dir;
+        string target_dir;
+
+        bool pgn_eval_side_to_move = false;
+        bool convert_no_eval_fens_as_score_zero = false;
+
+        string output_file_name = "shuffled_sfen.bin";
+
+        while (true)
+        {
+            string option;
+            is >> option;
+
+            if (option == "")
+                break;
+
+            if (option == "targetdir") is >> target_dir;
+            else if (option == "targetfile")
+            {
+                std::string filename;
+                is >> filename;
+                filenames.push_back(filename);
+            }
+
+            else if (option == "basedir")   is >> base_dir;
+
+            else if (option == "pgn_eval_side_to_move") is >> pgn_eval_side_to_move;
+            else if (option == "convert_no_eval_fens_as_score_zero") is >> convert_no_eval_fens_as_score_zero;
+            else if (option == "output_file_name") is >> output_file_name;
+            else
+            {
+                cout << "Unknown option: " << option << ". Ignoring.\n";
+            }
+        }
+
+        if (!target_dir.empty())
+        {
+            append_files_from_dir(filenames, base_dir, target_dir);
+        }
+        rebase_files(filenames, base_dir);
+
+        Eval::NNUE::init();
+
+        cout << "convert_bin_from_pgn-extract.." << endl;
+        convert_bin_from_pgn_extract(
+            filenames,
+            output_file_name,
+            pgn_eval_side_to_move,
+            convert_no_eval_fens_as_score_zero);
+    }
+
+    void convert_bin(std::istringstream& is)
+    {
+        std::vector<std::string> filenames;
+
+        string base_dir;
+        string target_dir;
+
+        int ply_minimum = 0;
+        int ply_maximum = 114514;
+        bool interpolate_eval = 0;
+        bool check_invalid_fen = false;
+        bool check_illegal_move = false;
+
+        bool pgn_eval_side_to_move = false;
+        bool convert_no_eval_fens_as_score_zero = false;
+
+        double src_score_min_value = 0.0;
+        double src_score_max_value = 1.0;
+        double dest_score_min_value = 0.0;
+        double dest_score_max_value = 1.0;
+
+        string output_file_name = "shuffled_sfen.bin";
+
+        while (true)
+        {
+            string option;
+            is >> option;
+
+            if (option == "")
+                break;
+
+            if (option == "targetdir") is >> target_dir;
+            else if (option == "targetfile")
+            {
+                std::string filename;
+                is >> filename;
+                filenames.push_back(filename);
+            }
+
+            else if (option == "basedir")   is >> base_dir;
+
+            else if (option == "ply_minimum") is >> ply_minimum;
+            else if (option == "ply_maximum") is >> ply_maximum;
+            else if (option == "interpolate_eval") is >> interpolate_eval;
+            else if (option == "check_invalid_fen") is >> check_invalid_fen;
+            else if (option == "check_illegal_move") is >> check_illegal_move;
+            else if (option == "pgn_eval_side_to_move") is >> pgn_eval_side_to_move;
+            else if (option == "convert_no_eval_fens_as_score_zero") is >> convert_no_eval_fens_as_score_zero;
+            else if (option == "src_score_min_value") is >> src_score_min_value;
+            else if (option == "src_score_max_value") is >> src_score_max_value;
+            else if (option == "dest_score_min_value") is >> dest_score_min_value;
+            else if (option == "dest_score_max_value") is >> dest_score_max_value;
+            else if (option == "output_file_name") is >> output_file_name;
+            else
+            {
+                cout << "Unknown option: " << option << ". Ignoring.\n";
+            }
+        }
+
+        if (!target_dir.empty())
+        {
+            append_files_from_dir(filenames, base_dir, target_dir);
+        }
+        rebase_files(filenames, base_dir);
+
+        Eval::NNUE::init();
+
+        cout << "convert_bin.." << endl;
+            convert_bin(
+                filenames,
+                output_file_name,
+                ply_minimum,
+                ply_maximum,
+                interpolate_eval,
+                src_score_min_value,
+                src_score_max_value,
+                dest_score_min_value,
+                dest_score_max_value,
+                check_invalid_fen,
+                check_illegal_move
+            );
+    }
+
+    void convert_plain(std::istringstream& is)
+    {
+        std::vector<std::string> filenames;
+
+        string base_dir;
+        string target_dir;
+
+        string output_file_name = "shuffled_sfen.bin";
+
+        while (true)
+        {
+            string option;
+            is >> option;
+
+            if (option == "")
+                break;
+
+            if (option == "targetdir") is >> target_dir;
+            else if (option == "targetfile")
+            {
+                std::string filename;
+                is >> filename;
+                filenames.push_back(filename);
+            }
+
+            else if (option == "basedir")   is >> base_dir;
+
+            else if (option == "output_file_name") is >> output_file_name;
+            else
+            {
+                cout << "Unknown option: " << option << ". Ignoring.\n";
+            }
+        }
+
+        if (!target_dir.empty())
+        {
+            append_files_from_dir(filenames, base_dir, target_dir);
+        }
+        rebase_files(filenames, base_dir);
+
+        Eval::NNUE::init();
+
+        cout << "convert_plain.." << endl;
+        convert_plain(filenames, output_file_name);
+    }
+}

src/tools/convert.h

@@ -0,0 +1,18 @@
+#ifndef _CONVERT_H_
+#define _CONVERT_H_
+
+#include <vector>
+#include <string>
+#include <sstream>
+
+namespace Stockfish::Tools {
+    void convert(std::istringstream& is);
+
+    void convert_bin_from_pgn_extract(std::istringstream& is);
+
+    void convert_bin(std::istringstream& is);
+
+    void convert_plain(std::istringstream& is);
+}
+
+#endif

src/tools/opening_book.cpp

@@ -0,0 +1,43 @@
+#include "opening_book.h"
+
+#include <fstream>
+
+namespace Stockfish::Tools {
+
+    EpdOpeningBook::EpdOpeningBook(const std::string& file, PRNG& prng) :
+        OpeningBook(file)
+    {
+        std::ifstream in(file);
+        if (!in)
+        {
+            return;
+        }
+
+        std::string line;
+        while (std::getline(in, line))
+        {
+            if (line.empty())
+                continue;
+
+            fens.emplace_back(line);
+        }
+
+        Algo::shuffle(fens, prng);
+    }
+
+    static bool ends_with(const std::string& lhs, const std::string& end)
+    {
+        if (end.size() > lhs.size()) return false;
+
+        return std::equal(end.rbegin(), end.rend(), lhs.rbegin());
+    }
+
+    std::unique_ptr<OpeningBook> open_opening_book(const std::string& filename, PRNG& prng)
+    {
+        if (ends_with(filename, ".epd"))
+            return std::make_unique<EpdOpeningBook>(filename, prng);
+
+        return nullptr;
+    }
+
+}

src/tools/opening_book.h

@@ -0,0 +1,60 @@
+#ifndef LEARN_OPENING_BOOK_H
+#define LEARN_OPENING_BOOK_H
+
+#include "misc.h"
+#include "position.h"
+#include "thread.h"
+
+#include <vector>
+#include <random>
+#include <optional>
+#include <string>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+
+namespace Stockfish::Tools {
+
+    struct OpeningBook {
+
+        const std::string& next_fen()
+        {
+            assert(fens.size() > 0);
+
+            std::unique_lock lock(mutex);
+
+            auto& fen = fens[current_index++];
+            if (current_index >= fens.size())
+                current_index = 0;
+
+            return fen;
+        }
+
+        std::size_t size() const { return fens.size(); }
+
+        const std::string& get_filename() const { return filename; }
+
+    protected:
+        OpeningBook(const std::string& file) :
+            filename(file),
+            current_index(0)
+        {
+        }
+
+
+        std::mutex mutex;
+        std::string filename;
+        std::vector<std::string> fens;
+        std::size_t current_index;
+    };
+
+    struct EpdOpeningBook : OpeningBook {
+
+        EpdOpeningBook(const std::string& file, PRNG& prng);
+    };
+
+    std::unique_ptr<OpeningBook> open_opening_book(const std::string& filename, PRNG& prng);
+
+}
+
+#endif

src/tools/packed_sfen.h

@@ -0,0 +1,46 @@
+#ifndef _PACKED_SFEN_H_
+#define _PACKED_SFEN_H_
+
+#include <vector>
+#include <cstdint>
+
+namespace Stockfish::Tools {
+
+    // packed sfen
+    struct PackedSfen { std::uint8_t data[32]; };
+
+    // Structure in which PackedSfen and evaluation value are integrated
+    // If you write different contents for each option, it will be a problem when reusing the teacher game
+    // For the time being, write all the following members regardless of the options.
+    struct PackedSfenValue
+    {
+        // phase
+        PackedSfen sfen;
+
+        // Evaluation value returned from Tools::search()
+        std::int16_t score;
+
+        // PV first move
+        // Used when finding the match rate with the teacher
+        std::uint16_t move;
+
+        // Trouble of the phase from the initial phase.
+        std::uint16_t gamePly;
+
+        // 1 if the player on this side ultimately wins the game. -1 if you are losing.
+        // 0 if a draw is reached.
+        // The draw is in the teacher position generation command gensfen,
+        // Only write if LEARN_GENSFEN_DRAW_RESULT is enabled.
+        std::int8_t game_result;
+
+        // When exchanging the file that wrote the teacher aspect with other people
+        //Because this structure size is not fixed, pad it so that it is 40 bytes in any environment.
+        std::uint8_t padding;
+
+        // 32 + 2 + 2 + 2 + 1 + 1 = 40bytes
+    };
+
+    // Phase array: PSVector stands for packed sfen vector.
+    using PSVector = std::vector<PackedSfenValue>;
+}
+#endif

src/tools/sfen_packer.cpp

@@ -0,0 +1,389 @@
+#include "sfen_packer.h"
+
+#include "packed_sfen.h"
+
+#include "misc.h"
+#include "position.h"
+
+#include <sstream>
+#include <fstream>
+#include <cstring> // std::memset()
+
+using namespace std;
+
+namespace Stockfish::Tools {
+
+    // Class that handles bitstream
+    // useful when doing aspect encoding
+    struct BitStream
+    {
+        // Set the memory to store the data in advance.
+        // Assume that memory is cleared to 0.
+        void set_data(std::uint8_t* data_) { data = data_; reset(); }
+
+        // Get the pointer passed in set_data().
+        uint8_t* get_data() const { return data; }
+
+        // Get the cursor.
+        int get_cursor() const { return bit_cursor; }
+
+        // reset the cursor
+        void reset() { bit_cursor = 0; }
+
+        // Write 1bit to the stream.
+        // If b is non-zero, write out 1. If 0, write 0.
+        void write_one_bit(int b)
+        {
+            if (b)
+                data[bit_cursor / 8] |= 1 << (bit_cursor & 7);
+
+            ++bit_cursor;
+        }
+
+        // Get 1 bit from the stream.
+        int read_one_bit()
+        {
+            int b = (data[bit_cursor / 8] >> (bit_cursor & 7)) & 1;
+            ++bit_cursor;
+
+            return b;
+        }
+
+        // write n bits of data
+        // Data shall be written out from the lower order of d.
+        void write_n_bit(int d, int n)
+        {
+            for (int i = 0; i <n; ++i)
+                write_one_bit(d & (1 << i));
+        }
+
+        // read n bits of data
+        // Reverse conversion of write_n_bit().
+        int read_n_bit(int n)
+        {
+            int result = 0;
+            for (int i = 0; i < n; ++i)
+                result |= read_one_bit() ? (1 << i) : 0;
+
+            return result;
+        }
+
+    private:
+        // Next bit position to read/write.
+        int bit_cursor;
+
+        // data entity
+        std::uint8_t* data;
+    };
+
+    // Class for compressing/decompressing sfen
+    // sfen can be packed to 256bit (32bytes) by Huffman coding.
+    // This is proven by mini. The above is Huffman coding.
+    //
+    // Internal format = 1-bit turn + 7-bit king position *2 + piece on board (Huffman coding) + hand piece (Huffman coding)
+    // Side to move (White = 0, Black = 1) (1bit)
+    // White King Position (6 bits)
+    // Black King Position (6 bits)
+    // Huffman Encoding of the board
+    // Castling availability (1 bit x 4)
+    // En passant square (1 or 1 + 6 bits)
+    // Rule 50 (6 bits)
+    // Game play (8 bits)
+    //
+    // TODO(someone): Rename SFEN to FEN.
+    //
+    struct SfenPacker
+    {
+        void pack(const Position& pos, bool resetCastlingRights);
+
+        // sfen packed by pack() (256bit = 32bytes)
+        // Or sfen to decode with unpack()
+        uint8_t *data; // uint8_t[32];
+
+        BitStream stream;
+
+        // Output the board pieces to stream.
+        void write_board_piece_to_stream(Piece pc);
+
+        // Read one board piece from stream
+        Piece read_board_piece_from_stream();
+    };
+
+
+    // Huffman coding
+    // * is simplified from mini encoding to make conversion easier.
+    //
+    // Huffman Encoding
+    //
+    // Empty  xxxxxxx0
+    // Pawn   xxxxx001 + 1 bit (Color)
+    // Knight xxxxx011 + 1 bit (Color)
+    // Bishop xxxxx101 + 1 bit (Color)
+    // Rook   xxxxx111 + 1 bit (Color)
+    // Queen   xxxx1001 + 1 bit (Color)
+    //
+    // Worst case:
+    // - 32 empty squares    32 bits
+    // - 30 pieces           150 bits
+    // - 2 kings             12 bits
+    // - castling rights     4 bits
+    // - ep square           7 bits
+    // - rule50              7 bits
+    // - game ply            16 bits
+    // - TOTAL               228 bits < 256 bits
+
+    struct HuffmanedPiece
+    {
+        int code; // how it will be coded
+        int bits; // How many bits do you have
+    };
+
+    constexpr HuffmanedPiece huffman_table[] =
+    {
+        {0b0000,1}, // NO_PIECE
+        {0b0001,4}, // PAWN
+        {0b0011,4}, // KNIGHT
+        {0b0101,4}, // BISHOP
+        {0b0111,4}, // ROOK
+        {0b1001,4}, // QUEEN
+    };
+
+    // Pack sfen and store in data[32].
+    void SfenPacker::pack(const Position& pos, bool resetCastlingRights)
+    {
+        memset(data, 0, 32 /* 256bit */);
+        stream.set_data(data);
+
+        // turn
+        // Side to move.
+        stream.write_one_bit((int)(pos.side_to_move()));
+
+        // 7-bit positions for leading and trailing balls
+        // White king and black king, 6 bits for each.
+        for(auto c: Colors)
+            stream.write_n_bit(pos.king_square(c), 6);
+
+        // Write the pieces on the board other than the kings.
+        for (Rank r = RANK_8; r >= RANK_1; --r)
+        {
+            for (File f = FILE_A; f <= FILE_H; ++f)
+            {
+                Piece pc = pos.piece_on(make_square(f, r));
+                if (type_of(pc) == KING)
+                    continue;
+                write_board_piece_to_stream(pc);
+            }
+        }
+
+        if (resetCastlingRights)
+        {
+            stream.write_n_bit(0, 4);
+        }
+        else
+        {
+            stream.write_one_bit(pos.can_castle(WHITE_OO));
+            stream.write_one_bit(pos.can_castle(WHITE_OOO));
+            stream.write_one_bit(pos.can_castle(BLACK_OO));
+            stream.write_one_bit(pos.can_castle(BLACK_OOO));
+        }
+
+        if (pos.ep_square() == SQ_NONE) {
+            stream.write_one_bit(0);
+        }
+        else {
+            stream.write_one_bit(1);
+            stream.write_n_bit(static_cast<int>(pos.ep_square()), 6);
+        }
+
+        stream.write_n_bit(pos.state()->rule50, 6);
+
+        const int fm = 1 + (pos.game_ply()-(pos.side_to_move() == BLACK)) / 2;
+        stream.write_n_bit(fm, 8);
+
+        // Write high bits of half move. This is a fix for the
+        // limited range of half move counter.
+        // This is backwards compatibile.
+        stream.write_n_bit(fm >> 8, 8);
+
+        // Write the highest bit of rule50 at the end. This is a backwards
+        // compatibile fix for rule50 having only 6 bits stored.
+        // This bit is just ignored by the old parsers.
+        stream.write_n_bit(pos.state()->rule50 >> 6, 1);
+
+        assert(stream.get_cursor() <= 256);
+    }
+
+    // Output the board pieces to stream.
+    void SfenPacker::write_board_piece_to_stream(Piece pc)
+    {
+        // piece type
+        PieceType pr = type_of(pc);
+        auto c = huffman_table[pr];
+        stream.write_n_bit(c.code, c.bits);
+
+        if (pc == NO_PIECE)
+            return;
+
+        // first and second flag
+        stream.write_one_bit(color_of(pc));
+    }
+
+    // Read one board piece from stream
+    Piece SfenPacker::read_board_piece_from_stream()
+    {
+        PieceType pr = NO_PIECE_TYPE;
+        int code = 0, bits = 0;
+        while (true)
+        {
+            code |= stream.read_one_bit() << bits;
+            ++bits;
+
+            assert(bits <= 6);
+
+            for (pr = NO_PIECE_TYPE; pr <KING; ++pr)
+                if (huffman_table[pr].code == code
+                    && huffman_table[pr].bits == bits)
+                    goto Found;
+        }
+    Found:;
+        if (pr == NO_PIECE_TYPE)
+            return NO_PIECE;
+
+        // first and second flag
+        Color c = (Color)stream.read_one_bit();
+
+        return make_piece(c, pr);
+    }
+
+    int set_from_packed_sfen(Position& pos, const PackedSfen& sfen, StateInfo* si, Thread* th, bool frc)
+    {
+        SfenPacker packer;
+        auto& stream = packer.stream;
+
+        // TODO: separate streams for writing and reading. Here we actually have to
+        // const_cast which is not safe in the long run.
+        stream.set_data(const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(&sfen)));
+
+        pos.clear();
+        std::memset(si, 0, sizeof(StateInfo));
+        si->accumulator.computed[WHITE] = false;
+        si->accumulator.computed[BLACK] = false;
+        pos.st = si;
+
+        // Active color
+        pos.sideToMove = (Color)stream.read_one_bit();
+
+        // First the position of the ball
+        for (auto c : Colors)
+            pos.board[stream.read_n_bit(6)] = make_piece(c, KING);
+
+        // Piece placement
+        for (Rank r = RANK_8; r >= RANK_1; --r)
+        {
+            for (File f = FILE_A; f <= FILE_H; ++f)
+            {
+                auto sq = make_square(f, r);
+
+                // it seems there are already balls
+                Piece pc;
+                if (type_of(pos.board[sq]) != KING)
+                {
+                    assert(pos.board[sq] == NO_PIECE);
+                    pc = packer.read_board_piece_from_stream();
+                }
+                else
+                {
+                    pc = pos.board[sq];
+                    // put_piece() will catch ASSERT unless you remove it all.
+                    pos.board[sq] = NO_PIECE;
+                }
+
+                // There may be no pieces, so skip in that case.
+                if (pc == NO_PIECE)
+                    continue;
+
+                pos.put_piece(Piece(pc), sq);
+
+                if (stream.get_cursor()> 256)
+                    return 1;
+            }
+        }
+
+        // Castling availability.
+        pos.st->castlingRights = 0;
+        if (stream.read_one_bit()) {
+            Square rsq;
+            for (rsq = relative_square(WHITE, SQ_H1); pos.piece_on(rsq) != W_ROOK; --rsq) {}
+            pos.set_castling_right(WHITE, rsq);
+        }
+        if (stream.read_one_bit()) {
+            Square rsq;
+            for (rsq = relative_square(WHITE, SQ_A1); pos.piece_on(rsq) != W_ROOK; ++rsq) {}
+            pos.set_castling_right(WHITE, rsq);
+        }
+        if (stream.read_one_bit()) {
+            Square rsq;
+            for (rsq = relative_square(BLACK, SQ_H1); pos.piece_on(rsq) != B_ROOK; --rsq) {}
+            pos.set_castling_right(BLACK, rsq);
+        }
+        if (stream.read_one_bit()) {
+            Square rsq;
+            for (rsq = relative_square(BLACK, SQ_A1); pos.piece_on(rsq) != B_ROOK; ++rsq) {}
+            pos.set_castling_right(BLACK, rsq);
+        }
+
+        // En passant square. Ignore if no pawn capture is possible
+        if (stream.read_one_bit()) {
+            Square ep_square = static_cast<Square>(stream.read_n_bit(6));
+            pos.st->epSquare = ep_square;
+
+            if (!(pos.attackers_to(pos.st->epSquare) & pos.pieces(pos.sideToMove, PAWN))
+                || !(pos.pieces(~pos.sideToMove, PAWN) & (pos.st->epSquare + pawn_push(~pos.sideToMove))))
+                pos.st->epSquare = SQ_NONE;
+        }
+        else {
+            pos.st->epSquare = SQ_NONE;
+        }
+
+        // Halfmove clock
+        pos.st->rule50 = stream.read_n_bit(6);
+
+        // Fullmove number
+        pos.gamePly = stream.read_n_bit(8);
+
+        // Read the highest bit of rule50. This was added as a fix for rule50
+        // counter having only 6 bits stored.
+        // In older entries this will just be a zero bit.
+        pos.gamePly |= stream.read_n_bit(8) << 8;
+
+        // Read the highest bit of rule50. This was added as a fix for rule50
+        // counter having only 6 bits stored.
+        // In older entries this will just be a zero bit.
+        pos.st->rule50 |= stream.read_n_bit(1) << 6;
+
+        // Convert from fullmove starting from 1 to gamePly starting from 0,
+        // handle also common incorrect FEN with fullmove = 0.
+        pos.gamePly = std::max(2 * (pos.gamePly - 1), 0) + (pos.sideToMove == BLACK);
+
+        assert(stream.get_cursor() <= 256);
+
+        pos.chess960 = frc;
+        pos.thisThread = th;
+        pos.set_state(pos.st);
+
+        assert(pos.pos_is_ok());
+
+        return 0;
+    }
+
+    PackedSfen sfen_pack(Position& pos, bool resetCastlingRights)
+    {
+        PackedSfen sfen;
+
+        SfenPacker sp;
+        sp.data = (uint8_t*)&sfen;
+        sp.pack(pos, resetCastlingRights);
+
+        return sfen;
+    }
+}

src/tools/sfen_packer.h

@@ -0,0 +1,22 @@
+#ifndef _SFEN_PACKER_H_
+#define _SFEN_PACKER_H_
+
+#include "types.h"
+
+#include "packed_sfen.h"
+
+#include <cstdint>
+
+namespace Stockfish {
+    class Position;
+    struct StateInfo;
+    class Thread;
+}
+
+namespace Stockfish::Tools {
+
+    int set_from_packed_sfen(Position& pos, const PackedSfen& sfen, StateInfo* si, Thread* th, bool frc);
+    PackedSfen sfen_pack(Position& pos, bool resetCastlingRights);
+}
+
+#endif

src/tools/sfen_reader.h

@@ -0,0 +1,352 @@
+#include "sfen_stream.h"
+
+#include "packed_sfen.h"
+
+#include "misc.h"
+
+#include <string>
+#include <vector>
+#include <deque>
+#include <memory>
+#include <mutex>
+#include <list>
+#include <atomic>
+#include <optional>
+#include <iostream>
+#include <cstdint>
+#include <thread>
+#include <functional>
+
+namespace Stockfish::Tools{
+
+    enum struct SfenReaderMode
+    {
+        Sequential,
+        Cyclic
+    };
+
+    // Sfen reader
+    struct SfenReader
+    {
+        // Number of phases buffered by each thread 0.1M phases. 4M phase at 40HT
+        static constexpr size_t DEFAULT_THREAD_BUFFER_SIZE = 10 * 1000;
+
+        // Buffer for reading files (If this is made larger,
+        // the shuffle becomes larger and the phases may vary.
+        // If it is too large, the memory consumption will increase.
+        // SFEN_READ_SIZE is a multiple of THREAD_BUFFER_SIZE.
+        static constexpr const size_t DEFAULT_SFEN_READ_SIZE = 1000 * 1000 * 10;
+
+        // Do not use std::random_device().
+        // Because it always the same integers on MinGW.
+        SfenReader(
+            const std::vector<std::string>& filenames_,
+            bool do_shuffle,
+            SfenReaderMode mode_,
+            int thread_num,
+            const std::string& seed,
+            size_t read_size = DEFAULT_SFEN_READ_SIZE,
+            size_t buffer_size = DEFAULT_THREAD_BUFFER_SIZE
+        ) :
+            filenames(filenames_.begin(), filenames_.end()),
+            mode(mode_),
+            // Due to the implementation of waiting for buffer empty a bit
+            // the read size must be at least twice the buffer size.
+            sfen_read_size(std::max(read_size, buffer_size * 2)),
+            thread_buffer_size(buffer_size),
+            prng(seed)
+        {
+            packed_sfens.resize(thread_num);
+            total_read = 0;
+            end_of_files = false;
+            shuffle = do_shuffle;
+            stop_flag = false;
+            num_buffers_in_pool.store(0);
+
+            file_worker_thread = std::thread([&] {
+                this->file_read_worker();
+            });
+        }
+
+        ~SfenReader()
+        {
+            stop_flag = true;
+
+            if (file_worker_thread.joinable())
+                file_worker_thread.join();
+        }
+
+        // Load the phase for calculation such as mse.
+        PSVector read_some(uint64_t count, uint64_t count_tries, std::function<bool(const PackedSfenValue&)> do_take)
+        {
+            PSVector psv;
+            psv.reserve(count);
+
+            for (uint64_t i = 0; i < count_tries; ++i)
+            {
+                PackedSfenValue ps;
+                if (!read_to_thread_buffer(0, ps))
+                {
+                    std::cout << "ERROR (sfen_reader): Reading failed." << std::endl;
+                    return psv;
+                }
+
+                if (do_take(ps))
+                {
+                    psv.push_back(ps);
+
+                    if (psv.size() >= count)
+                        break;
+                }
+            }
+
+            return psv;
+        }
+
+        // [ASYNC] Thread returns one aspect. Otherwise returns false.
+        bool read_to_thread_buffer(size_t thread_id, PackedSfenValue& ps)
+        {
+            // If there are any positions left in the thread buffer
+            // then retrieve one and return it.
+            auto& thread_ps = packed_sfens[thread_id];
+
+            // Fill the read buffer if there is no remaining buffer,
+            // but if it doesn't even exist, finish.
+            // If the buffer is empty, fill it.
+            if ((thread_ps == nullptr || thread_ps->empty())
+                && !read_to_thread_buffer_impl(thread_id))
+                return false;
+
+            // read_to_thread_buffer_impl() returned true,
+            // Since the filling of the thread buffer with the
+            // phase has been completed successfully
+            // thread_ps->rbegin() is alive.
+
+            ps = thread_ps->back();
+            thread_ps->pop_back();
+
+            // If you've run out of buffers, call delete yourself to free this buffer.
+            if (thread_ps->empty())
+            {
+                thread_ps.reset();
+            }
+
+            return true;
+        }
+
+        // [ASYNC] Read some aspects into thread buffer.
+        bool read_to_thread_buffer_impl(size_t thread_id)
+        {
+            while (true)
+            {
+                {
+                    std::unique_lock<std::mutex> lk(mutex);
+                    // If you can fill from the file buffer, that's fine.
+                    if (packed_sfens_pool.size() != 0)
+                    {
+                        // It seems that filling is possible, so fill and finish.
+
+                        packed_sfens[thread_id] = std::move(packed_sfens_pool.front());
+                        packed_sfens_pool.pop_front();
+                        num_buffers_in_pool.fetch_sub(1);
+
+                        total_read += thread_buffer_size;
+
+                        return true;
+                    }
+                }
+
+                // The file to read is already gone. No more use.
+                if (end_of_files)
+                    return false;
+
+                // Waiting for file worker to fill packed_sfens_pool.
+                // The mutex isn't locked, so it should fill up soon.
+                // Poor man's condition variable.
+                sleep(1);
+            }
+
+        }
+
+        void file_read_worker()
+        {
+            std::string currentFilename;
+            uint64_t numEntriesReadFromCurrentFile = 0;
+
+            auto open_next_file = [&]() {
+                // no more
+                for(;;)
+                {
+                    sfen_input_stream.reset();
+
+                    if (filenames.empty())
+                        return false;
+
+                    // Get the next file name.
+                    currentFilename = filenames.front();
+                    filenames.pop_front();
+
+                    numEntriesReadFromCurrentFile = 0;
+
+                    sfen_input_stream = open_sfen_input_file(currentFilename);
+
+                    auto out = sync_region_cout.new_region();
+                    if (sfen_input_stream == nullptr)
+                    {
+                        out << "INFO (sfen_reader): File does not exist: " << currentFilename << '\n';
+                    }
+                    else
+                    {
+                        out << "INFO (sfen_reader): Opened file for reading: " << currentFilename << '\n';
+
+                        // in case the file is empty or was deleted.
+                        if (sfen_input_stream->eof())
+                        {
+                            out << "  - File empty, nothing to read.\n";
+                        }
+                        else
+                        {
+                            return true;
+                        }
+                    }
+                }
+            };
+
+            if (sfen_input_stream == nullptr && !open_next_file())
+            {
+                auto out = sync_region_cout.new_region();
+                out << "INFO (sfen_reader): End of files." << std::endl;
+                end_of_files = true;
+                return;
+            }
+
+            // We want to set the `end_of_files` only after we read everything AND copy to the buffer pool.
+            bool local_end_of_files = false;
+            while (!local_end_of_files)
+            {
+                // Wait for the buffer to run out.
+                // This size() is read only, so you don't need to lock it.
+                while (!stop_flag && num_buffers_in_pool.load() >= sfen_read_size / thread_buffer_size)
+                    sleep(100);
+
+                if (stop_flag)
+                    return;
+
+                PSVector sfens;
+                sfens.reserve(sfen_read_size);
+
+                // Read from the file into the file buffer.
+                while (sfens.size() < sfen_read_size)
+                {
+                    std::optional<PackedSfenValue> p = sfen_input_stream->next();
+                    if (p.has_value())
+                    {
+                        sfens.push_back(*p);
+                        ++numEntriesReadFromCurrentFile;
+                    }
+                    else
+                    {
+                        if (mode == SfenReaderMode::Cyclic
+                            && numEntriesReadFromCurrentFile > 0)
+                        {
+                            // The file contained data so we add it again to the end of the queue.
+                            filenames.emplace_back(currentFilename);
+                        }
+
+                        if(!open_next_file())
+                        {
+                            // There was no next file. Abort.
+                            auto out = sync_region_cout.new_region();
+                            out << "INFO (sfen_reader): End of files." << std::endl;
+                            local_end_of_files = true;
+                            break;
+                        }
+                    }
+                }
+
+                // Shuffle the read phase data.
+                if (shuffle)
+                {
+                    Algo::shuffle(sfens, prng);
+                }
+
+                std::vector<std::unique_ptr<PSVector>> buffers;
+                for (size_t offset = 0; offset < sfens.size(); offset += thread_buffer_size)
+                {
+                    const size_t count =
+                        offset + thread_buffer_size > sfens.size()
+                        ? sfens.size() - offset
+                        : thread_buffer_size;
+
+                    // Delete this pointer on the receiving side.
+                    auto buf = std::make_unique<PSVector>();
+                    buf->resize(count);
+                    memcpy(
+                        buf->data(),
+                        &sfens[offset],
+                        sizeof(PackedSfenValue) * count);
+
+                    buffers.emplace_back(std::move(buf));
+                }
+
+                {
+                    std::unique_lock<std::mutex> lk(mutex);
+
+                    // The mutex lock is required because the%
+                    // contents of packed_sfens_pool are changed.
+
+                    for (auto& buf : buffers)
+                    {
+                        num_buffers_in_pool.fetch_add(1);
+                        packed_sfens_pool.emplace_back(std::move(buf));
+                    }
+                }
+            }
+
+            end_of_files = true;
+        }
+
+    protected:
+
+        // worker thread reading file in background
+        std::thread file_worker_thread;
+
+        // sfen files
+        std::deque<std::string> filenames;
+
+        std::atomic<bool> stop_flag;
+
+        // number of phases read (file to memory buffer)
+        std::atomic<uint64_t> total_read;
+
+        // Do not shuffle when reading the phase.
+        bool shuffle;
+
+        SfenReaderMode mode;
+
+        size_t sfen_read_size;
+        size_t thread_buffer_size;
+
+        // Random number to shuffle when reading the phase
+        PRNG prng;
+
+        // Did you read the files and reached the end?
+        std::atomic<bool> end_of_files;
+
+        // handle of sfen file
+        std::unique_ptr<BasicSfenInputStream> sfen_input_stream;
+
+        // sfen for each thread
+        // (When the thread is used up, the thread should call delete to release it.)
+        std::vector<std::unique_ptr<PSVector>> packed_sfens;
+
+        // Mutex when accessing packed_sfens_pool
+        std::mutex mutex;
+
+        // pool of sfen. The worker thread read from the file is added here.
+        // Each worker thread fills its own packed_sfens[thread_id] from here.
+        // * Lock and access the mutex.
+        std::list<std::unique_ptr<PSVector>> packed_sfens_pool;
+        std::atomic<size_t> num_buffers_in_pool;
+    };
+}

src/tools/sfen_stream.h

@@ -0,0 +1,222 @@
+#ifndef _SFEN_STREAM_H_
+#define _SFEN_STREAM_H_
+
+#include "packed_sfen.h"
+
+#include "extra/nnue_data_binpack_format.h"
+
+#include <optional>
+#include <fstream>
+#include <string>
+#include <memory>
+
+namespace Stockfish::Tools {
+
+    enum struct SfenOutputType
+    {
+        Bin,
+        Binpack
+    };
+
+    static bool ends_with(const std::string& lhs, const std::string& end)
+    {
+        if (end.size() > lhs.size()) return false;
+
+        return std::equal(end.rbegin(), end.rend(), lhs.rbegin());
+    }
+
+    static bool has_extension(const std::string& filename, const std::string& extension)
+    {
+        return ends_with(filename, "." + extension);
+    }
+
+    static std::string filename_with_extension(const std::string& filename, const std::string& ext)
+    {
+        if (ends_with(filename, ext))
+        {
+            return filename;
+        }
+        else
+        {
+            return filename + "." + ext;
+        }
+    }
+
+    struct BasicSfenInputStream
+    {
+        virtual std::optional<PackedSfenValue> next() = 0;
+        virtual bool eof() const = 0;
+        virtual ~BasicSfenInputStream() {}
+    };
+
+    struct BinSfenInputStream : BasicSfenInputStream
+    {
+        static constexpr auto openmode = std::ios::in | std::ios::binary;
+        static inline const std::string extension = "bin";
+
+        BinSfenInputStream(std::string filename) :
+            m_stream(filename, openmode),
+            m_eof(!m_stream)
+        {
+        }
+
+        std::optional<PackedSfenValue> next() override
+        {
+            PackedSfenValue e;
+            if(m_stream.read(reinterpret_cast<char*>(&e), sizeof(PackedSfenValue)))
+            {
+                return e;
+            }
+            else
+            {
+                m_eof = true;
+                return std::nullopt;
+            }
+        }
+
+        bool eof() const override
+        {
+            return m_eof;
+        }
+
+        ~BinSfenInputStream() override {}
+
+    private:
+        std::fstream m_stream;
+        bool m_eof;
+    };
+
+    struct BinpackSfenInputStream : BasicSfenInputStream
+    {
+        static constexpr auto openmode = std::ios::in | std::ios::binary;
+        static inline const std::string extension = "binpack";
+
+        BinpackSfenInputStream(std::string filename) :
+            m_stream(filename, openmode),
+            m_eof(!m_stream.hasNext())
+        {
+        }
+
+        std::optional<PackedSfenValue> next() override
+        {
+            static_assert(sizeof(binpack::nodchip::PackedSfenValue) == sizeof(PackedSfenValue));
+
+            if (!m_stream.hasNext())
+            {
+                m_eof = true;
+                return std::nullopt;
+            }
+
+            auto training_data_entry = m_stream.next();
+            auto v = binpack::trainingDataEntryToPackedSfenValue(training_data_entry);
+            PackedSfenValue psv;
+            // same layout, different types. One is from generic library.
+            std::memcpy(&psv, &v, sizeof(PackedSfenValue));
+
+            return psv;
+        }
+
+        bool eof() const override
+        {
+            return m_eof;
+        }
+
+        ~BinpackSfenInputStream() override {}
+
+    private:
+        binpack::CompressedTrainingDataEntryReader m_stream;
+        bool m_eof;
+    };
+
+    struct BasicSfenOutputStream
+    {
+        virtual void write(const PSVector& sfens) = 0;
+        virtual ~BasicSfenOutputStream() {}
+    };
+
+    struct BinSfenOutputStream : BasicSfenOutputStream
+    {
+        static constexpr auto openmode = std::ios::out | std::ios::binary | std::ios::app;
+        static inline const std::string extension = "bin";
+
+        BinSfenOutputStream(std::string filename) :
+            m_stream(filename_with_extension(filename, extension), openmode)
+        {
+        }
+
+        void write(const PSVector& sfens) override
+        {
+            m_stream.write(reinterpret_cast<const char*>(sfens.data()), sizeof(PackedSfenValue) * sfens.size());
+        }
+
+        ~BinSfenOutputStream() override {}
+
+    private:
+        std::fstream m_stream;
+    };
+
+    struct BinpackSfenOutputStream : BasicSfenOutputStream
+    {
+        static constexpr auto openmode = std::ios::out | std::ios::binary | std::ios::app;
+        static inline const std::string extension = "binpack";
+
+        BinpackSfenOutputStream(std::string filename) :
+            m_stream(filename_with_extension(filename, extension), openmode)
+        {
+        }
+
+        void write(const PSVector& sfens) override
+        {
+            static_assert(sizeof(binpack::nodchip::PackedSfenValue) == sizeof(PackedSfenValue));
+
+            for(auto& sfen : sfens)
+            {
+                // The library uses a type that's different but layout-compatibile.
+                binpack::nodchip::PackedSfenValue e;
+                std::memcpy(&e, &sfen, sizeof(binpack::nodchip::PackedSfenValue));
+                m_stream.addTrainingDataEntry(binpack::packedSfenValueToTrainingDataEntry(e));
+            }
+        }
+
+        ~BinpackSfenOutputStream() override {}
+
+    private:
+        binpack::CompressedTrainingDataEntryWriter m_stream;
+    };
+
+    inline std::unique_ptr<BasicSfenInputStream> open_sfen_input_file(const std::string& filename)
+    {
+        if (has_extension(filename, BinSfenInputStream::extension))
+            return std::make_unique<BinSfenInputStream>(filename);
+        else if (has_extension(filename, BinpackSfenInputStream::extension))
+            return std::make_unique<BinpackSfenInputStream>(filename);
+
+        return nullptr;
+    }
+
+    inline std::unique_ptr<BasicSfenOutputStream> create_new_sfen_output(const std::string& filename, SfenOutputType sfen_output_type)
+    {
+        switch(sfen_output_type)
+        {
+            case SfenOutputType::Bin:
+                return std::make_unique<BinSfenOutputStream>(filename);
+            case SfenOutputType::Binpack:
+                return std::make_unique<BinpackSfenOutputStream>(filename);
+        }
+
+        assert(false);
+        return nullptr;
+    }
+
+    inline std::unique_ptr<BasicSfenOutputStream> create_new_sfen_output(const std::string& filename)
+    {
+        if (has_extension(filename, BinSfenOutputStream::extension))
+            return std::make_unique<BinSfenOutputStream>(filename);
+        else if (has_extension(filename, BinpackSfenOutputStream::extension))
+            return std::make_unique<BinpackSfenOutputStream>(filename);
+
+        return nullptr;
+    }
+}
+
+#endif

src/tools/sfen_writer.h

@@ -0,0 +1,203 @@
+#include "packed_sfen.h"
+#include "sfen_stream.h"
+
+#include "misc.h"
+
+#include "extra/nnue_data_binpack_format.h"
+
+#include "syzygy/tbprobe.h"
+
+#include <cstring>
+#include <fstream>
+#include <limits>
+#include <list>
+#include <memory>
+#include <optional>
+#include <shared_mutex>
+#include <thread>
+#include <atomic>
+
+namespace Stockfish::Tools {
+
+    // Helper class for exporting Sfen
+    struct SfenWriter
+    {
+        // Amount of sfens required to flush the buffer.
+        static constexpr size_t SFEN_WRITE_SIZE = 5000;
+
+        // File name to write and number of threads to create
+        SfenWriter(std::string filename_, int thread_num, uint64_t save_count, SfenOutputType sfen_output_type)
+        {
+            sfen_buffers_pool.reserve((size_t)thread_num * 10);
+            sfen_buffers.resize(thread_num);
+
+            auto out = sync_region_cout.new_region();
+            out << "INFO (sfen_writer): Creating new data file at " << filename_ << std::endl;
+
+            sfen_format = sfen_output_type;
+            output_file_stream = create_new_sfen_output(filename_, sfen_format);
+            filename = filename_;
+            save_every = save_count;
+
+            finished = false;
+
+            file_worker_thread = std::thread([&] { this->file_write_worker(); });
+        }
+
+        ~SfenWriter()
+        {
+            flush();
+
+            finished = true;
+            file_worker_thread.join();
+            output_file_stream.reset();
+
+#if !defined(NDEBUG)
+            {
+                // All buffers should be empty since file_worker_thread
+                // should have written everything before exiting.
+                for (const auto& p : sfen_buffers) { assert(p == nullptr); (void)p ; }
+                assert(sfen_buffers_pool.empty());
+            }
+#endif
+        }
+
+        void write(size_t thread_id, const PackedSfenValue& psv)
+        {
+            // We have a buffer for each thread and add it there.
+            // If the buffer overflows, write it to a file.
+
+            // This buffer is prepared for each thread.
+            auto& buf = sfen_buffers[thread_id];
+
+            // Secure since there is no buf at the first time
+            // and immediately after writing the thread buffer.
+            if (!buf)
+            {
+                buf = std::make_unique<PSVector>();
+                buf->reserve(SFEN_WRITE_SIZE);
+            }
+
+            // Buffer is exclusive to this thread.
+            // There is no need for a critical section.
+            buf->push_back(psv);
+
+            if (buf->size() >= SFEN_WRITE_SIZE)
+            {
+                // If you load it in sfen_buffers_pool, the worker will do the rest.
+
+                // Critical section since sfen_buffers_pool is shared among threads.
+                std::unique_lock<std::mutex> lk(mutex);
+                sfen_buffers_pool.emplace_back(std::move(buf));
+            }
+        }
+
+        void flush()
+        {
+            for (size_t i = 0; i < sfen_buffers.size(); ++i)
+            {
+                flush(i);
+            }
+        }
+
+        // Move what remains in the buffer for your thread to a buffer for writing to a file.
+        void flush(size_t thread_id)
+        {
+            std::unique_lock<std::mutex> lk(mutex);
+
+            auto& buf = sfen_buffers[thread_id];
+
+            // There is a case that buf==nullptr, so that check is necessary.
+            if (buf && buf->size() != 0)
+            {
+                sfen_buffers_pool.emplace_back(std::move(buf));
+            }
+        }
+
+        // Dedicated thread to write to file
+        void file_write_worker()
+        {
+            while (!finished || sfen_buffers_pool.size())
+            {
+                std::vector<std::unique_ptr<PSVector>> buffers;
+                {
+                    std::unique_lock<std::mutex> lk(mutex);
+
+                    // Atomically swap take the filled buffers and
+                    // create a new buffer pool for threads to fill.
+                    buffers = std::move(sfen_buffers_pool);
+                    sfen_buffers_pool = std::vector<std::unique_ptr<PSVector>>();
+                }
+
+                if (!buffers.size())
+                {
+                    // Poor man's condition variable.
+                    sleep(100);
+                }
+                else
+                {
+                    for (auto& buf : buffers)
+                    {
+                        output_file_stream->write(*buf);
+
+                        sfen_write_count += buf->size();
+
+                        // Add the processed number here, and if it exceeds save_every,
+                        // change the file name and reset this counter.
+                        sfen_write_count_current_file += buf->size();
+                        if (sfen_write_count_current_file >= save_every)
+                        {
+                            sfen_write_count_current_file = 0;
+
+                            // Sequential number attached to the file
+                            int n = (int)(sfen_write_count / save_every);
+
+                            // Rename the file and open it again.
+                            // Add ios::app in consideration of overwriting.
+                            // (Depending on the operation, it may not be necessary.)
+                            std::string new_filename = filename + "_" + std::to_string(n);
+                            output_file_stream = create_new_sfen_output(new_filename, sfen_format);
+
+                            auto out = sync_region_cout.new_region();
+                            out << "INFO (sfen_writer): Creating new data file at " << new_filename << std::endl;
+                        }
+                    }
+                }
+            }
+        }
+
+    private:
+
+        std::unique_ptr<BasicSfenOutputStream> output_file_stream;
+
+        // A new net is saved after every save_every sfens are processed.
+        uint64_t save_every = std::numeric_limits<uint64_t>::max();
+
+        // File name passed in the constructor
+        std::string filename;
+
+        // Thread to write to the file
+        std::thread file_worker_thread;
+
+        // Flag that all threads have finished
+        std::atomic<bool> finished;
+
+        SfenOutputType sfen_format;
+
+        // buffer before writing to file
+        // sfen_buffers is the buffer for each thread
+        // sfen_buffers_pool is a buffer for writing.
+        // After loading the phase in the former buffer by SFEN_WRITE_SIZE,
+        // transfer it to the latter.
+        std::vector<std::unique_ptr<PSVector>> sfen_buffers;
+        std::vector<std::unique_ptr<PSVector>> sfen_buffers_pool;
+
+        // Mutex required to access sfen_buffers_pool
+        std::mutex mutex;
+
+        // Number of sfens written in total, and the
+        // number of sfens written in the current file.
+        uint64_t sfen_write_count = 0;
+        uint64_t sfen_write_count_current_file = 0;
+    };
+}

src/tools/stats.h

@@ -0,0 +1,12 @@
+#ifndef _STATS_H_
+#define _STATS_H_
+
+#include <sstream>
+
+namespace Stockfish::Tools::Stats {
+
+    void gather_statistics(std::istringstream& is);
+
+}
+
+#endif

src/tools/training_data_generator.cpp

@@ -0,0 +1,952 @@
+#include "training_data_generator.h"
+
+#include "sfen_writer.h"
+#include "packed_sfen.h"
+#include "opening_book.h"
+
+#include "misc.h"
+#include "position.h"
+#include "thread.h"
+#include "tt.h"
+#include "uci.h"
+
+#include "extra/nnue_data_binpack_format.h"
+
+#include "nnue/evaluate_nnue.h"
+
+#include "syzygy/tbprobe.h"
+
+#include <atomic>
+#include <chrono>
+#include <climits>
+#include <cmath>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <limits>
+#include <list>
+#include <memory>
+#include <optional>
+#include <random>
+#include <shared_mutex>
+#include <sstream>
+#include <unordered_set>
+
+using namespace std;
+
+namespace Stockfish::Tools
+{
+    // Class to generate sfen with multiple threads
+    struct TrainingDataGenerator
+    {
+        struct Params
+        {
+            // Min and max depths for search during gensfen
+            int search_depth_min = 3;
+            int search_depth_max = -1;
+
+            // Number of the nodes to be searched.
+            // 0 represents no limits.
+            uint64_t nodes = 0;
+
+            // Upper limit of evaluation value of generated situation
+            int eval_limit = 3000;
+
+            // minimum ply with random move
+            // maximum ply with random move
+            // Number of random moves in one station
+            int random_move_minply = 1;
+            int random_move_maxply = 24;
+            int random_move_count = 5;
+
+            // Move kings with a probability of 1/N when randomly moving like Apery software.
+            // When you move the king again, there is a 1/N chance that it will randomly moved
+            // once in the opponent's turn.
+            // Apery has N=2. Specifying 0 here disables this function.
+            int random_move_like_apery = 0;
+
+            // For when using multi pv instead of random move.
+            // random_multi_pv is the number of candidates for MultiPV.
+            // When adopting the move of the candidate move, the difference
+            // between the evaluation value of the move of the 1st place
+            // and the evaluation value of the move of the Nth place is.
+            // Must be in the range random_multi_pv_diff.
+            // random_multi_pv_depth is the search depth for MultiPV.
+            int random_multi_pv = 0;
+            int random_multi_pv_diff = 32000;
+            int random_multi_pv_depth = -1;
+            uint64_t random_multi_pv_nodes = 0;
+
+            // The minimum and maximum ply (number of steps from
+            // the initial phase) of the sfens to write out.
+            int write_minply = 16;
+            int write_maxply = 400;
+
+            uint64_t save_every = std::numeric_limits<uint64_t>::max();
+
+            std::string output_file_name = "training_data";
+
+            SfenOutputType sfen_format = SfenOutputType::Binpack;
+
+            std::string seed;
+
+            bool write_out_draw_game_in_training_data_generation = true;
+            bool detect_draw_by_consecutive_low_score = true;
+            bool detect_draw_by_insufficient_mating_material = true;
+
+            uint64_t num_threads;
+
+            std::string book;
+
+            void enforce_constraints()
+            {
+                search_depth_max = std::max(search_depth_min, search_depth_max);
+
+                // Limit the maximum to a one-stop score. (Otherwise you might not end the loop)
+                eval_limit = std::min(eval_limit, (int)mate_in(2));
+
+                save_every = std::max(save_every, REPORT_STATS_EVERY);
+
+                num_threads = Options["Threads"];
+
+                if (random_multi_pv_depth == -1)
+                    random_multi_pv_depth = search_depth_max;
+
+                if (random_multi_pv_nodes == 0)
+                    random_multi_pv_nodes = nodes;
+            }
+        };
+
+        // Hash to limit the export of identical sfens
+        static constexpr uint64_t GENSFEN_HASH_SIZE = 64 * 1024 * 1024;
+        // It must be 2**N because it will be used as the mask to calculate hash_index.
+        static_assert((GENSFEN_HASH_SIZE& (GENSFEN_HASH_SIZE - 1)) == 0);
+
+        static constexpr uint64_t REPORT_DOT_EVERY = 5000;
+        static constexpr uint64_t REPORT_STATS_EVERY = 200000;
+        static_assert(REPORT_STATS_EVERY % REPORT_DOT_EVERY == 0);
+
+        TrainingDataGenerator(
+            const Params& prm
+        ) :
+            params(prm),
+            sfen_writer(prm.output_file_name, prm.num_threads, prm.save_every, prm.sfen_format)
+        {
+            hash.resize(GENSFEN_HASH_SIZE);
+            prngs.reserve(prm.num_threads);
+            auto seed = prm.seed;
+            for (uint64_t i = 0; i < prm.num_threads; ++i)
+            {
+                prngs.emplace_back(seed);
+                seed = prngs.back().next_random_seed();
+            }
+
+            if (!prm.book.empty())
+            {
+                opening_book = open_opening_book(prm.book, prngs[0]);
+                if (opening_book == nullptr)
+                {
+                    std::cout << "WARNING: Failed to open opening book " << prm.book << ". Falling back to startpos.\n";
+                }
+            }
+
+            // Output seed to veryfy by the user if it's not identical by chance.
+            std::cout << prngs[0] << std::endl;
+        }
+
+        void generate(uint64_t limit, uint64_t limit_seconds);
+
+    private:
+        Params params;
+
+        std::vector<PRNG> prngs;
+
+        std::mutex stats_mutex;
+        TimePoint last_stats_report_time;
+
+        // sfen exporter
+        SfenWriter sfen_writer;
+
+        SynchronizedRegionLogger::Region out;
+
+        vector<Key> hash; // 64MB*sizeof(HASH_KEY) = 512MB
+
+        std::unique_ptr<OpeningBook> opening_book;
+
+        static void set_gensfen_search_limits();
+
+        void generate_worker(
+            Thread& th,
+            std::atomic<uint64_t>& counter,
+            uint64_t limit,
+            uint64_t limit_seconds);
+
+        bool was_seen_before(const Position& pos);
+
+        optional<int8_t> get_current_game_result(
+            Position& pos,
+            const vector<int>& move_hist_scores) const;
+
+        vector<uint8_t> generate_random_move_flags(PRNG& prng);
+
+        optional<Move> choose_random_move(
+            PRNG& prng,
+            Position& pos,
+            std::vector<uint8_t>& random_move_flag,
+            int ply,
+            int& random_move_c);
+
+        bool commit_psv(
+            Thread& th,
+            PSVector& sfens,
+            int8_t lastTurnIsWin,
+            std::atomic<uint64_t>& counter,
+            uint64_t limit,
+            Color result_color);
+
+        void initial_report();
+        void report(uint64_t done, uint64_t new_done);
+
+        void maybe_report(uint64_t done);
+    };
+
+    void TrainingDataGenerator::set_gensfen_search_limits()
+    {
+        // About Search::Limits
+        // Be careful because this member variable is global and affects other threads.
+        auto& limits = Search::Limits;
+
+        // Make the search equivalent to the "go infinite" command. (Because it is troublesome if time management is done)
+        limits.infinite = true;
+
+        // Since PV is an obstacle when displayed, erase it.
+        limits.silent = true;
+
+        // If you use this, it will be compared with the accumulated nodes of each thread. Therefore, do not use it.
+        limits.nodes = 0;
+
+        // depth is also processed by the one passed as an argument of Tools::search().
+        limits.depth = 0;
+    }
+
+    void TrainingDataGenerator::generate(uint64_t limit, uint64_t limit_seconds)
+    {
+        last_stats_report_time = 0;
+
+        initial_report();
+
+        set_gensfen_search_limits();
+
+        std::atomic<uint64_t> counter{0};
+        Threads.execute_with_workers([&counter, limit, limit_seconds, this](Thread& th) {
+            generate_worker(th, counter, limit, limit_seconds);
+        });
+        Threads.wait_for_workers_finished();
+
+        sfen_writer.flush();
+
+        report(counter.load(), counter.load() % REPORT_STATS_EVERY + 1);
+
+        std::cout << std::endl;
+    }
+
+    void TrainingDataGenerator::generate_worker(
+        Thread& th,
+        std::atomic<uint64_t>& counter,
+        uint64_t limit,
+        uint64_t limit_seconds)
+    {
+        // For the time being, it will be treated as a draw
+        // at the maximum number of steps to write.
+        // Maximum StateInfo + Search PV to advance to leaf buffer
+        std::vector<StateInfo, AlignedAllocator<StateInfo>> states(
+            params.write_maxply + MAX_PLY /* == search_depth_min + α */);
+
+        StateInfo si;
+
+        auto& prng = prngs[th.id()];
+
+        // end flag
+        bool quit = false;
+
+        const auto start_time = now();
+
+        const bool frc = Options["UCI_Chess960"];
+        // repeat until the specified number of times
+        while (!quit)
+        {
+            if (static_cast<uint64_t>(now() - start_time) / 1000 >= limit_seconds)
+            {
+                break;
+            }
+
+            // It is necessary to set a dependent thread for Position.
+            // When parallelizing, Threads (since this is a vector<Thread*>,
+            // Do the same for up to Threads[0]...Threads[thread_num-1].
+            auto& pos = th.rootPos;
+            if (opening_book != nullptr)
+            {
+                auto& fen = opening_book->next_fen();
+                pos.set(fen, frc, &si, &th);
+            }
+            else
+            {
+                pos.set(StartFEN, frc, &si, &th);
+            }
+
+            int resign_counter = 0;
+            bool should_resign = prng.rand(10) > 1;
+            // Vector for holding the sfens in the current simulated game.
+            PSVector packed_sfens;
+            packed_sfens.reserve(params.write_maxply + MAX_PLY);
+
+            // Precomputed flags. Used internally by choose_random_move.
+            vector<uint8_t> random_move_flag = generate_random_move_flags(prng);
+
+            // A counter that keeps track of the number of random moves
+            // When random_move_minply == -1, random moves are
+            // performed continuously, so use it at this time.
+            // Used internally by choose_random_move.
+            int actual_random_move_count = 0;
+
+            // Save history of move scores for adjudication
+            vector<int> move_hist_scores;
+
+            auto flush_psv = [&](int8_t result) {
+                quit = commit_psv(th, packed_sfens, result, counter, limit, pos.side_to_move());
+            };
+
+            for (int ply = 0; ; ++ply)
+            {
+                // Current search depth
+                const int depth = params.search_depth_min + (int)prng.rand(params.search_depth_max - params.search_depth_min + 1);
+
+                // Starting search calls init_for_search
+                auto [search_value, search_pv] = Search::search(pos, depth, 1, params.nodes);
+
+                // This has to be performed after search because it needs to know
+                // rootMoves which are filled in init_for_search.
+                const auto result = get_current_game_result(pos, move_hist_scores);
+                if (result.has_value())
+                {
+                    flush_psv(result.value());
+                    break;
+                }
+
+                // Always adjudivate by eval limit.
+                // Also because of this we don't have to check for TB/MATE scores
+                if (abs(search_value) >= params.eval_limit)
+                {
+                    resign_counter++;
+                    if ((should_resign && resign_counter >= 4) || abs(search_value) >= VALUE_KNOWN_WIN) {
+                        flush_psv((search_value >= params.eval_limit) ? 1 : -1);
+                        break;
+                    }
+                }
+                else
+                {
+                    resign_counter = 0;
+                }
+
+                // In case there is no PV and the game was not ended here
+                // there is nothing we can do, we can't continue the game,
+                // we don't know the result, so discard this game.
+                if (search_pv.empty())
+                {
+                    break;
+                }
+
+                // Save the move score for adjudication.
+                move_hist_scores.push_back(search_value);
+
+                // Discard stuff before write_minply is reached
+                // because it can harm training due to overfitting.
+                // Initial positions would be too common.
+                if (ply >= params.write_minply && !was_seen_before(pos))
+                {
+                    auto& psv = packed_sfens.emplace_back();
+
+                    // Here we only write the position data.
+                    // Result is added after the whole game is done.
+                    pos.sfen_pack(psv.sfen, pos.is_chess960());
+
+                    psv.score = search_value;
+                    psv.move = search_pv[0];
+                    psv.gamePly = ply;
+                }
+
+                // Update the next move according to best search result or random move.
+                auto random_move = choose_random_move(prng, pos, random_move_flag, ply, actual_random_move_count);
+                const Move next_move = random_move.has_value() ? *random_move : search_pv[0];
+
+                // We don't have the whole game yet, but it ended,
+                // so the writing process ends and the next game starts.
+                // This shouldn't really happen.
+                if (!is_ok(next_move))
+                {
+                    break;
+                }
+
+                // Do move.
+                pos.do_move(next_move, states[ply]);
+            }
+        }
+    }
+
+    bool TrainingDataGenerator::was_seen_before(const Position& pos)
+    {
+        // Look into the position hashtable to see if the same
+        // position was seen before.
+        // This is a good heuristic to exlude already seen
+        // positions without many false positives.
+        auto key = pos.key();
+        auto hash_index = (size_t)(key & (GENSFEN_HASH_SIZE - 1));
+        auto old_key = hash[hash_index];
+        if (key == old_key)
+        {
+            return true;
+        }
+        else
+        {
+            // Replace with the current key.
+            hash[hash_index] = key;
+            return false;
+        }
+    }
+
+    optional<int8_t> TrainingDataGenerator::get_current_game_result(
+        Position& pos,
+        const vector<int>& move_hist_scores) const
+    {
+        // Variables for draw adjudication.
+        // Todo: Make this as an option.
+
+        // start the adjudication when ply reaches this value
+        constexpr int adj_draw_ply = 80;
+
+        // 4 move scores for each side have to be checked
+        constexpr int adj_draw_cnt = 8;
+
+        // move score in CP
+        constexpr int adj_draw_score = 0;
+
+        // For the time being, it will be treated as a
+        // draw at the maximum number of steps to write.
+        const int ply = move_hist_scores.size();
+
+        // has it reached the max length or is a draw by fifty-move rule
+        // or by 3-fold repetition
+        if (ply >= params.write_maxply
+            || pos.is_fifty_move_draw()
+            || pos.is_three_fold_repetition())
+        {
+            return 0;
+        }
+
+        if(pos.this_thread()->rootMoves.empty())
+        {
+            // If there is no legal move
+            return pos.checkers()
+                ? -1 /* mate */
+                : 0 /* stalemate */;
+        }
+
+        // Adjudicate game to a draw if the last 4 scores of each engine is 0.
+        if (params.detect_draw_by_consecutive_low_score)
+        {
+            if (ply >= adj_draw_ply)
+            {
+                int num_cons_plies_within_draw_score = 0;
+                bool is_adj_draw = false;
+
+                for (auto it = move_hist_scores.rbegin();
+                    it != move_hist_scores.rend(); ++it)
+                {
+                    if (abs(*it) <= adj_draw_score)
+                    {
+                        num_cons_plies_within_draw_score++;
+                    }
+                    else
+                    {
+                        // Draw scores must happen on consecutive plies
+                        break;
+                    }
+
+                    if (num_cons_plies_within_draw_score >= adj_draw_cnt)
+                    {
+                        is_adj_draw = true;
+                        break;
+                    }
+                }
+
+                if (is_adj_draw)
+                {
+                    return 0;
+                }
+            }
+        }
+
+        // Draw by insufficient mating material
+        if (params.detect_draw_by_insufficient_mating_material)
+        {
+            if (pos.count<ALL_PIECES>() <= 4)
+            {
+                int num_pieces = pos.count<ALL_PIECES>();
+
+                // (1) KvK
+                if (num_pieces == 2)
+                {
+                    return 0;
+                }
+
+                // (2) KvK + 1 minor piece
+                if (num_pieces == 3)
+                {
+                    int minor_pc = pos.count<BISHOP>(WHITE) + pos.count<KNIGHT>(WHITE) +
+                        pos.count<BISHOP>(BLACK) + pos.count<KNIGHT>(BLACK);
+                    if (minor_pc == 1)
+                    {
+                        return 0;
+                    }
+                }
+
+                // (3) KBvKB, bishops of the same color
+                else if (num_pieces == 4)
+                {
+                    if (pos.count<BISHOP>(WHITE) == 1 && pos.count<BISHOP>(BLACK) == 1)
+                    {
+                        // Color of bishops is black.
+                        if ((pos.pieces(WHITE, BISHOP) & DarkSquares)
+                            && (pos.pieces(BLACK, BISHOP) & DarkSquares))
+                        {
+                            return 0;
+                        }
+                        // Color of bishops is white.
+                        if ((pos.pieces(WHITE, BISHOP) & ~DarkSquares)
+                            && (pos.pieces(BLACK, BISHOP) & ~DarkSquares))
+                        {
+                            return 0;
+                        }
+                    }
+                }
+            }
+        }
+
+        return nullopt;
+    }
+
+    vector<uint8_t> TrainingDataGenerator::generate_random_move_flags(PRNG& prng)
+    {
+        vector<uint8_t> random_move_flag;
+
+        // Depending on random move selection parameters setup
+        // the array of flags that indicates whether a random move
+        // be taken at a given ply.
+
+        // Make an array like a[0] = 0 ,a[1] = 1, ...
+        // Fisher-Yates shuffle and take out the first N items.
+        // Actually, I only want N pieces, so I only need
+        // to shuffle the first N pieces with Fisher-Yates.
+
+        vector<int> a;
+        a.reserve((size_t)params.random_move_maxply);
+
+        // random_move_minply ,random_move_maxply is specified by 1 origin,
+        // Note that we are handling 0 origin here.
+        for (int i = std::max(params.random_move_minply - 1, 0); i < params.random_move_maxply; ++i)
+        {
+            a.push_back(i);
+        }
+
+        // In case of Apery random move, insert() may be called random_move_count times.
+        // Reserve only the size considering it.
+        random_move_flag.resize((size_t)params.random_move_maxply + params.random_move_count);
+
+        // A random move that exceeds the size() of a[] cannot be applied, so limit it.
+        for (int i = 0; i < std::min(params.random_move_count, (int)a.size()); ++i)
+        {
+            swap(a[i], a[prng.rand((uint64_t)a.size() - i) + i]);
+            random_move_flag[a[i]] = true;
+        }
+
+        return random_move_flag;
+    }
+
+    optional<Move> TrainingDataGenerator::choose_random_move(
+        PRNG& prng,
+        Position& pos,
+        std::vector<uint8_t>& random_move_flag,
+        int ply,
+        int& random_move_c)
+    {
+        optional<Move> random_move;
+
+        // Randomly choose one from legal move
+        if (
+            // 1. Random move of random_move_count times from random_move_minply to random_move_maxply
+            (params.random_move_minply != -1 && ply < (int)random_move_flag.size() && random_move_flag[ply]) ||
+            // 2. A mode to perform random move of random_move_count times after leaving the startpos
+            (params.random_move_minply == -1 && random_move_c < params.random_move_count))
+        {
+            ++random_move_c;
+
+            // It's not a mate, so there should be one legal move...
+            if (params.random_multi_pv == 0)
+            {
+                // Normal random move
+                MoveList<LEGAL> list(pos);
+
+                // I don't really know the goodness and badness of making this the Apery method.
+                if (params.random_move_like_apery == 0
+                    || prng.rand(params.random_move_like_apery) != 0)
+                {
+                    // Normally one move from legal move
+                    random_move = list.at((size_t)prng.rand((uint64_t)list.size()));
+                }
+                else
+                {
+                    // if you can move the king, move the king
+                    Move moves[8]; // Near 8
+                    Move* p = &moves[0];
+                    for (auto& m : list)
+                    {
+                        if (type_of(pos.moved_piece(m)) == KING)
+                        {
+                            *(p++) = m;
+                        }
+                    }
+
+                    size_t n = p - &moves[0];
+                    if (n != 0)
+                    {
+                        // move to move the king
+                        random_move = moves[prng.rand(n)];
+
+                        // In Apery method, at this time there is a 1/2 chance
+                        // that the opponent will also move randomly
+                        if (prng.rand(2) == 0)
+                        {
+                            // Is it a simple hack to add a "1" next to random_move_flag[ply]?
+                            random_move_flag.insert(random_move_flag.begin() + ply + 1, 1, true);
+                        }
+                    }
+                    else
+                    {
+                        // Normally one move from legal move
+                        random_move = list.at((size_t)prng.rand((uint64_t)list.size()));
+                    }
+                }
+            }
+            else
+            {
+                Search::search(pos, params.random_multi_pv_depth, params.random_multi_pv, params.random_multi_pv_nodes);
+
+                // Select one from the top N hands of root Moves
+                auto& rm = pos.this_thread()->rootMoves;
+
+                uint64_t s = min((uint64_t)rm.size(), (uint64_t)params.random_multi_pv);
+                for (uint64_t i = 1; i < s; ++i)
+                {
+                    // The difference from the evaluation value of rm[0] must
+                    // be within the range of random_multi_pv_diff.
+                    // It can be assumed that rm[x].score is arranged in descending order.
+                    if (rm[0].score > rm[i].score + params.random_multi_pv_diff)
+                    {
+                        s = i;
+                        break;
+                    }
+                }
+
+                random_move = rm[prng.rand(s)].pv[0];
+            }
+        }
+
+        return random_move;
+    }
+
+    // Write out the phases loaded in sfens to a file.
+    // result: win/loss in the next phase after the final phase in sfens
+    // 1 when winning. -1 when losing. Pass 0 for a draw.
+    // Return value: true if the specified number of
+    // sfens has already been reached and the process ends.
+    bool TrainingDataGenerator::commit_psv(
+        Thread& th,
+        PSVector& sfens,
+        int8_t result,
+        std::atomic<uint64_t>& counter,
+        uint64_t limit,
+        Color result_color)
+    {
+        if (!params.write_out_draw_game_in_training_data_generation && result == 0)
+        {
+            // We didn't write anything so why quit.
+            return false;
+        }
+
+        auto side_to_move_from_sfen = [](auto& sfen){
+            return (Color)(sfen.sfen.data[0] & 1);
+        };
+
+        // From the final stage (one step before) to the first stage, give information on the outcome of the game for each stage.
+        // The phases stored in sfens are assumed to be continuous (in order).
+        for (auto it = sfens.rbegin(); it != sfens.rend(); ++it)
+        {
+            // The side to move is packed as the lowest bit of the first byte
+            const Color side_to_move = side_to_move_from_sfen(*it);
+            it->game_result = side_to_move == result_color ? result : -result;
+        }
+
+        const bool frc = th.rootPos.is_chess960();
+        // Write sfens in move order to make potential compression easier
+        for (auto& sfen : sfens)
+        {
+            // Skip positions with castling bestmove in FRC so that we don't
+            // need to support it in the trainer.
+            if (frc && type_of((Move)sfen.move) == CASTLING)
+            {
+                continue;
+            }
+
+            // Return true if there is already enough data generated.
+            const auto iter = counter.fetch_add(1);
+            if (iter >= limit)
+                return true;
+
+            // because `iter` was done, now we do one more
+            maybe_report(iter + 1);
+
+            // Write out one sfen.
+            sfen_writer.write(th.id(), sfen);
+        }
+
+        return false;
+    }
+
+    void TrainingDataGenerator::initial_report()
+    {
+        out = sync_region_cout.new_region();
+
+        const auto now_time = now();
+
+        out
+            << '\n'
+            << 0 << " sfens, "
+            << "at " << now_string() << endl;
+
+        last_stats_report_time = now_time;
+
+        out = sync_region_cout.new_region();
+    }
+
+    void TrainingDataGenerator::report(uint64_t done, uint64_t new_done)
+    {
+        const auto now_time = now();
+        const TimePoint elapsed = now_time - last_stats_report_time + 1;
+
+        out
+            << '\n'
+            << done << " sfens, "
+            << new_done * 1000 / elapsed << " sfens/second, "
+            << "at " << now_string() << endl;
+
+        last_stats_report_time = now_time;
+
+        out = sync_region_cout.new_region();
+    }
+
+    void TrainingDataGenerator::maybe_report(uint64_t done)
+    {
+        if (done % REPORT_DOT_EVERY == 0)
+        {
+            std::lock_guard lock(stats_mutex);
+
+            if (last_stats_report_time == 0)
+            {
+                last_stats_report_time = now();
+                out = sync_region_cout.new_region();
+            }
+
+            if (done != 0)
+            {
+                out << '.';
+
+                if (done % REPORT_STATS_EVERY == 0)
+                {
+                    report(done, REPORT_STATS_EVERY);
+                }
+            }
+        }
+    }
+
+    // Command to generate a game record
+    void generate_training_data(istringstream& is)
+    {
+        // Number of generated game records default = 8 billion phases (Ponanza specification)
+        uint64_t loop_max = 8000000000UL;
+        uint64_t time_max = 8000000000UL;
+
+        TrainingDataGenerator::Params params;
+
+        // Add a random number to the end of the file name.
+        bool random_file_name = false;
+        std::string sfen_format = "binpack";
+
+        string token;
+        while (true)
+        {
+            token = "";
+            is >> token;
+            if (token == "")
+                break;
+
+            if (token == "depth")
+            {
+                is >> params.search_depth_min;
+                params.search_depth_max = params.search_depth_min;
+            }
+            else if (token == "min_depth")
+                is >> params.search_depth_min;
+            else if (token == "max_depth")
+                is >> params.search_depth_max;
+            else if (token == "nodes")
+                is >> params.nodes;
+            else if (token == "count")
+                is >> loop_max;
+            else if (token == "max_time_seconds")
+                is >> time_max;
+            else if (token == "max_time_minutes")
+            {
+                is >> time_max;
+                time_max *= 60;
+            }
+            else if (token == "max_time_hours")
+            {
+                is >> time_max;
+                time_max *= 3600;
+            }
+            else if (token == "output_file_name")
+                is >> params.output_file_name;
+            else if (token == "eval_limit")
+                is >> params.eval_limit;
+            else if (token == "random_move_min_ply")
+                is >> params.random_move_minply;
+            else if (token == "random_move_max_ply")
+                is >> params.random_move_maxply;
+            else if (token == "random_move_count")
+                is >> params.random_move_count;
+            else if (token == "random_move_like_apery")
+                is >> params.random_move_like_apery;
+            else if (token == "random_multi_pv")
+                is >> params.random_multi_pv;
+            else if (token == "random_multi_pv_diff")
+                is >> params.random_multi_pv_diff;
+            else if (token == "random_multi_pv_depth")
+                is >> params.random_multi_pv_depth;
+            else if (token == "random_multi_pv_nodes")
+                is >> params.random_multi_pv_nodes;
+            else if (token == "write_min_ply")
+                is >> params.write_minply;
+            else if (token == "write_max_ply")
+                is >> params.write_maxply;
+            else if (token == "save_every")
+                is >> params.save_every;
+            else if (token == "book")
+                is >> params.book;
+            else if (token == "random_file_name")
+                is >> random_file_name;
+            else if (token == "keep_draws")
+                is >> params.write_out_draw_game_in_training_data_generation;
+            else if (token == "adjudicate_draws_by_score")
+                is >> params.detect_draw_by_consecutive_low_score;
+            else if (token == "adjudicate_draws_by_insufficient_material")
+                is >> params.detect_draw_by_insufficient_mating_material;
+            else if (token == "data_format")
+                is >> sfen_format;
+            else if (token == "seed")
+                is >> params.seed;
+            else if (token == "set_recommended_uci_options")
+            {
+                UCI::setoption("Skill Level", "20");
+                UCI::setoption("UCI_LimitStrength", "false");
+                UCI::setoption("PruneAtShallowDepth", "false");
+                UCI::setoption("EnableTranspositionTable", "true");
+            }
+            else
+            {
+                cout << "ERROR: Unknown option " << token << ". Exiting...\n";
+                return;
+            }
+        }
+
+        if (!sfen_format.empty())
+        {
+            if (sfen_format == "bin")
+                params.sfen_format = SfenOutputType::Bin;
+            else if (sfen_format == "binpack")
+                params.sfen_format = SfenOutputType::Binpack;
+            else
+                cout << "WARNING: Unknown sfen format `" << sfen_format << "`. Using bin\n";
+        }
+
+        if (random_file_name)
+        {
+            // Give a random number to output_file_name at this point.
+            // Do not use std::random_device().  Because it always the same integers on MinGW.
+            PRNG r(params.seed);
+
+            // Just in case, reassign the random numbers.
+            for (int i = 0; i < 10; ++i)
+                r.rand(1);
+
+            auto to_hex = [](uint64_t u) {
+                std::stringstream ss;
+                ss << std::hex << u;
+                return ss.str();
+            };
+
+            // I don't want to wear 64bit numbers by accident, so I'next_move going to make a 64bit number 2 just in case.
+            params.output_file_name += "_" + to_hex(r.rand<uint64_t>()) + to_hex(r.rand<uint64_t>());
+        }
+
+        params.enforce_constraints();
+
+        std::cout << "INFO: Executing generate_training_data command\n";
+
+        std::cout << "INFO: Parameters:\n";
+        std::cout
+            << "  - search_depth_min       = " << params.search_depth_min << endl
+            << "  - search_depth_max       = " << params.search_depth_max << endl
+            << "  - nodes                  = " << params.nodes << endl
+            << "  - count                  = " << loop_max << endl
+            << "  - max_time_seconds       = " << time_max << endl
+            << "  - eval_limit             = " << params.eval_limit << endl
+            << "  - num threads (UCI)      = " << params.num_threads << endl
+            << "  - random_move_min_ply    = " << params.random_move_minply << endl
+            << "  - random_move_max_ply    = " << params.random_move_maxply << endl
+            << "  - random_move_count      = " << params.random_move_count << endl
+            << "  - random_move_like_apery = " << params.random_move_like_apery << endl
+            << "  - random_multi_pv        = " << params.random_multi_pv << endl
+            << "  - random_multi_pv_diff   = " << params.random_multi_pv_diff << endl
+            << "  - random_multi_pv_depth  = " << params.random_multi_pv_depth << endl
+            << "  - random_multi_pv_nodes  = " << params.random_multi_pv_nodes << endl
+            << "  - write_min_ply          = " << params.write_minply << endl
+            << "  - write_max_ply          = " << params.write_maxply << endl
+            << "  - book                   = " << params.book << endl
+            << "  - output_file_name       = " << params.output_file_name << endl
+            << "  - save_every             = " << params.save_every << endl
+            << "  - random_file_name       = " << random_file_name << endl
+            << "  - write_drawn_games      = " << params.write_out_draw_game_in_training_data_generation << endl
+            << "  - draw by low score      = " << params.detect_draw_by_consecutive_low_score << endl
+            << "  - draw by insuff. mat.   = " << params.detect_draw_by_insufficient_mating_material << endl;
+
+        // Show if the training data generator uses NNUE.
+        Eval::NNUE::verify();
+
+        Threads.main()->ponder = false;
+
+        TrainingDataGenerator gensfen(params);
+        gensfen.generate(loop_max, time_max);
+
+        std::cout << "INFO: generate_training_data finished." << endl;
+    }
+}

src/tools/training_data_generator.h

@@ -0,0 +1,14 @@
+#ifndef _GENSFEN_H_
+#define _GENSFEN_H_
+
+#include "position.h"
+
+#include <sstream>
+
+namespace Stockfish::Tools {
+
+    // Automatic generation of teacher position
+    void generate_training_data(std::istringstream& is);
+}
+
+#endif

src/tools/training_data_generator_nonpv.cpp

@@ -0,0 +1,498 @@
+#include "training_data_generator_nonpv.h"
+
+#include "sfen_writer.h"
+#include "packed_sfen.h"
+#include "opening_book.h"
+
+#include "misc.h"
+#include "position.h"
+#include "thread.h"
+#include "tt.h"
+#include "uci.h"
+
+#include "extra/nnue_data_binpack_format.h"
+
+#include "nnue/evaluate_nnue.h"
+
+#include "syzygy/tbprobe.h"
+
+#include <atomic>
+#include <chrono>
+#include <climits>
+#include <cmath>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <limits>
+#include <list>
+#include <memory>
+#include <optional>
+#include <random>
+#include <shared_mutex>
+#include <sstream>
+#include <unordered_set>
+
+using namespace std;
+
+namespace Stockfish::Tools
+{
+    // Class to generate sfen with multiple threads
+    struct TrainingDataGeneratorNonPv
+    {
+        struct Params
+        {
+            // The depth for search on the fens gathered during exploration
+            int search_depth = 3;
+
+            // the min/max number of nodes to use for exploration per ply
+            int exploration_min_nodes = 5000;
+            int exploration_max_nodes = 15000;
+
+            // The pct of positions explored that are saved for rescoring
+            float exploration_save_rate = 0.01;
+
+            // Upper limit of evaluation value of generated situation
+            int eval_limit = 4000;
+
+            // the upper limit on evaluation during exploration selfplay
+            int exploration_eval_limit = 4000;
+
+            int exploration_max_ply = 200;
+
+            int exploration_min_pieces = 8;
+
+            std::string output_file_name = "training_data_nonpv";
+
+            SfenOutputType sfen_format = SfenOutputType::Binpack;
+
+            std::string seed;
+
+            int num_threads;
+
+            std::string book;
+
+            bool smart_fen_skipping = false;
+
+            void enforce_constraints()
+            {
+                // Limit the maximum to a one-stop score. (Otherwise you might not end the loop)
+                eval_limit = std::min(eval_limit, (int)mate_in(2));
+                exploration_eval_limit = std::min(eval_limit, (int)mate_in(2));
+                exploration_min_nodes = std::max(100, exploration_min_nodes);
+                exploration_max_nodes = std::max(exploration_min_nodes, exploration_max_nodes);
+
+                num_threads = Options["Threads"];
+            }
+        };
+
+        static constexpr uint64_t REPORT_DOT_EVERY = 5000;
+        static constexpr uint64_t REPORT_STATS_EVERY = 200000;
+        static_assert(REPORT_STATS_EVERY % REPORT_DOT_EVERY == 0);
+
+        TrainingDataGeneratorNonPv(
+            const Params& prm
+        ) :
+            params(prm),
+            prng(prm.seed),
+            sfen_writer(prm.output_file_name, prm.num_threads, std::numeric_limits<uint64_t>::max(), prm.sfen_format)
+        {
+            if (!prm.book.empty())
+            {
+                opening_book = open_opening_book(prm.book, prng);
+                if (opening_book == nullptr)
+                {
+                    std::cout << "WARNING: Failed to open opening book " << prm.book << ". Falling back to startpos.\n";
+                }
+            }
+
+            // Output seed to veryfy by the user if it's not identical by chance.
+            std::cout << prng << std::endl;
+        }
+
+        void generate(uint64_t limit);
+
+    private:
+        Params params;
+
+        PRNG prng;
+
+        std::mutex stats_mutex;
+        TimePoint last_stats_report_time;
+
+        // sfen exporter
+        SfenWriter sfen_writer;
+
+        SynchronizedRegionLogger::Region out;
+
+        std::unique_ptr<OpeningBook> opening_book;
+
+        static void set_gensfen_search_limits();
+
+        void generate_worker(
+            Thread& th,
+            std::atomic<uint64_t>& counter,
+            uint64_t limit);
+
+        bool commit_psv(
+            Thread& th,
+            PSVector& sfens,
+            std::atomic<uint64_t>& counter,
+            uint64_t limit);
+
+        PSVector do_exploration(
+            Thread& th,
+            int count);
+
+        void report(uint64_t done, uint64_t new_done);
+
+        void maybe_report(uint64_t done);
+    };
+
+    void TrainingDataGeneratorNonPv::set_gensfen_search_limits()
+    {
+        // About Search::Limits
+        // Be careful because this member variable is global and affects other threads.
+        auto& limits = Search::Limits;
+
+        // Make the search equivalent to the "go infinite" command. (Because it is troublesome if time management is done)
+        limits.infinite = true;
+
+        // Since PV is an obstacle when displayed, erase it.
+        limits.silent = true;
+
+        // If you use this, it will be compared with the accumulated nodes of each thread. Therefore, do not use it.
+        limits.nodes = 0;
+
+        // depth is also processed by the one passed as an argument of Tools::search().
+        limits.depth = 0;
+    }
+
+    void TrainingDataGeneratorNonPv::generate(uint64_t limit)
+    {
+        last_stats_report_time = 0;
+
+        set_gensfen_search_limits();
+
+        std::atomic<uint64_t> counter{0};
+        Threads.execute_with_workers([&counter, limit, this](Thread& th) {
+            generate_worker(th, counter, limit);
+        });
+        Threads.wait_for_workers_finished();
+
+        sfen_writer.flush();
+
+        if (limit % REPORT_STATS_EVERY != 0)
+        {
+            report(limit, limit % REPORT_STATS_EVERY);
+        }
+
+        std::cout << std::endl;
+    }
+
+    PSVector TrainingDataGeneratorNonPv::do_exploration(
+        Thread& th,
+        int count)
+    {
+        constexpr int max_depth = 30;
+
+        PSVector psv;
+
+        std::vector<StateInfo, AlignedAllocator<StateInfo>> states(
+            max_depth + MAX_PLY /* == search_depth_min + α */);
+
+        th.set_eval_callback([this, &psv](Position& pos) {
+            if ((double)prng.rand<uint64_t>() / std::numeric_limits<uint64_t>::max() < params.exploration_save_rate)
+            {
+                psv.emplace_back();
+                pos.sfen_pack(psv.back().sfen, pos.is_chess960());
+            }
+        });
+
+        auto& pos = th.rootPos;
+        StateInfo si;
+
+        const bool frc = Options["UCI_Chess960"];
+        for (int i = 0; i < count; ++i)
+        {
+            if (opening_book != nullptr)
+            {
+                auto& fen = opening_book->next_fen();
+                pos.set(fen, frc, &si, &th);
+            }
+            else
+            {
+                pos.set(StartFEN, frc, &si, &th);
+            }
+
+            for(int ply = 0; ply < params.exploration_max_ply; ++ply)
+            {
+                auto nodes = prng.rand(params.exploration_max_nodes - params.exploration_min_nodes + 1) + params.exploration_min_nodes;
+
+                auto [search_value, search_pv] = Search::search(pos, max_depth, 1, nodes);
+
+                if (search_pv.empty())
+                {
+                    break;
+                }
+
+                if (std::abs(search_value) > params.exploration_eval_limit)
+                {
+                    break;
+                }
+
+                pos.do_move(search_pv[0], states[ply]);
+
+                if (popcount(pos.pieces()) < params.exploration_min_pieces)
+                {
+                    break;
+                }
+            }
+        }
+
+        th.clear_eval_callback();
+
+        return psv;
+    }
+
+    void TrainingDataGeneratorNonPv::generate_worker(
+        Thread& th,
+        std::atomic<uint64_t>& counter,
+        uint64_t limit)
+    {
+        constexpr int exploration_batch_size = 1;
+
+        StateInfo si;
+
+        PSVector psv;
+
+        // end flag
+        bool quit = false;
+
+        const bool frc = Options["UCI_Chess960"];
+        // repeat until the specified number of times
+        while (!quit)
+        {
+            // It is necessary to set a dependent thread for Position.
+            // When parallelizing, Threads (since this is a vector<Thread*>,
+            // Do the same for up to Threads[0]...Threads[thread_num-1].
+            auto& pos = th.rootPos;
+
+            auto packed_sfens = do_exploration(th, exploration_batch_size);
+            psv.clear();
+
+            for (auto& ps : packed_sfens)
+            {
+                pos.set_from_packed_sfen(ps.sfen, &si, &th, frc);
+                pos.state()->rule50 = 0;
+
+                if (params.smart_fen_skipping && pos.checkers())
+                {
+                    continue;
+                }
+
+                auto [search_value, search_pv] = Search::search(pos, params.search_depth, 1);
+
+                if (search_pv.empty())
+                {
+                    continue;
+                }
+
+                if (std::abs(search_value) > params.eval_limit)
+                {
+                    continue;
+                }
+
+                if (params.smart_fen_skipping && pos.capture_or_promotion(search_pv[0]))
+                {
+                    continue;
+                }
+
+                auto& new_ps = psv.emplace_back();
+                pos.sfen_pack(new_ps.sfen, pos.is_chess960());
+                new_ps.score = search_value;
+                new_ps.move = search_pv[0];
+                new_ps.gamePly = 1;
+                new_ps.game_result = 0;
+                new_ps.padding = 0;
+            }
+
+            quit = commit_psv(th, psv, counter, limit);
+        }
+    }
+
+    // Write out the phases loaded in sfens to a file.
+    // result: win/loss in the next phase after the final phase in sfens
+    // 1 when winning. -1 when losing. Pass 0 for a draw.
+    // Return value: true if the specified number of
+    // sfens has already been reached and the process ends.
+    bool TrainingDataGeneratorNonPv::commit_psv(
+        Thread& th,
+        PSVector& sfens,
+        std::atomic<uint64_t>& counter,
+        uint64_t limit)
+    {
+        const bool frc = th.rootPos.is_chess960();
+        // Write sfens in move order to make potential compression easier
+        for (auto& sfen : sfens)
+        {
+            // Skip positions with castling bestmove in FRC so that we don't
+            // need to support it in the trainer.
+            if (frc && type_of((Move)sfen.move) == CASTLING)
+            {
+                continue;
+            }
+
+            // Return true if there is already enough data generated.
+            const auto iter = counter.fetch_add(1);
+            if (iter >= limit)
+                return true;
+
+            // because `iter` was done, now we do one more
+            maybe_report(iter + 1);
+
+            // Write out one sfen.
+            sfen_writer.write(th.id(), sfen);
+        }
+
+        return false;
+    }
+
+    void TrainingDataGeneratorNonPv::report(uint64_t done, uint64_t new_done)
+    {
+        const auto now_time = now();
+        const TimePoint elapsed = now_time - last_stats_report_time + 1;
+
+        out
+            << endl
+            << done << " sfens, "
+            << new_done * 1000 / elapsed << " sfens/second, "
+            << "at " << now_string() << sync_endl;
+
+        last_stats_report_time = now_time;
+
+        out = sync_region_cout.new_region();
+    }
+
+    void TrainingDataGeneratorNonPv::maybe_report(uint64_t done)
+    {
+        if (done % REPORT_DOT_EVERY == 0)
+        {
+            std::lock_guard lock(stats_mutex);
+
+            if (last_stats_report_time == 0)
+            {
+                last_stats_report_time = now();
+                out = sync_region_cout.new_region();
+            }
+
+            if (done != 0)
+            {
+                out << '.';
+
+                if (done % REPORT_STATS_EVERY == 0)
+                {
+                    report(done, REPORT_STATS_EVERY);
+                }
+            }
+        }
+    }
+
+    // Command to generate a game record
+    void generate_training_data_nonpv(istringstream& is)
+    {
+        // Number of generated game records default = 8 billion phases (Ponanza specification)
+        TrainingDataGeneratorNonPv::Params params;
+
+        uint64_t count = 1'000'000;
+
+        // Add a random number to the end of the file name.
+        std::string sfen_format = "binpack";
+
+        string token;
+        while (true)
+        {
+            token = "";
+            is >> token;
+            if (token == "")
+                break;
+
+            if (token == "depth")
+                is >> params.search_depth;
+            else if (token == "count")
+                is >> count;
+            else if (token == "output_file")
+                is >> params.output_file_name;
+            else if (token == "exploration_eval_limit")
+                is >> params.exploration_eval_limit;
+            else if (token == "eval_limit")
+                is >> params.eval_limit;
+            else if (token == "exploration_min_nodes")
+                is >> params.exploration_min_nodes;
+            else if (token == "exploration_max_nodes")
+                is >> params.exploration_max_nodes;
+            else if (token == "exploration_min_pieces")
+                is >> params.exploration_min_pieces;
+            else if (token == "exploration_save_rate")
+                is >> params.exploration_save_rate;
+            else if (token == "book")
+                is >> params.book;
+            else if (token == "data_format")
+                is >> sfen_format;
+            else if (token == "seed")
+                is >> params.seed;
+            else if (token == "smart_fen_skipping")
+                params.smart_fen_skipping = true;
+            else if (token == "set_recommended_uci_options")
+            {
+                UCI::setoption("Skill Level", "20");
+                UCI::setoption("UCI_LimitStrength", "false");
+                UCI::setoption("PruneAtShallowDepth", "false");
+                UCI::setoption("EnableTranspositionTable", "true");
+            }
+            else
+            {
+                cout << "ERROR: Unknown option " << token << ". Exiting...\n";
+                return;
+            }
+        }
+
+        if (!sfen_format.empty())
+        {
+            if (sfen_format == "bin")
+                params.sfen_format = SfenOutputType::Bin;
+            else if (sfen_format == "binpack")
+                params.sfen_format = SfenOutputType::Binpack;
+            else
+                cout << "WARNING: Unknown sfen format `" << sfen_format << "`. Using bin\n";
+        }
+
+        params.enforce_constraints();
+
+        std::cout << "INFO: Executing generate_training_data_nonpv command\n";
+
+        std::cout << "INFO: Parameters:\n";
+        std::cout
+            << "  - search_depth           = " << params.search_depth << endl
+            << "  - output_file            = " << params.output_file_name << endl
+            << "  - exploration_eval_limit = " << params.exploration_eval_limit << endl
+            << "  - eval_limit             = " << params.eval_limit << endl
+            << "  - exploration_min_nodes  = " << params.exploration_min_nodes << endl
+            << "  - exploration_max_nodes  = " << params.exploration_max_nodes << endl
+            << "  - exploration_min_pieces = " << params.exploration_min_pieces << endl
+            << "  - exploration_save_rate  = " << params.exploration_save_rate << endl
+            << "  - book                   = " << params.book << endl
+            << "  - data_format            = " << sfen_format << endl
+            << "  - seed                   = " << params.seed << endl
+            << "  - count                  = " << count << endl;
+
+        // Show if the training data generator uses NNUE.
+        Eval::NNUE::verify();
+
+        Threads.main()->ponder = false;
+
+        TrainingDataGeneratorNonPv gensfen(params);
+        gensfen.generate(count);
+
+        std::cout << "INFO: generate_training_data_nonpv finished." << endl;
+    }
+}

src/tools/training_data_generator_nonpv.h

@@ -0,0 +1,12 @@
+#ifndef _GENSFEN_NONPV_H_
+#define _GENSFEN_NONPV_H_
+
+#include <sstream>
+
+namespace Stockfish::Tools {
+
+    // Automatic generation of teacher position
+    void generate_training_data_nonpv(std::istringstream& is);
+}
+
+#endif

src/tools/transform.h

@@ -0,0 +1,12 @@
+#ifndef _TRANSFORM_H_
+#define _TRANSFORM_H_
+
+#include <sstream>
+
+namespace Stockfish::Tools {
+
+    void transform(std::istringstream& is);
+
+}
+
+#endif

src/tools/validate_training_data.cpp

@@ -0,0 +1,122 @@
+#include "validate_training_data.h"
+
+#include "uci.h"
+#include "misc.h"
+#include "thread.h"
+#include "position.h"
+#include "tt.h"
+
+#include "extra/nnue_data_binpack_format.h"
+
+#include "nnue/evaluate_nnue.h"
+
+#include "syzygy/tbprobe.h"
+
+#include <sstream>
+#include <fstream>
+#include <unordered_set>
+#include <iomanip>
+#include <list>
+#include <cmath>    // std::exp(),std::pow(),std::log()
+#include <cstring>  // memcpy()
+#include <memory>
+#include <limits>
+#include <optional>
+#include <chrono>
+#include <random>
+#include <regex>
+#include <filesystem>
+
+using namespace std;
+namespace sys = std::filesystem;
+
+namespace Stockfish::Tools
+{
+    static inline const std::string plain_extension = ".plain";
+    static inline const std::string bin_extension = ".bin";
+    static inline const std::string binpack_extension = ".binpack";
+
+    static bool file_exists(const std::string& name)
+    {
+        std::ifstream f(name);
+        return f.good();
+    }
+
+    static bool ends_with(const std::string& lhs, const std::string& end)
+    {
+        if (end.size() > lhs.size()) return false;
+
+        return std::equal(end.rbegin(), end.rend(), lhs.rbegin());
+    }
+
+    static bool is_validation_of_type(
+        const std::string& input_path,
+        const std::string& expected_input_extension)
+    {
+        return ends_with(input_path, expected_input_extension);
+    }
+
+    using ValidateFunctionType = void(std::string inputPath);
+
+    static ValidateFunctionType* get_validate_function(const std::string& input_path)
+    {
+        if (is_validation_of_type(input_path, plain_extension))
+            return binpack::validatePlain;
+
+        if (is_validation_of_type(input_path, bin_extension))
+            return binpack::validateBin;
+
+        if (is_validation_of_type(input_path, binpack_extension))
+            return binpack::validateBinpack;
+
+        return nullptr;
+    }
+
+    static void validate_training_data(const std::string& input_path)
+    {
+        if(!file_exists(input_path))
+        {
+            std::cerr << "Input file does not exist.\n";
+            return;
+        }
+
+        auto func = get_validate_function(input_path);
+        if (func != nullptr)
+        {
+            func(input_path);
+        }
+        else
+        {
+            std::cerr << "Validation of files of this type is not supported.\n";
+        }
+    }
+
+    static void validate_training_data(const std::vector<std::string>& args)
+    {
+        if (args.size() != 1)
+        {
+            std::cerr << "Invalid arguments.\n";
+            std::cerr << "Usage: validate in_path\n";
+            return;
+        }
+
+        validate_training_data(args[0]);
+    }
+
+    void validate_training_data(istringstream& is)
+    {
+        std::vector<std::string> args;
+
+        while (true)
+        {
+            std::string token = "";
+            is >> token;
+            if (token == "")
+                break;
+
+            args.push_back(token);
+        }
+
+        validate_training_data(args);
+    }
+}

src/tools/validate_training_data.h

@@ -0,0 +1,12 @@
+#ifndef _VALIDATE_TRAINING_DATA_H_
+#define _VALIDATE_TRAINING_DATA_H_
+
+#include <vector>
+#include <string>
+#include <sstream>
+
+namespace Stockfish::Tools {
+    void validate_training_data(std::istringstream& is);
+}
+
+#endif

src/tt.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -30,11 +30,16 @@ namespace Stockfish {
 
 TranspositionTable TT; // Our global transposition table
 
+bool TranspositionTable::enable_transposition_table = true;
+
 /// TTEntry::save() populates the TTEntry with a new node's data, possibly
 /// overwriting an old position. Update is not atomic and can be racy.
 
 void TTEntry::save(Key k, Value v, bool pv, Bound b, Depth d, Move m, Value ev) {
 
+  if (!TranspositionTable::enable_transposition_table) {
+      return;
+  }
   // Preserve any existing move for the same position
   if (m || (uint16_t)k != key16)
       move16 = (uint16_t)m;
@@ -87,7 +92,7 @@ void TranspositionTable::clear() {
 
   std::vector<std::thread> threads;
 
-  for (size_t idx = 0; idx < size_t(Options["Threads"]); ++idx)
+  for (size_t idx = 0; idx < Options["Threads"]; ++idx)
   {
       threads.emplace_back([this, idx]() {
 
@@ -98,7 +103,7 @@ void TranspositionTable::clear() {
           // Each thread will zero its part of the hash table
           const size_t stride = size_t(clusterCount / Options["Threads"]),
                        start  = size_t(stride * idx),
-                       len    = idx != size_t(Options["Threads"]) - 1 ?
+                       len    = idx != Options["Threads"] - 1 ?
                                 stride : clusterCount - start;
 
           std::memset(&table[start], 0, len * sizeof(Cluster));
@@ -119,6 +124,11 @@ void TranspositionTable::clear() {
 
 TTEntry* TranspositionTable::probe(const Key key, bool& found) const {
 
+  if (!enable_transposition_table) {
+      found = false;
+      return first_entry(0);
+  }
+
   TTEntry* const tte = first_entry(key);
   const uint16_t key16 = (uint16_t)key;  // Use the low 16 bits as key inside the cluster
 

src/tt.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -92,6 +92,8 @@ public:
     return &table[mul_hi64(key, clusterCount)].entry[0];
   }
 
+  static bool enable_transposition_table;
+
 private:
   friend struct TTEntry;
 

src/tune.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by

src/tune.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -26,8 +26,8 @@
 
 namespace Stockfish {
 
-using Range = std::pair<int, int>; // Option's min-max values
-using RangeFun = Range (int);
+typedef std::pair<int, int> Range; // Option's min-max values
+typedef Range (RangeFun) (int);
 
 // Default Range function, to calculate Option's min-max values
 inline Range default_range(int v) {
@@ -75,7 +75,7 @@ struct SetRange {
 
 class Tune {
 
-  using PostUpdate = void (); // Post-update function
+  typedef void (PostUpdate) (); // Post-update function
 
   Tune() { read_results(); }
   Tune(const Tune&) = delete;

src/types.h

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -103,8 +103,8 @@ constexpr bool Is64Bit = true;
 constexpr bool Is64Bit = false;
 #endif
 
-using Key = uint64_t;
-using Bitboard = uint64_t;
+typedef uint64_t Key;
+typedef uint64_t Bitboard;
 
 constexpr int MAX_MOVES = 256;
 constexpr int MAX_PLY   = 246;
@@ -137,6 +137,8 @@ enum Color {
   WHITE, BLACK, COLOR_NB = 2
 };
 
+constexpr Color Colors[2] = { WHITE, BLACK };
+
 enum CastlingRights {
   NO_CASTLING,
   WHITE_OO,
@@ -186,9 +188,6 @@ enum Value : int {
   VALUE_MATE_IN_MAX_PLY  =  VALUE_MATE - MAX_PLY,
   VALUE_MATED_IN_MAX_PLY = -VALUE_MATE_IN_MAX_PLY,
 
-  // In the code, we make the assumption that these values
-  // are such that non_pawn_material() can be used to uniquely
-  // identify the material on the board.
   PawnValueMg   = 126,   PawnValueEg   = 208,
   KnightValueMg = 781,   KnightValueEg = 854,
   BishopValueMg = 825,   BishopValueEg = 915,
@@ -218,7 +217,7 @@ constexpr Value PieceValue[PHASE_NB][PIECE_NB] = {
     VALUE_ZERO, PawnValueEg, KnightValueEg, BishopValueEg, RookValueEg, QueenValueEg, VALUE_ZERO, VALUE_ZERO }
 };
 
-using Depth = int;
+typedef int Depth;
 
 enum : int {
   DEPTH_QS_CHECKS     =  0,
@@ -416,10 +415,6 @@ inline Color color_of(Piece pc) {
   return Color(pc >> 3);
 }
 
-constexpr bool is_ok(Move m) {
-  return m != MOVE_NONE && m != MOVE_NULL;
-}
-
 constexpr bool is_ok(Square s) {
   return s >= SQ_A1 && s <= SQ_H8;
 }
@@ -449,15 +444,18 @@ constexpr Direction pawn_push(Color c) {
 }
 
 constexpr Square from_sq(Move m) {
-  assert(is_ok(m));
   return Square((m >> 6) & 0x3F);
 }
 
 constexpr Square to_sq(Move m) {
-  assert(is_ok(m));
   return Square(m & 0x3F);
 }
 
+// Return relative square when turning the board 180 degrees
+constexpr Square rotate180(Square sq) {
+    return (Square)(sq ^ 0x3F);
+}
+
 constexpr int from_to(Move m) {
   return m & 0xFFF;
 }
@@ -479,6 +477,10 @@ constexpr Move make(Square from, Square to, PieceType pt = KNIGHT) {
   return Move(T + ((pt - KNIGHT) << 12) + (from << 6) + to);
 }
 
+constexpr bool is_ok(Move m) {
+  return from_sq(m) != to_sq(m); // Catch MOVE_NULL and MOVE_NONE
+}
+
 /// Based on a congruential pseudo random number generator
 constexpr Key make_key(uint64_t seed) {
   return seed * 6364136223846793005ULL + 1442695040888963407ULL;

src/uci.cpp

@@ -1,6 +1,6 @@
 /*
   Stockfish, a UCI chess playing engine derived from Glaurung 2.1
-  Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
+  Copyright (C) 2004-2022 The Stockfish developers (see AUTHORS file)
 
   Stockfish is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
@@ -22,32 +22,36 @@
 #include <sstream>
 #include <string>
 
-#include "benchmark.h"
+#include "nnue/evaluate_nnue.h"
 #include "evaluate.h"
 #include "movegen.h"
 #include "position.h"
 #include "search.h"
+#include "syzygy/tbprobe.h"
 #include "thread.h"
 #include "timeman.h"
 #include "tt.h"
 #include "uci.h"
-#include "syzygy/tbprobe.h"
-#include "nnue/evaluate_nnue.h"
+
+#include "tools/validate_training_data.h"
+#include "tools/training_data_generator.h"
+#include "tools/training_data_generator_nonpv.h"
+#include "tools/convert.h"
+#include "tools/transform.h"
+#include "tools/stats.h"
 
 using namespace std;
 
 namespace Stockfish {
 
+extern vector<string> setup_bench(const Position&, istream&);
+
 namespace {
 
-  // FEN string for the initial position in standard chess
-  const char* StartFEN = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1";
-
-
-  // position() is called when the engine receives the "position" UCI command.
-  // It sets up the position that is described in the given FEN string ("fen") or
-  // the initial position ("startpos") and then makes the moves given in the following
-  // move list ("moves").
+  // position() is called when engine receives the "position" UCI command.
+  // The function sets up the position described in the given FEN string ("fen")
+  // or the starting position ("startpos") and then makes the moves given in the
+  // following move list ("moves").
 
   void position(Position& pos, istringstream& is, StateListPtr& states) {
 
@@ -59,7 +63,7 @@ namespace {
     if (token == "startpos")
     {
         fen = StartFEN;
-        is >> token; // Consume the "moves" token, if any
+        is >> token; // Consume "moves" token if any
     }
     else if (token == "fen")
         while (is >> token && token != "moves")
@@ -67,10 +71,10 @@ namespace {
     else
         return;
 
-    states = StateListPtr(new std::deque<StateInfo>(1)); // Drop the old state and create a new one
+    states = StateListPtr(new std::deque<StateInfo>(1)); // Drop old and create a new one
     pos.set(fen, Options["UCI_Chess960"], &states->back(), Threads.main());
 
-    // Parse the move list, if any
+    // Parse move list (if any)
     while (is >> token && (m = UCI::to_move(pos, token)) != MOVE_NONE)
     {
         states->emplace_back();
@@ -78,8 +82,8 @@ namespace {
     }
   }
 
-  // trace_eval() prints the evaluation of the current position, consistent with
-  // the UCI options set so far.
+  // trace_eval() prints the evaluation for the current position, consistent with the UCI
+  // options set so far.
 
   void trace_eval(Position& pos) {
 
@@ -93,33 +97,30 @@ namespace {
   }
 
 
-  // setoption() is called when the engine receives the "setoption" UCI command.
-  // The function updates the UCI option ("name") to the given value ("value").
+  // setoption() is called when engine receives the "setoption" UCI command. The
+  // function updates the UCI option ("name") to the given value ("value").
 
-  void setoption(istringstream& is) {
+  void setoption_from_stream(istringstream& is) {
 
     string token, name, value;
 
-    is >> token; // Consume the "name" token
+    is >> token; // Consume "name" token
 
-    // Read the option name (can contain spaces)
+    // Read option name (can contain spaces)
     while (is >> token && token != "value")
         name += (name.empty() ? "" : " ") + token;
 
-    // Read the option value (can contain spaces)
+    // Read option value (can contain spaces)
     while (is >> token)
         value += (value.empty() ? "" : " ") + token;
 
-    if (Options.count(name))
-        Options[name] = value;
-    else
-        sync_cout << "No such option: " << name << sync_endl;
+    UCI::setoption(name, value);
   }
 
 
-  // go() is called when the engine receives the "go" UCI command. The function
-  // sets the thinking time and other parameters from the input string, then starts
-  // with a search.
+  // go() is called when engine receives the "go" UCI command. The function sets
+  // the thinking time and other parameters from the input string, then starts
+  // the search.
 
   void go(Position& pos, istringstream& is, StateListPtr& states) {
 
@@ -127,7 +128,7 @@ namespace {
     string token;
     bool ponderMode = false;
 
-    limits.startTime = now(); // The search starts as early as possible
+    limits.startTime = now(); // As early as possible!
 
     while (is >> token)
         if (token == "searchmoves") // Needs to be the last command on the line
@@ -151,9 +152,9 @@ namespace {
   }
 
 
-  // bench() is called when the engine receives the "bench" command.
-  // Firstly, a list of UCI commands is set up according to the bench
-  // parameters, then it is run one by one, printing a summary at the end.
+  // bench() is called when engine receives the "bench" command. Firstly
+  // a list of UCI commands is setup according to bench parameters, then
+  // it is run one by one printing a summary at the end.
 
   void bench(Position& pos, istream& args, StateListPtr& states) {
 
@@ -161,7 +162,7 @@ namespace {
     uint64_t num, nodes = 0, cnt = 1;
 
     vector<string> list = setup_bench(pos, args);
-    num = count_if(list.begin(), list.end(), [](const string& s) { return s.find("go ") == 0 || s.find("eval") == 0; });
+    num = count_if(list.begin(), list.end(), [](string s) { return s.find("go ") == 0 || s.find("eval") == 0; });
 
     TimePoint elapsed = now();
 
@@ -182,14 +183,14 @@ namespace {
             else
                trace_eval(pos);
         }
-        else if (token == "setoption")  setoption(is);
+        else if (token == "setoption")  setoption_from_stream(is);
         else if (token == "position")   position(pos, is, states);
-        else if (token == "ucinewgame") { Search::clear(); elapsed = now(); } // Search::clear() may take a while
+        else if (token == "ucinewgame") { Search::clear(); elapsed = now(); } // Search::clear() may take some while
     }
 
     elapsed = now() - elapsed + 1; // Ensure positivity to avoid a 'divide by zero'
 
-    dbg_print();
+    dbg_print(); // Just before exiting
 
     cerr << "\n==========================="
          << "\nTotal time (ms) : " << elapsed
@@ -197,40 +198,110 @@ namespace {
          << "\nNodes/second    : " << 1000 * nodes / elapsed << endl;
   }
 
-  // The win rate model returns the probability of winning (in per mille units) given an
-  // eval and a game ply. It fits the LTC fishtest statistics rather accurately.
+} // namespace
+
+namespace UCI {
+
+  void setoption(const std::string& name, const std::string& value)
+  {
+      if (Options.count(name))
+          Options[name] = value;
+      else
+          sync_cout << "No such option: " << name << sync_endl;
+  }
+
+  // The win rate model returns the probability (per mille) of winning given an eval
+  // and a game-ply. The model fits rather accurately the LTC fishtest statistics.
   int win_rate_model(Value v, int ply) {
 
-     // The model only captures up to 240 plies, so limit the input and then rescale
+     // The model captures only up to 240 plies, so limit input (and rescale)
      double m = std::min(240, ply) / 64.0;
 
-     // The coefficients of a third-order polynomial fit is based on the fishtest data
-     // for two parameters that need to transform eval to the argument of a logistic
-     // function.
-     constexpr double as[] = {   0.38036525,   -2.82015070,   23.17882135,  307.36768407};
-     constexpr double bs[] = {  -2.29434733,   13.27689788,  -14.26828904,   63.45318330 };
-
-     // Enforce that NormalizeToPawnValue corresponds to a 50% win rate at ply 64
-     static_assert(UCI::NormalizeToPawnValue == int(as[0] + as[1] + as[2] + as[3]));
-
+     // Coefficients of a 3rd order polynomial fit based on fishtest data
+     // for two parameters needed to transform eval to the argument of a
+     // logistic function.
+     double as[] = {-1.17202460e-01, 5.94729104e-01, 1.12065546e+01, 1.22606222e+02};
+     double bs[] = {-1.79066759,  11.30759193, -17.43677612,  36.47147479};
      double a = (((as[0] * m + as[1]) * m + as[2]) * m) + as[3];
      double b = (((bs[0] * m + bs[1]) * m + bs[2]) * m) + bs[3];
 
-     // Transform the eval to centipawns with limited range
-     double x = std::clamp(double(v), -4000.0, 4000.0);
+     // Transform eval to centipawns with limited range
+     double x = std::clamp(double(100 * v) / PawnValueEg, -1000.0, 1000.0);
 
-     // Return the win rate in per mille units rounded to the nearest value
+     // Return win rate in per mille (rounded to nearest)
      return int(0.5 + 1000 / (1 + std::exp((a - x) / b)));
   }
 
 } // namespace
 
+// --------------------
+// Call qsearch(),search() directly for testing
+// --------------------
 
-/// UCI::loop() waits for a command from the stdin, parses it and then calls the appropriate
-/// function. It also intercepts an end-of-file (EOF) indication from the stdin to ensure a
-/// graceful exit if the GUI dies unexpectedly. When called with some command-line arguments,
-/// like running 'bench', the function returns immediately after the command is executed.
-/// In addition to the UCI ones, some additional debug commands are also supported.
+void qsearch_cmd(Position& pos)
+{
+  cout << "qsearch : ";
+  auto pv = Search::qsearch(pos);
+  cout << "Value = " << pv.first << " , " << UCI::value(pv.first) << " , PV = ";
+  for (auto m : pv.second)
+    cout << UCI::move(m, false) << " ";
+  cout << endl;
+}
+
+void search_cmd(Position& pos, istringstream& is)
+{
+  string token;
+  int depth = 1;
+  int multi_pv = (int)Options["MultiPV"];
+  while (is >> token)
+  {
+    if (token == "depth")
+      is >> depth;
+    if (token == "multipv")
+      is >> multi_pv;
+  }
+
+  cout << "search depth = " << depth << " , multi_pv = " << multi_pv << " : ";
+  auto pv = Search::search(pos, depth, multi_pv);
+  cout << "Value = " << pv.first << " , " << UCI::value(pv.first) << " , PV = ";
+  for (auto m : pv.second)
+    cout << UCI::move(m, false) << " ";
+  cout << endl;
+}
+
+void search_mcts_cmd(Position& pos, istringstream& is)
+{
+  string token;
+  int nodes = 1000;
+  int leafDepth = 3;
+  float explorationFactor = 0.25f;
+  while (is >> token)
+  {
+    if (token == "nodes")
+      is >> nodes;
+    if (token == "leaf_depth")
+      is >> leafDepth;
+    if (token == "exploration_factor")
+      is >> explorationFactor;
+  }
+
+  cout << "search nodes = " << nodes << " , leaf_depth = " << leafDepth << " :\n";
+  auto continuations = Search::MCTS::search_mcts_multipv(pos, nodes, leafDepth, explorationFactor);
+  for (auto&& [numVisits, value, actionValue, pv] : continuations)
+  {
+    cout << "NumVisits = " << numVisits << " , Value = " << UCI::value(value) << " , ActionValue = " << actionValue << " , PV = ";
+    for (auto m : pv)
+      cout << UCI::move(m, false) << " ";
+    cout << endl;
+  }
+  cout << endl;
+}
+
+/// UCI::loop() waits for a command from stdin, parses it and calls the appropriate
+/// function. Also intercepts EOF from stdin to ensure gracefully exiting if the
+/// GUI dies unexpectedly. When called with some command line arguments, e.g. to
+/// run 'bench', once the command is executed the function returns immediately.
+/// In addition to the UCI ones, also some additional debug commands are supported.
 
 void UCI::loop(int argc, char* argv[]) {
 
@@ -244,38 +315,38 @@ void UCI::loop(int argc, char* argv[]) {
       cmd += std::string(argv[i]) + " ";
 
   do {
-      if (argc == 1 && !getline(cin, cmd)) // Wait for an input or an end-of-file (EOF) indication
+      if (argc == 1 && !getline(cin, cmd)) // Block here waiting for input or EOF
           cmd = "quit";
 
       istringstream is(cmd);
 
-      token.clear(); // Avoid a stale if getline() returns nothing or a blank line
+      token.clear(); // Avoid a stale if getline() returns empty or blank line
       is >> skipws >> token;
 
       if (    token == "quit"
           ||  token == "stop")
           Threads.stop = true;
 
-      // The GUI sends 'ponderhit' to tell that the user has played the expected move.
-      // So, 'ponderhit' is sent if pondering was done on the same move that the user
-      // has played. The search should continue, but should also switch from pondering
-      // to the normal search.
+      // The GUI sends 'ponderhit' to tell us the user has played the expected move.
+      // So 'ponderhit' will be sent if we were told to ponder on the same move the
+      // user has played. We should continue searching but switch from pondering to
+      // normal search.
       else if (token == "ponderhit")
-          Threads.main()->ponder = false; // Switch to the normal search
+          Threads.main()->ponder = false; // Switch to normal search
 
       else if (token == "uci")
           sync_cout << "id name " << engine_info(true)
                     << "\n"       << Options
                     << "\nuciok"  << sync_endl;
 
-      else if (token == "setoption")  setoption(is);
+      else if (token == "setoption")  setoption_from_stream(is);
       else if (token == "go")         go(pos, is, states);
       else if (token == "position")   position(pos, is, states);
       else if (token == "ucinewgame") Search::clear();
       else if (token == "isready")    sync_cout << "readyok" << sync_endl;
 
-      // Add custom non-UCI commands, mainly for debugging purposes.
-      // These commands must not be used during a search!
+      // Additional custom non-UCI commands, mainly for debugging.
+      // Do not use these commands during a search!
       else if (token == "flip")     pos.flip();
       else if (token == "bench")    bench(pos, is, states);
       else if (token == "d")        sync_cout << pos << sync_endl;
@@ -289,25 +360,45 @@ void UCI::loop(int argc, char* argv[]) {
               filename = f;
           Eval::NNUE::save_eval(filename);
       }
+      else if (token == "generate_training_data") Tools::generate_training_data(is);
+      else if (token == "generate_training_data_nonpv") Tools::generate_training_data_nonpv(is);
+      else if (token == "convert") Tools::convert(is);
+      else if (token == "validate_training_data") Tools::validate_training_data(is);
+      else if (token == "convert_bin") Tools::convert_bin(is);
+      else if (token == "convert_plain") Tools::convert_plain(is);
+      else if (token == "convert_bin_from_pgn_extract") Tools::convert_bin_from_pgn_extract(is);
+      else if (token == "transform") Tools::transform(is);
+      else if (token == "gather_statistics") Tools::Stats::gather_statistics(is);
+
+      // Command to call qsearch(),search() directly for testing
+      else if (token == "qsearch") qsearch_cmd(pos);
+      else if (token == "search_mcts") search_mcts_cmd(pos, is);
+      else if (token == "search") search_cmd(pos, is);
+      else if (token == "tasktest")
+      {
+        Threads.execute_with_workers([](auto& th) {
+          std::cout << th.id() << '\n';
+        });
+      }
       else if (token == "--help" || token == "help" || token == "--license" || token == "license")
-          sync_cout << "\nStockfish is a powerful chess engine for playing and analyzing."
-                       "\nIt is released as free software licensed under the GNU GPLv3 License."
-                       "\nStockfish is normally used with a graphical user interface (GUI) and implements"
-                       "\nthe Universal Chess Interface (UCI) protocol to communicate with a GUI, an API, etc."
-                       "\nFor any further information, visit https://github.com/official-stockfish/Stockfish#readme"
-                       "\nor read the corresponding README.md and Copying.txt files distributed along with this program.\n" << sync_endl;
+          sync_cout << "\nStockfish is a powerful chess engine and free software licensed under the GNU GPLv3."
+                       "\nStockfish is normally used with a separate graphical user interface (GUI)."
+                       "\nStockfish implements the universal chess interface (UCI) to exchange information."
+                       "\nFor further information see https://github.com/official-stockfish/Stockfish#readme"
+                       "\nor the corresponding README.md and Copying.txt files distributed with this program.\n" << sync_endl;
       else if (!token.empty() && token[0] != '#')
           sync_cout << "Unknown command: '" << cmd << "'. Type help for more information." << sync_endl;
 
-  } while (token != "quit" && argc == 1); // The command-line arguments are one-shot
+  } while (token != "quit" && argc == 1); // Command line args are one-shot
 }
 
 
-/// UCI::value() converts a Value to a string by adhering to the UCI protocol specification:
+/// UCI::value() converts a Value to a string suitable for use with the UCI
+/// protocol specification:
 ///
 /// cp <x>    The score from the engine's point of view in centipawns.
-/// mate <y>  Mate in 'y' moves (not plies). If the engine is getting mated,
-///           uses negative values for 'y'.
+/// mate <y>  Mate in y moves, not plies. If the engine is getting mated
+///           use negative values for y.
 
 string UCI::value(Value v) {
 
@@ -315,13 +406,8 @@ string UCI::value(Value v) {
 
   stringstream ss;
 
-  if (abs(v) < VALUE_TB_WIN_IN_MAX_PLY)
-      ss << "cp " << v * 100 / NormalizeToPawnValue;
-  else if (abs(v) < VALUE_MATE_IN_MAX_PLY)
-  {
-      const int ply = VALUE_MATE_IN_MAX_PLY - 1 - std::abs(v);  // recompute ss->ply
-      ss << "cp " << (v > 0 ? 20000 - ply : -20000 + ply);
-  }
+  if (abs(v) < VALUE_MATE_IN_MAX_PLY)
+      ss << "cp " << v * 100 / PawnValueEg;
   else
       ss << "mate " << (v > 0 ? VALUE_MATE - v + 1 : -VALUE_MATE - v) / 2;
 
@@ -329,8 +415,8 @@ string UCI::value(Value v) {
 }
 
 
-/// UCI::wdl() reports the win-draw-loss (WDL) statistics given an evaluation
-/// and a game ply based on the data gathered for fishtest LTC games.
+/// UCI::wdl() report WDL statistics given an evaluation and a game ply, based on
+/// data gathered for fishtest LTC games.
 
 string UCI::wdl(Value v, int ply) {
 
@@ -353,21 +439,21 @@ std::string UCI::square(Square s) {
 
 
 /// UCI::move() converts a Move to a string in coordinate notation (g1f3, a7a8q).
-/// The only special case is castling where the e1g1 notation is printed in
-/// standard chess mode and in e1h1 notation it is printed in Chess960 mode.
-/// Internally, all castling moves are always encoded as 'king captures rook'.
+/// The only special case is castling, where we print in the e1g1 notation in
+/// normal chess mode, and in e1h1 notation in chess960 mode. Internally all
+/// castling moves are always encoded as 'king captures rook'.
 
 string UCI::move(Move m, bool chess960) {
 
+  Square from = from_sq(m);
+  Square to = to_sq(m);
+
   if (m == MOVE_NONE)
       return "(none)";
 
   if (m == MOVE_NULL)
       return "0000";
 
-  Square from = from_sq(m);
-  Square to = to_sq(m);
-
   if (type_of(m) == CASTLING && !chess960)
       to = make_square(to > from ? FILE_G : FILE_C, rank_of(from));
 
@@ -385,8 +471,8 @@ string UCI::move(Move m, bool chess960) {
 
 Move UCI::to_move(const Position& pos, string& str) {
 
-  if (str.length() == 5)
-      str[4] = char(tolower(str[4])); // The promotion piece character must be lowercased
+  if (str.length() == 5) // Junior could send promotion piece in uppercase
+      str[4] = char(tolower(str[4]));
 
   for (const auto& m : MoveList<LEGAL>(pos))
       if (str == UCI::move(m, pos.is_chess960()))