PascalCase -> snake_case for consistency with the rest of the codebase.

This commit is contained in:
Tomasz Sobczyk
2020-10-14 22:42:58 +02:00
committed by nodchip
parent 2398d34e87
commit 146a6b056e
37 changed files with 844 additions and 737 deletions
+14 -14
View File
@@ -964,7 +964,7 @@ namespace Learner
// Lock the evaluation function so that it is not used during updating. // Lock the evaluation function so that it is not used during updating.
lock_guard<shared_timed_mutex> write_lock(nn_mutex); lock_guard<shared_timed_mutex> write_lock(nn_mutex);
Eval::NNUE::UpdateParameters(); Eval::NNUE::update_parameters();
} }
++epoch; ++epoch;
@@ -998,7 +998,7 @@ namespace Learner
// loss calculation // loss calculation
calc_loss(thread_id, done); calc_loss(thread_id, done);
Eval::NNUE::CheckHealth(); Eval::NNUE::check_health();
// Make a note of how far you have totaled. // Make a note of how far you have totaled.
sr.last_done = sr.total_done; sr.last_done = sr.total_done;
@@ -1127,7 +1127,7 @@ namespace Learner
learn_sum_entropy_win += learn_entropy_win; learn_sum_entropy_win += learn_entropy_win;
learn_sum_entropy += learn_entropy; learn_sum_entropy += learn_entropy;
Eval::NNUE::AddExample(pos, rootColor, ps, 1.0); Eval::NNUE::add_example(pos, rootColor, ps, 1.0);
// Since the processing is completed, the counter of the processed number is incremented // Since the processing is completed, the counter of the processed number is incremented
sr.total_done++; sr.total_done++;
@@ -1194,7 +1194,7 @@ namespace Learner
{ {
cout << " < best (" << best_loss << "), accepted" << endl; cout << " < best (" << best_loss << "), accepted" << endl;
best_loss = latest_loss; best_loss = latest_loss;
best_nn_directory = Path::Combine((std::string)Options["EvalSaveDir"], dir_name); best_nn_directory = Path::combine((std::string)Options["EvalSaveDir"], dir_name);
trials = newbob_num_trials; trials = newbob_num_trials;
if (tot >= last_lr_drop + auto_lr_drop) if (tot >= last_lr_drop + auto_lr_drop)
@@ -1207,13 +1207,13 @@ namespace Learner
{ {
cout << " < best (" << best_loss << "), accepted" << endl; cout << " < best (" << best_loss << "), accepted" << endl;
best_loss = latest_loss; best_loss = latest_loss;
best_nn_directory = Path::Combine((std::string)Options["EvalSaveDir"], dir_name); best_nn_directory = Path::combine((std::string)Options["EvalSaveDir"], dir_name);
trials = newbob_num_trials; trials = newbob_num_trials;
} }
else else
{ {
cout << " >= best (" << best_loss << "), rejected" << endl; cout << " >= best (" << best_loss << "), rejected" << endl;
best_nn_directory = Path::Combine((std::string)Options["EvalSaveDir"], dir_name); best_nn_directory = Path::combine((std::string)Options["EvalSaveDir"], dir_name);
if (--trials > 0 && !is_final) if (--trials > 0 && !is_final)
{ {
@@ -1713,14 +1713,14 @@ namespace Learner
// Display learning game file // Display learning game file
if (target_dir != "") if (target_dir != "")
{ {
string kif_base_dir = Path::Combine(base_dir, target_dir); string kif_base_dir = Path::combine(base_dir, target_dir);
namespace sys = std::filesystem; namespace sys = std::filesystem;
sys::path p(kif_base_dir); // Origin of enumeration sys::path p(kif_base_dir); // Origin of enumeration
std::for_each(sys::directory_iterator(p), sys::directory_iterator(), std::for_each(sys::directory_iterator(p), sys::directory_iterator(),
[&](const sys::path& path) { [&](const sys::path& path) {
if (sys::is_regular_file(path)) if (sys::is_regular_file(path))
filenames.push_back(Path::Combine(target_dir, path.filename().generic_string())); filenames.push_back(Path::combine(target_dir, path.filename().generic_string()));
}); });
} }
@@ -1814,7 +1814,7 @@ namespace Learner
// order so I'll reverse it here. I'm sorry. // order so I'll reverse it here. I'm sorry.
for (auto it = filenames.rbegin(); it != filenames.rend(); ++it) for (auto it = filenames.rbegin(); it != filenames.rend(); ++it)
{ {
sr.filenames.push_back(Path::Combine(base_dir, *it)); sr.filenames.push_back(Path::combine(base_dir, *it));
} }
} }
@@ -1858,9 +1858,9 @@ namespace Learner
set_learning_search_limits(); set_learning_search_limits();
cout << "init_training.." << endl; cout << "init_training.." << endl;
Eval::NNUE::InitializeTraining(seed); Eval::NNUE::initialize_training(seed);
Eval::NNUE::SetBatchSize(nn_batch_size); Eval::NNUE::set_batch_size(nn_batch_size);
Eval::NNUE::SetOptions(nn_options); Eval::NNUE::set_options(nn_options);
if (newbob_decay != 1.0 && !Options["SkipLoadingEval"]) { if (newbob_decay != 1.0 && !Options["SkipLoadingEval"]) {
// Save the current net to [EvalSaveDir]\original. // Save the current net to [EvalSaveDir]\original.
Eval::NNUE::save_eval("original"); Eval::NNUE::save_eval("original");
@@ -1868,7 +1868,7 @@ namespace Learner
// Set the folder above to best_nn_directory so that the trainer can // Set the folder above to best_nn_directory so that the trainer can
// resotre the network parameters from the original net file. // resotre the network parameters from the original net file.
learn_think.best_nn_directory = learn_think.best_nn_directory =
Path::Combine(Options["EvalSaveDir"], "original"); Path::combine(Options["EvalSaveDir"], "original");
} }
cout << "init done." << endl; cout << "init done." << endl;
@@ -1925,7 +1925,7 @@ namespace Learner
// Start learning. // Start learning.
learn_think.go_think(); learn_think.go_think();
Eval::NNUE::FinalizeNet(); Eval::NNUE::finalize_net();
// Save once at the end. // Save once at the end.
learn_think.save(true); learn_think.save(true);
+2 -2
View File
@@ -299,7 +299,7 @@ struct Path
{ {
// Combine the path name and file name and return it. // Combine the path name and file name and return it.
// If the folder name is not an empty string, append it if there is no'/' or'\\' at the end. // If the folder name is not an empty string, append it if there is no'/' or'\\' at the end.
static std::string Combine(const std::string& folder, const std::string& filename) static std::string combine(const std::string& folder, const std::string& filename)
{ {
if (folder.length() >= 1 && *folder.rbegin() != '/' && *folder.rbegin() != '\\') if (folder.length() >= 1 && *folder.rbegin() != '/' && *folder.rbegin() != '\\')
return folder + "/" + filename; return folder + "/" + filename;
@@ -308,7 +308,7 @@ struct Path
} }
// Get the file name part (excluding the folder name) from the full path expression. // Get the file name part (excluding the folder name) from the full path expression.
static std::string GetFileName(const std::string& path) static std::string get_file_name(const std::string& path)
{ {
// I don't know which "\" or "/" is used. // I don't know which "\" or "/" is used.
auto path_index1 = path.find_last_of("\\") + 1; auto path_index1 = path.find_last_of("\\") + 1;
+291 -256
View File
@@ -1,303 +1,338 @@
/* /*
Stockfish, a UCI chess playing engine derived from Glaurung 2.1 Stockfish, a UCI chess playing engine derived from Glaurung 2.1
Copyright (C) 2004-2020 The Stockfish developers (see AUTHORS file) Copyright (C) 2004-2020 The Stockfish developers (see AUTHORS file)
Stockfish is free software: you can redistribute it and/or modify Stockfish is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
(at your option) any later version. (at your option) any later version.
Stockfish is distributed in the hope that it will be useful, Stockfish is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details. GNU General Public License for more details.
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
// Code for calculating NNUE evaluation function // Code for calculating NNUE evaluation function
#include "evaluate_nnue.h"
#include "position.h"
#include "misc.h"
#include "uci.h"
#include "types.h"
#include <iostream> #include <iostream>
#include <string> #include <string>
#include <fstream> #include <fstream>
#include <set> #include <set>
#include "../position.h"
#include "../misc.h"
#include "../uci.h"
#include "../types.h"
#include "evaluate_nnue.h"
namespace Eval::NNUE { namespace Eval::NNUE {
const uint32_t kpp_board_index[PIECE_NB][COLOR_NB] = { const uint32_t kpp_board_index[PIECE_NB][COLOR_NB] = {
// convention: W - us, B - them // convention: W - us, B - them
// viewed from other side, W and B are reversed // viewed from other side, W and B are reversed
{ PS_NONE, PS_NONE }, { PS_NONE, PS_NONE },
{ PS_W_PAWN, PS_B_PAWN }, { PS_W_PAWN, PS_B_PAWN },
{ PS_W_KNIGHT, PS_B_KNIGHT }, { PS_W_KNIGHT, PS_B_KNIGHT },
{ PS_W_BISHOP, PS_B_BISHOP }, { PS_W_BISHOP, PS_B_BISHOP },
{ PS_W_ROOK, PS_B_ROOK }, { PS_W_ROOK, PS_B_ROOK },
{ PS_W_QUEEN, PS_B_QUEEN }, { PS_W_QUEEN, PS_B_QUEEN },
{ PS_W_KING, PS_B_KING }, { PS_W_KING, PS_B_KING },
{ PS_NONE, PS_NONE }, { PS_NONE, PS_NONE },
{ PS_NONE, PS_NONE }, { PS_NONE, PS_NONE },
{ PS_B_PAWN, PS_W_PAWN }, { PS_B_PAWN, PS_W_PAWN },
{ PS_B_KNIGHT, PS_W_KNIGHT }, { PS_B_KNIGHT, PS_W_KNIGHT },
{ PS_B_BISHOP, PS_W_BISHOP }, { PS_B_BISHOP, PS_W_BISHOP },
{ PS_B_ROOK, PS_W_ROOK }, { PS_B_ROOK, PS_W_ROOK },
{ PS_B_QUEEN, PS_W_QUEEN }, { PS_B_QUEEN, PS_W_QUEEN },
{ PS_B_KING, PS_W_KING }, { PS_B_KING, PS_W_KING },
{ PS_NONE, PS_NONE } { PS_NONE, PS_NONE }
}; };
// Input feature converter // Input feature converter
LargePagePtr<FeatureTransformer> feature_transformer; LargePagePtr<FeatureTransformer> feature_transformer;
// Evaluation function // Evaluation function
AlignedPtr<Network> network; AlignedPtr<Network> network;
// Evaluation function file name // Evaluation function file name
std::string fileName; std::string fileName;
// Saved evaluation function file name // Saved evaluation function file name
std::string savedfileName = "nn.bin"; std::string savedfileName = "nn.bin";
// Get a string that represents the structure of the evaluation function // Get a string that represents the structure of the evaluation function
std::string GetArchitectureString() { std::string get_architecture_string() {
return "Features=" + FeatureTransformer::GetStructureString() + return "Features=" + FeatureTransformer::get_structure_string() +
",Network=" + Network::GetStructureString(); ",Network=" + Network::get_structure_string();
}
UseNNUEMode useNNUE;
std::string eval_file_loaded = "None";
namespace Detail {
// Initialize the evaluation function parameters
template <typename T>
void Initialize(AlignedPtr<T>& pointer) {
pointer.reset(reinterpret_cast<T*>(std_aligned_alloc(alignof(T), sizeof(T))));
std::memset(pointer.get(), 0, sizeof(T));
}
template <typename T>
void Initialize(LargePagePtr<T>& pointer) {
static_assert(alignof(T) <= 4096, "aligned_large_pages_alloc() may fail for such a big alignment requirement of T");
pointer.reset(reinterpret_cast<T*>(aligned_large_pages_alloc(sizeof(T))));
std::memset(pointer.get(), 0, sizeof(T));
}
// Read evaluation function parameters
template <typename T>
bool ReadParameters(std::istream& stream, T& reference) {
std::uint32_t header;
header = read_little_endian<std::uint32_t>(stream);
if (!stream || header != T::GetHashValue()) return false;
return reference.ReadParameters(stream);
}
// write evaluation function parameters
template <typename T>
bool WriteParameters(std::ostream& stream, const AlignedPtr<T>& pointer) {
constexpr std::uint32_t header = T::GetHashValue();
stream.write(reinterpret_cast<const char*>(&header), sizeof(header));
return pointer->WriteParameters(stream);
}
template <typename T>
bool WriteParameters(std::ostream& stream, const LargePagePtr<T>& pointer) {
constexpr std::uint32_t header = T::GetHashValue();
stream.write(reinterpret_cast<const char*>(&header), sizeof(header));
return pointer->WriteParameters(stream);
}
} // namespace Detail
// Initialize the evaluation function parameters
void Initialize() {
Detail::Initialize(feature_transformer);
Detail::Initialize(network);
}
// Read network header
bool ReadHeader(std::istream& stream, std::uint32_t* hash_value, std::string* architecture)
{
std::uint32_t version, size;
version = read_little_endian<std::uint32_t>(stream);
*hash_value = read_little_endian<std::uint32_t>(stream);
size = read_little_endian<std::uint32_t>(stream);
if (!stream || version != kVersion) return false;
architecture->resize(size);
stream.read(&(*architecture)[0], size);
return !stream.fail();
}
// write the header
bool WriteHeader(std::ostream& stream,
std::uint32_t hash_value, const std::string& architecture) {
stream.write(reinterpret_cast<const char*>(&kVersion), sizeof(kVersion));
stream.write(reinterpret_cast<const char*>(&hash_value), sizeof(hash_value));
const std::uint32_t size = static_cast<std::uint32_t>(architecture.size());
stream.write(reinterpret_cast<const char*>(&size), sizeof(size));
stream.write(architecture.data(), size);
return !stream.fail();
}
// Read network parameters
bool ReadParameters(std::istream& stream) {
std::uint32_t hash_value;
std::string architecture;
if (!ReadHeader(stream, &hash_value, &architecture)) return false;
if (hash_value != kHashValue) return false;
if (!Detail::ReadParameters(stream, *feature_transformer)) return false;
if (!Detail::ReadParameters(stream, *network)) return false;
return stream && stream.peek() == std::ios::traits_type::eof();
}
// write evaluation function parameters
bool WriteParameters(std::ostream& stream) {
if (!WriteHeader(stream, kHashValue, GetArchitectureString())) return false;
if (!Detail::WriteParameters(stream, feature_transformer)) return false;
if (!Detail::WriteParameters(stream, network)) return false;
return !stream.fail();
}
// Evaluation function. Perform differential calculation.
Value evaluate(const Position& pos) {
alignas(kCacheLineSize) TransformedFeatureType
transformed_features[FeatureTransformer::kBufferSize];
feature_transformer->Transform(pos, transformed_features);
alignas(kCacheLineSize) char buffer[Network::kBufferSize];
const auto output = network->Propagate(transformed_features, buffer);
return static_cast<Value>(output[0] / FV_SCALE);
}
// Load eval, from a file stream or a memory stream
bool load_eval(std::string name, std::istream& stream) {
Initialize();
fileName = name;
return ReadParameters(stream);
}
static UseNNUEMode nnue_mode_from_option(const UCI::Option& mode)
{
if (mode == "false")
return UseNNUEMode::False;
else if (mode == "true")
return UseNNUEMode::True;
else if (mode == "pure")
return UseNNUEMode::Pure;
return UseNNUEMode::False;
}
void init() {
useNNUE = nnue_mode_from_option(Options["Use NNUE"]);
if (Options["SkipLoadingEval"] || useNNUE == UseNNUEMode::False)
{
eval_file_loaded.clear();
return;
} }
std::string eval_file = std::string(Options["EvalFile"]); UseNNUEMode useNNUE;
std::string eval_file_loaded = "None";
namespace Detail {
// Initialize the evaluation function parameters
template <typename T>
void initialize(AlignedPtr<T>& pointer) {
pointer.reset(reinterpret_cast<T*>(std_aligned_alloc(alignof(T), sizeof(T))));
std::memset(pointer.get(), 0, sizeof(T));
}
template <typename T>
void initialize(LargePagePtr<T>& pointer) {
static_assert(alignof(T) <= 4096, "aligned_large_pages_alloc() may fail for such a big alignment requirement of T");
pointer.reset(reinterpret_cast<T*>(aligned_large_pages_alloc(sizeof(T))));
std::memset(pointer.get(), 0, sizeof(T));
}
// Read evaluation function parameters
template <typename T>
bool read_parameters(std::istream& stream, T& reference) {
std::uint32_t header;
header = read_little_endian<std::uint32_t>(stream);
if (!stream || header != T::get_hash_value())
return false;
return reference.read_parameters(stream);
}
// write evaluation function parameters
template <typename T>
bool write_parameters(std::ostream& stream, const AlignedPtr<T>& pointer) {
constexpr std::uint32_t header = T::get_hash_value();
stream.write(reinterpret_cast<const char*>(&header), sizeof(header));
return pointer->write_parameters(stream);
}
template <typename T>
bool write_parameters(std::ostream& stream, const LargePagePtr<T>& pointer) {
constexpr std::uint32_t header = T::get_hash_value();
stream.write(reinterpret_cast<const char*>(&header), sizeof(header));
return pointer->write_parameters(stream);
}
} // namespace Detail
// Initialize the evaluation function parameters
void initialize() {
Detail::initialize(feature_transformer);
Detail::initialize(network);
}
// Read network header
bool read_header(std::istream& stream, std::uint32_t* hash_value, std::string* architecture)
{
std::uint32_t version, size;
version = read_little_endian<std::uint32_t>(stream);
*hash_value = read_little_endian<std::uint32_t>(stream);
size = read_little_endian<std::uint32_t>(stream);
if (!stream || version != kVersion)
return false;
architecture->resize(size);
stream.read(&(*architecture)[0], size);
return !stream.fail();
}
// write the header
bool write_header(std::ostream& stream,
std::uint32_t hash_value, const std::string& architecture) {
stream.write(reinterpret_cast<const char*>(&kVersion), sizeof(kVersion));
stream.write(reinterpret_cast<const char*>(&hash_value), sizeof(hash_value));
const std::uint32_t size = static_cast<std::uint32_t>(architecture.size());
stream.write(reinterpret_cast<const char*>(&size), sizeof(size));
stream.write(architecture.data(), size);
return !stream.fail();
}
// Read network parameters
bool read_parameters(std::istream& stream) {
std::uint32_t hash_value;
std::string architecture;
if (!read_header(stream, &hash_value, &architecture))
return false;
if (hash_value != kHashValue)
return false;
if (!Detail::read_parameters(stream, *feature_transformer))
return false;
if (!Detail::read_parameters(stream, *network))
return false;
return stream && stream.peek() == std::ios::traits_type::eof();
}
// write evaluation function parameters
bool write_parameters(std::ostream& stream) {
if (!write_header(stream, kHashValue, get_architecture_string()))
return false;
if (!Detail::write_parameters(stream, feature_transformer))
return false;
if (!Detail::write_parameters(stream, network))
return false;
return !stream.fail();
}
// Evaluation function. Perform differential calculation.
Value evaluate(const Position& pos) {
alignas(kCacheLineSize) TransformedFeatureType
transformed_features[FeatureTransformer::kBufferSize];
feature_transformer->transform(pos, transformed_features);
alignas(kCacheLineSize) char buffer[Network::kBufferSize];
const auto output = network->propagate(transformed_features, buffer);
return static_cast<Value>(output[0] / FV_SCALE);
}
// Load eval, from a file stream or a memory stream
bool load_eval(std::string name, std::istream& stream) {
initialize();
fileName = name;
return read_parameters(stream);
}
static UseNNUEMode nnue_mode_from_option(const UCI::Option& mode)
{
if (mode == "false")
return UseNNUEMode::False;
else if (mode == "true")
return UseNNUEMode::True;
else if (mode == "pure")
return UseNNUEMode::Pure;
return UseNNUEMode::False;
}
void init() {
useNNUE = nnue_mode_from_option(Options["Use NNUE"]);
if (Options["SkipLoadingEval"] || useNNUE == UseNNUEMode::False)
{
eval_file_loaded.clear();
return;
}
std::string eval_file = std::string(Options["EvalFile"]);
#if defined(DEFAULT_NNUE_DIRECTORY) #if defined(DEFAULT_NNUE_DIRECTORY)
#define stringify2(x) #x #define stringify2(x) #x
#define stringify(x) stringify2(x) #define stringify(x) stringify2(x)
std::vector<std::string> dirs = { "" , CommandLine::binaryDirectory , stringify(DEFAULT_NNUE_DIRECTORY) }; std::vector<std::string> dirs = { "" , CommandLine::binaryDirectory , stringify(DEFAULT_NNUE_DIRECTORY) };
#else #else
std::vector<std::string> dirs = { "" , CommandLine::binaryDirectory }; std::vector<std::string> dirs = { "" , CommandLine::binaryDirectory };
#endif #endif
for (std::string directory : dirs) for (std::string directory : dirs)
if (eval_file_loaded != eval_file)
{ {
std::ifstream stream(directory + eval_file, std::ios::binary); if (eval_file_loaded != eval_file)
if (load_eval(eval_file, stream))
{ {
sync_cout << "info string Loaded eval file " << directory + eval_file << sync_endl; std::ifstream stream(directory + eval_file, std::ios::binary);
eval_file_loaded = eval_file; if (load_eval(eval_file, stream))
} {
else sync_cout << "info string Loaded eval file " << directory + eval_file << sync_endl;
{ eval_file_loaded = eval_file;
sync_cout << "info string ERROR: failed to load eval file " << directory + eval_file << sync_endl; }
eval_file_loaded.clear(); else
{
sync_cout << "info string ERROR: failed to load eval file " << directory + eval_file << sync_endl;
eval_file_loaded.clear();
}
} }
} }
#undef stringify2 #undef stringify2
#undef stringify #undef stringify
}
/// NNUE::verify() verifies that the last net used was loaded successfully
void verify_eval_file_loaded() {
std::string eval_file = std::string(Options["EvalFile"]);
if (useNNUE != UseNNUEMode::False && eval_file_loaded != eval_file)
{
UCI::OptionsMap defaults;
UCI::init(defaults);
std::string msg1 = "If the UCI option \"Use NNUE\" is set to true, network evaluation parameters compatible with the engine must be available.";
std::string msg2 = "The option is set to true, but the network file " + eval_file + " was not loaded successfully.";
std::string msg3 = "The UCI option EvalFile might need to specify the full path, including the directory name, to the network file.";
std::string msg4 = "The default net can be downloaded from: https://tests.stockfishchess.org/api/nn/" + std::string(defaults["EvalFile"]);
std::string msg5 = "The engine will be terminated now.";
sync_cout << "info string ERROR: " << msg1 << sync_endl;
sync_cout << "info string ERROR: " << msg2 << sync_endl;
sync_cout << "info string ERROR: " << msg3 << sync_endl;
sync_cout << "info string ERROR: " << msg4 << sync_endl;
sync_cout << "info string ERROR: " << msg5 << sync_endl;
std::exit(EXIT_FAILURE);
} }
if (useNNUE != UseNNUEMode::False) /// NNUE::verify() verifies that the last net used was loaded successfully
sync_cout << "info string NNUE evaluation using " << eval_file << " enabled" << sync_endl; void verify_eval_file_loaded() {
else
sync_cout << "info string classical evaluation enabled" << sync_endl;
}
/// In training we override eval file so this is useful. std::string eval_file = std::string(Options["EvalFile"]);
void verify_any_net_loaded() {
if (!Options["SkipLoadingEval"] && useNNUE != UseNNUEMode::False && eval_file_loaded.empty()) if (useNNUE != UseNNUEMode::False && eval_file_loaded != eval_file)
{ {
UCI::OptionsMap defaults; UCI::OptionsMap defaults;
UCI::init(defaults); UCI::init(defaults);
std::string msg1 = "If the UCI option \"Use NNUE\" is set to true, network evaluation parameters compatible with the engine must be available."; std::string msg1 = "If the UCI option \"Use NNUE\" is set to true, network evaluation parameters compatible with the engine must be available.";
std::string msg2 = "The option is set to true, but the network file was not loaded successfully."; std::string msg2 = "The option is set to true, but the network file " + eval_file + " was not loaded successfully.";
std::string msg3 = "The UCI option EvalFile might need to specify the full path, including the directory name, to the network file."; std::string msg3 = "The UCI option EvalFile might need to specify the full path, including the directory name, to the network file.";
std::string msg5 = "The engine will be terminated now."; std::string msg4 = "The default net can be downloaded from: https://tests.stockfishchess.org/api/nn/" + std::string(defaults["EvalFile"]);
std::string msg5 = "The engine will be terminated now.";
sync_cout << "info string ERROR: " << msg1 << sync_endl; sync_cout << "info string ERROR: " << msg1 << sync_endl;
sync_cout << "info string ERROR: " << msg2 << sync_endl; sync_cout << "info string ERROR: " << msg2 << sync_endl;
sync_cout << "info string ERROR: " << msg3 << sync_endl; sync_cout << "info string ERROR: " << msg3 << sync_endl;
sync_cout << "info string ERROR: " << msg5 << sync_endl; sync_cout << "info string ERROR: " << msg4 << sync_endl;
sync_cout << "info string ERROR: " << msg5 << sync_endl;
std::exit(EXIT_FAILURE); std::exit(EXIT_FAILURE);
}
if (useNNUE != UseNNUEMode::False)
sync_cout << "info string NNUE evaluation using " << eval_file << " enabled" << sync_endl;
else
sync_cout << "info string classical evaluation enabled" << sync_endl;
} }
if (useNNUE != UseNNUEMode::False) /// In training we override eval file so this is useful.
sync_cout << "info string NNUE evaluation using " << eval_file_loaded << " enabled" << sync_endl; void verify_any_net_loaded() {
else
sync_cout << "info string classical evaluation enabled" << sync_endl; if (!Options["SkipLoadingEval"] && useNNUE != UseNNUEMode::False && eval_file_loaded.empty())
} {
UCI::OptionsMap defaults;
UCI::init(defaults);
std::string msg1 = "If the UCI option \"Use NNUE\" is set to true, network evaluation parameters compatible with the engine must be available.";
std::string msg2 = "The option is set to true, but the network file was not loaded successfully.";
std::string msg3 = "The UCI option EvalFile might need to specify the full path, including the directory name, to the network file.";
std::string msg5 = "The engine will be terminated now.";
sync_cout << "info string ERROR: " << msg1 << sync_endl;
sync_cout << "info string ERROR: " << msg2 << sync_endl;
sync_cout << "info string ERROR: " << msg3 << sync_endl;
sync_cout << "info string ERROR: " << msg5 << sync_endl;
std::exit(EXIT_FAILURE);
}
if (useNNUE != UseNNUEMode::False)
sync_cout << "info string NNUE evaluation using " << eval_file_loaded << " enabled" << sync_endl;
else
sync_cout << "info string classical evaluation enabled" << sync_endl;
}
} // namespace Eval::NNUE } // namespace Eval::NNUE
+6 -6
View File
@@ -37,7 +37,7 @@ namespace Eval::NNUE {
// Hash value of evaluation function structure // Hash value of evaluation function structure
constexpr std::uint32_t kHashValue = constexpr std::uint32_t kHashValue =
FeatureTransformer::GetHashValue() ^ Network::GetHashValue(); FeatureTransformer::get_hash_value() ^ Network::get_hash_value();
// Deleter for automating release of memory area // Deleter for automating release of memory area
template <typename T> template <typename T>
@@ -79,21 +79,21 @@ namespace Eval::NNUE {
extern std::string eval_file_loaded; extern std::string eval_file_loaded;
// Get a string that represents the structure of the evaluation function // Get a string that represents the structure of the evaluation function
std::string GetArchitectureString(); std::string get_architecture_string();
// read the header // read the header
bool ReadHeader(std::istream& stream, bool read_header(std::istream& stream,
std::uint32_t* hash_value, std::string* architecture); std::uint32_t* hash_value, std::string* architecture);
// write the header // write the header
bool WriteHeader(std::ostream& stream, bool write_header(std::ostream& stream,
std::uint32_t hash_value, const std::string& architecture); std::uint32_t hash_value, const std::string& architecture);
// read evaluation function parameters // read evaluation function parameters
bool ReadParameters(std::istream& stream); bool read_parameters(std::istream& stream);
// write evaluation function parameters // write evaluation function parameters
bool WriteParameters(std::ostream& stream); bool write_parameters(std::ostream& stream);
Value evaluate(const Position& pos); Value evaluate(const Position& pos);
bool load_eval(std::string name, std::istream& stream); bool load_eval(std::string name, std::istream& stream);
+32 -32
View File
@@ -44,9 +44,9 @@ namespace Eval::NNUE {
std::shared_ptr<Trainer<Network>> trainer; std::shared_ptr<Trainer<Network>> trainer;
// Tell the learner options such as hyperparameters // Tell the learner options such as hyperparameters
void SendMessages(std::vector<Message> messages) { void send_messages(std::vector<Message> messages) {
for (auto& message : messages) { for (auto& message : messages) {
trainer->SendMessage(&message); trainer->send_message(&message);
assert(message.num_receivers > 0); assert(message.num_receivers > 0);
} }
} }
@@ -54,31 +54,31 @@ namespace Eval::NNUE {
} // namespace } // namespace
// Initialize learning // Initialize learning
void InitializeTraining(const std::string& seed) { void initialize_training(const std::string& seed) {
std::cout << "Initializing NN training for " std::cout << "Initializing NN training for "
<< GetArchitectureString() << std::endl; << get_architecture_string() << std::endl;
assert(feature_transformer); assert(feature_transformer);
assert(network); assert(network);
trainer = Trainer<Network>::Create(network.get(), feature_transformer.get()); trainer = Trainer<Network>::create(network.get(), feature_transformer.get());
rng.seed(PRNG(seed).rand<uint64_t>()); rng.seed(PRNG(seed).rand<uint64_t>());
if (Options["SkipLoadingEval"]) { if (Options["SkipLoadingEval"]) {
trainer->Initialize(rng); trainer->initialize(rng);
} }
} }
// set the number of samples in the mini-batch // set the number of samples in the mini-batch
void SetBatchSize(uint64_t size) { void set_batch_size(uint64_t size) {
assert(size > 0); assert(size > 0);
batch_size = size; batch_size = size;
} }
// Set options such as hyperparameters // Set options such as hyperparameters
void SetOptions(const std::string& options) { void set_options(const std::string& options) {
std::vector<Message> messages; std::vector<Message> messages;
for (const auto& option : Split(options, ',')) { for (const auto& option : Algo::split(options, ',')) {
const auto fields = Split(option, '='); const auto fields = Algo::split(option, '=');
assert(fields.size() == 1 || fields.size() == 2); assert(fields.size() == 1 || fields.size() == 2);
if (fields.size() == 1) { if (fields.size() == 1) {
@@ -88,30 +88,30 @@ namespace Eval::NNUE {
} }
} }
SendMessages(std::move(messages)); send_messages(std::move(messages));
} }
// Reread the evaluation function parameters for learning from the file // Reread the evaluation function parameters for learning from the file
void RestoreParameters(const std::string& dir_name) { void restore_parameters(const std::string& dir_name) {
const std::string file_name = Path::Combine(dir_name, NNUE::savedfileName); const std::string file_name = Path::combine(dir_name, NNUE::savedfileName);
std::ifstream stream(file_name, std::ios::binary); std::ifstream stream(file_name, std::ios::binary);
#ifndef NDEBUG #ifndef NDEBUG
bool result = bool result =
#endif #endif
ReadParameters(stream); read_parameters(stream);
#ifndef NDEBUG #ifndef NDEBUG
assert(result); assert(result);
#endif #endif
SendMessages({{"reset"}}); send_messages({{"reset"}});
} }
void FinalizeNet() { void finalize_net() {
SendMessages({{"clear_unobserved_feature_weights"}}); send_messages({{"clear_unobserved_feature_weights"}});
} }
// Add 1 sample of learning data // Add 1 sample of learning data
void AddExample(Position& pos, Color rootColor, void add_example(Position& pos, Color rootColor,
const Learner::PackedSfenValue& psv, double weight) { const Learner::PackedSfenValue& psv, double weight) {
Example example; Example example;
@@ -126,7 +126,7 @@ namespace Eval::NNUE {
Features::IndexList active_indices[2]; Features::IndexList active_indices[2];
for (const auto trigger : kRefreshTriggers) { for (const auto trigger : kRefreshTriggers) {
RawFeatures::AppendActiveIndices(pos, trigger, active_indices); RawFeatures::append_active_indices(pos, trigger, active_indices);
} }
if (pos.side_to_move() != WHITE) { if (pos.side_to_move() != WHITE) {
@@ -136,9 +136,9 @@ namespace Eval::NNUE {
for (const auto color : Colors) { for (const auto color : Colors) {
std::vector<TrainingFeature> training_features; std::vector<TrainingFeature> training_features;
for (const auto base_index : active_indices[color]) { for (const auto base_index : active_indices[color]) {
static_assert(Features::Factorizer<RawFeatures>::GetDimensions() < static_assert(Features::Factorizer<RawFeatures>::get_dimensions() <
(1 << TrainingFeature::kIndexBits), ""); (1 << TrainingFeature::kIndexBits), "");
Features::Factorizer<RawFeatures>::AppendTrainingFeatures( Features::Factorizer<RawFeatures>::append_training_features(
base_index, &training_features); base_index, &training_features);
} }
@@ -147,7 +147,7 @@ namespace Eval::NNUE {
auto& unique_features = example.training_features[color]; auto& unique_features = example.training_features[color];
for (const auto& feature : training_features) { for (const auto& feature : training_features) {
if (!unique_features.empty() && if (!unique_features.empty() &&
feature.GetIndex() == unique_features.back().GetIndex()) { feature.get_index() == unique_features.back().get_index()) {
unique_features.back() += feature; unique_features.back() += feature;
} else { } else {
@@ -161,7 +161,7 @@ namespace Eval::NNUE {
} }
// update the evaluation function parameters // update the evaluation function parameters
void UpdateParameters() { void update_parameters() {
assert(batch_size > 0); assert(batch_size > 0);
const auto learning_rate = static_cast<LearnFloatType>( const auto learning_rate = static_cast<LearnFloatType>(
@@ -173,30 +173,30 @@ namespace Eval::NNUE {
std::vector<Example> batch(examples.end() - batch_size, examples.end()); std::vector<Example> batch(examples.end() - batch_size, examples.end());
examples.resize(examples.size() - batch_size); examples.resize(examples.size() - batch_size);
const auto network_output = trainer->Propagate(batch); const auto network_output = trainer->propagate(batch);
std::vector<LearnFloatType> gradients(batch.size()); std::vector<LearnFloatType> gradients(batch.size());
for (std::size_t b = 0; b < batch.size(); ++b) { for (std::size_t b = 0; b < batch.size(); ++b) {
const auto shallow = static_cast<Value>(Round<std::int32_t>( const auto shallow = static_cast<Value>(round<std::int32_t>(
batch[b].sign * network_output[b] * kPonanzaConstant)); batch[b].sign * network_output[b] * kPonanzaConstant));
const auto& psv = batch[b].psv; const auto& psv = batch[b].psv;
const double gradient = batch[b].sign * Learner::calc_grad(shallow, psv); const double gradient = batch[b].sign * Learner::calc_grad(shallow, psv);
gradients[b] = static_cast<LearnFloatType>(gradient * batch[b].weight); gradients[b] = static_cast<LearnFloatType>(gradient * batch[b].weight);
} }
trainer->Backpropagate(gradients.data(), learning_rate); trainer->backpropagate(gradients.data(), learning_rate);
} }
SendMessages({{"quantize_parameters"}}); send_messages({{"quantize_parameters"}});
} }
// Check if there are any problems with learning // Check if there are any problems with learning
void CheckHealth() { void check_health() {
SendMessages({{"check_health"}}); send_messages({{"check_health"}});
} }
// save merit function parameters to a file // save merit function parameters to a file
void save_eval(std::string dir_name) { void save_eval(std::string dir_name) {
auto eval_dir = Path::Combine(Options["EvalSaveDir"], dir_name); auto eval_dir = Path::combine(Options["EvalSaveDir"], dir_name);
std::cout << "save_eval() start. folder = " << eval_dir << std::endl; std::cout << "save_eval() start. folder = " << eval_dir << std::endl;
// mkdir() will fail if this folder already exists, but // mkdir() will fail if this folder already exists, but
@@ -204,12 +204,12 @@ namespace Eval::NNUE {
// Also, assume that the folders up to EvalSaveDir have been dug. // Also, assume that the folders up to EvalSaveDir have been dug.
std::filesystem::create_directories(eval_dir); std::filesystem::create_directories(eval_dir);
const std::string file_name = Path::Combine(eval_dir, NNUE::savedfileName); const std::string file_name = Path::combine(eval_dir, NNUE::savedfileName);
std::ofstream stream(file_name, std::ios::binary); std::ofstream stream(file_name, std::ios::binary);
#ifndef NDEBUG #ifndef NDEBUG
bool result = bool result =
#endif #endif
WriteParameters(stream); write_parameters(stream);
#ifndef NDEBUG #ifndef NDEBUG
assert(result); assert(result);
#endif #endif
+12 -9
View File
@@ -7,28 +7,31 @@
namespace Eval::NNUE { namespace Eval::NNUE {
// Initialize learning // Initialize learning
void InitializeTraining(const std::string& seed); void initialize_training(const std::string& seed);
// set the number of samples in the mini-batch // set the number of samples in the mini-batch
void SetBatchSize(uint64_t size); void set_batch_size(uint64_t size);
// Set options such as hyperparameters // Set options such as hyperparameters
void SetOptions(const std::string& options); void set_options(const std::string& options);
// Reread the evaluation function parameters for learning from the file // Reread the evaluation function parameters for learning from the file
void RestoreParameters(const std::string& dir_name); void restore_parameters(const std::string& dir_name);
// Add 1 sample of learning data // Add 1 sample of learning data
void AddExample(Position& pos, Color rootColor, void add_example(
const Learner::PackedSfenValue& psv, double weight); Position& pos,
Color rootColor,
const Learner::PackedSfenValue& psv,
double weight);
// update the evaluation function parameters // update the evaluation function parameters
void UpdateParameters(); void update_parameters();
// Check if there are any problems with learning // Check if there are any problems with learning
void CheckHealth(); void check_health();
void FinalizeNet(); void finalize_net();
void save_eval(std::string suffix); void save_eval(std::string suffix);
} // namespace Eval::NNUE } // namespace Eval::NNUE
+10 -5
View File
@@ -5,8 +5,11 @@
namespace Eval::NNUE::Features { namespace Eval::NNUE::Features {
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
void CastlingRight::AppendActiveIndices( void CastlingRight::append_active_indices(
const Position& pos, Color perspective, IndexList* active) { const Position& pos,
Color perspective,
IndexList* active) {
// do nothing if array size is small to avoid compiler warning // do nothing if array size is small to avoid compiler warning
if (RawFeatures::kMaxActiveDimensions < kMaxActiveDimensions) return; if (RawFeatures::kMaxActiveDimensions < kMaxActiveDimensions) return;
@@ -29,9 +32,11 @@ namespace Eval::NNUE::Features {
} }
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
void CastlingRight::AppendChangedIndices( void CastlingRight::append_changed_indices(
const Position& pos, Color perspective, const Position& pos,
IndexList* removed, IndexList* /* added */) { Color perspective,
IndexList* removed,
IndexList* /* added */) {
int previous_castling_rights = pos.state()->previous->castlingRights; int previous_castling_rights = pos.state()->previous->castlingRights;
int current_castling_rights = pos.state()->castlingRights; int current_castling_rights = pos.state()->castlingRights;
+8 -3
View File
@@ -26,12 +26,17 @@ namespace Eval::NNUE::Features {
static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kNone; static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kNone;
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
static void AppendActiveIndices(const Position& pos, Color perspective, static void append_active_indices(
const Position& pos,
Color perspective,
IndexList* active); IndexList* active);
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
static void AppendChangedIndices(const Position& pos, Color perspective, static void append_changed_indices(
IndexList* removed, IndexList* added); const Position& pos,
Color perspective,
IndexList* removed,
IndexList* added);
}; };
} // namespace Eval::NNUE::Features } // namespace Eval::NNUE::Features
+9 -5
View File
@@ -5,8 +5,10 @@
namespace Eval::NNUE::Features { namespace Eval::NNUE::Features {
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
void EnPassant::AppendActiveIndices( void EnPassant::append_active_indices(
const Position& pos, Color /* perspective */, IndexList* active) { const Position& pos,
Color /* perspective */,
IndexList* active) {
// do nothing if array size is small to avoid compiler warning // do nothing if array size is small to avoid compiler warning
if (RawFeatures::kMaxActiveDimensions < kMaxActiveDimensions) if (RawFeatures::kMaxActiveDimensions < kMaxActiveDimensions)
@@ -21,9 +23,11 @@ namespace Eval::NNUE::Features {
} }
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
void EnPassant::AppendChangedIndices( void EnPassant::append_changed_indices(
const Position& pos, Color /* perspective */, const Position& pos,
IndexList* removed, IndexList* added) { Color /* perspective */,
IndexList* removed,
IndexList* added) {
auto previous_epSquare = pos.state()->previous->epSquare; auto previous_epSquare = pos.state()->previous->epSquare;
auto epSquare = pos.state()->epSquare; auto epSquare = pos.state()->epSquare;
+8 -3
View File
@@ -22,12 +22,17 @@ namespace Eval::NNUE::Features {
static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kNone; static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kNone;
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
static void AppendActiveIndices(const Position& pos, Color perspective, static void append_active_indices(
const Position& pos,
Color perspective,
IndexList* active); IndexList* active);
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
static void AppendChangedIndices(const Position& pos, Color perspective, static void append_changed_indices(
IndexList* removed, IndexList* added); const Position& pos,
Color perspective,
IndexList* removed,
IndexList* added);
}; };
} // namespace Eval::NNUE::Features } // namespace Eval::NNUE::Features
+45 -30
View File
@@ -33,8 +33,8 @@ namespace Eval::NNUE::Features {
template <typename T, T First, T... Remaining> template <typename T, T First, T... Remaining>
struct CompileTimeList<T, First, Remaining...> { struct CompileTimeList<T, First, Remaining...> {
static constexpr bool Contains(T value) { static constexpr bool contains(T value) {
return value == First || CompileTimeList<T, Remaining...>::Contains(value); return value == First || CompileTimeList<T, Remaining...>::contains(value);
} }
static constexpr std::array<T, sizeof...(Remaining) + 1> static constexpr std::array<T, sizeof...(Remaining) + 1>
@@ -47,7 +47,7 @@ namespace Eval::NNUE::Features {
template <typename T> template <typename T>
struct CompileTimeList<T> { struct CompileTimeList<T> {
static constexpr bool Contains(T /*value*/) { static constexpr bool contains(T /*value*/) {
return false; return false;
} }
static constexpr std::array<T, 0> kValues = { {} }; static constexpr std::array<T, 0> kValues = { {} };
@@ -70,7 +70,7 @@ namespace Eval::NNUE::Features {
struct InsertToSet<T, CompileTimeList<T, First, Remaining...>, AnotherValue> { struct InsertToSet<T, CompileTimeList<T, First, Remaining...>, AnotherValue> {
using Result = using Result =
std::conditional_t< std::conditional_t<
CompileTimeList<T, First, Remaining...>::Contains(AnotherValue), CompileTimeList<T, First, Remaining...>::contains(AnotherValue),
CompileTimeList<T, First, Remaining...>, CompileTimeList<T, First, Remaining...>,
std::conditional_t< std::conditional_t<
(AnotherValue < First), (AnotherValue < First),
@@ -95,20 +95,23 @@ namespace Eval::NNUE::Features {
public: public:
// Get a list of indices for active features // Get a list of indices for active features
template <typename IndexListType> template <typename IndexListType>
static void AppendActiveIndices( static void append_active_indices(
const Position& pos, TriggerEvent trigger, IndexListType active[2]) { const Position& pos, TriggerEvent trigger, IndexListType active[2]) {
for (Color perspective : { WHITE, BLACK }) { for (Color perspective : { WHITE, BLACK }) {
Derived::CollectActiveIndices( Derived::collect_active_indices(
pos, trigger, perspective, &active[perspective]); pos, trigger, perspective, &active[perspective]);
} }
} }
// Get a list of indices for recently changed features // Get a list of indices for recently changed features
template <typename PositionType, typename IndexListType> template <typename PositionType, typename IndexListType>
static void AppendChangedIndices( static void append_changed_indices(
const PositionType& pos, TriggerEvent trigger, const PositionType& pos,
IndexListType removed[2], IndexListType added[2], bool reset[2]) { TriggerEvent trigger,
IndexListType removed[2],
IndexListType added[2],
bool reset[2]) {
const auto& dp = pos.state()->dirtyPiece; const auto& dp = pos.state()->dirtyPiece;
@@ -137,10 +140,10 @@ namespace Eval::NNUE::Features {
} }
if (reset[perspective]) { if (reset[perspective]) {
Derived::CollectActiveIndices( Derived::collect_active_indices(
pos, trigger, perspective, &added[perspective]); pos, trigger, perspective, &added[perspective]);
} else { } else {
Derived::CollectChangedIndices( Derived::collect_changed_indices(
pos, trigger, perspective, pos, trigger, perspective,
&removed[perspective], &added[perspective]); &removed[perspective], &added[perspective]);
} }
@@ -180,20 +183,23 @@ namespace Eval::NNUE::Features {
static constexpr auto kRefreshTriggers = SortedTriggerSet::kValues; static constexpr auto kRefreshTriggers = SortedTriggerSet::kValues;
// Get the feature quantity name // Get the feature quantity name
static std::string GetName() { static std::string get_name() {
return std::string(Head::kName) + "+" + Tail::GetName(); return std::string(Head::kName) + "+" + Tail::get_name();
} }
private: private:
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
template <typename IndexListType> template <typename IndexListType>
static void CollectActiveIndices( static void collect_active_indices(
const Position& pos, const TriggerEvent trigger, const Color perspective, const Position& pos,
const TriggerEvent trigger,
const Color perspective,
IndexListType* const active) { IndexListType* const active) {
Tail::CollectActiveIndices(pos, trigger, perspective, active);
Tail::collect_active_indices(pos, trigger, perspective, active);
if (Head::kRefreshTrigger == trigger) { if (Head::kRefreshTrigger == trigger) {
const auto start = active->size(); const auto start = active->size();
Head::AppendActiveIndices(pos, perspective, active); Head::append_active_indices(pos, perspective, active);
for (auto i = start; i < active->size(); ++i) { for (auto i = start; i < active->size(); ++i) {
(*active)[i] += Tail::kDimensions; (*active)[i] += Tail::kDimensions;
@@ -203,14 +209,18 @@ namespace Eval::NNUE::Features {
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
template <typename IndexListType> template <typename IndexListType>
static void CollectChangedIndices( static void collect_changed_indices(
const Position& pos, const TriggerEvent trigger, const Color perspective, const Position& pos,
IndexListType* const removed, IndexListType* const added) { const TriggerEvent trigger,
Tail::CollectChangedIndices(pos, trigger, perspective, removed, added); const Color perspective,
IndexListType* const removed,
IndexListType* const added) {
Tail::collect_changed_indices(pos, trigger, perspective, removed, added);
if (Head::kRefreshTrigger == trigger) { if (Head::kRefreshTrigger == trigger) {
const auto start_removed = removed->size(); const auto start_removed = removed->size();
const auto start_added = added->size(); const auto start_added = added->size();
Head::AppendChangedIndices(pos, perspective, removed, added); Head::append_changed_indices(pos, perspective, removed, added);
for (auto i = start_removed; i < removed->size(); ++i) { for (auto i = start_removed; i < removed->size(); ++i) {
(*removed)[i] += Tail::kDimensions; (*removed)[i] += Tail::kDimensions;
@@ -251,28 +261,33 @@ namespace Eval::NNUE::Features {
static constexpr auto kRefreshTriggers = SortedTriggerSet::kValues; static constexpr auto kRefreshTriggers = SortedTriggerSet::kValues;
// Get the feature quantity name // Get the feature quantity name
static std::string GetName() { static std::string get_name() {
return FeatureType::kName; return FeatureType::kName;
} }
private: private:
// Get a list of indices for active features // Get a list of indices for active features
static void CollectActiveIndices( static void collect_active_indices(
const Position& pos, const TriggerEvent trigger, const Color perspective, const Position& pos,
const TriggerEvent trigger,
const Color perspective,
IndexList* const active) { IndexList* const active) {
if (FeatureType::kRefreshTrigger == trigger) { if (FeatureType::kRefreshTrigger == trigger) {
FeatureType::AppendActiveIndices(pos, perspective, active); FeatureType::append_active_indices(pos, perspective, active);
} }
} }
// Get a list of indices for recently changed features // Get a list of indices for recently changed features
static void CollectChangedIndices( static void collect_changed_indices(
const Position& pos, const TriggerEvent trigger, const Color perspective, const Position& pos,
IndexList* const removed, IndexList* const added) { const TriggerEvent trigger,
const Color perspective,
IndexList* const removed,
IndexList* const added) {
if (FeatureType::kRefreshTrigger == trigger) { if (FeatureType::kRefreshTrigger == trigger) {
FeatureType::AppendChangedIndices(pos, perspective, removed, added); FeatureType::append_changed_indices(pos, perspective, removed, added);
} }
} }
+22 -11
View File
@@ -30,30 +30,41 @@ namespace Eval::NNUE::Features {
// Find the index of the feature quantity from the king position and PieceSquare // Find the index of the feature quantity from the king position and PieceSquare
template <Side AssociatedKing> template <Side AssociatedKing>
inline IndexType HalfKP<AssociatedKing>::MakeIndex( inline IndexType HalfKP<AssociatedKing>::make_index(
Color perspective, Square s, Piece pc, Square ksq) { Color perspective,
Square s,
Piece pc,
Square ksq) {
return IndexType(orient(perspective, s) + kpp_board_index[pc][perspective] + PS_END * ksq); return IndexType(orient(perspective, s) + kpp_board_index[pc][perspective] + PS_END * ksq);
} }
// Get a list of indices for active features // Get a list of indices for active features
template <Side AssociatedKing> template <Side AssociatedKing>
void HalfKP<AssociatedKing>::AppendActiveIndices( void HalfKP<AssociatedKing>::append_active_indices(
const Position& pos, Color perspective, IndexList* active) { const Position& pos,
Color perspective,
IndexList* active) {
Square ksq = orient(
perspective,
pos.square<KING>(
AssociatedKing == Side::kFriend ? perspective : ~perspective));
Square ksq = orient(perspective, pos.square<KING>(AssociatedKing == Side::kFriend ? perspective : ~perspective));
Bitboard bb = pos.pieces() & ~pos.pieces(KING); Bitboard bb = pos.pieces() & ~pos.pieces(KING);
while (bb) { while (bb) {
Square s = pop_lsb(&bb); Square s = pop_lsb(&bb);
active->push_back(MakeIndex(perspective, s, pos.piece_on(s), ksq)); active->push_back(make_index(perspective, s, pos.piece_on(s), ksq));
} }
} }
// Get a list of indices for recently changed features // Get a list of indices for recently changed features
template <Side AssociatedKing> template <Side AssociatedKing>
void HalfKP<AssociatedKing>::AppendChangedIndices( void HalfKP<AssociatedKing>::append_changed_indices(
const Position& pos, Color perspective, const Position& pos,
IndexList* removed, IndexList* added) { Color perspective,
IndexList* removed,
IndexList* added) {
Square ksq = orient( Square ksq = orient(
perspective, perspective,
@@ -68,10 +79,10 @@ namespace Eval::NNUE::Features {
continue; continue;
if (dp.from[i] != SQ_NONE) if (dp.from[i] != SQ_NONE)
removed->push_back(MakeIndex(perspective, dp.from[i], pc, ksq)); removed->push_back(make_index(perspective, dp.from[i], pc, ksq));
if (dp.to[i] != SQ_NONE) if (dp.to[i] != SQ_NONE)
added->push_back(MakeIndex(perspective, dp.to[i], pc, ksq)); added->push_back(make_index(perspective, dp.to[i], pc, ksq));
} }
} }
+10 -5
View File
@@ -53,16 +53,21 @@ namespace Eval::NNUE::Features {
TriggerEvent::kFriendKingMoved : TriggerEvent::kEnemyKingMoved; TriggerEvent::kFriendKingMoved : TriggerEvent::kEnemyKingMoved;
// Get a list of indices for active features // Get a list of indices for active features
static void AppendActiveIndices(const Position& pos, Color perspective, static void append_active_indices(
IndexList* active); const Position& pos,
Color perspective,
IndexList* active);
// Get a list of indices for recently changed features // Get a list of indices for recently changed features
static void AppendChangedIndices(const Position& pos, Color perspective, static void append_changed_indices(
IndexList* removed, IndexList* added); const Position& pos,
Color perspective,
IndexList* removed,
IndexList* added);
private: private:
// Index of a feature for a given king position and another piece on some square // Index of a feature for a given king position and another piece on some square
static IndexType MakeIndex(Color perspective, Square s, Piece pc, Square sq_k); static IndexType make_index(Color perspective, Square s, Piece pc, Square sq_k);
}; };
} // namespace Eval::NNUE::Features } // namespace Eval::NNUE::Features
+22 -13
View File
@@ -11,16 +11,21 @@ namespace Eval::NNUE::Features {
// Find the index of the feature quantity from the ball position and PieceSquare // Find the index of the feature quantity from the ball position and PieceSquare
template <Side AssociatedKing> template <Side AssociatedKing>
inline IndexType HalfRelativeKP<AssociatedKing>::MakeIndex( inline IndexType HalfRelativeKP<AssociatedKing>::make_index(
Color perspective, Square s, Piece pc, Square sq_k) { Color perspective,
Square s,
Piece pc,
Square sq_k) {
const IndexType p = IndexType(orient(perspective, s) + kpp_board_index[pc][perspective]); const IndexType p = IndexType(orient(perspective, s) + kpp_board_index[pc][perspective]);
return MakeIndex(sq_k, p); return make_index(sq_k, p);
} }
// Find the index of the feature quantity from the ball position and PieceSquare // Find the index of the feature quantity from the ball position and PieceSquare
template <Side AssociatedKing> template <Side AssociatedKing>
inline IndexType HalfRelativeKP<AssociatedKing>::MakeIndex( inline IndexType HalfRelativeKP<AssociatedKing>::make_index(
Square sq_k, IndexType p) { Square sq_k,
IndexType p) {
constexpr IndexType W = kBoardWidth; constexpr IndexType W = kBoardWidth;
constexpr IndexType H = kBoardHeight; constexpr IndexType H = kBoardHeight;
@@ -33,8 +38,10 @@ namespace Eval::NNUE::Features {
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
template <Side AssociatedKing> template <Side AssociatedKing>
void HalfRelativeKP<AssociatedKing>::AppendActiveIndices( void HalfRelativeKP<AssociatedKing>::append_active_indices(
const Position& pos, Color perspective, IndexList* active) { const Position& pos,
Color perspective,
IndexList* active) {
Square ksq = orient( Square ksq = orient(
perspective, perspective,
@@ -44,15 +51,17 @@ namespace Eval::NNUE::Features {
Bitboard bb = pos.pieces() & ~pos.pieces(KING); Bitboard bb = pos.pieces() & ~pos.pieces(KING);
while (bb) { while (bb) {
Square s = pop_lsb(&bb); Square s = pop_lsb(&bb);
active->push_back(MakeIndex(perspective, s, pos.piece_on(s), ksq)); active->push_back(make_index(perspective, s, pos.piece_on(s), ksq));
} }
} }
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
template <Side AssociatedKing> template <Side AssociatedKing>
void HalfRelativeKP<AssociatedKing>::AppendChangedIndices( void HalfRelativeKP<AssociatedKing>::append_changed_indices(
const Position& pos, Color perspective, const Position& pos,
IndexList* removed, IndexList* added) { Color perspective,
IndexList* removed,
IndexList* added) {
Square ksq = orient( Square ksq = orient(
perspective, perspective,
@@ -67,10 +76,10 @@ namespace Eval::NNUE::Features {
continue; continue;
if (dp.from[i] != SQ_NONE) if (dp.from[i] != SQ_NONE)
removed->push_back(MakeIndex(perspective, dp.from[i], pc, ksq)); removed->push_back(make_index(perspective, dp.from[i], pc, ksq));
if (dp.to[i] != SQ_NONE) if (dp.to[i] != SQ_NONE)
added->push_back(MakeIndex(perspective, dp.to[i], pc, ksq)); added->push_back(make_index(perspective, dp.to[i], pc, ksq));
} }
} }
+11 -6
View File
@@ -42,18 +42,23 @@ namespace Eval::NNUE::Features {
TriggerEvent::kFriendKingMoved : TriggerEvent::kEnemyKingMoved; TriggerEvent::kFriendKingMoved : TriggerEvent::kEnemyKingMoved;
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
static void AppendActiveIndices(const Position& pos, Color perspective, static void append_active_indices(
IndexList* active); const Position& pos,
Color perspective,
IndexList* active);
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
static void AppendChangedIndices(const Position& pos, Color perspective, static void append_changed_indices(
IndexList* removed, IndexList* added); const Position& pos,
Color perspective,
IndexList* removed,
IndexList* added);
// Find the index of the feature quantity from the ball position and PieceSquare // Find the index of the feature quantity from the ball position and PieceSquare
static IndexType MakeIndex(Square s, IndexType p); static IndexType make_index(Square s, IndexType p);
// Find the index of the feature quantity from the ball position and PieceSquare // Find the index of the feature quantity from the ball position and PieceSquare
static IndexType MakeIndex(Color perspective, Square s, Piece pc, Square sq_k); static IndexType make_index(Color perspective, Square s, Piece pc, Square sq_k);
}; };
} // namespace Eval::NNUE::Features } // namespace Eval::NNUE::Features
+13 -9
View File
@@ -10,29 +10,33 @@ namespace Eval::NNUE::Features {
} }
// Index of a feature for a given king position. // Index of a feature for a given king position.
IndexType K::MakeIndex(Color perspective, Square s, Color king_color) { IndexType K::make_index(Color perspective, Square s, Color king_color) {
return IndexType(orient(perspective, s) + bool(perspective ^ king_color) * 64); return IndexType(orient(perspective, s) + bool(perspective ^ king_color) * 64);
} }
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
void K::AppendActiveIndices( void K::append_active_indices(
const Position& pos, Color perspective, IndexList* active) { const Position& pos,
Color perspective,
IndexList* active) {
for (auto color : Colors) { for (auto color : Colors) {
active->push_back(MakeIndex(perspective, pos.square<KING>(color), color)); active->push_back(make_index(perspective, pos.square<KING>(color), color));
} }
} }
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
void K::AppendChangedIndices( void K::append_changed_indices(
const Position& pos, Color perspective, const Position& pos,
IndexList* removed, IndexList* added) { Color perspective,
IndexList* removed,
IndexList* added) {
const auto& dp = pos.state()->dirtyPiece; const auto& dp = pos.state()->dirtyPiece;
if (type_of(dp.piece[0]) == KING) if (type_of(dp.piece[0]) == KING)
{ {
removed->push_back(MakeIndex(perspective, dp.from[0], color_of(dp.piece[0]))); removed->push_back(make_index(perspective, dp.from[0], color_of(dp.piece[0])));
added->push_back(MakeIndex(perspective, dp.to[0], color_of(dp.piece[0]))); added->push_back(make_index(perspective, dp.to[0], color_of(dp.piece[0])));
} }
} }
+28 -23
View File
@@ -8,36 +8,41 @@
//Definition of input feature quantity K of NNUE evaluation function //Definition of input feature quantity K of NNUE evaluation function
namespace Eval::NNUE::Features { namespace Eval::NNUE::Features {
// Feature K: Ball position // Feature K: Ball position
class K { class K {
public: public:
// feature quantity name // feature quantity name
static constexpr const char* kName = "K"; static constexpr const char* kName = "K";
// Hash value embedded in the evaluation function file // Hash value embedded in the evaluation function file
static constexpr std::uint32_t kHashValue = 0xD3CEE169u; static constexpr std::uint32_t kHashValue = 0xD3CEE169u;
// number of feature dimensions // number of feature dimensions
static constexpr IndexType kDimensions = SQUARE_NB * 2; static constexpr IndexType kDimensions = SQUARE_NB * 2;
// The maximum value of the number of indexes whose value is 1 at the same time among the feature values // The maximum value of the number of indexes whose value is 1 at the same time among the feature values
static constexpr IndexType kMaxActiveDimensions = 2; static constexpr IndexType kMaxActiveDimensions = 2;
// Timing of full calculation instead of difference calculation // Timing of full calculation instead of difference calculation
static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kNone; static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kNone;
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
static void AppendActiveIndices(const Position& pos, Color perspective, static void append_active_indices(
IndexList* active); const Position& pos,
Color perspective,
IndexList* active);
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
static void AppendChangedIndices(const Position& pos, Color perspective, static void append_changed_indices(
IndexList* removed, IndexList* added); const Position& pos,
Color perspective,
IndexList* removed,
IndexList* added);
private: private:
// Index of a feature for a given king position. // Index of a feature for a given king position.
static IndexType MakeIndex(Color perspective, Square s, Color king_color); static IndexType make_index(Color perspective, Square s, Color king_color);
}; };
} // namespace Eval::NNUE::Features } // namespace Eval::NNUE::Features
+13 -9
View File
@@ -10,26 +10,30 @@ namespace Eval::NNUE::Features {
} }
// Find the index of the feature quantity from the king position and PieceSquare // Find the index of the feature quantity from the king position and PieceSquare
inline IndexType P::MakeIndex( inline IndexType P::make_index(
Color perspective, Square s, Piece pc) { Color perspective, Square s, Piece pc) {
return IndexType(orient(perspective, s) + kpp_board_index[pc][perspective]); return IndexType(orient(perspective, s) + kpp_board_index[pc][perspective]);
} }
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
void P::AppendActiveIndices( void P::append_active_indices(
const Position& pos, Color perspective, IndexList* active) { const Position& pos,
Color perspective,
IndexList* active) {
Bitboard bb = pos.pieces() & ~pos.pieces(KING); Bitboard bb = pos.pieces() & ~pos.pieces(KING);
while (bb) { while (bb) {
Square s = pop_lsb(&bb); Square s = pop_lsb(&bb);
active->push_back(MakeIndex(perspective, s, pos.piece_on(s))); active->push_back(make_index(perspective, s, pos.piece_on(s)));
} }
} }
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
void P::AppendChangedIndices( void P::append_changed_indices(
const Position& pos, Color perspective, const Position& pos,
IndexList* removed, IndexList* added) { Color perspective,
IndexList* removed,
IndexList* added) {
const auto& dp = pos.state()->dirtyPiece; const auto& dp = pos.state()->dirtyPiece;
for (int i = 0; i < dp.dirty_num; ++i) { for (int i = 0; i < dp.dirty_num; ++i) {
@@ -39,10 +43,10 @@ namespace Eval::NNUE::Features {
continue; continue;
if (dp.from[i] != SQ_NONE) if (dp.from[i] != SQ_NONE)
removed->push_back(MakeIndex(perspective, dp.from[i], pc)); removed->push_back(make_index(perspective, dp.from[i], pc));
if (dp.to[i] != SQ_NONE) if (dp.to[i] != SQ_NONE)
added->push_back(MakeIndex(perspective, dp.to[i], pc)); added->push_back(make_index(perspective, dp.to[i], pc));
} }
} }
+28 -23
View File
@@ -8,36 +8,41 @@
//Definition of input feature P of NNUE evaluation function //Definition of input feature P of NNUE evaluation function
namespace Eval::NNUE::Features { namespace Eval::NNUE::Features {
// Feature P: PieceSquare of pieces other than balls // Feature P: PieceSquare of pieces other than balls
class P { class P {
public: public:
// feature quantity name // feature quantity name
static constexpr const char* kName = "P"; static constexpr const char* kName = "P";
// Hash value embedded in the evaluation function file // Hash value embedded in the evaluation function file
static constexpr std::uint32_t kHashValue = 0x764CFB4Bu; static constexpr std::uint32_t kHashValue = 0x764CFB4Bu;
// number of feature dimensions // number of feature dimensions
static constexpr IndexType kDimensions = PS_END; static constexpr IndexType kDimensions = PS_END;
// The maximum value of the number of indexes whose value is 1 at the same time among the feature values // The maximum value of the number of indexes whose value is 1 at the same time among the feature values
static constexpr IndexType kMaxActiveDimensions = 30; // Kings don't count static constexpr IndexType kMaxActiveDimensions = 30; // Kings don't count
// Timing of full calculation instead of difference calculation // Timing of full calculation instead of difference calculation
static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kNone; static constexpr TriggerEvent kRefreshTrigger = TriggerEvent::kNone;
// Get a list of indices with a value of 1 among the features // Get a list of indices with a value of 1 among the features
static void AppendActiveIndices(const Position& pos, Color perspective, static void append_active_indices(
IndexList* active); const Position& pos,
Color perspective,
IndexList* active);
// Get a list of indices whose values have changed from the previous one in the feature quantity // Get a list of indices whose values have changed from the previous one in the feature quantity
static void AppendChangedIndices(const Position& pos, Color perspective, static void append_changed_indices(
IndexList* removed, IndexList* added); const Position& pos,
Color perspective,
IndexList* removed,
IndexList* added);
private: private:
// Index of a feature for a given piece on some square // Index of a feature for a given piece on some square
static IndexType MakeIndex(Color perspective, Square s, Piece pc); static IndexType make_index(Color perspective, Square s, Piece pc);
}; };
} // namespace Eval::NNUE::Features } // namespace Eval::NNUE::Features
+13 -13
View File
@@ -47,36 +47,36 @@ namespace Eval::NNUE::Layers {
static constexpr IndexType kOutputDimensions = OutputDimensions; static constexpr IndexType kOutputDimensions = OutputDimensions;
static constexpr IndexType kPaddedInputDimensions = static constexpr IndexType kPaddedInputDimensions =
CeilToMultiple<IndexType>(kInputDimensions, kMaxSimdWidth); ceil_to_multiple<IndexType>(kInputDimensions, kMaxSimdWidth);
// Size of forward propagation buffer used in this layer // Size of forward propagation buffer used in this layer
static constexpr std::size_t kSelfBufferSize = static constexpr std::size_t kSelfBufferSize =
CeilToMultiple(kOutputDimensions * sizeof(OutputType), kCacheLineSize); ceil_to_multiple(kOutputDimensions * sizeof(OutputType), kCacheLineSize);
// Size of the forward propagation buffer used from the input layer to this layer // Size of the forward propagation buffer used from the input layer to this layer
static constexpr std::size_t kBufferSize = static constexpr std::size_t kBufferSize =
PreviousLayer::kBufferSize + kSelfBufferSize; PreviousLayer::kBufferSize + kSelfBufferSize;
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
std::uint32_t hash_value = 0xCC03DAE4u; std::uint32_t hash_value = 0xCC03DAE4u;
hash_value += kOutputDimensions; hash_value += kOutputDimensions;
hash_value ^= PreviousLayer::GetHashValue() >> 1; hash_value ^= PreviousLayer::get_hash_value() >> 1;
hash_value ^= PreviousLayer::GetHashValue() << 31; hash_value ^= PreviousLayer::get_hash_value() << 31;
return hash_value; return hash_value;
} }
// A string that represents the structure from the input layer to this layer // A string that represents the structure from the input layer to this layer
static std::string GetStructureString() { static std::string get_structure_string() {
return "AffineTransform[" + return "AffineTransform[" +
std::to_string(kOutputDimensions) + "<-" + std::to_string(kOutputDimensions) + "<-" +
std::to_string(kInputDimensions) + "](" + std::to_string(kInputDimensions) + "](" +
PreviousLayer::GetStructureString() + ")"; PreviousLayer::get_structure_string() + ")";
} }
// Read network parameters // Read network parameters
bool ReadParameters(std::istream& stream) { bool read_parameters(std::istream& stream) {
if (!previous_layer_.ReadParameters(stream)) if (!previous_layer_.read_parameters(stream))
return false; return false;
for (std::size_t i = 0; i < kOutputDimensions; ++i) for (std::size_t i = 0; i < kOutputDimensions; ++i)
@@ -89,8 +89,8 @@ namespace Eval::NNUE::Layers {
} }
// write parameters // write parameters
bool WriteParameters(std::ostream& stream) const { bool write_parameters(std::ostream& stream) const {
if (!previous_layer_.WriteParameters(stream)) if (!previous_layer_.write_parameters(stream))
return false; return false;
stream.write(reinterpret_cast<const char*>(biases_), stream.write(reinterpret_cast<const char*>(biases_),
@@ -104,10 +104,10 @@ namespace Eval::NNUE::Layers {
} }
// Forward propagation // Forward propagation
const OutputType* Propagate( const OutputType* propagate(
const TransformedFeatureType* transformed_features, char* buffer) const { const TransformedFeatureType* transformed_features, char* buffer) const {
const auto input = previous_layer_.Propagate( const auto input = previous_layer_.propagate(
transformed_features, buffer + kSelfBufferSize); transformed_features, buffer + kSelfBufferSize);
const auto output = reinterpret_cast<OutputType*>(buffer); const auto output = reinterpret_cast<OutputType*>(buffer);
+11 -11
View File
@@ -48,41 +48,41 @@ namespace Eval::NNUE::Layers {
// Size of forward propagation buffer used in this layer // Size of forward propagation buffer used in this layer
static constexpr std::size_t kSelfBufferSize = static constexpr std::size_t kSelfBufferSize =
CeilToMultiple(kOutputDimensions * sizeof(OutputType), kCacheLineSize); ceil_to_multiple(kOutputDimensions * sizeof(OutputType), kCacheLineSize);
// Size of the forward propagation buffer used from the input layer to this layer // Size of the forward propagation buffer used from the input layer to this layer
static constexpr std::size_t kBufferSize = static constexpr std::size_t kBufferSize =
PreviousLayer::kBufferSize + kSelfBufferSize; PreviousLayer::kBufferSize + kSelfBufferSize;
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
std::uint32_t hash_value = 0x538D24C7u; std::uint32_t hash_value = 0x538D24C7u;
hash_value += PreviousLayer::GetHashValue(); hash_value += PreviousLayer::get_hash_value();
return hash_value; return hash_value;
} }
// A string that represents the structure from the input layer to this layer // A string that represents the structure from the input layer to this layer
static std::string GetStructureString() { static std::string get_structure_string() {
return "ClippedReLU[" + return "ClippedReLU[" +
std::to_string(kOutputDimensions) + "](" + std::to_string(kOutputDimensions) + "](" +
PreviousLayer::GetStructureString() + ")"; PreviousLayer::get_structure_string() + ")";
} }
// Read network parameters // Read network parameters
bool ReadParameters(std::istream& stream) { bool read_parameters(std::istream& stream) {
return previous_layer_.ReadParameters(stream); return previous_layer_.read_parameters(stream);
} }
// write parameters // write parameters
bool WriteParameters(std::ostream& stream) const { bool write_parameters(std::ostream& stream) const {
return previous_layer_.WriteParameters(stream); return previous_layer_.write_parameters(stream);
} }
// Forward propagation // Forward propagation
const OutputType* Propagate( const OutputType* propagate(
const TransformedFeatureType* transformed_features, char* buffer) const { const TransformedFeatureType* transformed_features, char* buffer) const {
const auto input = previous_layer_.Propagate( const auto input = previous_layer_.propagate(
transformed_features, buffer + kSelfBufferSize); transformed_features, buffer + kSelfBufferSize);
const auto output = reinterpret_cast<OutputType*>(buffer); const auto output = reinterpret_cast<OutputType*>(buffer);
+5 -5
View File
@@ -45,31 +45,31 @@ namespace Eval::NNUE::Layers {
static constexpr std::size_t kBufferSize = 0; static constexpr std::size_t kBufferSize = 0;
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
std::uint32_t hash_value = 0xEC42E90Du; std::uint32_t hash_value = 0xEC42E90Du;
hash_value ^= kOutputDimensions ^ (Offset << 10); hash_value ^= kOutputDimensions ^ (Offset << 10);
return hash_value; return hash_value;
} }
// A string that represents the structure from the input layer to this layer // A string that represents the structure from the input layer to this layer
static std::string GetStructureString() { static std::string get_structure_string() {
return "InputSlice[" + std::to_string(kOutputDimensions) + "(" + return "InputSlice[" + std::to_string(kOutputDimensions) + "(" +
std::to_string(Offset) + ":" + std::to_string(Offset) + ":" +
std::to_string(Offset + kOutputDimensions) + ")]"; std::to_string(Offset + kOutputDimensions) + ")]";
} }
// Read network parameters // Read network parameters
bool ReadParameters(std::istream& /*stream*/) { bool read_parameters(std::istream& /*stream*/) {
return true; return true;
} }
// write parameters // write parameters
bool WriteParameters(std::ostream& /*stream*/) const { bool write_parameters(std::ostream& /*stream*/) const {
return true; return true;
} }
// Forward propagation // Forward propagation
const OutputType* Propagate( const OutputType* propagate(
const TransformedFeatureType* transformed_features, const TransformedFeatureType* transformed_features,
char* /*buffer*/) const { char* /*buffer*/) const {
+32 -32
View File
@@ -30,51 +30,51 @@ namespace Eval::NNUE::Layers {
// Size of forward propagation buffer used in this layer // Size of forward propagation buffer used in this layer
static constexpr std::size_t kSelfBufferSize = static constexpr std::size_t kSelfBufferSize =
CeilToMultiple(kOutputDimensions * sizeof(OutputType), kCacheLineSize); ceil_to_multiple(kOutputDimensions * sizeof(OutputType), kCacheLineSize);
// Size of the forward propagation buffer used from the input layer to this layer // Size of the forward propagation buffer used from the input layer to this layer
static constexpr std::size_t kBufferSize = static constexpr std::size_t kBufferSize =
std::max(Head::kBufferSize + kSelfBufferSize, Tail::kBufferSize); std::max(Head::kBufferSize + kSelfBufferSize, Tail::kBufferSize);
// Hash value embedded in the evaluation function file // Hash value embedded in the evaluation function file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
std::uint32_t hash_value = 0xBCE400B4u; std::uint32_t hash_value = 0xBCE400B4u;
hash_value ^= Head::GetHashValue() >> 1; hash_value ^= Head::get_hash_value() >> 1;
hash_value ^= Head::GetHashValue() << 31; hash_value ^= Head::get_hash_value() << 31;
hash_value ^= Tail::GetHashValue() >> 2; hash_value ^= Tail::get_hash_value() >> 2;
hash_value ^= Tail::GetHashValue() << 30; hash_value ^= Tail::get_hash_value() << 30;
return hash_value; return hash_value;
} }
// A string that represents the structure from the input layer to this layer // A string that represents the structure from the input layer to this layer
static std::string GetStructureString() { static std::string get_structure_string() {
return "Sum[" + return "Sum[" +
std::to_string(kOutputDimensions) + "](" + GetSummandsString() + ")"; std::to_string(kOutputDimensions) + "](" + get_summands_string() + ")";
} }
// read parameters // read parameters
bool ReadParameters(std::istream& stream) { bool read_parameters(std::istream& stream) {
if (!Tail::ReadParameters(stream)) if (!Tail::read_parameters(stream))
return false; return false;
return previous_layer_.ReadParameters(stream); return previous_layer_.read_parameters(stream);
} }
// write parameters // write parameters
bool WriteParameters(std::ostream& stream) const { bool write_parameters(std::ostream& stream) const {
if (!Tail::WriteParameters(stream)) if (!Tail::write_parameters(stream))
return false; return false;
return previous_layer_.WriteParameters(stream); return previous_layer_.write_parameters(stream);
} }
// forward propagation // forward propagation
const OutputType* Propagate( const OutputType* propagate(
const TransformedFeatureType* transformed_features, char* buffer) const { const TransformedFeatureType* transformed_features, char* buffer) const {
Tail::Propagate(transformed_features, buffer); Tail::propagate(transformed_features, buffer);
const auto head_output = previous_layer_.Propagate( const auto head_output = previous_layer_.propagate(
transformed_features, buffer + kSelfBufferSize); transformed_features, buffer + kSelfBufferSize);
const auto output = reinterpret_cast<OutputType*>(buffer); const auto output = reinterpret_cast<OutputType*>(buffer);
@@ -88,8 +88,8 @@ namespace Eval::NNUE::Layers {
protected: protected:
// A string that represents the list of layers to be summed // A string that represents the list of layers to be summed
static std::string GetSummandsString() { static std::string get_summands_string() {
return Head::GetStructureString() + "," + Tail::GetSummandsString(); return Head::get_structure_string() + "," + Tail::get_summands_string();
} }
// Make the learning class a friend // Make the learning class a friend
@@ -118,40 +118,40 @@ namespace Eval::NNUE::Layers {
static constexpr std::size_t kBufferSize = PreviousLayer::kBufferSize; static constexpr std::size_t kBufferSize = PreviousLayer::kBufferSize;
// Hash value embedded in the evaluation function file // Hash value embedded in the evaluation function file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
std::uint32_t hash_value = 0xBCE400B4u; std::uint32_t hash_value = 0xBCE400B4u;
hash_value ^= PreviousLayer::GetHashValue() >> 1; hash_value ^= PreviousLayer::get_hash_value() >> 1;
hash_value ^= PreviousLayer::GetHashValue() << 31; hash_value ^= PreviousLayer::get_hash_value() << 31;
return hash_value; return hash_value;
} }
// A string that represents the structure from the input layer to this layer // A string that represents the structure from the input layer to this layer
static std::string GetStructureString() { static std::string get_structure_string() {
return "Sum[" + return "Sum[" +
std::to_string(kOutputDimensions) + "](" + GetSummandsString() + ")"; std::to_string(kOutputDimensions) + "](" + get_summands_string() + ")";
} }
// read parameters // read parameters
bool ReadParameters(std::istream& stream) { bool read_parameters(std::istream& stream) {
return previous_layer_.ReadParameters(stream); return previous_layer_.read_parameters(stream);
} }
// write parameters // write parameters
bool WriteParameters(std::ostream& stream) const { bool write_parameters(std::ostream& stream) const {
return previous_layer_.WriteParameters(stream); return previous_layer_.write_parameters(stream);
} }
// forward propagation // forward propagation
const OutputType* Propagate( const OutputType* propagate(
const TransformedFeatureType* transformed_features, char* buffer) const { const TransformedFeatureType* transformed_features, char* buffer) const {
return previous_layer_.Propagate(transformed_features, buffer); return previous_layer_.propagate(transformed_features, buffer);
} }
protected: protected:
// A string that represents the list of layers to be summed // A string that represents the list of layers to be summed
static std::string GetSummandsString() { static std::string get_summands_string() {
return PreviousLayer::GetStructureString(); return PreviousLayer::get_structure_string();
} }
// Make the learning class a friend // Make the learning class a friend
+1 -1
View File
@@ -127,7 +127,7 @@ namespace Eval::NNUE {
// Round n up to be a multiple of base // Round n up to be a multiple of base
template <typename IntType> template <typename IntType>
constexpr IntType CeilToMultiple(IntType n, IntType base) { constexpr IntType ceil_to_multiple(IntType n, IntType base) {
return (n + base - 1) / base * base; return (n + base - 1) / base * base;
} }
+16 -16
View File
@@ -111,20 +111,20 @@ namespace Eval::NNUE {
kOutputDimensions * sizeof(OutputType); kOutputDimensions * sizeof(OutputType);
// Hash value embedded in the evaluation file // Hash value embedded in the evaluation file
static constexpr std::uint32_t GetHashValue() { static constexpr std::uint32_t get_hash_value() {
return RawFeatures::kHashValue ^ kOutputDimensions; return RawFeatures::kHashValue ^ kOutputDimensions;
} }
// a string representing the structure // a string representing the structure
static std::string GetStructureString() { static std::string get_structure_string() {
return RawFeatures::GetName() + "[" + return RawFeatures::get_name() + "[" +
std::to_string(kInputDimensions) + "->" + std::to_string(kInputDimensions) + "->" +
std::to_string(kHalfDimensions) + "x2]"; std::to_string(kHalfDimensions) + "x2]";
} }
// Read network parameters // Read network parameters
bool ReadParameters(std::istream& stream) { bool read_parameters(std::istream& stream) {
for (std::size_t i = 0; i < kHalfDimensions; ++i) for (std::size_t i = 0; i < kHalfDimensions; ++i)
biases_[i] = read_little_endian<BiasType>(stream); biases_[i] = read_little_endian<BiasType>(stream);
@@ -136,7 +136,7 @@ namespace Eval::NNUE {
} }
// write parameters // write parameters
bool WriteParameters(std::ostream& stream) const { bool write_parameters(std::ostream& stream) const {
stream.write(reinterpret_cast<const char*>(biases_), stream.write(reinterpret_cast<const char*>(biases_),
kHalfDimensions * sizeof(BiasType)); kHalfDimensions * sizeof(BiasType));
@@ -147,7 +147,7 @@ namespace Eval::NNUE {
} }
// Proceed with the difference calculation if possible // Proceed with the difference calculation if possible
bool UpdateAccumulatorIfPossible(const Position& pos) const { bool update_accumulator_if_possible(const Position& pos) const {
const auto now = pos.state(); const auto now = pos.state();
if (now->accumulator.computed_accumulation) if (now->accumulator.computed_accumulation)
@@ -155,7 +155,7 @@ namespace Eval::NNUE {
const auto prev = now->previous; const auto prev = now->previous;
if (prev && prev->accumulator.computed_accumulation) { if (prev && prev->accumulator.computed_accumulation) {
UpdateAccumulator(pos); update_accumulator(pos);
return true; return true;
} }
@@ -163,10 +163,10 @@ namespace Eval::NNUE {
} }
// Convert input features // Convert input features
void Transform(const Position& pos, OutputType* output) const { void transform(const Position& pos, OutputType* output) const {
if (!UpdateAccumulatorIfPossible(pos)) if (!update_accumulator_if_possible(pos))
RefreshAccumulator(pos); refresh_accumulator(pos);
const auto& accumulation = pos.state()->accumulator.accumulation; const auto& accumulation = pos.state()->accumulator.accumulation;
@@ -294,13 +294,13 @@ namespace Eval::NNUE {
private: private:
// Calculate cumulative value without using difference calculation // Calculate cumulative value without using difference calculation
void RefreshAccumulator(const Position& pos) const { void refresh_accumulator(const Position& pos) const {
auto& accumulator = pos.state()->accumulator; auto& accumulator = pos.state()->accumulator;
for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) { for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) {
Features::IndexList active_indices[2]; Features::IndexList active_indices[2];
RawFeatures::AppendActiveIndices(pos, kRefreshTriggers[i], RawFeatures::append_active_indices(pos, kRefreshTriggers[i],
active_indices); active_indices);
for (Color perspective : { WHITE, BLACK }) { for (Color perspective : { WHITE, BLACK }) {
#ifdef TILING #ifdef TILING
for (unsigned j = 0; j < kHalfDimensions / kTileHeight; ++j) { for (unsigned j = 0; j < kHalfDimensions / kTileHeight; ++j) {
@@ -357,15 +357,15 @@ namespace Eval::NNUE {
} }
// Calculate cumulative value using difference calculation // Calculate cumulative value using difference calculation
void UpdateAccumulator(const Position& pos) const { void update_accumulator(const Position& pos) const {
const auto& prev_accumulator = pos.state()->previous->accumulator; const auto& prev_accumulator = pos.state()->previous->accumulator;
auto& accumulator = pos.state()->accumulator; auto& accumulator = pos.state()->accumulator;
for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) { for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) {
Features::IndexList removed_indices[2], added_indices[2]; Features::IndexList removed_indices[2], added_indices[2];
bool reset[2] = { false, false }; bool reset[2] = { false, false };
RawFeatures::AppendChangedIndices(pos, kRefreshTriggers[i], RawFeatures::append_changed_indices(pos, kRefreshTriggers[i],
removed_indices, added_indices, reset); removed_indices, added_indices, reset);
#ifdef TILING #ifdef TILING
for (IndexType j = 0; j < kHalfDimensions / kTileHeight; ++j) { for (IndexType j = 0; j < kHalfDimensions / kTileHeight; ++j) {
+11 -11
View File
@@ -24,7 +24,7 @@ namespace Eval::NNUE {
namespace { namespace {
// Testing RawFeatures mainly for difference calculation // Testing RawFeatures mainly for difference calculation
void TestFeatures(Position& pos) { void test_features(Position& pos) {
const std::uint64_t num_games = 1000; const std::uint64_t num_games = 1000;
StateInfo si; StateInfo si;
pos.set(StartFEN, false, &si, Threads.main()); pos.set(StartFEN, false, &si, Threads.main());
@@ -47,7 +47,7 @@ namespace Eval::NNUE {
for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) { for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) {
Features::IndexList active_indices[2]; Features::IndexList active_indices[2];
RawFeatures::AppendActiveIndices(position, kRefreshTriggers[i], RawFeatures::append_active_indices(position, kRefreshTriggers[i],
active_indices); active_indices);
for (const auto perspective : Colors) { for (const auto perspective : Colors) {
@@ -68,7 +68,7 @@ namespace Eval::NNUE {
for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) { for (IndexType i = 0; i < kRefreshTriggers.size(); ++i) {
Features::IndexList removed_indices[2], added_indices[2]; Features::IndexList removed_indices[2], added_indices[2];
bool reset[2] = { false, false }; bool reset[2] = { false, false };
RawFeatures::AppendChangedIndices(position, kRefreshTriggers[i], RawFeatures::append_changed_indices(position, kRefreshTriggers[i],
removed_indices, added_indices, reset); removed_indices, added_indices, reset);
for (const auto perspective : Colors) { for (const auto perspective : Colors) {
if (reset[perspective]) { if (reset[perspective]) {
@@ -99,7 +99,7 @@ namespace Eval::NNUE {
} }
}; };
std::cout << "feature set: " << RawFeatures::GetName() std::cout << "feature set: " << RawFeatures::get_name()
<< "[" << RawFeatures::kDimensions << "]" << std::endl; << "[" << RawFeatures::kDimensions << "]" << std::endl;
std::cout << "start testing with random games"; std::cout << "start testing with random games";
@@ -154,8 +154,8 @@ namespace Eval::NNUE {
} }
// Output a string that represents the structure of the evaluation function // Output a string that represents the structure of the evaluation function
void PrintInfo(std::istream& stream) { void print_info(std::istream& stream) {
std::cout << "network architecture: " << GetArchitectureString() << std::endl; std::cout << "network architecture: " << get_architecture_string() << std::endl;
while (true) { while (true) {
std::string file_name; std::string file_name;
@@ -170,7 +170,7 @@ namespace Eval::NNUE {
if (!file_stream) if (!file_stream)
return false; return false;
if (!ReadHeader(file_stream, &hash_value, &architecture)) if (!read_header(file_stream, &hash_value, &architecture))
return false; return false;
return true; return true;
@@ -180,7 +180,7 @@ namespace Eval::NNUE {
if (success) { if (success) {
if (hash_value == kHashValue) { if (hash_value == kHashValue) {
std::cout << "matches with this binary"; std::cout << "matches with this binary";
if (architecture != GetArchitectureString()) { if (architecture != get_architecture_string()) {
std::cout << ", but architecture string differs: " << architecture; std::cout << ", but architecture string differs: " << architecture;
} }
@@ -197,14 +197,14 @@ namespace Eval::NNUE {
} // namespace } // namespace
// USI extended command for NNUE evaluation function // USI extended command for NNUE evaluation function
void TestCommand(Position& pos, std::istream& stream) { void test_command(Position& pos, std::istream& stream) {
std::string sub_command; std::string sub_command;
stream >> sub_command; stream >> sub_command;
if (sub_command == "test_features") { if (sub_command == "test_features") {
TestFeatures(pos); test_features(pos);
} else if (sub_command == "info") { } else if (sub_command == "info") {
PrintInfo(stream); print_info(stream);
} else { } else {
std::cout << "usage:" << std::endl; std::cout << "usage:" << std::endl;
std::cout << " test nnue test_features" << std::endl; std::cout << " test nnue test_features" << std::endl;
+1 -1
View File
@@ -5,7 +5,7 @@
namespace Eval::NNUE { namespace Eval::NNUE {
// USI extended command for NNUE evaluation function // USI extended command for NNUE evaluation function
void TestCommand(Position& pos, std::istream& stream); void test_command(Position& pos, std::istream& stream);
} // namespace Eval::NNUE } // namespace Eval::NNUE
+11 -11
View File
@@ -14,12 +14,12 @@ namespace Eval::NNUE::Features {
class Factorizer { class Factorizer {
public: public:
// Get the dimensionality of the learning feature // Get the dimensionality of the learning feature
static constexpr IndexType GetDimensions() { static constexpr IndexType get_dimensions() {
return FeatureType::kDimensions; return FeatureType::kDimensions;
} }
// Get index of learning feature and scale of learning rate // Get index of learning feature and scale of learning rate
static void AppendTrainingFeatures( static void append_training_features(
IndexType base_index, std::vector<TrainingFeature>* training_features) { IndexType base_index, std::vector<TrainingFeature>* training_features) {
assert(base_index <FeatureType::kDimensions); assert(base_index <FeatureType::kDimensions);
@@ -35,7 +35,7 @@ namespace Eval::NNUE::Features {
// Add the original input features to the learning features // Add the original input features to the learning features
template <typename FeatureType> template <typename FeatureType>
IndexType AppendBaseFeature( IndexType append_base_feature(
FeatureProperties properties, IndexType base_index, FeatureProperties properties, IndexType base_index,
std::vector<TrainingFeature>* training_features) { std::vector<TrainingFeature>* training_features) {
@@ -47,7 +47,7 @@ namespace Eval::NNUE::Features {
// If the learning rate scale is not 0, inherit other types of learning features // If the learning rate scale is not 0, inherit other types of learning features
template <typename FeatureType> template <typename FeatureType>
IndexType InheritFeaturesIfRequired( IndexType inherit_features_if_required(
IndexType index_offset, FeatureProperties properties, IndexType base_index, IndexType index_offset, FeatureProperties properties, IndexType base_index,
std::vector<TrainingFeature>* training_features) { std::vector<TrainingFeature>* training_features) {
@@ -55,17 +55,17 @@ namespace Eval::NNUE::Features {
return 0; return 0;
} }
assert(properties.dimensions == Factorizer<FeatureType>::GetDimensions()); assert(properties.dimensions == Factorizer<FeatureType>::get_dimensions());
assert(base_index < FeatureType::kDimensions); assert(base_index < FeatureType::kDimensions);
const auto start = training_features->size(); const auto start = training_features->size();
Factorizer<FeatureType>::AppendTrainingFeatures( Factorizer<FeatureType>::append_training_features(
base_index, training_features); base_index, training_features);
for (auto i = start; i < training_features->size(); ++i) { for (auto i = start; i < training_features->size(); ++i) {
auto& feature = (*training_features)[i]; auto& feature = (*training_features)[i];
assert(feature.GetIndex() < Factorizer<FeatureType>::GetDimensions()); assert(feature.get_index() < Factorizer<FeatureType>::get_dimensions());
feature.ShiftIndex(index_offset); feature.shift_index(index_offset);
} }
return properties.dimensions; return properties.dimensions;
@@ -73,7 +73,7 @@ namespace Eval::NNUE::Features {
// Return the index difference as needed, without adding learning features // Return the index difference as needed, without adding learning features
// Call instead of InheritFeaturesIfRequired() if there are no corresponding features // Call instead of InheritFeaturesIfRequired() if there are no corresponding features
IndexType SkipFeatures(FeatureProperties properties) { IndexType skip_features(FeatureProperties properties) {
if (!properties.active) if (!properties.active)
return 0; return 0;
@@ -82,7 +82,7 @@ namespace Eval::NNUE::Features {
// Get the dimensionality of the learning feature // Get the dimensionality of the learning feature
template <std::size_t N> template <std::size_t N>
constexpr IndexType GetActiveDimensions( constexpr IndexType get_active_dimensions(
const FeatureProperties (&properties)[N]) { const FeatureProperties (&properties)[N]) {
static_assert(N > 0, ""); static_assert(N > 0, "");
@@ -100,7 +100,7 @@ namespace Eval::NNUE::Features {
// get the number of elements in the array // get the number of elements in the array
template <typename T, std::size_t N> template <typename T, std::size_t N>
constexpr std::size_t GetArrayLength(const T (&/*array*/)[N]) { constexpr std::size_t get_array_length(const T (&/*array*/)[N]) {
return N; return N;
} }
@@ -22,12 +22,12 @@ namespace Eval::NNUE::Features {
FeatureSet<FirstFeatureType, RemainingFeatureTypes...>::kDimensions; FeatureSet<FirstFeatureType, RemainingFeatureTypes...>::kDimensions;
// Get the dimensionality of the learning feature // Get the dimensionality of the learning feature
static constexpr IndexType GetDimensions() { static constexpr IndexType get_dimensions() {
return Head::GetDimensions() + Tail::GetDimensions(); return Head::get_dimensions() + Tail::get_dimensions();
} }
// Get index of learning feature and scale of learning rate // Get index of learning feature and scale of learning rate
static void AppendTrainingFeatures( static void append_training_features(
IndexType base_index, std::vector<TrainingFeature>* training_features, IndexType base_index, std::vector<TrainingFeature>* training_features,
IndexType base_dimensions = kBaseDimensions) { IndexType base_dimensions = kBaseDimensions) {
@@ -36,29 +36,29 @@ namespace Eval::NNUE::Features {
constexpr auto boundary = FeatureSet<RemainingFeatureTypes...>::kDimensions; constexpr auto boundary = FeatureSet<RemainingFeatureTypes...>::kDimensions;
if (base_index < boundary) { if (base_index < boundary) {
Tail::AppendTrainingFeatures( Tail::append_training_features(
base_index, training_features, base_dimensions); base_index, training_features, base_dimensions);
} }
else { else {
const auto start = training_features->size(); const auto start = training_features->size();
Head::AppendTrainingFeatures( Head::append_training_features(
base_index - boundary, training_features, base_dimensions); base_index - boundary, training_features, base_dimensions);
for (auto i = start; i < training_features->size(); ++i) { for (auto i = start; i < training_features->size(); ++i) {
auto& feature = (*training_features)[i]; auto& feature = (*training_features)[i];
const auto index = feature.GetIndex(); const auto index = feature.get_index();
assert(index < Head::GetDimensions() || assert(index < Head::get_dimensions() ||
(index >= base_dimensions && (index >= base_dimensions &&
index < base_dimensions + index < base_dimensions +
Head::GetDimensions() - Head::kBaseDimensions)); Head::get_dimensions() - Head::kBaseDimensions));
if (index < Head::kBaseDimensions) { if (index < Head::kBaseDimensions) {
feature.ShiftIndex(Tail::kBaseDimensions); feature.shift_index(Tail::kBaseDimensions);
} }
else { else {
feature.ShiftIndex(Tail::GetDimensions() - Tail::kBaseDimensions); feature.shift_index(Tail::get_dimensions() - Tail::kBaseDimensions);
} }
} }
} }
@@ -74,12 +74,12 @@ namespace Eval::NNUE::Features {
static constexpr IndexType kBaseDimensions = FeatureType::kDimensions; static constexpr IndexType kBaseDimensions = FeatureType::kDimensions;
// Get the dimensionality of the learning feature // Get the dimensionality of the learning feature
static constexpr IndexType GetDimensions() { static constexpr IndexType get_dimensions() {
return Factorizer<FeatureType>::GetDimensions(); return Factorizer<FeatureType>::get_dimensions();
} }
// Get index of learning feature and scale of learning rate // Get index of learning feature and scale of learning rate
static void AppendTrainingFeatures( static void append_training_features(
IndexType base_index, std::vector<TrainingFeature>* training_features, IndexType base_index, std::vector<TrainingFeature>* training_features,
IndexType base_dimensions = kBaseDimensions) { IndexType base_dimensions = kBaseDimensions) {
@@ -87,14 +87,14 @@ namespace Eval::NNUE::Features {
const auto start = training_features->size(); const auto start = training_features->size();
Factorizer<FeatureType>::AppendTrainingFeatures( Factorizer<FeatureType>::append_training_features(
base_index, training_features); base_index, training_features);
for (auto i = start; i < training_features->size(); ++i) { for (auto i = start; i < training_features->size(); ++i) {
auto& feature = (*training_features)[i]; auto& feature = (*training_features)[i];
assert(feature.GetIndex() < Factorizer<FeatureType>::GetDimensions()); assert(feature.get_index() < Factorizer<FeatureType>::get_dimensions());
if (feature.GetIndex() >= kBaseDimensions) { if (feature.get_index() >= kBaseDimensions) {
feature.ShiftIndex(base_dimensions - kBaseDimensions); feature.shift_index(base_dimensions - kBaseDimensions);
} }
} }
} }
+12 -12
View File
@@ -37,25 +37,25 @@ namespace Eval::NNUE::Features {
// kFeaturesHalfK // kFeaturesHalfK
{true, SQUARE_NB}, {true, SQUARE_NB},
// kFeaturesP // kFeaturesP
{true, Factorizer<P>::GetDimensions()}, {true, Factorizer<P>::get_dimensions()},
// kFeaturesHalfRelativeKP // kFeaturesHalfRelativeKP
{true, Factorizer<HalfRelativeKP<AssociatedKing>>::GetDimensions()}, {true, Factorizer<HalfRelativeKP<AssociatedKing>>::get_dimensions()},
}; };
static_assert(GetArrayLength(kProperties) == kNumTrainingFeatureTypes, ""); static_assert(get_array_length(kProperties) == kNumTrainingFeatureTypes, "");
public: public:
// Get the dimensionality of the learning feature // Get the dimensionality of the learning feature
static constexpr IndexType GetDimensions() { static constexpr IndexType get_dimensions() {
return GetActiveDimensions(kProperties); return get_active_dimensions(kProperties);
} }
// Get index of learning feature and scale of learning rate // Get index of learning feature and scale of learning rate
static void AppendTrainingFeatures( static void append_training_features(
IndexType base_index, std::vector<TrainingFeature>* training_features) { IndexType base_index, std::vector<TrainingFeature>* training_features) {
// kFeaturesHalfKP // kFeaturesHalfKP
IndexType index_offset = AppendBaseFeature<FeatureType>( IndexType index_offset = append_base_feature<FeatureType>(
kProperties[kFeaturesHalfKP], base_index, training_features); kProperties[kFeaturesHalfKP], base_index, training_features);
const auto sq_k = static_cast<Square>(base_index / PS_END); const auto sq_k = static_cast<Square>(base_index / PS_END);
@@ -71,20 +71,20 @@ namespace Eval::NNUE::Features {
} }
// kFeaturesP // kFeaturesP
index_offset += InheritFeaturesIfRequired<P>( index_offset += inherit_features_if_required<P>(
index_offset, kProperties[kFeaturesP], p, training_features); index_offset, kProperties[kFeaturesP], p, training_features);
// kFeaturesHalfRelativeKP // kFeaturesHalfRelativeKP
if (p >= PS_W_PAWN) { if (p >= PS_W_PAWN) {
index_offset += InheritFeaturesIfRequired<HalfRelativeKP<AssociatedKing>>( index_offset += inherit_features_if_required<HalfRelativeKP<AssociatedKing>>(
index_offset, kProperties[kFeaturesHalfRelativeKP], index_offset, kProperties[kFeaturesHalfRelativeKP],
HalfRelativeKP<AssociatedKing>::MakeIndex(sq_k, p), HalfRelativeKP<AssociatedKing>::make_index(sq_k, p),
training_features); training_features);
} }
else { else {
index_offset += SkipFeatures(kProperties[kFeaturesHalfRelativeKP]); index_offset += skip_features(kProperties[kFeaturesHalfRelativeKP]);
} }
assert(index_offset == GetDimensions()); assert(index_offset == get_dimensions());
} }
}; };
+10 -23
View File
@@ -37,22 +37,22 @@ namespace Eval::NNUE {
} }
TrainingFeature& operator+=(const TrainingFeature& other) { TrainingFeature& operator+=(const TrainingFeature& other) {
assert(other.GetIndex() == GetIndex()); assert(other.get_index() == get_index());
assert(other.GetCount() + GetCount() < (1 << kCountBits)); assert(other.get_index() + get_count() < (1 << kCountBits));
index_and_count_ += other.GetCount(); index_and_count_ += other.get_count();
return *this; return *this;
} }
IndexType GetIndex() const { IndexType get_index() const {
return static_cast<IndexType>(index_and_count_ >> kCountBits); return static_cast<IndexType>(index_and_count_ >> kCountBits);
} }
void ShiftIndex(IndexType offset) { void shift_index(IndexType offset) {
assert(GetIndex() + offset < (1 << kIndexBits)); assert(get_index() + offset < (1 << kIndexBits));
index_and_count_ += offset << kCountBits; index_and_count_ += offset << kCountBits;
} }
IndexType GetCount() const { IndexType get_count() const {
return static_cast<IndexType>(index_and_count_ & ((1 << kCountBits) - 1)); return static_cast<IndexType>(index_and_count_ & ((1 << kCountBits) - 1));
} }
@@ -86,7 +86,7 @@ namespace Eval::NNUE {
}; };
// determine whether to accept the message // determine whether to accept the message
bool ReceiveMessage(const std::string& name, Message* message) { bool receive_message(const std::string& name, Message* message) {
const auto subscript = "[" + std::to_string(message->num_peekers) + "]"; const auto subscript = "[" + std::to_string(message->num_peekers) + "]";
if (message->name.substr(0, name.size() + 1) == name + "[") { if (message->name.substr(0, name.size() + 1) == name + "[") {
@@ -101,28 +101,15 @@ namespace Eval::NNUE {
return false; return false;
} }
// split the string
std::vector<std::string> Split(const std::string& input, char delimiter) {
std::istringstream stream(input);
std::string field;
std::vector<std::string> fields;
while (std::getline(stream, field, delimiter)) {
fields.push_back(field);
}
return fields;
}
// round a floating point number to an integer // round a floating point number to an integer
template <typename IntType> template <typename IntType>
IntType Round(double value) { IntType round(double value) {
return static_cast<IntType>(std::floor(value + 0.5)); return static_cast<IntType>(std::floor(value + 0.5));
} }
// make_shared with alignment // make_shared with alignment
template <typename T, typename... ArgumentTypes> template <typename T, typename... ArgumentTypes>
std::shared_ptr<T> MakeAlignedSharedPtr(ArgumentTypes&&... arguments) { std::shared_ptr<T> make_aligned_shared_ptr(ArgumentTypes&&... arguments) {
const auto ptr = new(std_aligned_alloc(alignof(T), sizeof(T))) const auto ptr = new(std_aligned_alloc(alignof(T), sizeof(T)))
T(std::forward<ArgumentTypes>(arguments)...); T(std::forward<ArgumentTypes>(arguments)...);
+22 -22
View File
@@ -21,7 +21,7 @@ namespace Eval::NNUE {
public: public:
// factory function // factory function
static std::shared_ptr<Trainer> Create( static std::shared_ptr<Trainer> create(
LayerType* target_layer, FeatureTransformer* ft) { LayerType* target_layer, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>( return std::shared_ptr<Trainer>(
@@ -29,31 +29,31 @@ namespace Eval::NNUE {
} }
// Set options such as hyperparameters // Set options such as hyperparameters
void SendMessage(Message* message) { void send_message(Message* message) {
previous_layer_trainer_->SendMessage(message); previous_layer_trainer_->send_message(message);
if (ReceiveMessage("momentum", message)) { if (receive_message("momentum", message)) {
momentum_ = static_cast<LearnFloatType>(std::stod(message->value)); momentum_ = static_cast<LearnFloatType>(std::stod(message->value));
} }
if (ReceiveMessage("learning_rate_scale", message)) { if (receive_message("learning_rate_scale", message)) {
learning_rate_scale_ = learning_rate_scale_ =
static_cast<LearnFloatType>(std::stod(message->value)); static_cast<LearnFloatType>(std::stod(message->value));
} }
if (ReceiveMessage("reset", message)) { if (receive_message("reset", message)) {
DequantizeParameters(); dequantize_parameters();
} }
if (ReceiveMessage("quantize_parameters", message)) { if (receive_message("quantize_parameters", message)) {
QuantizeParameters(); quantize_parameters();
} }
} }
// Initialize the parameters with random numbers // Initialize the parameters with random numbers
template <typename RNG> template <typename RNG>
void Initialize(RNG& rng) { void initialize(RNG& rng) {
previous_layer_trainer_->Initialize(rng); previous_layer_trainer_->initialize(rng);
if (kIsOutputLayer) { if (kIsOutputLayer) {
// Initialize output layer with 0 // Initialize output layer with 0
@@ -80,18 +80,18 @@ namespace Eval::NNUE {
} }
} }
QuantizeParameters(); quantize_parameters();
} }
// forward propagation // forward propagation
const LearnFloatType* Propagate(const std::vector<Example>& batch) { const LearnFloatType* propagate(const std::vector<Example>& batch) {
if (output_.size() < kOutputDimensions * batch.size()) { if (output_.size() < kOutputDimensions * batch.size()) {
output_.resize(kOutputDimensions * batch.size()); output_.resize(kOutputDimensions * batch.size());
gradients_.resize(kInputDimensions * batch.size()); gradients_.resize(kInputDimensions * batch.size());
} }
batch_size_ = static_cast<IndexType>(batch.size()); batch_size_ = static_cast<IndexType>(batch.size());
batch_input_ = previous_layer_trainer_->Propagate(batch); batch_input_ = previous_layer_trainer_->propagate(batch);
#if defined(USE_BLAS) #if defined(USE_BLAS)
for (IndexType b = 0; b < batch_size_; ++b) { for (IndexType b = 0; b < batch_size_; ++b) {
const IndexType batch_offset = kOutputDimensions * b; const IndexType batch_offset = kOutputDimensions * b;
@@ -123,7 +123,7 @@ namespace Eval::NNUE {
} }
// backpropagation // backpropagation
void Backpropagate(const LearnFloatType* gradients, void backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) { LearnFloatType learning_rate) {
const LearnFloatType local_learning_rate = const LearnFloatType local_learning_rate =
@@ -206,7 +206,7 @@ namespace Eval::NNUE {
} }
#endif #endif
previous_layer_trainer_->Backpropagate(gradients_.data(), learning_rate); previous_layer_trainer_->backpropagate(gradients_.data(), learning_rate);
} }
private: private:
@@ -214,7 +214,7 @@ namespace Eval::NNUE {
Trainer(LayerType* target_layer, FeatureTransformer* ft) : Trainer(LayerType* target_layer, FeatureTransformer* ft) :
batch_size_(0), batch_size_(0),
batch_input_(nullptr), batch_input_(nullptr),
previous_layer_trainer_(Trainer<PreviousLayer>::Create( previous_layer_trainer_(Trainer<PreviousLayer>::create(
&target_layer->previous_layer_, ft)), &target_layer->previous_layer_, ft)),
target_layer_(target_layer), target_layer_(target_layer),
biases_(), biases_(),
@@ -224,11 +224,11 @@ namespace Eval::NNUE {
momentum_(0.2), momentum_(0.2),
learning_rate_scale_(1.0) { learning_rate_scale_(1.0) {
DequantizeParameters(); dequantize_parameters();
} }
// Weight saturation and parameterization // Weight saturation and parameterization
void QuantizeParameters() { void quantize_parameters() {
for (IndexType i = 0; i < kOutputDimensions * kInputDimensions; ++i) { for (IndexType i = 0; i < kOutputDimensions * kInputDimensions; ++i) {
weights_[i] = std::max(-kMaxWeightMagnitude, weights_[i] = std::max(-kMaxWeightMagnitude,
std::min(+kMaxWeightMagnitude, weights_[i])); std::min(+kMaxWeightMagnitude, weights_[i]));
@@ -236,7 +236,7 @@ namespace Eval::NNUE {
for (IndexType i = 0; i < kOutputDimensions; ++i) { for (IndexType i = 0; i < kOutputDimensions; ++i) {
target_layer_->biases_[i] = target_layer_->biases_[i] =
Round<typename LayerType::BiasType>(biases_[i] * kBiasScale); round<typename LayerType::BiasType>(biases_[i] * kBiasScale);
} }
for (IndexType i = 0; i < kOutputDimensions; ++i) { for (IndexType i = 0; i < kOutputDimensions; ++i) {
@@ -244,14 +244,14 @@ namespace Eval::NNUE {
const auto padded_offset = LayerType::kPaddedInputDimensions * i; const auto padded_offset = LayerType::kPaddedInputDimensions * i;
for (IndexType j = 0; j < kInputDimensions; ++j) { for (IndexType j = 0; j < kInputDimensions; ++j) {
target_layer_->weights_[padded_offset + j] = target_layer_->weights_[padded_offset + j] =
Round<typename LayerType::WeightType>( round<typename LayerType::WeightType>(
weights_[offset + j] * kWeightScale); weights_[offset + j] * kWeightScale);
} }
} }
} }
// read parameterized integer // read parameterized integer
void DequantizeParameters() { void dequantize_parameters() {
for (IndexType i = 0; i < kOutputDimensions; ++i) { for (IndexType i = 0; i < kOutputDimensions; ++i) {
biases_[i] = static_cast<LearnFloatType>( biases_[i] = static_cast<LearnFloatType>(
target_layer_->biases_[i] / kBiasScale); target_layer_->biases_[i] / kBiasScale);
+13 -13
View File
@@ -19,7 +19,7 @@ namespace Eval::NNUE {
public: public:
// factory function // factory function
static std::shared_ptr<Trainer> Create( static std::shared_ptr<Trainer> create(
LayerType* target_layer, FeatureTransformer* ft) { LayerType* target_layer, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>( return std::shared_ptr<Trainer>(
@@ -27,27 +27,27 @@ namespace Eval::NNUE {
} }
// Set options such as hyperparameters // Set options such as hyperparameters
void SendMessage(Message* message) { void send_message(Message* message) {
previous_layer_trainer_->SendMessage(message); previous_layer_trainer_->send_message(message);
if (ReceiveMessage("check_health", message)) { if (receive_message("check_health", message)) {
CheckHealth(); check_health();
} }
} }
// Initialize the parameters with random numbers // Initialize the parameters with random numbers
template <typename RNG> template <typename RNG>
void Initialize(RNG& rng) { void initialize(RNG& rng) {
previous_layer_trainer_->Initialize(rng); previous_layer_trainer_->initialize(rng);
} }
// forward propagation // forward propagation
const LearnFloatType* Propagate(const std::vector<Example>& batch) { const LearnFloatType* propagate(const std::vector<Example>& batch) {
if (output_.size() < kOutputDimensions * batch.size()) { if (output_.size() < kOutputDimensions * batch.size()) {
output_.resize(kOutputDimensions * batch.size()); output_.resize(kOutputDimensions * batch.size());
gradients_.resize(kInputDimensions * batch.size()); gradients_.resize(kInputDimensions * batch.size());
} }
const auto input = previous_layer_trainer_->Propagate(batch); const auto input = previous_layer_trainer_->propagate(batch);
batch_size_ = static_cast<IndexType>(batch.size()); batch_size_ = static_cast<IndexType>(batch.size());
for (IndexType b = 0; b < batch_size_; ++b) { for (IndexType b = 0; b < batch_size_; ++b) {
const IndexType batch_offset = kOutputDimensions * b; const IndexType batch_offset = kOutputDimensions * b;
@@ -63,7 +63,7 @@ namespace Eval::NNUE {
} }
// backpropagation // backpropagation
void Backpropagate(const LearnFloatType* gradients, void backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) { LearnFloatType learning_rate) {
for (IndexType b = 0; b < batch_size_; ++b) { for (IndexType b = 0; b < batch_size_; ++b) {
@@ -75,14 +75,14 @@ namespace Eval::NNUE {
} }
} }
previous_layer_trainer_->Backpropagate(gradients_.data(), learning_rate); previous_layer_trainer_->backpropagate(gradients_.data(), learning_rate);
} }
private: private:
// constructor // constructor
Trainer(LayerType* target_layer, FeatureTransformer* ft) : Trainer(LayerType* target_layer, FeatureTransformer* ft) :
batch_size_(0), batch_size_(0),
previous_layer_trainer_(Trainer<PreviousLayer>::Create( previous_layer_trainer_(Trainer<PreviousLayer>::create(
&target_layer->previous_layer_, ft)), &target_layer->previous_layer_, ft)),
target_layer_(target_layer) { target_layer_(target_layer) {
@@ -93,7 +93,7 @@ namespace Eval::NNUE {
} }
// Check if there are any problems with learning // Check if there are any problems with learning
void CheckHealth() { void check_health() {
const auto largest_min_activation = *std::max_element( const auto largest_min_activation = *std::max_element(
std::begin(min_activations_), std::end(min_activations_)); std::begin(min_activations_), std::end(min_activations_));
const auto smallest_max_activation = *std::min_element( const auto smallest_max_activation = *std::min_element(
+39 -39
View File
@@ -34,44 +34,44 @@ namespace Eval::NNUE {
friend struct AlignedDeleter; friend struct AlignedDeleter;
template <typename T, typename... ArgumentTypes> template <typename T, typename... ArgumentTypes>
friend std::shared_ptr<T> MakeAlignedSharedPtr(ArgumentTypes&&... arguments); friend std::shared_ptr<T> make_aligned_shared_ptr(ArgumentTypes&&... arguments);
// factory function // factory function
static std::shared_ptr<Trainer> Create(LayerType* target_layer) { static std::shared_ptr<Trainer> create(LayerType* target_layer) {
return MakeAlignedSharedPtr<Trainer>(target_layer); return make_aligned_shared_ptr<Trainer>(target_layer);
} }
// Set options such as hyperparameters // Set options such as hyperparameters
void SendMessage(Message* message) { void send_message(Message* message) {
if (ReceiveMessage("momentum", message)) { if (receive_message("momentum", message)) {
momentum_ = static_cast<LearnFloatType>(std::stod(message->value)); momentum_ = static_cast<LearnFloatType>(std::stod(message->value));
} }
if (ReceiveMessage("learning_rate_scale", message)) { if (receive_message("learning_rate_scale", message)) {
learning_rate_scale_ = learning_rate_scale_ =
static_cast<LearnFloatType>(std::stod(message->value)); static_cast<LearnFloatType>(std::stod(message->value));
} }
if (ReceiveMessage("reset", message)) { if (receive_message("reset", message)) {
DequantizeParameters(); dequantize_parameters();
} }
if (ReceiveMessage("quantize_parameters", message)) { if (receive_message("quantize_parameters", message)) {
QuantizeParameters(); quantize_parameters();
} }
if (ReceiveMessage("clear_unobserved_feature_weights", message)) { if (receive_message("clear_unobserved_feature_weights", message)) {
ClearUnobservedFeatureWeights(); clear_unobserved_feature_weights();
} }
if (ReceiveMessage("check_health", message)) { if (receive_message("check_health", message)) {
CheckHealth(); check_health();
} }
} }
// Initialize the parameters with random numbers // Initialize the parameters with random numbers
template <typename RNG> template <typename RNG>
void Initialize(RNG& rng) { void initialize(RNG& rng) {
std::fill(std::begin(weights_), std::end(weights_), +kZero); std::fill(std::begin(weights_), std::end(weights_), +kZero);
const double kSigma = 0.1 / std::sqrt(RawFeatures::kMaxActiveDimensions); const double kSigma = 0.1 / std::sqrt(RawFeatures::kMaxActiveDimensions);
@@ -86,11 +86,11 @@ namespace Eval::NNUE {
biases_[i] = static_cast<LearnFloatType>(0.5); biases_[i] = static_cast<LearnFloatType>(0.5);
} }
QuantizeParameters(); quantize_parameters();
} }
// forward propagation // forward propagation
const LearnFloatType* Propagate(const std::vector<Example>& batch) { const LearnFloatType* propagate(const std::vector<Example>& batch) {
if (output_.size() < kOutputDimensions * batch.size()) { if (output_.size() < kOutputDimensions * batch.size()) {
output_.resize(kOutputDimensions * batch.size()); output_.resize(kOutputDimensions * batch.size());
gradients_.resize(kOutputDimensions * batch.size()); gradients_.resize(kOutputDimensions * batch.size());
@@ -106,8 +106,8 @@ namespace Eval::NNUE {
#if defined(USE_BLAS) #if defined(USE_BLAS)
cblas_scopy(kHalfDimensions, biases_, 1, &output_[output_offset], 1); cblas_scopy(kHalfDimensions, biases_, 1, &output_[output_offset], 1);
for (const auto& feature : batch[b].training_features[c]) { for (const auto& feature : batch[b].training_features[c]) {
const IndexType weights_offset = kHalfDimensions * feature.GetIndex(); const IndexType weights_offset = kHalfDimensions * feature.get_index();
cblas_saxpy(kHalfDimensions, (float)feature.GetCount(), cblas_saxpy(kHalfDimensions, (float)feature.get_count(),
&weights_[weights_offset], 1, &output_[output_offset], 1); &weights_[weights_offset], 1, &output_[output_offset], 1);
} }
#else #else
@@ -115,10 +115,10 @@ namespace Eval::NNUE {
output_[output_offset + i] = biases_[i]; output_[output_offset + i] = biases_[i];
} }
for (const auto& feature : batch[b].training_features[c]) { for (const auto& feature : batch[b].training_features[c]) {
const IndexType weights_offset = kHalfDimensions * feature.GetIndex(); const IndexType weights_offset = kHalfDimensions * feature.get_index();
for (IndexType i = 0; i < kHalfDimensions; ++i) { for (IndexType i = 0; i < kHalfDimensions; ++i) {
output_[output_offset + i] += output_[output_offset + i] +=
feature.GetCount() * weights_[weights_offset + i]; feature.get_count() * weights_[weights_offset + i];
} }
} }
#endif #endif
@@ -143,7 +143,7 @@ namespace Eval::NNUE {
} }
// backpropagation // backpropagation
void Backpropagate(const LearnFloatType* gradients, void backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) { LearnFloatType learning_rate) {
const LearnFloatType local_learning_rate = const LearnFloatType local_learning_rate =
@@ -188,13 +188,13 @@ namespace Eval::NNUE {
const IndexType output_offset = batch_offset + kHalfDimensions * c; const IndexType output_offset = batch_offset + kHalfDimensions * c;
for (const auto& feature : (*batch_)[b].training_features[c]) { for (const auto& feature : (*batch_)[b].training_features[c]) {
#if defined(_OPENMP) #if defined(_OPENMP)
if (feature.GetIndex() % num_threads != thread_index) if (feature.get_index() % num_threads != thread_index)
continue; continue;
#endif #endif
const IndexType weights_offset = const IndexType weights_offset =
kHalfDimensions * feature.GetIndex(); kHalfDimensions * feature.get_index();
const auto scale = static_cast<LearnFloatType>( const auto scale = static_cast<LearnFloatType>(
effective_learning_rate / feature.GetCount()); effective_learning_rate / feature.get_count());
cblas_saxpy(kHalfDimensions, -scale, cblas_saxpy(kHalfDimensions, -scale,
&gradients_[output_offset], 1, &gradients_[output_offset], 1,
@@ -228,9 +228,9 @@ namespace Eval::NNUE {
for (IndexType c = 0; c < 2; ++c) { for (IndexType c = 0; c < 2; ++c) {
const IndexType output_offset = batch_offset + kHalfDimensions * c; const IndexType output_offset = batch_offset + kHalfDimensions * c;
for (const auto& feature : (*batch_)[b].training_features[c]) { for (const auto& feature : (*batch_)[b].training_features[c]) {
const IndexType weights_offset = kHalfDimensions * feature.GetIndex(); const IndexType weights_offset = kHalfDimensions * feature.get_index();
const auto scale = static_cast<LearnFloatType>( const auto scale = static_cast<LearnFloatType>(
effective_learning_rate / feature.GetCount()); effective_learning_rate / feature.get_count());
for (IndexType i = 0; i < kHalfDimensions; ++i) { for (IndexType i = 0; i < kHalfDimensions; ++i) {
weights_[weights_offset + i] -= weights_[weights_offset + i] -=
@@ -244,7 +244,7 @@ namespace Eval::NNUE {
for (IndexType b = 0; b < batch_->size(); ++b) { for (IndexType b = 0; b < batch_->size(); ++b) {
for (IndexType c = 0; c < 2; ++c) { for (IndexType c = 0; c < 2; ++c) {
for (const auto& feature : (*batch_)[b].training_features[c]) { for (const auto& feature : (*batch_)[b].training_features[c]) {
observed_features.set(feature.GetIndex()); observed_features.set(feature.get_index());
} }
} }
} }
@@ -269,14 +269,14 @@ namespace Eval::NNUE {
std::fill(std::begin(max_activations_), std::end(max_activations_), std::fill(std::begin(max_activations_), std::end(max_activations_),
std::numeric_limits<LearnFloatType>::lowest()); std::numeric_limits<LearnFloatType>::lowest());
DequantizeParameters(); dequantize_parameters();
} }
// Weight saturation and parameterization // Weight saturation and parameterization
void QuantizeParameters() { void quantize_parameters() {
for (IndexType i = 0; i < kHalfDimensions; ++i) { for (IndexType i = 0; i < kHalfDimensions; ++i) {
target_layer_->biases_[i] = target_layer_->biases_[i] =
Round<typename LayerType::BiasType>(biases_[i] * kBiasScale); round<typename LayerType::BiasType>(biases_[i] * kBiasScale);
} }
std::vector<TrainingFeature> training_features; std::vector<TrainingFeature> training_features;
@@ -284,23 +284,23 @@ namespace Eval::NNUE {
#pragma omp parallel for private(training_features) #pragma omp parallel for private(training_features)
for (IndexType j = 0; j < RawFeatures::kDimensions; ++j) { for (IndexType j = 0; j < RawFeatures::kDimensions; ++j) {
training_features.clear(); training_features.clear();
Features::Factorizer<RawFeatures>::AppendTrainingFeatures( Features::Factorizer<RawFeatures>::append_training_features(
j, &training_features); j, &training_features);
for (IndexType i = 0; i < kHalfDimensions; ++i) { for (IndexType i = 0; i < kHalfDimensions; ++i) {
double sum = 0.0; double sum = 0.0;
for (const auto& feature : training_features) { for (const auto& feature : training_features) {
sum += weights_[kHalfDimensions * feature.GetIndex() + i]; sum += weights_[kHalfDimensions * feature.get_index() + i];
} }
target_layer_->weights_[kHalfDimensions * j + i] = target_layer_->weights_[kHalfDimensions * j + i] =
Round<typename LayerType::WeightType>(sum * kWeightScale); round<typename LayerType::WeightType>(sum * kWeightScale);
} }
} }
} }
// read parameterized integer // read parameterized integer
void DequantizeParameters() { void dequantize_parameters() {
for (IndexType i = 0; i < kHalfDimensions; ++i) { for (IndexType i = 0; i < kHalfDimensions; ++i) {
biases_[i] = static_cast<LearnFloatType>( biases_[i] = static_cast<LearnFloatType>(
target_layer_->biases_[i] / kBiasScale); target_layer_->biases_[i] / kBiasScale);
@@ -317,7 +317,7 @@ namespace Eval::NNUE {
} }
// Set the weight corresponding to the feature that does not appear in the learning data to 0 // Set the weight corresponding to the feature that does not appear in the learning data to 0
void ClearUnobservedFeatureWeights() { void clear_unobserved_feature_weights() {
for (IndexType i = 0; i < kInputDimensions; ++i) { for (IndexType i = 0; i < kInputDimensions; ++i) {
if (!observed_features.test(i)) { if (!observed_features.test(i)) {
std::fill(std::begin(weights_) + kHalfDimensions * i, std::fill(std::begin(weights_) + kHalfDimensions * i,
@@ -325,11 +325,11 @@ namespace Eval::NNUE {
} }
} }
QuantizeParameters(); quantize_parameters();
} }
// Check if there are any problems with learning // Check if there are any problems with learning
void CheckHealth() { void check_health() {
std::cout << "INFO: observed " << observed_features.count() std::cout << "INFO: observed " << observed_features.count()
<< " (out of " << kInputDimensions << ") features" << std::endl; << " (out of " << kInputDimensions << ") features" << std::endl;
@@ -359,7 +359,7 @@ namespace Eval::NNUE {
// number of input/output dimensions // number of input/output dimensions
static constexpr IndexType kInputDimensions = static constexpr IndexType kInputDimensions =
Features::Factorizer<RawFeatures>::GetDimensions(); Features::Factorizer<RawFeatures>::get_dimensions();
static constexpr IndexType kOutputDimensions = LayerType::kOutputDimensions; static constexpr IndexType kOutputDimensions = LayerType::kOutputDimensions;
static constexpr IndexType kHalfDimensions = LayerType::kHalfDimensions; static constexpr IndexType kHalfDimensions = LayerType::kHalfDimensions;
+21 -21
View File
@@ -14,7 +14,7 @@ namespace Eval::NNUE {
class SharedInputTrainer { class SharedInputTrainer {
public: public:
// factory function // factory function
static std::shared_ptr<SharedInputTrainer> Create( static std::shared_ptr<SharedInputTrainer> create(
FeatureTransformer* ft) { FeatureTransformer* ft) {
static std::shared_ptr<SharedInputTrainer> instance; static std::shared_ptr<SharedInputTrainer> instance;
@@ -29,10 +29,10 @@ namespace Eval::NNUE {
} }
// Set options such as hyperparameters // Set options such as hyperparameters
void SendMessage(Message* message) { void send_message(Message* message) {
if (num_calls_ == 0) { if (num_calls_ == 0) {
current_operation_ = Operation::kSendMessage; current_operation_ = Operation::kSendMessage;
feature_transformer_trainer_->SendMessage(message); feature_transformer_trainer_->send_message(message);
} }
assert(current_operation_ == Operation::kSendMessage); assert(current_operation_ == Operation::kSendMessage);
@@ -45,10 +45,10 @@ namespace Eval::NNUE {
// Initialize the parameters with random numbers // Initialize the parameters with random numbers
template <typename RNG> template <typename RNG>
void Initialize(RNG& rng) { void initialize(RNG& rng) {
if (num_calls_ == 0) { if (num_calls_ == 0) {
current_operation_ = Operation::kInitialize; current_operation_ = Operation::kInitialize;
feature_transformer_trainer_->Initialize(rng); feature_transformer_trainer_->initialize(rng);
} }
assert(current_operation_ == Operation::kInitialize); assert(current_operation_ == Operation::kInitialize);
@@ -60,7 +60,7 @@ namespace Eval::NNUE {
} }
// forward propagation // forward propagation
const LearnFloatType* Propagate(const std::vector<Example>& batch) { const LearnFloatType* propagate(const std::vector<Example>& batch) {
if (gradients_.size() < kInputDimensions * batch.size()) { if (gradients_.size() < kInputDimensions * batch.size()) {
gradients_.resize(kInputDimensions * batch.size()); gradients_.resize(kInputDimensions * batch.size());
} }
@@ -69,7 +69,7 @@ namespace Eval::NNUE {
if (num_calls_ == 0) { if (num_calls_ == 0) {
current_operation_ = Operation::kPropagate; current_operation_ = Operation::kPropagate;
output_ = feature_transformer_trainer_->Propagate(batch); output_ = feature_transformer_trainer_->propagate(batch);
} }
assert(current_operation_ == Operation::kPropagate); assert(current_operation_ == Operation::kPropagate);
@@ -83,11 +83,11 @@ namespace Eval::NNUE {
} }
// backpropagation // backpropagation
void Backpropagate(const LearnFloatType* gradients, void backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) { LearnFloatType learning_rate) {
if (num_referrers_ == 1) { if (num_referrers_ == 1) {
feature_transformer_trainer_->Backpropagate(gradients, learning_rate); feature_transformer_trainer_->backpropagate(gradients, learning_rate);
return; return;
} }
@@ -111,7 +111,7 @@ namespace Eval::NNUE {
} }
if (++num_calls_ == num_referrers_) { if (++num_calls_ == num_referrers_) {
feature_transformer_trainer_->Backpropagate( feature_transformer_trainer_->backpropagate(
gradients_.data(), learning_rate); gradients_.data(), learning_rate);
num_calls_ = 0; num_calls_ = 0;
current_operation_ = Operation::kNone; current_operation_ = Operation::kNone;
@@ -125,7 +125,7 @@ namespace Eval::NNUE {
num_referrers_(0), num_referrers_(0),
num_calls_(0), num_calls_(0),
current_operation_(Operation::kNone), current_operation_(Operation::kNone),
feature_transformer_trainer_(Trainer<FeatureTransformer>::Create( feature_transformer_trainer_(Trainer<FeatureTransformer>::create(
ft)), ft)),
output_(nullptr) { output_(nullptr) {
} }
@@ -175,25 +175,25 @@ namespace Eval::NNUE {
public: public:
// factory function // factory function
static std::shared_ptr<Trainer> Create( static std::shared_ptr<Trainer> create(
LayerType* /*target_layer*/, FeatureTransformer* ft) { LayerType* /*target_layer*/, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>(new Trainer(ft)); return std::shared_ptr<Trainer>(new Trainer(ft));
} }
// Set options such as hyperparameters // Set options such as hyperparameters
void SendMessage(Message* message) { void send_message(Message* message) {
shared_input_trainer_->SendMessage(message); shared_input_trainer_->send_message(message);
} }
// Initialize the parameters with random numbers // Initialize the parameters with random numbers
template <typename RNG> template <typename RNG>
void Initialize(RNG& rng) { void initialize(RNG& rng) {
shared_input_trainer_->Initialize(rng); shared_input_trainer_->initialize(rng);
} }
// forward propagation // forward propagation
const LearnFloatType* Propagate(const std::vector<Example>& batch) { const LearnFloatType* propagate(const std::vector<Example>& batch) {
if (output_.size() < kOutputDimensions * batch.size()) { if (output_.size() < kOutputDimensions * batch.size()) {
output_.resize(kOutputDimensions * batch.size()); output_.resize(kOutputDimensions * batch.size());
gradients_.resize(kInputDimensions * batch.size()); gradients_.resize(kInputDimensions * batch.size());
@@ -201,7 +201,7 @@ namespace Eval::NNUE {
batch_size_ = static_cast<IndexType>(batch.size()); batch_size_ = static_cast<IndexType>(batch.size());
const auto input = shared_input_trainer_->Propagate(batch); const auto input = shared_input_trainer_->propagate(batch);
for (IndexType b = 0; b < batch_size_; ++b) { for (IndexType b = 0; b < batch_size_; ++b) {
const IndexType input_offset = kInputDimensions * b; const IndexType input_offset = kInputDimensions * b;
const IndexType output_offset = kOutputDimensions * b; const IndexType output_offset = kOutputDimensions * b;
@@ -219,7 +219,7 @@ namespace Eval::NNUE {
} }
// backpropagation // backpropagation
void Backpropagate(const LearnFloatType* gradients, void backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) { LearnFloatType learning_rate) {
for (IndexType b = 0; b < batch_size_; ++b) { for (IndexType b = 0; b < batch_size_; ++b) {
@@ -233,14 +233,14 @@ namespace Eval::NNUE {
} }
} }
} }
shared_input_trainer_->Backpropagate(gradients_.data(), learning_rate); shared_input_trainer_->backpropagate(gradients_.data(), learning_rate);
} }
private: private:
// constructor // constructor
Trainer(FeatureTransformer* ft): Trainer(FeatureTransformer* ft):
batch_size_(0), batch_size_(0),
shared_input_trainer_(SharedInputTrainer::Create(ft)) { shared_input_trainer_(SharedInputTrainer::create(ft)) {
} }
// number of input/output dimensions // number of input/output dimensions
+24 -24
View File
@@ -21,7 +21,7 @@ namespace Eval::NNUE {
public: public:
// factory function // factory function
static std::shared_ptr<Trainer> Create( static std::shared_ptr<Trainer> create(
LayerType* target_layer, FeatureTransformer* ft) { LayerType* target_layer, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>( return std::shared_ptr<Trainer>(
@@ -29,26 +29,26 @@ namespace Eval::NNUE {
} }
// Set options such as hyperparameters // Set options such as hyperparameters
void SendMessage(Message* message) { void send_message(Message* message) {
// The results of other member functions do not depend on the processing order, so // The results of other member functions do not depend on the processing order, so
// Tail is processed first for the purpose of simplifying the implementation, but // Tail is processed first for the purpose of simplifying the implementation, but
// SendMessage processes Head first to make it easier to understand subscript correspondence // SendMessage processes Head first to make it easier to understand subscript correspondence
previous_layer_trainer_->SendMessage(message); previous_layer_trainer_->send_message(message);
Tail::SendMessage(message); Tail::send_message(message);
} }
// Initialize the parameters with random numbers // Initialize the parameters with random numbers
template <typename RNG> template <typename RNG>
void Initialize(RNG& rng) { void initialize(RNG& rng) {
Tail::Initialize(rng); Tail::initialize(rng);
previous_layer_trainer_->Initialize(rng); previous_layer_trainer_->initialize(rng);
} }
// forward propagation // forward propagation
/*const*/ LearnFloatType* Propagate(const std::vector<Example>& batch) { /*const*/ LearnFloatType* propagate(const std::vector<Example>& batch) {
batch_size_ = static_cast<IndexType>(batch.size()); batch_size_ = static_cast<IndexType>(batch.size());
auto output = Tail::Propagate(batch); auto output = Tail::propagate(batch);
const auto head_output = previous_layer_trainer_->Propagate(batch); const auto head_output = previous_layer_trainer_->propagate(batch);
#if defined(USE_BLAS) #if defined(USE_BLAS)
cblas_saxpy(kOutputDimensions * batch_size_, 1.0, cblas_saxpy(kOutputDimensions * batch_size_, 1.0,
@@ -66,11 +66,11 @@ namespace Eval::NNUE {
} }
// backpropagation // backpropagation
void Backpropagate(const LearnFloatType* gradients, void backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) { LearnFloatType learning_rate) {
Tail::Backpropagate(gradients, learning_rate); Tail::backpropagate(gradients, learning_rate);
previous_layer_trainer_->Backpropagate(gradients, learning_rate); previous_layer_trainer_->backpropagate(gradients, learning_rate);
} }
private: private:
@@ -78,7 +78,7 @@ namespace Eval::NNUE {
Trainer(LayerType* target_layer, FeatureTransformer* ft): Trainer(LayerType* target_layer, FeatureTransformer* ft):
Tail(target_layer, ft), Tail(target_layer, ft),
batch_size_(0), batch_size_(0),
previous_layer_trainer_(Trainer<FirstPreviousLayer>::Create( previous_layer_trainer_(Trainer<FirstPreviousLayer>::create(
&target_layer->previous_layer_, ft)), &target_layer->previous_layer_, ft)),
target_layer_(target_layer) { target_layer_(target_layer) {
} }
@@ -110,7 +110,7 @@ namespace Eval::NNUE {
public: public:
// factory function // factory function
static std::shared_ptr<Trainer> Create( static std::shared_ptr<Trainer> create(
LayerType* target_layer, FeatureTransformer* ft) { LayerType* target_layer, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>( return std::shared_ptr<Trainer>(
@@ -118,24 +118,24 @@ namespace Eval::NNUE {
} }
// Set options such as hyperparameters // Set options such as hyperparameters
void SendMessage(Message* message) { void send_message(Message* message) {
previous_layer_trainer_->SendMessage(message); previous_layer_trainer_->send_message(message);
} }
// Initialize the parameters with random numbers // Initialize the parameters with random numbers
template <typename RNG> template <typename RNG>
void Initialize(RNG& rng) { void initialize(RNG& rng) {
previous_layer_trainer_->Initialize(rng); previous_layer_trainer_->initialize(rng);
} }
// forward propagation // forward propagation
/*const*/ LearnFloatType* Propagate(const std::vector<Example>& batch) { /*const*/ LearnFloatType* propagate(const std::vector<Example>& batch) {
if (output_.size() < kOutputDimensions * batch.size()) { if (output_.size() < kOutputDimensions * batch.size()) {
output_.resize(kOutputDimensions * batch.size()); output_.resize(kOutputDimensions * batch.size());
} }
batch_size_ = static_cast<IndexType>(batch.size()); batch_size_ = static_cast<IndexType>(batch.size());
const auto output = previous_layer_trainer_->Propagate(batch); const auto output = previous_layer_trainer_->propagate(batch);
#if defined(USE_BLAS) #if defined(USE_BLAS)
cblas_scopy(kOutputDimensions * batch_size_, output, 1, &output_[0], 1); cblas_scopy(kOutputDimensions * batch_size_, output, 1, &output_[0], 1);
@@ -152,17 +152,17 @@ namespace Eval::NNUE {
} }
// backpropagation // backpropagation
void Backpropagate(const LearnFloatType* gradients, void backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) { LearnFloatType learning_rate) {
previous_layer_trainer_->Backpropagate(gradients, learning_rate); previous_layer_trainer_->backpropagate(gradients, learning_rate);
} }
private: private:
// constructor // constructor
Trainer(LayerType* target_layer, FeatureTransformer* ft) : Trainer(LayerType* target_layer, FeatureTransformer* ft) :
batch_size_(0), batch_size_(0),
previous_layer_trainer_(Trainer<PreviousLayer>::Create( previous_layer_trainer_(Trainer<PreviousLayer>::create(
&target_layer->previous_layer_, ft)), &target_layer->previous_layer_, ft)),
target_layer_(target_layer) { target_layer_(target_layer) {
} }
+1 -1
View File
@@ -53,7 +53,7 @@ void test_cmd(Position& pos, istringstream& is)
std::string param; std::string param;
is >> param; is >> param;
if (param == "nnue") Eval::NNUE::TestCommand(pos, is); if (param == "nnue") Eval::NNUE::test_command(pos, is);
} }
namespace { namespace {