opelly/Stockfish
1
0
Fork 0
mirror of https://github.com/opelly27/Stockfish.git synced 2026-05-20 06:17:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup trainer.

Browse Source
This commit is contained in:
Tomasz Sobczyk
2020-10-14 21:26:03 +02:00
committed by nodchip
parent ea8eb415de
commit c286f9cd7d
6 changed files with 1263 additions and 1153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nnue/trainer/trainer.h
+52 -39
View File
@@ -1,121 +1,134 @@
// Common header of class template for learning NNUE evaluation function
#ifndef _NNUE_TRAINER_H_
#ifndef _NNUE_TRAINER_H_
#define _NNUE_TRAINER_H_
#include "../nnue_common.h"
#include "../features/index_list.h"
#include "nnue/nnue_common.h"
#include "nnue/features/index_list.h"
#include <sstream>
#if defined(USE_BLAS)
static_assert(std::is_same<LearnFloatType, float>::value, "");
#include <cblas.h>
#endif
namespace Eval {
// Common header of class template for learning NNUE evaluation function
namespace Eval::NNUE {
namespace NNUE {
// Ponanza constant used in the relation between evaluation value and winning percentage
constexpr double kPonanzaConstant = 600.0;
// Ponanza constant used in the relation between evaluation value and winning percentage
constexpr double kPonanzaConstant = 600.0;
// Class that represents one index of learning feature
class TrainingFeature {
// Class that represents one index of learning feature
class TrainingFeature {
using StorageType = std::uint32_t;
static_assert(std::is_unsigned<StorageType>::value, "");
public:
static constexpr std::uint32_t kIndexBits = 24;
static_assert(kIndexBits < std::numeric_limits<StorageType>::digits, "");
static constexpr std::uint32_t kCountBits =
std::numeric_limits<StorageType>::digits - kIndexBits;
explicit TrainingFeature(IndexType index) :
index_and_count_((index << kCountBits) | 1) {
assert(index < (1 << kIndexBits));
}
TrainingFeature& operator+=(const TrainingFeature& other) {
assert(other.GetIndex() == GetIndex());
assert(other.GetCount() + GetCount() < (1 << kCountBits));
index_and_count_ += other.GetCount();
return *this;
}
IndexType GetIndex() const {
return static_cast<IndexType>(index_and_count_ >> kCountBits);
}
void ShiftIndex(IndexType offset) {
assert(GetIndex() + offset < (1 << kIndexBits));
index_and_count_ += offset << kCountBits;
}
IndexType GetCount() const {
return static_cast<IndexType>(index_and_count_ & ((1 << kCountBits) - 1));
}
bool operator<(const TrainingFeature& other) const {
return index_and_count_ < other.index_and_count_;
}
private:
StorageType index_and_count_;
};
};
// Structure that represents one sample of training data
struct Example {
// Structure that represents one sample of training data
struct Example {
std::vector<TrainingFeature> training_features[2];
Learner::PackedSfenValue psv;
int sign;
double weight;
};
};
// Message used for setting hyperparameters
struct Message {
Message(const std::string& message_name, const std::string& message_value = "") :
name(message_name), value(message_value), num_peekers(0), num_receivers(0)
{
}
// Message used for setting hyperparameters
struct Message {
Message(const std::string& message_name, const std::string& message_value = ""):
name(message_name), value(message_value), num_peekers(0), num_receivers(0) {}
const std::string name;
const std::string value;
std::uint32_t num_peekers;
std::uint32_t num_receivers;
};
};
// determine whether to accept the message
bool ReceiveMessage(const std::string& name, Message* message) {
// determine whether to accept the message
bool ReceiveMessage(const std::string& name, Message* message) {
const auto subscript = "[" + std::to_string(message->num_peekers) + "]";
if (message->name.substr(0, name.size() + 1) == name + "[") {
++message->num_peekers;
}
if (message->name == name || message->name == name + subscript) {
++message->num_receivers;
return true;
}
return false;
}
// split the string
std::vector<std::string> Split(const std::string& input, char delimiter) {
return false;
}
// split the string
std::vector<std::string> Split(const std::string& input, char delimiter) {
std::istringstream stream(input);
std::string field;
std::vector<std::string> fields;
while (std::getline(stream, field, delimiter)) {
fields.push_back(field);
}
return fields;
}
}
// round a floating point number to an integer
template <typename IntType>
IntType Round(double value) {
// round a floating point number to an integer
template <typename IntType>
IntType Round(double value) {
return static_cast<IntType>(std::floor(value + 0.5));
}
}
// make_shared with alignment
template <typename T, typename... ArgumentTypes>
std::shared_ptr<T> MakeAlignedSharedPtr(ArgumentTypes&&... arguments) {
// make_shared with alignment
template <typename T, typename... ArgumentTypes>
std::shared_ptr<T> MakeAlignedSharedPtr(ArgumentTypes&&... arguments) {
const auto ptr = new(std_aligned_alloc(alignof(T), sizeof(T)))
T(std::forward<ArgumentTypes>(arguments)...);
return std::shared_ptr<T>(ptr, AlignedDeleter<T>());
}
}
} // namespace NNUE
} // namespace Eval
} // namespace Eval::NNUE
#endif
src/nnue/trainer/trainer_affine_transform.h
+48 -16
View File
@@ -1,21 +1,20 @@
// Specialization of NNUE evaluation function learning class template for AffineTransform
#ifndef _NNUE_TRAINER_AFFINE_TRANSFORM_H_
#ifndef _NNUE_TRAINER_AFFINE_TRANSFORM_H_
#define _NNUE_TRAINER_AFFINE_TRANSFORM_H_
#include "../../learn/learn.h"
#include "../layers/affine_transform.h"
#include "trainer.h"
#include "learn/learn.h"
#include "nnue/layers/affine_transform.h"
#include <random>
namespace Eval {
// Specialization of NNUE evaluation function learning class template for AffineTransform
namespace Eval::NNUE {
namespace NNUE {
// Learning: Affine transformation layer
template <typename PreviousLayer, IndexType OutputDimensions>
class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
// Learning: Affine transformation layer
template <typename PreviousLayer, IndexType OutputDimensions>
class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
private:
// Type of layer to learn
using LayerType = Layers::AffineTransform<PreviousLayer, OutputDimensions>;
@@ -24,6 +23,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
// factory function
static std::shared_ptr<Trainer> Create(
LayerType* target_layer, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>(
new Trainer(target_layer, ft));
}
@@ -31,16 +31,20 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
// Set options such as hyperparameters
void SendMessage(Message* message) {
previous_layer_trainer_->SendMessage(message);
if (ReceiveMessage("momentum", message)) {
momentum_ = static_cast<LearnFloatType>(std::stod(message->value));
}
if (ReceiveMessage("learning_rate_scale", message)) {
learning_rate_scale_ =
static_cast<LearnFloatType>(std::stod(message->value));
}
if (ReceiveMessage("reset", message)) {
DequantizeParameters();
}
if (ReceiveMessage("quantize_parameters", message)) {
QuantizeParameters();
}
@@ -50,17 +54,20 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
template <typename RNG>
void Initialize(RNG& rng) {
previous_layer_trainer_->Initialize(rng);
if (kIsOutputLayer) {
// Initialize output layer with 0
std::fill(std::begin(biases_), std::end(biases_),
static_cast<LearnFloatType>(0.0));
std::fill(std::begin(weights_), std::end(weights_),
static_cast<LearnFloatType>(0.0));
} else {
}
else {
// Assuming that the input distribution is unit-mean 0.5, equal variance,
// Initialize the output distribution so that each unit has a mean of 0.5 and the same variance as the input
const double kSigma = 1.0 / std::sqrt(kInputDimensions);
auto distribution = std::normal_distribution<double>(0.0, kSigma);
for (IndexType i = 0; i < kOutputDimensions; ++i) {
double sum = 0.0;
for (IndexType j = 0; j < kInputDimensions; ++j) {
@@ -68,9 +75,11 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
weights_[kInputDimensions * i + j] = weight;
sum += weight;
}
biases_[i] = static_cast<LearnFloatType>(0.5 - 0.5 * sum);
}
}
QuantizeParameters();
}
@@ -80,6 +89,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
output_.resize(kOutputDimensions * batch.size());
gradients_.resize(kInputDimensions * batch.size());
}
batch_size_ = static_cast<IndexType>(batch.size());
batch_input_ = previous_layer_trainer_->Propagate(batch);
#if defined(USE_BLAS)
@@ -87,6 +97,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
const IndexType batch_offset = kOutputDimensions * b;
cblas_scopy(kOutputDimensions, biases_, 1, &output_[batch_offset], 1);
}
cblas_sgemm(CblasColMajor, CblasTrans, CblasNoTrans,
kOutputDimensions, batch_size_, kInputDimensions, 1.0,
weights_, kInputDimensions,
@@ -102,9 +113,11 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
const IndexType index = kInputDimensions * i + j;
sum += weights_[index] * batch_input_[input_batch_offset + j];
}
output_[output_batch_offset + i] = static_cast<LearnFloatType>(sum);
}
}
#endif
return output_.data();
}
@@ -112,8 +125,10 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
// backpropagation
void Backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) {
const LearnFloatType local_learning_rate =
learning_rate * learning_rate_scale_;
#if defined(USE_BLAS)
// backpropagate
cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans,
@@ -121,6 +136,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
weights_, kInputDimensions,
gradients, kOutputDimensions,
0.0, &gradients_[0], kInputDimensions);
// update
cblas_sscal(kOutputDimensions, momentum_, biases_diff_, 1);
for (IndexType b = 0; b < batch_size_; ++b) {
@@ -128,8 +144,10 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
cblas_saxpy(kOutputDimensions, 1.0,
&gradients[batch_offset], 1, biases_diff_, 1);
}
cblas_saxpy(kOutputDimensions, -local_learning_rate,
biases_diff_, 1, biases_, 1);
cblas_sgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
kOutputDimensions, kInputDimensions, batch_size_, 1.0,
gradients, kOutputDimensions,
@@ -137,6 +155,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
momentum_, weights_diff_, kInputDimensions);
cblas_saxpy(kOutputDimensions * kInputDimensions, -local_learning_rate,
weights_diff_, 1, weights_, 1);
#else
// backpropagate
for (IndexType b = 0; b < batch_size_; ++b) {
@@ -151,19 +170,24 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
gradients_[input_batch_offset + j] = static_cast<LearnFloatType>(sum);
}
}
// update
for (IndexType i = 0; i < kOutputDimensions; ++i) {
biases_diff_[i] *= momentum_;
}
for (IndexType i = 0; i < kOutputDimensions * kInputDimensions; ++i) {
weights_diff_[i] *= momentum_;
}
for (IndexType b = 0; b < batch_size_; ++b) {
const IndexType input_batch_offset = kInputDimensions * b;
const IndexType output_batch_offset = kOutputDimensions * b;
for (IndexType i = 0; i < kOutputDimensions; ++i) {
biases_diff_[i] += gradients[output_batch_offset + i];
}
for (IndexType i = 0; i < kOutputDimensions; ++i) {
for (IndexType j = 0; j < kInputDimensions; ++j) {
const IndexType index = kInputDimensions * i + j;
@@ -172,12 +196,15 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
}
}
}
for (IndexType i = 0; i < kOutputDimensions; ++i) {
biases_[i] -= local_learning_rate * biases_diff_[i];
}
for (IndexType i = 0; i < kOutputDimensions * kInputDimensions; ++i) {
weights_[i] -= local_learning_rate * weights_diff_[i];
}
#endif
previous_layer_trainer_->Backpropagate(gradients_.data(), learning_rate);
}
@@ -196,6 +223,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
weights_diff_(),
momentum_(0.2),
learning_rate_scale_(1.0) {
DequantizeParameters();
}
@@ -205,10 +233,12 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
weights_[i] = std::max(-kMaxWeightMagnitude,
std::min(+kMaxWeightMagnitude, weights_[i]));
}
for (IndexType i = 0; i < kOutputDimensions; ++i) {
target_layer_->biases_[i] =
Round<typename LayerType::BiasType>(biases_[i] * kBiasScale);
}
for (IndexType i = 0; i < kOutputDimensions; ++i) {
const auto offset = kInputDimensions * i;
const auto padded_offset = LayerType::kPaddedInputDimensions * i;
@@ -226,6 +256,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
biases_[i] = static_cast<LearnFloatType>(
target_layer_->biases_[i] / kBiasScale);
}
for (IndexType i = 0; i < kOutputDimensions; ++i) {
const auto offset = kInputDimensions * i;
const auto padded_offset = LayerType::kPaddedInputDimensions * i;
@@ -234,6 +265,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
target_layer_->weights_[padded_offset + j] / kWeightScale);
}
}
std::fill(std::begin(biases_diff_), std::end(biases_diff_),
static_cast<LearnFloatType>(0.0));
std::fill(std::begin(weights_diff_), std::end(weights_diff_),
@@ -250,9 +282,11 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
// Coefficient used for parameterization
static constexpr LearnFloatType kActivationScale =
std::numeric_limits<std::int8_t>::max();
static constexpr LearnFloatType kBiasScale = kIsOutputLayer ?
(kPonanzaConstant * FV_SCALE) :
((1 << kWeightScaleBits) * kActivationScale);
static constexpr LearnFloatType kWeightScale = kBiasScale / kActivationScale;
// Upper limit of absolute value of weight used to prevent overflow when parameterizing integers
@@ -288,10 +322,8 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
// hyper parameter
LearnFloatType momentum_;
LearnFloatType learning_rate_scale_;
};
};
} // namespace NNUE
} // namespace Eval
} // namespace Eval::NNUE
#endif
src/nnue/trainer/trainer_clipped_relu.h
+18 -14
View File
@@ -1,19 +1,18 @@
// Specialization of NNUE evaluation function learning class template for ClippedReLU
#ifndef _NNUE_TRAINER_CLIPPED_RELU_H_
#ifndef _NNUE_TRAINER_CLIPPED_RELU_H_
#define _NNUE_TRAINER_CLIPPED_RELU_H_
#include "../../learn/learn.h"
#include "../layers/clipped_relu.h"
#include "trainer.h"
namespace Eval {
#include "learn/learn.h"
namespace NNUE {
#include "nnue/layers/clipped_relu.h"
// Learning: Affine transformation layer
template <typename PreviousLayer>
class Trainer<Layers::ClippedReLU<PreviousLayer>> {
// Specialization of NNUE evaluation function learning class template for ClippedReLU
namespace Eval::NNUE {
// Learning: Affine transformation layer
template <typename PreviousLayer>
class Trainer<Layers::ClippedReLU<PreviousLayer>> {
private:
// Type of layer to learn
using LayerType = Layers::ClippedReLU<PreviousLayer>;
@@ -22,6 +21,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
// factory function
static std::shared_ptr<Trainer> Create(
LayerType* target_layer, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>(
new Trainer(target_layer, ft));
}
@@ -46,6 +46,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
output_.resize(kOutputDimensions * batch.size());
gradients_.resize(kInputDimensions * batch.size());
}
const auto input = previous_layer_trainer_->Propagate(batch);
batch_size_ = static_cast<IndexType>(batch.size());
for (IndexType b = 0; b < batch_size_; ++b) {
@@ -57,12 +58,14 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
max_activations_[i] = std::max(max_activations_[i], output_[index]);
}
}
return output_.data();
}
// backpropagation
void Backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) {
for (IndexType b = 0; b < batch_size_; ++b) {
const IndexType batch_offset = kOutputDimensions * b;
for (IndexType i = 0; i < kOutputDimensions; ++i) {
@@ -71,6 +74,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
(output_[index] > kZero) * (output_[index] < kOne);
}
}
previous_layer_trainer_->Backpropagate(gradients_.data(), learning_rate);
}
@@ -81,6 +85,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
previous_layer_trainer_(Trainer<PreviousLayer>::Create(
&target_layer->previous_layer_, ft)),
target_layer_(target_layer) {
std::fill(std::begin(min_activations_), std::end(min_activations_),
std::numeric_limits<LearnFloatType>::max());
std::fill(std::begin(max_activations_), std::end(max_activations_),
@@ -93,6 +98,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
std::begin(min_activations_), std::end(min_activations_));
const auto smallest_max_activation = *std::min_element(
std::begin(max_activations_), std::end(max_activations_));
std::cout << "INFO: largest min activation = " << largest_min_activation
<< ", smallest max activation = " << smallest_max_activation
<< std::endl;
@@ -129,10 +135,8 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
// Health check statistics
LearnFloatType min_activations_[kOutputDimensions];
LearnFloatType max_activations_[kOutputDimensions];
};
};
} // namespace NNUE
} // namespace Eval
} // namespace Eval::NNUE
#endif
src/nnue/trainer/trainer_feature_transformer.h
+53 -16
View File
@@ -1,13 +1,14 @@
// Specialization for feature transformer of learning class template of NNUE evaluation function
#ifndef _NNUE_TRAINER_FEATURE_TRANSFORMER_H_
#ifndef _NNUE_TRAINER_FEATURE_TRANSFORMER_H_
#define _NNUE_TRAINER_FEATURE_TRANSFORMER_H_
#include "../../learn/learn.h"
#include "../nnue_feature_transformer.h"
#include "trainer.h"
#include "features/factorizer_feature_set.h"
#include "learn/learn.h"
#include "nnue/nnue_feature_transformer.h"
#include <array>
#include <bitset>
#include <numeric>
@@ -18,13 +19,12 @@
#include <omp.h>
#endif
namespace Eval {
// Specialization for feature transformer of learning class template of NNUE evaluation function
namespace Eval::NNUE {
namespace NNUE {
// Learning: Input feature converter
template <>
class Trainer<FeatureTransformer> {
// Learning: Input feature converter
template <>
class Trainer<FeatureTransformer> {
private:
// Type of layer to learn
using LayerType = FeatureTransformer;
@@ -32,6 +32,7 @@ class Trainer<FeatureTransformer> {
public:
template <typename T>
friend struct AlignedDeleter;
template <typename T, typename... ArgumentTypes>
friend std::shared_ptr<T> MakeAlignedSharedPtr(ArgumentTypes&&... arguments);
@@ -45,19 +46,24 @@ class Trainer<FeatureTransformer> {
if (ReceiveMessage("momentum", message)) {
momentum_ = static_cast<LearnFloatType>(std::stod(message->value));
}
if (ReceiveMessage("learning_rate_scale", message)) {
learning_rate_scale_ =
static_cast<LearnFloatType>(std::stod(message->value));
}
if (ReceiveMessage("reset", message)) {
DequantizeParameters();
}
if (ReceiveMessage("quantize_parameters", message)) {
QuantizeParameters();
}
if (ReceiveMessage("clear_unobserved_feature_weights", message)) {
ClearUnobservedFeatureWeights();
}
if (ReceiveMessage("check_health", message)) {
CheckHealth();
}
@@ -67,15 +73,19 @@ class Trainer<FeatureTransformer> {
template <typename RNG>
void Initialize(RNG& rng) {
std::fill(std::begin(weights_), std::end(weights_), +kZero);
const double kSigma = 0.1 / std::sqrt(RawFeatures::kMaxActiveDimensions);
auto distribution = std::normal_distribution<double>(0.0, kSigma);
for (IndexType i = 0; i < kHalfDimensions * RawFeatures::kDimensions; ++i) {
const auto weight = static_cast<LearnFloatType>(distribution(rng));
weights_[i] = weight;
}
for (IndexType i = 0; i < kHalfDimensions; ++i) {
biases_[i] = static_cast<LearnFloatType>(0.5);
}
QuantizeParameters();
}
@@ -85,6 +95,7 @@ class Trainer<FeatureTransformer> {
output_.resize(kOutputDimensions * batch.size());
gradients_.resize(kOutputDimensions * batch.size());
}
batch_ = &batch;
// affine transform
#pragma omp parallel for
@@ -113,6 +124,7 @@ class Trainer<FeatureTransformer> {
#endif
}
}
// clipped ReLU
for (IndexType b = 0; b < batch.size(); ++b) {
const IndexType batch_offset = kOutputDimensions * b;
@@ -126,14 +138,17 @@ class Trainer<FeatureTransformer> {
max_activations_[t] = std::max(max_activations_[t], output_[index]);
}
}
return output_.data();
}
// backpropagation
void Backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) {
const LearnFloatType local_learning_rate =
learning_rate * learning_rate_scale_;
for (IndexType b = 0; b < batch_->size(); ++b) {
const IndexType batch_offset = kOutputDimensions * b;
for (IndexType i = 0; i < kOutputDimensions; ++i) {
@@ -142,6 +157,7 @@ class Trainer<FeatureTransformer> {
((output_[index] > kZero) * (output_[index] < kOne));
}
}
// Since the weight matrix updates only the columns corresponding to the features that appeared in the input,
// Correct the learning rate and adjust the scale without using momentum
const LearnFloatType effective_learning_rate =
@@ -156,8 +172,10 @@ class Trainer<FeatureTransformer> {
&gradients_[output_offset], 1, biases_diff_, 1);
}
}
cblas_saxpy(kHalfDimensions, -local_learning_rate,
biases_diff_, 1, biases_, 1);
#pragma omp parallel
{
#if defined(_OPENMP)
@@ -170,12 +188,14 @@ class Trainer<FeatureTransformer> {
const IndexType output_offset = batch_offset + kHalfDimensions * c;
for (const auto& feature : (*batch_)[b].training_features[c]) {
#if defined(_OPENMP)
if (feature.GetIndex() % num_threads != thread_index) continue;
if (feature.GetIndex() % num_threads != thread_index)
continue;
#endif
const IndexType weights_offset =
kHalfDimensions * feature.GetIndex();
const auto scale = static_cast<LearnFloatType>(
effective_learning_rate / feature.GetCount());
cblas_saxpy(kHalfDimensions, -scale,
&gradients_[output_offset], 1,
&weights_[weights_offset], 1);
@@ -183,10 +203,12 @@ class Trainer<FeatureTransformer> {
}
}
}
#else
for (IndexType i = 0; i < kHalfDimensions; ++i) {
biases_diff_[i] *= momentum_;
}
for (IndexType b = 0; b < batch_->size(); ++b) {
const IndexType batch_offset = kOutputDimensions * b;
for (IndexType c = 0; c < 2; ++c) {
@@ -196,9 +218,11 @@ class Trainer<FeatureTransformer> {
}
}
}
for (IndexType i = 0; i < kHalfDimensions; ++i) {
biases_[i] -= local_learning_rate * biases_diff_[i];
}
for (IndexType b = 0; b < batch_->size(); ++b) {
const IndexType batch_offset = kOutputDimensions * b;
for (IndexType c = 0; c < 2; ++c) {
@@ -207,6 +231,7 @@ class Trainer<FeatureTransformer> {
const IndexType weights_offset = kHalfDimensions * feature.GetIndex();
const auto scale = static_cast<LearnFloatType>(
effective_learning_rate / feature.GetCount());
for (IndexType i = 0; i < kHalfDimensions; ++i) {
weights_[weights_offset + i] -=
scale * gradients_[output_offset + i];
@@ -214,6 +239,7 @@ class Trainer<FeatureTransformer> {
}
}
}
#endif
for (IndexType b = 0; b < batch_->size(); ++b) {
for (IndexType c = 0; c < 2; ++c) {
@@ -234,12 +260,15 @@ class Trainer<FeatureTransformer> {
biases_diff_(),
momentum_(0.2),
learning_rate_scale_(1.0) {
min_pre_activation_ = std::numeric_limits<LearnFloatType>::max();
max_pre_activation_ = std::numeric_limits<LearnFloatType>::lowest();
std::fill(std::begin(min_activations_), std::end(min_activations_),
std::numeric_limits<LearnFloatType>::max());
std::fill(std::begin(max_activations_), std::end(max_activations_),
std::numeric_limits<LearnFloatType>::lowest());
DequantizeParameters();
}
@@ -249,17 +278,21 @@ class Trainer<FeatureTransformer> {
target_layer_->biases_[i] =
Round<typename LayerType::BiasType>(biases_[i] * kBiasScale);
}
std::vector<TrainingFeature> training_features;
#pragma omp parallel for private(training_features)
for (IndexType j = 0; j < RawFeatures::kDimensions; ++j) {
training_features.clear();
Features::Factorizer<RawFeatures>::AppendTrainingFeatures(
j, &training_features);
for (IndexType i = 0; i < kHalfDimensions; ++i) {
double sum = 0.0;
for (const auto& feature : training_features) {
sum += weights_[kHalfDimensions * feature.GetIndex() + i];
}
target_layer_->weights_[kHalfDimensions * j + i] =
Round<typename LayerType::WeightType>(sum * kWeightScale);
}
@@ -272,11 +305,14 @@ class Trainer<FeatureTransformer> {
biases_[i] = static_cast<LearnFloatType>(
target_layer_->biases_[i] / kBiasScale);
}
std::fill(std::begin(weights_), std::end(weights_), +kZero);
for (IndexType i = 0; i < kHalfDimensions * RawFeatures::kDimensions; ++i) {
weights_[i] = static_cast<LearnFloatType>(
target_layer_->weights_[i] / kWeightScale);
}
std::fill(std::begin(biases_diff_), std::end(biases_diff_), +kZero);
}
@@ -288,6 +324,7 @@ class Trainer<FeatureTransformer> {
std::begin(weights_) + kHalfDimensions * (i + 1), +kZero);
}
}
QuantizeParameters();
}
@@ -299,6 +336,7 @@ class Trainer<FeatureTransformer> {
constexpr LearnFloatType kPreActivationLimit =
std::numeric_limits<typename LayerType::WeightType>::max() /
kWeightScale;
std::cout << "INFO: (min, max) of pre-activations = "
<< min_pre_activation_ << ", "
<< max_pre_activation_ << " (limit = "
@@ -308,6 +346,7 @@ class Trainer<FeatureTransformer> {
std::begin(min_activations_), std::end(min_activations_));
const auto smallest_max_activation = *std::min_element(
std::begin(max_activations_), std::end(max_activations_));
std::cout << "INFO: largest min activation = " << largest_min_activation
<< ", smallest max activation = " << smallest_max_activation
<< std::endl;
@@ -364,10 +403,8 @@ class Trainer<FeatureTransformer> {
LearnFloatType max_pre_activation_;
LearnFloatType min_activations_[kHalfDimensions];
LearnFloatType max_activations_[kHalfDimensions];
};
};
} // namespace NNUE
} // namespace Eval
} // namespace Eval::NNUE
#endif
src/nnue/trainer/trainer_input_slice.h
+37 -17
View File
@@ -1,27 +1,30 @@
// Specialization of NNUE evaluation function learning class template for InputSlice
#ifndef _NNUE_TRAINER_INPUT_SLICE_H_
#ifndef _NNUE_TRAINER_INPUT_SLICE_H_
#define _NNUE_TRAINER_INPUT_SLICE_H_
#include "../../learn/learn.h"
#include "../layers/input_slice.h"
#include "trainer.h"
namespace Eval {
#include "learn/learn.h"
namespace NNUE {
#include "nnue/layers/input_slice.h"
// Learning: Input layer
class SharedInputTrainer {
// Specialization of NNUE evaluation function learning class template for InputSlice
namespace Eval::NNUE {
// Learning: Input layer
class SharedInputTrainer {
public:
// factory function
static std::shared_ptr<SharedInputTrainer> Create(
FeatureTransformer* ft) {
static std::shared_ptr<SharedInputTrainer> instance;
if (!instance) {
instance.reset(new SharedInputTrainer(ft));
}
++instance->num_referrers_;
return instance;
}
@@ -31,7 +34,9 @@ class SharedInputTrainer {
current_operation_ = Operation::kSendMessage;
feature_transformer_trainer_->SendMessage(message);
}
assert(current_operation_ == Operation::kSendMessage);
if (++num_calls_ == num_referrers_) {
num_calls_ = 0;
current_operation_ = Operation::kNone;
@@ -45,7 +50,9 @@ class SharedInputTrainer {
current_operation_ = Operation::kInitialize;
feature_transformer_trainer_->Initialize(rng);
}
assert(current_operation_ == Operation::kInitialize);
if (++num_calls_ == num_referrers_) {
num_calls_ = 0;
current_operation_ = Operation::kNone;
@@ -57,26 +64,33 @@ class SharedInputTrainer {
if (gradients_.size() < kInputDimensions * batch.size()) {
gradients_.resize(kInputDimensions * batch.size());
}
batch_size_ = static_cast<IndexType>(batch.size());
if (num_calls_ == 0) {
current_operation_ = Operation::kPropagate;
output_ = feature_transformer_trainer_->Propagate(batch);
}
assert(current_operation_ == Operation::kPropagate);
if (++num_calls_ == num_referrers_) {
num_calls_ = 0;
current_operation_ = Operation::kNone;
}
return output_;
}
// backpropagation
void Backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) {
if (num_referrers_ == 1) {
feature_transformer_trainer_->Backpropagate(gradients, learning_rate);
return;
}
if (num_calls_ == 0) {
current_operation_ = Operation::kBackPropagate;
for (IndexType b = 0; b < batch_size_; ++b) {
@@ -86,13 +100,16 @@ class SharedInputTrainer {
}
}
}
assert(current_operation_ == Operation::kBackPropagate);
for (IndexType b = 0; b < batch_size_; ++b) {
const IndexType batch_offset = kInputDimensions * b;
for (IndexType i = 0; i < kInputDimensions; ++i) {
gradients_[batch_offset + i] += gradients[batch_offset + i];
}
}
if (++num_calls_ == num_referrers_) {
feature_transformer_trainer_->Backpropagate(
gradients_.data(), learning_rate);
@@ -147,11 +164,11 @@ class SharedInputTrainer {
// buffer for back propagation
std::vector<LearnFloatType> gradients_;
};
};
// Learning: Input layer
template <IndexType OutputDimensions, IndexType Offset>
class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
// Learning: Input layer
template <IndexType OutputDimensions, IndexType Offset>
class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
private:
// Type of layer to learn
using LayerType = Layers::InputSlice<OutputDimensions, Offset>;
@@ -160,6 +177,7 @@ class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
// factory function
static std::shared_ptr<Trainer> Create(
LayerType* /*target_layer*/, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>(new Trainer(ft));
}
@@ -180,7 +198,9 @@ class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
output_.resize(kOutputDimensions * batch.size());
gradients_.resize(kInputDimensions * batch.size());
}
batch_size_ = static_cast<IndexType>(batch.size());
const auto input = shared_input_trainer_->Propagate(batch);
for (IndexType b = 0; b < batch_size_; ++b) {
const IndexType input_offset = kInputDimensions * b;
@@ -194,12 +214,14 @@ class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
}
#endif
}
return output_.data();
}
// backpropagation
void Backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) {
for (IndexType b = 0; b < batch_size_; ++b) {
const IndexType input_offset = kInputDimensions * b;
const IndexType output_offset = kOutputDimensions * b;
@@ -238,10 +260,8 @@ class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
// buffer for back propagation
std::vector<LearnFloatType> gradients_;
};
};
} // namespace NNUE
} // namespace Eval
} // namespace Eval::NNUE
#endif
src/nnue/trainer/trainer_sum.h
+21 -17
View File
@@ -1,19 +1,16 @@
// Specialization of NNUE evaluation function learning class template for Sum
#ifndef _NNUE_TRAINER_SUM_H_
#ifndef _NNUE_TRAINER_SUM_H_
#define _NNUE_TRAINER_SUM_H_
#include "../../learn/learn.h"
#include "../layers/sum.h"
#include "trainer.h"
namespace Eval {
// Specialization of NNUE evaluation function learning class template for Sum
namespace Eval::NNUE {
namespace NNUE {
// Learning: A layer that sums the outputs of multiple layers
template <typename FirstPreviousLayer, typename... RemainingPreviousLayers>
class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
// Learning: A layer that sums the outputs of multiple layers
template <typename FirstPreviousLayer, typename... RemainingPreviousLayers>
class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
Trainer<Layers::Sum<RemainingPreviousLayers...>> {
private:
// Type of layer to learn
@@ -24,6 +21,7 @@ class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
// factory function
static std::shared_ptr<Trainer> Create(
LayerType* target_layer, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>(
new Trainer(target_layer, ft));
}
@@ -49,6 +47,7 @@ class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
batch_size_ = static_cast<IndexType>(batch.size());
auto output = Tail::Propagate(batch);
const auto head_output = previous_layer_trainer_->Propagate(batch);
#if defined(USE_BLAS)
cblas_saxpy(kOutputDimensions * batch_size_, 1.0,
head_output, 1, output, 1);
@@ -59,6 +58,7 @@ class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
output[batch_offset + i] += head_output[batch_offset + i];
}
}
#endif
return output;
}
@@ -66,6 +66,7 @@ class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
// backpropagation
void Backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) {
Tail::Backpropagate(gradients, learning_rate);
previous_layer_trainer_->Backpropagate(gradients, learning_rate);
}
@@ -95,12 +96,12 @@ class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
// layer to learn
LayerType* const target_layer_;
};
};
// Learning: Layer that takes the sum of the outputs of multiple layers (when there is one template argument)
template <typename PreviousLayer>
class Trainer<Layers::Sum<PreviousLayer>> {
// Learning: Layer that takes the sum of the outputs of multiple layers (when there is one template argument)
template <typename PreviousLayer>
class Trainer<Layers::Sum<PreviousLayer>> {
private:
// Type of layer to learn
using LayerType = Layers::Sum<PreviousLayer>;
@@ -109,6 +110,7 @@ class Trainer<Layers::Sum<PreviousLayer>> {
// factory function
static std::shared_ptr<Trainer> Create(
LayerType* target_layer, FeatureTransformer* ft) {
return std::shared_ptr<Trainer>(
new Trainer(target_layer, ft));
}
@@ -129,8 +131,10 @@ class Trainer<Layers::Sum<PreviousLayer>> {
if (output_.size() < kOutputDimensions * batch.size()) {
output_.resize(kOutputDimensions * batch.size());
}
batch_size_ = static_cast<IndexType>(batch.size());
const auto output = previous_layer_trainer_->Propagate(batch);
#if defined(USE_BLAS)
cblas_scopy(kOutputDimensions * batch_size_, output, 1, &output_[0], 1);
#else
@@ -140,6 +144,7 @@ class Trainer<Layers::Sum<PreviousLayer>> {
output_[batch_offset + i] = output[batch_offset + i];
}
}
#endif
return output_.data();
}
@@ -147,6 +152,7 @@ class Trainer<Layers::Sum<PreviousLayer>> {
// backpropagation
void Backpropagate(const LearnFloatType* gradients,
LearnFloatType learning_rate) {
previous_layer_trainer_->Backpropagate(gradients, learning_rate);
}
@@ -177,10 +183,8 @@ class Trainer<Layers::Sum<PreviousLayer>> {
// Forward propagation buffer
std::vector<LearnFloatType> output_;
};
};
} // namespace NNUE
} // namespace Eval
} // namespace Eval::NNUE
#endif