Cleanup trainer.

src/nnue/trainer/trainer.h

@@ -1,20 +1,18 @@
-// Common header of class template for learning NNUE evaluation function
+#ifndef _NNUE_TRAINER_H_
-
-#ifndef _NNUE_TRAINER_H_
 #define _NNUE_TRAINER_H_
 
-#include "../nnue_common.h"
+#include "nnue/nnue_common.h"
-#include "../features/index_list.h"
+#include "nnue/features/index_list.h"
 
 #include <sstream>
+
 #if defined(USE_BLAS)
 static_assert(std::is_same<LearnFloatType, float>::value, "");
 #include <cblas.h>
 #endif
 
-namespace Eval {
+// Common header of class template for learning NNUE evaluation function
-
+namespace Eval::NNUE {
-namespace NNUE {
 
     // Ponanza constant used in the relation between evaluation value and winning percentage
     constexpr double kPonanzaConstant = 600.0;
@@ -26,30 +24,38 @@ class TrainingFeature {
 
     public:
         static constexpr std::uint32_t kIndexBits = 24;
+
         static_assert(kIndexBits < std::numeric_limits<StorageType>::digits, "");
+
         static constexpr std::uint32_t kCountBits =
             std::numeric_limits<StorageType>::digits - kIndexBits;
 
         explicit TrainingFeature(IndexType index) :
             index_and_count_((index << kCountBits) | 1) {
+
             assert(index < (1 << kIndexBits));
         }
+
         TrainingFeature& operator+=(const TrainingFeature& other) {
             assert(other.GetIndex() == GetIndex());
             assert(other.GetCount() + GetCount() < (1 << kCountBits));
             index_and_count_ += other.GetCount();
             return *this;
         }
+
         IndexType GetIndex() const {
             return static_cast<IndexType>(index_and_count_ >> kCountBits);
         }
+
         void ShiftIndex(IndexType offset) {
             assert(GetIndex() + offset < (1 << kIndexBits));
             index_and_count_ += offset << kCountBits;
         }
+
         IndexType GetCount() const {
             return static_cast<IndexType>(index_and_count_ & ((1 << kCountBits) - 1));
         }
+
         bool operator<(const TrainingFeature& other) const {
             return index_and_count_ < other.index_and_count_;
         }
@@ -69,7 +75,10 @@ struct Example {
     // Message used for setting hyperparameters
     struct Message {
         Message(const std::string& message_name, const std::string& message_value = "") :
-      name(message_name), value(message_value), num_peekers(0), num_receivers(0) {}
+            name(message_name), value(message_value), num_peekers(0), num_receivers(0)
+        {
+        }
+
         const std::string name;
         const std::string value;
         std::uint32_t num_peekers;
@@ -79,13 +88,16 @@ struct Message {
     // determine whether to accept the message
     bool ReceiveMessage(const std::string& name, Message* message) {
         const auto subscript = "[" + std::to_string(message->num_peekers) + "]";
+
         if (message->name.substr(0, name.size() + 1) == name + "[") {
             ++message->num_peekers;
         }
+
         if (message->name == name || message->name == name + subscript) {
             ++message->num_receivers;
             return true;
         }
+
         return false;
     }
 
@@ -94,9 +106,11 @@ std::vector<std::string> Split(const std::string& input, char delimiter) {
         std::istringstream stream(input);
         std::string field;
         std::vector<std::string> fields;
+
         while (std::getline(stream, field, delimiter)) {
             fields.push_back(field);
         }
+
         return fields;
     }
 
@@ -111,11 +125,10 @@ template <typename T, typename... ArgumentTypes>
     std::shared_ptr<T> MakeAlignedSharedPtr(ArgumentTypes&&... arguments) {
         const auto ptr = new(std_aligned_alloc(alignof(T), sizeof(T)))
             T(std::forward<ArgumentTypes>(arguments)...);
+
         return std::shared_ptr<T>(ptr, AlignedDeleter<T>());
     }
 
-}  // namespace NNUE
+}  // namespace Eval::NNUE
-
-}  // namespace Eval
 
 #endif

src/nnue/trainer/trainer_affine_transform.h

@@ -1,17 +1,16 @@
-// Specialization of NNUE evaluation function learning class template for AffineTransform
+#ifndef _NNUE_TRAINER_AFFINE_TRANSFORM_H_
-
-#ifndef _NNUE_TRAINER_AFFINE_TRANSFORM_H_
 #define _NNUE_TRAINER_AFFINE_TRANSFORM_H_
 
-#include "../../learn/learn.h"
-#include "../layers/affine_transform.h"
 #include "trainer.h"
 
+#include "learn/learn.h"
+
+#include "nnue/layers/affine_transform.h"
+
 #include <random>
 
-namespace Eval {
+// Specialization of NNUE evaluation function learning class template for AffineTransform
-
+namespace Eval::NNUE {
-namespace NNUE {
 
     // Learning: Affine transformation layer
     template <typename PreviousLayer, IndexType OutputDimensions>
@@ -24,6 +23,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
         // factory function
         static std::shared_ptr<Trainer> Create(
             LayerType* target_layer, FeatureTransformer* ft) {
+
             return std::shared_ptr<Trainer>(
                 new Trainer(target_layer, ft));
         }
@@ -31,16 +31,20 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
         // Set options such as hyperparameters
         void SendMessage(Message* message) {
             previous_layer_trainer_->SendMessage(message);
+
             if (ReceiveMessage("momentum", message)) {
                 momentum_ = static_cast<LearnFloatType>(std::stod(message->value));
             }
+
             if (ReceiveMessage("learning_rate_scale", message)) {
                 learning_rate_scale_ =
                     static_cast<LearnFloatType>(std::stod(message->value));
             }
+
             if (ReceiveMessage("reset", message)) {
                 DequantizeParameters();
             }
+
             if (ReceiveMessage("quantize_parameters", message)) {
                 QuantizeParameters();
             }
@@ -50,17 +54,20 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
         template <typename RNG>
         void Initialize(RNG& rng) {
             previous_layer_trainer_->Initialize(rng);
+
             if (kIsOutputLayer) {
                 // Initialize output layer with 0
                 std::fill(std::begin(biases_), std::end(biases_),
                           static_cast<LearnFloatType>(0.0));
                 std::fill(std::begin(weights_), std::end(weights_),
                           static_cast<LearnFloatType>(0.0));
-    } else {
+            }
+            else {
                 // Assuming that the input distribution is unit-mean 0.5, equal variance,
                 // Initialize the output distribution so that each unit has a mean of 0.5 and the same variance as the input
                 const double kSigma = 1.0 / std::sqrt(kInputDimensions);
                 auto distribution = std::normal_distribution<double>(0.0, kSigma);
+
                 for (IndexType i = 0; i < kOutputDimensions; ++i) {
                     double sum = 0.0;
                       for (IndexType j = 0; j < kInputDimensions; ++j) {
@@ -68,9 +75,11 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                           weights_[kInputDimensions * i + j] = weight;
                           sum += weight;
                       }
+
                     biases_[i] = static_cast<LearnFloatType>(0.5 - 0.5 * sum);
                 }
             }
+
             QuantizeParameters();
         }
 
@@ -80,6 +89,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                 output_.resize(kOutputDimensions * batch.size());
                 gradients_.resize(kInputDimensions * batch.size());
             }
+
             batch_size_ = static_cast<IndexType>(batch.size());
             batch_input_ = previous_layer_trainer_->Propagate(batch);
 #if defined(USE_BLAS)
@@ -87,6 +97,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                 const IndexType batch_offset = kOutputDimensions * b;
                 cblas_scopy(kOutputDimensions, biases_, 1, &output_[batch_offset], 1);
             }
+
             cblas_sgemm(CblasColMajor, CblasTrans, CblasNoTrans,
                         kOutputDimensions, batch_size_, kInputDimensions, 1.0,
                         weights_, kInputDimensions,
@@ -102,9 +113,11 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                         const IndexType index = kInputDimensions * i + j;
                         sum += weights_[index] * batch_input_[input_batch_offset + j];
                     }
+
                     output_[output_batch_offset + i] = static_cast<LearnFloatType>(sum);
                 }
             }
+
 #endif
             return output_.data();
         }
@@ -112,8 +125,10 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
         // backpropagation
         void Backpropagate(const LearnFloatType* gradients,
                            LearnFloatType learning_rate) {
+
             const LearnFloatType local_learning_rate =
                 learning_rate * learning_rate_scale_;
+
 #if defined(USE_BLAS)
             // backpropagate
             cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans,
@@ -121,6 +136,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                         weights_, kInputDimensions,
                         gradients, kOutputDimensions,
                         0.0, &gradients_[0], kInputDimensions);
+
             // update
             cblas_sscal(kOutputDimensions, momentum_, biases_diff_, 1);
             for (IndexType b = 0; b < batch_size_; ++b) {
@@ -128,8 +144,10 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                 cblas_saxpy(kOutputDimensions, 1.0,
                           &gradients[batch_offset], 1, biases_diff_, 1);
             }
+
             cblas_saxpy(kOutputDimensions, -local_learning_rate,
                         biases_diff_, 1, biases_, 1);
+
             cblas_sgemm(CblasRowMajor, CblasTrans, CblasNoTrans,
                         kOutputDimensions, kInputDimensions, batch_size_, 1.0,
                         gradients, kOutputDimensions,
@@ -137,6 +155,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                         momentum_, weights_diff_, kInputDimensions);
             cblas_saxpy(kOutputDimensions * kInputDimensions, -local_learning_rate,
                         weights_diff_, 1, weights_, 1);
+
 #else
             // backpropagate
             for (IndexType b = 0; b < batch_size_; ++b) {
@@ -151,19 +170,24 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                     gradients_[input_batch_offset + j] = static_cast<LearnFloatType>(sum);
                 }
             }
+
             // update
             for (IndexType i = 0; i < kOutputDimensions; ++i) {
                 biases_diff_[i] *= momentum_;
             }
+
             for (IndexType i = 0; i < kOutputDimensions * kInputDimensions; ++i) {
                 weights_diff_[i] *= momentum_;
             }
+
             for (IndexType b = 0; b < batch_size_; ++b) {
                 const IndexType input_batch_offset = kInputDimensions * b;
                 const IndexType output_batch_offset = kOutputDimensions * b;
+
                 for (IndexType i = 0; i < kOutputDimensions; ++i) {
                     biases_diff_[i] += gradients[output_batch_offset + i];
                 }
+
                 for (IndexType i = 0; i < kOutputDimensions; ++i) {
                     for (IndexType j = 0; j < kInputDimensions; ++j) {
                         const IndexType index = kInputDimensions * i + j;
@@ -172,12 +196,15 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                     }
                 }
             }
+
             for (IndexType i = 0; i < kOutputDimensions; ++i) {
                 biases_[i] -= local_learning_rate * biases_diff_[i];
             }
+
             for (IndexType i = 0; i < kOutputDimensions * kInputDimensions; ++i) {
                 weights_[i] -= local_learning_rate * weights_diff_[i];
             }
+
 #endif
             previous_layer_trainer_->Backpropagate(gradients_.data(), learning_rate);
         }
@@ -196,6 +223,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
             weights_diff_(),
             momentum_(0.2),
             learning_rate_scale_(1.0) {
+
             DequantizeParameters();
         }
 
@@ -205,10 +233,12 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                 weights_[i] = std::max(-kMaxWeightMagnitude,
                                        std::min(+kMaxWeightMagnitude, weights_[i]));
             }
+
             for (IndexType i = 0; i < kOutputDimensions; ++i) {
                 target_layer_->biases_[i] =
                     Round<typename LayerType::BiasType>(biases_[i] * kBiasScale);
             }
+
             for (IndexType i = 0; i < kOutputDimensions; ++i) {
                 const auto offset = kInputDimensions * i;
                 const auto padded_offset = LayerType::kPaddedInputDimensions * i;
@@ -226,6 +256,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                 biases_[i] = static_cast<LearnFloatType>(
                     target_layer_->biases_[i] / kBiasScale);
             }
+
             for (IndexType i = 0; i < kOutputDimensions; ++i) {
                 const auto offset = kInputDimensions * i;
                 const auto padded_offset = LayerType::kPaddedInputDimensions * i;
@@ -234,6 +265,7 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
                         target_layer_->weights_[padded_offset + j] / kWeightScale);
                 }
             }
+
             std::fill(std::begin(biases_diff_), std::end(biases_diff_),
                       static_cast<LearnFloatType>(0.0));
             std::fill(std::begin(weights_diff_), std::end(weights_diff_),
@@ -250,9 +282,11 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
         // Coefficient used for parameterization
         static constexpr LearnFloatType kActivationScale =
             std::numeric_limits<std::int8_t>::max();
+
         static constexpr LearnFloatType kBiasScale = kIsOutputLayer ?
             (kPonanzaConstant * FV_SCALE) :
             ((1 << kWeightScaleBits) * kActivationScale);
+
         static constexpr LearnFloatType kWeightScale = kBiasScale / kActivationScale;
 
         // Upper limit of absolute value of weight used to prevent overflow when parameterizing integers
@@ -290,8 +324,6 @@ class Trainer<Layers::AffineTransform<PreviousLayer, OutputDimensions>> {
         LearnFloatType learning_rate_scale_;
     };
 
-}  // namespace NNUE
+}  // namespace Eval::NNUE
-
-}  // namespace Eval
 
 #endif

src/nnue/trainer/trainer_clipped_relu.h

@@ -1,15 +1,14 @@
-// Specialization of NNUE evaluation function learning class template for ClippedReLU
+#ifndef _NNUE_TRAINER_CLIPPED_RELU_H_
-
-#ifndef _NNUE_TRAINER_CLIPPED_RELU_H_
 #define _NNUE_TRAINER_CLIPPED_RELU_H_
 
-#include "../../learn/learn.h"
-#include "../layers/clipped_relu.h"
 #include "trainer.h"
 
-namespace Eval {
+#include "learn/learn.h"
 
-namespace NNUE {
+#include "nnue/layers/clipped_relu.h"
+
+// Specialization of NNUE evaluation function learning class template for ClippedReLU
+namespace Eval::NNUE {
 
     // Learning: Affine transformation layer
     template <typename PreviousLayer>
@@ -22,6 +21,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
         // factory function
         static std::shared_ptr<Trainer> Create(
             LayerType* target_layer, FeatureTransformer* ft) {
+
             return std::shared_ptr<Trainer>(
                 new Trainer(target_layer, ft));
         }
@@ -46,6 +46,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
               output_.resize(kOutputDimensions * batch.size());
               gradients_.resize(kInputDimensions * batch.size());
             }
+
             const auto input = previous_layer_trainer_->Propagate(batch);
             batch_size_ = static_cast<IndexType>(batch.size());
             for (IndexType b = 0; b < batch_size_; ++b) {
@@ -57,12 +58,14 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
                     max_activations_[i] = std::max(max_activations_[i], output_[index]);
                 }
             }
+
             return output_.data();
         }
 
         // backpropagation
         void Backpropagate(const LearnFloatType* gradients,
                            LearnFloatType learning_rate) {
+
             for (IndexType b = 0; b < batch_size_; ++b) {
                 const IndexType batch_offset = kOutputDimensions * b;
                 for (IndexType i = 0; i < kOutputDimensions; ++i) {
@@ -71,6 +74,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
                         (output_[index] > kZero) * (output_[index] < kOne);
                 }
             }
+
             previous_layer_trainer_->Backpropagate(gradients_.data(), learning_rate);
         }
 
@@ -81,6 +85,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
             previous_layer_trainer_(Trainer<PreviousLayer>::Create(
                 &target_layer->previous_layer_, ft)),
             target_layer_(target_layer) {
+
             std::fill(std::begin(min_activations_), std::end(min_activations_),
                       std::numeric_limits<LearnFloatType>::max());
             std::fill(std::begin(max_activations_), std::end(max_activations_),
@@ -93,6 +98,7 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
                 std::begin(min_activations_), std::end(min_activations_));
             const auto smallest_max_activation = *std::min_element(
                 std::begin(max_activations_), std::end(max_activations_));
+
             std::cout << "INFO: largest min activation = " << largest_min_activation
                       << ", smallest max activation = " << smallest_max_activation
                       << std::endl;
@@ -131,8 +137,6 @@ class Trainer<Layers::ClippedReLU<PreviousLayer>> {
         LearnFloatType max_activations_[kOutputDimensions];
     };
 
-}  // namespace NNUE
+}  // namespace Eval::NNUE
-
-}  // namespace Eval
 
 #endif

src/nnue/trainer/trainer_feature_transformer.h

@@ -1,13 +1,14 @@
-// Specialization for feature transformer of learning class template of NNUE evaluation function
+#ifndef _NNUE_TRAINER_FEATURE_TRANSFORMER_H_
-
-#ifndef _NNUE_TRAINER_FEATURE_TRANSFORMER_H_
 #define _NNUE_TRAINER_FEATURE_TRANSFORMER_H_
 
-#include "../../learn/learn.h"
-#include "../nnue_feature_transformer.h"
 #include "trainer.h"
+
 #include "features/factorizer_feature_set.h"
 
+#include "learn/learn.h"
+
+#include "nnue/nnue_feature_transformer.h"
+
 #include <array>
 #include <bitset>
 #include <numeric>
@@ -18,9 +19,8 @@
 #include <omp.h>
 #endif
 
-namespace Eval {
+// Specialization for feature transformer of learning class template of NNUE evaluation function
-
+namespace Eval::NNUE {
-namespace NNUE {
 
     // Learning: Input feature converter
     template <>
@@ -32,6 +32,7 @@ class Trainer<FeatureTransformer> {
     public:
         template <typename T>
         friend struct AlignedDeleter;
+
         template <typename T, typename... ArgumentTypes>
         friend std::shared_ptr<T> MakeAlignedSharedPtr(ArgumentTypes&&... arguments);
 
@@ -45,19 +46,24 @@ class Trainer<FeatureTransformer> {
             if (ReceiveMessage("momentum", message)) {
                 momentum_ = static_cast<LearnFloatType>(std::stod(message->value));
             }
+
             if (ReceiveMessage("learning_rate_scale", message)) {
                 learning_rate_scale_ =
                     static_cast<LearnFloatType>(std::stod(message->value));
             }
+
             if (ReceiveMessage("reset", message)) {
                 DequantizeParameters();
             }
+
             if (ReceiveMessage("quantize_parameters", message)) {
                 QuantizeParameters();
             }
+
             if (ReceiveMessage("clear_unobserved_feature_weights", message)) {
                 ClearUnobservedFeatureWeights();
             }
+
             if (ReceiveMessage("check_health", message)) {
                 CheckHealth();
             }
@@ -67,15 +73,19 @@ class Trainer<FeatureTransformer> {
         template <typename RNG>
         void Initialize(RNG& rng) {
             std::fill(std::begin(weights_), std::end(weights_), +kZero);
+
             const double kSigma = 0.1 / std::sqrt(RawFeatures::kMaxActiveDimensions);
             auto distribution = std::normal_distribution<double>(0.0, kSigma);
+
             for (IndexType i = 0; i < kHalfDimensions * RawFeatures::kDimensions; ++i) {
                 const auto weight = static_cast<LearnFloatType>(distribution(rng));
                 weights_[i] = weight;
             }
+
             for (IndexType i = 0; i < kHalfDimensions; ++i) {
                 biases_[i] = static_cast<LearnFloatType>(0.5);
             }
+
             QuantizeParameters();
         }
 
@@ -85,6 +95,7 @@ class Trainer<FeatureTransformer> {
                 output_.resize(kOutputDimensions * batch.size());
                 gradients_.resize(kOutputDimensions * batch.size());
             }
+
             batch_ = &batch;
             // affine transform
 #pragma omp parallel for
@@ -113,6 +124,7 @@ class Trainer<FeatureTransformer> {
 #endif
                 }
             }
+
             // clipped ReLU
             for (IndexType b = 0; b < batch.size(); ++b) {
                 const IndexType batch_offset = kOutputDimensions * b;
@@ -126,14 +138,17 @@ class Trainer<FeatureTransformer> {
                     max_activations_[t] = std::max(max_activations_[t], output_[index]);
                 }
             }
+
             return output_.data();
         }
 
         // backpropagation
         void Backpropagate(const LearnFloatType* gradients,
                            LearnFloatType learning_rate) {
+
             const LearnFloatType local_learning_rate =
                 learning_rate * learning_rate_scale_;
+
             for (IndexType b = 0; b < batch_->size(); ++b) {
                 const IndexType batch_offset = kOutputDimensions * b;
                 for (IndexType i = 0; i < kOutputDimensions; ++i) {
@@ -142,6 +157,7 @@ class Trainer<FeatureTransformer> {
                         ((output_[index] > kZero) * (output_[index] < kOne));
                 }
             }
+
             // Since the weight matrix updates only the columns corresponding to the features that appeared in the input,
             // Correct the learning rate and adjust the scale without using momentum
             const LearnFloatType effective_learning_rate =
@@ -156,8 +172,10 @@ class Trainer<FeatureTransformer> {
                                 &gradients_[output_offset], 1, biases_diff_, 1);
                 }
             }
+
             cblas_saxpy(kHalfDimensions, -local_learning_rate,
                         biases_diff_, 1, biases_, 1);
+
 #pragma omp parallel
             {
 #if defined(_OPENMP)
@@ -170,12 +188,14 @@ class Trainer<FeatureTransformer> {
                         const IndexType output_offset = batch_offset + kHalfDimensions * c;
                         for (const auto& feature : (*batch_)[b].training_features[c]) {
 #if defined(_OPENMP)
-            if (feature.GetIndex() % num_threads != thread_index) continue;
+                            if (feature.GetIndex() % num_threads != thread_index)
+                                continue;
 #endif
                             const IndexType weights_offset =
                                 kHalfDimensions * feature.GetIndex();
                             const auto scale = static_cast<LearnFloatType>(
                                 effective_learning_rate / feature.GetCount());
+
                             cblas_saxpy(kHalfDimensions, -scale,
                                         &gradients_[output_offset], 1,
                                         &weights_[weights_offset], 1);
@@ -183,10 +203,12 @@ class Trainer<FeatureTransformer> {
                     }
                 }
             }
+
 #else
             for (IndexType i = 0; i < kHalfDimensions; ++i) {
                 biases_diff_[i] *= momentum_;
             }
+
             for (IndexType b = 0; b < batch_->size(); ++b) {
                 const IndexType batch_offset = kOutputDimensions * b;
                 for (IndexType c = 0; c < 2; ++c) {
@@ -196,9 +218,11 @@ class Trainer<FeatureTransformer> {
                     }
                 }
             }
+
             for (IndexType i = 0; i < kHalfDimensions; ++i) {
                 biases_[i] -= local_learning_rate * biases_diff_[i];
             }
+
             for (IndexType b = 0; b < batch_->size(); ++b) {
                 const IndexType batch_offset = kOutputDimensions * b;
                 for (IndexType c = 0; c < 2; ++c) {
@@ -207,6 +231,7 @@ class Trainer<FeatureTransformer> {
                         const IndexType weights_offset = kHalfDimensions * feature.GetIndex();
                         const auto scale = static_cast<LearnFloatType>(
                             effective_learning_rate / feature.GetCount());
+
                         for (IndexType i = 0; i < kHalfDimensions; ++i) {
                             weights_[weights_offset + i] -=
                                 scale * gradients_[output_offset + i];
@@ -214,6 +239,7 @@ class Trainer<FeatureTransformer> {
                     }
                 }
             }
+
 #endif
             for (IndexType b = 0; b < batch_->size(); ++b) {
                 for (IndexType c = 0; c < 2; ++c) {
@@ -234,12 +260,15 @@ class Trainer<FeatureTransformer> {
             biases_diff_(),
             momentum_(0.2),
             learning_rate_scale_(1.0) {
+
             min_pre_activation_ = std::numeric_limits<LearnFloatType>::max();
             max_pre_activation_ = std::numeric_limits<LearnFloatType>::lowest();
+
             std::fill(std::begin(min_activations_), std::end(min_activations_),
                       std::numeric_limits<LearnFloatType>::max());
             std::fill(std::begin(max_activations_), std::end(max_activations_),
                       std::numeric_limits<LearnFloatType>::lowest());
+
             DequantizeParameters();
         }
 
@@ -249,17 +278,21 @@ class Trainer<FeatureTransformer> {
                 target_layer_->biases_[i] =
                     Round<typename LayerType::BiasType>(biases_[i] * kBiasScale);
             }
+
             std::vector<TrainingFeature> training_features;
+
 #pragma omp parallel for private(training_features)
             for (IndexType j = 0; j < RawFeatures::kDimensions; ++j) {
                 training_features.clear();
                 Features::Factorizer<RawFeatures>::AppendTrainingFeatures(
                     j, &training_features);
+
                 for (IndexType i = 0; i < kHalfDimensions; ++i) {
                     double sum = 0.0;
                     for (const auto& feature : training_features) {
                         sum += weights_[kHalfDimensions * feature.GetIndex() + i];
                     }
+
                     target_layer_->weights_[kHalfDimensions * j + i] =
                         Round<typename LayerType::WeightType>(sum * kWeightScale);
                 }
@@ -272,11 +305,14 @@ class Trainer<FeatureTransformer> {
                 biases_[i] = static_cast<LearnFloatType>(
                     target_layer_->biases_[i] / kBiasScale);
             }
+
             std::fill(std::begin(weights_), std::end(weights_), +kZero);
+
             for (IndexType i = 0; i < kHalfDimensions * RawFeatures::kDimensions; ++i) {
                 weights_[i] = static_cast<LearnFloatType>(
                     target_layer_->weights_[i] / kWeightScale);
             }
+
             std::fill(std::begin(biases_diff_), std::end(biases_diff_), +kZero);
         }
 
@@ -288,6 +324,7 @@ class Trainer<FeatureTransformer> {
                               std::begin(weights_) + kHalfDimensions * (i + 1), +kZero);
                 }
             }
+
             QuantizeParameters();
         }
 
@@ -299,6 +336,7 @@ class Trainer<FeatureTransformer> {
             constexpr LearnFloatType kPreActivationLimit =
                 std::numeric_limits<typename LayerType::WeightType>::max() /
                 kWeightScale;
+
             std::cout << "INFO: (min, max) of pre-activations = "
                       << min_pre_activation_ << ", "
                       << max_pre_activation_ << " (limit = "
@@ -308,6 +346,7 @@ class Trainer<FeatureTransformer> {
                 std::begin(min_activations_), std::end(min_activations_));
             const auto smallest_max_activation = *std::min_element(
                 std::begin(max_activations_), std::end(max_activations_));
+
             std::cout << "INFO: largest min activation = " << largest_min_activation
                       << ", smallest max activation = " << smallest_max_activation
                       << std::endl;
@@ -366,8 +405,6 @@ class Trainer<FeatureTransformer> {
         LearnFloatType max_activations_[kHalfDimensions];
     };
 
-}  // namespace NNUE
+}  // namespace Eval::NNUE
-
-}  // namespace Eval
 
 #endif

src/nnue/trainer/trainer_input_slice.h

@@ -1,15 +1,14 @@
-// Specialization of NNUE evaluation function learning class template for InputSlice
+#ifndef _NNUE_TRAINER_INPUT_SLICE_H_
-
-#ifndef _NNUE_TRAINER_INPUT_SLICE_H_
 #define _NNUE_TRAINER_INPUT_SLICE_H_
 
-#include "../../learn/learn.h"
-#include "../layers/input_slice.h"
 #include "trainer.h"
 
-namespace Eval {
+#include "learn/learn.h"
 
-namespace NNUE {
+#include "nnue/layers/input_slice.h"
+
+// Specialization of NNUE evaluation function learning class template for InputSlice
+namespace Eval::NNUE {
 
     // Learning: Input layer
     class SharedInputTrainer {
@@ -17,11 +16,15 @@ class SharedInputTrainer {
         // factory function
         static std::shared_ptr<SharedInputTrainer> Create(
             FeatureTransformer* ft) {
+
             static std::shared_ptr<SharedInputTrainer> instance;
+
             if (!instance) {
                 instance.reset(new SharedInputTrainer(ft));
             }
+
             ++instance->num_referrers_;
+
             return instance;
         }
 
@@ -31,7 +34,9 @@ class SharedInputTrainer {
                 current_operation_ = Operation::kSendMessage;
                 feature_transformer_trainer_->SendMessage(message);
             }
+
             assert(current_operation_ == Operation::kSendMessage);
+
             if (++num_calls_ == num_referrers_) {
                 num_calls_ = 0;
                 current_operation_ = Operation::kNone;
@@ -45,7 +50,9 @@ class SharedInputTrainer {
                 current_operation_ = Operation::kInitialize;
                 feature_transformer_trainer_->Initialize(rng);
             }
+
             assert(current_operation_ == Operation::kInitialize);
+
             if (++num_calls_ == num_referrers_) {
                 num_calls_ = 0;
                 current_operation_ = Operation::kNone;
@@ -57,26 +64,33 @@ class SharedInputTrainer {
             if (gradients_.size() < kInputDimensions * batch.size()) {
                 gradients_.resize(kInputDimensions * batch.size());
             }
+
             batch_size_ = static_cast<IndexType>(batch.size());
+
             if (num_calls_ == 0) {
                 current_operation_ = Operation::kPropagate;
                 output_ = feature_transformer_trainer_->Propagate(batch);
             }
+
             assert(current_operation_ == Operation::kPropagate);
+
             if (++num_calls_ == num_referrers_) {
                 num_calls_ = 0;
                 current_operation_ = Operation::kNone;
             }
+
             return output_;
         }
 
         // backpropagation
         void Backpropagate(const LearnFloatType* gradients,
                            LearnFloatType learning_rate) {
+
             if (num_referrers_ == 1) {
                 feature_transformer_trainer_->Backpropagate(gradients, learning_rate);
                 return;
             }
+
             if (num_calls_ == 0) {
                 current_operation_ = Operation::kBackPropagate;
                 for (IndexType b = 0; b < batch_size_; ++b) {
@@ -86,13 +100,16 @@ class SharedInputTrainer {
                     }
                 }
             }
+
             assert(current_operation_ == Operation::kBackPropagate);
+
             for (IndexType b = 0; b < batch_size_; ++b) {
                 const IndexType batch_offset = kInputDimensions * b;
                 for (IndexType i = 0; i < kInputDimensions; ++i) {
                     gradients_[batch_offset + i] += gradients[batch_offset + i];
                 }
             }
+
             if (++num_calls_ == num_referrers_) {
                 feature_transformer_trainer_->Backpropagate(
                     gradients_.data(), learning_rate);
@@ -160,6 +177,7 @@ class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
         // factory function
         static std::shared_ptr<Trainer> Create(
             LayerType* /*target_layer*/, FeatureTransformer* ft) {
+
             return std::shared_ptr<Trainer>(new Trainer(ft));
         }
 
@@ -180,7 +198,9 @@ class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
               output_.resize(kOutputDimensions * batch.size());
               gradients_.resize(kInputDimensions * batch.size());
             }
+
             batch_size_ = static_cast<IndexType>(batch.size());
+
             const auto input = shared_input_trainer_->Propagate(batch);
             for (IndexType b = 0; b < batch_size_; ++b) {
                 const IndexType input_offset = kInputDimensions * b;
@@ -194,12 +214,14 @@ class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
                 }
 #endif
             }
+
             return output_.data();
         }
 
         // backpropagation
         void Backpropagate(const LearnFloatType* gradients,
                            LearnFloatType learning_rate) {
+
             for (IndexType b = 0; b < batch_size_; ++b) {
                 const IndexType input_offset = kInputDimensions * b;
                 const IndexType output_offset = kOutputDimensions * b;
@@ -240,8 +262,6 @@ class Trainer<Layers::InputSlice<OutputDimensions, Offset>> {
         std::vector<LearnFloatType> gradients_;
     };
 
-}  // namespace NNUE
+}  // namespace Eval::NNUE
-
-}  // namespace Eval
 
 #endif

src/nnue/trainer/trainer_sum.h

@@ -1,15 +1,12 @@
-// Specialization of NNUE evaluation function learning class template for Sum
+#ifndef _NNUE_TRAINER_SUM_H_
-
-#ifndef _NNUE_TRAINER_SUM_H_
 #define _NNUE_TRAINER_SUM_H_
 
 #include "../../learn/learn.h"
 #include "../layers/sum.h"
 #include "trainer.h"
 
-namespace Eval {
+// Specialization of NNUE evaluation function learning class template for Sum
-
+namespace Eval::NNUE {
-namespace NNUE {
 
     // Learning: A layer that sums the outputs of multiple layers
     template <typename FirstPreviousLayer, typename... RemainingPreviousLayers>
@@ -24,6 +21,7 @@ class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
         // factory function
         static std::shared_ptr<Trainer> Create(
             LayerType* target_layer, FeatureTransformer* ft) {
+
             return std::shared_ptr<Trainer>(
                 new Trainer(target_layer, ft));
         }
@@ -49,6 +47,7 @@ class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
             batch_size_ = static_cast<IndexType>(batch.size());
             auto output = Tail::Propagate(batch);
             const auto head_output = previous_layer_trainer_->Propagate(batch);
+
 #if defined(USE_BLAS)
             cblas_saxpy(kOutputDimensions * batch_size_, 1.0,
                         head_output, 1, output, 1);
@@ -59,6 +58,7 @@ class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
                     output[batch_offset + i] += head_output[batch_offset + i];
                 }
             }
+
 #endif
             return output;
         }
@@ -66,6 +66,7 @@ class Trainer<Layers::Sum<FirstPreviousLayer, RemainingPreviousLayers...>> :
         // backpropagation
         void Backpropagate(const LearnFloatType* gradients,
                            LearnFloatType learning_rate) {
+
             Tail::Backpropagate(gradients, learning_rate);
             previous_layer_trainer_->Backpropagate(gradients, learning_rate);
         }
@@ -109,6 +110,7 @@ class Trainer<Layers::Sum<PreviousLayer>> {
         // factory function
         static std::shared_ptr<Trainer> Create(
             LayerType* target_layer, FeatureTransformer* ft) {
+
             return std::shared_ptr<Trainer>(
                 new Trainer(target_layer, ft));
         }
@@ -129,8 +131,10 @@ class Trainer<Layers::Sum<PreviousLayer>> {
             if (output_.size() < kOutputDimensions * batch.size()) {
                 output_.resize(kOutputDimensions * batch.size());
             }
+
             batch_size_ = static_cast<IndexType>(batch.size());
             const auto output = previous_layer_trainer_->Propagate(batch);
+
 #if defined(USE_BLAS)
             cblas_scopy(kOutputDimensions * batch_size_, output, 1, &output_[0], 1);
 #else
@@ -140,6 +144,7 @@ class Trainer<Layers::Sum<PreviousLayer>> {
                     output_[batch_offset + i] = output[batch_offset + i];
                 }
             }
+
 #endif
             return output_.data();
         }
@@ -147,6 +152,7 @@ class Trainer<Layers::Sum<PreviousLayer>> {
         // backpropagation
         void Backpropagate(const LearnFloatType* gradients,
                            LearnFloatType learning_rate) {
+
             previous_layer_trainer_->Backpropagate(gradients, learning_rate);
         }
 
@@ -179,8 +185,6 @@ class Trainer<Layers::Sum<PreviousLayer>> {
         std::vector<LearnFloatType> output_;
     };
 
-}  // namespace NNUE
+}  // namespace Eval::NNUE
-
-}  // namespace Eval
 
 #endif