From 0e528995c279c773f5e6e5903bc4631586d8d27c Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Sat, 24 Oct 2020 20:03:40 +0200 Subject: [PATCH] Print avg bias/weight for affine trasform and feature transformer during training. --- src/nnue/trainer/trainer_affine_transform.h | 11 +++++++++++ src/nnue/trainer/trainer_feature_transformer.h | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/nnue/trainer/trainer_affine_transform.h b/src/nnue/trainer/trainer_affine_transform.h index 21e54f18..3179aeb0 100644 --- a/src/nnue/trainer/trainer_affine_transform.h +++ b/src/nnue/trainer/trainer_affine_transform.h @@ -241,6 +241,15 @@ namespace Eval::NNUE { void check_health() { + double abs_bias_sum = 0.0; + double abs_weight_sum = 0.0; + + for(auto b : biases_) + abs_bias_sum += std::abs(b); + + for(auto w : weights_) + abs_weight_sum += std::abs(w); + auto out = sync_region_cout.new_region(); out << "INFO (check_health):" @@ -248,7 +257,9 @@ namespace Eval::NNUE { << " - " << LayerType::get_name() << std::endl; + out << " - avg_abs_bias = " << abs_bias_sum / std::size(biases_) << std::endl; out << " - avg_abs_bias_diff = " << abs_biases_diff_sum_ / num_biases_diffs_ << std::endl; + out << " - avg_abs_weight = " << abs_weight_sum / std::size(weights_) << std::endl; out << " - avg_abs_weight_diff = " << abs_weights_diff_sum_ / num_weights_diffs_ << std::endl; out.unlock(); diff --git a/src/nnue/trainer/trainer_feature_transformer.h b/src/nnue/trainer/trainer_feature_transformer.h index 869ceb85..97b19c46 100644 --- a/src/nnue/trainer/trainer_feature_transformer.h +++ b/src/nnue/trainer/trainer_feature_transformer.h @@ -349,6 +349,15 @@ namespace Eval::NNUE { const auto smallest_max_activation = *std::min_element( std::begin(max_activations_), std::end(max_activations_)); + double abs_bias_sum = 0.0; + double abs_weight_sum = 0.0; + + for(auto b : biases_) + abs_bias_sum += std::abs(b); + + for(auto w : weights_) + abs_weight_sum += std::abs(w); + auto out = sync_region_cout.new_region(); out << "INFO (check_health):" @@ -370,6 +379,9 @@ namespace Eval::NNUE { << " , smallest max activation = " << smallest_max_activation << std::endl; + out << " - avg_abs_bias = " << abs_bias_sum / std::size(biases_) << std::endl; + out << " - avg_abs_weight = " << abs_weight_sum / std::size(weights_) << std::endl; + out << " - clipped " << static_cast(num_clipped_) / num_total_ * 100.0 << "% of outputs" << std::endl;