From 4f6fdca31f8baaebab761f0d356addffbf592fc4 Mon Sep 17 00:00:00 2001
From: Tomasz Sobczyk <tomasz.sobczyk1997@gmail.com>
Date: Thu, 24 Dec 2020 14:09:37 +0100
Subject: [PATCH] Reduce the amount of sfens buffered for the validation step.

Used to be 10M, now we bound it by a multiple of validation_count, and at most 1M. This reduces the RAM usage greatly.
---
 src/learn/learn.cpp     | 2 +-
 src/learn/sfen_reader.h | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp
index 8265a66f..4e70f61c 100644
--- a/src/learn/learn.cpp
+++ b/src/learn/learn.cpp
@@ -576,7 +576,7 @@ namespace Learner
                 SfenReaderMode::Cyclic,
                 1,
                 std::to_string(prng.next_random_seed()),
-                prm.sfen_read_size,
+                std::min<size_t>(prm.validation_count * 10, 1000000),
                 prm.thread_buffer_size),
             learn_loss_sum{}
         {
diff --git a/src/learn/sfen_reader.h b/src/learn/sfen_reader.h
index 1574f63a..10fb8404 100644
--- a/src/learn/sfen_reader.h
+++ b/src/learn/sfen_reader.h
@@ -50,7 +50,9 @@ namespace Learner{
         ) :
             filenames(filenames_.begin(), filenames_.end()),
             mode(mode_),
-            sfen_read_size(read_size),
+            // Due to the implementation of waiting for buffer empty a bit
+            // the read size must be at least twice the buffer size.
+            sfen_read_size(std::max(read_size, buffer_size * 2)),
             thread_buffer_size(buffer_size),
             prng(seed)
         {