Reduce the amount of sfens buffered for the validation step.

Used to be 10M, now we bound it by a multiple of validation_count, and at most 1M. This reduces the RAM usage greatly.
This commit is contained in:
Tomasz Sobczyk
2020-12-24 14:09:37 +01:00
committed by nodchip
parent 7636bcccd1
commit 4f6fdca31f
2 changed files with 4 additions and 2 deletions

View File

@@ -576,7 +576,7 @@ namespace Learner
SfenReaderMode::Cyclic,
1,
std::to_string(prng.next_random_seed()),
prm.sfen_read_size,
std::min<size_t>(prm.validation_count * 10, 1000000),
prm.thread_buffer_size),
learn_loss_sum{}
{

View File

@@ -50,7 +50,9 @@ namespace Learner{
) :
filenames(filenames_.begin(), filenames_.end()),
mode(mode_),
sfen_read_size(read_size),
// Due to the implementation of waiting for buffer empty a bit
// the read size must be at least twice the buffer size.
sfen_read_size(std::max(read_size, buffer_size * 2)),
thread_buffer_size(buffer_size),
prng(seed)
{