From 34510dd08a611762b9d826f9cd72cda72ad0ee13 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Wed, 25 Nov 2020 23:12:02 +0100 Subject: [PATCH] Remove used examples asyncronously. --- src/nnue/evaluate_nnue_learner.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/nnue/evaluate_nnue_learner.cpp b/src/nnue/evaluate_nnue_learner.cpp index 24ad2732..4a1a163d 100644 --- a/src/nnue/evaluate_nnue_learner.cpp +++ b/src/nnue/evaluate_nnue_learner.cpp @@ -214,9 +214,10 @@ namespace Eval::NNUE { std::vector abs_discrete_eval_sum_local(thread_pool.size(), 0.0); std::vector gradient_norm_local(thread_pool.size(), 0.0); - while (examples.size() >= batch_size) { - auto batch_begin = examples.end() - batch_size; - auto batch_end = examples.end(); + auto prev_batch_begin = examples.end(); + while (prev_batch_begin - examples.begin() >= batch_size) { + auto batch_begin = prev_batch_begin - batch_size; + auto batch_end = prev_batch_begin; auto size = batch_end - batch_begin; const auto network_output = trainer->step_start(thread_pool, batch_begin, batch_end); std::vector gradients(size); @@ -253,14 +254,20 @@ namespace Eval::NNUE { trainer->backpropagate(th, gradients.data(), offset, count); } ); + + // We can asyncronously erase the examples that we used in the previous + // step. This can be done safely because we're no longer using these + // examples and erase won't invalidate iterators. + examples.erase(prev_batch_begin, examples.end()); + prev_batch_begin = batch_begin; + thread_pool.wait_for_workers_finished(); trainer->step_end(thread_pool, learning_rate); - examples.resize(examples.size() - size); - collect_stats = false; } + examples.erase(prev_batch_begin, examples.end()); if (verbose) {