From 994eb5e183e76765af1ccd33c37bdfe192fe2141 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Mon, 21 Dec 2020 00:01:34 +0100 Subject: [PATCH] rescore_fen -> rescore. Make it work on .bin and .binpack inputs. --- docs/transform.md | 12 ++-- src/learn/transform.cpp | 134 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 131 insertions(+), 15 deletions(-) diff --git a/docs/transform.md b/docs/transform.md index 35ef16a1..a0a7ad75 100644 --- a/docs/transform.md +++ b/docs/transform.md @@ -20,17 +20,21 @@ Currently the following options are available: `interpolate` states that the output score should be a value interpolated between static eval and the score from the input file. After this token follows the interpolation constant `t`. `t` of 0 means that only static eval is used. `t` of 1 means that only score from the input file is used. `t` of 0.5 means that the static eval and input score are averaged. It accepts values outside of range `<0, 1>`, but the usefulness is questionable. -## `rescore_fen` +## `rescore` -`transform rescore_fen` takes named parameters in the form of `transform rescore_fen param_1_name param_1_value param_2_name param_2_value ...` and flag parameters which don't require values. +`transform rescore` takes named parameters in the form of `transform rescore param_1_name param_1_value param_2_name param_2_value ...` and flag parameters which don't require values. -This command takes a path to the input file which contains one FEN per line and outputs a .bin or .binpack file with these positions rescored with specified depth search. +This tool respects the UCI option `Threads` and uses all available threads. + +This command takes a path to the input file that is either a .epd file which contains one FEN per line or a .bin or .binpack file and outputs a .bin or .binpack file with these positions rescored with specified depth search. Currently the following options are available: -`input_file` - path to the input .epd file. Default: in.binpack. +`input_file` - path to the input file. Default: in.binpack. `output_file` - path to the output .bin or .binpack file. The file is opened in append mode. Default: out.binpack. `depth` - the search depth to use for rescoring. Default: 3. +`keep_moves` - whether to keep moves from the input file if available. Allows to keep compression in .binpack. Default: 1. + diff --git a/src/learn/transform.cpp b/src/learn/transform.cpp index 3e302e21..77671c65 100644 --- a/src/learn/transform.cpp +++ b/src/learn/transform.cpp @@ -2,6 +2,7 @@ #include "sfen_stream.h" #include "packed_sfen.h" +#include "sfen_writer.h" #include "thread.h" #include "position.h" @@ -47,11 +48,12 @@ namespace Learner } }; - struct RescoreFenParams + struct RescoreParams { std::string input_filename = "in.epd"; std::string output_filename = "out.binpack"; int depth = 3; + bool keep_moves = true; void enforce_constraints() { @@ -233,13 +235,11 @@ namespace Learner do_nudged_static(params); } - void do_rescore_fen(RescoreFenParams& params) + void do_rescore_epd(RescoreParams& params) { std::ifstream fens_file(params.input_filename); - auto next_fen = [&fens_file]() -> std::optional{ - static std::mutex mutex; - + auto next_fen = [&fens_file, mutex = std::mutex{}]() mutable -> std::optional{ std::string fen; std::unique_lock lock(mutex); @@ -333,9 +333,117 @@ namespace Learner std::cout << "Finished.\n"; } - void rescore_fen(std::istringstream& is) + void do_rescore_data(RescoreParams& params) { - RescoreFenParams params{}; + // TODO: Use SfenReader once it works correctly in sequential mode. See issue #271 + auto in = Learner::open_sfen_input_file(params.input_filename); + auto readsome = [&in, mutex = std::mutex{}](int n) mutable -> PSVector { + + PSVector psv; + psv.reserve(n); + + std::unique_lock lock(mutex); + + for (int i = 0; i < n; ++i) + { + auto ps_opt = in->next(); + if (ps_opt.has_value()) + { + psv.emplace_back(*ps_opt); + } + else + { + break; + } + } + + return psv; + }; + + auto sfen_format = ends_with(params.output_filename, ".binpack") ? SfenOutputType::Binpack : SfenOutputType::Bin; + + auto out = SfenWriter( + params.output_filename, + Threads.size(), + std::numeric_limits::max(), + sfen_format); + + // About Search::Limits + // Be careful because this member variable is global and affects other threads. + auto& limits = Search::Limits; + + // Make the search equivalent to the "go infinite" command. (Because it is troublesome if time management is done) + limits.infinite = true; + + // Since PV is an obstacle when displayed, erase it. + limits.silent = true; + + // If you use this, it will be compared with the accumulated nodes of each thread. Therefore, do not use it. + limits.nodes = 0; + + // depth is also processed by the one passed as an argument of Learner::search(). + limits.depth = 0; + + std::atomic num_processed = 0; + + Threads.execute_with_workers([&](auto& th){ + Position& pos = th.rootPos; + StateInfo si; + + for (;;) + { + PSVector psv = readsome(5000); + if (psv.empty()) + break; + + for(auto& ps : psv) + { + pos.set_from_packed_sfen(ps.sfen, &si, &th); + + auto [search_value, search_pv] = Search::search(pos, params.depth, 1); + if (search_pv.empty()) + continue; + + pos.sfen_pack(ps.sfen); + ps.score = search_value; + if (!params.keep_moves) + ps.move = search_pv[0]; + ps.padding = 0; + + out.write(th.thread_idx(), ps); + + auto p = num_processed.fetch_add(1) + 1; + if (p % 10000 == 0) + { + std::cout << "Processed " << p << " positions.\n"; + } + } + } + }); + Threads.wait_for_workers_finished(); + + std::cout << "Finished.\n"; + } + + void do_rescore(RescoreParams& params) + { + if (ends_with(params.input_filename, ".epd")) + { + do_rescore_epd(params); + } + else if (ends_with(params.input_filename, ".bin") || ends_with(params.input_filename, ".binpack")) + { + do_rescore_data(params); + } + else + { + std::cerr << "Invalid input file type.\n"; + } + } + + void rescore(std::istringstream& is) + { + RescoreParams params{}; while(true) { @@ -351,23 +459,27 @@ namespace Learner is >> params.input_filename; else if (token == "output_file") is >> params.output_filename; + else if (token == "keep_moves") + is >> params.keep_moves; } - std::cout << "Performing transform rescore_fen with parameters:\n"; + params.enforce_constraints(); + + std::cout << "Performing transform rescore with parameters:\n"; std::cout << "depth : " << params.depth << '\n'; std::cout << "input_file : " << params.input_filename << '\n'; std::cout << "output_file : " << params.output_filename << '\n'; + std::cout << "keep_moves : " << params.keep_moves << '\n'; std::cout << '\n'; - params.enforce_constraints(); - do_rescore_fen(params); + do_rescore(params); } void transform(std::istringstream& is) { const std::map subcommands = { { "nudged_static", &nudged_static }, - { "rescore_fen", &rescore_fen } + { "rescore", &rescore } }; Eval::NNUE::init();