diff --git a/src/learn/convert.cpp b/src/learn/convert.cpp index 59111dcf..a7528b02 100644 --- a/src/learn/convert.cpp +++ b/src/learn/convert.cpp @@ -10,6 +10,8 @@ #include "extra/nnue_data_binpack_format.h" +#include "nnue/evaluate_nnue.h" + #include "syzygy/tbprobe.h" #include @@ -600,4 +602,214 @@ namespace Learner convert(args); } + + static void append_files_from_dir( + std::vector& filenames, + const std::string& base_dir, + const std::string& target_dir) + { + string kif_base_dir = Path::combine(base_dir, target_dir); + + namespace sys = std::filesystem; + sys::path p(kif_base_dir); // Origin of enumeration + std::for_each(sys::directory_iterator(p), sys::directory_iterator(), + [&](const sys::path& path) { + if (sys::is_regular_file(path)) + filenames.push_back(Path::combine(target_dir, path.filename().generic_string())); + }); + } + + static void rebase_files( + std::vector& filenames, + const std::string& base_dir) + { + for (auto& file : filenames) + { + file = Path::combine(base_dir, file); + } + } + + void convert_bin_from_pgn_extract(std::istringstream& is) + { + std::vector filenames; + + string base_dir; + string target_dir; + + bool pgn_eval_side_to_move = false; + bool convert_no_eval_fens_as_score_zero = false; + + string output_file_name = "shuffled_sfen.bin"; + + while (true) + { + string option; + is >> option; + + if (option == "") + break; + + if (option == "targetdir") is >> target_dir; + else if (option == "targetfile") + { + std::string filename; + is >> filename; + filenames.push_back(filename); + } + + else if (option == "basedir") is >> base_dir; + + else if (option == "pgn_eval_side_to_move") is >> pgn_eval_side_to_move; + else if (option == "convert_no_eval_fens_as_score_zero") is >> convert_no_eval_fens_as_score_zero; + else if (option == "output_file_name") is >> output_file_name; + else + { + cout << "Unknown option: " << option << ". Ignoring.\n"; + } + } + + if (!target_dir.empty()) + { + append_files_from_dir(filenames, base_dir, target_dir); + } + rebase_files(filenames, base_dir); + + Eval::NNUE::init(); + + cout << "convert_bin_from_pgn-extract.." << endl; + convert_bin_from_pgn_extract( + filenames, + output_file_name, + pgn_eval_side_to_move, + convert_no_eval_fens_as_score_zero); + } + + void convert_bin(std::istringstream& is) + { + std::vector filenames; + + string base_dir; + string target_dir; + + int ply_minimum = 0; + int ply_maximum = 114514; + bool interpolate_eval = 0; + bool check_invalid_fen = false; + bool check_illegal_move = false; + + bool pgn_eval_side_to_move = false; + bool convert_no_eval_fens_as_score_zero = false; + + double src_score_min_value = 0.0; + double src_score_max_value = 1.0; + double dest_score_min_value = 0.0; + double dest_score_max_value = 1.0; + + string output_file_name = "shuffled_sfen.bin"; + + while (true) + { + string option; + is >> option; + + if (option == "") + break; + + if (option == "targetdir") is >> target_dir; + else if (option == "targetfile") + { + std::string filename; + is >> filename; + filenames.push_back(filename); + } + + else if (option == "basedir") is >> base_dir; + + else if (option == "ply_minimum") is >> ply_minimum; + else if (option == "ply_maximum") is >> ply_maximum; + else if (option == "interpolate_eval") is >> interpolate_eval; + else if (option == "check_invalid_fen") is >> check_invalid_fen; + else if (option == "check_illegal_move") is >> check_illegal_move; + else if (option == "pgn_eval_side_to_move") is >> pgn_eval_side_to_move; + else if (option == "convert_no_eval_fens_as_score_zero") is >> convert_no_eval_fens_as_score_zero; + else if (option == "src_score_min_value") is >> src_score_min_value; + else if (option == "src_score_max_value") is >> src_score_max_value; + else if (option == "dest_score_min_value") is >> dest_score_min_value; + else if (option == "dest_score_max_value") is >> dest_score_max_value; + else if (option == "output_file_name") is >> output_file_name; + else + { + cout << "Unknown option: " << option << ". Ignoring.\n"; + } + } + + if (!target_dir.empty()) + { + append_files_from_dir(filenames, base_dir, target_dir); + } + rebase_files(filenames, base_dir); + + Eval::NNUE::init(); + + cout << "convert_bin.." << endl; + convert_bin( + filenames, + output_file_name, + ply_minimum, + ply_maximum, + interpolate_eval, + src_score_min_value, + src_score_max_value, + dest_score_min_value, + dest_score_max_value, + check_invalid_fen, + check_illegal_move + ); + } + + void convert_plain(std::istringstream& is) + { + std::vector filenames; + + string base_dir; + string target_dir; + + string output_file_name = "shuffled_sfen.bin"; + + while (true) + { + string option; + is >> option; + + if (option == "") + break; + + if (option == "targetdir") is >> target_dir; + else if (option == "targetfile") + { + std::string filename; + is >> filename; + filenames.push_back(filename); + } + + else if (option == "basedir") is >> base_dir; + + else if (option == "output_file_name") is >> output_file_name; + else + { + cout << "Unknown option: " << option << ". Ignoring.\n"; + } + } + + if (!target_dir.empty()) + { + append_files_from_dir(filenames, base_dir, target_dir); + } + rebase_files(filenames, base_dir); + + Eval::NNUE::init(); + + cout << "convert_plain.." << endl; + convert_plain(filenames, output_file_name); + } } diff --git a/src/learn/convert.h b/src/learn/convert.h index a41885d9..227f0799 100644 --- a/src/learn/convert.h +++ b/src/learn/convert.h @@ -6,30 +6,13 @@ #include namespace Learner { - void convert_bin_from_pgn_extract( - const std::vector& filenames, - const std::string& output_file_name, - const bool pgn_eval_side_to_move, - const bool convert_no_eval_fens_as_score_zero); - - void convert_bin( - const std::vector& filenames, - const std::string& output_file_name, - const int ply_minimum, - const int ply_maximum, - const int interpolate_eval, - const int src_score_min_value, - const int src_score_max_value, - const int dest_score_min_value, - const int dest_score_max_value, - const bool check_invalid_fen, - const bool check_illegal_move); - - void convert_plain( - const std::vector& filenames, - const std::string& output_file_name); - void convert(std::istringstream& is); + + void convert_bin_from_pgn_extract(std::istringstream& is); + + void convert_bin(std::istringstream& is); + + void convert_plain(std::istringstream& is); } #endif diff --git a/src/learn/learn.cpp b/src/learn/learn.cpp index 2cab54b7..32aa986f 100644 --- a/src/learn/learn.cpp +++ b/src/learn/learn.cpp @@ -19,7 +19,6 @@ #include "learn.h" -#include "convert.h" #include "sfen_reader.h" #include "misc.h" @@ -940,29 +939,8 @@ namespace Learner // Game file storage folder (get game file with relative path from here) string base_dir; - string target_dir; - // --- Function that only shuffles the teacher aspect - - // normal shuffle - // Conversion of packed sfen. In plain, it consists of sfen(string), - // evaluation value (integer), move (eg 7g7f, string), result (loss-1, win 1, draw 0) - bool use_convert_plain = false; - // convert plain format teacher to Yaneura King's bin - bool use_convert_bin = false; - int ply_minimum = 0; - int ply_maximum = 114514; - bool interpolate_eval = 0; - bool check_invalid_fen = false; - bool check_illegal_move = false; - // convert teacher in pgn-extract format to Yaneura King's bin - bool use_convert_bin_from_pgn_extract = false; - bool pgn_eval_side_to_move = false; - bool convert_no_eval_fens_as_score_zero = false; - // File name to write in those cases (default is "shuffled_sfen.bin") - string output_file_name = "shuffled_sfen.bin"; - // If the absolute value of the evaluation value // in the deep search of the teacher phase exceeds this value, // that phase is discarded. @@ -1079,19 +1057,11 @@ namespace Learner else if (option == "loss_output_interval") is >> loss_output_interval; else if (option == "validation_set_file_name") is >> validation_set_file_name; - // Rabbit convert related - else if (option == "convert_plain") use_convert_plain = true; - else if (option == "convert_bin") use_convert_bin = true; - else if (option == "interpolate_eval") is >> interpolate_eval; - else if (option == "check_invalid_fen") is >> check_invalid_fen; - else if (option == "check_illegal_move") is >> check_illegal_move; - else if (option == "convert_bin_from_pgn-extract") use_convert_bin_from_pgn_extract = true; - else if (option == "pgn_eval_side_to_move") is >> pgn_eval_side_to_move; - else if (option == "convert_no_eval_fens_as_score_zero") is >> convert_no_eval_fens_as_score_zero; else if (option == "src_score_min_value") is >> src_score_min_value; else if (option == "src_score_max_value") is >> src_score_max_value; else if (option == "dest_score_min_value") is >> dest_score_min_value; else if (option == "dest_score_max_value") is >> dest_score_max_value; + else if (option == "seed") is >> seed; else if (option == "set_recommended_uci_options") { @@ -1123,8 +1093,8 @@ namespace Learner cout << "Warning! OpenMP disabled." << endl; #endif - LearnerThink learn_think(thread_num, seed); - + // Right now we only have the individual files. + // We need to apply base_dir here rebase_files(filenames, base_dir); if (!target_dir.empty()) { @@ -1144,48 +1114,6 @@ namespace Learner cout << "base dir : " << base_dir << endl; cout << "target dir : " << target_dir << endl; - if (use_convert_plain) - { - Eval::NNUE::init(); - cout << "convert_plain.." << endl; - convert_plain(filenames, output_file_name); - return; - } - - if (use_convert_bin) - { - Eval::NNUE::init(); - cout << "convert_bin.." << endl; - convert_bin( - filenames, - output_file_name, - ply_minimum, - ply_maximum, - interpolate_eval, - src_score_min_value, - src_score_max_value, - dest_score_min_value, - dest_score_max_value, - check_invalid_fen, - check_illegal_move); - - return; - - } - - if (use_convert_bin_from_pgn_extract) - { - Eval::NNUE::init(); - cout << "convert_bin_from_pgn-extract.." << endl; - convert_bin_from_pgn_extract( - filenames, - output_file_name, - pgn_eval_side_to_move, - convert_no_eval_fens_as_score_zero); - - return; - } - cout << "loop : " << loop << endl; cout << "eval_limit : " << eval_limit << endl; cout << "save_only_once : " << (save_only_once ? "true" : "false") << endl; @@ -1226,6 +1154,8 @@ namespace Learner cout << "init.." << endl; + LearnerThink learn_think(thread_num, seed); + Threads.main()->ponder = false; set_learning_search_limits(); @@ -1244,8 +1174,6 @@ namespace Learner Path::combine(Options["EvalSaveDir"], "original"); } - cout << "init done." << endl; - // Reflect other option settings. learn_think.eval_limit = eval_limit; learn_think.save_only_once = save_only_once; @@ -1271,6 +1199,8 @@ namespace Learner } } + cout << "init done." << endl; + // Start learning. learn_think.learn(); } diff --git a/src/uci.cpp b/src/uci.cpp index b05c7eeb..398fd01a 100644 --- a/src/uci.cpp +++ b/src/uci.cpp @@ -341,6 +341,9 @@ void UCI::loop(int argc, char* argv[]) { else if (token == "gensfen") Learner::gen_sfen(pos, is); else if (token == "learn") Learner::learn(pos, is); else if (token == "convert") Learner::convert(is); + else if (token == "convert_bin") Learner::convert_bin(is); + else if (token == "convert_plain") Learner::convert_plain(is); + else if (token == "convert_bin_from_pgn_extract") Learner::convert_bin_from_pgn_extract(is); // Command to call qsearch(),search() directly for testing else if (token == "qsearch") qsearch_cmd(pos);