From c76bb34a96ed36511e360a60bd9f33e364617139 Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Thu, 10 Sep 2020 15:18:47 +0200 Subject: [PATCH] Add convert UCI function that allows conversion of files between any of plain, bin, and binpack. Usage: convert infile outfile [append]. --- src/extra/nnue_data_binpack_format.h | 33 +++++---- src/learn/convert.cpp | 105 +++++++++++++++++++++++++++ src/uci.cpp | 3 + 3 files changed, 125 insertions(+), 16 deletions(-) diff --git a/src/extra/nnue_data_binpack_format.h b/src/extra/nnue_data_binpack_format.h index 839fc17c..2c555939 100644 --- a/src/extra/nnue_data_binpack_format.h +++ b/src/extra/nnue_data_binpack_format.h @@ -6915,7 +6915,7 @@ namespace binpack { constexpr std::size_t reportEveryNPositions = 100'000; - std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; CompressedTrainingDataEntryWriter writer(outputPath, om); TrainingDataEntry e; @@ -6961,13 +6961,15 @@ namespace binpack if (key == "ply"sv) e.ply = std::stoi(value); if (key == "result"sv) e.result = std::stoi(value); } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } inline void convertBinpackToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om) { constexpr std::size_t bufferSize = MiB; - std::cout << "Decompressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; CompressedTrainingDataEntryReader reader(inputPath); std::ofstream outputFile(outputPath, om); @@ -6999,6 +7001,8 @@ namespace binpack const auto cur = outputFile.tellp(); std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } @@ -7006,14 +7010,9 @@ namespace binpack { constexpr std::size_t reportEveryNPositions = 100'000; - std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; CompressedTrainingDataEntryWriter writer(outputPath, om); - TrainingDataEntry e; - - std::string key; - std::string value; - std::string move; std::ifstream inputFile(inputPath, std::ios_base::binary); const auto base = inputFile.tellg(); @@ -7037,13 +7036,15 @@ namespace binpack std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } inline void convertBinpackToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om) { constexpr std::size_t bufferSize = MiB; - std::cout << "Decompressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; CompressedTrainingDataEntryReader reader(inputPath); std::ofstream outputFile(outputPath, std::ios_base::binary | om); @@ -7075,6 +7076,8 @@ namespace binpack const auto cur = outputFile.tellp(); std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } inline void convertBinToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om) @@ -7083,12 +7086,6 @@ namespace binpack std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; - TrainingDataEntry e; - - std::string key; - std::string value; - std::string move; - std::ifstream inputFile(inputPath, std::ios_base::binary); const auto base = inputFile.tellg(); std::size_t numProcessedPositions = 0; @@ -7127,13 +7124,15 @@ namespace binpack const auto cur = outputFile.tellp(); std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } inline void convertPlainToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om) { constexpr std::size_t bufferSize = MiB; - std::cout << "Compressing " << inputPath << " to " << outputPath << '\n'; + std::cout << "Converting " << inputPath << " to " << outputPath << '\n'; std::ofstream outputFile(outputPath, std::ios_base::binary | om); std::vector buffer; @@ -7194,5 +7193,7 @@ namespace binpack const auto cur = outputFile.tellp(); std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n"; } + + std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n"; } } \ No newline at end of file diff --git a/src/learn/convert.cpp b/src/learn/convert.cpp index d07fc00c..364ad3dd 100644 --- a/src/learn/convert.cpp +++ b/src/learn/convert.cpp @@ -12,6 +12,8 @@ #include "../position.h" #include "../tt.h" +#include "../extra/nnue_data_binpack_format.h" + #include #include #include @@ -497,5 +499,108 @@ namespace Learner ofs.close(); std::cout << "all done" << std::endl; } + + static inline const std::string plain_extension = ".plain"; + static inline const std::string bin_extension = ".bin"; + static inline const std::string binpack_extension = ".binpack"; + + static bool file_exists(const std::string& name) + { + std::ifstream f(name); + return f.good(); + } + + static bool ends_with(const std::string& lhs, const std::string& end) + { + if (end.size() > lhs.size()) return false; + + return std::equal(end.rbegin(), end.rend(), lhs.rbegin()); + } + + static bool is_convert_of_type( + const std::string& input_path, + const std::string& output_path, + const std::string& expected_input_extension, + const std::string& expected_output_extension) + { + return ends_with(input_path, expected_input_extension) + && ends_with(output_path, expected_output_extension); + } + + using ConvertFunctionType = void(std::string inputPath, std::string outputPath, std::ios_base::openmode om); + + static ConvertFunctionType* get_convert_function(const std::string& input_path, const std::string& output_path) + { + if (is_convert_of_type(input_path, output_path, plain_extension, bin_extension)) + return binpack::convertPlainToBin; + if (is_convert_of_type(input_path, output_path, plain_extension, binpack_extension)) + return binpack::convertPlainToBinpack; + + if (is_convert_of_type(input_path, output_path, bin_extension, plain_extension)) + return binpack::convertBinToPlain; + if (is_convert_of_type(input_path, output_path, bin_extension, binpack_extension)) + return binpack::convertBinToBinpack; + + if (is_convert_of_type(input_path, output_path, binpack_extension, plain_extension)) + return binpack::convertBinpackToPlain; + if (is_convert_of_type(input_path, output_path, binpack_extension, bin_extension)) + return binpack::convertBinpackToBin; + + return nullptr; + } + + static void convert(const std::string& input_path, const std::string& output_path, std::ios_base::openmode om) + { + if(!file_exists(input_path)) + { + std::cerr << "Input file does not exist.\n"; + return; + } + + auto func = get_convert_function(input_path, output_path); + if (func != nullptr) + { + func(input_path, output_path, om); + } + else + { + std::cerr << "Conversion between files of these types is not supported.\n"; + } + } + + static void convert(const std::vector& args) + { + if (args.size() < 2 || args.size() > 3) + { + std::cerr << "Invalid arguments.\n"; + std::cerr << "Usage: convert from_path to_path [append]\n"; + return; + } + + const bool append = (args.size() == 3) && (args[2] == "append"); + const std::ios_base::openmode openmode = + append + ? std::ios_base::app + : std::ios_base::trunc; + + convert(args[0], args[1], openmode); + } + + void convert(istringstream& is) + { + std::vector args; + + while (true) + { + std::string token = ""; + is >> token; + if (token == "") + break; + + args.push_back(token); + } + + convert(args); + } } #endif diff --git a/src/uci.cpp b/src/uci.cpp index 6675f2e0..96adf927 100644 --- a/src/uci.cpp +++ b/src/uci.cpp @@ -50,6 +50,8 @@ namespace Learner // Learning from the generated game record void learn(Position& pos, istringstream& is); + void convert(istringstream& is); + // A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch(). typedef std::pair > ValueAndPV; @@ -352,6 +354,7 @@ void UCI::loop(int argc, char* argv[]) { #if defined (EVAL_LEARN) else if (token == "gensfen") Learner::gen_sfen(pos, is); else if (token == "learn") Learner::learn(pos, is); + else if (token == "convert") Learner::convert(is); // Command to call qsearch(),search() directly for testing else if (token == "qsearch") qsearch_cmd(pos);