Move the old convert stuff from learn to their own commands.

src/learn/convert.cpp

@@ -10,6 +10,8 @@
 
 #include "extra/nnue_data_binpack_format.h"
 
+#include "nnue/evaluate_nnue.h"
+
 #include "syzygy/tbprobe.h"
 
 #include <sstream>
@@ -600,4 +602,214 @@ namespace Learner
 
         convert(args);
     }
+
+    static void append_files_from_dir(
+        std::vector<std::string>& filenames,
+        const std::string& base_dir,
+        const std::string& target_dir)
+    {
+        string kif_base_dir = Path::combine(base_dir, target_dir);
+
+        namespace sys = std::filesystem;
+        sys::path p(kif_base_dir); // Origin of enumeration
+        std::for_each(sys::directory_iterator(p), sys::directory_iterator(),
+            [&](const sys::path& path) {
+                if (sys::is_regular_file(path))
+                    filenames.push_back(Path::combine(target_dir, path.filename().generic_string()));
+            });
+    }
+
+    static void rebase_files(
+        std::vector<std::string>& filenames,
+        const std::string& base_dir)
+    {
+        for (auto& file : filenames)
+        {
+            file = Path::combine(base_dir, file);
+        }
+    }
+
+    void convert_bin_from_pgn_extract(std::istringstream& is)
+    {
+        std::vector<std::string> filenames;
+
+        string base_dir;
+        string target_dir;
+
+        bool pgn_eval_side_to_move = false;
+        bool convert_no_eval_fens_as_score_zero = false;
+
+        string output_file_name = "shuffled_sfen.bin";
+
+        while (true)
+        {
+            string option;
+            is >> option;
+
+            if (option == "")
+                break;
+
+            if (option == "targetdir") is >> target_dir;
+            else if (option == "targetfile")
+            {
+                std::string filename;
+                is >> filename;
+                filenames.push_back(filename);
+            }
+
+            else if (option == "basedir")   is >> base_dir;
+
+            else if (option == "pgn_eval_side_to_move") is >> pgn_eval_side_to_move;
+            else if (option == "convert_no_eval_fens_as_score_zero") is >> convert_no_eval_fens_as_score_zero;
+            else if (option == "output_file_name") is >> output_file_name;
+            else
+            {
+                cout << "Unknown option: " << option << ". Ignoring.\n";
+            }
+        }
+
+        if (!target_dir.empty())
+        {
+            append_files_from_dir(filenames, base_dir, target_dir);
+        }
+        rebase_files(filenames, base_dir);
+
+        Eval::NNUE::init();
+
+        cout << "convert_bin_from_pgn-extract.." << endl;
+        convert_bin_from_pgn_extract(
+            filenames,
+            output_file_name,
+            pgn_eval_side_to_move,
+            convert_no_eval_fens_as_score_zero);
+    }
+
+    void convert_bin(std::istringstream& is)
+    {
+        std::vector<std::string> filenames;
+
+        string base_dir;
+        string target_dir;
+
+        int ply_minimum = 0;
+        int ply_maximum = 114514;
+        bool interpolate_eval = 0;
+        bool check_invalid_fen = false;
+        bool check_illegal_move = false;
+
+        bool pgn_eval_side_to_move = false;
+        bool convert_no_eval_fens_as_score_zero = false;
+
+        double src_score_min_value = 0.0;
+        double src_score_max_value = 1.0;
+        double dest_score_min_value = 0.0;
+        double dest_score_max_value = 1.0;
+
+        string output_file_name = "shuffled_sfen.bin";
+
+        while (true)
+        {
+            string option;
+            is >> option;
+
+            if (option == "")
+                break;
+
+            if (option == "targetdir") is >> target_dir;
+            else if (option == "targetfile")
+            {
+                std::string filename;
+                is >> filename;
+                filenames.push_back(filename);
+            }
+
+            else if (option == "basedir")   is >> base_dir;
+
+            else if (option == "ply_minimum") is >> ply_minimum;
+            else if (option == "ply_maximum") is >> ply_maximum;
+            else if (option == "interpolate_eval") is >> interpolate_eval;
+            else if (option == "check_invalid_fen") is >> check_invalid_fen;
+            else if (option == "check_illegal_move") is >> check_illegal_move;
+            else if (option == "pgn_eval_side_to_move") is >> pgn_eval_side_to_move;
+            else if (option == "convert_no_eval_fens_as_score_zero") is >> convert_no_eval_fens_as_score_zero;
+            else if (option == "src_score_min_value") is >> src_score_min_value;
+            else if (option == "src_score_max_value") is >> src_score_max_value;
+            else if (option == "dest_score_min_value") is >> dest_score_min_value;
+            else if (option == "dest_score_max_value") is >> dest_score_max_value;
+            else if (option == "output_file_name") is >> output_file_name;
+            else
+            {
+                cout << "Unknown option: " << option << ". Ignoring.\n";
+            }
+        }
+
+        if (!target_dir.empty())
+        {
+            append_files_from_dir(filenames, base_dir, target_dir);
+        }
+        rebase_files(filenames, base_dir);
+
+        Eval::NNUE::init();
+
+        cout << "convert_bin.." << endl;
+            convert_bin(
+                filenames,
+                output_file_name,
+                ply_minimum,
+                ply_maximum,
+                interpolate_eval,
+                src_score_min_value,
+                src_score_max_value,
+                dest_score_min_value,
+                dest_score_max_value,
+                check_invalid_fen,
+                check_illegal_move
+            );
+    }
+
+    void convert_plain(std::istringstream& is)
+    {
+        std::vector<std::string> filenames;
+
+        string base_dir;
+        string target_dir;
+
+        string output_file_name = "shuffled_sfen.bin";
+
+        while (true)
+        {
+            string option;
+            is >> option;
+
+            if (option == "")
+                break;
+
+            if (option == "targetdir") is >> target_dir;
+            else if (option == "targetfile")
+            {
+                std::string filename;
+                is >> filename;
+                filenames.push_back(filename);
+            }
+
+            else if (option == "basedir")   is >> base_dir;
+
+            else if (option == "output_file_name") is >> output_file_name;
+            else
+            {
+                cout << "Unknown option: " << option << ". Ignoring.\n";
+            }
+        }
+
+        if (!target_dir.empty())
+        {
+            append_files_from_dir(filenames, base_dir, target_dir);
+        }
+        rebase_files(filenames, base_dir);
+
+        Eval::NNUE::init();
+
+        cout << "convert_plain.." << endl;
+        convert_plain(filenames, output_file_name);
+    }
 }

src/learn/convert.h

@@ -6,30 +6,13 @@
 #include <sstream>
 
 namespace Learner {
-    void convert_bin_from_pgn_extract(
-        const std::vector<std::string>& filenames,
-        const std::string& output_file_name,
-        const bool pgn_eval_side_to_move,
-        const bool convert_no_eval_fens_as_score_zero);
-
-    void convert_bin(
-        const std::vector<std::string>& filenames,
-        const std::string& output_file_name,
-        const int ply_minimum,
-        const int ply_maximum,
-        const int interpolate_eval,
-        const int src_score_min_value,
-        const int src_score_max_value,
-        const int dest_score_min_value,
-        const int dest_score_max_value,
-        const bool check_invalid_fen,
-        const bool check_illegal_move);
-
-    void convert_plain(
-        const std::vector<std::string>& filenames,
-        const std::string& output_file_name);
-
     void convert(std::istringstream& is);
+
+    void convert_bin_from_pgn_extract(std::istringstream& is);
+
+    void convert_bin(std::istringstream& is);
+
+    void convert_plain(std::istringstream& is);
 }
 
 #endif

src/learn/learn.cpp

@@ -19,7 +19,6 @@
 
 #include "learn.h"
 
-#include "convert.h"
 #include "sfen_reader.h"
 
 #include "misc.h"
@@ -940,29 +939,8 @@ namespace Learner
 
         // Game file storage folder (get game file with relative path from here)
         string base_dir;
-
         string target_dir;
 
-        // --- Function that only shuffles the teacher aspect
-
-        // normal shuffle
-        // Conversion of packed sfen. In plain, it consists of sfen(string),
-        // evaluation value (integer), move (eg 7g7f, string), result (loss-1, win 1, draw 0)
-        bool use_convert_plain = false;
-        // convert plain format teacher to Yaneura King's bin
-        bool use_convert_bin = false;
-        int ply_minimum = 0;
-        int ply_maximum = 114514;
-        bool interpolate_eval = 0;
-        bool check_invalid_fen = false;
-        bool check_illegal_move = false;
-        // convert teacher in pgn-extract format to Yaneura King's bin
-        bool use_convert_bin_from_pgn_extract = false;
-        bool pgn_eval_side_to_move = false;
-        bool convert_no_eval_fens_as_score_zero = false;
-        // File name to write in those cases (default is "shuffled_sfen.bin")
-        string output_file_name = "shuffled_sfen.bin";
-
         // If the absolute value of the evaluation value
         // in the deep search of the teacher phase exceeds this value,
         // that phase is discarded.
@@ -1079,19 +1057,11 @@ namespace Learner
             else if (option == "loss_output_interval") is >> loss_output_interval;
             else if (option == "validation_set_file_name") is >> validation_set_file_name;
 
-            // Rabbit convert related
-            else if (option == "convert_plain") use_convert_plain = true;
-            else if (option == "convert_bin") use_convert_bin = true;
-            else if (option == "interpolate_eval") is >> interpolate_eval;
-            else if (option == "check_invalid_fen") is >> check_invalid_fen;
-            else if (option == "check_illegal_move") is >> check_illegal_move;
-            else if (option == "convert_bin_from_pgn-extract") use_convert_bin_from_pgn_extract = true;
-            else if (option == "pgn_eval_side_to_move") is >> pgn_eval_side_to_move;
-            else if (option == "convert_no_eval_fens_as_score_zero") is >> convert_no_eval_fens_as_score_zero;
             else if (option == "src_score_min_value") is >> src_score_min_value;
             else if (option == "src_score_max_value") is >> src_score_max_value;
             else if (option == "dest_score_min_value") is >> dest_score_min_value;
             else if (option == "dest_score_max_value") is >> dest_score_max_value;
+
             else if (option == "seed") is >> seed;
             else if (option == "set_recommended_uci_options")
             {
@@ -1123,8 +1093,8 @@ namespace Learner
         cout << "Warning! OpenMP disabled." << endl;
 #endif
 
-        LearnerThink learn_think(thread_num, seed);
-
+        // Right now we only have the individual files.
+        // We need to apply base_dir here
         rebase_files(filenames, base_dir);
         if (!target_dir.empty())
         {
@@ -1144,48 +1114,6 @@ namespace Learner
         cout << "base dir        : " << base_dir << endl;
         cout << "target dir      : " << target_dir << endl;
 
-        if (use_convert_plain)
-        {
-            Eval::NNUE::init();
-            cout << "convert_plain.." << endl;
-            convert_plain(filenames, output_file_name);
-            return;
-        }
-
-        if (use_convert_bin)
-        {
-            Eval::NNUE::init();
-            cout << "convert_bin.." << endl;
-            convert_bin(
-                filenames,
-                output_file_name,
-                ply_minimum,
-                ply_maximum,
-                interpolate_eval,
-                src_score_min_value,
-                src_score_max_value,
-                dest_score_min_value,
-                dest_score_max_value,
-                check_invalid_fen,
-                check_illegal_move);
-
-            return;
-
-        }
-
-        if (use_convert_bin_from_pgn_extract)
-        {
-            Eval::NNUE::init();
-            cout << "convert_bin_from_pgn-extract.." << endl;
-            convert_bin_from_pgn_extract(
-                filenames,
-                output_file_name,
-                pgn_eval_side_to_move,
-                convert_no_eval_fens_as_score_zero);
-
-            return;
-        }
-
         cout << "loop              : " << loop << endl;
         cout << "eval_limit        : " << eval_limit << endl;
         cout << "save_only_once    : " << (save_only_once ? "true" : "false") << endl;
@@ -1226,6 +1154,8 @@ namespace Learner
 
         cout << "init.." << endl;
 
+        LearnerThink learn_think(thread_num, seed);
+
         Threads.main()->ponder = false;
 
         set_learning_search_limits();
@@ -1244,8 +1174,6 @@ namespace Learner
                 Path::combine(Options["EvalSaveDir"], "original");
         }
 
-        cout << "init done." << endl;
-
         // Reflect other option settings.
         learn_think.eval_limit = eval_limit;
         learn_think.save_only_once = save_only_once;
@@ -1271,6 +1199,8 @@ namespace Learner
             }
         }
 
+        cout << "init done." << endl;
+
         // Start learning.
         learn_think.learn();
     }

src/uci.cpp

@@ -341,6 +341,9 @@ void UCI::loop(int argc, char* argv[]) {
       else if (token == "gensfen") Learner::gen_sfen(pos, is);
       else if (token == "learn") Learner::learn(pos, is);
       else if (token == "convert") Learner::convert(is);
+      else if (token == "convert_bin") Learner::convert_bin(is);
+      else if (token == "convert_plain") Learner::convert_plain(is);
+      else if (token == "convert_bin_from_pgn_extract") Learner::convert_bin_from_pgn_extract(is);
 
       // Command to call qsearch(),search() directly for testing
       else if (token == "qsearch") qsearch_cmd(pos);