Add convert UCI function that allows conversion of files between any of plain, bin, and binpack. Usage: convert infile outfile [append].

src/extra/nnue_data_binpack_format.h

@@ -6915,7 +6915,7 @@ namespace binpack
     {
         constexpr std::size_t reportEveryNPositions = 100'000;
 
-        std::cout << "Compressing " << inputPath << " to " << outputPath << '\n';
+        std::cout << "Converting " << inputPath << " to " << outputPath << '\n';
 
         CompressedTrainingDataEntryWriter writer(outputPath, om);
         TrainingDataEntry e;
@@ -6961,13 +6961,15 @@ namespace binpack
             if (key == "ply"sv) e.ply = std::stoi(value);
             if (key == "result"sv) e.result = std::stoi(value);
         }
+
+        std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n";
     }
 
     inline void convertBinpackToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om)
     {
         constexpr std::size_t bufferSize = MiB;
 
-        std::cout << "Decompressing " << inputPath << " to " << outputPath << '\n';
+        std::cout << "Converting " << inputPath << " to " << outputPath << '\n';
 
         CompressedTrainingDataEntryReader reader(inputPath);
         std::ofstream outputFile(outputPath, om);
@@ -6999,6 +7001,8 @@ namespace binpack
             const auto cur = outputFile.tellp();
             std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n";
         }
+
+        std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n";
     }
 
 
@@ -7006,14 +7010,9 @@ namespace binpack
     {
         constexpr std::size_t reportEveryNPositions = 100'000;
 
-        std::cout << "Compressing " << inputPath << " to " << outputPath << '\n';
+        std::cout << "Converting " << inputPath << " to " << outputPath << '\n';
 
         CompressedTrainingDataEntryWriter writer(outputPath, om);
-        TrainingDataEntry e;
-
-        std::string key;
-        std::string value;
-        std::string move;
 
         std::ifstream inputFile(inputPath, std::ios_base::binary);
         const auto base = inputFile.tellg();
@@ -7037,13 +7036,15 @@ namespace binpack
                 std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n";
             }
         }
+
+        std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n";
     }
 
     inline void convertBinpackToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om)
     {
         constexpr std::size_t bufferSize = MiB;
 
-        std::cout << "Decompressing " << inputPath << " to " << outputPath << '\n';
+        std::cout << "Converting " << inputPath << " to " << outputPath << '\n';
 
         CompressedTrainingDataEntryReader reader(inputPath);
         std::ofstream outputFile(outputPath, std::ios_base::binary | om);
@@ -7075,6 +7076,8 @@ namespace binpack
             const auto cur = outputFile.tellp();
             std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n";
         }
+
+        std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n";
     }
 
     inline void convertBinToPlain(std::string inputPath, std::string outputPath, std::ios_base::openmode om)
@@ -7083,12 +7086,6 @@ namespace binpack
 
         std::cout << "Converting " << inputPath << " to " << outputPath << '\n';
 
-        TrainingDataEntry e;
-
-        std::string key;
-        std::string value;
-        std::string move;
-
         std::ifstream inputFile(inputPath, std::ios_base::binary);
         const auto base = inputFile.tellg();
         std::size_t numProcessedPositions = 0;
@@ -7127,13 +7124,15 @@ namespace binpack
             const auto cur = outputFile.tellp();
             std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n";
         }
+
+        std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n";
     }
 
     inline void convertPlainToBin(std::string inputPath, std::string outputPath, std::ios_base::openmode om)
     {
         constexpr std::size_t bufferSize = MiB;
 
-        std::cout << "Compressing " << inputPath << " to " << outputPath << '\n';
+        std::cout << "Converting " << inputPath << " to " << outputPath << '\n';
 
         std::ofstream outputFile(outputPath, std::ios_base::binary | om);
         std::vector<char> buffer;
@@ -7194,5 +7193,7 @@ namespace binpack
             const auto cur = outputFile.tellp();
             std::cout << "Processed " << (cur - base) << " bytes and " << numProcessedPositions << " positions.\n";
         }
+
+        std::cout << "Finished. Converted " << numProcessedPositions << " positions.\n";
     }
 }

src/learn/convert.cpp

@@ -12,6 +12,8 @@
 #include "../position.h"
 #include "../tt.h"
 
+#include "../extra/nnue_data_binpack_format.h"
+
 #include <sstream>
 #include <fstream>
 #include <unordered_set>
@@ -497,5 +499,108 @@ namespace Learner
         ofs.close();
         std::cout << "all done" << std::endl;
     }
+
+    static inline const std::string plain_extension = ".plain";
+    static inline const std::string bin_extension = ".bin";
+    static inline const std::string binpack_extension = ".binpack";
+
+    static bool file_exists(const std::string& name)
+    {
+        std::ifstream f(name);
+        return f.good();
+    }
+
+    static bool ends_with(const std::string& lhs, const std::string& end)
+    {
+        if (end.size() > lhs.size()) return false;
+
+        return std::equal(end.rbegin(), end.rend(), lhs.rbegin());
+    }
+
+    static bool is_convert_of_type(
+        const std::string& input_path,
+        const std::string& output_path,
+        const std::string& expected_input_extension,
+        const std::string& expected_output_extension)
+    {
+        return ends_with(input_path, expected_input_extension)
+            && ends_with(output_path, expected_output_extension);
+    }
+
+    using ConvertFunctionType = void(std::string inputPath, std::string outputPath, std::ios_base::openmode om);
+
+    static ConvertFunctionType* get_convert_function(const std::string& input_path, const std::string& output_path)
+    {
+        if (is_convert_of_type(input_path, output_path, plain_extension, bin_extension))
+            return binpack::convertPlainToBin;
+        if (is_convert_of_type(input_path, output_path, plain_extension, binpack_extension))
+            return binpack::convertPlainToBinpack;
+
+        if (is_convert_of_type(input_path, output_path, bin_extension, plain_extension))
+            return binpack::convertBinToPlain;
+        if (is_convert_of_type(input_path, output_path, bin_extension, binpack_extension))
+            return binpack::convertBinToBinpack;
+
+        if (is_convert_of_type(input_path, output_path, binpack_extension, plain_extension))
+            return binpack::convertBinpackToPlain;
+        if (is_convert_of_type(input_path, output_path, binpack_extension, bin_extension))
+            return binpack::convertBinpackToBin;
+
+        return nullptr;
+    }
+
+    static void convert(const std::string& input_path, const std::string& output_path, std::ios_base::openmode om)
+    {
+        if(!file_exists(input_path))
+        {
+            std::cerr << "Input file does not exist.\n";
+            return;
+        }
+
+        auto func = get_convert_function(input_path, output_path);
+        if (func != nullptr)
+        {
+            func(input_path, output_path, om);
+        }
+        else
+        {
+            std::cerr << "Conversion between files of these types is not supported.\n";
+        }
+    }
+
+    static void convert(const std::vector<std::string>& args)
+    {
+        if (args.size() < 2 || args.size() > 3)
+        {
+            std::cerr << "Invalid arguments.\n";
+            std::cerr << "Usage: convert from_path to_path [append]\n";
+            return;
+        }
+
+        const bool append = (args.size() == 3) && (args[2] == "append");
+        const std::ios_base::openmode openmode =
+            append
+            ? std::ios_base::app
+            : std::ios_base::trunc;
+
+        convert(args[0], args[1], openmode);
+    }
+
+    void convert(istringstream& is)
+    {
+        std::vector<std::string> args;
+
+        while (true)
+        {
+            std::string token = "";
+            is >> token;
+            if (token == "")
+                break;
+
+            args.push_back(token);
+        }
+
+        convert(args);
+    }
 }
 #endif

src/uci.cpp

@@ -50,6 +50,8 @@ namespace Learner
   // Learning from the generated game record
   void learn(Position& pos, istringstream& is);
 
+  void convert(istringstream& is);
+
   // A pair of reader and evaluation value. Returned by Learner::search(),Learner::qsearch().
   typedef std::pair<Value, std::vector<Move> > ValueAndPV;
 
@@ -352,6 +354,7 @@ void UCI::loop(int argc, char* argv[]) {
 #if defined (EVAL_LEARN)
       else if (token == "gensfen") Learner::gen_sfen(pos, is);
       else if (token == "learn") Learner::learn(pos, is);
+      else if (token == "convert") Learner::convert(is);
 
       // Command to call qsearch(),search() directly for testing
       else if (token == "qsearch") qsearch_cmd(pos);