End generation inside the classifier, build grammars and tool parsers once, reuse sampling buffers, and bump to 0.16.0

Cargo.toml

@@ -18,7 +18,7 @@ members = [
 
 [workspace.package]
 edition = "2024"
-version = "0.15.1"
+version = "0.16.0"
 license = "Apache-2.0"
 repository = "https://github.com/intentee/llama-cpp-bindings"
 
@@ -33,18 +33,18 @@ find_cuda_helper = "=0.2.0"
 hf-hub = "=0.5.0"
 inventory = "=0.3.24"
 libtest-mimic = "=0.8.2"
-llama-cpp-bindings = { path = "llama-cpp-bindings", version = "=0.15.1" }
-llama-cpp-bindings-build = { path = "llama-cpp-bindings-build", version = "=0.15.1" }
-llama-cpp-bindings-sys = { path = "llama-cpp-bindings-sys", version = "=0.15.1" }
-llama-cpp-bindings-types = { path = "llama-cpp-bindings-types", version = "=0.15.1" }
-llama-cpp-error-recorder = { path = "llama-cpp-error-recorder", version = "=0.15.1" }
-llama-cpp-ffi-status = { path = "llama-cpp-ffi-status", version = "=0.15.1" }
-llama-cpp-gbnf = { path = "llama-cpp-gbnf", version = "=0.15.1" }
-llama-cpp-log-decoder = { path = "llama-cpp-log-decoder", version = "=0.15.1" }
-llama-cpp-test-harness = { path = "llama-cpp-test-harness", version = "=0.15.1" }
-llama-cpp-test-harness-macros = { path = "llama-cpp-test-harness-macros", version = "=0.15.1" }
+llama-cpp-bindings = { path = "llama-cpp-bindings", version = "=0.16.0" }
+llama-cpp-bindings-build = { path = "llama-cpp-bindings-build", version = "=0.16.0" }
+llama-cpp-bindings-sys = { path = "llama-cpp-bindings-sys", version = "=0.16.0" }
+llama-cpp-bindings-types = { path = "llama-cpp-bindings-types", version = "=0.16.0" }
+llama-cpp-error-recorder = { path = "llama-cpp-error-recorder", version = "=0.16.0" }
+llama-cpp-ffi-status = { path = "llama-cpp-ffi-status", version = "=0.16.0" }
+llama-cpp-gbnf = { path = "llama-cpp-gbnf", version = "=0.16.0" }
+llama-cpp-log-decoder = { path = "llama-cpp-log-decoder", version = "=0.16.0" }
+llama-cpp-test-harness = { path = "llama-cpp-test-harness", version = "=0.16.0" }
+llama-cpp-test-harness-macros = { path = "llama-cpp-test-harness-macros", version = "=0.16.0" }
 llama-cpp-wrapper-error-fixture = { path = "llama-cpp-wrapper-error-fixture" }
-llama-cpp-wrapper-sources = { path = "llama-cpp-wrapper-sources", version = "=0.15.1" }
+llama-cpp-wrapper-sources = { path = "llama-cpp-wrapper-sources", version = "=0.16.0" }
 llguidance = "=1.7.0"
 log = "=0.4.29"
 nom = "=8.0.0"

llama-cpp-bindings-build/src/cmake_config.rs

@@ -246,6 +246,10 @@ fn configure_gpu_backends(config: &mut Config, target_os: TargetOs) -> Result<()
         if cfg!(feature = "cuda-no-vmm") {
             config.define("GGML_CUDA_NO_VMM", "ON");
         }
+
+        if let Some(cuda_architectures) = optional_env("CUDAARCHS")? {
+            config.define("CMAKE_CUDA_ARCHITECTURES", cuda_architectures);
+        }
     }
 
     if cfg!(feature = "rocm") {

llama-cpp-bindings-build/src/rebuild_tracking.rs

@@ -5,6 +5,7 @@ use llama_cpp_wrapper_sources::wrapper_sources::WRAPPER_SOURCES;
 
 pub fn register_rebuild_triggers(llama_src: &Path) {
     println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-env-changed=CUDAARCHS");
 
     for path in WRAPPER_HEADERS.iter().chain(WRAPPER_SOURCES) {
         println!("cargo:rerun-if-changed={path}");

llama-cpp-bindings-sys/wrapper.h

@@ -3,6 +3,7 @@
 #include "wrapper_chat_apply.h"
 #include "wrapper_chat_parse.h"
 #include "wrapper_common.h"
+#include "wrapper_context.h"
 #include "wrapper_fit.h"
 #include "wrapper_gbnf.h"
 #include "wrapper_mtmd.h"

llama-cpp-bindings-sys/wrapper_chat_parse.cpp

@@ -25,6 +25,10 @@ struct llama_rs_chat_parser {
     autoparser::autoparser parser;
 };
 
+struct llama_rs_chat_tools_parser {
+    common_peg_arena arena;
+};
+
 namespace {
 void dup_or_set_alloc_flag(const std::string & source, char ** out_dup, bool * out_alloc_failed) {
     *out_dup = llama_rs_dup_string(source);
@@ -49,18 +53,6 @@ auto report_current_parse_exception(
     return thrown_status;
 }
 
-auto build_tools_parser(const llama_rs_chat_parser & parser, const char * tools_json) -> common_peg_arena {
-    autoparser::generation_params inputs;
-
-    if ((tools_json != nullptr) && *tools_json != '\0') {
-        inputs.tools = common_json::parse(tools_json);
-    } else {
-        inputs.tools = common_json::array();
-    }
-
-    return parser.parser.build_parser(inputs, std::string());
-}
-
 auto parse_with_tools_parser(
     const common_peg_arena & chat_parser,
     const char * input,
@@ -176,9 +168,94 @@ extern "C" auto llama_rs_chat_parser_free(
     }
 }
 
-extern "C" auto llama_rs_parse_chat_message(
+extern "C" auto llama_rs_chat_tools_parser_create(
     llama_rs_chat_parser_handle parser,
     const char * tools_json,
+    llama_rs_chat_tools_parser_handle * out_tools_parser,
+    char ** out_error) -> llama_rs_chat_tools_parser_create_status {
+    if (out_tools_parser != nullptr) {
+        *out_tools_parser = nullptr;
+    }
+    if (out_error != nullptr) {
+        *out_error = nullptr;
+    }
+    if (parser == nullptr) {
+        return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG;
+    }
+    if (tools_json == nullptr) {
+        return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG;
+    }
+    if (out_tools_parser == nullptr) {
+        return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG;
+    }
+    if (out_error == nullptr) {
+        return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG;
+    }
+
+    try {
+        autoparser::generation_params inputs;
+
+        inputs.tools = common_json::parse(tools_json);
+
+        if (!inputs.tools.is_array()) {
+            return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY;
+        }
+
+        auto tools_parser_handle = std::make_unique<llama_rs_chat_tools_parser>();
+        tools_parser_handle->arena = parser->parser.build_parser(inputs, std::string());
+
+        *out_tools_parser = tools_parser_handle.release();
+
+        return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK;
+    } catch (const std::bad_alloc &) {
+        return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY;
+    } catch (const std::exception & ex) {
+        *out_error = llama_rs_dup_string(std::string(ex.what()));
+        if (*out_error == nullptr) {
+            return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED;
+        }
+        return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION;
+    } catch (...) {
+        *out_error = llama_rs_dup_string(std::string("unknown c++ exception"));
+        if (*out_error == nullptr) {
+            return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED;
+        }
+        return LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION;
+    }
+}
+
+extern "C" auto llama_rs_chat_tools_parser_free(
+    llama_rs_chat_tools_parser_handle tools_parser,
+    char ** out_error) -> llama_rs_chat_tools_parser_free_status {
+    if (out_error != nullptr) {
+        *out_error = nullptr;
+    }
+    try {
+        const std::unique_ptr<llama_rs_chat_tools_parser> reclaimed(tools_parser);
+        return LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK;
+    } catch (const std::bad_alloc &) {
+        return LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY;
+    } catch (const std::exception & err) {
+        if (out_error != nullptr) {
+            *out_error = llama_rs_dup_string(err.what());
+            if (*out_error == nullptr) {
+                return LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED;
+            }
+        }
+        return LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION;
+    } catch (...) {
+        if (out_error != nullptr) {
+            *out_error = llama_rs_dup_string("unknown c++ exception");
+            if (*out_error == nullptr) {
+                return LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED;
+            }
+        }
+        return LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION;
+    }
+}
+
+extern "C" auto llama_rs_parse_chat_message(
+    llama_rs_chat_tools_parser_handle tools_parser,
     const char * input,
     int is_partial,
     llama_rs_parsed_chat_handle * out_handle,
@@ -189,8 +266,8 @@ extern "C" auto llama_rs_parse_chat_message(
     if (out_error != nullptr) {
         *out_error = nullptr;
     }
-    if (parser == nullptr) {
-        return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_PARSER_ARG;
+    if (tools_parser == nullptr) {
+        return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG;
     }
     if (input == nullptr) {
         return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG;
@@ -202,14 +279,7 @@ extern "C" auto llama_rs_parse_chat_message(
         return LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG;
     }
 
-    try {
-        common_peg_arena const chat_parser = build_tools_parser(*parser, tools_json);
-
-        return parse_with_tools_parser(chat_parser, input, is_partial, out_handle, out_error);
-    } catch (...) {
-        return report_current_parse_exception(
-            out_error, LLAMA_RS_PARSE_CHAT_MESSAGE_TOOLS_PARSER_BUILD_THREW_CXX_EXCEPTION);
-    }
+    return parse_with_tools_parser(tools_parser->arena, input, is_partial, out_handle, out_error);
 }
 
 extern "C" auto llama_rs_parsed_chat_free(

llama-cpp-bindings-sys/wrapper_chat_parse.h

@@ -15,6 +15,9 @@ typedef struct llama_rs_parsed_chat * llama_rs_parsed_chat_handle;
 struct llama_rs_chat_parser;
 typedef struct llama_rs_chat_parser * llama_rs_chat_parser_handle;
 
+struct llama_rs_chat_tools_parser;
+typedef struct llama_rs_chat_tools_parser * llama_rs_chat_tools_parser_handle;
+
 typedef enum llama_rs_chat_parser_create_status {
     LLAMA_RS_CHAT_PARSER_CREATE_OK = 0,
     LLAMA_RS_CHAT_PARSER_CREATE_NULL_MODEL_ARG,
@@ -43,21 +46,48 @@ llama_rs_chat_parser_free_status llama_rs_chat_parser_free(
     llama_rs_chat_parser_handle parser,
     char ** out_error);
 
+typedef enum llama_rs_chat_tools_parser_create_status {
+    LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_OK = 0,
+    LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_PARSER_ARG,
+    LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_TOOLS_JSON_ARG,
+    LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_TOOLS_PARSER_ARG,
+    LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_NULL_OUT_ERROR_ARG,
+    LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_TOOLS_NOT_AN_ARRAY,
+    LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_ERROR_STRING_ALLOCATION_FAILED,
+    LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_OUT_OF_MEMORY,
+    LLAMA_RS_CHAT_TOOLS_PARSER_CREATE_LLAMA_CPP_THREW_CXX_EXCEPTION,
+} llama_rs_chat_tools_parser_create_status;
+
+llama_rs_chat_tools_parser_create_status llama_rs_chat_tools_parser_create(
+    llama_rs_chat_parser_handle parser,
+    const char * tools_json,
+    llama_rs_chat_tools_parser_handle * out_tools_parser,
+    char ** out_error);
+
+typedef enum llama_rs_chat_tools_parser_free_status {
+    LLAMA_RS_CHAT_TOOLS_PARSER_FREE_OK = 0,
+    LLAMA_RS_CHAT_TOOLS_PARSER_FREE_ERROR_STRING_ALLOCATION_FAILED,
+    LLAMA_RS_CHAT_TOOLS_PARSER_FREE_LLAMA_CPP_OUT_OF_MEMORY,
+    LLAMA_RS_CHAT_TOOLS_PARSER_FREE_DESTRUCTOR_THREW_CXX_EXCEPTION,
+} llama_rs_chat_tools_parser_free_status;
+
+llama_rs_chat_tools_parser_free_status llama_rs_chat_tools_parser_free(
+    llama_rs_chat_tools_parser_handle tools_parser,
+    char ** out_error);
+
 typedef enum llama_rs_parse_chat_message_status {
     LLAMA_RS_PARSE_CHAT_MESSAGE_OK = 0,
-    LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_PARSER_ARG,
+    LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_TOOLS_PARSER_ARG,
     LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_INPUT_ARG,
     LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_HANDLE_ARG,
     LLAMA_RS_PARSE_CHAT_MESSAGE_NULL_OUT_ERROR_ARG,
     LLAMA_RS_PARSE_CHAT_MESSAGE_ERROR_STRING_ALLOCATION_FAILED,
     LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_OUT_OF_MEMORY,
     LLAMA_RS_PARSE_CHAT_MESSAGE_LLAMA_CPP_THREW_CXX_EXCEPTION,
-    LLAMA_RS_PARSE_CHAT_MESSAGE_TOOLS_PARSER_BUILD_THREW_CXX_EXCEPTION,
 } llama_rs_parse_chat_message_status;
 
 llama_rs_parse_chat_message_status llama_rs_parse_chat_message(
-    llama_rs_chat_parser_handle parser,
-    const char * tools_json,
+    llama_rs_chat_tools_parser_handle tools_parser,
     const char * input,
     int is_partial,
     llama_rs_parsed_chat_handle * out_handle,