|
|
|
|
@@ -1,15 +1,15 @@
|
|
|
|
|
From 64b3fbb150d12b3ca63ac2fb4e57bc46f41d2ccd Mon Sep 17 00:00:00 2001
|
|
|
|
|
From 087cf3300e973d7790db8f7cad01d2a790de38be Mon Sep 17 00:00:00 2001
|
|
|
|
|
From: Daniel Hiltgen <daniel@ollama.com>
|
|
|
|
|
Date: Mon, 13 Nov 2023 12:25:58 -0800
|
|
|
|
|
Subject: [PATCH] Expose callable API for server
|
|
|
|
|
|
|
|
|
|
This adds an extern "C" interface within the example server
|
|
|
|
|
---
|
|
|
|
|
examples/server/CMakeLists.txt | 24 ++++
|
|
|
|
|
examples/server/server.cpp | 247 +++++++++++++++++++++++++++++++++
|
|
|
|
|
examples/server/server.h | 83 +++++++++++
|
|
|
|
|
examples/server/CMakeLists.txt | 24 +++
|
|
|
|
|
examples/server/server.cpp | 274 +++++++++++++++++++++++++++++++++
|
|
|
|
|
examples/server/server.h | 89 +++++++++++
|
|
|
|
|
ggml-cuda.cu | 1 +
|
|
|
|
|
4 files changed, 355 insertions(+)
|
|
|
|
|
4 files changed, 388 insertions(+)
|
|
|
|
|
create mode 100644 examples/server/server.h
|
|
|
|
|
|
|
|
|
|
diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt
|
|
|
|
|
@@ -46,7 +46,7 @@ index 859cd12..4ea47a7 100644
|
|
|
|
|
+endif()
|
|
|
|
|
\ No newline at end of file
|
|
|
|
|
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
|
|
|
|
|
index 895f751..f939590 100644
|
|
|
|
|
index d0cd8e1..5f5d4c5 100644
|
|
|
|
|
--- a/examples/server/server.cpp
|
|
|
|
|
+++ b/examples/server/server.cpp
|
|
|
|
|
@@ -5,6 +5,9 @@
|
|
|
|
|
@@ -59,7 +59,7 @@ index 895f751..f939590 100644
|
|
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
|
// crash the server in debug mode, otherwise send an http 500 error
|
|
|
|
|
@@ -2631,6 +2634,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con
|
|
|
|
|
@@ -2632,6 +2635,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -67,84 +67,84 @@ index 895f751..f939590 100644
|
|
|
|
|
int main(int argc, char **argv)
|
|
|
|
|
{
|
|
|
|
|
// own arguments required by this example
|
|
|
|
|
@@ -3065,3 +3069,246 @@ int main(int argc, char **argv)
|
|
|
|
|
@@ -3066,3 +3070,273 @@ int main(int argc, char **argv)
|
|
|
|
|
llama_backend_free();
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
+
|
|
|
|
|
+#else // LLAMA_SERVER_LIBRARY
|
|
|
|
|
+// Expose the llama server as a callable extern "C" API
|
|
|
|
|
+llama_server_context llama;
|
|
|
|
|
+llama_server_context *llama = NULL;
|
|
|
|
|
+std::atomic<bool> ext_server_running(false);
|
|
|
|
|
+std::thread ext_server_thread;
|
|
|
|
|
+inline ext_server_err makeErr(uint32_t code, std::string msg) {
|
|
|
|
|
+ if (code == 0) {
|
|
|
|
|
+ return ext_server_err{0, NULL};
|
|
|
|
|
+ }
|
|
|
|
|
+ const std::string::size_type size = msg.size();
|
|
|
|
|
+ ext_server_err ret = {
|
|
|
|
|
+ code,
|
|
|
|
|
+ new char[size + 1],
|
|
|
|
|
+ };
|
|
|
|
|
+ memcpy(ret.err, msg.c_str(), size + 1);
|
|
|
|
|
+ return ret;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+ext_server_err llama_server_init(ext_server_params *sparams)
|
|
|
|
|
+void llama_server_init(ext_server_params *sparams, ext_server_resp_t *err)
|
|
|
|
|
+{
|
|
|
|
|
+ log_set_target(stdout);
|
|
|
|
|
+ gpt_params params;
|
|
|
|
|
+ params.n_ctx = sparams->n_ctx;
|
|
|
|
|
+ params.n_batch = sparams->n_batch;
|
|
|
|
|
+ params.n_threads = sparams->n_threads;
|
|
|
|
|
+ params.n_parallel = sparams->n_parallel;
|
|
|
|
|
+ params.rope_freq_base = sparams->rope_freq_base;
|
|
|
|
|
+ params.rope_freq_scale = sparams->rope_freq_scale;
|
|
|
|
|
+
|
|
|
|
|
+ if (sparams->memory_f16) {
|
|
|
|
|
+ params.cache_type_k = "f16";
|
|
|
|
|
+ params.cache_type_v = "f16";
|
|
|
|
|
+ } else {
|
|
|
|
|
+ params.cache_type_k = "f32";
|
|
|
|
|
+ params.cache_type_v = "f32";
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ params.n_gpu_layers = sparams->n_gpu_layers;
|
|
|
|
|
+ params.main_gpu = sparams->main_gpu;
|
|
|
|
|
+ params.use_mlock = sparams->use_mlock;
|
|
|
|
|
+ params.use_mmap = sparams->use_mmap;
|
|
|
|
|
+ params.numa = sparams->numa;
|
|
|
|
|
+ params.embedding = sparams->embedding;
|
|
|
|
|
+ if (sparams->model != NULL) {
|
|
|
|
|
+ params.model = sparams->model;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL; la = la->next) {
|
|
|
|
|
+ params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale));
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ assert(err != NULL && sparams != NULL);
|
|
|
|
|
+ err->id = 0;
|
|
|
|
|
+ err->msg[0] = '\0';
|
|
|
|
|
+ try {
|
|
|
|
|
+ llama = new llama_server_context;
|
|
|
|
|
+ log_set_target(stdout);
|
|
|
|
|
+ gpt_params params;
|
|
|
|
|
+ params.n_ctx = sparams->n_ctx;
|
|
|
|
|
+ params.n_batch = sparams->n_batch;
|
|
|
|
|
+ params.n_threads = sparams->n_threads;
|
|
|
|
|
+ params.n_parallel = sparams->n_parallel;
|
|
|
|
|
+ params.rope_freq_base = sparams->rope_freq_base;
|
|
|
|
|
+ params.rope_freq_scale = sparams->rope_freq_scale;
|
|
|
|
|
+
|
|
|
|
|
+ if (sparams->memory_f16) {
|
|
|
|
|
+ params.cache_type_k = "f16";
|
|
|
|
|
+ params.cache_type_v = "f16";
|
|
|
|
|
+ } else {
|
|
|
|
|
+ params.cache_type_k = "f32";
|
|
|
|
|
+ params.cache_type_v = "f32";
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ params.n_gpu_layers = sparams->n_gpu_layers;
|
|
|
|
|
+ params.main_gpu = sparams->main_gpu;
|
|
|
|
|
+ params.use_mlock = sparams->use_mlock;
|
|
|
|
|
+ params.use_mmap = sparams->use_mmap;
|
|
|
|
|
+ params.numa = sparams->numa;
|
|
|
|
|
+ params.embedding = sparams->embedding;
|
|
|
|
|
+ if (sparams->model != NULL) {
|
|
|
|
|
+ params.model = sparams->model;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL; la = la->next) {
|
|
|
|
|
+ params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale));
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if (sparams->mmproj != NULL) {
|
|
|
|
|
+ params.mmproj = std::string(sparams->mmproj);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ llama_backend_init(params.numa);
|
|
|
|
|
+
|
|
|
|
|
+ // load the model
|
|
|
|
|
+ if (!llama.load_model(params))
|
|
|
|
|
+ if (!llama->load_model(params))
|
|
|
|
|
+ {
|
|
|
|
|
+ // TODO - consider modifying the logging logic or patching load_model so we can capture more detailed error messages
|
|
|
|
|
+ // and pass them back to the caller for better UX
|
|
|
|
|
+ return makeErr(1, "error loading model " + params.model);
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "error loading model %s", params.model.c_str());
|
|
|
|
|
+ return;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ llama.initialize();
|
|
|
|
|
+ llama->initialize();
|
|
|
|
|
+ } catch (std::exception &e) {
|
|
|
|
|
+ return makeErr(1, e.what());
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
|
|
|
|
|
+ } catch (...) {
|
|
|
|
|
+ return makeErr(1, "Unknown Exception initializing llama server");
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "Unknown exception initializing llama server");
|
|
|
|
|
+ }
|
|
|
|
|
+ return makeErr(0, "");
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+void llama_server_start()
|
|
|
|
|
+{
|
|
|
|
|
+ assert(llama != NULL);
|
|
|
|
|
+ // TODO mutex to protect thread creation
|
|
|
|
|
+ ext_server_thread = std::thread([&]()
|
|
|
|
|
+ {
|
|
|
|
|
@@ -154,7 +154,7 @@ index 895f751..f939590 100644
|
|
|
|
|
+ ggml_time_init();
|
|
|
|
|
+ while (ext_server_running.load())
|
|
|
|
|
+ {
|
|
|
|
|
+ if (!llama.update_slots()) {
|
|
|
|
|
+ if (!llama->update_slots()) {
|
|
|
|
|
+ LOG_TEE("unexpected error in llama server update_slots - exiting main loop\n");
|
|
|
|
|
+ break;
|
|
|
|
|
+ }
|
|
|
|
|
@@ -170,124 +170,150 @@ index 895f751..f939590 100644
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+void llama_server_stop() {
|
|
|
|
|
+ assert(llama != NULL);
|
|
|
|
|
+ // TODO - too verbose, remove once things are solid
|
|
|
|
|
+ LOG_TEE("requesting llama server shutdown\n");
|
|
|
|
|
+ ext_server_running = false;
|
|
|
|
|
+ ext_server_thread.join();
|
|
|
|
|
+ delete llama;
|
|
|
|
|
+ llama = NULL;
|
|
|
|
|
+ LOG_TEE("llama server shutdown complete\n");
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+ext_server_completion_resp llama_server_completion(const char *json_req) {
|
|
|
|
|
+ std::string msg;
|
|
|
|
|
+ ext_server_completion_resp resp = {
|
|
|
|
|
+ 0,
|
|
|
|
|
+ NULL,
|
|
|
|
|
+ };
|
|
|
|
|
+void llama_server_completion(const char *json_req, ext_server_resp_t *resp) {
|
|
|
|
|
+ assert(llama != NULL && json_req != NULL && resp != NULL);
|
|
|
|
|
+ resp->id = -1;
|
|
|
|
|
+ resp->msg[0] = '\0';
|
|
|
|
|
+ try {
|
|
|
|
|
+ json data = json::parse(json_req);
|
|
|
|
|
+ resp.task_id = llama.request_completion(data, false, false, -1);
|
|
|
|
|
+ return resp;
|
|
|
|
|
+ resp->id = llama->request_completion(data, false, false, -1);
|
|
|
|
|
+ } catch (std::exception &e) {
|
|
|
|
|
+ msg = e.what();
|
|
|
|
|
+ snprintf(resp->msg, resp->msg_len, "exception %s", e.what());
|
|
|
|
|
+ } catch (...) {
|
|
|
|
|
+ msg = "Unknown Exception during completion";
|
|
|
|
|
+ snprintf(resp->msg, resp->msg_len, "Unknown exception during completion");
|
|
|
|
|
+ }
|
|
|
|
|
+ const std::string::size_type size = msg.size();
|
|
|
|
|
+ resp.task_id = 0;
|
|
|
|
|
+ resp.err = new char[size + 1];
|
|
|
|
|
+ memcpy(resp.err, msg.c_str(), size + 1);
|
|
|
|
|
+ return resp;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+ext_task_result llama_server_completion_next_result(const int task_id) {
|
|
|
|
|
+void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *resp) {
|
|
|
|
|
+ assert(llama != NULL && resp != NULL);
|
|
|
|
|
+ std::string msg;
|
|
|
|
|
+ ext_task_result resp = {-1,false,false,NULL};
|
|
|
|
|
+ try {
|
|
|
|
|
+ task_result result = llama.next_result(task_id);
|
|
|
|
|
+ std::string result_json = result.result_json.dump(-1, ' ', false, json::error_handler_t::replace);
|
|
|
|
|
+ const std::string::size_type size = result_json.size();
|
|
|
|
|
+ resp.id = result.id;
|
|
|
|
|
+ resp.stop = result.stop;
|
|
|
|
|
+ resp.error = result.error;
|
|
|
|
|
+ resp.result_json = new char[size + 1];
|
|
|
|
|
+ memcpy(resp.result_json, result_json.c_str(), size + 1);
|
|
|
|
|
+ if (result.error) {
|
|
|
|
|
+ llama.request_cancel(task_id);
|
|
|
|
|
+ } else if (result.stop) {
|
|
|
|
|
+ llama.request_cancel(task_id);
|
|
|
|
|
+ }
|
|
|
|
|
+ return resp;
|
|
|
|
|
+ } catch (std::exception &e) {
|
|
|
|
|
+ msg = e.what(); // TODO - json?
|
|
|
|
|
+ } catch (...) {
|
|
|
|
|
+ msg = "Unknown Exception during completion";
|
|
|
|
|
+ }
|
|
|
|
|
+ resp.error = true;
|
|
|
|
|
+ const std::string::size_type size = msg.size();
|
|
|
|
|
+ resp.result_json = new char[size + 1];
|
|
|
|
|
+ memcpy(resp.result_json, msg.c_str(), size + 1);
|
|
|
|
|
+ return resp;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+ext_server_err llama_server_completion_cancel(const int task_id) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ llama.request_cancel(task_id);
|
|
|
|
|
+ } catch (std::exception &e) {
|
|
|
|
|
+ return makeErr(1, e.what());
|
|
|
|
|
+ } catch (...) {
|
|
|
|
|
+ return makeErr(1, "Unknown Exception running llama server");
|
|
|
|
|
+ }
|
|
|
|
|
+ return makeErr(0, "");
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ext_server_err llama_server_tokenize(const char *json_req, ext_server_resp *resp) {
|
|
|
|
|
+ resp->id = -1;
|
|
|
|
|
+ resp->stop = false;
|
|
|
|
|
+ resp->error = false;
|
|
|
|
|
+ resp->json_resp = NULL;
|
|
|
|
|
+ std::string result_json;
|
|
|
|
|
+ try {
|
|
|
|
|
+ task_result result = llama->next_result(task_id);
|
|
|
|
|
+ result_json = result.result_json.dump(-1, ' ', false, json::error_handler_t::replace);
|
|
|
|
|
+ resp->id = result.id;
|
|
|
|
|
+ resp->stop = result.stop;
|
|
|
|
|
+ resp->error = result.error;
|
|
|
|
|
+ if (result.error) {
|
|
|
|
|
+ llama->request_cancel(task_id);
|
|
|
|
|
+ } else if (result.stop) {
|
|
|
|
|
+ llama->request_cancel(task_id);
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (std::exception &e) {
|
|
|
|
|
+ resp->error = true;
|
|
|
|
|
+ resp->id = -1;
|
|
|
|
|
+ result_json = "{\"error\":\"exception " + std::string(e.what()) + "\"}";
|
|
|
|
|
+ } catch (...) {
|
|
|
|
|
+ resp->error = true;
|
|
|
|
|
+ resp->id = -1;
|
|
|
|
|
+ result_json = "{\"error\":\"Unknown exception during completion\"}";
|
|
|
|
|
+ }
|
|
|
|
|
+ const std::string::size_type size = result_json.size() + 1;
|
|
|
|
|
+ resp->json_resp = new char[size];
|
|
|
|
|
+ snprintf(resp->json_resp, size, "%s", result_json.c_str());
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+void llama_server_release_task_result(ext_server_task_result_t *result) {
|
|
|
|
|
+ if (result == NULL || result->json_resp == NULL) {
|
|
|
|
|
+ return;
|
|
|
|
|
+ }
|
|
|
|
|
+ delete[] result->json_resp;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err) {
|
|
|
|
|
+ assert(llama != NULL && err != NULL);
|
|
|
|
|
+ err->id = 0;
|
|
|
|
|
+ err->msg[0] = '\0';
|
|
|
|
|
+ try {
|
|
|
|
|
+ llama->request_cancel(task_id);
|
|
|
|
|
+ } catch (std::exception &e) {
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
|
|
|
|
|
+ } catch (...) {
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "Unknown exception completion cancel in llama server");
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
|
|
|
|
|
+ assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
|
|
|
|
|
+ *json_resp = NULL;
|
|
|
|
|
+ err->id = 0;
|
|
|
|
|
+ err->msg[0] = '\0';
|
|
|
|
|
+ try {
|
|
|
|
|
+ const json body = json::parse(json_req);
|
|
|
|
|
+ std::vector<llama_token> tokens;
|
|
|
|
|
+ if (body.count("content") != 0)
|
|
|
|
|
+ {
|
|
|
|
|
+ tokens = llama.tokenize(body["content"], false);
|
|
|
|
|
+ tokens = llama->tokenize(body["content"], false);
|
|
|
|
|
+ }
|
|
|
|
|
+ const json data = format_tokenizer_response(tokens);
|
|
|
|
|
+ std::string result_json = data.dump();
|
|
|
|
|
+ const std::string::size_type size = result_json.size();
|
|
|
|
|
+ resp->json_resp = new char[size + 1];
|
|
|
|
|
+ memcpy(resp->json_resp, result_json.c_str(), size + 1);
|
|
|
|
|
+ const std::string::size_type size = result_json.size() + 1;
|
|
|
|
|
+ *json_resp = new char[size];
|
|
|
|
|
+ snprintf(*json_resp, size, "%s", result_json.c_str());
|
|
|
|
|
+ } catch (std::exception &e) {
|
|
|
|
|
+ return makeErr(1, e.what());
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
|
|
|
|
|
+ } catch (...) {
|
|
|
|
|
+ return makeErr(1, "Unknown Exception during tokenize");
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "Unknown exception during tokenize");
|
|
|
|
|
+ }
|
|
|
|
|
+ return makeErr(0, "");
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+ext_server_err llama_server_detokenize(const char *json_req, ext_server_resp *resp) {
|
|
|
|
|
+ resp->json_resp = NULL;
|
|
|
|
|
+void llama_server_release_json_resp(char **json_resp) {
|
|
|
|
|
+ if (json_resp == NULL || *json_resp == NULL) {
|
|
|
|
|
+ return;
|
|
|
|
|
+ }
|
|
|
|
|
+ delete[] *json_resp;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
|
|
|
|
|
+ assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
|
|
|
|
|
+ *json_resp = NULL;
|
|
|
|
|
+ err->id = 0;
|
|
|
|
|
+ err->msg[0] = '\0';
|
|
|
|
|
+ try {
|
|
|
|
|
+ const json body = json::parse(json_req);
|
|
|
|
|
+ std::string content;
|
|
|
|
|
+ if (body.count("tokens") != 0)
|
|
|
|
|
+ {
|
|
|
|
|
+ const std::vector<llama_token> tokens = body["tokens"];
|
|
|
|
|
+ content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend());
|
|
|
|
|
+ content = tokens_to_str(llama->ctx, tokens.cbegin(), tokens.cend());
|
|
|
|
|
+ }
|
|
|
|
|
+ const json data = format_detokenized_response(content);
|
|
|
|
|
+ std::string result_json = data.dump();
|
|
|
|
|
+ const std::string::size_type size = result_json.size();
|
|
|
|
|
+ resp->json_resp = new char[size + 1];
|
|
|
|
|
+ memcpy(resp->json_resp, result_json.c_str(), size + 1);
|
|
|
|
|
+ const std::string::size_type size = result_json.size() + 1;
|
|
|
|
|
+ *json_resp = new char[size];
|
|
|
|
|
+ snprintf(*json_resp, size, "%s", result_json.c_str());
|
|
|
|
|
+ } catch (std::exception &e) {
|
|
|
|
|
+ return makeErr(1, e.what());
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
|
|
|
|
|
+ } catch (...) {
|
|
|
|
|
+ return makeErr(1, "Unknown Exception during detokenize");
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "Unknown exception during detokenize");
|
|
|
|
|
+ }
|
|
|
|
|
+ return makeErr(0, "");
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+ext_server_err llama_server_embedding(const char *json_req, ext_server_resp *resp) {
|
|
|
|
|
+ resp->json_resp = NULL;
|
|
|
|
|
+void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err) {
|
|
|
|
|
+ assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
|
|
|
|
|
+ *json_resp = NULL;
|
|
|
|
|
+ err->id = 0;
|
|
|
|
|
+ err->msg[0] = '\0';
|
|
|
|
|
+ try {
|
|
|
|
|
+ const json body = json::parse(json_req);
|
|
|
|
|
+ json prompt;
|
|
|
|
|
@@ -299,28 +325,29 @@ index 895f751..f939590 100644
|
|
|
|
|
+ {
|
|
|
|
|
+ prompt = "";
|
|
|
|
|
+ }
|
|
|
|
|
+ const int task_id = llama.request_completion({ {"prompt", prompt}, { "n_predict", 0} }, false, true, -1);
|
|
|
|
|
+ task_result result = llama.next_result(task_id);
|
|
|
|
|
+ const int task_id = llama->request_completion({ {"prompt", prompt}, { "n_predict", 0} }, false, true, -1);
|
|
|
|
|
+ task_result result = llama->next_result(task_id);
|
|
|
|
|
+ std::string result_json = result.result_json.dump();
|
|
|
|
|
+ const std::string::size_type size = result_json.size();
|
|
|
|
|
+ resp->json_resp = new char[size + 1];
|
|
|
|
|
+ memcpy(resp->json_resp, result_json.c_str(), size + 1);
|
|
|
|
|
+ const std::string::size_type size = result_json.size() + 1;
|
|
|
|
|
+ *json_resp = new char[size];
|
|
|
|
|
+ snprintf(*json_resp, size, "%s", result_json.c_str());
|
|
|
|
|
+ } catch (std::exception &e) {
|
|
|
|
|
+ return makeErr(1, e.what());
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "exception %s", e.what());
|
|
|
|
|
+ } catch (...) {
|
|
|
|
|
+ return makeErr(1, "Unknown Exception during detokenize");
|
|
|
|
|
+ err->id = -1;
|
|
|
|
|
+ snprintf(err->msg, err->msg_len, "Unknown exception during embedding");
|
|
|
|
|
+ }
|
|
|
|
|
+ return makeErr(0, "");
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+#endif // LLAMA_SERVER_LIBRARY
|
|
|
|
|
\ No newline at end of file
|
|
|
|
|
diff --git a/examples/server/server.h b/examples/server/server.h
|
|
|
|
|
new file mode 100644
|
|
|
|
|
index 0000000..4d03b1e
|
|
|
|
|
index 0000000..d22f1b6
|
|
|
|
|
--- /dev/null
|
|
|
|
|
+++ b/examples/server/server.h
|
|
|
|
|
@@ -0,0 +1,83 @@
|
|
|
|
|
@@ -0,0 +1,89 @@
|
|
|
|
|
+#if defined(LLAMA_SERVER_LIBRARY)
|
|
|
|
|
+#ifndef LLAMA_SERVER_H
|
|
|
|
|
+#define LLAMA_SERVER_H
|
|
|
|
|
@@ -336,20 +363,23 @@ index 0000000..4d03b1e
|
|
|
|
|
+extern "C"
|
|
|
|
|
+{
|
|
|
|
|
+#endif
|
|
|
|
|
+ // TODO - clean the type def's up a bit for better consistency
|
|
|
|
|
+ typedef struct ext_server_err {
|
|
|
|
|
+ uint32_t code; // 0 on success, > 0 on error
|
|
|
|
|
+ char *err; // null if code == 0; else contains error message. Caller responsible for freeing memory
|
|
|
|
|
+ } ext_server_err;
|
|
|
|
|
+ typedef struct ext_server_resp {
|
|
|
|
|
+ int id; // < 0 on error
|
|
|
|
|
+ size_t msg_len; // caller must allocate msg and set msg_len
|
|
|
|
|
+ char *msg;
|
|
|
|
|
+ } ext_server_resp_t;
|
|
|
|
|
+
|
|
|
|
|
+ // Allocated and freed by caller
|
|
|
|
|
+ typedef struct ext_server_lora_adapter {
|
|
|
|
|
+ char *adapter;
|
|
|
|
|
+ float scale;
|
|
|
|
|
+ struct ext_server_lora_adapter *next;
|
|
|
|
|
+ } ext_server_lora_adapter;
|
|
|
|
|
+ } ext_server_lora_adapter_t;
|
|
|
|
|
+
|
|
|
|
|
+ // Allocated and freed by caller
|
|
|
|
|
+ typedef struct ext_server_params
|
|
|
|
|
+ {
|
|
|
|
|
+ char *model;
|
|
|
|
|
+ char *model;
|
|
|
|
|
+ uint32_t n_ctx; // text context, 0 = from model
|
|
|
|
|
+ uint32_t n_batch; // prompt processing maximum batch size
|
|
|
|
|
+ uint32_t n_threads; // number of threads to use for generation
|
|
|
|
|
@@ -363,40 +393,43 @@ index 0000000..4d03b1e
|
|
|
|
|
+ bool use_mmap; // use mmap if possible
|
|
|
|
|
+ bool numa; // attempt optimizations that help on some NUMA systems
|
|
|
|
|
+ bool embedding; // get only sentence embedding
|
|
|
|
|
+ ext_server_lora_adapter* lora_adapters;
|
|
|
|
|
+ } ext_server_params;
|
|
|
|
|
+ ext_server_lora_adapter_t* lora_adapters;
|
|
|
|
|
+ char *mmproj;
|
|
|
|
|
+ } ext_server_params_t;
|
|
|
|
|
+
|
|
|
|
|
+ // Initialize the server once per process
|
|
|
|
|
+ ext_server_err llama_server_init(ext_server_params *sparams);
|
|
|
|
|
+
|
|
|
|
|
+ // Run the main loop
|
|
|
|
|
+ void llama_server_start();
|
|
|
|
|
+ // Stop the main loop
|
|
|
|
|
+ void llama_server_stop();
|
|
|
|
|
+
|
|
|
|
|
+ typedef struct ext_task_result
|
|
|
|
|
+ typedef struct ext_server_task_result
|
|
|
|
|
+ {
|
|
|
|
|
+ int id;
|
|
|
|
|
+ bool stop;
|
|
|
|
|
+ bool error;
|
|
|
|
|
+ char* result_json; // caller responsible to free this memory
|
|
|
|
|
+ } ext_task_result;
|
|
|
|
|
+
|
|
|
|
|
+ typedef struct ext_server_completion_resp {
|
|
|
|
|
+ int task_id; // < 0 on error, >= 0 on success
|
|
|
|
|
+ char *err; // null if task_id >= 0; else contains error message. Caller responsible for freeing memory
|
|
|
|
|
+ } ext_server_completion_resp;
|
|
|
|
|
+ ext_server_completion_resp llama_server_completion(const char *json_req);
|
|
|
|
|
+ ext_task_result llama_server_completion_next_result(const int task_id);
|
|
|
|
|
+ ext_server_err llama_server_completion_cancel(const int task_id);
|
|
|
|
|
+ char* json_resp; // null terminated, memory managed by ext_server
|
|
|
|
|
+ } ext_server_task_result_t;
|
|
|
|
|
+
|
|
|
|
|
+ // Caller responsible for freeing json_resp
|
|
|
|
|
+ typedef struct ext_server_resp {
|
|
|
|
|
+ char *json_resp; // Caller responsible for freeing string
|
|
|
|
|
+ } ext_server_resp;
|
|
|
|
|
+ ext_server_err llama_server_tokenize(const char *json_req, ext_server_resp *resp);
|
|
|
|
|
+ ext_server_err llama_server_detokenize(const char *json_req, ext_server_resp *resp);
|
|
|
|
|
+ ext_server_err llama_server_embedding(const char *json_req, ext_server_resp *resp);
|
|
|
|
|
+ // Initialize the server once per process
|
|
|
|
|
+ // err->id = 0 for success and err->msg[0] = NULL
|
|
|
|
|
+ // err->id != 0 for failure, and err->msg contains error message
|
|
|
|
|
+ void llama_server_init(ext_server_params_t *sparams, ext_server_resp_t *err);
|
|
|
|
|
+
|
|
|
|
|
+ // Run the main loop, called once per init
|
|
|
|
|
+ void llama_server_start();
|
|
|
|
|
+ // Stop the main loop and free up resources allocated in init and start. Init must be called again to reuse
|
|
|
|
|
+ void llama_server_stop();
|
|
|
|
|
+
|
|
|
|
|
+ // json_req null terminated string, memory managed by caller
|
|
|
|
|
+ // resp->id >= 0 on success (task ID)
|
|
|
|
|
+ // resp->id < 0 on error, and resp->msg contains error message
|
|
|
|
|
+ void llama_server_completion(const char *json_req, ext_server_resp_t *resp);
|
|
|
|
|
+
|
|
|
|
|
+ // Caller must call llama_server_release_task_result to free resp->json_resp
|
|
|
|
|
+ void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *result);
|
|
|
|
|
+ void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err);
|
|
|
|
|
+ void llama_server_release_task_result(ext_server_task_result_t *result);
|
|
|
|
|
+
|
|
|
|
|
+ // Caller must call llama_server_releaes_json_resp to free json_resp if err.id < 0
|
|
|
|
|
+ void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
|
|
|
|
|
+ void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
|
|
|
|
|
+ void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err);
|
|
|
|
|
+ void llama_server_release_json_resp(char **json_resp);
|
|
|
|
|
+
|
|
|
|
|
+#ifdef __cplusplus
|
|
|
|
|
+}
|
|
|
|
|
@@ -406,10 +439,10 @@ index 0000000..4d03b1e
|
|
|
|
|
+#endif // LLAMA_SERVER_LIBRARY
|
|
|
|
|
\ No newline at end of file
|
|
|
|
|
diff --git a/ggml-cuda.cu b/ggml-cuda.cu
|
|
|
|
|
index 85f7a29..ce51364 100644
|
|
|
|
|
index 9e1acd3..ea64b55 100644
|
|
|
|
|
--- a/ggml-cuda.cu
|
|
|
|
|
+++ b/ggml-cuda.cu
|
|
|
|
|
@@ -6410,6 +6410,7 @@ static cudaError_t ggml_cuda_cpy_tensor_2d(
|
|
|
|
|
@@ -6505,6 +6505,7 @@ static cudaError_t ggml_cuda_cpy_tensor_2d(
|
|
|
|
|
CUDA_CHECK(cudaGetDevice(&id));
|
|
|
|
|
src_ptr = (char *) extra->data_device[id];
|
|
|
|
|
} else {
|
|
|
|
|
|
Daniel Hiltgen