mirror of
https://github.com/yacy/yacy_search_server.git
synced 2026-10-25 05:54:15 +00:00
refactoring of llm classes
This commit is contained in:
Michael Peter Christen committed
1 parent
d61b3b85e7
commit
1ab5ee3476
5 files changed
+357
-388
No files matched your search
@@ -0,0 +1,339 @@
|
||||
/**
|
||||
* LLMEndpoint
|
||||
* Copyright 2024 by Michael Peter Christen
|
||||
* First released 17.05.2024 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.ai;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
public class LLM {
|
||||
|
||||
private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};
|
||||
|
||||
public static enum LLMType {
|
||||
OPENAI("https://api.openai.com"),
|
||||
OLLAMA("http://localhost:11434"),
|
||||
LMSTUDIO("http://localhost:1234"),
|
||||
OPENROUTER("https://openrouter.ai/api"),
|
||||
OTHER(null);
|
||||
public String hoststub;
|
||||
private LLMType(String hoststub) {
|
||||
this.hoststub = hoststub;
|
||||
}
|
||||
}
|
||||
|
||||
public final String hoststub;
|
||||
public final String key; // the apikey
|
||||
public final int max_tokens; // the max_tokens as configured by the endpoint for all models
|
||||
public final LLMType type;
|
||||
|
||||
public LLM(final String hoststub, String key, final int max_tokens, final LLMType type) {
|
||||
this.hoststub = hoststub.endsWith("/") ? hoststub.substring(0, hoststub.length() - 1) : hoststub;
|
||||
this.key = key == null ? "" : key;
|
||||
this.max_tokens = max_tokens <= 0 ? 4096 : max_tokens;
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public String getHoststub() {
|
||||
return this.hoststub;
|
||||
}
|
||||
|
||||
|
||||
// API Helper Methods
|
||||
|
||||
public static String sendPostRequest(final String urls, final JSONObject data) throws IOException, URISyntaxException {
|
||||
final URL url = new URI(urls).toURL();
|
||||
final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setRequestProperty("Content-Type", "application/json");
|
||||
conn.setDoOutput(true);
|
||||
|
||||
try (OutputStream os = conn.getOutputStream()) {
|
||||
final byte[] input = data.toString().getBytes("utf-8");
|
||||
os.write(input, 0, input.length);
|
||||
}
|
||||
|
||||
final int responseCode = conn.getResponseCode();
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
|
||||
final StringBuilder response = new StringBuilder();
|
||||
String responseLine;
|
||||
while ((responseLine = br.readLine()) != null) {
|
||||
response.append(responseLine.trim());
|
||||
}
|
||||
return response.toString();
|
||||
}
|
||||
} else {
|
||||
throw new IOException("Request failed with response code " + responseCode);
|
||||
}
|
||||
}
|
||||
|
||||
public static String sendGetRequest(final String urls) throws IOException, URISyntaxException {
|
||||
final URL url = new URI(urls).toURL();
|
||||
final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("GET");
|
||||
|
||||
final int responseCode = conn.getResponseCode();
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
|
||||
final StringBuilder response = new StringBuilder();
|
||||
String responseLine;
|
||||
while ((responseLine = br.readLine()) != null) {
|
||||
response.append(responseLine.trim());
|
||||
}
|
||||
return response.toString();
|
||||
}
|
||||
} else {
|
||||
throw new IOException("Request failed with response code " + responseCode);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public LinkedHashMap<String, Long> listOllamaModels() {
|
||||
final LinkedHashMap<String, Long> sortedMap = new LinkedHashMap<>();
|
||||
try {
|
||||
final String response = sendGetRequest(this.hoststub + "/api/tags");
|
||||
final JSONObject responseObject = new JSONObject(response);
|
||||
final JSONArray models = responseObject.getJSONArray("models");
|
||||
|
||||
final List<Map.Entry<String, Long>> list = new ArrayList<>();
|
||||
for (int i = 0; i < models.length(); i++) {
|
||||
final JSONObject model = models.getJSONObject(i);
|
||||
final String name = model.optString("name", "");
|
||||
final long size = model.optLong("size", 0);
|
||||
list.add(new AbstractMap.SimpleEntry<>(name, size));
|
||||
}
|
||||
|
||||
// Sort the list in descending order based on the values
|
||||
list.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue()));
|
||||
|
||||
// Create a new LinkedHashMap and add the sorted entries
|
||||
for (final Map.Entry<String, Long> entry : list) {
|
||||
sortedMap.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
} catch (JSONException | URISyntaxException | IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return sortedMap;
|
||||
}
|
||||
|
||||
public boolean ollamaModelExists(final String name) {
|
||||
final JSONObject data = new JSONObject();
|
||||
try {
|
||||
data.put("name", name);
|
||||
sendPostRequest(this.hoststub + "/api/show", data);
|
||||
return true;
|
||||
} catch (JSONException | URISyntaxException | IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean pullOllamaModel(final String name) {
|
||||
final JSONObject data = new JSONObject();
|
||||
try {
|
||||
data.put("name", name);
|
||||
data.put("stream", false);
|
||||
final String response = sendPostRequest(this.hoststub + "/api/pull", data);
|
||||
// this sends {"status": "success"} in case of success
|
||||
final JSONObject responseObject = new JSONObject(response);
|
||||
final String status = responseObject.optString("status", "");
|
||||
return status.equals("success");
|
||||
} catch (JSONException | URISyntaxException | IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// chat endpoints
|
||||
|
||||
public static class Context extends JSONArray {
|
||||
public Context(String systemPrompt) throws JSONException {
|
||||
super();
|
||||
final JSONObject systemPromptObject = new JSONObject(true);
|
||||
systemPromptObject.put("role", "system");
|
||||
systemPromptObject.put("content", systemPrompt);
|
||||
this.put(systemPromptObject);
|
||||
}
|
||||
public void addDialog(String user, String assistant) throws JSONException {
|
||||
final JSONObject userPromptObject = new JSONObject(true);
|
||||
userPromptObject.put("role", "user");
|
||||
userPromptObject.put("content", user);
|
||||
this.put(userPromptObject);
|
||||
final JSONObject assistantPromptObject = new JSONObject(true);
|
||||
assistantPromptObject.put("role", "assistant");
|
||||
assistantPromptObject.put("content", assistant);
|
||||
this.put(assistantPromptObject);
|
||||
}
|
||||
public void addPrompt(String userPrompt) throws JSONException {
|
||||
final JSONObject userPromptObject = new JSONObject(true);
|
||||
userPromptObject.put("role", "user");
|
||||
userPromptObject.put("content", userPrompt);
|
||||
this.put(userPromptObject);
|
||||
}
|
||||
}
|
||||
|
||||
// OpenAI chat client, works with llama.cpp and Ollama
|
||||
public String chat(final String model, final Context context, JSONObject schema, final int max_tokens) throws IOException {
|
||||
final JSONObject data = new JSONObject();
|
||||
|
||||
try {
|
||||
data.put("model", model);
|
||||
data.put("temperature", 0.1);
|
||||
data.put("max_tokens", max_tokens);
|
||||
data.put("messages", context);
|
||||
data.put("stop", new JSONArray(STOPTOKENS));
|
||||
data.put("stream", false);
|
||||
|
||||
if (schema != null) {
|
||||
System.out.println(schema.toString());
|
||||
JSONObject json_schema = new JSONObject(true);
|
||||
json_schema.put("strict", true);
|
||||
json_schema.put("schema", schema);
|
||||
JSONObject response_format = new JSONObject();
|
||||
response_format.put("type", "json_schema");
|
||||
response_format.put("json_schema", json_schema);
|
||||
data.put("response_format", response_format);
|
||||
}
|
||||
|
||||
final String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data);
|
||||
final JSONObject responseObject = new JSONObject(response);
|
||||
final JSONArray choices = responseObject.getJSONArray("choices");
|
||||
final JSONObject choice = choices.getJSONObject(0);
|
||||
final JSONObject message = choice.getJSONObject("message");
|
||||
final String content = message.optString("content", "");
|
||||
return content;
|
||||
} catch (JSONException | URISyntaxException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public String chat(final String model, final String systemPrompt, final String userPrompt, final int max_tokens) throws IOException {
|
||||
try {
|
||||
Context context = new Context(systemPrompt);
|
||||
context.addPrompt(userPrompt);
|
||||
return chat(model, context, null, max_tokens);
|
||||
} catch (JSONException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public static String[] stringsFromChat(String chatanswer) throws JSONException {
|
||||
JSONArray ja = new JSONArray(chatanswer);
|
||||
List<String> list = new ArrayList<>();
|
||||
// parse the JSON array and extract strings
|
||||
for (int i = 0; i < ja.length(); i++) {
|
||||
Object item = ja.get(i);
|
||||
if (item instanceof String) {
|
||||
list.add((String) item);
|
||||
} else if (item instanceof JSONObject) {
|
||||
JSONObject jo = (JSONObject) item;
|
||||
String answer = jo.optString("answer", null);
|
||||
if (answer != null) {
|
||||
list.add(answer);
|
||||
} else {
|
||||
// take any string value from the object
|
||||
for (String key : jo.keySet()) {
|
||||
Object value = jo.optString(key, null);
|
||||
if (value != null && value instanceof String) {
|
||||
list.add((String) value);
|
||||
break; // take the first string found
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// convert the list to an array
|
||||
String[] result = new String[list.size()];
|
||||
return list.toArray(result);
|
||||
}
|
||||
|
||||
public final static JSONObject listSchema = new JSONObject(Map.of(
|
||||
"title", "Answer List",
|
||||
"type", "array",
|
||||
"properties", Map.of(
|
||||
"answer", Map.of("type", "string")
|
||||
),
|
||||
"required", List.of("answer")
|
||||
));
|
||||
|
||||
public static void main(final String[] args) {
|
||||
final LLM llm = new LLM(LLMType.OLLAMA.hoststub, null, 4069, LLMType.OLLAMA);
|
||||
|
||||
final LinkedHashMap<String, Long> models = llm.listOllamaModels();
|
||||
System.out.println(models.toString());
|
||||
|
||||
// check if model exists
|
||||
final String model = "qwen2.5:0.5b";
|
||||
if (llm.ollamaModelExists(model))
|
||||
System.out.println("model " + model + " exists");
|
||||
else
|
||||
System.out.println("model " + model + " does not exist");
|
||||
|
||||
// pull a model
|
||||
final boolean success = llm.pullOllamaModel(model);
|
||||
System.out.println("pulled model: " + model + ": " + success);
|
||||
|
||||
String response;
|
||||
try {
|
||||
response = llm.chat(model, "You are a helpful assistant.", "What is the capital of France?", 1000);
|
||||
System.out.println("Chat response: " + response);
|
||||
} catch (IOException e) {
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
// make chat completion with model
|
||||
String question = "Who invented the wheel?";
|
||||
try {
|
||||
final String answer = llm.chat(model, "Make short answers.", question, 200);
|
||||
System.out.println(answer);
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
// try the json parser from chat results
|
||||
question = "Make a list of four names from Star Wars movies. Use a JSON Array.";
|
||||
try {
|
||||
Context context = new Context("Make short answers");
|
||||
context.addPrompt(question);
|
||||
final String[] a = stringsFromChat(llm.chat(model, context, listSchema, 1000));
|
||||
for (String s : a) {
|
||||
System.out.println(s);
|
||||
}
|
||||
} catch (final IOException | JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -28,26 +28,22 @@ import java.util.Map;
|
||||
import java.util.Random;
|
||||
|
||||
|
||||
public class OllamaSwarm {
|
||||
public class LLMSwarm {
|
||||
|
||||
Random random = new Random();
|
||||
List<OllamaClient> swarm;;
|
||||
List<LLM> swarm;;
|
||||
|
||||
public OllamaSwarm() {
|
||||
public LLMSwarm() {
|
||||
this.swarm = new ArrayList<>();
|
||||
}
|
||||
|
||||
public void addOllamaClient(final OllamaClient client) {
|
||||
public void addOllamaClient(final LLM client) {
|
||||
this.swarm.add(client);
|
||||
}
|
||||
|
||||
public void addOllamaClient(final String hoststub) {
|
||||
this.swarm.add(new OllamaClient(hoststub));
|
||||
}
|
||||
|
||||
public List<String> getSwarm() {
|
||||
List<String> hoststubs = new ArrayList<>();
|
||||
for (OllamaClient client : this.swarm) {
|
||||
for (LLM client : this.swarm) {
|
||||
hoststubs.add(client.getHoststub());
|
||||
}
|
||||
return hoststubs;
|
||||
@@ -59,7 +55,7 @@ public class OllamaSwarm {
|
||||
*/
|
||||
public LinkedHashMap<String, Integer> listOllamaModels() {
|
||||
final LinkedHashMap<String, Integer> modelCountMap = new LinkedHashMap<>();
|
||||
for (OllamaClient client: this.swarm) {
|
||||
for (LLM client: this.swarm) {
|
||||
LinkedHashMap<String, Long> models = client.listOllamaModels();
|
||||
for (Map.Entry<String, Long> entry : models.entrySet()) {
|
||||
modelCountMap.merge(entry.getKey(), 1, Integer::sum);
|
||||
@@ -74,7 +70,7 @@ public class OllamaSwarm {
|
||||
* @return true if at least one client has the model, false otherwise
|
||||
*/
|
||||
public boolean ollamaModelExists(final String name) {
|
||||
for (OllamaClient client : this.swarm) {
|
||||
for (LLM client : this.swarm) {
|
||||
if (client.ollamaModelExists(name)) {
|
||||
return true;
|
||||
}
|
||||
@@ -91,7 +87,7 @@ public class OllamaSwarm {
|
||||
public boolean pullOllamaModel(final String name, boolean all) {
|
||||
boolean pulled = false;
|
||||
if (all) {
|
||||
for (OllamaClient client: this.swarm) {
|
||||
for (LLM client: this.swarm) {
|
||||
if (client.pullOllamaModel(name)) {
|
||||
pulled = true;
|
||||
}
|
||||
@@ -100,7 +96,7 @@ public class OllamaSwarm {
|
||||
// we try several times in case one client is not available
|
||||
for (int i = 0; i < this.swarm.size(); i++) {
|
||||
int index = random.nextInt(this.swarm.size());
|
||||
OllamaClient client = this.swarm.get(index);
|
||||
LLM client = this.swarm.get(index);
|
||||
pulled = client.pullOllamaModel(name);
|
||||
if (pulled) return true;
|
||||
}
|
||||
@@ -113,9 +109,9 @@ public class OllamaSwarm {
|
||||
* @param name
|
||||
* @return a List of OllamaClient objects that have the specified model
|
||||
*/
|
||||
public List<OllamaClient> getSwarm(String name) {
|
||||
List<OllamaClient> clientsWithModel = new ArrayList<>();
|
||||
for (OllamaClient client : this.swarm) {
|
||||
public List<LLM> getSwarm(String name) {
|
||||
List<LLM> clientsWithModel = new ArrayList<>();
|
||||
for (LLM client : this.swarm) {
|
||||
if (client.ollamaModelExists(name)) {
|
||||
clientsWithModel.add(client);
|
||||
}
|
||||
@@ -132,11 +128,11 @@ public class OllamaSwarm {
|
||||
* @throws IOException
|
||||
*/
|
||||
public String getAnswer(String model, String systemPrompt, String userPrompt) throws IOException {
|
||||
List<OllamaClient> clientsWithModel = getSwarm(model);
|
||||
List<LLM> clientsWithModel = getSwarm(model);
|
||||
if (clientsWithModel.isEmpty()) {
|
||||
return "No client with model " + model + " found.";
|
||||
}
|
||||
OllamaClient client = clientsWithModel.get(random.nextInt(clientsWithModel.size()));
|
||||
LLM client = clientsWithModel.get(random.nextInt(clientsWithModel.size()));
|
||||
return client.chat(model, systemPrompt, userPrompt, 4096);
|
||||
}
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
/**
|
||||
* OllamaClient
|
||||
* Copyright 2024 by Michael Peter Christen
|
||||
* First released 17.05.2024 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.ai;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
public class OllamaClient extends OpenAIClient {
|
||||
|
||||
public static String OLLAMA_API_HOST = "http://localhost:11434";
|
||||
|
||||
public OllamaClient(final String hoststub) {
|
||||
super(hoststub);
|
||||
}
|
||||
|
||||
public String getHoststub() {
|
||||
return this.hoststub;
|
||||
}
|
||||
|
||||
public LinkedHashMap<String, Long> listOllamaModels() {
|
||||
final LinkedHashMap<String, Long> sortedMap = new LinkedHashMap<>();
|
||||
try {
|
||||
final String response = OpenAIClient.sendGetRequest(this.hoststub + "/api/tags");
|
||||
final JSONObject responseObject = new JSONObject(response);
|
||||
final JSONArray models = responseObject.getJSONArray("models");
|
||||
|
||||
final List<Map.Entry<String, Long>> list = new ArrayList<>();
|
||||
for (int i = 0; i < models.length(); i++) {
|
||||
final JSONObject model = models.getJSONObject(i);
|
||||
final String name = model.optString("name", "");
|
||||
final long size = model.optLong("size", 0);
|
||||
list.add(new AbstractMap.SimpleEntry<>(name, size));
|
||||
}
|
||||
|
||||
// Sort the list in descending order based on the values
|
||||
list.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue()));
|
||||
|
||||
// Create a new LinkedHashMap and add the sorted entries
|
||||
for (final Map.Entry<String, Long> entry : list) {
|
||||
sortedMap.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
} catch (JSONException | URISyntaxException | IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return sortedMap;
|
||||
}
|
||||
|
||||
public boolean ollamaModelExists(final String name) {
|
||||
final JSONObject data = new JSONObject();
|
||||
try {
|
||||
data.put("name", name);
|
||||
OpenAIClient.sendPostRequest(this.hoststub + "/api/show", data);
|
||||
return true;
|
||||
} catch (JSONException | URISyntaxException | IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean pullOllamaModel(final String name) {
|
||||
final JSONObject data = new JSONObject();
|
||||
try {
|
||||
data.put("name", name);
|
||||
data.put("stream", false);
|
||||
final String response = OpenAIClient.sendPostRequest(this.hoststub + "/api/pull", data);
|
||||
// this sends {"status": "success"} in case of success
|
||||
final JSONObject responseObject = new JSONObject(response);
|
||||
final String status = responseObject.optString("status", "");
|
||||
return status.equals("success");
|
||||
} catch (JSONException | URISyntaxException | IOException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(final String[] args) {
|
||||
final OllamaClient oc = new OllamaClient(OLLAMA_API_HOST);
|
||||
|
||||
final LinkedHashMap<String, Long> models = oc.listOllamaModels();
|
||||
System.out.println(models.toString());
|
||||
|
||||
// check if model exists
|
||||
final String model = "qwen2.5:0.5b";
|
||||
if (oc.ollamaModelExists(model))
|
||||
System.out.println("model " + model + " exists");
|
||||
else
|
||||
System.out.println("model " + model + " does not exist");
|
||||
|
||||
// pull a model
|
||||
final boolean success = oc.pullOllamaModel(model);
|
||||
System.out.println("pulled model: " + model + ": " + success);
|
||||
|
||||
String response;
|
||||
try {
|
||||
response = oc.chat(model, "You are a helpful assistant.", "What is the capital of France?", 1000);
|
||||
System.out.println("Chat response: " + response);
|
||||
} catch (IOException e) {
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
/**
|
||||
* OpenAIClient
|
||||
* Copyright 2024 by Michael Peter Christen
|
||||
* First released 17.05.2024 at https://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package net.yacy.ai;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
|
||||
public class OpenAIClient {
|
||||
|
||||
private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};
|
||||
|
||||
protected final String hoststub;
|
||||
|
||||
public OpenAIClient(final String hoststub) {
|
||||
this.hoststub = hoststub;
|
||||
}
|
||||
|
||||
|
||||
// API Helper Methods
|
||||
|
||||
public static String sendPostRequest(final String endpoint, final JSONObject data) throws IOException, URISyntaxException {
|
||||
final URL url = new URI(endpoint).toURL();
|
||||
final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setRequestProperty("Content-Type", "application/json");
|
||||
conn.setDoOutput(true);
|
||||
|
||||
try (OutputStream os = conn.getOutputStream()) {
|
||||
final byte[] input = data.toString().getBytes("utf-8");
|
||||
os.write(input, 0, input.length);
|
||||
}
|
||||
|
||||
final int responseCode = conn.getResponseCode();
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
|
||||
final StringBuilder response = new StringBuilder();
|
||||
String responseLine;
|
||||
while ((responseLine = br.readLine()) != null) {
|
||||
response.append(responseLine.trim());
|
||||
}
|
||||
return response.toString();
|
||||
}
|
||||
} else {
|
||||
throw new IOException("Request failed with response code " + responseCode);
|
||||
}
|
||||
}
|
||||
|
||||
public static String sendGetRequest(final String endpoint) throws IOException, URISyntaxException {
|
||||
final URL url = new URI(endpoint).toURL();
|
||||
final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("GET");
|
||||
|
||||
final int responseCode = conn.getResponseCode();
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
|
||||
final StringBuilder response = new StringBuilder();
|
||||
String responseLine;
|
||||
while ((responseLine = br.readLine()) != null) {
|
||||
response.append(responseLine.trim());
|
||||
}
|
||||
return response.toString();
|
||||
}
|
||||
} else {
|
||||
throw new IOException("Request failed with response code " + responseCode);
|
||||
}
|
||||
}
|
||||
|
||||
public static class Context extends JSONArray {
|
||||
public Context(String systemPrompt) throws JSONException {
|
||||
super();
|
||||
final JSONObject systemPromptObject = new JSONObject(true);
|
||||
systemPromptObject.put("role", "system");
|
||||
systemPromptObject.put("content", systemPrompt);
|
||||
this.put(systemPromptObject);
|
||||
}
|
||||
public void addDialog(String user, String assistant) throws JSONException {
|
||||
final JSONObject userPromptObject = new JSONObject(true);
|
||||
userPromptObject.put("role", "user");
|
||||
userPromptObject.put("content", user);
|
||||
this.put(userPromptObject);
|
||||
final JSONObject assistantPromptObject = new JSONObject(true);
|
||||
assistantPromptObject.put("role", "assistant");
|
||||
assistantPromptObject.put("content", assistant);
|
||||
this.put(assistantPromptObject);
|
||||
}
|
||||
public void addPrompt(String userPrompt) throws JSONException {
|
||||
final JSONObject userPromptObject = new JSONObject(true);
|
||||
userPromptObject.put("role", "user");
|
||||
userPromptObject.put("content", userPrompt);
|
||||
this.put(userPromptObject);
|
||||
}
|
||||
}
|
||||
|
||||
// OpenAI chat client, works with llama.cpp and Ollama
|
||||
|
||||
public String chat(final String model, final Context context, JSONObject schema, final int max_tokens) throws IOException {
|
||||
final JSONObject data = new JSONObject();
|
||||
|
||||
try {
|
||||
data.put("model", model);
|
||||
data.put("temperature", 0.1);
|
||||
data.put("max_tokens", max_tokens);
|
||||
data.put("messages", context);
|
||||
data.put("stop", new JSONArray(STOPTOKENS));
|
||||
data.put("stream", false);
|
||||
|
||||
if (schema != null) {
|
||||
System.out.println(schema.toString());
|
||||
JSONObject json_schema = new JSONObject(true);
|
||||
json_schema.put("strict", true);
|
||||
json_schema.put("schema", schema);
|
||||
JSONObject response_format = new JSONObject();
|
||||
response_format.put("type", "json_schema");
|
||||
response_format.put("json_schema", json_schema);
|
||||
data.put("response_format", response_format);
|
||||
}
|
||||
|
||||
final String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data);
|
||||
final JSONObject responseObject = new JSONObject(response);
|
||||
final JSONArray choices = responseObject.getJSONArray("choices");
|
||||
final JSONObject choice = choices.getJSONObject(0);
|
||||
final JSONObject message = choice.getJSONObject("message");
|
||||
final String content = message.optString("content", "");
|
||||
return content;
|
||||
} catch (JSONException | URISyntaxException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public String chat(final String model, final String systemPrompt, final String userPrompt, final int max_tokens) throws IOException {
|
||||
try {
|
||||
Context context = new Context(systemPrompt);
|
||||
context.addPrompt(userPrompt);
|
||||
return chat(model, context, null, max_tokens);
|
||||
} catch (JSONException e) {
|
||||
throw new IOException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public static String[] stringsFromChat(String chatanswer) throws JSONException {
|
||||
JSONArray ja = new JSONArray(chatanswer);
|
||||
List<String> list = new ArrayList<>();
|
||||
// parse the JSON array and extract strings
|
||||
for (int i = 0; i < ja.length(); i++) {
|
||||
Object item = ja.get(i);
|
||||
if (item instanceof String) {
|
||||
list.add((String) item);
|
||||
} else if (item instanceof JSONObject) {
|
||||
JSONObject jo = (JSONObject) item;
|
||||
String answer = jo.optString("answer", null);
|
||||
if (answer != null) {
|
||||
list.add(answer);
|
||||
} else {
|
||||
// take any string value from the object
|
||||
for (String key : jo.keySet()) {
|
||||
Object value = jo.optString(key, null);
|
||||
if (value != null && value instanceof String) {
|
||||
list.add((String) value);
|
||||
break; // take the first string found
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// convert the list to an array
|
||||
String[] result = new String[list.size()];
|
||||
return list.toArray(result);
|
||||
}
|
||||
|
||||
public final static JSONObject listSchema = new JSONObject(Map.of(
|
||||
"title", "Answer List",
|
||||
"type", "array",
|
||||
"properties", Map.of(
|
||||
"answer", Map.of("type", "string")
|
||||
),
|
||||
"required", List.of("answer")
|
||||
));
|
||||
|
||||
public static void main(final String[] args) {
|
||||
final String model = "qwen2.5:0.5b";
|
||||
final OpenAIClient oaic = new OpenAIClient(OllamaClient.OLLAMA_API_HOST);
|
||||
// make chat completion with model
|
||||
String question = "Who invented the wheel?";
|
||||
try {
|
||||
final String answer = oaic.chat(model, "Make short answers.", question, 200);
|
||||
System.out.println(answer);
|
||||
} catch (final IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
// try the json parser from chat results
|
||||
question = "Make a list of four names from Star Wars movies. Use a JSON Array.";
|
||||
try {
|
||||
Context context = new Context("Make short answers");
|
||||
context.addPrompt(question);
|
||||
final String[] a = stringsFromChat(oaic.chat(model, context, listSchema, 1000));
|
||||
for (String s : a) {
|
||||
System.out.println(s);
|
||||
}
|
||||
} catch (final IOException | JSONException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -48,7 +48,7 @@ import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import net.yacy.ai.OpenAIClient;
|
||||
import net.yacy.ai.LLM;
|
||||
import net.yacy.cora.federate.solr.SolrType;
|
||||
import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
|
||||
import net.yacy.search.Switchboard;
|
||||
@@ -257,10 +257,10 @@ public class RAGProxyServlet extends HttpServlet {
|
||||
StringBuilder query = new StringBuilder();
|
||||
String question = "Make a list of a maximum of four search words for the following question; use a JSON Array: " + prompt;
|
||||
try {
|
||||
OpenAIClient oaic = new OpenAIClient(LLM_API_HOST);
|
||||
OpenAIClient.Context context = new OpenAIClient.Context(LLM_SYSTEM_PREFIX);
|
||||
LLM llm = new LLM(LLM_API_HOST, null, 4096, LLM.LLMType.OLLAMA);
|
||||
LLM.Context context = new LLM.Context(LLM_SYSTEM_PREFIX);
|
||||
context.addPrompt(question);
|
||||
String[] a = OpenAIClient.stringsFromChat(oaic.chat(model, context, OpenAIClient.listSchema, 80));
|
||||
String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 80));
|
||||
for (String s : a)
|
||||
query.append(s).append(' ');
|
||||
return query.toString().trim();
|
||||
|
||||
Reference in new issue
Block a user