Merge branch 'feature/openailike-lmstudio' into main

private_gpt/components/llm/llm_component.py

@@ -123,6 +123,9 @@ class LLMComponent:
                     max_new_tokens=settings.llm.max_new_tokens,
                     messages_to_prompt=prompt_style.messages_to_prompt,
                     completion_to_prompt=prompt_style.completion_to_prompt,
+                    tokenizer=settings.llm.tokenizer,
+                    timeout=openai_settings.request_timeout,
+                    reuse_client=False,
                 )
             case "ollama":
                 try:

private_gpt/settings/settings.py

@@ -205,6 +205,10 @@ class OpenAISettings(BaseModel):
         "gpt-3.5-turbo",
         description="OpenAI Model to use. Example: 'gpt-4'.",
     )
+    request_timeout: float = Field(
+        120.0,
+        description="Time elapsed until openailike server times out the request. Default is 120s. Format is float. ",
+    )
 
 
 class OllamaSettings(BaseModel):

settings-vllm.yaml

@@ -3,6 +3,9 @@ server:
 
 llm:
   mode: openailike
+  max_new_tokens: 512
+  tokenizer: mistralai/Mistral-7B-Instruct-v0.2
+  temperature: 0.1
 
 embedding:
   mode: huggingface
@@ -15,3 +18,4 @@ openai:
   api_base: http://localhost:8000/v1
   api_key: EMPTY
   model: facebook/opt-125m
+  request_timeout: 600.0