SystemSh0cker/private-gpt
1
0
Fork 0
mirror of https://github.com/zylon-ai/private-gpt.git synced 2025-12-22 07:40:12 +01:00
Code Issues Projects Releases Packages Activity

Use a different text splitter to improve results. Ingest takes an argument pointing to the doc to ingest.

Browse source
This commit is contained in:
Iván Martínez 2023-05-05 17:32:31 +02:00
parent a05186b598
commit 92244a90b4
2 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

7
ingest.py
View file

@ -1,13 +1,14 @@
from langchain.document_loaders import TextLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import Chroma
from langchain.embeddings import LlamaCppEmbeddings
from sys import argv
def main():
# Load document and split in chunks
loader = TextLoader('./source_documents/state_of_the_union.txt', encoding='utf8')
loader = TextLoader(argv[1], encoding="utf8")
documents = loader.load()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
text_splitter = CharacterTextSplitter(chunk_size=500, chunk_overlap=50)
texts = text_splitter.split_documents(documents)
# Create embeddings
llama = LlamaCppEmbeddings(model_path="./models/ggml-model-q4_0.bin")