SystemSh0cker/private-gpt
1
0
Fork 0
mirror of https://github.com/zylon-ai/private-gpt.git synced 2025-12-22 13:55:41 +01:00
Code Issues Projects Releases Packages Activity

Sanitize null bytes before ingestion

Browse source
This commit is contained in:
Dmitri Tsiu 2024-09-24 01:34:21 +03:00
parent 77461b96cf
commit 43a9dbe21b
1 changed files with 6 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

7
private_gpt/components/ingest/ingest_helper.py
View file

@ -92,7 +92,12 @@ class IngestionHelper:
return string_reader.load_data([file_data.read_text()]) return string_reader.load_data([file_data.read_text()])
logger.debug("Specific reader found for extension=%s", extension) logger.debug("Specific reader found for extension=%s", extension)
return reader_cls().load_data(file_data) documents = reader_cls().load_data(file_data)
for i in range(len(documents)):
documents[i].text = documents[i].text.replace("\u0000", "")
return documents
@staticmethod @staticmethod
def _exclude_metadata(documents: list[Document]) -> None: def _exclude_metadata(documents: list[Document]) -> None: