Added documents upload table

This commit is contained in:
Saurab-Shrestha 2024-02-09 20:37:37 +05:45
parent 2ce2b794c0
commit 85eddaf471
24 changed files with 339 additions and 129 deletions

View file

@ -14,6 +14,7 @@ from private_gpt.users.models.role import Role
from private_gpt.users.models.user_role import UserRole
from private_gpt.users.models.subscription import Subscription
from private_gpt.users.models.company import Company
from private_gpt.users.models.documents import Documents
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
@ -25,7 +26,7 @@ if config.config_file_name is not None:
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# from myapp import mymodel1w
# target_metadata = mymodel.Base.metadata
target_metadata = Base.metadata
@ -33,7 +34,9 @@ target_metadata = Base.metadata
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
config.set_section_option(config.config_ini_section, "sqlalchemy.url", SQLALCHEMY_DATABASE_URI)
config.set_section_option(config.config_ini_section,
"sqlalchemy.url", SQLALCHEMY_DATABASE_URI)
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode.

View file

@ -0,0 +1,32 @@
"""Updated documents primary key
Revision ID: 0ef554491192
Revises: 9ce6871961b5
Create Date: 2024-02-08 17:06:32.957163
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '0ef554491192'
down_revision: Union[str, None] = '9ce6871961b5'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(op.f('ix_document_id'), 'document', ['id'], unique=False)
# op.create_unique_constraint('unique_user_role', 'user_roles', ['user_id', 'role_id', 'company_id'])
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# op.drop_constraint('unique_user_role', 'user_roles', type_='unique')
op.drop_index(op.f('ix_document_id'), table_name='document')
# ### end Alembic commands ###

View file

@ -0,0 +1,36 @@
"""Remove company from the users table
Revision ID: 488f28da0939
Revises: 0ef554491192
Create Date: 2024-02-09 19:08:12.002449
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '488f28da0939'
down_revision: Union[str, None] = '0ef554491192'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_unique_constraint(None, 'document', ['filename'])
# op.create_unique_constraint('unique_user_role', 'user_roles', ['user_id', 'role_id', 'company_id'])
op.drop_constraint('users_company_id_fkey', 'users', type_='foreignkey')
op.drop_column('users', 'company_id')
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('users', sa.Column('company_id', sa.INTEGER(), autoincrement=False, nullable=True))
op.create_foreign_key('users_company_id_fkey', 'users', 'companies', ['company_id'], ['id'])
# op.drop_constraint('unique_user_role', 'user_roles', type_='unique')
op.drop_constraint(None, 'document', type_='unique')
# ### end Alembic commands ###

View file

@ -1,88 +0,0 @@
"""Create models
Revision ID: 864a15f1ee0a
Revises:
Create Date: 2024-01-29 15:34:21.797387
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '864a15f1ee0a'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('companies',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_companies_id'), 'companies', ['id'], unique=False)
op.create_index(op.f('ix_companies_name'), 'companies', ['name'], unique=True)
op.create_table('roles',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=100), nullable=True),
sa.Column('description', sa.Text(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_roles_id'), 'roles', ['id'], unique=False)
op.create_index(op.f('ix_roles_name'), 'roles', ['name'], unique=False)
op.create_table('subscriptions',
sa.Column('sub_id', sa.Integer(), nullable=False),
sa.Column('company_id', sa.Integer(), nullable=True),
sa.Column('start_date', sa.DateTime(), nullable=True),
sa.Column('end_date', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['company_id'], ['companies.id'], ),
sa.PrimaryKeyConstraint('sub_id')
)
op.create_index(op.f('ix_subscriptions_sub_id'), 'subscriptions', ['sub_id'], unique=False)
op.create_table('users',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('email', sa.String(length=225), nullable=False),
sa.Column('hashed_password', sa.String(), nullable=False),
sa.Column('fullname', sa.String(length=225), nullable=False),
sa.Column('is_active', sa.Boolean(), nullable=True),
sa.Column('last_login', sa.DateTime(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('company_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['company_id'], ['companies.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('email'),
sa.UniqueConstraint('fullname'),
sa.UniqueConstraint('fullname', name='unique_username_no_spacing')
)
op.create_table('user_roles',
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('role_id', sa.Integer(), nullable=False),
sa.Column('company_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['company_id'], ['companies.id'], ),
sa.ForeignKeyConstraint(['role_id'], ['roles.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
sa.PrimaryKeyConstraint('user_id', 'role_id', 'company_id'),
sa.UniqueConstraint('user_id', 'role_id', 'company_id', name='unique_user_role')
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('user_roles')
op.drop_table('users')
op.drop_index(op.f('ix_subscriptions_sub_id'), table_name='subscriptions')
op.drop_table('subscriptions')
op.drop_index(op.f('ix_roles_name'), table_name='roles')
op.drop_index(op.f('ix_roles_id'), table_name='roles')
op.drop_table('roles')
op.drop_index(op.f('ix_companies_name'), table_name='companies')
op.drop_index(op.f('ix_companies_id'), table_name='companies')
op.drop_table('companies')
# ### end Alembic commands ###

View file

@ -0,0 +1,39 @@
"""Create document model
Revision ID: 9ce6871961b5
Revises:
Create Date: 2024-02-08 16:16:31.072913
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '9ce6871961b5'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('document',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('filename', sa.String(length=225), nullable=False),
sa.Column('uploaded_by', sa.Integer(), nullable=False),
sa.Column('uploaded_at', sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(['uploaded_by'], ['users.id'], ),
sa.PrimaryKeyConstraint('id', 'uploaded_by')
)
# op.create_unique_constraint('unique_user_role', 'user_roles', ['user_id', 'role_id', 'company_id'])
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# op.drop_constraint('unique_user_role', 'user_roles', type_='unique')
op.drop_table('document')
# ### end Alembic commands ###

View file

@ -0,0 +1,34 @@
"""Company column users table
Revision ID: ce0f3c4dd625
Revises: 488f28da0939
Create Date: 2024-02-09 19:35:09.546805
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'ce0f3c4dd625'
down_revision: Union[str, None] = '488f28da0939'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
# op.create_unique_constraint('unique_user_role', 'user_roles', ['user_id', 'role_id', 'company_id'])
op.add_column('users', sa.Column('company_id', sa.Integer(), nullable=True))
op.create_foreign_key(None, 'users', 'companies', ['company_id'], ['id'])
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(None, 'users', type_='foreignkey')
op.drop_column('users', 'company_id')
# op.drop_constraint('unique_user_role', 'user_roles', type_='unique')
# ### end Alembic commands ###

View file

@ -39,7 +39,7 @@ SOURCES_SEPARATOR = "\n Sources: \n"
MODES = ["Query Docs", "Search in Docs", "LLM Chat"]
DEFAULT_MODE = MODES[0]
chat_router = APIRouter(prefix="/v1", tags=["Chat"])
home_router = APIRouter(prefix="/v1", tags=["Chat"])
class ListFilesResponse(BaseModel):
uploaded_files: List[str]
@ -165,7 +165,7 @@ def get_home_instance(request: Request) -> Home:
return home_instance
@chat_router.post("/chat")
@home_router.post("/chat")
async def chat_endpoint(
home_instance: Home = Depends(get_home_instance),
message: str = Body(...), mode: str = Body(DEFAULT_MODE),
@ -180,7 +180,7 @@ async def chat_endpoint(
)
@chat_router.get("/list_files")
@home_router.get("/list_files")
async def list_files(
home_instance: Home = Depends(get_home_instance),
current_user: models.User = Security(

View file

@ -14,7 +14,7 @@ from private_gpt.server.ingest.ingest_router import ingest_router
from private_gpt.users.api.v1.api import api_router
from private_gpt.settings.settings import Settings
from private_gpt.home import chat_router
from private_gpt.home import home_router
logger = logging.getLogger(__name__)
@ -33,7 +33,7 @@ def create_app(root_injector: Injector) -> FastAPI:
app.include_router(health_router)
app.include_router(api_router)
app.include_router(chat_router)
app.include_router(home_router)
settings = root_injector.get(Settings)
if settings.server.cors.enabled:
logger.debug("Setting up CORS middleware")

View file

@ -1,7 +1,8 @@
import logging
from pathlib import Path
from typing import Literal
from typing import Literal, Optional
from sqlalchemy.orm import Session
from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File, status, Security, Body
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field
@ -136,8 +137,6 @@ def delete_file(
The document will be effectively deleted from your storage context.
"""
filename = delete_input.filename
print(filename)
service = request.state.injector.get(IngestService)
try:
doc_ids = service.get_doc_ids_by_filename(filename)
@ -159,6 +158,7 @@ def delete_file(
@ingest_router.post("/ingest/file", response_model=IngestResponse, tags=["Ingestion"])
def ingest_file(
request: Request,
db: Session = Depends(deps.get_db),
file: UploadFile = File(...),
current_user: models.User = Security(
deps.get_current_user,
@ -167,10 +167,27 @@ def ingest_file(
service = request.state.injector.get(IngestService)
try:
file_ingested = crud.documents.get_by_filename(db, file_name=file.filename)
if file_ingested:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="File already exists. Choose a different file.",
)
if file.filename is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail="No file name provided")
status_code=status.HTTP_400_BAD_REQUEST,
detail="No file name provided",
)
try:
docs_in = schemas.DocumentCreate(filename=file.filename, uploaded_by=current_user.id)
crud.documents.create(db=db, obj_in=docs_in)
except Exception as e:
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Unable to upload file.",
)
upload_path = Path(f"{UPLOAD_DIR}/{file.filename}")
with open(upload_path, "wb") as f:
@ -178,6 +195,7 @@ def ingest_file(
with open(upload_path, "rb") as f:
ingested_documents = service.ingest_bin_data(file.filename, f)
logger.info(f"{file.filename} is uploaded by the {current_user.fullname}.")
return IngestResponse(object="list", model="private-gpt", data=ingested_documents)
except Exception as e:

View file

@ -88,7 +88,7 @@ def login_access_token(
user_in = schemas.UserUpdate(
email=user.email,
fullname=user.fullname,
company_id=user.company_id,
company_id=user.user_role.company_id,
last_login=datetime.now()
)
user = crud.user.update(db, db_obj=user, obj_in=user_in)
@ -176,21 +176,11 @@ def register(
status_code=404,
detail="Company not found.",
)
if current_user.user_role.role.name not in {Role.SUPER_ADMIN["name"], Role.ADMIN["name"]}:
raise HTTPException(
status_code=403,
detail="You do not have permission to register users!",
)
user = register_user(db, email, fullname, random_password, company)
user_role_name = role_name or Role.GUEST["name"]
user_role = create_user_role(db, user, user_role_name, company)
else:
if current_user.user_role.role.name != Role.SUPER_ADMIN["name"]:
raise HTTPException(
status_code=403,
detail="You do not have permission to register users without a company.",
)
user = register_user(db, email, fullname, random_password, None)
user_role_name = role_name or Role.ADMIN["name"]
user_role = create_user_role(db, user, user_role_name, None)

View file

@ -285,3 +285,58 @@ def delete_user(
content={"message": "User deleted successfully"},
)
@router.post("/update_user")
def admin_update_user(
*,
db: Session = Depends(deps.get_db),
user_update: schemas.UserAdminUpdate,
current_user: models.User = Security(
deps.get_current_user,
scopes=[Role.ADMIN["name"], Role.SUPER_ADMIN["name"]],
),
) -> Any:
"""
Update the user by the Admin/Super_ADMIN
"""
existing_user = crud.user.get(db, id=user_update.id)
if existing_user is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"User not found with id: {user_update.id}",
)
if existing_user.fullname == user_update.fullname:
pass
else:
fullname = crud.user.get_by_name(db, name=user_update.fullname)
if fullname:
raise HTTPException(
status_code=409,
detail="The user with this username already exists!",
)
role = crud.role.get_by_name(db, name=user_update.role)
if role.id == 1:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Cannot create SUPER ADMIN!",
)
user_role = crud.user_role.get_by_user_id(db, user_id=existing_user.id)
role_in = schemas.UserRoleUpdate(
user_id=existing_user.id,
role_id=role.id,
)
role = crud.user_role.update(db, db_obj=user_role, obj_in=role_in)
user_in = schemas.UserUpdate(fullname=user_update.fullname,
email=existing_user.email, company_id=existing_user.user_role.company_id)
print("User in: ", user_in)
user = crud.user.update(db, db_obj=existing_user, obj_in=user_in)
return JSONResponse(
status_code=status.HTTP_200_OK,
content={"message": "User updated successfully",
}
)

View file

@ -3,6 +3,13 @@ from typing import Any, Dict, Optional
from pydantic_settings import BaseSettings
SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://{username}:{password}@{host}:{port}/{db_name}".format(
host='localhost',
port='5432',
db_name='QuickGpt',
username='postgres',
password="quick",
)
class Settings(BaseSettings):
PROJECT_NAME: str = "AUTHENTICATION AND AUTHORIZATION"

View file

@ -3,3 +3,4 @@ from .user_crud import user
from .user_role_crud import user_role
from .company_crud import company
from .subscription_crud import subscription
from .document_crud import documents

View file

@ -0,0 +1,16 @@
from sqlalchemy.orm import Session
from private_gpt.users.schemas.documents import DocumentCreate, DocumentUpdate
from private_gpt.users.models.documents import Documents
from private_gpt.users.crud.base import CRUDBase
from typing import Optional
class CRUDDocuments(CRUDBase[Documents, DocumentCreate, DocumentUpdate]):
def get_by_id(self, db: Session, *, id: str) -> Optional[Documents]:
return db.query(self.model).filter(Documents.id == id).first()
def get_by_filename(self, db: Session, *, file_name: str) -> Optional[Documents]:
return db.query(self.model).filter(Documents.filename == file_name).first()
documents = CRUDDocuments(Documents)

View file

@ -3,12 +3,13 @@ from typing import Any, Dict, List, Optional, Union
from private_gpt.users.core.security import get_password_hash, verify_password
from private_gpt.users.crud.base import CRUDBase
from private_gpt.users.models.user import User
from private_gpt.users.schemas.user import UserCreate, UserUpdate, AdminUpdate
from private_gpt.users.schemas.user import UserCreate, UserUpdate
from private_gpt.users.models.user_role import UserRole
from private_gpt.users.models.role import Role
from sqlalchemy.orm import Session
from sqlalchemy.orm import joinedload
class CRUDUser(CRUDBase[User, UserCreate, UserUpdate]):
def get_by_email(self, db: Session, *, email: str) -> Optional[User]:
return db.query(self.model).filter(User.email == email).first()
@ -75,7 +76,6 @@ class CRUDUser(CRUDBase[User, UserCreate, UserUpdate]):
.all()
)
def get_multi_by_company_id(
self, db: Session, *, company_id: str, skip: int = 0, limit: int = 100
) -> List[User]:
@ -89,5 +89,7 @@ class CRUDUser(CRUDBase[User, UserCreate, UserUpdate]):
.all()
)
def get_by_name(self, db: Session, *, name: str) -> Optional[User]:
return db.query(self.model).filter(User.fullname == name).first()
user = CRUDUser(User)

View file

@ -1,6 +1,6 @@
from private_gpt.users.core.config import settings
from private_gpt.users.core.config import SQLALCHEMY_DATABASE_URI
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
engine = create_engine(settings.SQLALCHEMY_DATABASE_URI, echo=True, future=True, pool_pre_ping=True)
engine = create_engine(SQLALCHEMY_DATABASE_URI, echo=True, future=True, pool_pre_ping=True)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

View file

@ -2,3 +2,5 @@ from .user import User
from .company import Company
from .user_role import UserRole
from .role import Role
from .documents import Documents
from .subscription import Subscription

View file

@ -0,0 +1,22 @@
from private_gpt.users.db.base_class import Base
from datetime import datetime, timedelta
from sqlalchemy.orm import relationship
from sqlalchemy import Column, Integer, String, Boolean, Float, ForeignKey, DateTime
class Documents(Base):
"""Models a user table"""
_tablename_ = "document"
id = Column(Integer, primary_key=True, index=True)
filename = Column(String(225), nullable=False, unique=True)
uploaded_by = Column(
Integer,
ForeignKey("users.id"),
primary_key=True,
nullable=False,
)
uploaded_at = Column(
DateTime,
default=datetime.utcnow,
)
uploaded_by_user = relationship("User", back_populates="uploaded_documents")

View file

@ -36,7 +36,7 @@ class User(Base):
company_id = Column(Integer, ForeignKey("companies.id"), nullable=True)
company = relationship("Company", back_populates="users")
uploaded_documents = relationship("Documents", back_populates="uploaded_by_user")
user_role = relationship(
"UserRole", back_populates="user", uselist=False, cascade="all, delete-orphan")

View file

@ -1,6 +1,6 @@
from .role import Role, RoleCreate, RoleInDB, RoleUpdate
from .token import TokenSchema, TokenPayload
from .user import User, UserCreate, UserInDB, UserUpdate, UserBaseSchema, Profile, UsernameUpdate, DeleteUser, AdminUpdate
from .user import User, UserCreate, UserInDB, UserUpdate, UserBaseSchema, Profile, UsernameUpdate, DeleteUser, UserAdminUpdate
from .user_role import UserRole, UserRoleCreate, UserRoleInDB, UserRoleUpdate
from .subscription import Subscription, SubscriptionBase, SubscriptionCreate, SubscriptionUpdate
from .company import Company, CompanyBase, CompanyCreate, CompanyUpdate

View file

@ -0,0 +1,24 @@
from pydantic import BaseModel
from datetime import datetime
from typing import Optional
class DocumentsBase(BaseModel):
filename: str
class DocumentCreate(DocumentsBase):
uploaded_by: int
class DocumentUpdate(DocumentsBase):
pass
class Document(DocumentsBase):
id: int
uploaded_by: int
uploaded_at: datetime
class Config:
orm_mode = True

View file

@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, EmailStr
from private_gpt.users.schemas.user_role import UserRole
from private_gpt.users.schemas.company import Company
class UserBaseSchema(BaseModel):
email: EmailStr
fullname: str
@ -14,12 +15,15 @@ class UserBaseSchema(BaseModel):
class Config:
arbitrary_types_allowed = True
class UserCreate(UserBaseSchema):
password: str = Field(alias="password")
class UsernameUpdate(BaseModel):
fullname: str
class UserUpdate(UserBaseSchema):
last_login: Optional[datetime] = None
@ -44,10 +48,14 @@ class UserSchema(UserBaseSchema):
orm_mode = True
# Additional properties to return via API
class User(UserSchema):
pass
# Additional properties stored in DB
class UserInDB(UserSchema):
hashed_password: str
@ -55,9 +63,12 @@ class UserInDB(UserSchema):
class Profile(UserBaseSchema):
role: str
class DeleteUser(BaseModel):
id: int
class AdminUpdate(BaseModel):
class UserAdminUpdate(BaseModel):
id: int
fullname: str
role: int
role: str

View file

@ -4,6 +4,8 @@ from private_gpt.users.schemas.role import Role
from pydantic import BaseModel
# Shared properties
class UserRoleBase(BaseModel):
user_id: Optional[int]
role_id: Optional[int]
@ -18,12 +20,16 @@ class UserRoleCreate(UserRoleBase):
pass
# Properties to receive via API on update
class UserRoleUpdate(BaseModel):
user_id: int
role_id: int
class Config:
arbitrary_types_allowed = True
class UserRoleInDBBase(UserRoleBase):
role: Role