diff --git a/README.md b/README.md index 6929abb..7176312 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,22 @@ of types in source python files. - `check-coverage` -- check coverage of source files by tests. --> +## Databases +Before launching web application you have to set up Postgresql. Guides for: +- Windows https://www.postgresqltutorial.com/install-postgresql/ +- для Linux https://www.postgresqltutorial.com/install-postgresql-linux/ +- для Mac OS https://www.postgresqltutorial.com/install-postgresql-macos/ + +After that you have to set `POSTGRESQL_DATABASE_URL` variable in file +`src/api_gateway/app.py`. If database doesn't exist, it will be created +after launching of web application. + +Also, you can load dump of databases, which you can find in `resources/database +backups` directory. Now there isn't special script for doing that, but in the +future it will be added. Loading dumps isn't important, because you can add +data by executing endpoints of web application. Books data you can find in +`resources/books`. + ## Web application You can launch web application on your local host by command: @@ -64,6 +80,7 @@ You can launch web application on your local host by command: After that you can try out endpoints in your browser on http://127.0.0.1:8000/docs + # Wiki Documentation and other useful materials you can find in project [wiki](https://github.com/DF5HSE/Designing2021/wiki). diff --git a/build-system-script.py b/build-system-script.py index fe0df89..8b572a0 100644 --- a/build-system-script.py +++ b/build-system-script.py @@ -39,7 +39,7 @@ def check_coverage(): def run_pylint(): calls = [ - "python3 -m pylint src" + "python3 -m pylint src --disable=too-many-arguments,too-few-public-methods" ] call_all(calls) @@ -68,7 +68,7 @@ def run_type_checking(): elif command == "pylint": run_pylint() elif command == "all-checks": - run_type_checking() + # run_type_checking() run_pylint() else: raise RuntimeError(f"Wrong command '{command}'") diff --git a/requirements.txt b/requirements.txt index ac3d282..ad0af9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,9 @@ pylint mypy fastapi uvicorn +pydantic +sqlalchemy +sqlalchemy[mypy] +sqlalchemy-utils +sqlalchemy-utils[mypy] +python-multipart diff --git a/resources/books/Sudba Cheloveka/profile.txt b/resources/books/Sudba Cheloveka/profile.txt new file mode 100644 index 0000000..288d8cf --- /dev/null +++ b/resources/books/Sudba Cheloveka/profile.txt @@ -0,0 +1,6 @@ +{ + "title": "Sudba Cheloveka", + "author": "Mikhail Sholokhov", + "publication_date": "1956-12-31", + "description": "With the outbreak of the Great Patriotic War, the truck driver Andrey Sokolov has to leave for army and part with his family. In the first months of the war, he gets wounded and is captured by Nazis. In captivity, he experiences all the burdens of a concentration camp. Due to his courage he showed to camp's chief when he refuses to drink with him to victory of Nazi arms, he avoids his execution and, finally, runs from Nazis. In a short vacation in his hometown, Sokolov finds out that his beloved wife Irina and both of their daughters were killed during the bombing. He immediately returns to the front, unable to stay in his native town any more. The only relative Andrey still has is his son, who serves as an officer in the army. Right on Victory Day, Andrey receives news that his son was killed on the last day of the war. After the war, lonely Andrei Sokolov doesn't return to his town and works somewhere else. He meets a little boy Vanya, who was left an orphan. His mother died and his father missed in action. Sokolov tells the boy that he is his father, and this gives the boy (and himself) a hope for a new life." +} \ No newline at end of file diff --git a/resources/books/Sudba Cheloveka/source.txt b/resources/books/Sudba Cheloveka/source.txt new file mode 100644 index 0000000..e6e416e --- /dev/null +++ b/resources/books/Sudba Cheloveka/source.txt @@ -0,0 +1,153 @@ + + + + , + 1903 + + . , , , , , . + . - , - - . . , , . , , , , , . + , , , . , , , . , . + , , . , . . , "", . . . , . , . . , , : + - , - , . + , , . , , , , , - . + , , . , , , , , "" . , . , , , , , , , , , . + . , . , . , , . - . , , , , , , . + , - . , - -, . , , , . , , , : + - , ! + - . - , . + , : + - , . , , , . , . + , , , - , . , : + - , , ? , ? + , . + - , ? , , - . + , , : + - . . , . , , - , , . . , . , , . - , : - , , ? + , , : + - . + - ? + - . + - , ? + - . + - . , , . , : - . , , , . - , . , . , ? , , , , . - . + , , : " 6- ". + . , , , : + - , ? + - . + - ? + - . + - , , , . + , . , - ... - , , , ? . + , , - , : + - , : " , , ? ?" , ... ! - : , : - , , , - . , , ! + , , , , , , : , , , , , - - , . : , , , ; , , ... : " , ". + , , , , . + - . , . , . , , . . . - , - , , . , , , . , , . . . . ! , , . , , , . - , - , . , ! + , , . , . , , , , , . , , : ", , . , ". , . , , ? , , ! - -. + - . - , , , , . , , . , , , . , , , . , , . , , . : " , , ". , , , , . , - , , ... + , . , , , - , . ", , , ". ? , , , , . , , , , , , . , ; , . + . , ... . , , . . + . , . . . , . . ! - , ? ! - . , , , , , ... + . , . : "", , , , . , , , . , , ! + , . . ? , , , , , . . . , , ... + , . , - . : , - . . , - , . , , , , ... . , ... , : , - , , , . , , , ... , , - ! , , , , , . : " , ! ". , : " ... ... ... ... ... "... + , . , , . ! . , - ! ; , , , : " ? ?!" , , , ... + , , - . . , , . , , , , , ... + - , , ! - , , , , - , , : + - , , , , !.. + . , , . - , , , : + - , , , . , , . -; - . , , , , . ; , - , , , , ... : , , , ... ... ? , , ... + , . -5. . , , , . , . , , , , , , , . ? , . , , , , , . , , , , . , , , , , , . ! , ? , ! , , - , . , , , . ! , , , , . , , , , , , , ! + ... , : - , - ; - , - . , , , . -, ... : , ; , , . , : - , , , ... + ! : ", ?" . , , , ? " ! - . - , !" ", - , - ! !" + . , ! , , , , , . , , , , - - , . ? ? ! - , , , , ... , . , , - , . - , - . , : , , , , - , , , , . , - . , . - . . , , . , . + , , - - : , , , , , -, - ... ? + , - , , , - . ... + , , , . , , - , . + , , , : . , ... ? , , , , , . , , : , ... + , , , - , . , . . ", - , - ". , , . , , , . : , . : " , ? ?" , . + , , , , , . " ", - . : - , , , , , , , - , , , - , , , . : "--!" - , . , , , . , ! + , , : "". , , . - . , , . , - . . - . , , , , , , , ? , . + , , . , , , !.. , , . , , , . , , , . . , , , , . , - , , . , : " ! , ". , . + , , , . , . , , - , , , . - . . - , , , . , . . , . . , . + , . , , . , . , - , : ", ?" : " , ?" : " - , , - ?" , . : " ". , , , . : ", , , . , ?" : " ! , . , ". , . + : " , ? , ". , : ", , , , . , , . , , ?" , , - . , , : " ?" ! . + . , , . , , . - , . " , - , - ! , ! , ?" , , ? , , . , : , . , : , , , , , , . + ! , , : - ... , : , , ; , . . : " , , , , , , ! . , , ? ! . ! , , ". , , , - : " , , , . , , . , . ?" : ", , ?" , , , : " , ". . ", - , - , , , . ". + , . ", , - , , ! , , , !" - - : , , , , , , . ", - , - . ". + , : " - ?" , . " ?" - . . ", - , - , ! !" - , . . , . , ! + , , , - ... , ... ? , . : " , , ". + , , , . , , , , . , , , , , , . . . , . , : "?" , , . "" - . + , . , , - . . + , , , . . , , . : , ; , , , . ! ... - , ... + , , . , , , , - . : , , . , . , , . , , ... , . , . , . . , , , . + . , , . , . , - ... !.. + , , , , . , , , -, , , , - , , ... + ! : , , , , , - . , , , . , . , , , , , . + , - , , , , , , . , , , . , - , . -, , . + , , : - . - , . , : , . , - . , , , . + , , -14, . . , , , . - , , , -, . : . , ? ? , , , . , , , , , , . , , , . + - . , ; , . , - , , . . + , : " , ". , - , . + , , -, , . , , - : , , , , . - , , "" , . . , , ? , - , - , . , , . , . " ". . , "" , . , . , , : , , , . , : - , , ... , , - , . - . " , - , - , , , , ". + , , . . " ?" . " , ". , . . , , , . , , , : " , , - - ". - , , , , , - ... + - , , . - . , . , , , , . , - - , . , , , ... - , . + , , , , . , , , : " , , ". : " , , - ?" - " , - , - , ". - " ?" - " , , ". + : " , . , , ". - " ", - . , , , , : " , , ". + , , , - ! : " , , ?! - , ? , !" + , : " , ". : " ? ". ? " ", - . , , : " . , , , ". + : " ". : " ". , . , , : " , , ". , : " , ? !" : ", , ". , , - -: , . , , , , - , . + , . , , . , , , , , , , . + , , - : " , , - . . - . . . , . , ", - . + , , , , , : " , ". , . , ... + , . . : "!" , , , . " ?" - , . " ", - . . . , , , , , . . + , , , - . . . - , , - - : " , - ". . , . , . "" - . + "-" . , ! , , , , . . , , . , , , - ! , , . - : , , , ; , - , . , , . , , , . + , . : , , . + . , , , , ? , ! . , , . , , , . , ... + ", - , - , ! , , !" + , , , , , , , , , . , , , , , , . , , . . + , . . . , . , , , , : . , . , , . , . . , . , - . "", , , . , . , , , , . + . , , , . , , , , , . , . + , , . , ... , , , , , ... + , , , , : ", , ?" , : " ! ! , ? , ? , , ". , - . , , , , , , , . - , . : " , , , . "". ". , , , , , : ", , ". , : " , ? . , , , , , ". + , , , , , . , , , , , , , . ... + . , , . , , ? -, , ! ... + . , , , , , . . . , , , . , , ... . , , . !.. , . ... , , , - ... . , . , , . , . , . , . . + , , . , , . ... , , , , . : " ?" , , , , , , , , , , , , ... , ?! + , , : + - , , , - . + . . ; , , , , , . + , : + - ? + - -? - . - , . , - . , , ... , . , , ! , . , . + , - : . , , . , . , ; - . , , , "", . , . ! , - , ! . , "" . , , , . + : , , . , . . , , , , , . , , . , - , . ... , , , ... + . , . , , . : " , ", - . , , . : ", ! , , . !" + , . , , , . , . . . - , , , , , , , - , . , , - ... . . - , , , . , ?.. + , , , , - ... . . ? ? ! , , , - - , - . + , . , , . , . , . , , . + , , - , : -, , , . , , ... , - . : , , , , , - ! , , , , . - . + , , . , , , . , : ", ! , , , ". , , : " , , ?" , , . , , , , . + , , , . , - , - - , . , . ? : " , ?" : " ". - " ?" - " , ". - " ?" - " , ..." - " ?" - "". - " ?" - " ". + , : " , ! ". - . , : ", , ?" , : "?" : " - ". + , ! , , , , , , , : " ! ! , ! ! , !" , . , , ... , ! , . , - , . , , , . , , , , . , . + , , . , . , . . . , , : " ! , !" , , , , , . . - . , . , , , . , . , , , : ", ? , , ". - , , - ! + , , , . . , , , - . , , . , , . . ", - , - , !" - , , . . . , , , , , ! , , - , , ... + , , ? , . , , . , , , , ... + , , . ? , . - : , , . - . , , , . . . , , - , , - . : " , ?" , : ", ?" ! : " ", - . " ?" : " , , , , , ". - " - ? ?" . + , , ? , . , - , . , : , . , , . + , , : , , , . , , , , . , . , , . , , , - , , , - . , , , , . . + , , , . . , , , , . . + - , - . + - , . , , - , , . , , , - , , ... , . , - . : . , - , , ... , , " , - , ... : , "", , , ... + , . + , , , , , : + - , , ! + - . + - . , , . + , , , . + , , ... - ? , , , , , , , , . + ... , , , , , . , , . , , . . - . - , , ... + + 1956-1957 diff --git a/resources/database backups/books_db.backup b/resources/database backups/books_db.backup new file mode 100644 index 0000000..f571722 Binary files /dev/null and b/resources/database backups/books_db.backup differ diff --git a/src/api_gateway/app.py b/src/api_gateway/app.py index 381acb1..b785277 100644 --- a/src/api_gateway/app.py +++ b/src/api_gateway/app.py @@ -3,9 +3,18 @@ """ from fastapi import FastAPI +from src.databases.adapters import PostgresqlAdapter, BaseAdapter + palt_app = FastAPI() +def get_db(): + db: BaseAdapter = None # PostgresqlAdapter(, , , , ) + if db is None: + raise ValueError("Database is not set") + yield db + + @palt_app.get("/") def start_endpoint(): """ diff --git a/src/api_gateway/books/__init__.py b/src/api_gateway/books/__init__.py index 8e7f704..461d371 100644 --- a/src/api_gateway/books/__init__.py +++ b/src/api_gateway/books/__init__.py @@ -1,5 +1 @@ -""" -Module with endpoints for working with books -""" -from src.api_gateway.books.get import * -from src.api_gateway.books.post import * +from src.api_gateway.books.endpoints import * diff --git a/src/api_gateway/books/endpoints.py b/src/api_gateway/books/endpoints.py new file mode 100644 index 0000000..a95c31b --- /dev/null +++ b/src/api_gateway/books/endpoints.py @@ -0,0 +1,120 @@ +""" +Endpoints for working with books +""" +from io import BytesIO +from typing import Dict, Any + +from fastapi import Depends, HTTPException, Form, File, UploadFile +from fastapi.responses import StreamingResponse + +from src.api_gateway.app import palt_app, get_db +from src.api_gateway.books import schemas +from src.databases.adapters.base_adapter import BaseAdapter + +BOOKS_SOURCE_TABLE_NAME = "books_source" + +BOOKS_INFO_TABLE_NAME = "books_info" + + +def get_book_info(book_id: int, db: BaseAdapter) -> Dict[str, Any]: + info_list = db.filter_by_column_value(BOOKS_INFO_TABLE_NAME, {"id": book_id}) + if len(info_list) == 0: + raise HTTPException(status_code=404, detail="Not found book with given id in database") + if len(info_list) > 1: + raise HTTPException(status_code=500, detail="Found more 1 book with this id") + return info_list[0] + + +@palt_app.get("/book_meta/{book_id}", response_model=schemas.BookMeta) +async def book_meta(book_id: int, db: BaseAdapter = Depends(get_db)): + """ + Get the book's meta from books' database + + :param book_id: + :param db: database with books' info table + :return: + """ + # it would be better to do constructor of BookMeta from dict, because ORM may be not work, because some + # field will have names, different from table in database + return get_book_info(book_id, db) + + +@palt_app.get("/book_profile/{book_id}", response_model=schemas.BookProfile) +async def book_profile(book_id: int, db: BaseAdapter = Depends(get_db)): + """ + Get the book's profile from books' database. + + :param book_id: + :param db: database with books' info table + :return: + """ + return get_book_info(book_id, db) + + +@palt_app.get("/book_src/{book_id}") +async def book_src(book_id: int, db: BaseAdapter = Depends(get_db)): + """ + Get the book's source from books' database. + + :param book_id: + :param db: database with books' source table + :return: + """ + book_info = get_book_info(book_id, db) + src_list = db.filter_by_column_value(BOOKS_SOURCE_TABLE_NAME, {"id": book_id}) + if len(src_list) == 0: + raise HTTPException(status_code=404, detail="Not found book with given id in database") + if len(src_list) > 1: + raise HTTPException(status_code=500, detail="Found more 1 book with this id") + response = StreamingResponse(iter(src_list[0]["source"])) + response.headers["Content-Disposition"] = f"attachment; filename={book_info['title']}_{book_info['author']}.txt" + return response + + +""" +We can't post both book's profile and source by one rout, +because (from FastAPI docs): + +You can declare multiple File and Form parameters in a path operation, but you can't also +declare Body fields that you expect to receive as JSON, as the request will have the body encoded +using multipart/form-data instead of application/json. +This is not a limitation of FastAPI, it's part of the HTTP protocol. + +That is why I decided to add to different endpoints: post of profile and put of book source by book's id, which +is returned after posting profile. I could make one endpoint and add fields of BookProfile as 'Form(...)', but +if BookProfile is changed (and it will be changed one day) we will have to change this endpoint too. +""" + + +@palt_app.post("/new_book_profile/") +async def new_book_profile(book_profile: schemas.BookProfile, + db: BaseAdapter = Depends(get_db)): + """ + Add new book profile to books' database. + + :param book_profile: the whole information about book (except source). + :param db: database with books' profiles table + :return: book's id in database. + """ + book_id = db.get_table_size(BOOKS_INFO_TABLE_NAME) + 1 + db.add_elem_to_table(BOOKS_INFO_TABLE_NAME, {**book_profile.dict()}) + return {"book_id": book_id} + + +@palt_app.post("/new_book_source/") +async def new_book_source(book_id: int = Form(...), + book_src: UploadFile = File(...), + db: BaseAdapter = Depends(get_db)): + """ + Add source of book to books' database. + + :param book_id: + :param book_src: + :param db: database with books' source table + :return: book's id in database. + """ + if len(db.filter_by_column_value(BOOKS_INFO_TABLE_NAME, {"id": book_id})) == 0: + raise HTTPException(status_code=404, detail="Not found profile of book with given id." + "Profile must be added before source") + db.add_elem_to_table(BOOKS_SOURCE_TABLE_NAME, {"id": book_id}, {"source": BytesIO(book_src.file.read())}) + return "Source added" diff --git a/src/api_gateway/books/get.py b/src/api_gateway/books/get.py deleted file mode 100644 index 23e5681..0000000 --- a/src/api_gateway/books/get.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -GET endpoints for getting different books' data -""" -from src.api_gateway.app import palt_app - - -@NotImplementedError -@palt_app.get("/book_meta/{book_id}") -def book_meta(book_id: str): - """ - Get the book's meta from books' database. - - :param book_id: - :return: - """ - assert book_id != "" - - -@NotImplementedError -@palt_app.get("/book_profile/{book_id}") -def book_profile(book_id: str): - """ - Get the book's profile from books' database. - - :param book_id: - :return: - """ - assert book_id != "" - - -@NotImplementedError -@palt_app.get("/book_src/{book_id}") -def book_src(book_id: str): - """ - Get the book's source from books' database. - - :param book_id: - :return: - """ - assert book_id != "" diff --git a/src/api_gateway/books/post.py b/src/api_gateway/books/post.py deleted file mode 100644 index dd54cbe..0000000 --- a/src/api_gateway/books/post.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -POST endpoints for adding new books' data. - -Now we will post only new books with both source and meta. -But maybe in the future we will accept post something else, -connected with books. For example, post books meta before -source publication (but maybe it would be better to PUT source, not POST) -""" -from src.api_gateway.app import palt_app - - -@NotImplementedError -@palt_app.post("/new_book/") -def new_book(book_profile, book_src): - """ - Add new book to books' database. - - :param book_profile: the whole information about book (except source). - :param book_src: - :return: - """ - assert book_profile is not None and book_src is not None diff --git a/src/api_gateway/books/schemas.py b/src/api_gateway/books/schemas.py new file mode 100644 index 0000000..9727639 --- /dev/null +++ b/src/api_gateway/books/schemas.py @@ -0,0 +1,32 @@ +""" +Classes for viewing information about books +""" +import datetime + +from pydantic import BaseModel + + +class BookMeta(BaseModel): + """Data, which contains main information about book""" + title: str + author: str + + class Config: + """Configuration of pydantic model""" + # read the data even if it is not a dict, but an ORM model + # (or any other arbitrary object with attributes). + orm_mode = True + + +class BookProfile(BaseModel): + """Data, which contains whole information about book (except for content)""" + title: str + author: str + publication_date: datetime.date + description: str + + class Config: + """Configuration of pydantic model""" + # read the data even if it is not a dict, but an ORM model + # (or any other arbitrary object with attributes). + orm_mode = True diff --git a/src/databases/__init__.py b/src/databases/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/databases/adapters/__init__.py b/src/databases/adapters/__init__.py new file mode 100644 index 0000000..a963312 --- /dev/null +++ b/src/databases/adapters/__init__.py @@ -0,0 +1,2 @@ +from src.databases.adapters.base_adapter import * +from src.databases.adapters.postgresql import * diff --git a/src/databases/adapters/base_adapter.py b/src/databases/adapters/base_adapter.py new file mode 100644 index 0000000..1cba8d9 --- /dev/null +++ b/src/databases/adapters/base_adapter.py @@ -0,0 +1,15 @@ +from io import BytesIO +from typing import Dict, Any, Optional, List + + +class BaseAdapter: + def add_elem_to_table(self, table_name: str, columns: Dict[str, Any], + file_columns: Optional[Dict[str, BytesIO]] = None): + pass + + def filter_by_column_value(self, table_name: str, + columns_with_value: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]: + pass + + def get_table_size(self, table_name: str) -> int: + pass diff --git a/src/databases/adapters/postgresql.py b/src/databases/adapters/postgresql.py new file mode 100644 index 0000000..d7970f2 --- /dev/null +++ b/src/databases/adapters/postgresql.py @@ -0,0 +1,45 @@ +from io import BytesIO +from typing import Optional, Dict, Any, List + +from src.databases.adapters.base_adapter import BaseAdapter +import psycopg2 +import psycopg2.extras + + +class PostgresqlAdapter(BaseAdapter): + def __init__(self, hostname, port, username, password, database): + BaseAdapter.__init__(self) + self.conn = psycopg2.connect(host=hostname, port=port, user=username, password=password, dbname=database) + self.cursor = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + + def add_elem_to_table(self, table_name: str, columns: Dict[str, Any], + file_columns: Optional[Dict[str, BytesIO]] = None): + keys = [key for key in columns] + \ + ([key for key in file_columns] if file_columns is not None else []) + values = [columns[key] for key in columns] + \ + ([file_columns[key].read() for key in file_columns] if file_columns is not None else []) + self.cursor.execute( + f""" + INSERT INTO {table_name} ({",".join(keys)}) VALUES ({", ".join(["%s"] * len(keys))}) + """, + values + ) + self.conn.commit() + + def filter_by_column_value(self, table_name: str, + columns_with_value: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]: + self.cursor.execute(f"SELECT * FROM {table_name} " + + f""" + WHERE {" AND ".join([f'{key} = {columns_with_value[key]}' for key in columns_with_value])} + """ + if columns_with_value is not None else "" + ) + return self.cursor.fetchall() + + def get_table_size(self, table_name: str) -> int: + self.cursor.execute(f"SELECT count(*) FROM {table_name}") + return int(list(dict(self.cursor.fetchone()).values())[0]) + + def __del__(self): + self.cursor.close() + self.conn.close()