From 6e2056a68599ccafddf682b68c641b266bacd5ca Mon Sep 17 00:00:00 2001 From: mst Date: Fri, 28 Feb 2025 07:07:52 +0300 Subject: [PATCH] updating the code to work with postgres --- app.py | 81 +++++++++++++++++----------------- functions.py | 112 +++++++++++++++++++++++++---------------------- requirements.txt | 3 +- schema.sql | 37 ++++++++-------- 4 files changed, 120 insertions(+), 113 deletions(-) diff --git a/app.py b/app.py index 4f90bd4..04955c1 100644 --- a/app.py +++ b/app.py @@ -60,7 +60,7 @@ admin_bp = Blueprint('admin', const.appName) @app.cli.command("init-db") def initDatabase(): dbName = os.environ.get("DB_NAME") - print(f"Attempting to connect to database {dbName}...") + print("Attempting to connect to database {dbName}...") try: conn = func.connectToDb() cursor = conn.cursor() @@ -68,34 +68,30 @@ def initDatabase(): conn.database = dbName - except mysql.connector.Error as error: - if error.errno == errorcode.ER_ACCESS_DENIED_ERROR: - print("Bad credentials") - elif error.errno == errorcode.ER_BAD_DB_ERROR: - dbPort = os.environ.get("DB_PORT") - if not dbPort: - dbPort = 3306 - - conn = mysql.connector.connect( - user=os.environ.get("DB_USER"), - password=os.environ.get("DB_PASS"), - host=os.environ.get("DB_HOST"), - port=dbPort, - database='mysql' - ) - cursor = conn.cursor() - func.createDatabase(cursor, dbName) - conn.database = dbName - else: - print("Error:", error) - return + except psycopg.Error as error: + app.logger.error("Database error:", error) + # if error.errno == errorcode.ER_ACCESS_DENIED_ERROR: + # print("Bad credentials") + # elif error.errno == errorcode.ER_BAD_DB_ERROR: + # dbPort = os.environ.get("DB_PORT") + # if not dbPort: + # dbPort = 3306 + # + # conn = func.connectToDb() + # cursor = conn.cursor() + # func.createDatabase(cursor, dbName) + # conn.database = dbName + # else: + # print("Error:", error) + # return with open('schema.sql', 'r') as schema_file: schema = schema_file.read() try: - cursor.execute(schema, multi=True) + cursor.execute(schema) + conn.commit() print(f"Database {dbName} was successfully initialized!") - except mysql.connector.Error as error: + except psycopg.Error as error: print(f"Failed to initialize database {dbName}: {error}") finally: cursor.close() @@ -209,7 +205,7 @@ def viewQuestion(question_id): @app.route('/questions/', methods=['GET']) def seeAskedQuestions(): conn = func.connectToDb() - cursor = conn.cursor(dictionary=True) + cursor = conn.cursor() cursor.execute("SELECT * FROM questions WHERE asker_id=%s ORDER BY creation_date DESC", (asker_id,)) questions = cursor.fetchall() @@ -411,7 +407,7 @@ def addQuestion(): def deleteQuestion(): question_id = request.args.get('question_id', '') if not question_id: - abort(400, "Missing 'question_id' attribute or 'question_id' is empty") + abort(400, _("Missing 'question_id' attribute or 'question_id' is empty")) conn = func.connectToDb() cursor = conn.cursor() @@ -419,17 +415,18 @@ def deleteQuestion(): app.logger.debug("[CatAsk/API/delete_question] DELETE'ing a question from database") cursor.execute("DELETE FROM questions WHERE id=%s", (question_id,)) + conn.commit() cursor.close() conn.close() - return {'message': 'Successfully deleted question.'}, 200 + return {'message': _("Successfully deleted question.")}, 200 @api_bp.route('/return_to_inbox/', methods=['POST']) @loginRequired def returnToInbox(): question_id = request.args.get('question_id', '') if not question_id: - abort(400, "Missing 'question_id' attribute or 'question_id' is empty") + abort(400, _("Missing 'question_id' attribute or 'question_id' is empty")) conn = func.connectToDb() cursor = conn.cursor() @@ -437,14 +434,14 @@ def returnToInbox(): app.logger.debug("[CatAsk/API/return_to_inbox] SELECT'ing a question from database") cursor.execute("SELECT from_who, content, creation_date, cw FROM questions WHERE id=%s", (question_id,)) - row = cursor.fetchone() + question = cursor.fetchone() - question = { - 'from_who': row[0], - 'content': row[1], - 'creation_date': row[2], - 'cw': row[3] - } + # question = { + # 'from_who': row[0], + # 'content': row[1], + # 'creation_date': row[2], + # 'cw': row[3] + # } app.logger.debug("[CatAsk/API/return_to_inbox] DELETE'ing a question from database") @@ -453,17 +450,18 @@ def returnToInbox(): app.logger.debug("[CatAsk/API/return_to_inbox] INSERT'ing a question into database") cursor.execute("INSERT INTO questions (from_who, content, creation_date, answered, cw) VALUES (%s, %s, %s, %s, %s)", (question["from_who"], question["content"], question["creation_date"], False, question['cw'])) + conn.commit() cursor.close() conn.close() - return {'message': 'Successfully returned question to inbox.'}, 200 + return {'message': _("Successfully returned question to inbox.")}, 200 @api_bp.route('/pin_question/', methods=['POST']) @loginRequired def pinQuestion(): question_id = request.args.get('question_id', '') if not question_id: - abort(400, "Missing 'question_id' attribute or 'question_id' is empty") + abort(400, _("Missing 'question_id' attribute or 'question_id' is empty")) conn = func.connectToDb() cursor = conn.cursor() @@ -471,17 +469,18 @@ def pinQuestion(): app.logger.debug("[CatAsk/API/pin_question] UPDATE'ing a question to pin it") cursor.execute("UPDATE questions SET pinned=%s WHERE id=%s", (True, question_id)) + conn.commit() cursor.close() conn.close() - return {'message': 'Successfully pinned question.'}, 200 + return {'message': _("Successfully pinned question.")}, 200 @api_bp.route('/unpin_question/', methods=['POST']) @loginRequired def unpinQuestion(): question_id = request.args.get('question_id', '') if not question_id: - abort(400, "Missing 'question_id' attribute or 'question_id' is empty") + abort(400, _("Missing 'question_id' attribute or 'question_id' is empty")) conn = func.connectToDb() cursor = conn.cursor() @@ -489,10 +488,12 @@ def unpinQuestion(): app.logger.debug("[CatAsk/API/unpin_question] UPDATE'ing a question to unpin it") cursor.execute("UPDATE questions SET pinned=%s WHERE id=%s", (False, question_id)) + + conn.commit() cursor.close() conn.close() - return {'message': 'Successfully unpinned question.'}, 200 + return {'message': _("Successfully unpinned question.")}, 200 @api_bp.route('/add_answer/', methods=['POST']) @loginRequired diff --git a/functions.py b/functions.py index 44aa708..84b7541 100644 --- a/functions.py +++ b/functions.py @@ -101,37 +101,31 @@ dbPort = os.environ.get("DB_PORT") if not dbPort: dbPort = 3306 -def createDatabase(cursor, dbName): +def createDatabase(cursor, dbName) -> None: try: - cursor.execute("CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(dbName)) + cursor.execute("CREATE DATABASE {} OWNER {}".format(dbName, dbUser)) print(f"Database {dbName} created successfully") - except mysql.connector.Error as error: + except psycopg.Error as error: print("Failed to create database:", error) exit(1) def connectToDb(): - conn = mysql.connector.connect( - host=dbHost, - user=dbUser, - password=dbPass, - database=dbName, - port=dbPort, - autocommit=True - ) - return conn + # using dict_row factory here because its easier than modifying now-legacy mysql code + return psycopg.connect(f"postgresql://{dbUser}:{dbPass}@{dbHost}/{dbName}", row_factory=dict_row) -def getQuestion(question_id: int): +def getQuestion(question_id: int) -> dict: conn = connectToDb() - cursor = conn.cursor(dictionary=True) + cursor = conn.cursor() cursor.execute("SELECT * FROM questions WHERE id=%s", (question_id,)) question = cursor.fetchone() + question['creation_date'] = question['creation_date'].replace(microsecond=0).replace(tzinfo=None) cursor.close() conn.close() return question -def getAllQuestions(): +def getAllQuestions(limit: int = None, offset: int = None) -> dict: conn = connectToDb() - cursor = conn.cursor(dictionary=True) + cursor = conn.cursor() app.logger.debug("[CatAsk/functions/getAllQuestions] SELECT'ing all questions with latest answers") @@ -147,11 +141,18 @@ def getAllQuestions(): ORDER BY q.pinned DESC, (a.creation_date IS NULL), a.creation_date DESC, q.creation_date DESC """ - cursor.execute(query, (True,)) + params = [True] + if limit is not None: + query += " LIMIT %s" + params.append(limit) + if offset is not None: + query += " OFFSET %s" + params.append(offset) + + cursor.execute(query, tuple(params)) questions = cursor.fetchall() app.logger.debug("[CatAsk/functions/getAllQuestions] SELECT'ing answers") - cursor.execute("SELECT * FROM answers ORDER BY creation_date DESC") answers = cursor.fetchall() @@ -159,6 +160,9 @@ def getAllQuestions(): combined = [] for question in questions: + question['creation_date'] = question['creation_date'].replace(microsecond=0).replace(tzinfo=None) + for answer in answers: + answer['creation_date'] = answer['creation_date'].replace(microsecond=0).replace(tzinfo=None) question_answers = [answer for answer in answers if answer['question_id'] == question['id']] combined.append({ 'question': question, @@ -201,7 +205,7 @@ def addQuestion(from_who, question, cw, noAntispam=False): json_r = r.json() success = json_r['success'] if not success: - return {'error': 'An error has occurred'}, 500 + return {'error': _('An error has occurred')}, 500 elif cfg['antispam']['type'] == 'turnstile': r = requests.post( 'https://challenges.cloudflare.com/turnstile/v0/siteverify', @@ -210,7 +214,7 @@ def addQuestion(from_who, question, cw, noAntispam=False): json_r = r.json() success = json_r['success'] if not success: - return {'error': 'An error has occurred'}, 500 + return {'error': _('An error has occurred')}, 500 elif cfg['antispam']['type'] == 'frc': url = 'https://global.frcapi.com/api/v2/captcha/siteverify' headers = {'X-API-Key': cfg['antispam']['frc']['apikey']} @@ -219,14 +223,14 @@ def addQuestion(from_who, question, cw, noAntispam=False): json_r = r.json() success = json_r['success'] if not success: - return {'error': 'An error has occurred'}, 500 + return {'error': _('An error has occurred')}, 500 blacklist = readPlainFile(const.blacklistFile, split=True) for bad_word in blacklist: if bad_word in question or bad_word in from_who: # return a generic error message so bad actors wouldn't figure out the blacklist - return {'error': 'An error has occurred'}, 500 + return {'error': _('An error has occurred')}, 500 conn = connectToDb() cursor = conn.cursor() @@ -234,45 +238,48 @@ def addQuestion(from_who, question, cw, noAntispam=False): app.logger.debug("[CatAsk/API/add_question] INSERT'ing new question into database") cursor.execute("INSERT INTO questions (from_who, content, answered, cw) VALUES (%s, %s, %s, %s) RETURNING id", (from_who, question, False, cw)) - question_id = cursor.fetchone()[0] + question_id = cursor.fetchone()['id'] + conn.commit() cursor.close() conn.close() - return {'message': 'Question asked successfully!'}, 201, question_id + return {'message': _('Question asked successfully!')}, 201, question_id -def getAnswer(question_id: int): +def getAnswer(question_id: int) -> dict: conn = connectToDb() - cursor = conn.cursor(dictionary=True) + cursor = conn.cursor() cursor.execute("SELECT * FROM answers WHERE question_id=%s", (question_id,)) answer = cursor.fetchone() + answer['creation_date'] = answer['creation_date'].replace(microsecond=0).replace(tzinfo=None) cursor.close() conn.close() return answer -def addAnswer(question_id, answer, cw): +def addAnswer(question_id: int, answer: str, cw: str) -> dict: conn = connectToDb() try: cursor = conn.cursor() app.logger.debug("[CatAsk/API/add_answer] INSERT'ing an answer into database") - cursor.execute("INSERT INTO answers (question_id, content, cw) VALUES (%s, %s, %s)", (question_id, answer, cw)) - answer_id = cursor.lastrowid + cursor.execute("INSERT INTO answers (question_id, content, cw) VALUES (%s, %s, %s) RETURNING id", (question_id, answer, cw)) + answer_id = cursor.fetchone()['id'] app.logger.debug("[CatAsk/API/add_answer] UPDATE'ing question to set answered and answer_id") cursor.execute("UPDATE questions SET answered=%s, answer_id=%s WHERE id=%s", (True, answer_id, question_id)) conn.commit() - except Exception as e: - conn.rollback() - return jsonify({'error': str(e)}), 500 + # except Exception as e: + # conn.rollback() + # app.logger.error(e) + # return jsonify({'error': str(e)}), 500 finally: cursor.close() conn.close() - return jsonify({'message': 'Answer added successfully!'}), 201 + return jsonify({'message': _('Answer added successfully!')}), 201 -def ntfySend(cw, return_val, from_who, question): +def ntfySend(cw, return_val, from_who, question) -> None: app.logger.debug("[CatAsk/functions/ntfySend] started ntfy flow") ntfy_cw = f" [CW: {cw}]" if cw else "" ntfy_host = cfg['ntfy']['host'] @@ -636,12 +643,13 @@ def createExport(): # Export database to SQL file dump_file = temp_dir / 'database.sql' result = subprocess.Popen( - f'mysqldump --quote-names -u {dbUser} -p{dbPass} {dbName} --result-file={dump_file}', + f'pg_dump -U {dbUser} -d {dbName} -F c -E UTF8 -f {dump_file}', stdin=subprocess.PIPE, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding="utf-8" + encoding="utf-8", + env=dict(os.environ, PGPASSWORD=dbPass) ) # absolutely dumb workaround for an error time.sleep(1) @@ -677,14 +685,14 @@ def createExport(): appendToJSON(export_data, const.exportsFile) shutil.rmtree(temp_dir) - return {'message': 'Export created successfully!'} - except mysql.connector.Error as e: + return {'message': _('Export created successfully!')} + except psycopg.Error as e: return {'error': str(e)}, 500 except Exception as e: return {'error': str(e)}, 500 -def importData(export_file): +def importData(export_file) -> dict: try: shutil.unpack_archive(export_file, const.tempDir) @@ -706,25 +714,23 @@ def importData(export_file): conn = connectToDb() cursor = conn.cursor() - with open(const.tempDir / 'database.sql', 'r') as schema_file: - try: - # for some reason `cursor.execute(schema, multi=True)` doesn't work, so we use this instead - schema = schema_file.read() - queries = schema.split(';') - for query in queries: - cursor.execute(query) - except mysql.connector.Error as e: - return {'error': str(e)}, 500 - finally: - cursor.close() - conn.close() - shutil.rmtree(const.tempDir) + dump_file = const.tempDir / 'database.sql' + process = subprocess.Popen( + f'pg_restore --clean -U {dbUser} -d {dbName} {dump_file}', + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + env=dict(os.environ, PGPASSWORD=dbPass) + ) + shutil.rmtree(const.tempDir) - return {'message': 'Data imported successfully!'} + return {'message': _('Data imported successfully!')} except Exception as e: return {'error': str(e)}, 500 # will probably get to it in 1.8.0 because my brain can't do it rn +# 2.1.0 maybe -1/12/25 """ def retrospringImport(export_file): shutil.unpack_archive(export_file, const.tempDir) diff --git a/requirements.txt b/requirements.txt index a762be0..ad13b40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ flask python-dotenv -mysql-connector-python==8.2.0 +psycopg humanize mistune bleach @@ -9,3 +9,4 @@ Flask-Compress gunicorn pillow requests +Flask-Babel diff --git a/schema.sql b/schema.sql index 8fa32e3..06b72e5 100644 --- a/schema.sql +++ b/schema.sql @@ -1,25 +1,24 @@ CREATE TABLE IF NOT EXISTS answers ( - id INT PRIMARY KEY AUTO_INCREMENT, - question_id INT NOT NULL, - creation_date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - content TEXT NOT NULL, - cw VARCHAR(255) NOT NULL DEFAULT '' -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + id SERIAL PRIMARY KEY, + question_id INTEGER NOT NULL, + creation_date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + content TEXT NOT NULL, + cw VARCHAR(255) NOT NULL DEFAULT '' +); CREATE TABLE IF NOT EXISTS questions ( - id INT PRIMARY KEY AUTO_INCREMENT, - from_who VARCHAR(255) NOT NULL, - creation_date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - content TEXT NOT NULL, - answered BOOLEAN NOT NULL DEFAULT FALSE, - answer_id INT, - pinned BOOLEAN NOT NULL DEFAULT FALSE, - cw VARCHAR(255) NOT NULL DEFAULT '', - unread BOOLEAN NOT NULL DEFAULT TRUE - -- below is reserved for version 1.7.0 or later - -- private BOOLEAN NOT NULL DEFAULT FALSE, - -- user_ip VARBINARY(16) NOT NULL DEFAULT '' -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + id SERIAL PRIMARY KEY, + from_who VARCHAR(255) NOT NULL, + creation_date TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + content TEXT NOT NULL, + answered BOOLEAN NOT NULL DEFAULT FALSE, + answer_id INTEGER, + pinned BOOLEAN NOT NULL DEFAULT FALSE, + cw VARCHAR(255) NOT NULL DEFAULT '', + unread BOOLEAN NOT NULL DEFAULT TRUE + -- private BOOLEAN NOT NULL DEFAULT FALSE, -- For later use + -- user_ip BYTEA NOT NULL DEFAULT '' -- For later use +); ALTER TABLE questions ADD CONSTRAINT fk_answer_id FOREIGN KEY (answer_id) REFERENCES answers(id) ON DELETE CASCADE;