diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..f899050 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "shared_migrations"] + path = shared_migrations + url = git@github.com:Code4GovTech/shared-models-migrations.git diff --git a/app.log b/app.log new file mode 100644 index 0000000..e69de29 diff --git a/app.py b/app.py index fd6c755..9a1ca7e 100644 --- a/app.py +++ b/app.py @@ -7,7 +7,7 @@ from utils.dispatcher import dispatch_event from utils.link_pr_issue import AddIssueId from utils.webhook_auth import verify_github_webhook -from utils.db import SupabaseInterface,PostgresORM +# from utils.db import SupabaseInterface,PostgresORM from events.ticketEventHandler import TicketEventHandler from events.ticketFeedbackHandler import TicketFeedbackHandler from githubdatapipeline.pull_request.scraper import getNewPRs @@ -24,6 +24,7 @@ from quart_cors import cors from utils.migrate_tickets import MigrateTickets from utils.migrate_users import MigrateContributors +from shared_migrations.db.server import ServerQueries scheduler = AsyncIOScheduler() @@ -78,7 +79,7 @@ async def get_github_data(code, discord_id): async def comment_cleaner(): while True: await asyncio.sleep(5) - comments = await PostgresORM().readAll("app_comments") + comments = await ServerQueries().readAll("app_comments") for comment in comments: utc_now = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc) update_time = dateutil.parser.parse(comment["updated_at"]) @@ -90,7 +91,7 @@ async def comment_cleaner(): issue_id = comment["issue_id"] comment = await TicketFeedbackHandler().deleteComment(owner, repo, comment_id) print(f"Print Delete Task,{comment}", file=sys.stderr) - print(await PostgresORM().deleteComment(issue_id,"app_comments")) + print(await ServerQueries().deleteComment(issue_id,"app_comments")) async def fetch_github_issues_from_repo(owner, repo): try: @@ -162,7 +163,7 @@ async def verify(githubUsername): @app.route("/misc_actions") async def addIssues(): - tickets = await PostgresORM().readAll("ccbp_tickets") + tickets = await ServerQueries().readAll("ccbp_tickets") count =1 for ticket in tickets: print(f'{count}/{len(tickets)}') @@ -178,7 +179,7 @@ async def addIssues(): @app.route("/update_profile", methods=["POST"]) async def updateGithubStats(): webhook_data = await request.json - data = await PostgresORM().read("github_profile_data", filters={"dpg_points": ("gt", 0)}) + data = await ServerQueries().read("github_profile_data", filters={"dpg_points": ("gt", 0)}) GithubProfileDisplay().update(data) return 'Done' @@ -189,7 +190,7 @@ async def do_update(): while True: print("Starting Update") await asyncio.sleep(21600) - data = await PostgresORM().read("github_profile_data", filters={"dpg_points": ("gt", 0)}) + data = await ServerQueries().read("github_profile_data", filters={"dpg_points": ("gt", 0)}) GithubProfileDisplay().update(data) @@ -219,7 +220,7 @@ async def test(): @app.route("/register/") async def register(discord_userdata): print("🛠️SUCCESSFULLY REDIECTED FROM GITHUB TO SERVER", locals(), file=sys.stderr) - postgres_client = PostgresORM() + postgres_client = ServerQueries() discord_id = discord_userdata print("🛠️SUCCESFULLY DEFINED FUNCTION TO POST TO SUPABASE", locals(), file=sys.stderr) @@ -250,7 +251,7 @@ async def event_handler(): verification_result, error_message = await verify_github_webhook(request,secret_key) - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries() event_type = request.headers.get("X-GitHub-Event") await dispatch_event(event_type, data, postgres_client) @@ -275,7 +276,7 @@ async def discord_metrics(): } discord_data.append(data) - data = await PostgresORM().add_discord_metrics(discord_data) + data = await ServerQueries().add_discord_metrics(discord_data) return data @app.route("/metrics/github", methods = ['POST']) @@ -294,12 +295,12 @@ async def github_metrics(): } github_data.append(data) - data = await PostgresORM().add_github_metrics(github_data) + data = await ServerQueries().add_github_metrics(github_data) return data @app.route("/role-master") async def get_role_master(): - role_masters = await PostgresORM().readAll("role_master") + role_masters = await ServerQueries().readAll("role_master") print('role master ', role_masters) return role_masters.data @@ -311,7 +312,7 @@ async def get_program_tickets_user(): filter = '' if request_data: filter = json.loads(request_data.decode('utf-8')) - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries() all_issues = await postgres_client.fetch_filtered_issues(filter) print('length of all issue ', len(all_issues)) diff --git a/events/ticketEventHandler.py b/events/ticketEventHandler.py index afe24e5..1b28c82 100644 --- a/events/ticketEventHandler.py +++ b/events/ticketEventHandler.py @@ -3,7 +3,8 @@ import aiohttp import os, sys, datetime, json -from utils.db import PostgresORM +# from utils.db import PostgresORM +from shared_migrations.db.server import ServerQueries from utils.markdown_handler import MarkdownHeaders from utils.github_api import GithubAPI from utils.jwt_generator import GenerateJWT @@ -80,8 +81,8 @@ def matchProduct(enteredProductName): async def send_message(ticket_data): - discord_channels = await PostgresORM().readAll("discord_channels") - products = await PostgresORM().readAll("product") + discord_channels = await ServerQueries().readAll("discord_channels") + products = await ServerQueries().readAll("product") url = None # for product in products: @@ -139,7 +140,7 @@ async def get_pull_request(owner, repo, number): class TicketEventHandler: def __init__(self): - self.postgres_client = PostgresORM() + self.postgres_client = ServerQueries() self.ticket_points = { "hard":30, "easy":10, @@ -255,11 +256,11 @@ async def onTicketCreate(self, eventData): repo = url_components[-3] owner = url_components[-4] try: - await PostgresORM().add_data({"issue_id":issue["id"],"updated_at": datetime.utcnow().isoformat()},"app_comments") + await self.postgres_client.add_data({"issue_id":issue["id"],"updated_at": datetime.utcnow().isoformat()},"app_comments") comment = await TicketFeedbackHandler().createComment(owner, repo, issue_number, markdown_contents) if comment: - await PostgresORM().update_data({ + await self.postgres_client.update_data({ "api_url":comment["url"], "comment_id":comment["id"], "issue_id":issue["id"], @@ -343,17 +344,17 @@ async def onTicketEdit(self, eventData): if added_contributor: print('contributors data added') - if await PostgresORM().check_record_exists("app_comments","issue_id",issue["id"]) and ticketType=="ccbp": + if await self.postgres_client.check_record_exists("app_comments","issue_id",issue["id"]) and ticketType=="ccbp": url_components = issue["url"].split('/') repo = url_components[-3] owner = url_components[-4] - comments = await PostgresORM().get_data("issue_id","app_comments",issue["id"],None) + comments = await self.postgres_client.get_data("issue_id","app_comments",issue["id"],None) comment_id = comments[0]["comment_id"] if TicketFeedbackHandler().evaluateDict(markdown_contents): comment = await TicketFeedbackHandler().updateComment(owner, repo, comment_id, markdown_contents) if comment: - await PostgresORM.get_instance().update_data({ + await self.postgres_client.update_data({ "updated_at": datetime.utcnow().isoformat(), "issue_id": issue["id"] },"issue_id","app_comments") @@ -361,7 +362,7 @@ async def onTicketEdit(self, eventData): try: comment = await TicketFeedbackHandler().deleteComment(owner, repo, comment_id) print(f"Print Delete Task,{comment}", file=sys.stderr) - print(await PostgresORM.get_instance().deleteComment(issue["id"],"app_comments")) + print(await self.postgres_client.deleteComment(issue["id"],"app_comments")) except: print("Error in deletion") elif ticketType=="ccbp": @@ -373,14 +374,14 @@ async def onTicketEdit(self, eventData): try: - await PostgresORM().add_data({ + await self.postgres_client.add_data({ "issue_id":issue["id"], "updated_at": datetime.utcnow().isoformat() },"app_comments") comment = await TicketFeedbackHandler().createComment(owner, repo, issue_number, markdown_contents) if comment: - await PostgresORM().update_data({ + await self.postgres_client.update_data({ "api_url":comment["url"], "comment_id":comment["id"], "issue_id":issue["id"], diff --git a/handlers/issue_comment_handler.py b/handlers/issue_comment_handler.py index 488ce7c..7b2718e 100644 --- a/handlers/issue_comment_handler.py +++ b/handlers/issue_comment_handler.py @@ -3,8 +3,11 @@ from utils.logging_file import logger import logging from utils.user_activity import UserActivity +from shared_migrations.db.server import ServerQueries class Issue_commentHandler(EventHandler): + def __init__(self): + self.postgres_client = ServerQueries() async def handle_event(self, data, postgres_client): try: @@ -16,7 +19,7 @@ async def handle_event(self, data, postgres_client): if next((l for l in labels if l['name'].lower() == 'c4gt community'), None): handler_method = getattr(self, f'handle_issue_comment_{module_name}', None) if handler_method: - await handler_method(data, postgres_client) + await handler_method(data) await UserActivity.log_user_activity(data, 'comment') else: logging.info(f"No handler found for module: {module_name}") @@ -28,7 +31,7 @@ async def handle_event(self, data, postgres_client): logging.info(e) raise Exception - async def handle_issue_comment_created(self, data, postgres_client): + async def handle_issue_comment_created(self, data): try: #generate sample dict for ticket comment table print(f'creating comment with {data["issue"]}') @@ -52,7 +55,7 @@ async def handle_issue_comment_created(self, data, postgres_client): print('comments data ', comment_data) - save_data = await postgres_client.add_data(comment_data,"ticket_comments") + save_data = await self.postgres_client.add_data(comment_data,"ticket_comments") print('saved data in comments created ', save_data) if save_data == None: logger.info(f"{datetime.now()}--- Failed to save data in ticket_comments") @@ -62,7 +65,7 @@ async def handle_issue_comment_created(self, data, postgres_client): raise Exception - async def handle_issue_comment_edited(self, data, postgres_client): + async def handle_issue_comment_edited(self, data): try: #generate sample dict for ticket comment table print(f'editing comment with {data["issue"]}') @@ -72,7 +75,7 @@ async def handle_issue_comment_edited(self, data, postgres_client): 'updated_at':str(datetime.now()) } - save_data = await postgres_client.update_data(comment_data, "id", "ticket_comments") + save_data = await self.postgres_client.update_data(comment_data, "id", "ticket_comments") print('saved data in comments edited ', save_data) if save_data == None: logger.info(f"{datetime.now()}--- Failed to save data in ticket_comments") @@ -81,12 +84,12 @@ async def handle_issue_comment_edited(self, data, postgres_client): logger.info(f"{datetime.now()}---{e}") raise Exception - async def handle_issue_comment_deleted(self, data, postgres_client): + async def handle_issue_comment_deleted(self, data): try: print(f'deleting comment with {data["issue"]}') comment_id = data['comment']['id'] # data = await postgres_client.deleteIssueComment(comment_id) - await postgres_client.delete("ticket_comments","id", comment_id) + await self.postgres_client.delete("ticket_comments","id", comment_id) print('data in comment deleted', data) except Exception as e: print('Exception occured ', e) diff --git a/handlers/issues_handler.py b/handlers/issues_handler.py index 93b89c0..c57fa29 100644 --- a/handlers/issues_handler.py +++ b/handlers/issues_handler.py @@ -2,8 +2,9 @@ import json from handlers.EventHandler import EventHandler from events.ticketEventHandler import TicketEventHandler -from utils.db import PostgresORM +# from utils.db import PostgresORM from utils.user_activity import UserActivity +from shared_migrations.db.server import ServerQueries class IssuesHandler(EventHandler): async def handle_event(self, data, postgres_client): @@ -40,7 +41,7 @@ async def handle_event(self, data, postgres_client): async def handle_issue_created(self, data): # Implement your logic for handling issue events here try: - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries if data.get("issue"): issue = data["issue"] print('inside issue created with', issue) @@ -56,7 +57,7 @@ async def handle_issue_created(self, data): async def handle_issue_opened(self, data): # Implement your logic for handling issue events here try: - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries if data.get("issue"): issue = data["issue"] print('inside issue opened with', issue) @@ -69,7 +70,7 @@ async def handle_issue_opened(self, data): async def handle_issue_labeled(self, data): try: print(json.dumps(data, indent=4)) - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries issue = data["issue"] print('inside issue labeled with', issue) db_issue = await postgres_client.get_data('id', 'issues', issue["id"]) @@ -90,7 +91,7 @@ async def handle_issue_labeled(self, data): async def handle_issue_unlabeled(self, data): try: - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries if data["action"] == "unlabeled": issue = data["issue"] db_issue = await postgres_client.get_issue_from_issue_id(issue["id"]) @@ -112,7 +113,7 @@ async def handle_issue_unlabeled(self, data): async def handle_issue_edited(self, data): try: - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries print(json.dumps(data, indent=4)) issue = data["issue"] print('inside issue edited with', issue) @@ -134,7 +135,7 @@ async def handle_issue_edited(self, data): async def handle_issue_closed(self, data): try: - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries issue = data["issue"] print('inside issue closed with', issue) issue_exist = await postgres_client.get_data('issue_id', 'issues', issue["id"]) @@ -148,7 +149,7 @@ async def handle_issue_closed(self, data): async def handle_issue_assigned(self, data): try: - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries issue = data["issue"] print('inside issue closed with', issue) @@ -160,7 +161,7 @@ async def handle_issue_assigned(self, data): async def handle_issue_unassigned(self, data): try: - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries issue = data["issue"] db_issue = await postgres_client.get_issue_from_issue_id(issue["id"]) print('db issue in unlabeled is ', db_issue) @@ -173,7 +174,7 @@ async def handle_issue_unassigned(self, data): async def log_user_activity(self, data): try: - postgres_client = PostgresORM.get_instance() + postgres_client = ServerQueries issue = data["issue"] print('inside user activity', issue) issue = await postgres_client.get_data('issue_id', 'issues', issue["id"]) diff --git a/handlers/pull_request_handler.py b/handlers/pull_request_handler.py index d7d8f22..3942022 100644 --- a/handlers/pull_request_handler.py +++ b/handlers/pull_request_handler.py @@ -2,8 +2,11 @@ from handlers.EventHandler import EventHandler from datetime import datetime from utils.user_activity import UserActivity +from shared_migrations.db.server import ServerQueries class Pull_requestHandler(EventHandler): + def __init__(self): + self.postgres_client = ServerQueries() def convert_to_datetime(self, date_str): return datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%SZ') @@ -88,11 +91,11 @@ async def handle_event(self, data, postgres_client): print('PR data ', pr_data) - pr_exist = await postgres_client.get_data('pr_id', 'pr_history', data['pull_request']['id']) + pr_exist = await self.postgres_client.get_data('pr_id', 'pr_history', data['pull_request']['id']) if pr_exist: - save_data = await postgres_client.update_pr_history(pr_data["pr_id"],pr_data) + save_data = await self.postgres_client.update_pr_history(pr_data["pr_id"],pr_data) else: - save_data = await postgres_client.add_data(pr_data,"pr_history") + save_data = await self.postgres_client.add_data(pr_data,"pr_history") print('saved data in PR ', save_data) if save_data == None: logging.info("Failed to save data in pr_history") @@ -100,14 +103,14 @@ async def handle_event(self, data, postgres_client): user_id = data['pull_request']['user']['id'] #get contributor_id and save to supabase - contributor = await postgres_client.get_data('github_id', 'contributors_registration', user_id) + contributor = await self.postgres_client.get_data('github_id', 'contributors_registration', user_id) if not contributor: print('could not add contributors data contributor does not exist') return pr_data contributor_id = contributor[0]["id"] issue_url = data['pull_request']['issue_url'] - issue = await postgres_client.get_data('link', 'issues', issue_url, '*') + issue = await self.postgres_client.get_data('link', 'issues', issue_url, '*') #save activity to user_activity await UserActivity.log_user_activity(data, 'pull_request') diff --git a/shared_migrations b/shared_migrations new file mode 160000 index 0000000..8ca6de8 --- /dev/null +++ b/shared_migrations @@ -0,0 +1 @@ +Subproject commit 8ca6de83f1ab522e124652c6d0fd7f3e09ce088a diff --git a/utils/migrate_users.py b/utils/migrate_users.py index 0096646..5f548a9 100644 --- a/utils/migrate_users.py +++ b/utils/migrate_users.py @@ -1,9 +1,10 @@ from utils.db import PostgresORM, SupabaseInterface from datetime import datetime +from shared_migrations.db.server import ServerQueries class MigrateContributors: def __init__(self): - self.postgres_client = PostgresORM.get_instance() + self.postgres_client = ServerQueries() self.supabase_client = SupabaseInterface.get_instance() return diff --git a/utils/user_activity.py b/utils/user_activity.py index bdc5907..90f1b43 100644 --- a/utils/user_activity.py +++ b/utils/user_activity.py @@ -1,19 +1,24 @@ import logging from utils.db import PostgresORM +from shared_migrations.db.server import ServerQueries class UserActivity: - async def log_user_activity(data, activity): + def __init__(self): + self.postgres_client = ServerQueries() + return + + async def log_user_activity(self, data, activity): try: - postgres_client = PostgresORM.get_instance() + issue = data["issue"] print('inside user activity', issue) - issue = await postgres_client.get_data('issue_id', 'issues', issue["id"]) + issue = await self.postgres_client.get_data('issue_id', 'issues', issue["id"]) user_id = data['issue']['user']['id'] - contributor = await postgres_client.get_data('github_id', 'contributors_registration', user_id, '*') + contributor = await self.postgres_client.get_data('github_id', 'contributors_registration', user_id, '*') contributor_id = contributor[0]["id"] - mentor = await postgres_client.get_data('issue_id', 'issue_mentors',issue[0]["id"]) + mentor = await self.postgres_client.get_data('issue_id', 'issue_mentors',issue[0]["id"]) activity_data = { "issue_id": issue[0]["id"], "activity": f"{activity}_{data['action']}", @@ -22,7 +27,7 @@ async def log_user_activity(data, activity): "contributor_id": contributor_id, "mentor_id": mentor[0]["angel_mentor_id"] if mentor else None } - saved_activity_data = await postgres_client.add_data(activity_data,"user_activity") + saved_activity_data = await self.postgres_client.add_data(activity_data,"user_activity") return saved_activity_data except Exception as e: