diff --git a/collectoss/api/README.md b/collectoss/api/README.md index af8c4561e..9c71cb40d 100644 --- a/collectoss/api/README.md +++ b/collectoss/api/README.md @@ -19,7 +19,7 @@ So then Gunicorn uses this app to load the server. Note: Those three lines above ### Config -The config located in `collectoss/api/gunicorn_conf.py` loads a default configuration and then if the config table in the augur_operation schema contains gunicorn config values they override the defaults. +The config located in `collectoss/api/gunicorn_conf.py` loads a default configuration and then if the config table in the operations schema contains gunicorn config values they override the defaults. ### Routes diff --git a/collectoss/api/routes/config.py b/collectoss/api/routes/config.py index c6b68a5d1..c648ec5c8 100644 --- a/collectoss/api/routes/config.py +++ b/collectoss/api/routes/config.py @@ -1,21 +1,27 @@ -#SPDX-License-Identifier: MIT +# SPDX-License-Identifier: MIT """ Creates routes for config functionality """ import logging -from flask import request, jsonify, current_app +import httpx + import sqlalchemy as s +from flask import current_app, jsonify, request # Disable the requirement for SSL by setting env["COLLECTOSS_DEV"] = True from collectoss.application.config import get_development_flag -from collectoss.application.db.lib import get_session +from collectoss.application.db.lib import get_session, remove_setting, remove_worker_oauth_key +from collectoss.api.util import ssl_required, admin_required from collectoss.application.db.models import Config from collectoss.application.config import SystemConfig from collectoss.application.db.session import DatabaseSession +from collectoss.application.db.models.operations import WorkerOauth +from keyman.KeyClient import KeyPublisher +from collectoss.tasks.github.util.github_api_key_handler import GithubApiKeyHandler +from collectoss.tasks.gitlab.gitlab_api_key_handler import GitlabApiKeyHandler from ..server import app logger = logging.getLogger(__name__) -development = get_development_flag() from collectoss.api.routes import API_VERSION @@ -28,34 +34,57 @@ def generate_upgrade_request(): return response, 426 @app.route(f"/{API_VERSION}/config/get", methods=['GET', 'POST']) +@ssl_required def get_config(): - if not development and not request.is_secure: - return generate_upgrade_request() - with DatabaseSession(logger, engine=current_app.engine) as session: config_dict = SystemConfig(logger, session).config.load_config() return jsonify(config_dict), 200 +@app.route(f"/{API_VERSION}/config/set", methods=['GET', 'POST']) +@ssl_required +@admin_required +def set_config_item(): + setting = request.args.get("setting") + section = request.args.get("section") + value = request.values.get("value") + + result = { + "section_name": section, + "setting_name": setting, + "value": value + } + + if not setting or not section or not value: + return jsonify({"status": "Missing argument"}), 400 + + with get_session() as session: + config = SystemConfig(logger, session) + config.add_or_update_settings([result]) + + return jsonify({"status": "success"}) @app.route(f"/{API_VERSION}/config/update", methods=['POST']) +@ssl_required def update_config(): - if not development and not request.is_secure: - return generate_upgrade_request() - update_dict = request.get_json() with get_session() as session: - for section, data in update_dict.items(): - for key, value in data.items(): - try: - config_setting = session.query(Config).filter(Config.section_name == section, Config.setting_name == key).one() + config_setting = ( + session.query(Config) + .filter( + Config.section_name == section, Config.setting_name == key + ) + .one() + ) except s.orm.exc.NoResultFound: - return jsonify({"status": "Bad Request", "section": section, "setting": key}), 400 + return jsonify( + {"status": "Bad Request", "section": section, "setting": key} + ), 400 config_setting.value = value @@ -66,3 +95,159 @@ def update_config(): return jsonify({"status": "success"}), 200 +@app.route(f"/{API_VERSION}/workeroauth/get/keys", methods=['GET']) +@ssl_required +@admin_required +def get_oauth_keys(): + """ + Retrieve all worker oauth keys from the configuration table in the database. + The keys from the "Keys" section are normalized and returned such that they follow the format: + { + "github_api_key": "ghp_XXXXXXXXXXXXXXX", + "gitlab_api_key": "glpat_XXXXXXXXXXXXXXX" + } + """ + # Open a database session using the current application engine + with DatabaseSession(logger, engine=current_app.engine) as session: + config = AugurConfig(logger, session) + # Get the Keys section if it exists; otherwise, key list remains empty. + if config.is_section_in_config("Keys"): + keys_section = config.get_section("Keys") + else: + keys_section = {} + + # Normalize the key names (append '_api_key' if needed) + keys_dict = {} + for platform, key_value in keys_section.items(): + platform_lower = platform.lower() + if "github" in platform_lower: + platform_lower = "github" + elif "gitlab" in platform_lower: + platform_lower = "gitlab" + keys_dict[f"{platform_lower}_api_key"] = key_value + + return jsonify(keys_dict), 200 + + +@app.route(f"/{API_VERSION}/workeroauth/get/invalidkeys", methods=["GET"]) +@ssl_required +@admin_required +def get_invalid_keys(): + """ + Retrieve all invalid worker OAuth keys by comparing the keys loaded + in the KeyPublisher at startup with those stored in the database, + and by checking the live keys with the is_bad_api_key function. + + A key is considered valid if it appears in the set of live keys and passes + the is_bad_api_key test; any key that fails is considered invalid. + """ + keypub = KeyPublisher() + + invalid_keys = {} + live_keys = {} + for platform in keypub.list_platforms(): + platform_lower = platform.lower() + tokens = keypub.list_keys(platform) + if tokens is not None: + live_keys[platform_lower] = set(tokens) + + # Instantiate the API key handlers and HTTP client. + ghkeyman = GithubApiKeyHandler(logger) + glkeyman = GitlabApiKeyHandler(logger) + client = httpx.Client() + + github_db_keys = ghkeyman.get_api_keys_from_database() + # For GitHub, we check keys published under both channels. + github_live = live_keys.get("github_rest", set()) | live_keys.get("github_graphql", set()) + for token in github_db_keys: + if token not in github_live or ghkeyman.is_bad_api_key(client, token): + invalid_keys.setdefault("github", []).append(token) + + gitlab_db_keys = glkeyman.get_api_keys_from_database() + gitlab_live = live_keys.get("gitlab_rest", set()) + for token in gitlab_db_keys: + if token not in gitlab_live or glkeyman.is_bad_api_key(client, token): + invalid_keys.setdefault("gitlab", []).append(token) + + # This ensures that even if a key is live, we verify it using is_bad_api_key. + for token in live_keys.get("github_rest", set()): + if ghkeyman.is_bad_api_key(client, token) and token not in invalid_keys.get("github", []): + invalid_keys.setdefault("github", []).append(token) + for token in live_keys.get("github_graphql", set()): + if ghkeyman.is_bad_api_key(client, token) and token not in invalid_keys.get("github", []): + invalid_keys.setdefault("github", []).append(token) + for token in live_keys.get("gitlab_rest", set()): + if glkeyman.is_bad_api_key(client, token) and token not in invalid_keys.get("gitlab", []): + invalid_keys.setdefault("gitlab", []).append(token) + + for platform, tokens in invalid_keys.items(): + invalid_keys[platform] = [{"id": token, "token": token} for token in tokens] + + return jsonify(invalid_keys), 200 + + + +@app.route(f"/{API_VERSION}/workeroauth/delete/key", methods=["POST"]) +@ssl_required +@admin_required +def delete_oauth_key(): + """ + Delete a worker oauth key from the KeyPublisher, config table and worker oauth table + Expects a JSON payload with the platform and token properties. + """ + data = request.get_json() + + if not data or "platform" not in data or "token" not in data: + return jsonify( + {"status": "Bad Request", "message": "Missing platform or token"} + ), 400 + + keypub = KeyPublisher() + + platform = data.get("platform").lower() + if platform == "github": + keypub.unpublish(data["token"], "github_rest") + keypub.unpublish(data["token"], "github_graphql") + elif platform == "gitlab": + keypub.unpublish(data["token"], "gitlab_rest") + + remove_worker_oauth_key(platform=data["platform"].lower()) + remove_setting(section_name="Keys", setting_name=data["platform"].lower() + "_api_key") + + return jsonify({"status": "success"}), 200 + + + +@app.route(f"/{API_VERSION}/workeroauth/new/keys", methods=["POST"]) +@ssl_required +@admin_required +def new_oauth_keys(): + """ + Add new worker oauth keys to the KeyPublisher. + Expects a JSON payload with the platform and token properties. + """ + data = request.get_json() + + if not data or "platform" not in data or "token" not in data: + return jsonify( + {"status": "Bad Request", "message": "Missing platform or token"} + ), 400 + + ghkeyman = GithubApiKeyHandler(logger) + glkeyman = GitlabApiKeyHandler(logger) + + keypub = KeyPublisher() + client = httpx.Client() + + if data.get("platform") == "github": + if ghkeyman.is_bad_api_key(client, data["token"]): + return jsonify({"status": "Bad Request", "message": "Invalid GitHub API key"}), 400 + keypub.publish(data["token"], "github_rest") + keypub.publish(data["token"], "github_graphql") + + elif data.get("platform") == "gitlab": + if glkeyman.is_bad_api_key(client, data["token"]): + return jsonify({"status": "Bad Request", "message": "Invalid GitLab API key"}), 400 + keypub.publish(data["token"], "gitlab_rest") + + return jsonify({"status": "success"}), 200 diff --git a/collectoss/api/ssl/README.md b/collectoss/api/ssl/README.md index 10c3c9597..dfc3df7b8 100644 --- a/collectoss/api/ssl/README.md +++ b/collectoss/api/ssl/README.md @@ -22,9 +22,3 @@ Let's Encrypt/Certbot ~~~~~~~~~~~~~~~~~~~~~ The easiest way to get an HTTPS server up is to make use of `Let's Encrypt `_'s `Certbot `_ tool. It is an open source tool that is so good it will even alter the nginx configuration for you automatically to enable HTTPS. Following their guide for ``Ubuntu 20.04``, run ``sudo snap install --classic certbot``, ``sudo ln -s /snap/bin/certbot /usr/bin/certbot``, and then ``sudo certbot --nginx``. - -~~~~~~~~~~~~~~~~~~~~ -Fixing the Backend -~~~~~~~~~~~~~~~~~~~~ - -Now our server is configured properly and our frontend is being served over HTTPS, but there's an extra problem: the backend APIs are still being served over HTTP resulting in a ``blocked loading mixed active content`` error. This issue is currently being looked into by our developers. Some files that are candidates for causing issues here are ``collectoss/application.py``, ``frontend/src/AugurAPI.ts``, and ``frontend/src/router.ts``. diff --git a/collectoss/api/util.py b/collectoss/api/util.py index f2dea539b..44aaadf3d 100644 --- a/collectoss/api/util.py +++ b/collectoss/api/util.py @@ -6,7 +6,7 @@ import re import beaker -from flask import request, jsonify, current_app +from flask import request, jsonify, current_app, abort from collectoss.application.db import get_session from functools import wraps @@ -14,6 +14,8 @@ from collectoss.application.config import get_development_flag from collectoss.application.db.models import ClientApplication +from flask_login import login_required, current_user + development = get_development_flag() __ROOT = os.path.abspath(os.path.dirname(__file__)) @@ -112,7 +114,6 @@ def get_client_token(): return token - # usage: """ @app.route("/path") @@ -155,4 +156,22 @@ def wrapper(*args, **kwargs): return generate_upgrade_request() return fun(*args, **kwargs) - return wrapper \ No newline at end of file + return wrapper + +def admin_required(func): + @login_required + @wraps(func) + def inner_function(*args, **kwargs): + if current_user.admin: + return func(*args, **kwargs) + else: + abort(403) + return inner_function + +def development_required(func): + @wraps(func) + def inner_function(*args, **kwargs): + if not development: + abort(403) + return func(*args, **kwargs) + return inner_function \ No newline at end of file diff --git a/collectoss/api/view/api_view.py b/collectoss/api/view/api_view.py index b2f5a2925..0eccef7b2 100644 --- a/collectoss/api/view/api_view.py +++ b/collectoss/api/view/api_view.py @@ -1,10 +1,12 @@ from flask import render_template, redirect, url_for, session, request, jsonify -from flask_login import LoginManager +from flask_login import LoginManager, current_user, login_required from io import StringIO from .utils import * from .init import logger from .url_converters import * +from functools import wraps + # from .server import User from ..server import app, db_session from collectoss.application.db.models import User, UserSessionToken @@ -38,6 +40,13 @@ def unsupported_method(error): return render_message("405 - Method not supported", "The resource you are trying to access does not support the request method used"), 405 +@app.errorhandler(403) +def forbidden(error): + if API_VERSION in str(request.url_rule): + return jsonify({"status": "Forbidden"}), 403 + + return render_message("403 - Forbidden", "You do not have permission to view this page"), 403 + @app.errorhandler(500) def internal_server_error(error): if API_VERSION in str(request.path): @@ -52,8 +61,21 @@ def internal_server_error(error): errout.close() except Exception as e: logger.error(e) + raise e - return render_message("500 - Internal Server Error", "An error occurred while trying to service your request. Please try again, and if the issue persists, please file a GitHub issue with the below error message:", error=stacktrace), 500 + return render_message("500 - Internal Server Error", """An error occurred while trying to service your request. + Please try again, and if the error persists, please file a GitHub issue with a description + of what you were doing before this error occurred accompanied by the below error message:""", error=stacktrace), 500 + +@app.template_filter("escape_ID") +def escape_HTML_ID(data: str) -> str: + # Done this way in case we want to add more replacements in the future + data = data.replace(".", "\\.") + return data + +@app.template_filter("quoted") +def quote_surrounded(data: str) -> str: + return '"' + data + '"' @login_manager.unauthorized_handler def unauthorized(): @@ -98,19 +120,16 @@ def load_user(user_id): @login_manager.request_loader def load_user_request(request): token = get_bearer_token() - current_time = int(time.time()) - token = db_session.query(UserSessionToken).filter(UserSessionToken.token == token, UserSessionToken.expiration >= current_time).first() - if token: - print("Valid user") + token = db_session.query(UserSessionToken).filter(UserSessionToken.token == token, UserSessionToken.expiration >= current_time).first() + if token: user = token.user user._is_authenticated = True user._is_active = True - return user - + return None @app.template_filter('as_datetime') diff --git a/collectoss/api/view/routes.py b/collectoss/api/view/routes.py index 15ab991b3..3d4776a70 100644 --- a/collectoss/api/view/routes.py +++ b/collectoss/api/view/routes.py @@ -3,16 +3,23 @@ """ import logging import math +import os +import signal from flask import render_template, request, redirect, url_for, session, flash from .utils import * +from collectoss.api.util import admin_required, development_required from flask_login import login_user, logout_user, current_user, login_required +from sqlalchemy.exc import OperationalError from collectoss.application.db.models import User, Repo, ClientApplication from .server import LoginException from collectoss.application.util import * from collectoss.application.db.lib import get_value +from collectoss.application.config import SystemConfig from ..server import app, db_session +from collectoss.application.db.lib import get_session + logger = logging.getLogger(__name__) @@ -318,6 +325,7 @@ def user_group_view(group = None): return render_module("user-group-repos-table", title="Repos", repos=data, query_key=query, activePage=params["page"], pages=page_count, offset=pagination_offset, PS="user_group_view", reverse = rev, sorting = params.get("sort"), group=group) @app.route('/error') +@development_required def throw_exception(): raise Exception("This Exception intentionally raised") @@ -326,6 +334,7 @@ def throw_exception(): View the admin dashboard. """ @app.route('/dashboard') +@admin_required def dashboard_view(): empty = [ { "title": "Placeholder", "settings": [ @@ -337,6 +346,14 @@ def dashboard_view(): ]} ] - backend_config = requestJson("config/get", False) + backend_config = SystemConfig(logger, db_session).load_config() + + with get_session() as session: + try: + users = session.query(User).all() + except OperationalError as e: + # Instruct Gunicorn to reboot workers to resolve database connection instability + os.kill(os.getpid(), signal.SIGTERM) + return "reloading" - return render_template('admin-dashboard.j2', sections = empty, config = backend_config) + return render_template('admin-dashboard.j2', sections = empty, config = backend_config, users = users) diff --git a/collectoss/application/cli/config.py b/collectoss/application/cli/config.py index 681c9d201..e1b0a0cf2 100644 --- a/collectoss/application/cli/config.py +++ b/collectoss/application/cli/config.py @@ -20,7 +20,7 @@ def get_transitional_envs(name: str) -> list: return [ENVVAR_PREFIX + name, "AUGUR_" + name] -@click.group('config', short_help='Generate an augur.config.json') +@click.group('config', short_help='Generate a collectoss.config.json') @click.pass_context def cli(ctx): ctx.obj = DatabaseContext() diff --git a/collectoss/application/db/lib.py b/collectoss/application/db/lib.py index 719b97c4c..6dc756ecc 100644 --- a/collectoss/application/db/lib.py +++ b/collectoss/application/db/lib.py @@ -10,7 +10,7 @@ from typing_extensions import deprecated from collectoss.application.db.models import Config, Repo, Commit, WorkerOauth, Issue, PullRequest, PullRequestReview, ContributorsAlias,UnresolvedCommitEmail, Contributor, CollectionStatus, UserGroup, RepoGroup -# TODO: CollectionState should be moved to augur/application/db/ to eliminate +# TODO: CollectionState should be moved to collectoss/application/db/ to eliminate # this cross-layer dependency — same issue as the correction.py import above. from collectoss.tasks.util.collection_state import CollectionState from collectoss.application.db.timestamp_utils import correct_timestamp @@ -20,7 +20,7 @@ logger = logging.getLogger("db_lib") -@deprecated("This is a legacy method. Use AugurConfig.get_value instead") +@deprecated("This is a legacy method. Use SystemConfig.get_value instead") def get_value(section_name: str, setting_name: str) -> Optional[Any]: """Get the value of a setting from the config. @@ -146,6 +146,37 @@ def get_gitlab_repo_by_src_id(src_id): repo = execute_session_query(query, 'first') return repo + +def remove_setting(section_name: str, setting_name: str) -> bool: + """Remove a setting from the config. + + Args: + section_name: The name of the section that the setting belongs to + setting_name: The name of the setting to remove + + Returns: + True if the setting was successfully removed, False if it wasn't found + """ + with get_session() as session: + try: + query = session.query(Config).filter( + Config.section_name == section_name, + Config.setting_name == setting_name + ) + config_setting = execute_session_query(query, 'one') + + session.delete(config_setting) + session.commit() + logger.info(f"Removed setting '{setting_name}' from section '{section_name}'") + return True + + except s.orm.exc.NoResultFound: + logger.warning(f"Setting '{setting_name}' in section '{section_name}' not found") + return False + except Exception as e: + logger.error(f"Error removing setting '{setting_name}' from section '{section_name}': {e}") + session.rollback() + return False def remove_working_commits_by_repo_id_and_hashes(repo_id, commit_hashes): @@ -210,6 +241,18 @@ def get_worker_oauth_keys(platform: str): return [row.access_token for row in results] +def remove_worker_oauth_key(platform: str): + with get_session() as session: + + results = session.query(WorkerOauth).filter(WorkerOauth.platform == platform).order_by(func.random()).all() + + for row in results: + session.delete(row) + + session.commit() + + return [row.access_token for row in results] + def get_active_repo_count(collection_type): with get_session() as session: diff --git a/collectoss/static/css/dashboard.css b/collectoss/static/css/dashboard.css index ef111c32a..c89815488 100644 --- a/collectoss/static/css/dashboard.css +++ b/collectoss/static/css/dashboard.css @@ -1,7 +1,9 @@ :root { --color-bg: #1A233A; --color-bg-light: #272E48; + --color-bg-contrast: #646683; --color-fg: white; + --color-fg-dark: #b0bdd6; --color-fg-contrast: black; --color-accent: #6f42c1; --color-accent-dark: #6134b3; @@ -25,7 +27,35 @@ body { margin-bottom: 10px; } -.nav-pills .nav-link.active, .nav-pills .show > .nav-link { +.input-textbox { + color: var(--color-fg); + background-color: var(--color-bg); + border-color: var(--color-accent-dark); +} + +.input-group-text { + color: var(--color-fg); + background-color: var(--color-bg-light); + border-color: var(--color-accent-dark); + border-right: none; +} + +.input-textbox::placeholder { + color: var(--color-fg-dark); +} + +.input-textbox:focus { + color: var(--color-fg); + background-color: var(--color-bg); + border-color: var(--color-accent-dark); +} + +.input-textbox:focus::placeholder { + color: var(--color-fg-dark); +} + +.nav-pills .nav-link.active, +.nav-pills .show>.nav-link { background-color: var(--color-accent) } @@ -52,6 +82,10 @@ body { padding-right: 10px !important; } +.modal-dialog { + color: var(--color-fg-contrast); +} + .dashboard-form-control { border: 1px solid #596280; -webkit-border-radius: 2px; @@ -62,17 +96,59 @@ body { color: #bcd0f7; } +.contrast-card { + background-color: var(--color-bg); +} + +.card:has(.contrast-card) { + border: none; +} + +.accordion-item { + background-color: var(--color-bg-light); + color: var(--color-fg); +} + +.accordion-button { + background-color: var(--color-accent-dark); + color: var(--color-fg); +} + +.accordion-button:not(.collapsed) { + background-color: var(--color-accent); + color: var(--color-fg); +} + +.accordion-button::after { + filter: saturate(0%) brightness(10); +} + +.accordion-button:not(.collapsed)::after { + filter: saturate(0%) brightness(10); +} + +.accordion-button:focus { + box-shadow: none; + border-color: var(--color-accent-dark); +} + .circle-opaque { - border-radius: 50%; /* Make it a circle */ - display: inline-block; - position: absolute; /* Able to position it, overlaying the other image */ - left:0px; /* Customise the position, but make sure it */ - top:0px; /* is the same as .circle-transparent */ - z-index: -1; /* Makes the image sit *behind* .circle-transparent */ + border-radius: 50%; + /* Make it a circle */ + display: inline-block; + position: absolute; + /* Able to position it, overlaying the other image */ + left: 0px; + /* Customise the position, but make sure it */ + top: 0px; + /* is the same as .circle-transparent */ + z-index: -1; + /* Makes the image sit *behind* .circle-transparent */ } .circle-opaque img { - border-radius: 50%; /* Make it a circle */ + border-radius: 50%; + /* Make it a circle */ z-index: -1; } @@ -95,4 +171,47 @@ table { #toast-placeholder { display: none; z-index: 100; +} + +@-webkit-keyframes rotating + +/* Safari and Chrome */ + { + from { + -webkit-transform: rotate(0deg); + -o-transform: rotate(0deg); + transform: rotate(0deg); + } + + to { + -webkit-transform: rotate(360deg); + -o-transform: rotate(360deg); + transform: rotate(360deg); + } +} + +@keyframes rotating { + from { + -ms-transform: rotate(0deg); + -moz-transform: rotate(0deg); + -webkit-transform: rotate(0deg); + -o-transform: rotate(0deg); + transform: rotate(0deg); + } + + to { + -ms-transform: rotate(360deg); + -moz-transform: rotate(360deg); + -webkit-transform: rotate(360deg); + -o-transform: rotate(360deg); + transform: rotate(360deg); + } +} + +.rotating { + -webkit-animation: rotating 1s linear infinite; + -moz-animation: rotating 1s linear infinite; + -ms-animation: rotating 1s linear infinite; + -o-animation: rotating 1s linear infinite; + animation: rotating 1s linear infinite; } \ No newline at end of file diff --git a/collectoss/templates/admin-dashboard.j2 b/collectoss/templates/admin-dashboard.j2 index ee547b46d..dbb083789 100644 --- a/collectoss/templates/admin-dashboard.j2 +++ b/collectoss/templates/admin-dashboard.j2 @@ -1,13 +1,16 @@ + - - - - + + + @@ -15,164 +18,619 @@ - + + Dasboard - CollectOSS View - + - -
-
-
-
- Dashboard -
-
- -
-