Admin changes rebased

collectoss/api/README.md

@@ -19,7 +19,7 @@ So then Gunicorn uses this app to load the server. Note: Those three lines above
 
 ### Config
 
-The config located in `collectoss/api/gunicorn_conf.py` loads a default configuration and then if the config table in the augur_operation schema contains gunicorn config values they override the defaults. 
+The config located in `collectoss/api/gunicorn_conf.py` loads a default configuration and then if the config table in the operations schema contains gunicorn config values they override the defaults. 
 
 ### Routes
 

collectoss/api/routes/config.py

@@ -1,21 +1,27 @@
-#SPDX-License-Identifier: MIT
+# SPDX-License-Identifier: MIT
 """
 Creates routes for config functionality
 """
 import logging
-from flask import request, jsonify, current_app
+import httpx
+
 import sqlalchemy as s
+from flask import current_app, jsonify, request
 
 # Disable the requirement for SSL by setting env["COLLECTOSS_DEV"] = True
 from collectoss.application.config import get_development_flag
-from collectoss.application.db.lib import get_session
+from collectoss.application.db.lib import get_session, remove_setting, remove_worker_oauth_key
+from collectoss.api.util import ssl_required, admin_required
 from collectoss.application.db.models import Config
 from collectoss.application.config import SystemConfig
 from collectoss.application.db.session import DatabaseSession
+from collectoss.application.db.models.operations import WorkerOauth
+from keyman.KeyClient import KeyPublisher
+from collectoss.tasks.github.util.github_api_key_handler import GithubApiKeyHandler
+from collectoss.tasks.gitlab.gitlab_api_key_handler import GitlabApiKeyHandler
 from ..server import app
 
 logger = logging.getLogger(__name__)
-development = get_development_flag()
 
 from collectoss.api.routes import API_VERSION
 
@@ -28,34 +34,57 @@ def generate_upgrade_request():
     return response, 426
 
 @app.route(f"/{API_VERSION}/config/get", methods=['GET', 'POST'])
+@ssl_required
 def get_config():
-    if not development and not request.is_secure:
-        return generate_upgrade_request()
-
     with DatabaseSession(logger, engine=current_app.engine) as session:
 
         config_dict = SystemConfig(logger, session).config.load_config()
 
     return jsonify(config_dict), 200
 
+@app.route(f"/{API_VERSION}/config/set", methods=['GET', 'POST'])
+@ssl_required
+@admin_required
+def set_config_item():
+    setting = request.args.get("setting")
+    section = request.args.get("section")
+    value = request.values.get("value")
+
+    result = {
+        "section_name": section,
+        "setting_name": setting,
+        "value": value
+    }
+
+    if not setting or not section or not value:
+        return jsonify({"status": "Missing argument"}), 400
+
+    with get_session() as session:
+        config = SystemConfig(logger, session)
+        config.add_or_update_settings([result])
+
+    return jsonify({"status": "success"})
 
 @app.route(f"/{API_VERSION}/config/update", methods=['POST'])
+@ssl_required
 def update_config():
-    if not development and not request.is_secure:
-        return generate_upgrade_request()
-
     update_dict = request.get_json()
 
     with get_session() as session:
-
         for section, data in update_dict.items():
-
             for key, value in data.items():
-
                 try:
-                    config_setting = session.query(Config).filter(Config.section_name == section, Config.setting_name == key).one()
+                    config_setting = (
+                        session.query(Config)
+                        .filter(
+                            Config.section_name == section, Config.setting_name == key
+                        )
+                        .one()
+                    )
                 except s.orm.exc.NoResultFound:
-                    return jsonify({"status": "Bad Request", "section": section, "setting": key}), 400
+                    return jsonify(
+                        {"status": "Bad Request", "section": section, "setting": key}
+                    ), 400
 
                 config_setting.value = value
 
@@ -66,3 +95,159 @@ def update_config():
     return jsonify({"status": "success"}), 200
 
 
+@app.route(f"/{API_VERSION}/workeroauth/get/keys", methods=['GET'])
+@ssl_required
+@admin_required
+def get_oauth_keys():
+    """
+    Retrieve all worker oauth keys from the configuration table in the database.
+    The keys from the "Keys" section are normalized and returned such that they follow the format:
+      {
+         "github_api_key": "ghp_XXXXXXXXXXXXXXX",
+         "gitlab_api_key": "glpat_XXXXXXXXXXXXXXX"
+      }
+    """
+    # Open a database session using the current application engine
+    with DatabaseSession(logger, engine=current_app.engine) as session:
+        config = AugurConfig(logger, session)
+        # Get the Keys section if it exists; otherwise, key list remains empty.
+        if config.is_section_in_config("Keys"):
+            keys_section = config.get_section("Keys")
+        else:
+            keys_section = {}
+
+    # Normalize the key names (append '_api_key' if needed)
+    keys_dict = {}
+    for platform, key_value in keys_section.items():
+        platform_lower = platform.lower()
+        if "github" in platform_lower:
+            platform_lower = "github"
+        elif "gitlab" in platform_lower:
+            platform_lower = "gitlab"
+        keys_dict[f"{platform_lower}_api_key"] = key_value
+
+    return jsonify(keys_dict), 200
+
+
+@app.route(f"/{API_VERSION}/workeroauth/get/invalidkeys", methods=["GET"])
+@ssl_required
+@admin_required
+def get_invalid_keys():
+    """
+    Retrieve all invalid worker OAuth keys by comparing the keys loaded 
+    in the KeyPublisher at startup with those stored in the database, 
+    and by checking the live keys with the is_bad_api_key function.
+
+    A key is considered valid if it appears in the set of live keys and passes 
+    the is_bad_api_key test; any key that fails is considered invalid.
+    """
+    keypub = KeyPublisher()
+
+    invalid_keys = {}
+    live_keys = {}
+    for platform in keypub.list_platforms():
+        platform_lower = platform.lower()
+        tokens = keypub.list_keys(platform)
+        if tokens is not None:
+            live_keys[platform_lower] = set(tokens)
+
+    # Instantiate the API key handlers and HTTP client.
+    ghkeyman = GithubApiKeyHandler(logger)
+    glkeyman = GitlabApiKeyHandler(logger)
+    client = httpx.Client()
+
+    github_db_keys = ghkeyman.get_api_keys_from_database()
+    # For GitHub, we check keys published under both channels.
+    github_live = live_keys.get("github_rest", set()) | live_keys.get("github_graphql", set())
+    for token in github_db_keys:
+        if token not in github_live or ghkeyman.is_bad_api_key(client, token):
+            invalid_keys.setdefault("github", []).append(token)
+
+    gitlab_db_keys = glkeyman.get_api_keys_from_database()
+    gitlab_live = live_keys.get("gitlab_rest", set())
+    for token in gitlab_db_keys:
+        if token not in gitlab_live or glkeyman.is_bad_api_key(client, token):
+            invalid_keys.setdefault("gitlab", []).append(token)
+
+    # This ensures that even if a key is live, we verify it using is_bad_api_key.
+    for token in live_keys.get("github_rest", set()):
+        if ghkeyman.is_bad_api_key(client, token) and token not in invalid_keys.get("github", []):
+            invalid_keys.setdefault("github", []).append(token)
+    for token in live_keys.get("github_graphql", set()):
+        if ghkeyman.is_bad_api_key(client, token) and token not in invalid_keys.get("github", []):
+            invalid_keys.setdefault("github", []).append(token)
+    for token in live_keys.get("gitlab_rest", set()):
+        if glkeyman.is_bad_api_key(client, token) and token not in invalid_keys.get("gitlab", []):
+            invalid_keys.setdefault("gitlab", []).append(token)
+
+    for platform, tokens in invalid_keys.items():
+        invalid_keys[platform] = [{"id": token, "token": token} for token in tokens]
+
+    return jsonify(invalid_keys), 200
+
+
+
+@app.route(f"/{API_VERSION}/workeroauth/delete/key", methods=["POST"])
+@ssl_required
+@admin_required
+def delete_oauth_key():
+    """
+    Delete a worker oauth key from the KeyPublisher, config table and worker oauth table
+    Expects a JSON payload with the platform and token properties.
+    """
+    data = request.get_json()
+
+    if not data or "platform" not in data or "token" not in data:
+        return jsonify(
+            {"status": "Bad Request", "message": "Missing platform or token"}
+        ), 400
+
+    keypub = KeyPublisher()
+
+    platform = data.get("platform").lower()
+    if platform == "github":
+        keypub.unpublish(data["token"], "github_rest")
+        keypub.unpublish(data["token"], "github_graphql")
+    elif platform == "gitlab":
+        keypub.unpublish(data["token"], "gitlab_rest")
+
+    remove_worker_oauth_key(platform=data["platform"].lower())
+    remove_setting(section_name="Keys", setting_name=data["platform"].lower() + "_api_key")
+
+    return jsonify({"status": "success"}), 200
+
+
+
+@app.route(f"/{API_VERSION}/workeroauth/new/keys", methods=["POST"])
+@ssl_required
+@admin_required
+def new_oauth_keys():
+    """
+    Add new worker oauth keys to the KeyPublisher.
+    Expects a JSON payload with the platform and token properties.
+    """
+    data = request.get_json()
+
+    if not data or "platform" not in data or "token" not in data:
+        return jsonify(
+            {"status": "Bad Request", "message": "Missing platform or token"}
+        ), 400
+
+    ghkeyman = GithubApiKeyHandler(logger)
+    glkeyman = GitlabApiKeyHandler(logger)
+
+    keypub = KeyPublisher()
+    client = httpx.Client()
+
+    if data.get("platform") == "github":
+        if ghkeyman.is_bad_api_key(client, data["token"]):
+            return jsonify({"status": "Bad Request", "message": "Invalid GitHub API key"}), 400
+        keypub.publish(data["token"], "github_rest")
+        keypub.publish(data["token"], "github_graphql")
+
+    elif data.get("platform") == "gitlab":
+        if glkeyman.is_bad_api_key(client, data["token"]):
+            return jsonify({"status": "Bad Request", "message": "Invalid GitLab API key"}), 400
+        keypub.publish(data["token"], "gitlab_rest")
+
+    return jsonify({"status": "success"}), 200

collectoss/api/ssl/README.md

@@ -22,9 +22,3 @@ Let's Encrypt/Certbot
 ~~~~~~~~~~~~~~~~~~~~~
 
 The easiest way to get an HTTPS server up is to make use of `Let's Encrypt <https://letsencrypt.org/>`_'s `Certbot <https://certbot.eff.org/>`_ tool. It is an open source tool that is so good it will even alter the nginx configuration for you automatically to enable HTTPS. Following their guide for ``Ubuntu 20.04``, run ``sudo snap install --classic certbot``, ``sudo ln -s /snap/bin/certbot /usr/bin/certbot``, and then ``sudo certbot --nginx``.
-
-~~~~~~~~~~~~~~~~~~~~
-Fixing the Backend
-~~~~~~~~~~~~~~~~~~~~
-
-Now our server is configured properly and our frontend is being served over HTTPS, but there's an extra problem: the backend APIs are still being served over HTTP resulting in a ``blocked loading mixed active content`` error. This issue is currently being looked into by our developers. Some files that are candidates for causing issues here are ``collectoss/application.py``, ``frontend/src/AugurAPI.ts``, and ``frontend/src/router.ts``.

collectoss/api/util.py

@@ -6,14 +6,16 @@
 import re
 import beaker
 
-from flask import request, jsonify, current_app
+from flask import request, jsonify, current_app, abort
 
 from collectoss.application.db import get_session
 from functools import wraps
 from sqlalchemy.orm.exc import NoResultFound
 from collectoss.application.config import get_development_flag
 from collectoss.application.db.models import ClientApplication
 
+from flask_login import login_required, current_user
+
 development = get_development_flag()
 
 __ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -112,7 +114,6 @@ def get_client_token():
 
     return token
 
-
 # usage:
 """
 @app.route("/path")
@@ -155,4 +156,22 @@ def wrapper(*args, **kwargs):
             return generate_upgrade_request()
         return fun(*args, **kwargs)
 
-    return wrapper
+    return wrapper
+
+def admin_required(func):
+    @login_required
+    @wraps(func)
+    def inner_function(*args, **kwargs):
+        if current_user.admin:
+            return func(*args, **kwargs)
+        else:
+            abort(403)
+    return inner_function
+
+def development_required(func):
+    @wraps(func)
+    def inner_function(*args, **kwargs):
+        if not development:
+            abort(403)
+        return func(*args, **kwargs)
+    return inner_function