Implement graceful move+migration of Facade repo path calculation

collectoss/application/db/lib.py

@@ -109,7 +109,26 @@ def get_repo_by_repo_id(repo_id):
         repo = execute_session_query(query, 'one')
 
         return repo
-
+
+def get_clone_path_by_repo_id(repo_id: int) -> Optional[str]:
+
+    with get_session() as session:
+
+        query = session.query(CollectionStatus).filter(CollectionStatus.repo_id == repo_id)
+        ops_repo = query.one_or_none()
+        return ops_repo.facade_clone_path if ops_repo else None
+
+def set_clone_path_by_repo_id(repo_id: int, clone_path: str):
+
+    with get_session() as session:
+
+        query = session.query(CollectionStatus).filter(CollectionStatus.repo_id == repo_id)
+        entry = query.one_or_none()
+        if entry:
+            entry.facade_clone_path = clone_path
+            session.commit()
+
+
 def get_github_repo_by_src_id(src_id):
 
     with get_session() as session:

collectoss/application/db/models/data.py

@@ -855,8 +855,8 @@ class Repo(Base):
     repo_git = Column(String, nullable=False)
 
     #TODO: repo_path and repo_name should be generated columns in postgresql
-    repo_path = Column(String)
-    repo_name = Column(String)
+    repo_path = Column(String, comment="deprecated by the addition of facade_clone_path in the collection_status table")
+    repo_name = Column(String, comment="the current repo name")
     repo_added = Column(
         TIMESTAMP(precision=0), nullable=False, server_default=text("CURRENT_TIMESTAMP")
     )

collectoss/application/db/models/operations.py

@@ -1216,6 +1216,7 @@ class CollectionStatus(Base):
     facade_status = Column(String,nullable=False, server_default=text("'Pending'"))
     facade_data_last_collected = Column(TIMESTAMP)
     facade_task_id = Column(String)
+    facade_clone_path = Column(String, server_default=None)
 
     ml_status = Column(String,nullable=False, server_default=text("'Pending'"))
     ml_data_last_collected = Column(TIMESTAMP)

collectoss/application/schema/alembic/versions/45_add_facade_path_column.py

@@ -0,0 +1,53 @@
+"""add facade path column
+
+Revision ID: 45
+Revises: 44
+Create Date: 2026-09-28 11:44:57.974030
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '45'
+down_revision = '44'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('collection_status', sa.Column('facade_clone_path', sa.String(), nullable=True), schema='operations')
+    op.alter_column('repo', 'repo_path',
+               existing_type=sa.VARCHAR(),
+               comment='deprecated by the addition of facade_clone_path in the collection_status table',
+               existing_nullable=True,
+               existing_server_default=sa.text("'NULL'::character varying"),
+               schema='data')
+    op.alter_column('repo', 'repo_name',
+               existing_type=sa.VARCHAR(),
+               comment='the current repo name',
+               existing_nullable=True,
+               existing_server_default=sa.text("'NULL'::character varying"),
+               schema='data')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('repo', 'repo_name',
+               existing_type=sa.VARCHAR(),
+               comment=None,
+               existing_comment='the current repo name',
+               existing_nullable=True,
+               existing_server_default=sa.text("'NULL'::character varying"),
+               schema='data')
+    op.alter_column('repo', 'repo_path',
+               existing_type=sa.VARCHAR(),
+               comment=None,
+               existing_comment='deprecated by the addition of facade_clone_path in the collection_status table',
+               existing_nullable=True,
+               existing_server_default=sa.text("'NULL'::character varying"),
+               schema='data')
+    op.drop_column('collection_status', 'facade_clone_path', schema='operations')
+    # ### end Alembic commands ###

collectoss/tasks/git/dependency_libyear_tasks/core.py

@@ -2,7 +2,7 @@
 from collectoss.application.db.models import *
 from collectoss.application.db.lib import get_value, bulk_insert_dicts, get_repo_by_repo_git
 from collectoss.tasks.git.dependency_libyear_tasks.libyear_util.util import get_deps_libyear_data
-from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path
+from collectoss.tasks.git.util.helpers import get_absolute_clone_path
 
 def deps_libyear_model(logger,repo_git):
         """ Data collection and storage method
@@ -15,7 +15,7 @@ def deps_libyear_model(logger,repo_git):
 
         repo = get_repo_by_repo_git(repo_git)
 
-        absolute_repo_path = get_absolute_repo_path(get_value("Facade", "repo_directory"),repo.repo_id,repo.repo_path,repo.repo_name)
+        absolute_repo_path = get_absolute_clone_path(get_value("Facade", "repo_directory"), repo.repo_id)
         #config.get_section("Facade")['repo_directory'] + relative_repo_path#self.config['repo_directory'] + relative_repo_path
 
         generate_deps_libyear_data(logger, repo.repo_id, absolute_repo_path)

collectoss/tasks/git/dependency_tasks/core.py

@@ -3,10 +3,10 @@
 from collectoss.application.db.models import *
 from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_value
 from collectoss.application.environment import SystemEnv
+from collectoss.tasks.git.util.helpers import get_absolute_clone_path
 from collectoss.tasks.github.util.github_api_key_handler import GithubApiKeyHandler
 from collectoss.tasks.git.dependency_tasks.dependency_util import dependency_calculator as dep_calc
 from collectoss.tasks.util.worker_util import parse_json_from_subprocess_call
-from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path
 from collectoss.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
 from collectoss.tasks.util.metadata_exception import MetadataException
 
@@ -22,7 +22,7 @@ def generate_deps_data(logger, repo_git):
         repo = get_repo_by_repo_git(repo_git)
         repo_id = repo.repo_id
 
-        path = get_absolute_repo_path(get_value("Facade", "repo_directory"),repo.repo_id,repo.repo_path,repo.repo_name)
+        path = get_absolute_clone_path(get_value("Facade", "repo_directory"), repo.repo_id)
 
         logger.debug(f"This is the deps model repo: {repo_git}.")
 

collectoss/tasks/git/facade_tasks.py

@@ -8,10 +8,11 @@
 from collectoss.application.db.lib import get_session, get_repo_by_repo_git, get_repo_by_repo_id, remove_working_commits_by_repo_id_and_hashes, get_working_commits_by_repo_id, facade_bulk_insert_commits, bulk_insert_dicts, get_missing_commit_message_hashes
 
 from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import trim_commits
-from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path, get_parent_commits_set, get_existing_commits_set
+from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_parent_commits_set, get_existing_commits_set
 from collectoss.tasks.git.util.facade_worker.facade_worker.analyzecommit import analyze_commit
 from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_repo_commit_count, update_facade_scheduling_fields, get_facade_weight_with_commit_count
 
+from collectoss.tasks.git.util.helpers import get_absolute_clone_path
 from collectoss.tasks.github.facade_github.tasks import *
 from collectoss.tasks.git.util.facade_worker.facade_worker.config import FacadeHelper
 from collectoss.tasks.util.collection_state import CollectionState
@@ -117,8 +118,8 @@ def trim_commits_post_analysis_facade_task(repo_git):
     repo = get_repo_by_repo_git(repo_git)
 
     #Get the huge list of commits to process.
-    absolute_path = get_absolute_repo_path(facade_helper.repo_base_directory, repo.repo_id, repo.repo_path,repo.repo_name)
-    repo_loc = (f"{absolute_path}/.git")
+    absolute_path = get_absolute_clone_path(facade_helper.repo_base_directory, repo.repo_id)
+    repo_loc = absolute_path.joinpath(".git")
     # Grab the parents of HEAD
 
     parent_commits = get_parent_commits_set(repo_loc, facade_helper)
@@ -190,8 +191,8 @@ def facade_fetch_missing_commit_messages(repo_git):
         except (TypeError, IndexError):
             escaped_hash = hash
 
-        absolute_path = get_absolute_repo_path(facade_helper.repo_base_directory, repo.repo_id, repo.repo_path, repo.repo_name)
-        repo_loc = (f"{absolute_path}/.git")
+        absolute_path = get_absolute_clone_path(facade_helper.repo_base_directory, repo.repo_id)
+        repo_loc = absolute_path.joinpath(".git")
 
         try: 
             commit_message = check_output(
@@ -240,8 +241,8 @@ def analyze_commits_in_parallel(repo_git, multithreaded: bool)-> None:
     repo = get_repo_by_repo_id(repo_id)
 
     #Get the huge list of commits to process.
-    absolute_path = get_absolute_repo_path(facade_helper.repo_base_directory, repo.repo_id, repo.repo_path, repo.repo_name)
-    repo_loc = (f"{absolute_path}/.git")
+    absolute_path = get_absolute_clone_path(facade_helper.repo_base_directory, repo.repo_id)
+    repo_loc = absolute_path.joinpath(".git")
     # Grab the parents of HEAD
 
     parent_commits = get_parent_commits_set(repo_loc, facade_helper)
@@ -262,8 +263,8 @@ def analyze_commits_in_parallel(repo_git, multithreaded: bool)-> None:
     queue = list(missing_commits)
 
     logger.info(f"Got to analysis!")
-    absolute_path = get_absolute_repo_path(facade_helper.repo_base_directory, repo.repo_id, repo.repo_path,repo.repo_name)
-    repo_loc = (f"{absolute_path}/.git")
+    absolute_path = get_absolute_clone_path(facade_helper.repo_base_directory, repo.repo_id)
+    repo_loc = absolute_path.joinpath(".git")
 
     pendingCommitRecordsToInsert = []
     pendingCommitMessageRecordsToInsert = []

collectoss/tasks/git/scc_value_tasks/core.py

@@ -3,8 +3,8 @@
 from collectoss.application.db.models import *
 from collectoss.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_value
 from collectoss.application.environment import SystemEnv
+from collectoss.tasks.git.util.helpers import get_absolute_clone_path
 from collectoss.tasks.util.worker_util import parse_json_from_subprocess_call
-from collectoss.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path
 
 def value_model(logger,repo_git):
     """Runs scc on repo and stores data in database
@@ -15,7 +15,7 @@ def value_model(logger,repo_git):
     repo = get_repo_by_repo_git(repo_git)
     repo_id = repo.repo_id
 
-    path = get_absolute_repo_path(get_value("Facade", "repo_directory"),repo_id,repo.repo_path,repo.repo_name)
+    path = get_absolute_clone_path(get_value("Facade", "repo_directory"), repo.repo_id)
 
     logger.info('Generating value data for repo')
     logger.info(f"Repo ID: {repo_id}, Path: {path}")

collectoss/tasks/git/util/facade_worker/facade_worker/repofetch.py

@@ -30,12 +30,14 @@
 import os
 import pathlib
 import sqlalchemy as s
-from .utilitymethods import update_repo_log, get_absolute_repo_path
+
+from collectoss.tasks.git.util.helpers import get_absolute_clone_path
+from .utilitymethods import update_repo_log
 from sqlalchemy.orm.exc import NoResultFound
 from collectoss.application.db.models.data import *
 from collectoss.application.db.models.operations import CollectionStatus
 from collectoss.application.db.util import execute_session_query
-from collectoss.application.db.lib import execute_sql, get_repo_by_repo_git
+from collectoss.application.db.lib import execute_sql, get_repo_by_repo_git, set_clone_path_by_repo_id
 from typing_extensions import deprecated
 
 class GitCloneError(Exception):
@@ -122,6 +124,8 @@ def git_repo_initialize(facade_helper, session, repo_git):
             repo_name=:nameParam WHERE repo_id=:idParam
             """).bindparams(pathParam=path_identifier, nameParam=repo_name, idParam=row.repo_id)
 
+            set_clone_path_by_repo_id(row.repo_id, f"{row.repo_id}-{path_identifier}")
+
             execute_sql(query)
             return
 
@@ -162,6 +166,7 @@ def git_repo_initialize(facade_helper, session, repo_git):
             # Mark the entire project for an update, so that under normal
             # circumstances caches are rebuilt only once per waiting period.
             update_repo_log(logger, facade_helper, row.repo_id, 'Up-to-date')
+            set_clone_path_by_repo_id(row.repo_id, f"{row.repo_id}-{path_identifier}/{repo_name}")
             facade_helper.log_activity('Info', f"Cloned {git}")
 
         else:
@@ -313,8 +318,7 @@ def git_repo_updates(facade_helper, repo_git):
 
     # default_branch = ''
 
-    absolute_path = get_absolute_repo_path(
-        facade_helper.repo_base_directory, repo.repo_id, repo.repo_path, repo.repo_name)
+    absolute_path = get_absolute_clone_path(facade_helper.repo_base_directory, repo.repo_id)
 
     while attempt < 2:
 

collectoss/tasks/git/util/facade_worker/facade_worker/utilitymethods.py

@@ -30,6 +30,7 @@
 import os
 import sqlalchemy as s
 from collectoss.application.db.models import *
+from collectoss.tasks.git.util.helpers import get_absolute_clone_path
 from .config import FacadeHelper as FacadeHelper
 from collectoss.tasks.util.worker_util import calculate_date_weight_from_timestamps
 from collectoss.application.db.lib import execute_sql, fetchall_data_from_sql_text, remove_working_commits_by_repo_id_and_hashes, remove_commits_by_repo_id_and_hashes, get_repo_by_repo_git, get_session
@@ -102,7 +103,8 @@ def trim_author(facade_helper, email):
 
 	facade_helper.log_activity('Debug',f"Trimmed working author: {email}")
 
-def get_absolute_repo_path(repo_base_dir, repo_id, repo_path,repo_name):
+@deprecated("This method of storing the repo path is legacy and is being replaced")
+def get_absolute_repo_path(repo_base_dir, repo_id, repo_path, repo_name):
 
 	return f"{repo_base_dir}{repo_id}-{repo_path}/{repo_name}"
 
@@ -150,8 +152,8 @@ def get_repo_commit_count(logger, facade_helper, repo_git):
 
 	repo = get_repo_by_repo_git(repo_git)
 
-	absolute_path = get_absolute_repo_path(facade_helper.repo_base_directory, repo.repo_id, repo.repo_path,repo.repo_name)
-	repo_loc = (f"{absolute_path}/.git")
+	absolute_path = get_absolute_clone_path(facade_helper.repo_base_directory, repo.repo_id)
+	repo_loc = absolute_path.joinpath(".git")
 
 	logger.debug(f"loc: {repo_loc}")
 	logger.debug(f"path: {repo.repo_path}")

collectoss/tasks/git/util/helpers.py

@@ -0,0 +1,59 @@
+from pathlib import Path
+
+from collectoss.application.db.lib import get_clone_path_by_repo_id, get_repo_by_repo_id, set_clone_path_by_repo_id
+
+def is_git_repo(path:Path) -> bool:
+    if not path.exists():
+        return False
+
+    gitdir = path.joinpath(".git")
+    if not gitdir.exists():
+        return False
+    return True
+
+def get_absolute_clone_path(facade_base_directory: str | Path, repo_id: int) -> Path:
+    """Returns the absolute path to the clone of the repo on disk
+
+    This method tries several methods to get the clone path and will automatically 
+    update the database with the correct path if a less-than-ideal method is used.
+
+    This method expects that the clone directory already exists.
+    See `git_repo_initialize` for the method that creates the clone directory.
+
+    Args:
+        facade_base_directory (str): the configured base directory that all facade clones paths are relative to
+        repo_id (int): the id of the repo to get the clone path for
+    """
+
+    base_dir = Path(facade_base_directory)
+
+    # check if [configured facade base dir] + [path from db operations table] exists and is a git repo (happy path/ideal case)
+    clone_path = get_clone_path_by_repo_id(repo_id)
+    if clone_path and is_git_repo(base_dir.joinpath(clone_path)):
+        return base_dir.joinpath(clone_path)
+    # if not, use the current path building technique ( [configured facade base dir] + [path from db data table] + [repo name]). if success, rewrite the facade path and return it
+    repo = get_repo_by_repo_id(repo_id)
+
+    # absolute_path = get_absolute_repo_path(base_dir, repo.repo_id, repo.repo_path,repo.repo_name)
+    legacy_path = f"{repo_id}-{repo.repo_path}/{repo.repo_name}"
+    if legacy_path and is_git_repo(base_dir.joinpath(legacy_path)):
+        set_clone_path_by_repo_id(repo_id, legacy_path)
+        return base_dir.joinpath(legacy_path)
+
+    # if not, discover it (check just the facade path from step 1, if it contains just one dir, use that and update the database else fail)
+    discover_path = base_dir.joinpath(f"{repo_id}-{repo.repo_path}")
+    discovered_directories = []
+    if discover_path.exists():
+        discovered_directories = [x for x in discover_path.iterdir() if x.is_dir()]
+        if len(discovered_directories) == 1 and is_git_repo(discover_path.joinpath(discovered_directories[0])):
+            discovered_repo = discovered_directories[0]  # already an absolute Path
+            set_clone_path_by_repo_id(repo_id, str(discovered_repo.relative_to(base_dir)))
+            return discovered_repo
+
+
+    raise ValueError(f"""No valid git repo path found for repo {repo_id} ({repo.repo_git}).
+    Attempted paths:
+    - {base_dir.joinpath(clone_path) if clone_path else '(not set)'}
+    - {base_dir.joinpath(legacy_path)}
+    - {discover_path} ({len(discovered_directories)} children)
+    """)

docs/source/development-guide/workers/creating_a_new_worker.rst

@@ -8,6 +8,29 @@ Worker Setup
 1. If you are hitting an API on a platform like GitHub, or GitLab, follow the pattern in those workers.
 2. If you are analyzing CollectOSS data, the `value_worker` provides a good example.
 
+Accessing Cloned Repositories on Disk
+--------------------------------------
+
+If your task needs to access a repository's local clone (e.g. to run a tool against the source code or git repo), use the ``get_absolute_clone_path`` function from ``collectoss.tasks.git.util.helpers``:
+
+.. code-block:: python
+
+    from collectoss.tasks.git.util.helpers import get_absolute_clone_path
+    from collectoss.application.config import SystemConfig
+    from collectoss.application.db.session import DatabaseSession
+    from collectoss.application.db import get_engine
+
+    with DatabaseSession(logger, get_engine()) as session:
+        config = SystemConfig(logger, session)
+        repo_directory = config.get_value("Facade", "repo_directory")
+
+    repo_path = get_absolute_clone_path(repo_directory, repo_id)
+
+This function returns a ``pathlib.Path`` object pointing to the root of the cloned repository. It reads the path from ``collection_status.facade_clone_path``.
+
+.. warning::
+    Do **not** construct clone paths manually from ``repo.repo_path`` and ``repo.repo_name``. Those columns are deprecated for path resolution purposes and ``repo.repo_name`` will change when a repository is moved. Always use the ``get_absolute_clone_path`` function.
+
 What are the key sections?
 -----------------------------------
 

docs/source/getting-started/collecting-data.rst

@@ -70,7 +70,7 @@ Next up are the configuration options specific to some collection tasks (but som
 ``Facade``
 ::::::::::::::::::
 
-- ``repo_directory``, which is the local directory where the facade tasks will clone the repositories it needs to analyze. You should have been prompted for this during installation, but if you need to change it, make sure that it's an absolute path (environment variables like ``$HOME`` are not supported) and that the directory already exists. Defaults to ``repos/``, but it's highly recommended you change this.
+- ``repo_directory``, which is the local directory where the facade tasks will clone the repositories it needs to analyze. Make sure that it's an absolute path (environment variables like ``$HOME`` are not yet supported) and that the directory already exists. Defaults to ``repos/``, but it's highly recommended you change this. The exact subdirectory layout within ``repo_directory`` is managed automatically by CollectOSS.
 - ``limited_run``, toggle between 0 and 1 to determine whether to run all facade tasks or not. Runs all tasks if set to 0
 - ``pull_repos``, toggle whether to pull updates from repos after cloning them. If turned off updates to repos will not be collected.
 - ``run_analysis``, toggle whether to process commit data at all. If turned off will only clone repos and run tertiary tasks such as resolving contributors from any existing commits or collecting dependency relationships. Mainly used for testing.