Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

- fix: restore Windows DLL search paths after failed loads by @remote-controlled-man in #2425
- fix(ci): remove unsupported macOS 14 release runners by @abetlen in #2416

## [0.3.37]
Expand Down
62 changes: 40 additions & 22 deletions llama_cpp/_ctypes_extensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import ctypes
import functools
import pathlib
from contextlib import ExitStack

from typing import (
Any,
Expand Down Expand Up @@ -53,21 +54,6 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path):

cdll_args = dict() # type: ignore

# Add the library directory to the DLL search path on Windows (if needed)
if sys.platform == "win32":
os.add_dll_directory(str(base_path))
os.environ["PATH"] = str(base_path) + os.pathsep + os.environ["PATH"]

if sys.platform == "win32" and sys.version_info >= (3, 8):
os.add_dll_directory(str(base_path))
if "CUDA_PATH" in os.environ:
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin"))
os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib"))
if "HIP_PATH" in os.environ:
os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin"))
os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib"))
cdll_args["winmode"] = ctypes.RTLD_GLOBAL

if sys.platform == "emscripten":
cdll_args["mode"] = ctypes.RTLD_GLOBAL
lib_dir = str(base_path)
Expand Down Expand Up @@ -95,13 +81,45 @@ def load_shared_library(lib_base_name: str, base_path: pathlib.Path):
f"Failed to load shared library '{dependency_path}': {e}"
)

# Try to load the shared library, handling potential errors
for lib_path in lib_paths:
if lib_path.exists():
try:
return ctypes.CDLL(str(lib_path), **cdll_args) # type: ignore
except Exception as e:
raise RuntimeError(f"Failed to load shared library '{lib_path}': {e}")
with ExitStack() as dll_search:
if sys.platform == "win32":
lib_dir = str(base_path)
original_path = os.environ.get("PATH")
if lib_dir not in (original_path or "").split(os.pathsep):
if original_path is None:
dll_search.callback(os.environ.pop, "PATH", None)
else:
dll_search.callback(os.environ.__setitem__, "PATH", original_path)
os.environ["PATH"] = (
lib_dir
if not original_path
else lib_dir + os.pathsep + original_path
)

dll_dirs = [lib_dir]
for sdk in ("CUDA_PATH", "HIP_PATH"):
if sdk in os.environ:
dll_dirs.extend(
os.path.join(os.environ[sdk], subdir)
for subdir in ("bin", "lib")
)
for dll_dir in dict.fromkeys(dll_dirs):
handle = os.add_dll_directory(dll_dir)
dll_search.callback(handle.close)
cdll_args["winmode"] = ctypes.RTLD_GLOBAL

# Try to load the shared library, handling potential errors
for lib_path in lib_paths:
if lib_path.exists():
try:
lib = ctypes.CDLL(str(lib_path), **cdll_args) # type: ignore
except Exception as e:
raise RuntimeError(
f"Failed to load shared library '{lib_path}': {e}"
)
# Keep dependency search paths after a successful load.
dll_search.pop_all()
return lib

raise FileNotFoundError(
f"Shared library with base name '{lib_base_name}' not found"
Expand Down
190 changes: 190 additions & 0 deletions tests/test_ctypes_extensions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
import os
from types import SimpleNamespace
from unittest.mock import Mock

import pytest

import llama_cpp._ctypes_extensions as ctypes_extensions


@pytest.fixture
def windows_loader(monkeypatch, tmp_path):
handles = []

def add_dll_directory(path):
handle = SimpleNamespace(path=path, close=Mock())
handles.append(handle)
return handle

environ = {"PATH": "original-path"}
cdll = Mock(return_value=object())
monkeypatch.setattr(
ctypes_extensions,
"sys",
SimpleNamespace(platform="win32", version_info=(3, 12)),
)
monkeypatch.setattr(
ctypes_extensions,
"os",
SimpleNamespace(
environ=environ,
path=os.path,
pathsep=";",
add_dll_directory=add_dll_directory,
),
)
monkeypatch.setattr(
ctypes_extensions, "ctypes", SimpleNamespace(CDLL=cdll, RTLD_GLOBAL=0)
)
(tmp_path / "llama.dll").touch()
return SimpleNamespace(
base_path=tmp_path, environ=environ, handles=handles, cdll=cdll
)


@pytest.mark.parametrize("sdk", [None, "CUDA_PATH", "HIP_PATH"])
def test_windows_failed_load_restores_search_paths(windows_loader, sdk):
loader = windows_loader
if sdk:
loader.environ[sdk] = str(loader.base_path / "sdk")
original_environ = loader.environ.copy()

def fail_load(path, **kwargs):
assert path == str(loader.base_path / "llama.dll")
assert kwargs == {"winmode": 0}
assert loader.environ["PATH"].split(";")[0] == str(loader.base_path)
assert all(
not handle.close.called for handle in loader.handles[-(3 if sdk else 1) :]
)
raise OSError("missing dependency")

loader.cdll.side_effect = fail_load
for _ in range(3):
with pytest.raises(RuntimeError, match="missing dependency"):
ctypes_extensions.load_shared_library("llama", loader.base_path)
assert loader.environ == original_environ
assert all(handle.close.call_count == 1 for handle in loader.handles)

expected_directories = [str(loader.base_path)]
if sdk:
expected_directories += [
os.path.join(loader.environ[sdk], "bin"),
os.path.join(loader.environ[sdk], "lib"),
]
assert [handle.path for handle in loader.handles] == expected_directories * 3


def test_windows_missing_library_restores_search_paths(windows_loader):
loader = windows_loader
(loader.base_path / "llama.dll").unlink()
with pytest.raises(FileNotFoundError, match="not found"):
ctypes_extensions.load_shared_library("llama", loader.base_path)
assert loader.environ == {"PATH": "original-path"}
assert all(handle.close.call_count == 1 for handle in loader.handles)
loader.cdll.assert_not_called()


def test_windows_failed_load_preserves_previous_success(windows_loader):
loader = windows_loader
ctypes_extensions.load_shared_library("llama", loader.base_path)
original_environ = loader.environ.copy()
successful_handles = loader.handles.copy()
loader.cdll.side_effect = OSError("missing dependency")

for _ in range(3):
with pytest.raises(RuntimeError, match="missing dependency"):
ctypes_extensions.load_shared_library("llama", loader.base_path)
assert loader.environ == original_environ
assert all(not handle.close.called for handle in successful_handles)
assert all(handle.close.call_count == 1 for handle in loader.handles[1:])


def test_windows_shared_sdk_directory_is_registered_once(windows_loader):
loader = windows_loader
sdk_path = str(loader.base_path / "sdk")
loader.environ.update(CUDA_PATH=sdk_path, HIP_PATH=sdk_path)
loader.cdll.side_effect = OSError("missing dependency")
with pytest.raises(RuntimeError, match="missing dependency"):
ctypes_extensions.load_shared_library("llama", loader.base_path)
assert [handle.path for handle in loader.handles] == [
str(loader.base_path),
os.path.join(sdk_path, "bin"),
os.path.join(sdk_path, "lib"),
]
assert all(handle.close.call_count == 1 for handle in loader.handles)
assert loader.environ == {
"PATH": "original-path",
"CUDA_PATH": sdk_path,
"HIP_PATH": sdk_path,
}


def test_windows_partial_setup_failure_restores_search_paths(
windows_loader, monkeypatch
):
loader = windows_loader
loader.environ["CUDA_PATH"] = str(loader.base_path / "cuda")
original_environ = loader.environ.copy()
add_directory = ctypes_extensions.os.add_dll_directory

def fail_second_directory(path):
if path != str(loader.base_path):
raise OSError("invalid SDK directory")
return add_directory(path)

monkeypatch.setattr(
ctypes_extensions.os, "add_dll_directory", fail_second_directory
)
with pytest.raises(OSError, match="invalid SDK directory"):
ctypes_extensions.load_shared_library("llama", loader.base_path)
assert loader.environ == original_environ
assert len(loader.handles) == 1
loader.handles[0].close.assert_called_once_with()
loader.cdll.assert_not_called()


@pytest.mark.parametrize(
"original_path", [None, "", "original-path", "already-present"]
)
@pytest.mark.parametrize("succeeds", [False, True])
def test_windows_path_and_successful_load(windows_loader, original_path, succeeds):
loader = windows_loader
if original_path is None:
loader.environ.pop("PATH")
elif original_path == "already-present":
loader.environ["PATH"] = "first;" + str(loader.base_path) + ";last"
else:
loader.environ["PATH"] = original_path
original_environ = loader.environ.copy()
if not succeeds:
loader.cdll.side_effect = OSError("missing dependency")

for _ in range(2):
if succeeds:
assert (
ctypes_extensions.load_shared_library("llama", loader.base_path)
is loader.cdll.return_value
)
assert loader.environ["PATH"].split(";").count(str(loader.base_path)) == 1
assert all(not handle.close.called for handle in loader.handles)
else:
with pytest.raises(RuntimeError, match="missing dependency"):
ctypes_extensions.load_shared_library("llama", loader.base_path)
assert loader.environ == original_environ
assert all(handle.close.call_count == 1 for handle in loader.handles)


@pytest.mark.parametrize("platform", ["linux", "freebsd14", "darwin"])
def test_non_windows_load_does_not_change_search_paths(
windows_loader, monkeypatch, platform
):
loader = windows_loader
monkeypatch.setattr(ctypes_extensions, "sys", SimpleNamespace(platform=platform))
(loader.base_path / "libllama.so").touch()
assert (
ctypes_extensions.load_shared_library("llama", loader.base_path)
is loader.cdll.return_value
)
loader.cdll.assert_called_once_with(str(loader.base_path / "libllama.so"))
assert loader.environ == {"PATH": "original-path"}
assert not loader.handles