From fcd9309d87ef401121699d1fd3bf90ba7a1356e0 Mon Sep 17 00:00:00 2001 From: Steven Ingram Date: Mon, 14 Sep 2026 17:47:23 +0000 Subject: [PATCH 01/14] Added chip concurrency --- MaxKernel/auto_agent/config.py | 39 ++++++++++++++++ .../auto_agent/server_utils/server_config.py | 33 +++++++------- MaxKernel/auto_agent/server_utils/setup.sh | 44 +++++++++++++++++-- .../auto_agent/server_utils/tpu_server.py | 6 ++- MaxKernel/hitl_agent/config.py | 39 ++++++++++++++++ MaxKernel/prepare_maxkernel.sh | 43 +++++++++++++----- 6 files changed, 169 insertions(+), 35 deletions(-) diff --git a/MaxKernel/auto_agent/config.py b/MaxKernel/auto_agent/config.py index 21c356a5..383b41aa 100644 --- a/MaxKernel/auto_agent/config.py +++ b/MaxKernel/auto_agent/config.py @@ -52,3 +52,42 @@ def get_thinking_planner(level: str = "high") -> BuiltInPlanner: thinking_level=level, ) ) + + +# MONKEY PATCH GENERATE_CONTENT to handle rate limits +try: + import tenacity + from google import genai + import logging + + def get_retry_decorator(): + return tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=4, max=60), + stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(Exception), + before_sleep=tenacity.before_sleep_log(logging.getLogger(__name__), logging.WARNING) + ) + + if not hasattr(genai.models.Models, "_original_generate_content"): + orig_sync = genai.models.Models.generate_content + genai.models.Models._original_generate_content = orig_sync + + @get_retry_decorator() + def wrapped_sync(self, *args, **kwargs): + return orig_sync(self, *args, **kwargs) + + genai.models.Models.generate_content = wrapped_sync + + if not hasattr(genai.models.AsyncModels, "_original_generate_content"): + orig_async = genai.models.AsyncModels.generate_content + genai.models.AsyncModels._original_generate_content = orig_async + + @get_retry_decorator() + async def wrapped_async(self, *args, **kwargs): + return await orig_async(self, *args, **kwargs) + + genai.models.AsyncModels.generate_content = wrapped_async +except ImportError: + pass +# END MONKEY PATCH + diff --git a/MaxKernel/auto_agent/server_utils/server_config.py b/MaxKernel/auto_agent/server_utils/server_config.py index b4c47b1a..e634fa87 100644 --- a/MaxKernel/auto_agent/server_utils/server_config.py +++ b/MaxKernel/auto_agent/server_utils/server_config.py @@ -35,35 +35,29 @@ def _resolve_config_path(cfg_path: str) -> Optional[str]: return None -def get_local_tpu_port(cfg_path: str = "eval_config.yaml") -> Optional[int]: - """Checks eval_config.yaml and returns the port if a local TPU server is needed.""" + +def get_local_tpu_ports(cfg_path: str = "eval_config.yaml") -> list[int]: + """Checks eval_config.yaml and returns the ports if local TPU servers are needed.""" resolved_path = _resolve_config_path(cfg_path) if not resolved_path: - return None + return [] try: with open(resolved_path, "r") as file: config = yaml.safe_load(file) or {} except Exception as e: logging.error(f"Config file {resolved_path} error: {e}") - return None + return [] if not isinstance(config, dict): - raise ValueError( - f"Invalid configuration format in {resolved_path}: " - "Expected a YAML dictionary at the root level." - ) + return [] backends = config.get("backends", []) if not isinstance(backends, list): - raise ValueError( - f"Invalid configuration format in {resolved_path}: " - "'backends' must be a list." - ) + return [] local_ip = get_local_ip() - # Find all backends that are local TPUs local_tpu_backends = [ b for b in backends @@ -73,11 +67,12 @@ def get_local_tpu_port(cfg_path: str = "eval_config.yaml") -> Optional[int]: and "tpu_vm" not in b ] - if not local_tpu_backends: - return None + return [b.get("port", TPU_SERVER_PORT) for b in local_tpu_backends] + +def get_local_tpu_port(cfg_path: str = "eval_config.yaml"): + ports = get_local_tpu_ports(cfg_path) + return ports[0] if ports else None - port = local_tpu_backends[0].get("port") - return port if port is not None else TPU_SERVER_PORT def get_local_cpu_port(cfg_path: str = "eval_config.yaml") -> Optional[int]: @@ -159,7 +154,9 @@ def get_bastion_config( if __name__ == "__main__": - tpu_p = get_local_tpu_port() + tpu_ports = get_local_tpu_ports() + tpu_p = tpu_ports[0] if tpu_ports else None + print(f"LOCAL_TPU_PORTS={' '.join(map(str, tpu_ports))}") cpu_p = get_local_cpu_port() b = get_bastion_config() diff --git a/MaxKernel/auto_agent/server_utils/setup.sh b/MaxKernel/auto_agent/server_utils/setup.sh index 46e499f7..945a2fbb 100644 --- a/MaxKernel/auto_agent/server_utils/setup.sh +++ b/MaxKernel/auto_agent/server_utils/setup.sh @@ -93,19 +93,55 @@ elif [ "$1" = "--start-gke" ]; then exit 1 fi elif [ "$1" = "--start-local" ] || [ "$1" = "--start-gce" ]; then + CHIPS=1 + while [[ "$#" -gt 0 ]]; do + case "$1" in + --chips) CHIPS="$2"; shift ;; + esac + shift + done + + # Generate eval_config.yaml dynamically if CHIPS is specified + if [ "$CHIPS" -gt 1 ]; then + echo "Dynamically generating eval_config.yaml for $CHIPS TPU chips..." + HOSTNAME_IP="127.0.0.1" + target="$SCRIPT_DIR/eval_config.yaml" + echo "backends:" > "$target" + local tpu_port=5463 + for (( i=0; i> "$target" + echo " ip: $HOSTNAME_IP" >> "$target" + echo " port: $tpu_port" >> "$target" + echo " type: tpu" >> "$target" + ((tpu_port++)) + done + local cpu_port=5464 + if [ $CHIPS -gt 1 ]; then + cpu_port=$tpu_port + fi + echo " - name: cpu-0" >> "$target" + echo " ip: $HOSTNAME_IP" >> "$target" + echo " port: $cpu_port" >> "$target" + echo " type: cpu" >> "$target" + fi + load_config # Start all local execution/evaluation servers (needed for local or GCE cases) echo "Starting local background servers (CPU, TPU, Eval)..." - if [ -n "$LOCAL_TPU_PORT" ]; then - nohup python3 tpu_server.py > output_tpu_server.txt 2>&1 & + if [ -n "$LOCAL_TPU_PORTS" ]; then + for port in $LOCAL_TPU_PORTS; do + PORT=$port nohup python3 tpu_server.py > output_tpu_server_${port}.txt 2>&1 & + done fi if [ -n "$LOCAL_CPU_PORT" ]; then nohup python3 cpu_server.py > output_cpu_server.txt 2>&1 & fi nohup python3 eval_server.py > output_eval_server.txt 2>&1 & - if [ -n "$LOCAL_TPU_PORT" ]; then - wait_for_server_health "TPU server" "$LOCAL_TPU_PORT" "output_tpu_server.txt" || exit 1 + if [ -n "$LOCAL_TPU_PORTS" ]; then + for port in $LOCAL_TPU_PORTS; do + wait_for_server_health "TPU server" "$port" "output_tpu_server_${port}.txt" || exit 1 + done fi if [ -n "$LOCAL_CPU_PORT" ]; then wait_for_server_health "CPU server" "$LOCAL_CPU_PORT" "output_cpu_server.txt" || exit 1 diff --git a/MaxKernel/auto_agent/server_utils/tpu_server.py b/MaxKernel/auto_agent/server_utils/tpu_server.py index 97dff1ef..aafba50e 100644 --- a/MaxKernel/auto_agent/server_utils/tpu_server.py +++ b/MaxKernel/auto_agent/server_utils/tpu_server.py @@ -515,7 +515,11 @@ def get_tpu_version() -> dict: if __name__ == "__main__": - tpu_port = get_local_tpu_port() + port_env = os.environ.get("PORT") + if port_env: + tpu_port = int(port_env) + else: + tpu_port = get_local_tpu_port() if tpu_port is None: logging.info( diff --git a/MaxKernel/hitl_agent/config.py b/MaxKernel/hitl_agent/config.py index 99519012..1394bf18 100644 --- a/MaxKernel/hitl_agent/config.py +++ b/MaxKernel/hitl_agent/config.py @@ -28,3 +28,42 @@ thinking_level="high", ) ) + + +# MONKEY PATCH GENERATE_CONTENT to handle rate limits +try: + import tenacity + from google import genai + import logging + + def get_retry_decorator(): + return tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=4, max=60), + stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(Exception), + before_sleep=tenacity.before_sleep_log(logging.getLogger(__name__), logging.WARNING) + ) + + if not hasattr(genai.models.Models, "_original_generate_content"): + orig_sync = genai.models.Models.generate_content + genai.models.Models._original_generate_content = orig_sync + + @get_retry_decorator() + def wrapped_sync(self, *args, **kwargs): + return orig_sync(self, *args, **kwargs) + + genai.models.Models.generate_content = wrapped_sync + + if not hasattr(genai.models.AsyncModels, "_original_generate_content"): + orig_async = genai.models.AsyncModels.generate_content + genai.models.AsyncModels._original_generate_content = orig_async + + @get_retry_decorator() + async def wrapped_async(self, *args, **kwargs): + return await orig_async(self, *args, **kwargs) + + genai.models.AsyncModels.generate_content = wrapped_async +except ImportError: + pass +# END MONKEY PATCH + diff --git a/MaxKernel/prepare_maxkernel.sh b/MaxKernel/prepare_maxkernel.sh index aea69c4c..8b4c8e1b 100644 --- a/MaxKernel/prepare_maxkernel.sh +++ b/MaxKernel/prepare_maxkernel.sh @@ -12,6 +12,17 @@ YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color + +CHIPS=1 +# Parse command line arguments +while [[ "$#" -gt 0 ]]; do + case "$1" in + --chips) CHIPS="$2"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + # Function to print colored output print_info() { echo -e "${BLUE}[INFO]${NC} $1" @@ -463,6 +474,7 @@ EOF } + # Function to create eval_config.yaml for both auto_agent and hitl_agent create_eval_config() { print_info "Creating eval_config.yaml for evaluation servers..." @@ -484,24 +496,31 @@ create_eval_config() { local target_dir target_dir="$(dirname "$target")" if [ -d "$target_dir" ]; then - cat > "$target" << EOF -backends: - - name: tpu-0 - ip: $HOSTNAME_IP - port: 5463 - type: tpu - - name: cpu-0 - ip: $HOSTNAME_IP - port: 5464 - type: cpu -EOF + echo "backends:" > "$target" + local tpu_port=5463 + for (( i=0; i> "$target" + echo " ip: $HOSTNAME_IP" >> "$target" + echo " port: $tpu_port" >> "$target" + echo " type: tpu" >> "$target" + ((tpu_port++)) + done + local cpu_port=5464 + if [ $CHIPS -gt 1 ]; then + cpu_port=$tpu_port + fi + echo " - name: cpu-0" >> "$target" + echo " ip: $HOSTNAME_IP" >> "$target" + echo " port: $cpu_port" >> "$target" + echo " type: cpu" >> "$target" fi done - print_success "Created eval_config.yaml for both auto_agent and hitl_agent (TPU:5463, CPU:5464, IP: $HOSTNAME_IP)" + print_success "Created eval_config.yaml with $CHIPS TPU chips." } + # Main execution main() { print_info "Starting MaxKernel Agent setup..." From 533ac34097ae1de598778b5bd3222752ac11214d Mon Sep 17 00:00:00 2001 From: Steven Ingram Date: Mon, 14 Sep 2026 18:15:59 +0000 Subject: [PATCH 02/14] bug fix --- MaxKernel/auto_agent/server_utils/server_config.py | 2 +- MaxKernel/auto_agent/server_utils/setup.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MaxKernel/auto_agent/server_utils/server_config.py b/MaxKernel/auto_agent/server_utils/server_config.py index e634fa87..6152f0b8 100644 --- a/MaxKernel/auto_agent/server_utils/server_config.py +++ b/MaxKernel/auto_agent/server_utils/server_config.py @@ -156,7 +156,7 @@ def get_bastion_config( if __name__ == "__main__": tpu_ports = get_local_tpu_ports() tpu_p = tpu_ports[0] if tpu_ports else None - print(f"LOCAL_TPU_PORTS={' '.join(map(str, tpu_ports))}") + print(f"LOCAL_TPU_PORTS='{ ' '.join(map(str, tpu_ports)) }'") cpu_p = get_local_cpu_port() b = get_bastion_config() diff --git a/MaxKernel/auto_agent/server_utils/setup.sh b/MaxKernel/auto_agent/server_utils/setup.sh index 945a2fbb..08a72f04 100644 --- a/MaxKernel/auto_agent/server_utils/setup.sh +++ b/MaxKernel/auto_agent/server_utils/setup.sh @@ -107,7 +107,7 @@ elif [ "$1" = "--start-local" ] || [ "$1" = "--start-gce" ]; then HOSTNAME_IP="127.0.0.1" target="$SCRIPT_DIR/eval_config.yaml" echo "backends:" > "$target" - local tpu_port=5463 + tpu_port=5463 for (( i=0; i> "$target" echo " ip: $HOSTNAME_IP" >> "$target" @@ -115,7 +115,7 @@ elif [ "$1" = "--start-local" ] || [ "$1" = "--start-gce" ]; then echo " type: tpu" >> "$target" ((tpu_port++)) done - local cpu_port=5464 + cpu_port=5464 if [ $CHIPS -gt 1 ]; then cpu_port=$tpu_port fi From 610372287d1bc8e99df0af813fc28949d86d995c Mon Sep 17 00:00:00 2001 From: Steven Ingram Date: Mon, 14 Sep 2026 20:27:08 +0000 Subject: [PATCH 03/14] allocate chips --- MaxKernel/auto_agent/server_utils/setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MaxKernel/auto_agent/server_utils/setup.sh b/MaxKernel/auto_agent/server_utils/setup.sh index 08a72f04..6b99dbe7 100644 --- a/MaxKernel/auto_agent/server_utils/setup.sh +++ b/MaxKernel/auto_agent/server_utils/setup.sh @@ -129,8 +129,10 @@ elif [ "$1" = "--start-local" ] || [ "$1" = "--start-gce" ]; then # Start all local execution/evaluation servers (needed for local or GCE cases) echo "Starting local background servers (CPU, TPU, Eval)..." if [ -n "$LOCAL_TPU_PORTS" ]; then + tpu_index=0 for port in $LOCAL_TPU_PORTS; do - PORT=$port nohup python3 tpu_server.py > output_tpu_server_${port}.txt 2>&1 & + TPU_VISIBLE_DEVICES=$tpu_index TPU_CHIPS_PER_HOST_BOUNDS=1,1,1 TPU_HOST_BOUNDS=1,1,1 PORT=$port nohup python3 tpu_server.py > output_tpu_server_${port}.txt 2>&1 & + ((tpu_index++)) done fi if [ -n "$LOCAL_CPU_PORT" ]; then From 0f72203e703a187b188cfcb4d5be982dcf13bc61 Mon Sep 17 00:00:00 2001 From: Steven Ingram Date: Mon, 14 Sep 2026 23:05:20 +0000 Subject: [PATCH 04/14] Updated data for multiple chips. --- MaxKernel/auto_search/run_search.py | 4 +++- MaxKernel/auto_search/utils/analyze_timing.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/MaxKernel/auto_search/run_search.py b/MaxKernel/auto_search/run_search.py index d0609a7b..a9c80a1a 100644 --- a/MaxKernel/auto_search/run_search.py +++ b/MaxKernel/auto_search/run_search.py @@ -3,6 +3,7 @@ import json import logging import os +import time import sys import time from typing import Any, Optional, Tuple @@ -172,7 +173,8 @@ async def run_search( try: logger.info("Generating timing summary...") - summary_text = analyze_path(dest_dir) + run_duration = time.time() - global_start_time + summary_text = analyze_path(dest_dir, real_wall_time=run_duration) out_file = os.path.join(dest_dir, "timing_summary.md") with open(out_file, "w") as f: f.write("```text\n" + summary_text + "\n```\n") diff --git a/MaxKernel/auto_search/utils/analyze_timing.py b/MaxKernel/auto_search/utils/analyze_timing.py index 9d02435e..8d762156 100644 --- a/MaxKernel/auto_search/utils/analyze_timing.py +++ b/MaxKernel/auto_search/utils/analyze_timing.py @@ -164,7 +164,7 @@ def process_file_metrics(file_path): return file_stats -def analyze_path(target_path: str): +def analyze_path(target_path: str, real_wall_time: float = None): path = Path(target_path) if path.is_file(): files_to_process = [path] @@ -254,9 +254,13 @@ def log(msg=""): log(" MACRO SUMMARY (ACROSS ALL DISCOVERED NODES) ") log("============================================================") log(f"Total Nodes/Attempts Analyzed : {total_runs}") - log( - f"Aggregated Pipeline Time : {global_pipeline:>7.2f}s computation-hours" - ) + if real_wall_time: + concurrency_factor = (global_pipeline / real_wall_time) if real_wall_time > 0 else 0 + log(f"Aggregated Pipeline Time : {global_pipeline/60:>7.2f} computation-minutes") + log(f"Real-World Wall Time : {real_wall_time/60:>7.2f} minutes") + log(f"Concurrency Acceleration : {concurrency_factor:>7.2f}x speedup") + else: + log(f"Aggregated Pipeline Time : {global_pipeline:>7.2f}s computation-hours") if global_pipeline > 0: log( From ff01a1b2327a643e0d6d99202eb4cf5afe20d756 Mon Sep 17 00:00:00 2001 From: Steven Ingram Date: Mon, 14 Sep 2026 23:16:52 +0000 Subject: [PATCH 05/14] add ruff install --- MaxKernel/prepare_maxkernel.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MaxKernel/prepare_maxkernel.sh b/MaxKernel/prepare_maxkernel.sh index 8b4c8e1b..8b67bb19 100644 --- a/MaxKernel/prepare_maxkernel.sh +++ b/MaxKernel/prepare_maxkernel.sh @@ -224,6 +224,12 @@ install_dependencies() { pip install -e "$REPO_ROOT" fi + # Install ruff formatter/linter + if ! command -v ruff &> /dev/null; then + print_info "Installing ruff..." + pip install ruff + fi + # Check if npx is installed if ! command -v npx &> /dev/null; then print_info "npx not found. Installing nodejs and npm via nvm..." @@ -608,6 +614,7 @@ STEP 2: Install Dependencies pip install -r dependency/main_requirements.txt pip install -r dependency/agent_requirements.txt pip install -e . +pip install ruff STEP 3: Set Environment Variables ---------------------------------- From 4b559a2168bb67ed2c01b8880995ab6774a12340 Mon Sep 17 00:00:00 2001 From: Steven Ingram Date: Tue, 15 Sep 2026 15:53:43 +0000 Subject: [PATCH 06/14] global time fix --- MaxKernel/auto_agent/config.py | 56 ++++++++++--------- .../auto_agent/server_utils/server_config.py | 5 +- MaxKernel/auto_search/run_search.py | 2 +- MaxKernel/auto_search/utils/analyze_timing.py | 14 +++-- MaxKernel/hitl_agent/config.py | 54 +++++++++--------- 5 files changed, 70 insertions(+), 61 deletions(-) diff --git a/MaxKernel/auto_agent/config.py b/MaxKernel/auto_agent/config.py index 383b41aa..19ddeaef 100644 --- a/MaxKernel/auto_agent/config.py +++ b/MaxKernel/auto_agent/config.py @@ -56,38 +56,40 @@ def get_thinking_planner(level: str = "high") -> BuiltInPlanner: # MONKEY PATCH GENERATE_CONTENT to handle rate limits try: - import tenacity - from google import genai - import logging - - def get_retry_decorator(): - return tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=4, max=60), - stop=tenacity.stop_after_attempt(10), - retry=tenacity.retry_if_exception_type(Exception), - before_sleep=tenacity.before_sleep_log(logging.getLogger(__name__), logging.WARNING) - ) + import logging + + import tenacity + from google import genai + + def get_retry_decorator(): + return tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=4, max=60), + stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(Exception), + before_sleep=tenacity.before_sleep_log( + logging.getLogger(__name__), logging.WARNING + ), + ) - if not hasattr(genai.models.Models, "_original_generate_content"): - orig_sync = genai.models.Models.generate_content - genai.models.Models._original_generate_content = orig_sync + if not hasattr(genai.models.Models, "_original_generate_content"): + orig_sync = genai.models.Models.generate_content + genai.models.Models._original_generate_content = orig_sync - @get_retry_decorator() - def wrapped_sync(self, *args, **kwargs): - return orig_sync(self, *args, **kwargs) + @get_retry_decorator() + def wrapped_sync(self, *args, **kwargs): + return orig_sync(self, *args, **kwargs) - genai.models.Models.generate_content = wrapped_sync + genai.models.Models.generate_content = wrapped_sync - if not hasattr(genai.models.AsyncModels, "_original_generate_content"): - orig_async = genai.models.AsyncModels.generate_content - genai.models.AsyncModels._original_generate_content = orig_async + if not hasattr(genai.models.AsyncModels, "_original_generate_content"): + orig_async = genai.models.AsyncModels.generate_content + genai.models.AsyncModels._original_generate_content = orig_async - @get_retry_decorator() - async def wrapped_async(self, *args, **kwargs): - return await orig_async(self, *args, **kwargs) + @get_retry_decorator() + async def wrapped_async(self, *args, **kwargs): + return await orig_async(self, *args, **kwargs) - genai.models.AsyncModels.generate_content = wrapped_async + genai.models.AsyncModels.generate_content = wrapped_async except ImportError: - pass + pass # END MONKEY PATCH - diff --git a/MaxKernel/auto_agent/server_utils/server_config.py b/MaxKernel/auto_agent/server_utils/server_config.py index 6152f0b8..4070f45b 100644 --- a/MaxKernel/auto_agent/server_utils/server_config.py +++ b/MaxKernel/auto_agent/server_utils/server_config.py @@ -35,7 +35,6 @@ def _resolve_config_path(cfg_path: str) -> Optional[str]: return None - def get_local_tpu_ports(cfg_path: str = "eval_config.yaml") -> list[int]: """Checks eval_config.yaml and returns the ports if local TPU servers are needed.""" resolved_path = _resolve_config_path(cfg_path) @@ -69,12 +68,12 @@ def get_local_tpu_ports(cfg_path: str = "eval_config.yaml") -> list[int]: return [b.get("port", TPU_SERVER_PORT) for b in local_tpu_backends] + def get_local_tpu_port(cfg_path: str = "eval_config.yaml"): ports = get_local_tpu_ports(cfg_path) return ports[0] if ports else None - def get_local_cpu_port(cfg_path: str = "eval_config.yaml") -> Optional[int]: """Checks eval_config.yaml and returns the port if a local CPU server is needed.""" resolved_path = _resolve_config_path(cfg_path) @@ -156,7 +155,7 @@ def get_bastion_config( if __name__ == "__main__": tpu_ports = get_local_tpu_ports() tpu_p = tpu_ports[0] if tpu_ports else None - print(f"LOCAL_TPU_PORTS='{ ' '.join(map(str, tpu_ports)) }'") + print(f"LOCAL_TPU_PORTS='{' '.join(map(str, tpu_ports))}'") cpu_p = get_local_cpu_port() b = get_bastion_config() diff --git a/MaxKernel/auto_search/run_search.py b/MaxKernel/auto_search/run_search.py index a9c80a1a..2ea4a497 100644 --- a/MaxKernel/auto_search/run_search.py +++ b/MaxKernel/auto_search/run_search.py @@ -3,7 +3,6 @@ import json import logging import os -import time import sys import time from typing import Any, Optional, Tuple @@ -106,6 +105,7 @@ async def run_search( **kwargs: Any, ) -> Tuple[str, str]: """Executes the search algorithm asynchronously for a single reference file.""" + global_start_time = time.time() problem_dir = os.path.dirname(os.path.abspath(reference_file_path)) default_problem_id, ext = os.path.splitext( os.path.basename(reference_file_path) diff --git a/MaxKernel/auto_search/utils/analyze_timing.py b/MaxKernel/auto_search/utils/analyze_timing.py index 8d762156..4cec0b8a 100644 --- a/MaxKernel/auto_search/utils/analyze_timing.py +++ b/MaxKernel/auto_search/utils/analyze_timing.py @@ -255,12 +255,18 @@ def log(msg=""): log("============================================================") log(f"Total Nodes/Attempts Analyzed : {total_runs}") if real_wall_time: - concurrency_factor = (global_pipeline / real_wall_time) if real_wall_time > 0 else 0 - log(f"Aggregated Pipeline Time : {global_pipeline/60:>7.2f} computation-minutes") - log(f"Real-World Wall Time : {real_wall_time/60:>7.2f} minutes") + concurrency_factor = ( + (global_pipeline / real_wall_time) if real_wall_time > 0 else 0 + ) + log( + f"Aggregated Pipeline Time : {global_pipeline / 60:>7.2f} computation-minutes" + ) + log(f"Real-World Wall Time : {real_wall_time / 60:>7.2f} minutes") log(f"Concurrency Acceleration : {concurrency_factor:>7.2f}x speedup") else: - log(f"Aggregated Pipeline Time : {global_pipeline:>7.2f}s computation-hours") + log( + f"Aggregated Pipeline Time : {global_pipeline:>7.2f}s computation-hours" + ) if global_pipeline > 0: log( diff --git a/MaxKernel/hitl_agent/config.py b/MaxKernel/hitl_agent/config.py index 1394bf18..1de39dae 100644 --- a/MaxKernel/hitl_agent/config.py +++ b/MaxKernel/hitl_agent/config.py @@ -32,38 +32,40 @@ # MONKEY PATCH GENERATE_CONTENT to handle rate limits try: - import tenacity - from google import genai - import logging + import logging - def get_retry_decorator(): - return tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=4, max=60), - stop=tenacity.stop_after_attempt(10), - retry=tenacity.retry_if_exception_type(Exception), - before_sleep=tenacity.before_sleep_log(logging.getLogger(__name__), logging.WARNING) - ) + import tenacity + from google import genai - if not hasattr(genai.models.Models, "_original_generate_content"): - orig_sync = genai.models.Models.generate_content - genai.models.Models._original_generate_content = orig_sync + def get_retry_decorator(): + return tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=4, max=60), + stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(Exception), + before_sleep=tenacity.before_sleep_log( + logging.getLogger(__name__), logging.WARNING + ), + ) - @get_retry_decorator() - def wrapped_sync(self, *args, **kwargs): - return orig_sync(self, *args, **kwargs) + if not hasattr(genai.models.Models, "_original_generate_content"): + orig_sync = genai.models.Models.generate_content + genai.models.Models._original_generate_content = orig_sync - genai.models.Models.generate_content = wrapped_sync + @get_retry_decorator() + def wrapped_sync(self, *args, **kwargs): + return orig_sync(self, *args, **kwargs) - if not hasattr(genai.models.AsyncModels, "_original_generate_content"): - orig_async = genai.models.AsyncModels.generate_content - genai.models.AsyncModels._original_generate_content = orig_async + genai.models.Models.generate_content = wrapped_sync - @get_retry_decorator() - async def wrapped_async(self, *args, **kwargs): - return await orig_async(self, *args, **kwargs) + if not hasattr(genai.models.AsyncModels, "_original_generate_content"): + orig_async = genai.models.AsyncModels.generate_content + genai.models.AsyncModels._original_generate_content = orig_async - genai.models.AsyncModels.generate_content = wrapped_async + @get_retry_decorator() + async def wrapped_async(self, *args, **kwargs): + return await orig_async(self, *args, **kwargs) + + genai.models.AsyncModels.generate_content = wrapped_async except ImportError: - pass + pass # END MONKEY PATCH - From ae5f530fee27edb1be6f57dc926a2520a7481aa5 Mon Sep 17 00:00:00 2001 From: Steven Ingram Date: Tue, 15 Sep 2026 18:42:29 +0000 Subject: [PATCH 07/14] bug fix --- MaxKernel/auto_agent/config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/MaxKernel/auto_agent/config.py b/MaxKernel/auto_agent/config.py index 19ddeaef..13f92413 100644 --- a/MaxKernel/auto_agent/config.py +++ b/MaxKernel/auto_agent/config.py @@ -56,6 +56,7 @@ def get_thinking_planner(level: str = "high") -> BuiltInPlanner: # MONKEY PATCH GENERATE_CONTENT to handle rate limits try: + import asyncio import logging import tenacity @@ -87,7 +88,12 @@ def wrapped_sync(self, *args, **kwargs): @get_retry_decorator() async def wrapped_async(self, *args, **kwargs): - return await orig_async(self, *args, **kwargs) + # Gemini API occasionally hangs indefinitely on concurrent quotas. + # Force a 90 second hard timeout so it triggers a tenacity retry + # instead of infinitely blocking the orchestrator. + return await asyncio.wait_for( + orig_async(self, *args, **kwargs), timeout=90 + ) genai.models.AsyncModels.generate_content = wrapped_async except ImportError: From bfa08b13ee3eae7319492e5bfa26061519e887e2 Mon Sep 17 00:00:00 2001 From: stingram Date: Mon, 14 Sep 2026 18:17:27 +0000 Subject: [PATCH 08/14] bug fix --- MaxKernel/auto_agent/server_utils/setup.sh | 0 MaxKernel/prepare_maxkernel.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 MaxKernel/auto_agent/server_utils/setup.sh mode change 100644 => 100755 MaxKernel/prepare_maxkernel.sh diff --git a/MaxKernel/auto_agent/server_utils/setup.sh b/MaxKernel/auto_agent/server_utils/setup.sh old mode 100644 new mode 100755 diff --git a/MaxKernel/prepare_maxkernel.sh b/MaxKernel/prepare_maxkernel.sh old mode 100644 new mode 100755 From a7d91a41e677ce99bd6c711261c6ba8473110aef Mon Sep 17 00:00:00 2001 From: Steven Date: Tue, 15 Sep 2026 15:51:16 -0700 Subject: [PATCH 09/14] Update MaxKernel/hitl_agent/config.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- MaxKernel/hitl_agent/config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/MaxKernel/hitl_agent/config.py b/MaxKernel/hitl_agent/config.py index 1de39dae..ca5d0f6f 100644 --- a/MaxKernel/hitl_agent/config.py +++ b/MaxKernel/hitl_agent/config.py @@ -63,7 +63,13 @@ def wrapped_sync(self, *args, **kwargs): @get_retry_decorator() async def wrapped_async(self, *args, **kwargs): - return await orig_async(self, *args, **kwargs) + import asyncio + # Gemini API occasionally hangs indefinitely on concurrent quotas. + # Force a 90 second hard timeout so it triggers a tenacity retry + # instead of infinitely blocking the orchestrator. + return await asyncio.wait_for( + orig_async(self, *args, **kwargs), timeout=90 + ) genai.models.AsyncModels.generate_content = wrapped_async except ImportError: From a75682fe4066ce732c8459e54e16a0521c3e881d Mon Sep 17 00:00:00 2001 From: Steven Date: Tue, 15 Sep 2026 15:51:37 -0700 Subject: [PATCH 10/14] Update MaxKernel/auto_agent/server_utils/server_config.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- MaxKernel/auto_agent/server_utils/server_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MaxKernel/auto_agent/server_utils/server_config.py b/MaxKernel/auto_agent/server_utils/server_config.py index 4070f45b..7386b693 100644 --- a/MaxKernel/auto_agent/server_utils/server_config.py +++ b/MaxKernel/auto_agent/server_utils/server_config.py @@ -69,7 +69,7 @@ def get_local_tpu_ports(cfg_path: str = "eval_config.yaml") -> list[int]: return [b.get("port", TPU_SERVER_PORT) for b in local_tpu_backends] -def get_local_tpu_port(cfg_path: str = "eval_config.yaml"): +def get_local_tpu_port(cfg_path: str = "eval_config.yaml") -> Optional[int]: ports = get_local_tpu_ports(cfg_path) return ports[0] if ports else None From 49daf09158a5c67d11aa919f4ab870317e89e0d1 Mon Sep 17 00:00:00 2001 From: Steven Date: Tue, 15 Sep 2026 15:52:06 -0700 Subject: [PATCH 11/14] Update MaxKernel/auto_search/utils/analyze_timing.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- MaxKernel/auto_search/utils/analyze_timing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MaxKernel/auto_search/utils/analyze_timing.py b/MaxKernel/auto_search/utils/analyze_timing.py index 4cec0b8a..6cafec5d 100644 --- a/MaxKernel/auto_search/utils/analyze_timing.py +++ b/MaxKernel/auto_search/utils/analyze_timing.py @@ -265,7 +265,7 @@ def log(msg=""): log(f"Concurrency Acceleration : {concurrency_factor:>7.2f}x speedup") else: log( - f"Aggregated Pipeline Time : {global_pipeline:>7.2f}s computation-hours" + f"Aggregated Pipeline Time : {global_pipeline / 3600:>7.2f} computation-hours" ) if global_pipeline > 0: From d588ccf7f422e239128f4569b4bad54ec88217f2 Mon Sep 17 00:00:00 2001 From: Steven Date: Tue, 15 Sep 2026 15:52:19 -0700 Subject: [PATCH 12/14] Update MaxKernel/prepare_maxkernel.sh Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- MaxKernel/prepare_maxkernel.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MaxKernel/prepare_maxkernel.sh b/MaxKernel/prepare_maxkernel.sh index 8b67bb19..f1703301 100755 --- a/MaxKernel/prepare_maxkernel.sh +++ b/MaxKernel/prepare_maxkernel.sh @@ -23,6 +23,11 @@ while [[ "$#" -gt 0 ]]; do shift done +if ! [[ "$CHIPS" =~ ^[0-9]+$ ]] || [ "$CHIPS" -lt 1 ]; then + echo "Error: --chips must be a positive integer." + exit 1 +fi + # Function to print colored output print_info() { echo -e "${BLUE}[INFO]${NC} $1" From 0de7cd0c9e19a04ca4d865585730812fd675c94e Mon Sep 17 00:00:00 2001 From: stingram Date: Tue, 15 Sep 2026 22:56:57 +0000 Subject: [PATCH 13/14] bug fix --- MaxKernel/auto_agent/server_utils/tpu_server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/MaxKernel/auto_agent/server_utils/tpu_server.py b/MaxKernel/auto_agent/server_utils/tpu_server.py index aafba50e..38e3dc1e 100644 --- a/MaxKernel/auto_agent/server_utils/tpu_server.py +++ b/MaxKernel/auto_agent/server_utils/tpu_server.py @@ -517,7 +517,11 @@ def get_tpu_version() -> dict: if __name__ == "__main__": port_env = os.environ.get("PORT") if port_env: - tpu_port = int(port_env) + try: + tpu_port = int(port_env) + except ValueError: + logging.error(f"Invalid PORT environment variable: {port_env}. Must be an integer.") + sys.exit(1) else: tpu_port = get_local_tpu_port() From d8462d4c724e8d56afcbaa8d78e6e572f690eafb Mon Sep 17 00:00:00 2001 From: stingram Date: Tue, 15 Sep 2026 23:02:34 +0000 Subject: [PATCH 14/14] formatting --- MaxKernel/auto_agent/server_utils/tpu_server.py | 4 +++- MaxKernel/hitl_agent/config.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/MaxKernel/auto_agent/server_utils/tpu_server.py b/MaxKernel/auto_agent/server_utils/tpu_server.py index 38e3dc1e..23764b05 100644 --- a/MaxKernel/auto_agent/server_utils/tpu_server.py +++ b/MaxKernel/auto_agent/server_utils/tpu_server.py @@ -520,7 +520,9 @@ def get_tpu_version() -> dict: try: tpu_port = int(port_env) except ValueError: - logging.error(f"Invalid PORT environment variable: {port_env}. Must be an integer.") + logging.error( + f"Invalid PORT environment variable: {port_env}. Must be an integer." + ) sys.exit(1) else: tpu_port = get_local_tpu_port() diff --git a/MaxKernel/hitl_agent/config.py b/MaxKernel/hitl_agent/config.py index ca5d0f6f..3d696b97 100644 --- a/MaxKernel/hitl_agent/config.py +++ b/MaxKernel/hitl_agent/config.py @@ -64,6 +64,7 @@ def wrapped_sync(self, *args, **kwargs): @get_retry_decorator() async def wrapped_async(self, *args, **kwargs): import asyncio + # Gemini API occasionally hangs indefinitely on concurrent quotas. # Force a 90 second hard timeout so it triggers a tenacity retry # instead of infinitely blocking the orchestrator.