From 1faa544ead581f845cfd052eab6b889e01f915de Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Mon, 31 Aug 2026 00:46:36 -0500 Subject: [PATCH] Make async git checkout return immediately Once the job is dispatched the agent cloned synchronously even with async=1, so a slow clone blew past the controller's 5s dispatch timeout and the job was marked failed while the worker kept building. Run the clone in the background thread and return 202 + job id right away. --- agent/build_agent.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/agent/build_agent.py b/agent/build_agent.py index 762d415..992839b 100755 --- a/agent/build_agent.py +++ b/agent/build_agent.py @@ -228,15 +228,18 @@ def checkout_project(): build_root = os.path.join(TMP_DIR, BUILD_DIR) repo_dir = os.path.join(build_root, job_id) start_time = datetime.datetime.now() + if async_mode: + os.makedirs(build_root, exist_ok=True) + _set_step(job_id, "cloning_repo", repo_url=repo_url) + system_status['status'] = "cloning_repo" + _run_checkout_build_background(job_id, repo_url, repo_dir, start_time) + return jsonify({"id": job_id, "async": True, "message": "Build started"}), 202 try: os.makedirs(build_root, exist_ok=True) _set_step(job_id, "cloning_repo", repo_url=repo_url) system_status['status'] = "cloning_repo" subprocess.check_call(['git', 'clone', repo_url, repo_dir]) system_status['status'] = "ready" - if async_mode: - _run_build_background(job_id, repo_dir, start_time) - return jsonify({"id": job_id, "async": True, "message": "Build started"}), 202 return install_and_build(repo_dir, job_id, start_time) except Exception as e: print(f"Error processing checkout: {e}") @@ -251,6 +254,25 @@ def checkout_project(): if not async_mode: build_lock.release() + +def _run_checkout_build_background(job_id, repo_url, repo_dir, start_time): + """Run git clone + install_and_build in a background thread so async + checkouts return immediately. Assumes build_lock is held; the thread + releases it when the work finishes.""" + def runner(): + try: + subprocess.check_call(['git', 'clone', repo_url, repo_dir]) + install_and_build(repo_dir, job_id, start_time) + except Exception as e: + print(f"Background checkout/build failed for {job_id}: {e}") + system_status['status'] = "ready" + system_status['running_job'] = None + _clear_progress(job_id) + shutil.rmtree(repo_dir, ignore_errors=True) + finally: + build_lock.release() + threading.Thread(target=runner, name=f"checkout-{job_id}", daemon=True).start() + def _run_build_background(job_id, project_path, start_time): """Run install_and_build in a background thread. Assumes build_lock is held; the thread releases it when the build finishes."""