From f06d418e29421835caf4a49901226fc3d72daa4c Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Thu, 6 Mar 2025 00:16:13 -0600 Subject: [PATCH] Fixed issue with updates after builds and restarting process --- agent_manager.py | 36 ++++++++++++++++++++++-------------- build_agent.py | 34 +++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/agent_manager.py b/agent_manager.py index 66f8883..07b9ced 100644 --- a/agent_manager.py +++ b/agent_manager.py @@ -15,6 +15,8 @@ import requests from build_agent import build_agent_version from zeroconf_server import ZeroconfServer +TIMEOUT = 10 + def find_server_ips(): ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), 9001) hostnames = [] @@ -47,12 +49,13 @@ def get_all_servers_status(): def get_worker_status(hostname): """Fetch worker status from the given hostname.""" try: - response = requests.get(f"http://{hostname}:9001/status", timeout=5) + response = requests.get(f"http://{hostname}:9001/status", timeout=TIMEOUT) status = response.json() if status['hostname'] != hostname and status['ip'] != hostname: status['ip'] = socket.gethostbyname(hostname) return status - except requests.exceptions.RequestException: + except requests.exceptions.RequestException as e: + print(f"EXCEPTION: {e}") return {"hostname": hostname, "status": "offline"} @@ -71,8 +74,7 @@ def zip_project(source_dir, output_zip): def send_build_request(zip_file, server_ip, download_after=False): """Uploads the zip file to the given server.""" upload_url = f"http://{server_ip}:9001/upload" - print(f"Submitting build request to URL: {upload_url}") - print("Please wait. This may take a few minutes...") + print(f"Submitting build request to URL: {upload_url} - Please wait. This may take a few minutes...") with open(zip_file, 'rb') as f: response = requests.post(upload_url, files={"file": f}) @@ -189,7 +191,7 @@ def update_worker(server): response_json = response.json() if response_json.get('updated_files') and not response_json.get('error_files'): try: - requests.get(f"http://{server['ip']}:9001/restart", timeout=2) + requests.get(f"http://{server['ip']}:9001/restart", timeout=TIMEOUT) except requests.exceptions.ConnectionError: pass return server @@ -223,7 +225,7 @@ def update_build_workers(server_data): while unverified_servers and datetime.now() < end_time: for server_ip in list(unverified_servers.keys()): # Iterate over a copy to avoid modification issues try: - response = requests.get(f"http://{server_ip}:9001/status", timeout=2) + response = requests.get(f"http://{server_ip}:9001/status") response.raise_for_status() server_info = unverified_servers[server_ip] # Get full server details agent_version = response.json().get('agent_version') @@ -254,7 +256,7 @@ def main(): parser.add_argument("-restart", type=str, help="Hostname to restart") parser.add_argument("-restart-all", action="store_true", help="Restart all agents") parser.add_argument("-shutdown", type=str, help="Hostname to shutdown") - # parser.add_argument("-worker", type=str, help="Update Agents") + parser.add_argument("-shutdown-all", action="store_true", help="Restart all agents") args = parser.parse_args() if args.status: @@ -267,11 +269,11 @@ def main(): for server_ip in find_server_ips(): restart_agent(server_ip) elif args.shutdown: - print(f"Shutting down hostname: {args.shutdown}") - try: - requests.get(f"http://{args.shutdown}:9001/shutdown") - except requests.exceptions.ConnectionError: - pass + shutdown_agent(args.shutdown) + elif args.shutdown_all: + print("Shutting down all agents...") + for server_ip in find_server_ips(): + shutdown_agent(server_ip) elif args.delete_cache: delete_cache(get_all_servers_status()) elif args.path: @@ -281,12 +283,18 @@ def main(): else: print("No path given!") +def shutdown_agent(hostname): + print(f"Shutting down hostname: {hostname}") + try: + requests.get(f"http://{hostname}:9001/shutdown", timeout=TIMEOUT) + except (requests.exceptions.ConnectionError, TimeoutError): + pass def restart_agent(hostname): print(f"Restarting agent: {hostname}") try: - requests.get(f"http://{hostname}:9001/restart") - except requests.exceptions.ConnectionError: + requests.get(f"http://{hostname}:9001/restart", timeout=TIMEOUT) + except (requests.exceptions.ConnectionError, TimeoutError): pass diff --git a/build_agent.py b/build_agent.py index 260d91a..d9dde86 100644 --- a/build_agent.py +++ b/build_agent.py @@ -16,9 +16,11 @@ import platform from zeroconf_server import ZeroconfServer from version import APP_VERSION, APP_NAME -build_agent_version = "0.1.23" +build_agent_version = "0.1.32" app = Flask(__name__) +launch_time = datetime.datetime.now() +LAUNCH_DIR = os.curdir SCRIPT_PATH = os.path.basename(__file__) LOCAL_DIR = os.path.dirname(__file__) BUILD_DIR = "pybuild-data" @@ -69,7 +71,6 @@ def update_files(): check=True) print("Update complete") - return jsonify({'updated_files': updated_files, 'error_files': error_files}), 200 if not error_files else 500 @app.get("/restart") @@ -100,6 +101,7 @@ def restart(): try: return jsonify({"message": "=== Restarting ==="}), 200 finally: + time.sleep(0.1) os.kill(os.getpid(), signal.SIGTERM) @app.get("/shutdown") @@ -109,7 +111,7 @@ def shutdown(): system_status['status'] = "shutting_down" return jsonify({"message": "Shutting down"}), 200 finally: - time.sleep(1) + time.sleep(0.1) os.kill(os.getpid(), signal.SIGTERM) @app.get("/") @@ -117,7 +119,7 @@ def status_page(): version = platform.mac_ver()[0] if platform.mac_ver() else platform.version() hostname = socket.gethostname() return (f"{APP_NAME} - Build Agent {build_agent_version} - \n" - f"{platform.system()} | {cpu_arch()} | {version} | {hostname} | {ZeroconfServer.get_local_ip()}") + f"{system_os()} | {cpu_arch()} | {version} | {hostname} | {ZeroconfServer.get_local_ip()}") @app.get("/status") def status(): @@ -137,9 +139,17 @@ def status(): size_in_bytes /= 1024 hostname = socket.gethostname() - return jsonify({"status": system_status['status'], "agent_version": build_agent_version, "os": platform.system(), "cpu": cpu_arch(), - "python_version": platform.python_version(), "hostname": hostname, "ip": ZeroconfServer.get_local_ip(), - "running_job": system_status['running_job'], "job_cache": len(job_cache()), "job_cache_size": format_size(get_directory_size(TMP_DIR))}) + return jsonify({"status": system_status['status'], + "agent_version": build_agent_version, + "os": system_os(), + "cpu": cpu_arch(), + "python": platform.python_version(), + "hostname": hostname, + "ip": ZeroconfServer.get_local_ip(), + "job_id": system_status['running_job'], + "cache_size": format_size(get_directory_size(TMP_DIR)), + "uptime": str(datetime.datetime.now() - launch_time) + }) @app.route('/upload', methods=['POST']) @@ -220,7 +230,6 @@ def install_and_build(project_path, job_id, start_time): # Compile with PyInstaller print(f"\n========== Compiling spec file {index+1} of {len(spec_files)} - {spec_file} ==========") simple_name = os.path.splitext(os.path.basename(spec_file))[0] - os.chdir(project_path) dist_path = os.path.join(project_path, "dist") work_path = os.path.join(project_path, "build") log_file_path = os.path.join(project_path, f"build-{simple_name}.log") @@ -231,12 +240,10 @@ def install_and_build(project_path, job_id, start_time): [py_exec, "-m", "PyInstaller", spec_file, "--distpath", dist_path, "--workpath", work_path], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - for line in process.stdout: print(line, end="") # Print to console log_file.write(line) # Save to log file log_file.flush() # Ensure real-time writing - process.wait() # Wait for the process to complete print(f"\n========== Compilation of spec file {spec_file} complete ==========\n") except Exception as e: @@ -256,7 +263,7 @@ def install_and_build(project_path, job_id, start_time): "output_folder": dist_path, "duration": str(datetime.datetime.now() - start_time), "cpu": cpu_arch(), - "os": platform.system(), + "os": system_os(), "hostname": socket.gethostname() }), 200 @@ -273,6 +280,9 @@ def cpu_arch(): arch = arch.replace(x, y) return arch +def system_os(): + return platform.system().replace("Darwin", "macOS") + @app.route('/download/', methods=['GET']) def download_binaries(job_id): """Handles downloading the compiled PyInstaller binaries for a given job.""" @@ -369,13 +379,11 @@ if __name__ == "__main__": print("Another instance is running. Waiting until it exits.") time.sleep(1) - os.system('cls' if os.name == 'nt' else 'clear') print(f"===== {APP_NAME} {APP_VERSION} Build Agent (v{build_agent_version}) =====") ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), 9001) try: ZeroconfServer.start() app.run(host="0.0.0.0", port=9001, threaded=True) - except KeyboardInterrupt: pass finally: