From b0d3660881b6cbe9162d1aa897e037ce18de6de4 Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Wed, 12 Mar 2025 00:20:25 -0500 Subject: [PATCH] Ability to checkout git repos, custom ports, and setup for pip --- README.md | 40 +++-- cross-py-builder/__init__.py | 0 .../build_agent.py | 62 ++++++-- .../cross-py-builder.py | 137 +++++++++++------- .../zeroconf_server.py | 1 - setup.py | 26 ++++ 6 files changed, 181 insertions(+), 85 deletions(-) create mode 100644 cross-py-builder/__init__.py rename build_agent.py => cross-py-builder/build_agent.py (87%) rename agent_manager.py => cross-py-builder/cross-py-builder.py (73%) rename zeroconf_server.py => cross-py-builder/zeroconf_server.py (99%) create mode 100644 setup.py diff --git a/README.md b/README.md index f6ec52d..31552ad 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# Cross-Py-Build +# Cross-Py-Builder -Cross-Py-Build is a simple remote build tool for compiling Python projects with PyInstaller on different platforms on a local network. +Cross-Py-Builder is a simple remote build tool for compiling Python projects with PyInstaller on different platforms on a local network. ## System Requirements - **Ubuntu/Debian & macOS:** 2GB+ RAM - **Windows:** 4GB+ RAM -- **Python 3** installed +- **Python 3.10 or later** installed - **Local Network** for build agents --- @@ -14,7 +14,7 @@ Cross-Py-Build is a simple remote build tool for compiling Python projects with ```sh - ./agent_manager.py --build /path/to/repo -cpu x64 -os windows + ./cross-py-builder.py --build /path/to/repo -cpu x64 -os windows options: -h, --help show this help message and exit @@ -28,7 +28,7 @@ options: --restart RESTART Hostname to restart --restart-all Restart all agents --shutdown SHUTDOWN Hostname to shutdown - --shutdown-all Restart all agents + --shutdown-all Shutdown all agents ``` --- @@ -50,29 +50,23 @@ This guide provides steps to set up a worker VM on **Ubuntu/Debian, macOS, or Wi ```sh xcode-select --install # Install xcode tools ``` - - Install Python 3.x from [python.org](https://www.python.org/downloads/) - - **Windows:** - - Install Python 3.x from [python.org](https://www.python.org/downloads/) - - -3. **Set up a virtual environment:** - +3. **Install Python** >=3.10 from [python.org](https://www.python.org/downloads/) (Windows and macOS - macOS) +4. **Set up a virtual environment:** - **Ubuntu/Debian/macOS:** - ```sh - python3 -m venv venv - source venv/bin/activate - ``` + ```sh + python3 -m venv venv + source venv/bin/activate + ``` - **Windows:** - ```sh - python3 -m venv venv - source venv\Scripts\activate - ``` -4. **Copy project files** and install dependencies: + ```sh + python3 -m venv venv + venv\Scripts\activate + ``` +5. **Copy project files** and install dependencies: ```sh pip install -r requirements.txt ``` - -5. **Start Build Agent**: +6. **Start Build Agent**: - **Ubuntu/Debian/macOS:** ```sh python3 build_agent.py diff --git a/cross-py-builder/__init__.py b/cross-py-builder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build_agent.py b/cross-py-builder/build_agent.py similarity index 87% rename from build_agent.py rename to cross-py-builder/build_agent.py index 6258e4c..70a46db 100755 --- a/build_agent.py +++ b/cross-py-builder/build_agent.py @@ -18,7 +18,8 @@ import platform from zeroconf_server import ZeroconfServer APP_NAME = "cross-py-builder" -build_agent_version = "0.1.33" +build_agent_version = "0.1.34" +app_port = 9001 app = Flask(__name__) launch_time = datetime.datetime.now() @@ -148,12 +149,39 @@ def status(): "python": platform.python_version(), "hostname": hostname, "ip": ZeroconfServer.get_local_ip(), + "port": app_port, "job_id": system_status['running_job'], "cache_size": format_size(get_directory_size(TMP_DIR)), "uptime": str(datetime.datetime.now() - launch_time) }) +def generate_job_id(): + return str(uuid.uuid4()).split('-')[-1] + + +@app.route("/checkout_git", methods=['POST']) +def checkout_project(): + start_time = datetime.datetime.now() + repo_url = request.json.get('repo_url') + if not repo_url: + return jsonify({'error': 'Repository URL is required'}), 400 + + print(f"\n========== Checking Out Git Project ==========") + + job_id = generate_job_id() + repo_dir = os.path.join(TMP_DIR, job_id) + try: + system_status['status'] = "cloning_repo" + subprocess.check_call(['git', 'clone', repo_url, repo_dir]) + system_status['status'] = "ready" + except subprocess.CalledProcessError as e: + print(f"Error cloning repo: {e}") + system_status['status'] = "ready" + return jsonify({'error': 'Failed to clone repository'}), 500 + + return install_and_build(repo_dir, job_id, start_time) + @app.route('/upload', methods=['POST']) def upload_project(): try: @@ -161,8 +189,9 @@ def upload_project(): if 'file' not in request.files: return jsonify({"error": "No file uploaded"}), 400 + system_status['status'] = "processing_files" print(f"\n========== Processing Incoming Project ==========") - job_id = str(uuid.uuid4()).split('-')[-1] + job_id = generate_job_id() working_dir = os.path.join(TMP_DIR, BUILD_DIR, job_id) file = request.files['file'] @@ -180,7 +209,8 @@ def upload_project(): return install_and_build(working_dir, job_id, start_time) except Exception as e: print(f"Uncaught error processing job: {e}") - jsonify({"error": f"Uncaught error processing job: {e}"}), 500 + system_status['status'] = "ready" + return jsonify({"error": f"Uncaught error processing job: {e}"}), 500 def install_and_build(project_path, job_id, start_time): @@ -196,6 +226,7 @@ def install_and_build(project_path, job_id, start_time): # Set up virtual environment venv_path = os.path.join(project_path, "venv") try: + system_status['status'] = "creating_venv" print(f"\n========== Configuring Virtual Environment ({venv_path}) ==========") python_exec = "python" if is_windows() else "python3" subprocess.run([python_exec, "-m", "venv", venv_path], check=True) @@ -212,6 +243,7 @@ def install_and_build(project_path, job_id, start_time): # Install requirements try: + system_status['status'] = "installing_packages" subprocess.run([py_exec, "-m", "pip", "install", "--upgrade", "pip"], check=True) subprocess.run([py_exec, "-m", "pip", "install", "pyinstaller", "pyinstaller_versionfile", "--prefer-binary"], check=True) requirements_path = os.path.join(project_path, "requirements.txt") @@ -230,6 +262,7 @@ def install_and_build(project_path, job_id, start_time): try: for index, spec_file in enumerate(spec_files): # Compile with PyInstaller + system_status['status'] = "compiling" print(f"\n========== Compiling spec file {index+1} of {len(spec_files)} - {spec_file} ==========") simple_name = os.path.splitext(os.path.basename(spec_file))[0] dist_path = os.path.join(project_path, "dist") @@ -242,17 +275,26 @@ def install_and_build(project_path, job_id, start_time): [py_exec, "-m", "PyInstaller", spec_file, "--distpath", dist_path, "--workpath", work_path], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + last_line = None for line in process.stdout: - print(line, end="") # Print to console - log_file.write(line) # Save to log file - log_file.flush() # Ensure real-time writing - process.wait() # Wait for the process to complete + last_line = line + print(line, end="") + log_file.write(line) + log_file.flush() + process.wait() + if process.returncode != 0: + raise RuntimeError( + f"PyInstaller failed with exit code {process.returncode}. Last line: {str(last_line).strip()}") + print(f"\n========== Compilation of spec file {spec_file} complete ==========\n") except Exception as e: print(f"Error compiling project: {e}") system_status['status'] = "ready" system_status['running_job'] = None - os.remove(project_path) + try: + os.remove(project_path) + except PermissionError: + pass return jsonify({"error": f"Error compiling project: {e}"}), 500 dist_path = os.path.join(project_path, "dist") @@ -355,10 +397,10 @@ def delete_cache(): if __name__ == "__main__": print(f"===== {APP_NAME} Build Agent (v{build_agent_version}) =====") - ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), 9001) + ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), app_port) try: ZeroconfServer.start() - app.run(host="0.0.0.0", port=9001, threaded=True) + app.run(host="0.0.0.0", port=app_port, threaded=True) except KeyboardInterrupt: pass finally: diff --git a/agent_manager.py b/cross-py-builder/cross-py-builder.py similarity index 73% rename from agent_manager.py rename to cross-py-builder/cross-py-builder.py index 2229f3d..1a366c1 100755 --- a/agent_manager.py +++ b/cross-py-builder/cross-py-builder.py @@ -9,18 +9,18 @@ import time import zipfile from datetime import datetime, timedelta +import requests from packaging.version import Version from tabulate import tabulate -import requests - from build_agent import build_agent_version from zeroconf_server import ZeroconfServer +DEFAULT_PORT = 9001 TIMEOUT = 10 def find_server_ips(): - ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), 9001) + ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), DEFAULT_PORT) hostnames = [] try: ZeroconfServer.start(listen_only=True) @@ -32,32 +32,44 @@ def find_server_ips(): ZeroconfServer.stop() # get known hosts - with open("known_hosts", "r") as file: + with open("../known_hosts", "r") as file: lines = file.readlines() - - lines = [line.split(':')[0].strip() for line in lines] hostnames.extend(lines) return hostnames + def get_all_servers_status(): table_data = [] server_ips = find_server_ips() + with concurrent.futures.ThreadPoolExecutor() as executor: - results = executor.map(get_worker_status, server_ips) # Runs in parallel - table_data.extend(results) + futures = [] + for server in server_ips: + ip = server.split(':')[0] + port = server.split(":")[-1].strip() if ":" in server else DEFAULT_PORT + futures.append(executor.submit(get_worker_status, ip, port)) + + for future in concurrent.futures.as_completed(futures): + try: + result = future.result() # Get the result of the thread + table_data.append(result) + except Exception as e: + print(f"Error fetching status from server: {e}") # Handle potential errors return table_data -def get_worker_status(hostname): + +def get_worker_status(hostname, port=DEFAULT_PORT): """Fetch worker status from the given hostname.""" try: - response = requests.get(f"http://{hostname}:9001/status", timeout=TIMEOUT) + response = requests.get(f"http://{hostname}:{port}/status", timeout=TIMEOUT) status = response.json() + status['port'] = port if status['hostname'] != hostname and status['ip'] != hostname: status['ip'] = socket.gethostbyname(hostname) return status except requests.exceptions.RequestException as e: - return {"hostname": hostname, "status": "offline"} + return {"hostname": hostname, "port": port, "status": "offline"} def zip_project(source_dir, output_zip): @@ -72,26 +84,36 @@ def zip_project(source_dir, output_zip): zipf.write(file_path, os.path.relpath(file_path, source_dir)) -def send_build_request(zip_file, server_ip, download_after=False): - """Uploads the zip file to the given server.""" - upload_url = f"http://{server_ip}:9001/upload" - print(f"Submitting build request to URL: {upload_url} - Please wait. This may take a few minutes...") - with open(zip_file, 'rb') as f: - response = requests.post(upload_url, files={"file": f}) +def send_build_request(server_ip, server_port=DEFAULT_PORT, zip_file=None, git_url=None, download_after=True, version=None): + + if zip_file: + upload_url = f"http://{server_ip}:{server_port}/upload" + print(f"Submitting build request to URL: {upload_url} - Please wait. This may take a few minutes...") + with open(zip_file, 'rb') as f: + response = requests.post(upload_url, files={"file": f}) + elif git_url: + checkout_url = f"http://{server_ip}:{server_port}/checkout_git" + response = requests.post(checkout_url, json={"repo_url": git_url}) + else: + raise ValueError("Missing zip file or git url!") if response.status_code == 200: response_data = response.json() + print(response_data) print(f"Build successful. ID: {response_data['id']} Hostname: {response_data['hostname']} OS: {response_data['os']} CPU: {response_data['cpu']} Spec files: {len(response_data['spec_files'])} - Elapsed time: {response_data['duration']}" ) if download_after: - download_url = f"http://{server_ip}:9001/download/{response_data.get('id')}" + download_url = f"http://{server_ip}:{server_port}/download/{response_data.get('id')}" try: - save_name = f"{os.path.splitext(os.path.basename(zip_file))[0]}-{response_data['os'].lower()}-{response_data['cpu'].lower()}.zip" + base_name = os.path.splitext(os.path.basename(zip_file))[0] + version_string = ("-" + version) if version else "" + save_name = f"{base_name}{version_string}-{response_data['os'].lower()}-{response_data['cpu'].lower()}.zip" download_zip(download_url, save_name=save_name) except Exception as e: print(f"Error downloading zip: {e}") else: print("Upload failed:", response.status_code, response.text) + def download_zip(url, save_name, save_dir="."): """Download a ZIP file from a URL and save it with its original filename.""" response = requests.get(url, stream=True) @@ -116,6 +138,7 @@ def select_server(servers, cpu=None, os_name=None): available = [s for s in available if os_name.lower() in s["os"].lower()] return available[0] if available else None # Return first matching server or None + def process_new_job(args, server_data): available_servers = [s for s in server_data if s["status"] == "ready"] @@ -139,19 +162,23 @@ def process_new_job(args, server_data): print(f"Found {len(available_servers)} servers available to build") print(tabulate(available_servers, headers="keys", tablefmt="grid")) - project_path = args.build - tmp_dir = tempfile.gettempdir() - zip_file = os.path.join(tmp_dir, f"{os.path.basename(project_path)}.zip") - zip_project(project_path, zip_file) - print(f"Zipped {project_path} to {zip_file}") + zip_file = None + if args.build: + project_path = args.build + tmp_dir = tempfile.gettempdir() + zip_file = os.path.join(tmp_dir, f"{os.path.basename(project_path)}.zip") + zip_project(project_path, zip_file) + print(f"Zipped {project_path} to {zip_file}") # Start builds on all matching servers with concurrent.futures.ThreadPoolExecutor() as executor: print("Submitting builds to:") + download = True for server in available_servers: print(f"\t{server['hostname']} - {server['os']} - {server['cpu']}") - futures = {executor.submit(send_build_request, zip_file, server["ip"], args.download): server for server in - available_servers} + futures = {executor.submit( + send_build_request,server["ip"], DEFAULT_PORT, zip_file, args.checkout, download, args.version): + server for server in available_servers} # Collect results for future in concurrent.futures.as_completed(futures): @@ -162,16 +189,18 @@ def process_new_job(args, server_data): print(f"Build failed on {server['hostname']}: {e}") try: - os.remove(zip_file) + if zip_file: + os.remove(zip_file) except Exception as e: print(f"Error removing zip file: {e}") + def delete_cache(server_data): available_servers = [s for s in server_data if s["status"] == "ready"] print(f"Deleting cache in from all available servers ({len(available_servers)})") for server in available_servers: try: - response = requests.get(f"http://{server['ip']}:9001/delete_cache") + response = requests.get(f"http://{server['ip']}:{server.get('port', DEFAULT_PORT)}/delete_cache") response.raise_for_status() print(f"Cache cleared on {server['hostname']}") except Exception as e: @@ -182,18 +211,19 @@ def update_worker(server): try: print(f"Updating {server['hostname']} from {server.get('agent_version')} => {build_agent_version}") - with open("build_agent.py", "rb") as file1, open("requirements.txt", "rb") as file2: + with open("build_agent.py", "rb") as file1, open("../requirements.txt", "rb") as file2: update_files = { "file1": open("build_agent.py", "rb"), - "file2": open("requirements.txt", "rb") + "file2": open("../requirements.txt", "rb") } - response = requests.post(f"http://{server['ip']}:9001/update", files=update_files) + response = requests.post(f"http://{server['ip']}:{server.get('port', DEFAULT_PORT)}/update", + files=update_files) response.raise_for_status() response_json = response.json() if response_json.get('updated_files') and not response_json.get('error_files'): try: - requests.get(f"http://{server['ip']}:9001/restart", timeout=TIMEOUT) + requests.get(f"http://{server['ip']}:{server.get('port', DEFAULT_PORT)}/restart", timeout=TIMEOUT) except requests.exceptions.ConnectionError: pass return server @@ -227,7 +257,7 @@ def update_build_workers(server_data): while unverified_servers and datetime.now() < end_time: for server_ip in list(unverified_servers.keys()): # Iterate over a copy to avoid modification issues try: - response = requests.get(f"http://{server_ip}:9001/status") + response = requests.get(f"http://{server_ip}:{server.get('port', DEFAULT_PORT)}/status") response.raise_for_status() server_info = unverified_servers[server_ip] # Get full server details agent_version = response.json().get('agent_version') @@ -246,23 +276,42 @@ def update_build_workers(server_data): print("Update complete") +def shutdown_agent(hostname): + print(f"Shutting down hostname: {hostname}") + try: + requests.get(f"http://{hostname}:{DEFAULT_PORT}/shutdown", timeout=TIMEOUT) + except (requests.exceptions.ConnectionError, TimeoutError): + pass + + +def restart_agent(hostname): + print(f"Restarting agent: {hostname}") + try: + requests.get(f"http://{hostname}:{DEFAULT_PORT}/restart", timeout=TIMEOUT) + except (requests.exceptions.ConnectionError, TimeoutError): + pass + + def main(): parser = argparse.ArgumentParser(description="Build agent manager for cross-py-builder") parser.add_argument("--status", action="store_true", help="Get status of available servers") parser.add_argument("--build", type=str, help="Path to the project to build") + parser.add_argument("--checkout", type=str, help="Url to Git repo for checkout") parser.add_argument("-cpu", type=str, help="CPU architecture") parser.add_argument("-os", type=str, help="Operating system") - parser.add_argument("-d", '--download', action="store_true", help="Download after build") + parser.add_argument("-version", type=str, help="Version number for build") parser.add_argument("--delete-cache", action="store_true", help="Delete cache") parser.add_argument("--update-all", action="store_true", help="Update build agent") parser.add_argument("--restart", type=str, help="Hostname to restart") parser.add_argument("--restart-all", action="store_true", help="Restart all agents") parser.add_argument("--shutdown", type=str, help="Hostname to shutdown") - parser.add_argument("--shutdown-all", action="store_true", help="Restart all agents") + parser.add_argument("--shutdown-all", action="store_true", help="Shutdown all agents") args = parser.parse_args() if args.status: - print(tabulate(get_all_servers_status(), headers="keys", tablefmt="grid")) + server_data = get_all_servers_status() + server_data = [x for x in server_data if x['status'] != 'offline'] + print(tabulate(server_data, headers="keys", tablefmt="grid")) return elif args.restart: restart_agent(args.restart) @@ -278,27 +327,13 @@ def main(): shutdown_agent(server_ip) elif args.delete_cache: delete_cache(get_all_servers_status()) - elif args.build: + elif args.build or args.checkout: process_new_job(args, get_all_servers_status()) elif args.update_all: update_build_workers(get_all_servers_status()) else: print("No path given!") -def shutdown_agent(hostname): - print(f"Shutting down hostname: {hostname}") - try: - requests.get(f"http://{hostname}:9001/shutdown", timeout=TIMEOUT) - except (requests.exceptions.ConnectionError, TimeoutError): - pass - -def restart_agent(hostname): - print(f"Restarting agent: {hostname}") - try: - requests.get(f"http://{hostname}:9001/restart", timeout=TIMEOUT) - except (requests.exceptions.ConnectionError, TimeoutError): - pass - if __name__ == "__main__": main() \ No newline at end of file diff --git a/zeroconf_server.py b/cross-py-builder/zeroconf_server.py similarity index 99% rename from zeroconf_server.py rename to cross-py-builder/zeroconf_server.py index 803ca63..0de8d73 100644 --- a/zeroconf_server.py +++ b/cross-py-builder/zeroconf_server.py @@ -53,7 +53,6 @@ class ZeroconfServer: @classmethod def _register_service(cls): try: - info = ServiceInfo( cls.service_type, f"{cls.server_name}.{cls.service_type}", diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..2aecc75 --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +from setuptools import setup, find_packages + +setup( + name="cross-py-builder", + version="0.0.1", + packages=find_packages(), + install_requires=open("requirements.txt").read().splitlines(), + entry_points={ + "console_scripts": [ + "cross-py-agent=cross_py_builder.build_agent:main", + "cross-py-builder=cross_py_builder.agent_manager:main", + ], + }, + author="Brett Williams", + author_email="blw1138@mac.com", + description="A cross-platform Python build system using PyInstaller", + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + url="https://github.com/blw1138/cross-py-builder", + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires='>=3.10', +)