brett/cross-py-builder
1
0
Fork 0
mirror of https://github.com/blw1138/cross-py-builder.git synced 2025-12-17 08:38:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ability to checkout git repos, custom ports, and setup for pip

Browse Source
This commit is contained in:
Brett Williams
2025-03-12 00:20:25 -05:00
parent 94e0ecda83
commit b0d3660881
6 changed files with 181 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
README.md
View File

@@ -1,11 +1,11 @@
# Cross-Py-Build
# Cross-Py-Builder
Cross-Py-Build is a simple remote build tool for compiling Python projects with PyInstaller on different platforms on a local network.
Cross-Py-Builder is a simple remote build tool for compiling Python projects with PyInstaller on different platforms on a local network.
## System Requirements
- **Ubuntu/Debian & macOS:** 2GB+ RAM
- **Windows:** 4GB+ RAM
- **Python 3** installed
- **Python 3.10 or later** installed
- **Local Network** for build agents
---
@@ -14,7 +14,7 @@ Cross-Py-Build is a simple remote build tool for compiling Python projects with
```sh
./agent_manager.py --build /path/to/repo -cpu x64 -os windows
./cross-py-builder.py --build /path/to/repo -cpu x64 -os windows
options:
-h, --help show this help message and exit
@@ -28,7 +28,7 @@ options:
--restart RESTART Hostname to restart
--restart-all Restart all agents
--shutdown SHUTDOWN Hostname to shutdown
--shutdown-all Restart all agents
--shutdown-all Shutdown all agents
```
---
@@ -50,13 +50,8 @@ This guide provides steps to set up a worker VM on **Ubuntu/Debian, macOS, or Wi
```sh
xcode-select --install # Install xcode tools
```
- Install Python 3.x from [python.org](https://www.python.org/downloads/)
- **Windows:**
- Install Python 3.x from [python.org](https://www.python.org/downloads/)
3. **Set up a virtual environment:**
3. **Install Python** >=3.10 from [python.org](https://www.python.org/downloads/) (Windows and macOS - macOS)
4. **Set up a virtual environment:**
- **Ubuntu/Debian/macOS:**
```sh
python3 -m venv venv
@@ -65,14 +60,13 @@ This guide provides steps to set up a worker VM on **Ubuntu/Debian, macOS, or Wi
- **Windows:**
```sh
python3 -m venv venv
source venv\Scripts\activate
venv\Scripts\activate
```
4. **Copy project files** and install dependencies:
5. **Copy project files** and install dependencies:
```sh
pip install -r requirements.txt
```
5. **Start Build Agent**:
6. **Start Build Agent**:
- **Ubuntu/Debian/macOS:**
```sh
python3 build_agent.py

0
cross-py-builder/__init__.py Normal file
View File

60
build_agent.py → cross-py-builder/build_agent.py
View File

@@ -18,7 +18,8 @@ import platform
from zeroconf_server import ZeroconfServer
APP_NAME = "cross-py-builder"
build_agent_version = "0.1.33"
build_agent_version = "0.1.34"
app_port = 9001
app = Flask(__name__)
launch_time = datetime.datetime.now()
@@ -148,12 +149,39 @@ def status():
"python": platform.python_version(),
"hostname": hostname,
"ip": ZeroconfServer.get_local_ip(),
"port": app_port,
"job_id": system_status['running_job'],
"cache_size": format_size(get_directory_size(TMP_DIR)),
"uptime": str(datetime.datetime.now() - launch_time)
})
def generate_job_id():
return str(uuid.uuid4()).split('-')[-1]
@app.route("/checkout_git", methods=['POST'])
def checkout_project():
start_time = datetime.datetime.now()
repo_url = request.json.get('repo_url')
if not repo_url:
return jsonify({'error': 'Repository URL is required'}), 400
print(f"\n========== Checking Out Git Project ==========")
job_id = generate_job_id()
repo_dir = os.path.join(TMP_DIR, job_id)
try:
system_status['status'] = "cloning_repo"
subprocess.check_call(['git', 'clone', repo_url, repo_dir])
system_status['status'] = "ready"
except subprocess.CalledProcessError as e:
print(f"Error cloning repo: {e}")
system_status['status'] = "ready"
return jsonify({'error': 'Failed to clone repository'}), 500
return install_and_build(repo_dir, job_id, start_time)
@app.route('/upload', methods=['POST'])
def upload_project():
try:
@@ -161,8 +189,9 @@ def upload_project():
if 'file' not in request.files:
return jsonify({"error": "No file uploaded"}), 400
system_status['status'] = "processing_files"
print(f"\n========== Processing Incoming Project ==========")
job_id = str(uuid.uuid4()).split('-')[-1]
job_id = generate_job_id()
working_dir = os.path.join(TMP_DIR, BUILD_DIR, job_id)
file = request.files['file']
@@ -180,7 +209,8 @@ def upload_project():
return install_and_build(working_dir, job_id, start_time)
except Exception as e:
print(f"Uncaught error processing job: {e}")
jsonify({"error": f"Uncaught error processing job: {e}"}), 500
system_status['status'] = "ready"
return jsonify({"error": f"Uncaught error processing job: {e}"}), 500
def install_and_build(project_path, job_id, start_time):
@@ -196,6 +226,7 @@ def install_and_build(project_path, job_id, start_time):
# Set up virtual environment
venv_path = os.path.join(project_path, "venv")
try:
system_status['status'] = "creating_venv"
print(f"\n========== Configuring Virtual Environment ({venv_path}) ==========")
python_exec = "python" if is_windows() else "python3"
subprocess.run([python_exec, "-m", "venv", venv_path], check=True)
@@ -212,6 +243,7 @@ def install_and_build(project_path, job_id, start_time):
# Install requirements
try:
system_status['status'] = "installing_packages"
subprocess.run([py_exec, "-m", "pip", "install", "--upgrade", "pip"], check=True)
subprocess.run([py_exec, "-m", "pip", "install", "pyinstaller", "pyinstaller_versionfile", "--prefer-binary"], check=True)
requirements_path = os.path.join(project_path, "requirements.txt")
@@ -230,6 +262,7 @@ def install_and_build(project_path, job_id, start_time):
try:
for index, spec_file in enumerate(spec_files):
# Compile with PyInstaller
system_status['status'] = "compiling"
print(f"\n========== Compiling spec file {index+1} of {len(spec_files)} - {spec_file} ==========")
simple_name = os.path.splitext(os.path.basename(spec_file))[0]
dist_path = os.path.join(project_path, "dist")
@@ -242,17 +275,26 @@ def install_and_build(project_path, job_id, start_time):
[py_exec, "-m", "PyInstaller", spec_file, "--distpath", dist_path, "--workpath", work_path],
text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
)
last_line = None
for line in process.stdout:
print(line, end="") # Print to console
log_file.write(line) # Save to log file
log_file.flush() # Ensure real-time writing
process.wait() # Wait for the process to complete
last_line = line
print(line, end="")
log_file.write(line)
log_file.flush()
process.wait()
if process.returncode != 0:
raise RuntimeError(
f"PyInstaller failed with exit code {process.returncode}. Last line: {str(last_line).strip()}")
print(f"\n========== Compilation of spec file {spec_file} complete ==========\n")
except Exception as e:
print(f"Error compiling project: {e}")
system_status['status'] = "ready"
system_status['running_job'] = None
try:
os.remove(project_path)
except PermissionError:
pass
return jsonify({"error": f"Error compiling project: {e}"}), 500
dist_path = os.path.join(project_path, "dist")
@@ -355,10 +397,10 @@ def delete_cache():
if __name__ == "__main__":
print(f"===== {APP_NAME} Build Agent (v{build_agent_version}) =====")
ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), 9001)
ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), app_port)
try:
ZeroconfServer.start()
app.run(host="0.0.0.0", port=9001, threaded=True)
app.run(host="0.0.0.0", port=app_port, threaded=True)
except KeyboardInterrupt:
pass
finally:

119
agent_manager.py → cross-py-builder/cross-py-builder.py
View File

@@ -9,18 +9,18 @@ import time
import zipfile
from datetime import datetime, timedelta
import requests
from packaging.version import Version
from tabulate import tabulate
import requests
from build_agent import build_agent_version
from zeroconf_server import ZeroconfServer
DEFAULT_PORT = 9001
TIMEOUT = 10
def find_server_ips():
ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), 9001)
ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), DEFAULT_PORT)
hostnames = []
try:
ZeroconfServer.start(listen_only=True)
@@ -32,32 +32,44 @@ def find_server_ips():
ZeroconfServer.stop()
# get known hosts
with open("known_hosts", "r") as file:
with open("../known_hosts", "r") as file:
lines = file.readlines()
lines = [line.split(':')[0].strip() for line in lines]
hostnames.extend(lines)
return hostnames
def get_all_servers_status():
table_data = []
server_ips = find_server_ips()
with concurrent.futures.ThreadPoolExecutor() as executor:
results = executor.map(get_worker_status, server_ips) # Runs in parallel
table_data.extend(results)
futures = []
for server in server_ips:
ip = server.split(':')[0]
port = server.split(":")[-1].strip() if ":" in server else DEFAULT_PORT
futures.append(executor.submit(get_worker_status, ip, port))
for future in concurrent.futures.as_completed(futures):
try:
result = future.result() # Get the result of the thread
table_data.append(result)
except Exception as e:
print(f"Error fetching status from server: {e}") # Handle potential errors
return table_data
def get_worker_status(hostname):
def get_worker_status(hostname, port=DEFAULT_PORT):
"""Fetch worker status from the given hostname."""
try:
response = requests.get(f"http://{hostname}:9001/status", timeout=TIMEOUT)
response = requests.get(f"http://{hostname}:{port}/status", timeout=TIMEOUT)
status = response.json()
status['port'] = port
if status['hostname'] != hostname and status['ip'] != hostname:
status['ip'] = socket.gethostbyname(hostname)
return status
except requests.exceptions.RequestException as e:
return {"hostname": hostname, "status": "offline"}
return {"hostname": hostname, "port": port, "status": "offline"}
def zip_project(source_dir, output_zip):
@@ -72,26 +84,36 @@ def zip_project(source_dir, output_zip):
zipf.write(file_path, os.path.relpath(file_path, source_dir))
def send_build_request(zip_file, server_ip, download_after=False):
"""Uploads the zip file to the given server."""
upload_url = f"http://{server_ip}:9001/upload"
def send_build_request(server_ip, server_port=DEFAULT_PORT, zip_file=None, git_url=None, download_after=True, version=None):
if zip_file:
upload_url = f"http://{server_ip}:{server_port}/upload"
print(f"Submitting build request to URL: {upload_url} - Please wait. This may take a few minutes...")
with open(zip_file, 'rb') as f:
response = requests.post(upload_url, files={"file": f})
elif git_url:
checkout_url = f"http://{server_ip}:{server_port}/checkout_git"
response = requests.post(checkout_url, json={"repo_url": git_url})
else:
raise ValueError("Missing zip file or git url!")
if response.status_code == 200:
response_data = response.json()
print(response_data)
print(f"Build successful. ID: {response_data['id']} Hostname: {response_data['hostname']} OS: {response_data['os']} CPU: {response_data['cpu']} Spec files: {len(response_data['spec_files'])} - Elapsed time: {response_data['duration']}" )
if download_after:
download_url = f"http://{server_ip}:9001/download/{response_data.get('id')}"
download_url = f"http://{server_ip}:{server_port}/download/{response_data.get('id')}"
try:
save_name = f"{os.path.splitext(os.path.basename(zip_file))[0]}-{response_data['os'].lower()}-{response_data['cpu'].lower()}.zip"
base_name = os.path.splitext(os.path.basename(zip_file))[0]
version_string = ("-" + version) if version else ""
save_name = f"{base_name}{version_string}-{response_data['os'].lower()}-{response_data['cpu'].lower()}.zip"
download_zip(download_url, save_name=save_name)
except Exception as e:
print(f"Error downloading zip: {e}")
else:
print("Upload failed:", response.status_code, response.text)
def download_zip(url, save_name, save_dir="."):
"""Download a ZIP file from a URL and save it with its original filename."""
response = requests.get(url, stream=True)
@@ -116,6 +138,7 @@ def select_server(servers, cpu=None, os_name=None):
available = [s for s in available if os_name.lower() in s["os"].lower()]
return available[0] if available else None # Return first matching server or None
def process_new_job(args, server_data):
available_servers = [s for s in server_data if s["status"] == "ready"]
@@ -139,6 +162,8 @@ def process_new_job(args, server_data):
print(f"Found {len(available_servers)} servers available to build")
print(tabulate(available_servers, headers="keys", tablefmt="grid"))
zip_file = None
if args.build:
project_path = args.build
tmp_dir = tempfile.gettempdir()
zip_file = os.path.join(tmp_dir, f"{os.path.basename(project_path)}.zip")
@@ -148,10 +173,12 @@ def process_new_job(args, server_data):
# Start builds on all matching servers
with concurrent.futures.ThreadPoolExecutor() as executor:
print("Submitting builds to:")
download = True
for server in available_servers:
print(f"\t{server['hostname']} - {server['os']} - {server['cpu']}")
futures = {executor.submit(send_build_request, zip_file, server["ip"], args.download): server for server in
available_servers}
futures = {executor.submit(
send_build_request,server["ip"], DEFAULT_PORT, zip_file, args.checkout, download, args.version):
server for server in available_servers}
# Collect results
for future in concurrent.futures.as_completed(futures):
@@ -162,16 +189,18 @@ def process_new_job(args, server_data):
print(f"Build failed on {server['hostname']}: {e}")
try:
if zip_file:
os.remove(zip_file)
except Exception as e:
print(f"Error removing zip file: {e}")
def delete_cache(server_data):
available_servers = [s for s in server_data if s["status"] == "ready"]
print(f"Deleting cache in from all available servers ({len(available_servers)})")
for server in available_servers:
try:
response = requests.get(f"http://{server['ip']}:9001/delete_cache")
response = requests.get(f"http://{server['ip']}:{server.get('port', DEFAULT_PORT)}/delete_cache")
response.raise_for_status()
print(f"Cache cleared on {server['hostname']}")
except Exception as e:
@@ -182,18 +211,19 @@ def update_worker(server):
try:
print(f"Updating {server['hostname']} from {server.get('agent_version')} => {build_agent_version}")
with open("build_agent.py", "rb") as file1, open("requirements.txt", "rb") as file2:
with open("build_agent.py", "rb") as file1, open("../requirements.txt", "rb") as file2:
update_files = {
"file1": open("build_agent.py", "rb"),
"file2": open("requirements.txt", "rb")
"file2": open("../requirements.txt", "rb")
}
response = requests.post(f"http://{server['ip']}:9001/update", files=update_files)
response = requests.post(f"http://{server['ip']}:{server.get('port', DEFAULT_PORT)}/update",
files=update_files)
response.raise_for_status()
response_json = response.json()
if response_json.get('updated_files') and not response_json.get('error_files'):
try:
requests.get(f"http://{server['ip']}:9001/restart", timeout=TIMEOUT)
requests.get(f"http://{server['ip']}:{server.get('port', DEFAULT_PORT)}/restart", timeout=TIMEOUT)
except requests.exceptions.ConnectionError:
pass
return server
@@ -227,7 +257,7 @@ def update_build_workers(server_data):
while unverified_servers and datetime.now() < end_time:
for server_ip in list(unverified_servers.keys()): # Iterate over a copy to avoid modification issues
try:
response = requests.get(f"http://{server_ip}:9001/status")
response = requests.get(f"http://{server_ip}:{server.get('port', DEFAULT_PORT)}/status")
response.raise_for_status()
server_info = unverified_servers[server_ip] # Get full server details
agent_version = response.json().get('agent_version')
@@ -246,23 +276,42 @@ def update_build_workers(server_data):
print("Update complete")
def shutdown_agent(hostname):
print(f"Shutting down hostname: {hostname}")
try:
requests.get(f"http://{hostname}:{DEFAULT_PORT}/shutdown", timeout=TIMEOUT)
except (requests.exceptions.ConnectionError, TimeoutError):
pass
def restart_agent(hostname):
print(f"Restarting agent: {hostname}")
try:
requests.get(f"http://{hostname}:{DEFAULT_PORT}/restart", timeout=TIMEOUT)
except (requests.exceptions.ConnectionError, TimeoutError):
pass
def main():
parser = argparse.ArgumentParser(description="Build agent manager for cross-py-builder")
parser.add_argument("--status", action="store_true", help="Get status of available servers")
parser.add_argument("--build", type=str, help="Path to the project to build")
parser.add_argument("--checkout", type=str, help="Url to Git repo for checkout")
parser.add_argument("-cpu", type=str, help="CPU architecture")
parser.add_argument("-os", type=str, help="Operating system")
parser.add_argument("-d", '--download', action="store_true", help="Download after build")
parser.add_argument("-version", type=str, help="Version number for build")
parser.add_argument("--delete-cache", action="store_true", help="Delete cache")
parser.add_argument("--update-all", action="store_true", help="Update build agent")
parser.add_argument("--restart", type=str, help="Hostname to restart")
parser.add_argument("--restart-all", action="store_true", help="Restart all agents")
parser.add_argument("--shutdown", type=str, help="Hostname to shutdown")
parser.add_argument("--shutdown-all", action="store_true", help="Restart all agents")
parser.add_argument("--shutdown-all", action="store_true", help="Shutdown all agents")
args = parser.parse_args()
if args.status:
print(tabulate(get_all_servers_status(), headers="keys", tablefmt="grid"))
server_data = get_all_servers_status()
server_data = [x for x in server_data if x['status'] != 'offline']
print(tabulate(server_data, headers="keys", tablefmt="grid"))
return
elif args.restart:
restart_agent(args.restart)
@@ -278,27 +327,13 @@ def main():
shutdown_agent(server_ip)
elif args.delete_cache:
delete_cache(get_all_servers_status())
elif args.build:
elif args.build or args.checkout:
process_new_job(args, get_all_servers_status())
elif args.update_all:
update_build_workers(get_all_servers_status())
else:
print("No path given!")
def shutdown_agent(hostname):
print(f"Shutting down hostname: {hostname}")
try:
requests.get(f"http://{hostname}:9001/shutdown", timeout=TIMEOUT)
except (requests.exceptions.ConnectionError, TimeoutError):
pass
def restart_agent(hostname):
print(f"Restarting agent: {hostname}")
try:
requests.get(f"http://{hostname}:9001/restart", timeout=TIMEOUT)
except (requests.exceptions.ConnectionError, TimeoutError):
pass
if __name__ == "__main__":
main()

1
zeroconf_server.py → cross-py-builder/zeroconf_server.py
View File

@@ -53,7 +53,6 @@ class ZeroconfServer:
@classmethod
def _register_service(cls):
try:
info = ServiceInfo(
cls.service_type,
f"{cls.server_name}.{cls.service_type}",

26
setup.py Normal file
View File

@@ -0,0 +1,26 @@
from setuptools import setup, find_packages
setup(
name="cross-py-builder",
version="0.0.1",
packages=find_packages(),
install_requires=open("requirements.txt").read().splitlines(),
entry_points={
"console_scripts": [
"cross-py-agent=cross_py_builder.build_agent:main",
"cross-py-builder=cross_py_builder.agent_manager:main",
],
},
author="Brett Williams",
author_email="blw1138@mac.com",
description="A cross-platform Python build system using PyInstaller",
long_description=open("README.md").read(),
long_description_content_type="text/markdown",
url="https://github.com/blw1138/cross-py-builder",
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
],
python_requires='>=3.10',
)