Ability to checkout git repos, custom ports, and setup for pip

README.md

@@ -1,11 +1,11 @@
-# Cross-Py-Build
+# Cross-Py-Builder
 
-Cross-Py-Build is a simple remote build tool for compiling Python projects with PyInstaller on different platforms on a local network.
+Cross-Py-Builder is a simple remote build tool for compiling Python projects with PyInstaller on different platforms on a local network.
 
 ## System Requirements
 - **Ubuntu/Debian & macOS:** 2GB+ RAM  
 - **Windows:** 4GB+ RAM  
-- **Python 3** installed
+- **Python 3.10 or later** installed
 - **Local Network** for build agents
 
 ---
@@ -14,7 +14,7 @@ Cross-Py-Build is a simple remote build tool for compiling Python projects with
 
 
  ```sh
- ./agent_manager.py --build /path/to/repo -cpu x64 -os windows
+ ./cross-py-builder.py --build /path/to/repo -cpu x64 -os windows
 
 options:
   -h, --help           show this help message and exit
@@ -28,7 +28,7 @@ options:
   --restart RESTART    Hostname to restart
   --restart-all        Restart all agents
   --shutdown SHUTDOWN  Hostname to shutdown
-  --shutdown-all       Restart all agents
+  --shutdown-all       Shutdown all agents
 ```
 
 ---
@@ -50,13 +50,8 @@ This guide provides steps to set up a worker VM on **Ubuntu/Debian, macOS, or Wi
      ```sh
      xcode-select --install  # Install xcode tools
      ```
-     - Install Python 3.x from [python.org](https://www.python.org/downloads/)
+3. **Install Python** >=3.10 from [python.org](https://www.python.org/downloads/) (Windows and macOS - macOS)
-   - **Windows:**
+4. **Set up a virtual environment:**
-     - Install Python 3.x from [python.org](https://www.python.org/downloads/)
-
-
-3. **Set up a virtual environment:**
-
    - **Ubuntu/Debian/macOS:**
        ```sh
        python3 -m venv venv
@@ -65,14 +60,13 @@ This guide provides steps to set up a worker VM on **Ubuntu/Debian, macOS, or Wi
    - **Windows:**
        ```sh
        python3 -m venv venv
-   source venv\Scripts\activate
+       venv\Scripts\activate
        ```
-4. **Copy project files** and install dependencies:
+5. **Copy project files** and install dependencies:
    ```sh
    pip install -r requirements.txt
    ```
-   
+6. **Start Build Agent**:
-5. **Start Build Agent**:
    - **Ubuntu/Debian/macOS:**
       ```sh
       python3 build_agent.py

cross-py-builder/build_agent.py

@@ -18,7 +18,8 @@ import platform
 from zeroconf_server import ZeroconfServer
 
 APP_NAME = "cross-py-builder"
-build_agent_version = "0.1.33"
+build_agent_version = "0.1.34"
+app_port = 9001
 
 app = Flask(__name__)
 launch_time = datetime.datetime.now()
@@ -148,12 +149,39 @@ def status():
                     "python": platform.python_version(),
                     "hostname": hostname,
                     "ip": ZeroconfServer.get_local_ip(),
+                    "port": app_port,
                     "job_id": system_status['running_job'],
                     "cache_size": format_size(get_directory_size(TMP_DIR)),
                     "uptime": str(datetime.datetime.now() - launch_time)
                     })
 
 
+def generate_job_id():
+    return str(uuid.uuid4()).split('-')[-1]
+
+
+@app.route("/checkout_git", methods=['POST'])
+def checkout_project():
+    start_time = datetime.datetime.now()
+    repo_url = request.json.get('repo_url')
+    if not repo_url:
+        return jsonify({'error': 'Repository URL is required'}), 400
+
+    print(f"\n========== Checking Out Git Project ==========")
+
+    job_id = generate_job_id()
+    repo_dir = os.path.join(TMP_DIR, job_id)
+    try:
+        system_status['status'] = "cloning_repo"
+        subprocess.check_call(['git', 'clone', repo_url, repo_dir])
+        system_status['status'] = "ready"
+    except subprocess.CalledProcessError as e:
+        print(f"Error cloning repo: {e}")
+        system_status['status'] = "ready"
+        return jsonify({'error': 'Failed to clone repository'}), 500
+
+    return install_and_build(repo_dir, job_id, start_time)
+
 @app.route('/upload', methods=['POST'])
 def upload_project():
     try:
@@ -161,8 +189,9 @@ def upload_project():
         if 'file' not in request.files:
             return jsonify({"error": "No file uploaded"}), 400
 
+        system_status['status'] = "processing_files"
         print(f"\n========== Processing Incoming Project ==========")
-        job_id = str(uuid.uuid4()).split('-')[-1]
+        job_id = generate_job_id()
         working_dir = os.path.join(TMP_DIR, BUILD_DIR, job_id)
 
         file = request.files['file']
@@ -180,7 +209,8 @@ def upload_project():
         return install_and_build(working_dir, job_id, start_time)
     except Exception as e:
         print(f"Uncaught error processing job: {e}")
-        jsonify({"error": f"Uncaught error processing job: {e}"}), 500
+        system_status['status'] = "ready"
+        return jsonify({"error": f"Uncaught error processing job: {e}"}), 500
 
 def install_and_build(project_path, job_id, start_time):
 
@@ -196,6 +226,7 @@ def install_and_build(project_path, job_id, start_time):
     # Set up virtual environment
     venv_path = os.path.join(project_path, "venv")
     try:
+        system_status['status'] = "creating_venv"
         print(f"\n========== Configuring Virtual Environment ({venv_path}) ==========")
         python_exec = "python" if is_windows() else "python3"
         subprocess.run([python_exec, "-m", "venv", venv_path], check=True)
@@ -212,6 +243,7 @@ def install_and_build(project_path, job_id, start_time):
 
     # Install requirements
     try:
+        system_status['status'] = "installing_packages"
         subprocess.run([py_exec, "-m", "pip", "install", "--upgrade", "pip"], check=True)
         subprocess.run([py_exec, "-m", "pip", "install", "pyinstaller", "pyinstaller_versionfile", "--prefer-binary"], check=True)
         requirements_path = os.path.join(project_path, "requirements.txt")
@@ -230,6 +262,7 @@ def install_and_build(project_path, job_id, start_time):
     try:
         for index, spec_file in enumerate(spec_files):
             # Compile with PyInstaller
+            system_status['status'] = "compiling"
             print(f"\n========== Compiling spec file {index+1} of {len(spec_files)} - {spec_file} ==========")
             simple_name = os.path.splitext(os.path.basename(spec_file))[0]
             dist_path = os.path.join(project_path, "dist")
@@ -242,17 +275,26 @@ def install_and_build(project_path, job_id, start_time):
                     [py_exec, "-m", "PyInstaller", spec_file, "--distpath", dist_path, "--workpath", work_path],
                     text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
                 )
+                last_line = None
                 for line in process.stdout:
-                    print(line, end="")  # Print to console
+                    last_line = line
-                    log_file.write(line)  # Save to log file
+                    print(line, end="")
-                    log_file.flush()  # Ensure real-time writing
+                    log_file.write(line)
-                process.wait()  # Wait for the process to complete
+                    log_file.flush()
+                process.wait()
+                if process.returncode != 0:
+                    raise RuntimeError(
+                        f"PyInstaller failed with exit code {process.returncode}. Last line: {str(last_line).strip()}")
+
             print(f"\n========== Compilation of spec file {spec_file} complete ==========\n")
     except Exception as e:
         print(f"Error compiling project: {e}")
         system_status['status'] = "ready"
         system_status['running_job'] = None
+        try:
             os.remove(project_path)
+        except PermissionError:
+            pass
         return jsonify({"error": f"Error compiling project: {e}"}), 500
 
     dist_path = os.path.join(project_path, "dist")
@@ -355,10 +397,10 @@ def delete_cache():
 if __name__ == "__main__":
 
     print(f"===== {APP_NAME} Build Agent (v{build_agent_version}) =====")
-    ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), 9001)
+    ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), app_port)
     try:
         ZeroconfServer.start()
-        app.run(host="0.0.0.0", port=9001, threaded=True)
+        app.run(host="0.0.0.0", port=app_port, threaded=True)
     except KeyboardInterrupt:
         pass
     finally:

cross-py-builder/cross-py-builder.py

@@ -9,18 +9,18 @@ import time
 import zipfile
 from datetime import datetime, timedelta
 
+import requests
 from packaging.version import Version
 from tabulate import tabulate
 
-import requests
-
 from build_agent import build_agent_version
 from zeroconf_server import ZeroconfServer
 
+DEFAULT_PORT = 9001
 TIMEOUT = 10
 
 def find_server_ips():
-    ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), 9001)
+    ZeroconfServer.configure("_crosspybuilder._tcp.local.", socket.gethostname(), DEFAULT_PORT)
     hostnames = []
     try:
         ZeroconfServer.start(listen_only=True)
@@ -32,32 +32,44 @@ def find_server_ips():
         ZeroconfServer.stop()
 
     # get known hosts
-    with open("known_hosts", "r") as file:
+    with open("../known_hosts", "r") as file:
         lines = file.readlines()
-
-    lines = [line.split(':')[0].strip() for line in lines]
     hostnames.extend(lines)
 
     return hostnames
 
+
 def get_all_servers_status():
     table_data = []
     server_ips = find_server_ips()
+
     with concurrent.futures.ThreadPoolExecutor() as executor:
-        results = executor.map(get_worker_status, server_ips)  # Runs in parallel
+        futures = []
-    table_data.extend(results)
+        for server in server_ips:
+            ip = server.split(':')[0]
+            port = server.split(":")[-1].strip() if ":" in server else DEFAULT_PORT
+            futures.append(executor.submit(get_worker_status, ip, port))
+
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                result = future.result()  # Get the result of the thread
+                table_data.append(result)
+            except Exception as e:
+                print(f"Error fetching status from server: {e}")  # Handle potential errors
     return table_data
 
-def get_worker_status(hostname):
+
+def get_worker_status(hostname, port=DEFAULT_PORT):
     """Fetch worker status from the given hostname."""
     try:
-        response = requests.get(f"http://{hostname}:9001/status", timeout=TIMEOUT)
+        response = requests.get(f"http://{hostname}:{port}/status", timeout=TIMEOUT)
         status = response.json()
+        status['port'] = port
         if status['hostname'] != hostname and status['ip'] != hostname:
             status['ip'] = socket.gethostbyname(hostname)
         return status
     except requests.exceptions.RequestException as e:
-        return {"hostname": hostname, "status": "offline"}
+        return {"hostname": hostname, "port": port, "status": "offline"}
 
 
 def zip_project(source_dir, output_zip):
@@ -72,26 +84,36 @@ def zip_project(source_dir, output_zip):
                 zipf.write(file_path, os.path.relpath(file_path, source_dir))
 
 
-def send_build_request(zip_file, server_ip, download_after=False):
+def send_build_request(server_ip, server_port=DEFAULT_PORT, zip_file=None, git_url=None, download_after=True, version=None):
-    """Uploads the zip file to the given server."""
+
-    upload_url = f"http://{server_ip}:9001/upload"
+    if zip_file:
+        upload_url = f"http://{server_ip}:{server_port}/upload"
         print(f"Submitting build request to URL: {upload_url} - Please wait. This may take a few minutes...")
         with open(zip_file, 'rb') as f:
             response = requests.post(upload_url, files={"file": f})
+    elif git_url:
+        checkout_url = f"http://{server_ip}:{server_port}/checkout_git"
+        response = requests.post(checkout_url, json={"repo_url": git_url})
+    else:
+        raise ValueError("Missing zip file or git url!")
 
     if response.status_code == 200:
         response_data = response.json()
+        print(response_data)
         print(f"Build successful. ID: {response_data['id']} Hostname: {response_data['hostname']} OS: {response_data['os']} CPU: {response_data['cpu']} Spec files: {len(response_data['spec_files'])} - Elapsed time: {response_data['duration']}" )
         if download_after:
-            download_url = f"http://{server_ip}:9001/download/{response_data.get('id')}"
+            download_url = f"http://{server_ip}:{server_port}/download/{response_data.get('id')}"
             try:
-                save_name = f"{os.path.splitext(os.path.basename(zip_file))[0]}-{response_data['os'].lower()}-{response_data['cpu'].lower()}.zip"
+                base_name = os.path.splitext(os.path.basename(zip_file))[0]
+                version_string = ("-" + version) if version else ""
+                save_name = f"{base_name}{version_string}-{response_data['os'].lower()}-{response_data['cpu'].lower()}.zip"
                 download_zip(download_url, save_name=save_name)
             except Exception as e:
                 print(f"Error downloading zip: {e}")
     else:
         print("Upload failed:", response.status_code, response.text)
 
+
 def download_zip(url, save_name, save_dir="."):
     """Download a ZIP file from a URL and save it with its original filename."""
     response = requests.get(url, stream=True)
@@ -116,6 +138,7 @@ def select_server(servers, cpu=None, os_name=None):
         available = [s for s in available if os_name.lower() in s["os"].lower()]
     return available[0] if available else None  # Return first matching server or None
 
+
 def process_new_job(args, server_data):
     available_servers = [s for s in server_data if s["status"] == "ready"]
 
@@ -139,6 +162,8 @@ def process_new_job(args, server_data):
     print(f"Found {len(available_servers)} servers available to build")
     print(tabulate(available_servers, headers="keys", tablefmt="grid"))
 
+    zip_file = None
+    if args.build:
         project_path = args.build
         tmp_dir = tempfile.gettempdir()
         zip_file = os.path.join(tmp_dir, f"{os.path.basename(project_path)}.zip")
@@ -148,10 +173,12 @@ def process_new_job(args, server_data):
     # Start builds on all matching servers
     with concurrent.futures.ThreadPoolExecutor() as executor:
         print("Submitting builds to:")
+        download = True
         for server in available_servers:
             print(f"\t{server['hostname']} - {server['os']} - {server['cpu']}")
-        futures = {executor.submit(send_build_request, zip_file, server["ip"], args.download): server for server in
+        futures = {executor.submit(
-                   available_servers}
+            send_build_request,server["ip"], DEFAULT_PORT, zip_file, args.checkout, download, args.version):
+                       server for server in available_servers}
 
     # Collect results
     for future in concurrent.futures.as_completed(futures):
@@ -162,16 +189,18 @@ def process_new_job(args, server_data):
             print(f"Build failed on {server['hostname']}: {e}")
 
     try:
+        if zip_file:
             os.remove(zip_file)
     except Exception as e:
         print(f"Error removing zip file: {e}")
 
+
 def delete_cache(server_data):
     available_servers = [s for s in server_data if s["status"] == "ready"]
     print(f"Deleting cache in from all available servers ({len(available_servers)})")
     for server in available_servers:
         try:
-            response = requests.get(f"http://{server['ip']}:9001/delete_cache")
+            response = requests.get(f"http://{server['ip']}:{server.get('port', DEFAULT_PORT)}/delete_cache")
             response.raise_for_status()
             print(f"Cache cleared on {server['hostname']}")
         except Exception as e:
@@ -182,18 +211,19 @@ def update_worker(server):
     try:
         print(f"Updating {server['hostname']} from {server.get('agent_version')} => {build_agent_version}")
 
-        with open("build_agent.py", "rb") as file1, open("requirements.txt", "rb") as file2:
+        with open("build_agent.py", "rb") as file1, open("../requirements.txt", "rb") as file2:
             update_files = {
                 "file1": open("build_agent.py", "rb"),
-                "file2": open("requirements.txt", "rb")
+                "file2": open("../requirements.txt", "rb")
             }
-            response = requests.post(f"http://{server['ip']}:9001/update", files=update_files)
+            response = requests.post(f"http://{server['ip']}:{server.get('port', DEFAULT_PORT)}/update",
+                                     files=update_files)
             response.raise_for_status()
 
         response_json = response.json()
         if response_json.get('updated_files') and not response_json.get('error_files'):
             try:
-                requests.get(f"http://{server['ip']}:9001/restart", timeout=TIMEOUT)
+                requests.get(f"http://{server['ip']}:{server.get('port', DEFAULT_PORT)}/restart", timeout=TIMEOUT)
             except requests.exceptions.ConnectionError:
                 pass
             return server
@@ -227,7 +257,7 @@ def update_build_workers(server_data):
         while unverified_servers and datetime.now() < end_time:
             for server_ip in list(unverified_servers.keys()):  # Iterate over a copy to avoid modification issues
                 try:
-                    response = requests.get(f"http://{server_ip}:9001/status")
+                    response = requests.get(f"http://{server_ip}:{server.get('port', DEFAULT_PORT)}/status")
                     response.raise_for_status()
                     server_info = unverified_servers[server_ip]  # Get full server details
                     agent_version = response.json().get('agent_version')
@@ -246,23 +276,42 @@ def update_build_workers(server_data):
         print("Update complete")
 
 
+def shutdown_agent(hostname):
+    print(f"Shutting down hostname: {hostname}")
+    try:
+        requests.get(f"http://{hostname}:{DEFAULT_PORT}/shutdown", timeout=TIMEOUT)
+    except (requests.exceptions.ConnectionError, TimeoutError):
+        pass
+
+
+def restart_agent(hostname):
+    print(f"Restarting agent: {hostname}")
+    try:
+        requests.get(f"http://{hostname}:{DEFAULT_PORT}/restart", timeout=TIMEOUT)
+    except (requests.exceptions.ConnectionError, TimeoutError):
+        pass
+
+
 def main():
     parser = argparse.ArgumentParser(description="Build agent manager for cross-py-builder")
     parser.add_argument("--status", action="store_true", help="Get status of available servers")
     parser.add_argument("--build", type=str, help="Path to the project to build")
+    parser.add_argument("--checkout", type=str, help="Url to Git repo for checkout")
     parser.add_argument("-cpu", type=str, help="CPU architecture")
     parser.add_argument("-os", type=str, help="Operating system")
-    parser.add_argument("-d", '--download', action="store_true", help="Download after build")
+    parser.add_argument("-version", type=str, help="Version number for build")
     parser.add_argument("--delete-cache",  action="store_true", help="Delete cache")
     parser.add_argument("--update-all",  action="store_true", help="Update build agent")
     parser.add_argument("--restart",  type=str, help="Hostname to restart")
     parser.add_argument("--restart-all",  action="store_true", help="Restart all agents")
     parser.add_argument("--shutdown",  type=str, help="Hostname to shutdown")
-    parser.add_argument("--shutdown-all",  action="store_true", help="Restart all agents")
+    parser.add_argument("--shutdown-all",  action="store_true", help="Shutdown all agents")
     args = parser.parse_args()
 
     if args.status:
-        print(tabulate(get_all_servers_status(), headers="keys", tablefmt="grid"))
+        server_data = get_all_servers_status()
+        server_data = [x for x in server_data if x['status'] != 'offline']
+        print(tabulate(server_data, headers="keys", tablefmt="grid"))
         return
     elif args.restart:
         restart_agent(args.restart)
@@ -278,27 +327,13 @@ def main():
             shutdown_agent(server_ip)
     elif args.delete_cache:
         delete_cache(get_all_servers_status())
-    elif args.build:
+    elif args.build or args.checkout:
         process_new_job(args, get_all_servers_status())
     elif args.update_all:
         update_build_workers(get_all_servers_status())
     else:
         print("No path given!")
 
-def shutdown_agent(hostname):
-    print(f"Shutting down hostname: {hostname}")
-    try:
-        requests.get(f"http://{hostname}:9001/shutdown", timeout=TIMEOUT)
-    except (requests.exceptions.ConnectionError, TimeoutError):
-        pass
-
-def restart_agent(hostname):
-    print(f"Restarting agent: {hostname}")
-    try:
-        requests.get(f"http://{hostname}:9001/restart", timeout=TIMEOUT)
-    except (requests.exceptions.ConnectionError, TimeoutError):
-        pass
-
 
 if __name__ == "__main__":
     main()

cross-py-builder/zeroconf_server.py

@@ -53,7 +53,6 @@ class ZeroconfServer:
     @classmethod
     def _register_service(cls):
         try:
-
             info = ServiceInfo(
                 cls.service_type,
                 f"{cls.server_name}.{cls.service_type}",

setup.py

@@ -0,0 +1,26 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="cross-py-builder",
+    version="0.0.1",
+    packages=find_packages(),
+    install_requires=open("requirements.txt").read().splitlines(),
+    entry_points={
+        "console_scripts": [
+            "cross-py-agent=cross_py_builder.build_agent:main",
+            "cross-py-builder=cross_py_builder.agent_manager:main",
+        ],
+    },
+    author="Brett Williams",
+    author_email="blw1138@mac.com",
+    description="A cross-platform Python build system using PyInstaller",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    url="https://github.com/blw1138/cross-py-builder",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.10',
+)