brett/Zordon
1
0
Fork 0
mirror of https://github.com/blw1138/Zordon.git synced 2026-02-04 21:26:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Job Submission CLI (#122)

Browse Source 
* Initial commit of job submission cli tool, with minor fixes in API code

* Refactored and further decoupled server / client code

* Clean up ServerProxy to not use hardcoded loopback addresses
This commit is contained in:
Brett
2025-12-27 18:36:34 -06:00
committed by GitHub
parent 6bfa5629d5
commit 574c6f0755
10 changed files with 350 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/create-executables.yml vendored
View File

@@ -13,7 +13,7 @@ jobs:
uses: sayyid5416/pyinstaller@v1
with:
python_ver: '3.11'
spec: 'main.spec'
spec: 'client.spec'
requirements: 'requirements.txt'
upload_exe_with_name: 'Zordon'
pyinstaller-build-linux:
@@ -23,7 +23,7 @@ jobs:
uses: sayyid5416/pyinstaller@v1
with:
python_ver: '3.11'
spec: 'main.spec'
spec: 'client.spec'
requirements: 'requirements.txt'
upload_exe_with_name: 'Zordon'
pyinstaller-build-macos:
@@ -33,6 +33,6 @@ jobs:
uses: sayyid5416/pyinstaller@v1
with:
python_ver: '3.11'
spec: 'main.spec'
spec: 'client.spec'
requirements: 'requirements.txt'
upload_exe_with_name: 'Zordon'

63
client.py Executable file
View File

@@ -0,0 +1,63 @@
#!/usr/bin/env python3
import logging
import threading
from collections import deque
from server import start_server
logger = logging.getLogger()
def __setup_buffer_handler():
# lazy load GUI frameworks
from PyQt6.QtCore import QObject, pyqtSignal
class BufferingHandler(logging.Handler, QObject):
new_record = pyqtSignal(str)
def __init__(self, capacity=100):
logging.Handler.__init__(self)
QObject.__init__(self)
self.buffer = deque(maxlen=capacity) # Define a buffer with a fixed capacity
def emit(self, record):
try:
msg = self.format(record)
self.buffer.append(msg) # Add message to the buffer
self.new_record.emit(msg) # Emit signal
except RuntimeError:
pass
def get_buffer(self):
return list(self.buffer) # Return a copy of the buffer
buffer_handler = BufferingHandler()
buffer_handler.setFormatter(logging.getLogger().handlers[0].formatter)
new_logger = logging.getLogger()
new_logger.addHandler(buffer_handler)
return buffer_handler
def __show_gui(buffer_handler):
# lazy load GUI frameworks
from PyQt6.QtWidgets import QApplication
# load application
app: QApplication = QApplication(sys.argv)
if app.style().objectName() != 'macos':
app.setStyle('Fusion')
# configure main window
from src.ui.main_window import MainWindow
window: MainWindow = MainWindow()
window.buffer_handler = buffer_handler
window.show()
return app.exec()
if __name__ == '__main__':
import sys
local_server_thread = threading.Thread(target=start_server, args=[True], daemon=True)
local_server_thread.start()
__show_gui(__setup_buffer_handler())
sys.exit()

0
main.spec → client.spec
View File

120
job_launcher.py Normal file
View File

@@ -0,0 +1,120 @@
#!/usr/bin/env python3
import argparse
import logging
import os
import socket
import sys
import threading
import time
from server import start_server
from src.api.serverproxy_manager import ServerProxyManager
logger = logging.getLogger()
def main():
parser = argparse.ArgumentParser(
description="Zordon CLI tool for preparing/submitting a render job",
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
# Required arguments
parser.add_argument("scene_file", help="Path to the scene file (e.g., .blend, .max, .mp4)")
parser.add_argument("engine", help="Desired render engine", choices=['blender', 'ffmpeg'])
# Frame range
parser.add_argument("--start", type=int, default=1, help="Start frame")
parser.add_argument("--end", type=int, default=1, help="End frame")
# Job metadata
parser.add_argument("--name", default=None, help="Job name")
# Output
parser.add_argument("--output", default="", help="Output path/pattern (e.g., /renders/frame_####.exr)")
# Target OS and Engine Version
parser.add_argument(
"--os",
choices=["any", "windows", "linux", "macos"],
default="any",
help="Target operating system for render workers"
)
parser.add_argument(
"--engine-version",
default="latest",
help="Required renderer/engine version number (e.g., '4.2', '5.0')"
)
# Optional flags
parser.add_argument("--dry-run", action="store_true", help="Print job details without submitting")
args = parser.parse_args()
# Basic validation
if not os.path.exists(args.scene_file):
print(f"Error: Scene file '{args.scene_file}' not found!", file=sys.stderr)
sys.exit(1)
if args.start > args.end:
print("Error: Start frame cannot be greater than end frame!", file=sys.stderr)
sys.exit(1)
# Calculate total frames
total_frames = len(range(args.start, args.end + 1))
job_name = args.name or os.path.basename(args.scene_file)
file_path = os.path.abspath(args.scene_file)
# Print job summary
print("Render Job Summary:")
print(f" Job Name : {job_name}")
print(f" Scene File : {file_path}")
print(f" Engine : {args.engine}")
print(f" Frames : {args.start}-{args.end} → {total_frames} frames")
print(f" Output Path : {args.output or '(default from scene)'}")
print(f" Target OS : {args.os}")
print(f" Engine Version : {args.engine_version}")
if args.dry_run:
print("\nDry run complete (no submission performed).")
return
local_hostname = socket.gethostname()
local_hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
found_proxy = ServerProxyManager.get_proxy_for_hostname(local_hostname)
is_connected = found_proxy.check_connection()
if not is_connected:
local_server_thread = threading.Thread(target=start_server, args=[True], daemon=True)
local_server_thread.start()
while not is_connected:
# todo: add timeout
# is_connected = found_proxy.check_connection()
time.sleep(1)
new_job = {"name": job_name, "renderer": args.engine}
response = found_proxy.post_job_to_server(file_path, [new_job])
if response and response.ok:
print(f"Uploaded to {found_proxy.hostname} successfully!")
running_job_data = response.json()[0]
job_id = running_job_data.get('id')
print(f"Job {job_id} Summary:")
print(f" Status : {running_job_data.get('status')}")
print(f" Engine : {running_job_data.get('renderer')}-{running_job_data.get('renderer_version')}")
print("\nWaiting for render to complete...")
percent_complete = 0.0
while percent_complete < 1.0:
# add checks for errors
time.sleep(1)
running_job_data = found_proxy.get_job_info(job_id)
percent_complete = running_job_data['percent_complete']
sys.stdout.write("\x1b[1A") # Move up 1
sys.stdout.write("\x1b[0J") # Clear from cursor to end of screen (optional)
print(f"Percent Complete: {percent_complete:.2%}")
sys.stdout.flush()
print("Finished rendering successfully!")
if __name__ == "__main__":
main()

7
main.py
View File

@@ -1,7 +0,0 @@
#!/usr/bin/env python3
from src import init
if __name__ == '__main__':
import sys
sys.exit(init.run())

141
server.py
View File

@@ -1,5 +1,140 @@
#!/usr/bin/env python3
from src.init import run
import logging
import multiprocessing
import os
import socket
import sys
import threading
import cpuinfo
import psutil
from src.api.api_server import API_VERSION
from src.api.api_server import start_api_server
from src.api.preview_manager import PreviewManager
from src.api.serverproxy_manager import ServerProxyManager
from src.distributed_job_manager import DistributedJobManager
from src.engines.engine_manager import EngineManager
from src.render_queue import RenderQueue
from src.utilities.config import Config
from src.utilities.misc_helper import (get_gpu_info, system_safe_path, current_system_cpu, current_system_os,
current_system_os_version, check_for_updates)
from src.utilities.zeroconf_server import ZeroconfServer
from src.version import APP_NAME, APP_VERSION, APP_REPO_NAME, APP_REPO_OWNER
logger = logging.getLogger()
def start_server(skip_updates=False) -> int:
"""Initializes the application and runs it.
Args:
server_only: Run in server-only CLI mode. Default is False (runs in GUI mode).
Returns:
int: The exit status code.
"""
def existing_process(process_name):
import psutil
current_pid = os.getpid()
current_process = psutil.Process(current_pid)
for proc in psutil.process_iter(['pid', 'name', 'ppid']):
proc_name = proc.info['name'].lower().rstrip('.exe')
if proc_name == process_name.lower() and proc.info['pid'] != current_pid:
if proc.info['pid'] == current_process.ppid():
continue # parent process
elif proc.info['ppid'] == current_pid:
continue # child process
else:
return proc # unrelated process
return None
# setup logging
logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S',
level=Config.server_log_level.upper())
logging.getLogger("requests").setLevel(logging.WARNING) # suppress noisy requests/urllib3 logging
logging.getLogger("urllib3").setLevel(logging.WARNING)
# check for existing instance
existing_proc = existing_process(APP_NAME)
if existing_proc:
logger.fatal(f"Another instance of {APP_NAME} is already running (pid: {existing_proc.pid})")
sys.exit(1)
# check for updates
if not skip_updates:
update_thread = threading.Thread(target=check_for_updates, args=(APP_REPO_NAME, APP_REPO_OWNER, APP_NAME,
APP_VERSION))
update_thread.start()
# main start
logger.info(f"Starting {APP_NAME} Render Server")
return_code = 0
try:
# Load Config YAML
Config.setup_config_dir()
Config.load_config(system_safe_path(os.path.join(Config.config_dir(), 'config.yaml')))
# configure default paths
EngineManager.engines_path = system_safe_path(
os.path.join(os.path.join(os.path.expanduser(Config.upload_folder),
'engines')))
os.makedirs(EngineManager.engines_path, exist_ok=True)
PreviewManager.storage_path = system_safe_path(
os.path.join(os.path.expanduser(Config.upload_folder), 'previews'))
# Debug info
logger.debug(f"Upload directory: {os.path.expanduser(Config.upload_folder)}")
logger.debug(f"Thumbs directory: {PreviewManager.storage_path}")
logger.debug(f"Engines directory: {EngineManager.engines_path}")
# Set up the RenderQueue object
RenderQueue.load_state(database_directory=system_safe_path(os.path.expanduser(Config.upload_folder)))
ServerProxyManager.subscribe_to_listener()
DistributedJobManager.subscribe_to_listener()
# check for updates for render engines if configured or on first launch
if Config.update_engines_on_launch or not EngineManager.get_engines():
EngineManager.update_all_engines()
# get hostname
local_hostname = socket.gethostname()
# configure and start API server
api_server = threading.Thread(target=start_api_server, args=(local_hostname,))
api_server.daemon = True
api_server.start()
# start zeroconf server
ZeroconfServer.configure(f"_{APP_NAME.lower()}._tcp.local.", local_hostname, Config.port_number)
ZeroconfServer.properties = {'system_cpu': current_system_cpu(),
'system_cpu_brand': cpuinfo.get_cpu_info()['brand_raw'],
'system_cpu_cores': multiprocessing.cpu_count(),
'system_os': current_system_os(),
'system_os_version': current_system_os_version(),
'system_memory': round(psutil.virtual_memory().total / (1024**3)), # in GB
'gpu_info': get_gpu_info(),
'api_version': API_VERSION}
ZeroconfServer.start()
logger.info(f"{APP_NAME} Render Server started - Hostname: {local_hostname}")
RenderQueue.start() # Start evaluating the render queue
api_server.join()
except KeyboardInterrupt:
pass
except Exception as e:
logging.error(f"Unhandled exception: {e}")
return_code = 1
finally:
# shut down gracefully
logger.info(f"{APP_NAME} Render Server is preparing to shut down")
try:
RenderQueue.prepare_for_shutdown()
except Exception as e:
logger.exception(f"Exception during prepare for shutdown: {e}")
ZeroconfServer.stop()
logger.info(f"{APP_NAME} Render Server has shut down")
return sys.exit(return_code)
if __name__ == '__main__':
run(server_only=True)
start_server()

2
src/api/add_job_helpers.py
View File

@@ -38,7 +38,7 @@ def handle_uploaded_project_files(request, jobs_list, upload_directory):
uploaded_project = request.files.get('file', None)
project_url = jobs_list[0].get('url', None)
local_path = jobs_list[0].get('local_path', None)
renderer = jobs_list[0].get('renderer')
renderer = jobs_list[0]['renderer']
downloaded_file_url = None
if uploaded_project and uploaded_project.filename:

6
src/api/api_server.py
View File

@@ -34,7 +34,7 @@ ssl._create_default_https_context = ssl._create_unverified_context # disable SS
API_VERSION = "1"
def start_server(hostname=None):
def start_api_server(hostname=None):
# get hostname
if not hostname:
@@ -463,6 +463,10 @@ def get_renderer_help(renderer):
# --------------------------------------------
# Miscellaneous:
# --------------------------------------------
@server.get('/api/heartbeat')
def heartbeat():
return datetime.now().isoformat(), 200
@server.post('/api/job/<job_id>/send_subjob_update_notification')
def subjob_update_notification(job_id):
subjob_details = request.json

41
src/api/server_proxy.py
View File

@@ -21,7 +21,6 @@ categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStat
logger = logging.getLogger()
OFFLINE_MAX = 4
LOOPBACK = '127.0.0.1'
class RenderServerProxy:
@@ -55,14 +54,18 @@ class RenderServerProxy:
def __repr__(self):
return f"<RenderServerProxy - {self.hostname}>"
def connect(self):
return self.status()
def check_connection(self):
try:
return self.request("heartbeat").ok
except Exception:
pass
return False
def is_online(self):
if self.__update_in_background:
return self.__offline_flags < OFFLINE_MAX
else:
return self.get_status() is not None
return self.check_connection()
def status(self):
if not self.is_online():
@@ -102,8 +105,7 @@ class RenderServerProxy:
def request(self, payload, timeout=5):
from src.api.api_server import API_VERSION
hostname = LOOPBACK if self.is_localhost else self.hostname
return requests.get(f'http://{hostname}:{self.port}/api/{payload}', timeout=timeout,
return requests.get(f'http://{self.hostname}:{self.port}/api/{payload}', timeout=timeout,
headers={"X-API-Version": str(API_VERSION)})
# --------------------------------------------
@@ -203,7 +205,7 @@ class RenderServerProxy:
if self.is_localhost:
jobs_with_path = [{'local_path': file_path, **item} for item in job_list]
job_data = json.dumps(jobs_with_path)
url = urljoin(f'http://{LOOPBACK}:{self.port}', '/api/add_job')
url = urljoin(f'http://{self.hostname}:{self.port}', '/api/add_job')
headers = {'Content-Type': 'application/json'}
return requests.post(url, data=job_data, headers=headers)
@@ -245,32 +247,32 @@ class RenderServerProxy:
Returns:
Response: The response from the server.
"""
hostname = LOOPBACK if self.is_localhost else self.hostname
return requests.post(f'http://{hostname}:{self.port}/api/job/{parent_id}/send_subjob_update_notification',
return requests.post(f'http://{self.hostname}:{self.port}/api/job/{parent_id}/send_subjob_update_notification',
json=subjob.json())
# --------------------------------------------
# Renderers:
# Engines:
# --------------------------------------------
def is_engine_available(self, engine_name):
return self.request_data(f'{engine_name}/is_available')
def get_all_engines(self):
# todo: this doesnt work
return self.request_data('all_engines')
def get_renderer_info(self, response_type='standard', timeout=5):
def get_engine_info(self, response_type='standard', timeout=5):
"""
Fetches renderer information from the server.
Fetches engine information from the server.
Args:
response_type (str, optional): Returns standard or full version of renderer info
response_type (str, optional): Returns standard or full version of engine info
timeout (int, optional): The number of seconds to wait for a response from the server. Defaults to 5.
Returns:
dict: A dictionary containing the renderer information.
dict: A dictionary containing the engine information.
"""
all_data = self.request_data(f"renderer_info?response_type={response_type}", timeout=timeout)
all_data = self.request_data(f"engine_info?response_type={response_type}", timeout=timeout)
return all_data
def delete_engine(self, engine, version, system_cpu=None):
@@ -286,21 +288,18 @@ class RenderServerProxy:
Response: The response from the server.
"""
form_data = {'engine': engine, 'version': version, 'system_cpu': system_cpu}
hostname = LOOPBACK if self.is_localhost else self.hostname
return requests.post(f'http://{hostname}:{self.port}/api/delete_engine', json=form_data)
return requests.post(f'http://{self.hostname}:{self.port}/api/delete_engine', json=form_data)
# --------------------------------------------
# Download Files:
# --------------------------------------------
def download_all_job_files(self, job_id, save_path):
hostname = LOOPBACK if self.is_localhost else self.hostname
url = f"http://{hostname}:{self.port}/api/job/{job_id}/download_all"
url = f"http://{self.hostname}:{self.port}/api/job/{job_id}/download_all"
return self.__download_file_from_url(url, output_filepath=save_path)
def download_job_file(self, job_id, job_filename, save_path):
hostname = LOOPBACK if self.is_localhost else self.hostname
url = f"http://{hostname}:{self.port}/api/job/{job_id}/download?filename={job_filename}"
url = f"http://{self.hostname}:{self.port}/api/job/{job_id}/download?filename={job_filename}"
return self.__download_file_from_url(url, output_filepath=save_path)
@staticmethod

195
src/init.py
View File

@@ -1,195 +0,0 @@
import logging
import multiprocessing
import os
import socket
import sys
import threading
from collections import deque
import cpuinfo
import psutil
from src.api.api_server import API_VERSION
from src.api.api_server import start_server
from src.api.preview_manager import PreviewManager
from src.api.serverproxy_manager import ServerProxyManager
from src.distributed_job_manager import DistributedJobManager
from src.engines.engine_manager import EngineManager
from src.render_queue import RenderQueue
from src.utilities.config import Config
from src.utilities.misc_helper import (get_gpu_info, system_safe_path, current_system_cpu, current_system_os,
current_system_os_version, check_for_updates)
from src.utilities.zeroconf_server import ZeroconfServer
from src.version import APP_NAME, APP_VERSION, APP_REPO_NAME, APP_REPO_OWNER
logger = logging.getLogger()
def run(server_only=False) -> int:
"""Initializes the application and runs it.
Args:
server_only: Run in server-only CLI mode. Default is False (runs in GUI mode).
Returns:
int: The exit status code.
"""
def existing_process(process_name):
import psutil
current_pid = os.getpid()
current_process = psutil.Process(current_pid)
for proc in psutil.process_iter(['pid', 'name', 'ppid']):
proc_name = proc.info['name'].lower().rstrip('.exe')
if proc_name == process_name.lower() and proc.info['pid'] != current_pid:
if proc.info['pid'] == current_process.ppid():
continue # parent process
elif proc.info['ppid'] == current_pid:
continue # child process
else:
return proc # unrelated process
return None
# setup logging
logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S',
level=Config.server_log_level.upper())
logging.getLogger("requests").setLevel(logging.WARNING) # suppress noisy requests/urllib3 logging
logging.getLogger("urllib3").setLevel(logging.WARNING)
# check for existing instance
existing_proc = existing_process(APP_NAME)
if existing_proc:
logger.fatal(f"Another instance of {APP_NAME} is already running (pid: {existing_proc.pid})")
sys.exit(1)
# Setup logging for console ui
buffer_handler = __setup_buffer_handler() if not server_only else None
# check for updates
update_thread = threading.Thread(target=check_for_updates, args=(APP_REPO_NAME, APP_REPO_OWNER, APP_NAME,
APP_VERSION))
update_thread.start()
# main start
logger.info(f"Starting {APP_NAME} Render Server")
return_code = 0
try:
# Load Config YAML
Config.setup_config_dir()
Config.load_config(system_safe_path(os.path.join(Config.config_dir(), 'config.yaml')))
# configure default paths
EngineManager.engines_path = system_safe_path(
os.path.join(os.path.join(os.path.expanduser(Config.upload_folder),
'engines')))
os.makedirs(EngineManager.engines_path, exist_ok=True)
PreviewManager.storage_path = system_safe_path(
os.path.join(os.path.expanduser(Config.upload_folder), 'previews'))
# Debug info
logger.debug(f"Upload directory: {os.path.expanduser(Config.upload_folder)}")
logger.debug(f"Thumbs directory: {PreviewManager.storage_path}")
logger.debug(f"Engines directory: {EngineManager.engines_path}")
# Set up the RenderQueue object
RenderQueue.load_state(database_directory=system_safe_path(os.path.expanduser(Config.upload_folder)))
ServerProxyManager.subscribe_to_listener()
DistributedJobManager.subscribe_to_listener()
# check for updates for render engines if configured or on first launch
if Config.update_engines_on_launch or not EngineManager.get_engines():
EngineManager.update_all_engines()
# get hostname
local_hostname = socket.gethostname()
local_hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
# configure and start API server
api_server = threading.Thread(target=start_server, args=(local_hostname,))
api_server.daemon = True
api_server.start()
# start zeroconf server
ZeroconfServer.configure(f"_{APP_NAME.lower()}._tcp.local.", local_hostname, Config.port_number)
ZeroconfServer.properties = {'system_cpu': current_system_cpu(),
'system_cpu_brand': cpuinfo.get_cpu_info()['brand_raw'],
'system_cpu_cores': multiprocessing.cpu_count(),
'system_os': current_system_os(),
'system_os_version': current_system_os_version(),
'system_memory': round(psutil.virtual_memory().total / (1024**3)), # in GB
'gpu_info': get_gpu_info(),
'api_version': API_VERSION}
ZeroconfServer.start()
logger.info(f"{APP_NAME} Render Server started - Hostname: {local_hostname}")
RenderQueue.start() # Start evaluating the render queue
# start in gui or server only (cli) mode
logger.debug(f"Launching in {'server only' if server_only else 'GUI'} mode")
if server_only: # CLI only
api_server.join()
else: # GUI
return_code = __show_gui(buffer_handler)
except KeyboardInterrupt:
pass
except Exception as e:
logging.error(f"Unhandled exception: {e}")
return_code = 1
finally:
# shut down gracefully
logger.info(f"{APP_NAME} Render Server is preparing to shut down")
try:
RenderQueue.prepare_for_shutdown()
except Exception as e:
logger.exception(f"Exception during prepare for shutdown: {e}")
ZeroconfServer.stop()
logger.info(f"{APP_NAME} Render Server has shut down")
return sys.exit(return_code)
def __setup_buffer_handler():
# lazy load GUI frameworks
from PyQt6.QtCore import QObject, pyqtSignal
class BufferingHandler(logging.Handler, QObject):
new_record = pyqtSignal(str)
def __init__(self, capacity=100):
logging.Handler.__init__(self)
QObject.__init__(self)
self.buffer = deque(maxlen=capacity) # Define a buffer with a fixed capacity
def emit(self, record):
try:
msg = self.format(record)
self.buffer.append(msg) # Add message to the buffer
self.new_record.emit(msg) # Emit signal
except RuntimeError:
pass
def get_buffer(self):
return list(self.buffer) # Return a copy of the buffer
buffer_handler = BufferingHandler()
buffer_handler.setFormatter(logging.getLogger().handlers[0].formatter)
new_logger = logging.getLogger()
new_logger.addHandler(buffer_handler)
return buffer_handler
def __show_gui(buffer_handler):
# lazy load GUI frameworks
from PyQt6.QtWidgets import QApplication
# load application
app: QApplication = QApplication(sys.argv)
if app.style().objectName() != 'macos':
app.setStyle('Fusion')
# configure main window
from src.ui.main_window import MainWindow
window: MainWindow = MainWindow()
window.buffer_handler = buffer_handler
window.show()
return app.exec()