brett/Zordon
1
0
Fork 0
mirror of https://github.com/blw1138/Zordon.git synced 2026-02-05 13:46:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Job Submission CLI (#122)

Browse Source 
* Initial commit of job submission cli tool, with minor fixes in API code

* Refactored and further decoupled server / client code

* Clean up ServerProxy to not use hardcoded loopback addresses
This commit is contained in:
Brett
2025-12-27 18:36:34 -06:00
committed by GitHub
parent 6bfa5629d5
commit 574c6f0755
10 changed files with 350 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/api/add_job_helpers.py
View File

@@ -38,7 +38,7 @@ def handle_uploaded_project_files(request, jobs_list, upload_directory):
uploaded_project = request.files.get('file', None)
project_url = jobs_list[0].get('url', None)
local_path = jobs_list[0].get('local_path', None)
renderer = jobs_list[0].get('renderer')
renderer = jobs_list[0]['renderer']
downloaded_file_url = None
if uploaded_project and uploaded_project.filename:

6
src/api/api_server.py
View File

@@ -34,7 +34,7 @@ ssl._create_default_https_context = ssl._create_unverified_context # disable SS
API_VERSION = "1"
def start_server(hostname=None):
def start_api_server(hostname=None):
# get hostname
if not hostname:
@@ -463,6 +463,10 @@ def get_renderer_help(renderer):
# --------------------------------------------
# Miscellaneous:
# --------------------------------------------
@server.get('/api/heartbeat')
def heartbeat():
return datetime.now().isoformat(), 200
@server.post('/api/job/<job_id>/send_subjob_update_notification')
def subjob_update_notification(job_id):
subjob_details = request.json

41
src/api/server_proxy.py
View File

@@ -21,7 +21,6 @@ categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStat
logger = logging.getLogger()
OFFLINE_MAX = 4
LOOPBACK = '127.0.0.1'
class RenderServerProxy:
@@ -55,14 +54,18 @@ class RenderServerProxy:
def __repr__(self):
return f"<RenderServerProxy - {self.hostname}>"
def connect(self):
return self.status()
def check_connection(self):
try:
return self.request("heartbeat").ok
except Exception:
pass
return False
def is_online(self):
if self.__update_in_background:
return self.__offline_flags < OFFLINE_MAX
else:
return self.get_status() is not None
return self.check_connection()
def status(self):
if not self.is_online():
@@ -102,8 +105,7 @@ class RenderServerProxy:
def request(self, payload, timeout=5):
from src.api.api_server import API_VERSION
hostname = LOOPBACK if self.is_localhost else self.hostname
return requests.get(f'http://{hostname}:{self.port}/api/{payload}', timeout=timeout,
return requests.get(f'http://{self.hostname}:{self.port}/api/{payload}', timeout=timeout,
headers={"X-API-Version": str(API_VERSION)})
# --------------------------------------------
@@ -203,7 +205,7 @@ class RenderServerProxy:
if self.is_localhost:
jobs_with_path = [{'local_path': file_path, **item} for item in job_list]
job_data = json.dumps(jobs_with_path)
url = urljoin(f'http://{LOOPBACK}:{self.port}', '/api/add_job')
url = urljoin(f'http://{self.hostname}:{self.port}', '/api/add_job')
headers = {'Content-Type': 'application/json'}
return requests.post(url, data=job_data, headers=headers)
@@ -245,32 +247,32 @@ class RenderServerProxy:
Returns:
Response: The response from the server.
"""
hostname = LOOPBACK if self.is_localhost else self.hostname
return requests.post(f'http://{hostname}:{self.port}/api/job/{parent_id}/send_subjob_update_notification',
return requests.post(f'http://{self.hostname}:{self.port}/api/job/{parent_id}/send_subjob_update_notification',
json=subjob.json())
# --------------------------------------------
# Renderers:
# Engines:
# --------------------------------------------
def is_engine_available(self, engine_name):
return self.request_data(f'{engine_name}/is_available')
def get_all_engines(self):
# todo: this doesnt work
return self.request_data('all_engines')
def get_renderer_info(self, response_type='standard', timeout=5):
def get_engine_info(self, response_type='standard', timeout=5):
"""
Fetches renderer information from the server.
Fetches engine information from the server.
Args:
response_type (str, optional): Returns standard or full version of renderer info
response_type (str, optional): Returns standard or full version of engine info
timeout (int, optional): The number of seconds to wait for a response from the server. Defaults to 5.
Returns:
dict: A dictionary containing the renderer information.
dict: A dictionary containing the engine information.
"""
all_data = self.request_data(f"renderer_info?response_type={response_type}", timeout=timeout)
all_data = self.request_data(f"engine_info?response_type={response_type}", timeout=timeout)
return all_data
def delete_engine(self, engine, version, system_cpu=None):
@@ -286,21 +288,18 @@ class RenderServerProxy:
Response: The response from the server.
"""
form_data = {'engine': engine, 'version': version, 'system_cpu': system_cpu}
hostname = LOOPBACK if self.is_localhost else self.hostname
return requests.post(f'http://{hostname}:{self.port}/api/delete_engine', json=form_data)
return requests.post(f'http://{self.hostname}:{self.port}/api/delete_engine', json=form_data)
# --------------------------------------------
# Download Files:
# --------------------------------------------
def download_all_job_files(self, job_id, save_path):
hostname = LOOPBACK if self.is_localhost else self.hostname
url = f"http://{hostname}:{self.port}/api/job/{job_id}/download_all"
url = f"http://{self.hostname}:{self.port}/api/job/{job_id}/download_all"
return self.__download_file_from_url(url, output_filepath=save_path)
def download_job_file(self, job_id, job_filename, save_path):
hostname = LOOPBACK if self.is_localhost else self.hostname
url = f"http://{hostname}:{self.port}/api/job/{job_id}/download?filename={job_filename}"
url = f"http://{self.hostname}:{self.port}/api/job/{job_id}/download?filename={job_filename}"
return self.__download_file_from_url(url, output_filepath=save_path)
@staticmethod

195
src/init.py
View File

@@ -1,195 +0,0 @@
import logging
import multiprocessing
import os
import socket
import sys
import threading
from collections import deque
import cpuinfo
import psutil
from src.api.api_server import API_VERSION
from src.api.api_server import start_server
from src.api.preview_manager import PreviewManager
from src.api.serverproxy_manager import ServerProxyManager
from src.distributed_job_manager import DistributedJobManager
from src.engines.engine_manager import EngineManager
from src.render_queue import RenderQueue
from src.utilities.config import Config
from src.utilities.misc_helper import (get_gpu_info, system_safe_path, current_system_cpu, current_system_os,
current_system_os_version, check_for_updates)
from src.utilities.zeroconf_server import ZeroconfServer
from src.version import APP_NAME, APP_VERSION, APP_REPO_NAME, APP_REPO_OWNER
logger = logging.getLogger()
def run(server_only=False) -> int:
"""Initializes the application and runs it.
Args:
server_only: Run in server-only CLI mode. Default is False (runs in GUI mode).
Returns:
int: The exit status code.
"""
def existing_process(process_name):
import psutil
current_pid = os.getpid()
current_process = psutil.Process(current_pid)
for proc in psutil.process_iter(['pid', 'name', 'ppid']):
proc_name = proc.info['name'].lower().rstrip('.exe')
if proc_name == process_name.lower() and proc.info['pid'] != current_pid:
if proc.info['pid'] == current_process.ppid():
continue # parent process
elif proc.info['ppid'] == current_pid:
continue # child process
else:
return proc # unrelated process
return None
# setup logging
logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S',
level=Config.server_log_level.upper())
logging.getLogger("requests").setLevel(logging.WARNING) # suppress noisy requests/urllib3 logging
logging.getLogger("urllib3").setLevel(logging.WARNING)
# check for existing instance
existing_proc = existing_process(APP_NAME)
if existing_proc:
logger.fatal(f"Another instance of {APP_NAME} is already running (pid: {existing_proc.pid})")
sys.exit(1)
# Setup logging for console ui
buffer_handler = __setup_buffer_handler() if not server_only else None
# check for updates
update_thread = threading.Thread(target=check_for_updates, args=(APP_REPO_NAME, APP_REPO_OWNER, APP_NAME,
APP_VERSION))
update_thread.start()
# main start
logger.info(f"Starting {APP_NAME} Render Server")
return_code = 0
try:
# Load Config YAML
Config.setup_config_dir()
Config.load_config(system_safe_path(os.path.join(Config.config_dir(), 'config.yaml')))
# configure default paths
EngineManager.engines_path = system_safe_path(
os.path.join(os.path.join(os.path.expanduser(Config.upload_folder),
'engines')))
os.makedirs(EngineManager.engines_path, exist_ok=True)
PreviewManager.storage_path = system_safe_path(
os.path.join(os.path.expanduser(Config.upload_folder), 'previews'))
# Debug info
logger.debug(f"Upload directory: {os.path.expanduser(Config.upload_folder)}")
logger.debug(f"Thumbs directory: {PreviewManager.storage_path}")
logger.debug(f"Engines directory: {EngineManager.engines_path}")
# Set up the RenderQueue object
RenderQueue.load_state(database_directory=system_safe_path(os.path.expanduser(Config.upload_folder)))
ServerProxyManager.subscribe_to_listener()
DistributedJobManager.subscribe_to_listener()
# check for updates for render engines if configured or on first launch
if Config.update_engines_on_launch or not EngineManager.get_engines():
EngineManager.update_all_engines()
# get hostname
local_hostname = socket.gethostname()
local_hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
# configure and start API server
api_server = threading.Thread(target=start_server, args=(local_hostname,))
api_server.daemon = True
api_server.start()
# start zeroconf server
ZeroconfServer.configure(f"_{APP_NAME.lower()}._tcp.local.", local_hostname, Config.port_number)
ZeroconfServer.properties = {'system_cpu': current_system_cpu(),
'system_cpu_brand': cpuinfo.get_cpu_info()['brand_raw'],
'system_cpu_cores': multiprocessing.cpu_count(),
'system_os': current_system_os(),
'system_os_version': current_system_os_version(),
'system_memory': round(psutil.virtual_memory().total / (1024**3)), # in GB
'gpu_info': get_gpu_info(),
'api_version': API_VERSION}
ZeroconfServer.start()
logger.info(f"{APP_NAME} Render Server started - Hostname: {local_hostname}")
RenderQueue.start() # Start evaluating the render queue
# start in gui or server only (cli) mode
logger.debug(f"Launching in {'server only' if server_only else 'GUI'} mode")
if server_only: # CLI only
api_server.join()
else: # GUI
return_code = __show_gui(buffer_handler)
except KeyboardInterrupt:
pass
except Exception as e:
logging.error(f"Unhandled exception: {e}")
return_code = 1
finally:
# shut down gracefully
logger.info(f"{APP_NAME} Render Server is preparing to shut down")
try:
RenderQueue.prepare_for_shutdown()
except Exception as e:
logger.exception(f"Exception during prepare for shutdown: {e}")
ZeroconfServer.stop()
logger.info(f"{APP_NAME} Render Server has shut down")
return sys.exit(return_code)
def __setup_buffer_handler():
# lazy load GUI frameworks
from PyQt6.QtCore import QObject, pyqtSignal
class BufferingHandler(logging.Handler, QObject):
new_record = pyqtSignal(str)
def __init__(self, capacity=100):
logging.Handler.__init__(self)
QObject.__init__(self)
self.buffer = deque(maxlen=capacity) # Define a buffer with a fixed capacity
def emit(self, record):
try:
msg = self.format(record)
self.buffer.append(msg) # Add message to the buffer
self.new_record.emit(msg) # Emit signal
except RuntimeError:
pass
def get_buffer(self):
return list(self.buffer) # Return a copy of the buffer
buffer_handler = BufferingHandler()
buffer_handler.setFormatter(logging.getLogger().handlers[0].formatter)
new_logger = logging.getLogger()
new_logger.addHandler(buffer_handler)
return buffer_handler
def __show_gui(buffer_handler):
# lazy load GUI frameworks
from PyQt6.QtWidgets import QApplication
# load application
app: QApplication = QApplication(sys.argv)
if app.style().objectName() != 'macos':
app.setStyle('Fusion')
# configure main window
from src.ui.main_window import MainWindow
window: MainWindow = MainWindow()
window.buffer_handler = buffer_handler
window.show()
return app.exec()