Misc cleanup (#73)

* Stop previously running zeroconf instances

* Lots of formatting fixes

* Use f-strings for time delta

* More line fixes

* Update requirements.txt

* More misc cleanup

* Simplify README.md
This commit is contained in:
2024-01-27 22:56:33 -06:00
committed by GitHub
parent d216ae822e
commit d673d7d4bf
21 changed files with 136 additions and 106 deletions

View File

@@ -1,19 +1,10 @@
# 🎬 Zordon - Render Management Tools 🎬
# 🎬 Zordon - Render Management Tools
Welcome to Zordon! This is a hobby project written with fellow filmmakers in mind. It's a local network render farm manager, aiming to streamline and simplify the rendering process across multiple home computers.
Welcome to Zordon! It's a local network render farm manager, aiming to streamline and simplify the rendering process across multiple home computers.
## 📦 Installation
Make sure to install the necessary dependencies: `pip3 install -r requirements.txt`
## 🚀 How to Use
Zordon has two main files: `start_server.py` and `start_client.py`.
- **start_server.py**: Run this on any computer you want to render jobs. It manages the incoming job queue and kicks off the appropriate render jobs when ready.
- **start_client.py**: Run this to administer your render servers. It lets you manage and submit jobs.
When the server is running, the job queue can be accessed via a web browser on the server's hostname (default port is 8080). You can also access it via the GUI client or a simple view-only dashboard.
Install the necessary dependencies: `pip3 install -r requirements.txt`
## 🎨 Supported Renderers

View File

@@ -1,15 +1,35 @@
requests==2.31.0
psutil==5.9.6
PyYAML==6.0.1
Flask==3.0.0
rich==13.6.0
Werkzeug~=3.0.1
json2html~=1.3.0
SQLAlchemy~=2.0.15
Pillow==10.1.0
zeroconf==0.119.0
Pypubsub~=4.0.3
tqdm==4.66.1
plyer==2.1.0
PyQt6~=6.6.0
PySide6~=6.6.0
PyQt6>=6.6.1
psutil>=5.9.8
requests>=2.31.0
Pillow>=10.2.0
json2html>=1.3.0
PyYAML>=6.0.1
flask>=3.0.1
tqdm>=4.66.1
werkzeug>=3.0.1
Pypubsub>=4.0.3
zeroconf>=0.131.0
SQLAlchemy>=2.0.25
plyer>=2.1.0
pytz>=2023.3.post1
future>=0.18.3
rich>=13.7.0
pytest>=8.0.0
numpy>=1.26.3
setuptools>=69.0.3
pandas>=2.2.0
matplotlib>=3.8.2
MarkupSafe>=2.1.4
python-dateutil>=2.8.2
certifi>=2023.11.17
PySide6>=6.6.1
shiboken6>=6.6.1
Pygments>=2.17.2
cycler>=0.12.1
contourpy>=1.2.0
packaging>=23.2
fonttools>=4.47.2
Jinja2>=3.1.3
pyparsing>=3.1.1
kiwisolver>=1.4.5
attrs>=23.2.0

View File

@@ -86,7 +86,6 @@ def download_project_from_url(project_url):
# This nested function is to handle downloading from a URL
logger.info(f"Downloading project from url: {project_url}")
referred_name = os.path.basename(project_url)
downloaded_file_url = None
try:
response = requests.get(project_url, stream=True)
@@ -156,7 +155,7 @@ def process_zipped_project(zip_path):
return extracted_project_path
def create_render_jobs(jobs_list, loaded_project_local_path, job_dir):
def create_render_jobs(jobs_list, loaded_project_local_path):
"""
Creates render jobs.
@@ -166,7 +165,6 @@ def create_render_jobs(jobs_list, loaded_project_local_path, job_dir):
Args:
jobs_list (list): A list of job data.
loaded_project_local_path (str): The local path to the loaded project.
job_dir (str): The job directory.
Returns:
list: A list of results from creating the render jobs.

View File

@@ -181,7 +181,7 @@ def make_job_ready(job_id):
RenderQueue.save_state()
return found_job.json(), 200
except Exception as e:
return "Error making job ready: {e}", 500
return f"Error making job ready: {e}", 500
return "Not valid command", 405
@@ -213,8 +213,8 @@ def download_all(job_id):
def presets():
presets_path = system_safe_path('config/presets.yaml')
with open(presets_path) as f:
presets = yaml.load(f, Loader=yaml.FullLoader)
return presets
loaded_presets = yaml.load(f, Loader=yaml.FullLoader)
return loaded_presets
@server.get('/api/full_status')
@@ -268,7 +268,7 @@ def add_job_handler():
if loaded_project_local_path.lower().endswith('.zip'):
loaded_project_local_path = process_zipped_project(loaded_project_local_path)
results = create_render_jobs(jobs_list, loaded_project_local_path, referred_name)
results = create_render_jobs(jobs_list, loaded_project_local_path)
for response in results:
if response.get('error', None):
return results, 400
@@ -376,7 +376,8 @@ def renderer_info():
if installed_versions:
# fixme: using system versions only because downloaded versions may have permissions issues
system_installed_versions = [x for x in installed_versions if x['type'] == 'system']
install_path = system_installed_versions[0]['path'] if system_installed_versions else installed_versions[0]['path']
install_path = system_installed_versions[0]['path'] if system_installed_versions else (
installed_versions)[0]['path']
renderer_data[engine.name()] = {'is_available': RenderQueue.is_available_for_job(engine.name()),
'versions': installed_versions,
'supported_extensions': engine.supported_extensions(),
@@ -482,7 +483,7 @@ def start_server():
flask_log = logging.getLogger('werkzeug')
flask_log.setLevel(Config.flask_log_level.upper())
# check for updates for render engines if config'd or on first launch
# check for updates for render engines if configured or on first launch
if Config.update_engines_on_launch or not EngineManager.all_engines():
EngineManager.update_all_engines()

View File

@@ -32,4 +32,3 @@ class ServerProxyManager:
cls.server_proxys[hostname] = new_proxy
found_proxy = new_proxy
return found_proxy

View File

@@ -221,8 +221,9 @@ class DistributedJobManager:
"""
Splits a job into subjobs and distributes them among available servers.
This method checks the availability of servers, distributes the work among them, and creates subjobs on each server.
If a server is the local host, it adjusts the frame range of the parent job instead of creating a subjob.
This method checks the availability of servers, distributes the work among them, and creates subjobs on each
server. If a server is the local host, it adjusts the frame range of the parent job instead of creating a
subjob.
Args:
worker (Worker): The worker that is handling the job.
@@ -303,8 +304,8 @@ class DistributedJobManager:
Defaults to 'cpu_count'.
Returns:
list: A list of server dictionaries where each dictionary includes the frame range and total number of frames
to be rendered by the server.
list: A list of server dictionaries where each dictionary includes the frame range and total number of
frames to be rendered by the server.
"""
# Calculate respective frames for each server
def divide_frames_by_cpu_count(frame_start, frame_end, servers):

View File

@@ -15,7 +15,6 @@ supported_formats = ['.zip', '.tar.xz', '.dmg']
class BlenderDownloader(EngineDownloader):
engine = Blender
@staticmethod
@@ -43,11 +42,13 @@ class BlenderDownloader(EngineDownloader):
response = requests.get(base_url, timeout=5)
response.raise_for_status()
versions_pattern = r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
versions_pattern = \
r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)]
# Filter to just the supported formats
versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)]
versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in
supported_formats)]
# Filter down OS and CPU
system_os = system_os or current_system_os()
@@ -105,11 +106,12 @@ class BlenderDownloader(EngineDownloader):
try:
logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}")
major_version = '.'.join(version.split('.')[:2])
minor_versions = [x for x in cls.__get_minor_versions(major_version, system_os, cpu) if x['version'] == version]
minor_versions = [x for x in cls.__get_minor_versions(major_version, system_os, cpu) if
x['version'] == version]
# we get the URL instead of calculating it ourselves. May change this
cls.download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location,
timeout=timeout)
timeout=timeout)
except IndexError:
logger.error("Cannot find requested engine")
@@ -117,5 +119,4 @@ class BlenderDownloader(EngineDownloader):
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
print(BlenderDownloader.__get_major_versions())
print(BlenderDownloader.find_most_recent_version())

View File

@@ -98,7 +98,7 @@ class EngineDownloader:
zip_ref.extractall(download_location)
logger.info(
f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
except zipfile.BadZipFile as e:
except zipfile.BadZipFile:
logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.')
except FileNotFoundError:
logger.error(f'File not found: {temp_downloaded_file_path}')
@@ -110,7 +110,8 @@ class EngineDownloader:
for mount_point in dmg.attach():
try:
copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name))
logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}')
logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} '
f'to {download_location}')
except FileNotFoundError:
logger.error(f'Error: The source .app bundle does not exist.')
except PermissionError:

View File

@@ -71,4 +71,3 @@ class BaseRenderEngine(object):
def perform_presubmission_tasks(self, project_path):
return project_path

View File

@@ -227,7 +227,8 @@ class BaseRenderWorker(Base):
return
if not return_code:
message = f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in {self.time_elapsed()}"
message = (f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in "
f"{self.time_elapsed()}")
f.write(message)
break

View File

@@ -86,7 +86,8 @@ class EngineManager:
cpu = cpu or current_system_cpu()
try:
filtered = [x for x in cls.all_versions_for_engine(engine) if x['system_os'] == system_os and x['cpu'] == cpu]
filtered = [x for x in cls.all_versions_for_engine(engine) if x['system_os'] == system_os and
x['cpu'] == cpu]
return filtered[0]
except IndexError:
logger.error(f"Cannot find newest engine version for {engine}-{system_os}-{cpu}")
@@ -98,7 +99,8 @@ class EngineManager:
cpu = cpu or current_system_cpu()
filtered = [x for x in cls.all_engines() if
x['engine'] == engine and x['system_os'] == system_os and x['cpu'] == cpu and x['version'] == version]
x['engine'] == engine and x['system_os'] == system_os and x['cpu'] == cpu and
x['version'] == version]
return filtered[0] if filtered else False
@classmethod
@@ -107,6 +109,7 @@ class EngineManager:
downloader = cls.engine_with_name(engine).downloader()
return downloader.version_is_available_to_download(version=version, system_os=system_os, cpu=cpu)
except Exception as e:
logger.debug(f"Exception in version_is_available_to_download: {e}")
return None
@classmethod
@@ -115,10 +118,11 @@ class EngineManager:
downloader = cls.engine_with_name(engine).downloader()
return downloader.find_most_recent_version(system_os=system_os, cpu=cpu)
except Exception as e:
logger.debug(f"Exception in find_most_recent_version: {e}")
return None
@classmethod
def is_already_downloading(cls, engine, version, system_os=None, cpu=None):
def get_existing_download_task(cls, engine, version, system_os=None, cpu=None):
for task in cls.download_tasks:
task_parts = task.name.split('-')
task_engine, task_version, task_system_os, task_cpu = task_parts[:4]
@@ -126,17 +130,17 @@ class EngineManager:
if engine == task_engine and version == task_version:
if system_os in (task_system_os, None) and cpu in (task_cpu, None):
return task
return False
return None
@classmethod
def download_engine(cls, engine, version, system_os=None, cpu=None, background=False):
engine_to_download = cls.engine_with_name(engine)
existing_task = cls.is_already_downloading(engine, version, system_os, cpu)
existing_task = cls.get_existing_download_task(engine, version, system_os, cpu)
if existing_task:
logger.debug(f"Already downloading {engine} {version}")
if not background:
existing_task.join() # If download task exists, wait until its done downloading
existing_task.join() # If download task exists, wait until it's done downloading
return
elif not engine_to_download.downloader():
logger.warning("No valid downloader for this engine. Please update this software manually.")
@@ -157,7 +161,6 @@ class EngineManager:
logger.error(f"Error downloading {engine}")
return found_engine
@classmethod
def delete_engine_download(cls, engine, version, system_os=None, cpu=None):
logger.info(f"Requested deletion of engine: {engine}-{version}")
@@ -180,14 +183,14 @@ class EngineManager:
@classmethod
def update_all_engines(cls):
def engine_update_task(engine):
logger.debug(f"Checking for updates to {engine.name()}")
latest_version = engine.downloader().find_most_recent_version()
def engine_update_task(engine_class):
logger.debug(f"Checking for updates to {engine_class.name()}")
latest_version = engine_class.downloader().find_most_recent_version()
if latest_version:
logger.debug(f"Latest version of {engine.name()} available: {latest_version.get('version')}")
if not cls.is_version_downloaded(engine.name(), latest_version.get('version')):
logger.info(f"Downloading latest version of {engine.name()}...")
cls.download_engine(engine=engine.name(), version=latest_version['version'], background=True)
logger.debug(f"Latest version of {engine_class.name()} available: {latest_version.get('version')}")
if not cls.is_version_downloaded(engine_class.name(), latest_version.get('version')):
logger.info(f"Downloading latest version of {engine_class.name()}...")
cls.download_engine(engine=engine_class.name(), version=latest_version['version'], background=True)
else:
logger.warning(f"Unable to get check for updates for {engine.name()}")
@@ -199,7 +202,6 @@ class EngineManager:
threads.append(thread)
thread.start()
@classmethod
def create_worker(cls, renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None):

View File

@@ -182,4 +182,5 @@ if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# print(FFMPEGDownloader.download_engine('6.0', '/Users/brett/zordon-uploads/engines/'))
# print(FFMPEGDownloader.find_most_recent_version(system_os='linux'))
print(FFMPEGDownloader.download_engine(version='6.0', download_location='/Users/brett/zordon-uploads/engines/', system_os='linux', cpu='x64'))
print(FFMPEGDownloader.download_engine(version='6.0', download_location='/Users/brett/zordon-uploads/engines/',
system_os='linux', cpu='x64'))

View File

@@ -5,7 +5,6 @@ from src.engines.core.base_engine import *
class FFMPEG(BaseRenderEngine):
binary_names = {'linux': 'ffmpeg', 'windows': 'ffmpeg.exe', 'macos': 'ffmpeg'}
@staticmethod
@@ -83,7 +82,7 @@ class FFMPEG(BaseRenderEngine):
def get_encoders(self):
raw_stdout = subprocess.check_output([self.renderer_path(), '-encoders'], stderr=subprocess.DEVNULL,
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
pattern = '(?P<type>[VASFXBD.]{6})\s+(?P<name>\S{2,})\s+(?P<description>.*)'
pattern = r'(?P<type>[VASFXBD.]{6})\s+(?P<name>\S{2,})\s+(?P<description>.*)'
encoders = [m.groupdict() for m in re.finditer(pattern, raw_stdout)]
return encoders
@@ -94,7 +93,7 @@ class FFMPEG(BaseRenderEngine):
def get_all_formats(self):
try:
formats_raw = subprocess.check_output([self.renderer_path(), '-formats'], stderr=subprocess.DEVNULL,
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
pattern = '(?P<type>[DE]{1,2})\s+(?P<id>\S{2,})\s+(?P<name>.*)'
all_formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)]
return all_formats
@@ -119,8 +118,8 @@ class FFMPEG(BaseRenderEngine):
def get_frame_count(self, path_to_file):
raw_stdout = subprocess.check_output([self.renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy',
'-f', 'null', '-'], stderr=subprocess.STDOUT,
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
'-f', 'null', '-'], stderr=subprocess.STDOUT,
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
match = re.findall(r'frame=\s*(\d+)', raw_stdout)
if match:
frame_number = int(match[-1])
@@ -155,4 +154,4 @@ class FFMPEG(BaseRenderEngine):
if __name__ == "__main__":
print(FFMPEG().get_all_formats())
print(FFMPEG().supported_extensions())

View File

@@ -2,4 +2,4 @@ class FFMPEGUI:
@staticmethod
def get_options(instance):
options = []
return options
return options

View File

@@ -1,6 +1,5 @@
#!/usr/bin/env python3
import re
import subprocess
from src.engines.core.base_worker import BaseRenderWorker
from src.engines.ffmpeg.ffmpeg_engine import FFMPEG

View File

@@ -21,6 +21,12 @@ from src.utilities.zeroconf_server import ZeroconfServer
class NewRenderJobForm(QWidget):
def __init__(self, project_path=None):
super().__init__()
self.notes_group = None
self.frame_rate_input = None
self.resolution_x_input = None
self.renderer_group = None
self.output_settings_group = None
self.resolution_y_input = None
self.project_path = project_path
# UI
@@ -358,7 +364,7 @@ class NewRenderJobForm(QWidget):
text_box = QLineEdit()
h_layout.addWidget(text_box)
self.renderer_options_layout.addLayout(h_layout)
except AttributeError as e:
except AttributeError:
pass
def toggle_renderer_enablement(self, enabled=False):

View File

@@ -48,6 +48,11 @@ class MainWindow(QMainWindow):
super().__init__()
# Load the queue
self.job_list_view = None
self.server_info_ram = None
self.server_info_cpu = None
self.server_info_os = None
self.server_info_hostname = None
self.engine_browser_window = None
self.server_info_group = None
self.current_hostname = None
@@ -300,7 +305,7 @@ class MainWindow(QMainWindow):
except ConnectionError as e:
logger.error(f"Connection error fetching image: {e}")
except Exception as e:
logger.error(f"Error fetching image: {e}")
logger.exception(f"Error fetching image: {e}")
job_id = self.selected_job_ids()[0] if self.selected_job_ids() else None
local_server = is_localhost(self.current_hostname)
@@ -339,12 +344,15 @@ class MainWindow(QMainWindow):
self.topbar.actions_call['Open Files'].setVisible(False)
def selected_job_ids(self):
selected_rows = self.job_list_view.selectionModel().selectedRows()
job_ids = []
for selected_row in selected_rows:
id_item = self.job_list_view.item(selected_row.row(), 0)
job_ids.append(id_item.text())
return job_ids
try:
selected_rows = self.job_list_view.selectionModel().selectedRows()
job_ids = []
for selected_row in selected_rows:
id_item = self.job_list_view.item(selected_row.row(), 0)
job_ids.append(id_item.text())
return job_ids
except AttributeError:
return []
def refresh_job_headers(self):
self.job_list_view.setHorizontalHeaderLabels(["ID", "Name", "Renderer", "Priority", "Status",

View File

@@ -38,7 +38,7 @@ class Config:
@classmethod
def config_dir(cls):
# Setup the config path
# Set up the config path
if current_system_os() == 'macos':
local_config_path = os.path.expanduser('~/Library/Application Support/Zordon')
elif current_system_os() == 'windows':
@@ -49,7 +49,7 @@ class Config:
@classmethod
def setup_config_dir(cls):
# Setup the config path
# Set up the config path
local_config_dir = cls.config_dir()
if os.path.exists(local_config_dir):
return
@@ -71,4 +71,4 @@ class Config:
except Exception as e:
print(f"An error occurred while setting up the config directory: {e}")
raise
raise

View File

@@ -4,9 +4,10 @@ from src.engines.ffmpeg.ffmpeg_engine import FFMPEG
def image_sequence_to_video(source_glob_pattern, output_path, framerate=24, encoder="prores_ks", profile=4,
start_frame=1):
subprocess.run([FFMPEG.default_renderer_path(), "-framerate", str(framerate), "-start_number", str(start_frame), "-i",
f"{source_glob_pattern}", "-c:v", encoder, "-profile:v", str(profile), '-pix_fmt', 'yuva444p10le',
output_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
subprocess.run([FFMPEG.default_renderer_path(), "-framerate", str(framerate), "-start_number",
str(start_frame), "-i", f"{source_glob_pattern}", "-c:v", encoder, "-profile:v", str(profile),
'-pix_fmt', 'yuva444p10le', output_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
check=True)
def save_first_frame(source_path, dest_path, max_width=1280):

View File

@@ -36,9 +36,9 @@ def file_exists_in_mounts(filepath):
path = os.path.normpath(path)
components = []
while True:
path, component = os.path.split(path)
if component:
components.append(component)
path, comp = os.path.split(path)
if comp:
components.append(comp)
else:
if path:
components.append(path)
@@ -64,20 +64,17 @@ def file_exists_in_mounts(filepath):
def get_time_elapsed(start_time=None, end_time=None):
from string import Template
class DeltaTemplate(Template):
delimiter = "%"
def strfdelta(tdelta, fmt='%H:%M:%S'):
d = {"D": tdelta.days}
days = tdelta.days
hours, rem = divmod(tdelta.seconds, 3600)
minutes, seconds = divmod(rem, 60)
d["H"] = '{:02d}'.format(hours)
d["M"] = '{:02d}'.format(minutes)
d["S"] = '{:02d}'.format(seconds)
t = DeltaTemplate(fmt)
return t.substitute(**d)
# Using f-strings for formatting
formatted_str = fmt.replace('%D', f'{days}')
formatted_str = formatted_str.replace('%H', f'{hours:02d}')
formatted_str = formatted_str.replace('%M', f'{minutes:02d}')
formatted_str = formatted_str.replace('%S', f'{seconds:02d}')
return formatted_str
# calculate elapsed time
elapsed_time = None
@@ -95,7 +92,7 @@ def get_time_elapsed(start_time=None, end_time=None):
def get_file_size_human(file_path):
size_in_bytes = os.path.getsize(file_path)
# Convert size to a human readable format
# Convert size to a human-readable format
if size_in_bytes < 1024:
return f"{size_in_bytes} B"
elif size_in_bytes < 1024 ** 2:

View File

@@ -22,6 +22,10 @@ class ZeroconfServer:
cls.service_type = service_type
cls.server_name = server_name
cls.server_port = server_port
try: # Stop any previously running instances
socket.gethostbyname(socket.gethostname())
except socket.gaierror:
cls.stop()
@classmethod
def start(cls, listen_only=False):
@@ -82,7 +86,7 @@ class ZeroconfServer:
def found_hostnames(cls):
fetched_hostnames = [x.split(f'.{cls.service_type}')[0] for x in cls.client_cache.keys()]
local_hostname = socket.gethostname()
# Define a sort key function
def sort_key(hostname):
# Return 0 if it's the local hostname so it comes first, else return 1
return False if hostname == local_hostname else True
@@ -98,6 +102,7 @@ class ZeroconfServer:
decoded_server_info = {key.decode('utf-8'): value.decode('utf-8') for key, value in server_info.items()}
return decoded_server_info
# Example usage:
if __name__ == "__main__":
ZeroconfServer.configure("_zordon._tcp.local.", "foobar.local", 8080)