Misc cleanup (#73)

* Stop previously running zeroconf instances

* Lots of formatting fixes

* Use f-strings for time delta

* More line fixes

* Update requirements.txt

* More misc cleanup

* Simplify README.md

README.md

@@ -1,19 +1,10 @@
-# 🎬 Zordon - Render Management Tools 🎬
+# 🎬 Zordon - Render Management Tools
 
-Welcome to Zordon! This is a hobby project written with fellow filmmakers in mind. It's a local network render farm manager, aiming to streamline and simplify the rendering process across multiple home computers.
+Welcome to Zordon! It's a local network render farm manager, aiming to streamline and simplify the rendering process across multiple home computers.
 
 ## 📦 Installation
 
-Make sure to install the necessary dependencies: `pip3 install -r requirements.txt`
+Install the necessary dependencies: `pip3 install -r requirements.txt`
-
-## 🚀 How to Use
-
-Zordon has two main files: `start_server.py` and `start_client.py`.
-
-- **start_server.py**: Run this on any computer you want to render jobs. It manages the incoming job queue and kicks off the appropriate render jobs when ready.
-- **start_client.py**: Run this to administer your render servers. It lets you manage and submit jobs.
-
-When the server is running, the job queue can be accessed via a web browser on the server's hostname (default port is 8080). You can also access it via the GUI client or a simple view-only dashboard.
 
 ## 🎨 Supported Renderers
 

requirements.txt

@@ -1,15 +1,35 @@
-requests==2.31.0
+PyQt6>=6.6.1
-psutil==5.9.6
+psutil>=5.9.8
-PyYAML==6.0.1
+requests>=2.31.0
-Flask==3.0.0
+Pillow>=10.2.0
-rich==13.6.0
+json2html>=1.3.0
-Werkzeug~=3.0.1
+PyYAML>=6.0.1
-json2html~=1.3.0
+flask>=3.0.1
-SQLAlchemy~=2.0.15
+tqdm>=4.66.1
-Pillow==10.1.0
+werkzeug>=3.0.1
-zeroconf==0.119.0
+Pypubsub>=4.0.3
-Pypubsub~=4.0.3
+zeroconf>=0.131.0
-tqdm==4.66.1
+SQLAlchemy>=2.0.25
-plyer==2.1.0
+plyer>=2.1.0
-PyQt6~=6.6.0
+pytz>=2023.3.post1
-PySide6~=6.6.0
+future>=0.18.3
+rich>=13.7.0
+pytest>=8.0.0
+numpy>=1.26.3
+setuptools>=69.0.3
+pandas>=2.2.0
+matplotlib>=3.8.2
+MarkupSafe>=2.1.4
+python-dateutil>=2.8.2
+certifi>=2023.11.17
+PySide6>=6.6.1
+shiboken6>=6.6.1
+Pygments>=2.17.2
+cycler>=0.12.1
+contourpy>=1.2.0
+packaging>=23.2
+fonttools>=4.47.2
+Jinja2>=3.1.3
+pyparsing>=3.1.1
+kiwisolver>=1.4.5
+attrs>=23.2.0

src/api/add_job_helpers.py

@@ -86,7 +86,6 @@ def download_project_from_url(project_url):
     # This nested function is to handle downloading from a URL
     logger.info(f"Downloading project from url: {project_url}")
     referred_name = os.path.basename(project_url)
-    downloaded_file_url = None
 
     try:
         response = requests.get(project_url, stream=True)
@@ -156,7 +155,7 @@ def process_zipped_project(zip_path):
     return extracted_project_path
 
 
-def create_render_jobs(jobs_list, loaded_project_local_path, job_dir):
+def create_render_jobs(jobs_list, loaded_project_local_path):
     """
     Creates render jobs.
 
@@ -166,7 +165,6 @@ def create_render_jobs(jobs_list, loaded_project_local_path, job_dir):
     Args:
         jobs_list (list): A list of job data.
         loaded_project_local_path (str): The local path to the loaded project.
-        job_dir (str): The job directory.
 
     Returns:
         list: A list of results from creating the render jobs.

src/api/api_server.py

@@ -181,7 +181,7 @@ def make_job_ready(job_id):
             RenderQueue.save_state()
             return found_job.json(), 200
     except Exception as e:
-        return "Error making job ready: {e}", 500
+        return f"Error making job ready: {e}", 500
     return "Not valid command", 405
 
 
@@ -213,8 +213,8 @@ def download_all(job_id):
 def presets():
     presets_path = system_safe_path('config/presets.yaml')
     with open(presets_path) as f:
-        presets = yaml.load(f, Loader=yaml.FullLoader)
+        loaded_presets = yaml.load(f, Loader=yaml.FullLoader)
-    return presets
+    return loaded_presets
 
 
 @server.get('/api/full_status')
@@ -268,7 +268,7 @@ def add_job_handler():
         if loaded_project_local_path.lower().endswith('.zip'):
             loaded_project_local_path = process_zipped_project(loaded_project_local_path)
 
-        results = create_render_jobs(jobs_list, loaded_project_local_path, referred_name)
+        results = create_render_jobs(jobs_list, loaded_project_local_path)
         for response in results:
             if response.get('error', None):
                 return results, 400
@@ -376,7 +376,8 @@ def renderer_info():
         if installed_versions:
             # fixme: using system versions only because downloaded versions may have permissions issues
             system_installed_versions = [x for x in installed_versions if x['type'] == 'system']
-            install_path = system_installed_versions[0]['path'] if system_installed_versions else installed_versions[0]['path']
+            install_path = system_installed_versions[0]['path'] if system_installed_versions else (
+                installed_versions)[0]['path']
             renderer_data[engine.name()] = {'is_available': RenderQueue.is_available_for_job(engine.name()),
                                             'versions': installed_versions,
                                             'supported_extensions': engine.supported_extensions(),
@@ -482,7 +483,7 @@ def start_server():
     flask_log = logging.getLogger('werkzeug')
     flask_log.setLevel(Config.flask_log_level.upper())
 
-    # check for updates for render engines if config'd or on first launch
+    # check for updates for render engines if configured or on first launch
     if Config.update_engines_on_launch or not EngineManager.all_engines():
         EngineManager.update_all_engines()
 

src/api/serverproxy_manager.py

@@ -32,4 +32,3 @@ class ServerProxyManager:
             cls.server_proxys[hostname] = new_proxy
             found_proxy = new_proxy
         return found_proxy
-

src/distributed_job_manager.py

@@ -221,8 +221,9 @@ class DistributedJobManager:
         """
         Splits a job into subjobs and distributes them among available servers.
 
-        This method checks the availability of servers, distributes the work among them, and creates subjobs on each server.
+        This method checks the availability of servers, distributes the work among them, and creates subjobs on each
-        If a server is the local host, it adjusts the frame range of the parent job instead of creating a subjob.
+        server. If a server is the local host, it adjusts the frame range of the parent job instead of creating a
+        subjob.
 
         Args:
             worker (Worker): The worker that is handling the job.
@@ -303,8 +304,8 @@ class DistributedJobManager:
             Defaults to 'cpu_count'.
 
         Returns:
-            list: A list of server dictionaries where each dictionary includes the frame range and total number of frames
+            list: A list of server dictionaries where each dictionary includes the frame range and total number of
-            to be rendered by the server.
+            frames to be rendered by the server.
         """
         # Calculate respective frames for each server
         def divide_frames_by_cpu_count(frame_start, frame_end, servers):

src/engines/blender/blender_downloader.py

@@ -15,7 +15,6 @@ supported_formats = ['.zip', '.tar.xz', '.dmg']
 
 
 class BlenderDownloader(EngineDownloader):
-
     engine = Blender
 
     @staticmethod
@@ -43,11 +42,13 @@ class BlenderDownloader(EngineDownloader):
             response = requests.get(base_url, timeout=5)
             response.raise_for_status()
 
-            versions_pattern = r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
+            versions_pattern = \
+                r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
             versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)]
 
             # Filter to just the supported formats
-            versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)]
+            versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in
+                                                                   supported_formats)]
 
             # Filter down OS and CPU
             system_os = system_os or current_system_os()
@@ -105,7 +106,8 @@ class BlenderDownloader(EngineDownloader):
         try:
             logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}")
             major_version = '.'.join(version.split('.')[:2])
-            minor_versions = [x for x in cls.__get_minor_versions(major_version, system_os, cpu) if x['version'] == version]
+            minor_versions = [x for x in cls.__get_minor_versions(major_version, system_os, cpu) if
+                              x['version'] == version]
             # we get the URL instead of calculating it ourselves. May change this
 
             cls.download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location,
@@ -117,5 +119,4 @@ class BlenderDownloader(EngineDownloader):
 if __name__ == '__main__':
     logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 
-    print(BlenderDownloader.__get_major_versions())
+    print(BlenderDownloader.find_most_recent_version())
-

src/engines/core/base_downloader.py

@@ -98,7 +98,7 @@ class EngineDownloader:
                         zip_ref.extractall(download_location)
                     logger.info(
                         f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
-                except zipfile.BadZipFile as e:
+                except zipfile.BadZipFile:
                     logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.')
                 except FileNotFoundError:
                     logger.error(f'File not found: {temp_downloaded_file_path}')
@@ -110,7 +110,8 @@ class EngineDownloader:
                 for mount_point in dmg.attach():
                     try:
                         copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name))
-                        logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}')
+                        logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} '
+                                    f'to {download_location}')
                     except FileNotFoundError:
                         logger.error(f'Error: The source .app bundle does not exist.')
                     except PermissionError:

src/engines/core/base_engine.py

@@ -71,4 +71,3 @@ class BaseRenderEngine(object):
 
     def perform_presubmission_tasks(self, project_path):
         return project_path
-

src/engines/core/base_worker.py

@@ -227,7 +227,8 @@ class BaseRenderWorker(Base):
                     return
 
                 if not return_code:
-                    message = f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in {self.time_elapsed()}"
+                    message = (f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in "
+                               f"{self.time_elapsed()}")
                     f.write(message)
                     break
 

src/engines/engine_manager.py

@@ -86,7 +86,8 @@ class EngineManager:
         cpu = cpu or current_system_cpu()
 
         try:
-            filtered = [x for x in cls.all_versions_for_engine(engine) if x['system_os'] == system_os and x['cpu'] == cpu]
+            filtered = [x for x in cls.all_versions_for_engine(engine) if x['system_os'] == system_os and
+                        x['cpu'] == cpu]
             return filtered[0]
         except IndexError:
             logger.error(f"Cannot find newest engine version for {engine}-{system_os}-{cpu}")
@@ -98,7 +99,8 @@ class EngineManager:
         cpu = cpu or current_system_cpu()
 
         filtered = [x for x in cls.all_engines() if
-                    x['engine'] == engine and x['system_os'] == system_os and x['cpu'] == cpu and x['version'] == version]
+                    x['engine'] == engine and x['system_os'] == system_os and x['cpu'] == cpu and
+                    x['version'] == version]
         return filtered[0] if filtered else False
 
     @classmethod
@@ -107,6 +109,7 @@ class EngineManager:
             downloader = cls.engine_with_name(engine).downloader()
             return downloader.version_is_available_to_download(version=version, system_os=system_os, cpu=cpu)
         except Exception as e:
+            logger.debug(f"Exception in version_is_available_to_download: {e}")
             return None
 
     @classmethod
@@ -115,10 +118,11 @@ class EngineManager:
             downloader = cls.engine_with_name(engine).downloader()
             return downloader.find_most_recent_version(system_os=system_os, cpu=cpu)
         except Exception as e:
+            logger.debug(f"Exception in find_most_recent_version: {e}")
             return None
 
     @classmethod
-    def is_already_downloading(cls, engine, version, system_os=None, cpu=None):
+    def get_existing_download_task(cls, engine, version, system_os=None, cpu=None):
         for task in cls.download_tasks:
             task_parts = task.name.split('-')
             task_engine, task_version, task_system_os, task_cpu = task_parts[:4]
@@ -126,17 +130,17 @@ class EngineManager:
             if engine == task_engine and version == task_version:
                 if system_os in (task_system_os, None) and cpu in (task_cpu, None):
                     return task
-        return False
+        return None
 
     @classmethod
     def download_engine(cls, engine, version, system_os=None, cpu=None, background=False):
 
         engine_to_download = cls.engine_with_name(engine)
-        existing_task = cls.is_already_downloading(engine, version, system_os, cpu)
+        existing_task = cls.get_existing_download_task(engine, version, system_os, cpu)
         if existing_task:
             logger.debug(f"Already downloading {engine} {version}")
             if not background:
-                existing_task.join()  # If download task exists, wait until its done downloading
+                existing_task.join()  # If download task exists, wait until it's done downloading
             return
         elif not engine_to_download.downloader():
             logger.warning("No valid downloader for this engine. Please update this software manually.")
@@ -157,7 +161,6 @@ class EngineManager:
                 logger.error(f"Error downloading {engine}")
             return found_engine
 
-
     @classmethod
     def delete_engine_download(cls, engine, version, system_os=None, cpu=None):
         logger.info(f"Requested deletion of engine: {engine}-{version}")
@@ -180,14 +183,14 @@ class EngineManager:
 
     @classmethod
     def update_all_engines(cls):
-        def engine_update_task(engine):
+        def engine_update_task(engine_class):
-            logger.debug(f"Checking for updates to {engine.name()}")
+            logger.debug(f"Checking for updates to {engine_class.name()}")
-            latest_version = engine.downloader().find_most_recent_version()
+            latest_version = engine_class.downloader().find_most_recent_version()
             if latest_version:
-                logger.debug(f"Latest version of {engine.name()} available: {latest_version.get('version')}")
+                logger.debug(f"Latest version of {engine_class.name()} available: {latest_version.get('version')}")
-                if not cls.is_version_downloaded(engine.name(), latest_version.get('version')):
+                if not cls.is_version_downloaded(engine_class.name(), latest_version.get('version')):
-                    logger.info(f"Downloading latest version of {engine.name()}...")
+                    logger.info(f"Downloading latest version of {engine_class.name()}...")
-                    cls.download_engine(engine=engine.name(), version=latest_version['version'], background=True)
+                    cls.download_engine(engine=engine_class.name(), version=latest_version['version'], background=True)
             else:
                 logger.warning(f"Unable to get check for updates for {engine.name()}")
 
@@ -199,7 +202,6 @@ class EngineManager:
                 threads.append(thread)
                 thread.start()
 
-
     @classmethod
     def create_worker(cls, renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None):
 

src/engines/ffmpeg/ffmpeg_downloader.py

@@ -182,4 +182,5 @@ if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
     # print(FFMPEGDownloader.download_engine('6.0', '/Users/brett/zordon-uploads/engines/'))
     # print(FFMPEGDownloader.find_most_recent_version(system_os='linux'))
-    print(FFMPEGDownloader.download_engine(version='6.0', download_location='/Users/brett/zordon-uploads/engines/', system_os='linux', cpu='x64'))
+    print(FFMPEGDownloader.download_engine(version='6.0', download_location='/Users/brett/zordon-uploads/engines/',
+                                           system_os='linux', cpu='x64'))

src/engines/ffmpeg/ffmpeg_engine.py

@@ -5,7 +5,6 @@ from src.engines.core.base_engine import *
 
 
 class FFMPEG(BaseRenderEngine):
-
     binary_names = {'linux': 'ffmpeg', 'windows': 'ffmpeg.exe', 'macos': 'ffmpeg'}
 
     @staticmethod
@@ -83,7 +82,7 @@ class FFMPEG(BaseRenderEngine):
     def get_encoders(self):
         raw_stdout = subprocess.check_output([self.renderer_path(), '-encoders'], stderr=subprocess.DEVNULL,
                                              timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
-        pattern = '(?P<type>[VASFXBD.]{6})\s+(?P<name>\S{2,})\s+(?P<description>.*)'
+        pattern = r'(?P<type>[VASFXBD.]{6})\s+(?P<name>\S{2,})\s+(?P<description>.*)'
         encoders = [m.groupdict() for m in re.finditer(pattern, raw_stdout)]
         return encoders
 
@@ -155,4 +154,4 @@ class FFMPEG(BaseRenderEngine):
 
 
 if __name__ == "__main__":
-    print(FFMPEG().get_all_formats())
+    print(FFMPEG().supported_extensions())

src/engines/ffmpeg/ffmpeg_worker.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 import re
-import subprocess
 
 from src.engines.core.base_worker import BaseRenderWorker
 from src.engines.ffmpeg.ffmpeg_engine import FFMPEG

src/ui/add_job.py

@@ -21,6 +21,12 @@ from src.utilities.zeroconf_server import ZeroconfServer
 class NewRenderJobForm(QWidget):
     def __init__(self, project_path=None):
         super().__init__()
+        self.notes_group = None
+        self.frame_rate_input = None
+        self.resolution_x_input = None
+        self.renderer_group = None
+        self.output_settings_group = None
+        self.resolution_y_input = None
         self.project_path = project_path
 
         # UI
@@ -358,7 +364,7 @@ class NewRenderJobForm(QWidget):
                     text_box = QLineEdit()
                     h_layout.addWidget(text_box)
                 self.renderer_options_layout.addLayout(h_layout)
-        except AttributeError as e:
+        except AttributeError:
             pass
 
     def toggle_renderer_enablement(self, enabled=False):

src/ui/main_window.py

@@ -48,6 +48,11 @@ class MainWindow(QMainWindow):
         super().__init__()
 
         # Load the queue
+        self.job_list_view = None
+        self.server_info_ram = None
+        self.server_info_cpu = None
+        self.server_info_os = None
+        self.server_info_hostname = None
         self.engine_browser_window = None
         self.server_info_group = None
         self.current_hostname = None
@@ -300,7 +305,7 @@ class MainWindow(QMainWindow):
             except ConnectionError as e:
                 logger.error(f"Connection error fetching image: {e}")
             except Exception as e:
-                logger.error(f"Error fetching image: {e}")
+                logger.exception(f"Error fetching image: {e}")
 
         job_id = self.selected_job_ids()[0] if self.selected_job_ids() else None
         local_server = is_localhost(self.current_hostname)
@@ -339,12 +344,15 @@ class MainWindow(QMainWindow):
             self.topbar.actions_call['Open Files'].setVisible(False)
 
     def selected_job_ids(self):
+        try:
             selected_rows = self.job_list_view.selectionModel().selectedRows()
             job_ids = []
             for selected_row in selected_rows:
                 id_item = self.job_list_view.item(selected_row.row(), 0)
                 job_ids.append(id_item.text())
             return job_ids
+        except AttributeError:
+            return []
 
     def refresh_job_headers(self):
         self.job_list_view.setHorizontalHeaderLabels(["ID", "Name", "Renderer", "Priority", "Status",

src/utilities/ffmpeg_helper.py

@@ -4,9 +4,10 @@ from src.engines.ffmpeg.ffmpeg_engine import FFMPEG
 
 def image_sequence_to_video(source_glob_pattern, output_path, framerate=24, encoder="prores_ks", profile=4,
                             start_frame=1):
-    subprocess.run([FFMPEG.default_renderer_path(), "-framerate", str(framerate), "-start_number", str(start_frame), "-i",
+    subprocess.run([FFMPEG.default_renderer_path(), "-framerate", str(framerate), "-start_number",
-                    f"{source_glob_pattern}", "-c:v", encoder, "-profile:v", str(profile), '-pix_fmt', 'yuva444p10le',
+                    str(start_frame), "-i", f"{source_glob_pattern}", "-c:v", encoder, "-profile:v", str(profile),
-                    output_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
+                    '-pix_fmt', 'yuva444p10le', output_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                   check=True)
 
 
 def save_first_frame(source_path, dest_path, max_width=1280):

src/utilities/misc_helper.py

@@ -36,9 +36,9 @@ def file_exists_in_mounts(filepath):
         path = os.path.normpath(path)
         components = []
         while True:
-            path, component = os.path.split(path)
+            path, comp = os.path.split(path)
-            if component:
+            if comp:
-                components.append(component)
+                components.append(comp)
             else:
                 if path:
                     components.append(path)
@@ -64,20 +64,17 @@ def file_exists_in_mounts(filepath):
 
 def get_time_elapsed(start_time=None, end_time=None):
 
-    from string import Template
-
-    class DeltaTemplate(Template):
-        delimiter = "%"
-
     def strfdelta(tdelta, fmt='%H:%M:%S'):
-        d = {"D": tdelta.days}
+        days = tdelta.days
         hours, rem = divmod(tdelta.seconds, 3600)
         minutes, seconds = divmod(rem, 60)
-        d["H"] = '{:02d}'.format(hours)
+
-        d["M"] = '{:02d}'.format(minutes)
+        # Using f-strings for formatting
-        d["S"] = '{:02d}'.format(seconds)
+        formatted_str = fmt.replace('%D', f'{days}')
-        t = DeltaTemplate(fmt)
+        formatted_str = formatted_str.replace('%H', f'{hours:02d}')
-        return t.substitute(**d)
+        formatted_str = formatted_str.replace('%M', f'{minutes:02d}')
+        formatted_str = formatted_str.replace('%S', f'{seconds:02d}')
+        return formatted_str
 
     # calculate elapsed time
     elapsed_time = None
@@ -95,7 +92,7 @@ def get_time_elapsed(start_time=None, end_time=None):
 def get_file_size_human(file_path):
     size_in_bytes = os.path.getsize(file_path)
 
-    # Convert size to a human readable format
+    # Convert size to a human-readable format
     if size_in_bytes < 1024:
         return f"{size_in_bytes} B"
     elif size_in_bytes < 1024 ** 2:

src/utilities/zeroconf_server.py

@@ -22,6 +22,10 @@ class ZeroconfServer:
         cls.service_type = service_type
         cls.server_name = server_name
         cls.server_port = server_port
+        try:  # Stop any previously running instances
+            socket.gethostbyname(socket.gethostname())
+        except socket.gaierror:
+            cls.stop()
 
     @classmethod
     def start(cls, listen_only=False):
@@ -82,7 +86,7 @@ class ZeroconfServer:
     def found_hostnames(cls):
         fetched_hostnames = [x.split(f'.{cls.service_type}')[0] for x in cls.client_cache.keys()]
         local_hostname = socket.gethostname()
-        # Define a sort key function
+
         def sort_key(hostname):
             # Return 0 if it's the local hostname so it comes first, else return 1
             return False if hostname == local_hostname else True
@@ -98,6 +102,7 @@ class ZeroconfServer:
         decoded_server_info = {key.decode('utf-8'): value.decode('utf-8') for key, value in server_info.items()}
         return decoded_server_info
 
+
 # Example usage:
 if __name__ == "__main__":
     ZeroconfServer.configure("_zordon._tcp.local.", "foobar.local", 8080)