Major file reorganization (#26)

* Major file reorganization

* Rearrange imports

* Fix default log level

dashboard.py

@@ -17,9 +17,9 @@ from rich.table import Table
 from rich.text import Text
 from rich.tree import Tree
 
-from lib.workers.base_worker import RenderStatus, string_to_status
+from src.workers.base_worker import RenderStatus, string_to_status
-from lib.server.server_proxy import RenderServerProxy
+from src.server_proxy import RenderServerProxy
-from lib.utilities.misc_helper import get_time_elapsed
+from src.utilities.misc_helper import get_time_elapsed
 from start_server import start_server
 
 """

requirements.txt

@@ -1,15 +1,12 @@
-click~=8.1.3
 requests==2.31.0
 psutil==5.9.5
 PyYAML~=6.0
 Flask==2.3.2
 rich==13.4.1
 Werkzeug==2.3.5
-tkinterdnd2~=0.3.0
 future==0.18.3
 json2html~=1.3.0
 SQLAlchemy~=2.0.15
 Pillow==9.5.0
 zeroconf==0.64.1
-requests-toolbelt~=1.0
+Pypubsub~=4.0.3
-arrow~=1.2.3

src/api_server.py

@@ -20,16 +20,16 @@ import yaml
 from flask import Flask, request, render_template, send_file, after_this_request, Response, redirect, url_for, abort
 from werkzeug.utils import secure_filename
 
-from lib.distributed_job_manager import DistributedJobManager
+from src.distributed_job_manager import DistributedJobManager
-from lib.render_queue import RenderQueue, JobNotFoundError
+from src.render_queue import RenderQueue, JobNotFoundError
-from lib.server.server_proxy import RenderServerProxy
+from src.server_proxy import RenderServerProxy
-from lib.server.zeroconf_server import ZeroconfServer
+from src.utilities.server_helper import generate_thumbnail_for_job
-from lib.utilities.server_helper import generate_thumbnail_for_job
+from src.utilities.zeroconf_server import ZeroconfServer
-from lib.workers.base_worker import string_to_status, RenderStatus
+from src.worker_factory import RenderWorkerFactory
-from lib.workers.worker_factory import RenderWorkerFactory
+from src.workers.base_worker import string_to_status, RenderStatus
 
 logger = logging.getLogger()
-server = Flask(__name__, template_folder='templates', static_folder='static')
+server = Flask(__name__, template_folder='web/templates', static_folder='web/static')
 ssl._create_default_https_context = ssl._create_unverified_context  # disable SSL for downloads
 
 categories = [RenderStatus.RUNNING, RenderStatus.ERROR, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED,
@@ -53,13 +53,12 @@ def sorted_jobs(all_jobs, sort_by_date=True):
 @server.route('/')
 @server.route('/index')
 def index():
-
     with open('config/presets.yaml') as f:
-        presets = yaml.load(f, Loader=yaml.FullLoader)
+        render_presets = yaml.load(f, Loader=yaml.FullLoader)
 
     return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.all_jobs()),
                            hostname=server.config['HOSTNAME'], renderer_info=renderer_info(),
-                           render_clients=[server.config['HOSTNAME']], preset_list=presets)
+                           render_clients=[server.config['HOSTNAME']], preset_list=render_presets)
 
 
 @server.get('/api/jobs')
@@ -131,15 +130,15 @@ def job_thumbnail(job_id):
 
         # Misc status icons
         if found_job.status == RenderStatus.RUNNING:
-            return send_file('static/images/gears.png', mimetype="image/png")
+            return send_file('web/static/images/gears.png', mimetype="image/png")
         elif found_job.status == RenderStatus.CANCELLED:
-            return send_file('static/images/cancelled.png', mimetype="image/png")
+            return send_file('web/static/images/cancelled.png', mimetype="image/png")
         elif found_job.status == RenderStatus.SCHEDULED:
-            return send_file('static/images/scheduled.png', mimetype="image/png")
+            return send_file('web/static/images/scheduled.png', mimetype="image/png")
         elif found_job.status == RenderStatus.NOT_STARTED:
-            return send_file('static/images/not_started.png', mimetype="image/png")
+            return send_file('web/static/images/not_started.png', mimetype="image/png")
     # errors
-    return send_file('static/images/error.png', mimetype="image/png")
+    return send_file('web/static/images/error.png', mimetype="image/png")
 
 
 # Get job file routing
@@ -498,7 +497,6 @@ def clear_history():
 
 @server.route('/api/status')
 def status():
-
     renderer_data = {}
     for render_class in RenderWorkerFactory.supported_classes():
         if render_class.engine.renderer_path():  # only return renderers installed on host

src/client/dashboard_window.py

@@ -1,17 +1,20 @@
 import datetime
 import logging
-import tkinter as tk
+import os
+import socket
 import threading
 import time
-import socket
+import tkinter as tk
-import os
 from tkinter import ttk, messagebox, simpledialog
+
 from PIL import Image, ImageTk
-from lib.client.new_job_window import NewJobWindow
+
-from lib.server.server_proxy import RenderServerProxy
+from src.client.new_job_window import NewJobWindow
-from lib.server.zeroconf_server import ZeroconfServer
+# from src.client.server_details import create_server_popup
-from lib.workers.base_worker import RenderStatus
+from src.server_proxy import RenderServerProxy
-from lib.utilities.misc_helper import launch_url, file_exists_in_mounts, get_time_elapsed
+from src.utilities.misc_helper import launch_url, file_exists_in_mounts, get_time_elapsed
+from src.utilities.zeroconf_server import ZeroconfServer
+from src.workers.base_worker import RenderStatus
 
 logger = logging.getLogger()
 
@@ -31,7 +34,7 @@ def make_sortable(tree):
 class DashboardWindow:
 
     lib_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-    image_path = os.path.join(lib_path, 'server', 'static', 'images')
+    image_path = os.path.join(lib_path, 'web', 'static', 'images')
     default_image = Image.open(os.path.join(image_path, 'desktop.png'))
 
     def __init__(self):

src/client/new_job_window.py

@@ -4,16 +4,16 @@ import logging
 import os.path
 import pathlib
 import socket
+import threading
 from tkinter import *
 from tkinter import filedialog, messagebox
 from tkinter.ttk import Frame, Label, Entry, Combobox, Progressbar
 
 import psutil
-import requests
+
-import threading
+from src.server_proxy import RenderServerProxy
-from lib.workers.blender_worker import Blender
+from src.workers.blender_worker import Blender
-from lib.workers.ffmpeg_worker import FFMPEG
+from src.workers.ffmpeg_worker import FFMPEG
-from lib.server.server_proxy import RenderServerProxy
 
 logger = logging.getLogger()
 

src/distributed_job_manager.py

@@ -5,11 +5,12 @@ import time
 import zipfile
 
 from pubsub import pub
-from lib.render_queue import RenderQueue
+
-from lib.server.server_proxy import RenderServerProxy
+from src.render_queue import RenderQueue
-from lib.server.zeroconf_server import ZeroconfServer
+from src.server_proxy import RenderServerProxy
-from lib.utilities.misc_helper import get_file_size_human
+from src.utilities.misc_helper import get_file_size_human
-from lib.workers.base_worker import RenderStatus, string_to_status
+from src.utilities.status_utils import RenderStatus, string_to_status
+from src.utilities.zeroconf_server import ZeroconfServer
 
 logger = logging.getLogger()
 

src/render_queue.py

@@ -4,8 +4,9 @@ from datetime import datetime
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
-from lib.workers.base_worker import RenderStatus, BaseRenderWorker, Base
+from src.utilities.status_utils import RenderStatus
-from lib.workers.worker_factory import RenderWorkerFactory
+from src.worker_factory import RenderWorkerFactory
+from src.workers.base_worker import Base
 
 logger = logging.getLogger()
 
@@ -83,12 +84,20 @@ class RenderQueue:
 
     @classmethod
     def load_state(cls):
+        from src.workers.base_worker import BaseRenderWorker
         cls.job_queue = cls.session.query(BaseRenderWorker).all()
 
     @classmethod
     def save_state(cls):
         cls.session.commit()
 
+    @classmethod
+    def prepare_for_shutdown(cls):
+        running_jobs = cls.jobs_with_status(RenderStatus.RUNNING)  # cancel all running jobs
+        for job in running_jobs:
+            cls.cancel_job(job)
+        cls.save_state()
+
     @classmethod
     def is_available_for_job(cls, renderer, priority=2):
         if not RenderWorkerFactory.class_for_name(renderer).engine.renderer_path():
@@ -96,8 +105,9 @@ class RenderQueue:
 
         instances = cls.renderer_instances()
         higher_priority_jobs = [x for x in cls.running_jobs() if x.priority < priority]
-        max_renderers = renderer in instances.keys() and instances[renderer] >= cls.maximum_renderer_instances.get(renderer, 1)
+        max_allowed_instances = cls.maximum_renderer_instances.get(renderer, 1)
-        return not max_renderers and not higher_priority_jobs
+        maxed_out_instances = renderer in instances.keys() and instances[renderer] >= max_allowed_instances
+        return not maxed_out_instances and not higher_priority_jobs
 
     @classmethod
     def evaluate_queue(cls):

src/server_proxy.py

@@ -1,19 +1,21 @@
+import json
 import logging
 import os
-import json
-import requests
 import socket
-import time
 import threading
-from lib.workers.base_worker import RenderStatus
+import time
+
+import requests
 from requests_toolbelt.multipart import MultipartEncoder, MultipartEncoderMonitor
 
+from src.utilities.status_utils import RenderStatus
+
 status_colors = {RenderStatus.ERROR: "red", RenderStatus.CANCELLED: 'orange1', RenderStatus.COMPLETED: 'green',
                  RenderStatus.NOT_STARTED: "yellow", RenderStatus.SCHEDULED: 'purple',
                  RenderStatus.RUNNING: 'cyan', RenderStatus.WAITING_FOR_SUBJOBS: 'blue'}
 
-categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStatus.ERROR, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED,
+categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStatus.ERROR, RenderStatus.NOT_STARTED,
-              RenderStatus.COMPLETED, RenderStatus.CANCELLED, RenderStatus.UNDEFINED]
+              RenderStatus.SCHEDULED, RenderStatus.COMPLETED, RenderStatus.CANCELLED, RenderStatus.UNDEFINED]
 
 logger = logging.getLogger()
 OFFLINE_MAX = 2

src/utilities/ffmpeg_helper.py

@@ -1,5 +1,5 @@
 import subprocess
-from lib.engines.ffmpeg_engine import FFMPEG
+from src.workers.engines.ffmpeg_engine import FFMPEG
 
 
 def image_sequence_to_video(source_glob_pattern, output_path, framerate=24, encoder="prores_ks", profile=4,

src/utilities/server_helper.py

@@ -1,13 +1,9 @@
 import logging
 import os
-import socket
 import subprocess
 import threading
 
-import psutil
+from src.utilities.ffmpeg_helper import generate_thumbnail, save_first_frame
-
-from lib.server.server_proxy import RenderServerProxy
-from lib.utilities.ffmpeg_helper import generate_thumbnail, save_first_frame
 
 logger = logging.getLogger()
 
@@ -21,8 +17,8 @@ def generate_thumbnail_for_job(job, thumb_video_path, thumb_image_path, max_widt
         try:
             logger.debug(f"Generating video thumbnail for {source}")
             generate_thumbnail(source_path=source, dest_path=thumb_video_path, max_width=max_width)
-        except subprocess.CalledProcessError as e:
+        except subprocess.CalledProcessError as err:
-            logger.error(f"Error generating video thumbnail for {source}: {e}")
+            logger.error(f"Error generating video thumbnail for {source}: {err}")
 
         try:
             os.remove(in_progress_path)

src/utilities/status_utils.py

@@ -0,0 +1,20 @@
+from enum import Enum
+
+
+class RenderStatus(Enum):
+    NOT_STARTED = "not_started"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    CANCELLED = "cancelled"
+    ERROR = "error"
+    SCHEDULED = "scheduled"
+    WAITING_FOR_SUBJOBS = "waiting_for_subjobs"
+    CONFIGURING = "configuring"
+    UNDEFINED = "undefined"
+
+
+def string_to_status(string):
+    for stat in RenderStatus:
+        if stat.value == string:
+            return stat
+    return RenderStatus.UNDEFINED

src/utilities/zeroconf_server.py

@@ -59,6 +59,7 @@ class ZeroconfServer:
     @classmethod
     def _browse_services(cls):
         browser = ServiceBrowser(cls.zeroconf, cls.service_type, [cls._on_service_discovered])
+        browser.is_alive()
 
     @classmethod
     def _on_service_discovered(cls, zeroconf, service_type, name, state_change):

src/worker_factory.py

@@ -3,9 +3,9 @@ class RenderWorkerFactory:
     @staticmethod
     def supported_classes():
         # to add support for any additional RenderWorker classes, import their classes and add to list here
-        from .blender_worker import BlenderRenderWorker
+        from src.workers.blender_worker import BlenderRenderWorker
-        from .aerender_worker import AERenderWorker
+        from src.workers.aerender_worker import AERenderWorker
-        from .ffmpeg_worker import FFMPEGRenderWorker
+        from src.workers.ffmpeg_worker import FFMPEGRenderWorker
         classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker]
         return classes
 

src/workers/aerender_worker.py

@@ -1,11 +1,14 @@
 #!/usr/bin/env python3
 import glob
 import json
+import logging
+import os
 import re
 import time
 
-from .base_worker import *
+from src.workers.base_worker import BaseRenderWorker, timecode_to_frames
-from ..engines.aerender_engine import AERender
+from src.workers.engines.aerender_engine import AERender
+
 
 def aerender_path():
     paths = glob.glob('/Applications/*After Effects*/aerender')

src/workers/base_worker.py

@@ -6,38 +6,19 @@ import os
 import subprocess
 import threading
 from datetime import datetime
-from enum import Enum
 
 import psutil
 from pubsub import pub
 from sqlalchemy import Column, Integer, String, DateTime, JSON
 from sqlalchemy.ext.declarative import declarative_base
 
-from lib.utilities.misc_helper import get_time_elapsed
+from src.utilities.misc_helper import get_time_elapsed
+from src.utilities.status_utils import RenderStatus, string_to_status
 
 logger = logging.getLogger()
 Base = declarative_base()
 
 
-class RenderStatus(Enum):
-    NOT_STARTED = "not_started"
-    RUNNING = "running"
-    COMPLETED = "completed"
-    CANCELLED = "cancelled"
-    ERROR = "error"
-    SCHEDULED = "scheduled"
-    WAITING_FOR_SUBJOBS = "waiting_for_subjobs"
-    CONFIGURING = "configuring"
-    UNDEFINED = "undefined"
-
-
-def string_to_status(string):
-    for stat in RenderStatus:
-        if stat.value == string:
-            return stat
-    return RenderStatus.UNDEFINED
-
-
 class BaseRenderWorker(Base):
     __tablename__ = 'render_workers'
 
@@ -121,7 +102,7 @@ class BaseRenderWorker(Base):
     @status.setter
     def status(self, new_status):
         if self._status != new_status.value:
-            old_status = self._status
+            old_status = self.status
             self._status = new_status.value
             pub.sendMessage('status_change', job_id=self.id, old_status=old_status, new_status=new_status)
 
@@ -186,7 +167,8 @@ class BaseRenderWorker(Base):
 
         self.status = RenderStatus.RUNNING
         self.start_time = datetime.now()
-        logger.info(f'Starting {self.engine.name()} {self.engine.version()} Render for {self.input_path} | Frame Count: {self.total_frames}')
+        logger.info(f'Starting {self.engine.name()} {self.engine.version()} Render for {self.input_path} | '
+                    f'Frame Count: {self.total_frames}')
         self.__thread.start()
 
     def run(self):
@@ -207,7 +189,7 @@ class BaseRenderWorker(Base):
             f.write(f"Running command: {' '.join(subprocess_cmds)}\n")
             for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"):  # or another encoding
                 f.write(c)
-                logger.debug(f"{self.engine.name()}Worker: {c.strip()}")
+                # logger.debug(f"{self.engine.name()}Worker: {c.strip()}")
                 self.last_output = c.strip()
                 self._parse_stdout(c.strip())
             f.write('\n')
@@ -220,7 +202,7 @@ class BaseRenderWorker(Base):
                 return
 
             if return_code:
-                message = f"{self.engine.name()} render failed with return_code {return_code} after {self.time_elapsed()}"
+                message = f"{self.engine.name()} render failed with code {return_code} after {self.time_elapsed()}"
                 logger.error(message)
                 f.write(message)
                 self.status = RenderStatus.ERROR
@@ -233,7 +215,7 @@ class BaseRenderWorker(Base):
             f.write(message)
 
         if self.children:
-            from lib.distributed_job_manager import DistributedJobManager
+            from src.distributed_job_manager import DistributedJobManager
             DistributedJobManager.wait_for_subjobs(local_job=self)
 
         # Post Render Work

src/workers/blender_worker.py

@@ -2,9 +2,9 @@
 import re
 from collections import Counter
 
-from lib.engines.blender_engine import Blender
+from src.workers.engines.blender_engine import Blender
-from lib.utilities.ffmpeg_helper import image_sequence_to_video
+from src.utilities.ffmpeg_helper import image_sequence_to_video
-from lib.workers.base_worker import *
+from src.workers.base_worker import *
 
 
 class BlenderRenderWorker(BaseRenderWorker):
@@ -69,17 +69,6 @@ class BlenderRenderWorker(BaseRenderWorker):
                 samples = re.sub(r'[^\d/]', '', sample_string)
                 self.__frame_percent_complete = int(samples.split('/')[0]) / int(samples.split('/')[-1])
 
-            # Calculate rough percent based on cycles
-            # EEVEE
-            # 10-Apr-22 22:42:06 - RENDERER: Fra:0 Mem:857.99M (Peak 928.55M) | Time:00:03.96 | Rendering 1 / 65 samples
-            # 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:07.92 | Rendering 26 / 64 samples
-            # 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:08.17 | Rendering 51 / 64 samples
-            # 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:08.31 | Rendering 64 / 64 samples
-
-            # CYCLES
-            # 10-Apr-22 22:43:22 - RENDERER: Fra:0 Mem:836.30M (Peak 1726.13M) | Time:00:01.56 | Remaining:00:30.65 | Mem:588.68M, Peak:588.68M | Scene, View Layer | Sample 1/150
-            # 10-Apr-22 22:43:43 - RENDERER: Fra:0 Mem:836.30M (Peak 1726.13M) | Time:00:22.01 | Remaining:00:03.36 | Mem:588.68M, Peak:588.68M | Scene, View Layer | Sample 129/150
-
             if int(stats['frame']) > self.current_frame:
                 self.current_frame = int(stats['frame'])
                 logger.debug(

src/workers/ffmpeg_worker.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
 import re
-from .base_worker import *
+import subprocess
-from ..engines.ffmpeg_engine import FFMPEG
+
+from src.workers.base_worker import BaseRenderWorker
+from src.workers.engines.ffmpeg_engine import FFMPEG
 
 
 class FFMPEGRenderWorker(BaseRenderWorker):
@@ -48,18 +50,3 @@ class FFMPEGRenderWorker(BaseRenderWorker):
             time_elapsed = stats['time_elapsed']
         elif "not found" in line:
             self.log_error(line)
-
-
-if __name__ == '__main__':
-
-    print(FFMPEG.full_report())
-
-    # logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG)
-    #
-    # test_movie = '/Users/brettwilliams/Desktop/dark_knight_rises.mp4'
-    #
-    # r = FFMPEGRenderWorker(test_movie, '/Users/brettwilliams/Desktop/test-ffmpeg.mp4', args=['-c:v', 'libx265', '-vtag', 'hvc1'])
-    # # r = FFMPEGRenderer(test_movie, '/Users/brettwilliams/Desktop/dark_knight_rises-output.mp4')
-    # r.start()
-    # while r.is_running():
-    #     time.sleep(1)

start_client.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-from lib.client.dashboard_window import start_client
+from src.client.dashboard_window import start_client
 
 if __name__ == '__main__':
     start_client()

start_server.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-from lib.server.api_server import start_server
+from src.api_server import start_server
 
 if __name__ == '__main__':
     start_server()