Remove Old Multi-Client Code / Refactoring (#13)

* Remove a lot of old code from render_queue regarding clients

* More code cleanup

* More code cleanup

* Move everything around

* Minor log change

config/config.yaml

@@ -3,4 +3,5 @@ max_content_path: 100000000
 server_log_level: info
 flask_log_level: error
 flask_debug_enable: false
-queue_eval_seconds: 1
+queue_eval_seconds: 1
+port_number: 8080

dashboard.py

@@ -17,7 +17,7 @@ from rich.table import Table
 from rich.text import Text
 from rich.tree import Tree
 
-from lib.render_workers.base_worker import RenderStatus, string_to_status
+from lib.workers.base_worker import RenderStatus, string_to_status
 from lib.server.server_proxy import RenderServerProxy
 from lib.utilities.misc_helper import get_time_elapsed
 from start_server import start_server

lib/client/new_job_window.py

@@ -11,25 +11,13 @@ from tkinter.ttk import Frame, Label, Entry, Combobox, Progressbar
 import psutil
 import requests
 import threading
-from lib.render_workers.blender_worker import Blender
+from lib.workers.blender_worker import Blender
 from lib.server.server_proxy import RenderServerProxy
 
 logger = logging.getLogger()
 
-prefs_name = 'config/.scheduler_prefs'
 label_width = 9
 header_padding = 6
-server_setup_timeout = 5
-
-
-def request_data(server_ip, payload, server_port=8080, timeout=2):
-    try:
-        req = requests.get(f'http://{server_ip}:{server_port}/api/{payload}', timeout=timeout)
-        if req.ok:
-            return req.json()
-    except Exception as e:
-        pass
-    return None
 
 
 # CheckListBox source - https://stackoverflow.com/questions/50398649/python-tkinter-tk-support-checklist-box

lib/engines/blender_engine.py

@@ -61,7 +61,7 @@ class Blender(BaseRenderEngine):
 
     @classmethod
     def get_scene_info(cls, project_path, timeout=10):
-        scene_info = None
+        scene_info = {}
         try:
             results = cls.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                             'scripts', 'blender', 'get_file_info.py'), timeout=timeout)
@@ -71,6 +71,8 @@ class Blender(BaseRenderEngine):
                     raw_data = line.split('SCENE_DATA:')[-1]
                     scene_info = json.loads(raw_data)
                     break
+                elif line.startswith('Error'):
+                    logger.error(f"get_scene_info error: {line.strip()}")
         except Exception as e:
             logger.error(f'Error getting file details for .blend file: {e}')
         return scene_info

lib/render_queue.py

@@ -1,13 +1,10 @@
 import logging
-import platform
 from datetime import datetime
 
-import psutil
-import requests
-from sqlalchemy import create_engine, Column, String, Integer
+from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
-from .render_workers.base_worker import RenderStatus, BaseRenderWorker, Base
+from .workers.base_worker import RenderStatus, BaseRenderWorker, Base
 
 logger = logging.getLogger()
 
@@ -18,27 +15,6 @@ class JobNotFoundError(Exception):
         self.job_id = job_id
 
 
-class RenderClient(Base):
-    __tablename__ = 'render_clients'
-    id = Column(Integer, primary_key=True)
-    hostname = Column(String)
-
-    def __init__(self, hostname):
-        self.hostname = hostname
-
-    def is_available(self, timeout=3):
-        try:
-            response = requests.get(f"http://{self.hostname}:8080/api/status", timeout=timeout)
-            if response.ok:
-                return True
-        except requests.ConnectionError as e:
-            pass
-        return False
-
-    def __repr__(self):
-        return "client stuff"
-
-
 class RenderQueue:
     engine = create_engine('sqlite:///database.db')
     Base.metadata.create_all(engine)
@@ -46,30 +22,19 @@ class RenderQueue:
     session = Session()
     job_queue = []
     maximum_renderer_instances = {'blender': 1, 'aerender': 1, 'ffmpeg': 4}
-    hostname = None
-    port = 8080
-    client_mode = False
-    server_hostname = None
-
     last_saved_counts = {}
 
     def __init__(self):
         pass
 
     @classmethod
-    def add_to_render_queue(cls, render_job, force_start=False, client=None):
-
-        if not client or render_job.client == cls.hostname:
-            logger.debug('Adding priority {} job to render queue: {}'.format(render_job.priority, render_job))
-            render_job.client = cls.hostname
-            cls.job_queue.append(render_job)
-            if force_start:
-                cls.start_job(render_job)
-            cls.session.add(render_job)
-            cls.save_state()
-        else:
-            # todo: implement client rendering
-            logger.warning('remote client rendering not implemented yet')
+    def add_to_render_queue(cls, render_job, force_start=False):
+        logger.debug('Adding priority {} job to render queue: {}'.format(render_job.priority, render_job))
+        cls.job_queue.append(render_job)
+        if force_start:
+            cls.start_job(render_job)
+        cls.session.add(render_job)
+        cls.save_state()
 
     @classmethod
     def all_jobs(cls):
@@ -169,75 +134,3 @@ class RenderQueue:
         for job_status in RenderStatus:
             job_counts[job_status.value] = len(cls.jobs_with_status(job_status))
         return job_counts
-
-    @classmethod
-    def status(cls):
-        return {"timestamp": datetime.now().isoformat(),
-                "platform": platform.platform(),
-                "cpu_percent": psutil.cpu_percent(percpu=False),
-                "cpu_percent_per_cpu": psutil.cpu_percent(percpu=True),
-                "cpu_count": psutil.cpu_count(),
-                "memory_total": psutil.virtual_memory().total,
-                "memory_available": psutil.virtual_memory().available,
-                "memory_percent": psutil.virtual_memory().percent,
-                "job_counts": cls.job_counts(),
-                "host_name": cls.hostname
-                }
-
-    @classmethod
-    def render_clients(cls):
-        all_clients = cls.session.query(RenderClient).all()
-        if not all_clients:
-            cls.session.add(RenderClient(hostname=cls.hostname))
-            cls.save_state()
-            all_clients = cls.session.query(RenderClient).all()
-        return all_clients
-
-    @classmethod
-    def client_with_hostname(cls, hostname):
-        return cls.session.query(RenderClient).filter(RenderClient.hostname == hostname).first()
-
-    @classmethod
-    def register_client(cls, hostname):
-        new_client = None
-        err_msg = None
-
-        if hostname == cls.hostname:
-            err_msg = "Cannot register same hostname as server"
-        elif cls.client_with_hostname(hostname):
-            err_msg = f"Client '{hostname}' already registered"
-        else:
-            new_client = RenderClient(hostname=hostname)
-            if not new_client.is_available():
-                cls.session.add(new_client)
-                logger.info(f"Client '{hostname}' successfully registered")
-                cls.save_state()
-            else:
-                err_msg = f"Cannot connect to client at hostname: {hostname}"
-
-        if err_msg:
-            logger.warning(err_msg)
-            return err_msg, 400
-        else:
-            return new_client.hostname
-
-    @classmethod
-    def unregister_client(cls, hostname):
-        success = False
-        client = cls.client_with_hostname(hostname)
-        if client and hostname != cls.hostname:
-            cls.session.delete(client)
-            cls.save_state()
-            logger.info(f"Client '{hostname}' successfully unregistered")
-            success = True
-        return str(success)
-
-    @staticmethod
-    def is_client_available(client_hostname, timeout=3):
-        try:
-            response = requests.get(f"http://{client_hostname}:8080/api/status", timeout=timeout)
-            if response.ok:
-                return True
-        except requests.ConnectionError as e:
-            pass
-        return False

lib/server/api_server.py

@@ -3,28 +3,27 @@ import json
 import logging
 import os
 import pathlib
+import platform
 import shutil
 import socket
 import threading
 import time
 import zipfile
 from datetime import datetime
+from urllib.request import urlretrieve
 from zipfile import ZipFile
 
 import json2html
-import requests
+import psutil
 import yaml
 from flask import Flask, request, render_template, send_file, after_this_request, Response, redirect, url_for, abort
-from urllib.parse import urlparse
-from urllib.request import urlretrieve
 from werkzeug.utils import secure_filename
 
-from lib.server.zeroconf_server import ZeroconfServer
 from lib.render_queue import RenderQueue, JobNotFoundError
-from lib.render_workers.worker_factory import RenderWorkerFactory
-from lib.render_workers.base_worker import string_to_status, RenderStatus
+from lib.workers.base_worker import string_to_status, RenderStatus
+from lib.workers.worker_factory import RenderWorkerFactory
+from lib.server.zeroconf_server import ZeroconfServer
 from lib.utilities.server_helper import generate_thumbnail_for_job
-from lib.server.server_proxy import RenderServerProxy
 
 logger = logging.getLogger()
 server = Flask(__name__, template_folder='templates', static_folder='static')
@@ -55,8 +54,8 @@ def index():
         presets = yaml.load(f, Loader=yaml.FullLoader)
 
     return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.all_jobs()),
-                           hostname=RenderQueue.hostname, renderer_info=renderer_info(),
-                           render_clients=render_clients(), preset_list=presets)
+                           hostname=server.config['HOSTNAME'], renderer_info=renderer_info(),
+                           render_clients=[server.config['HOSTNAME']], preset_list=presets)
 
 
 @server.get('/api/jobs')
@@ -85,7 +84,7 @@ def job_detail(job_id):
         media_basename = os.path.basename(found_job.file_list()[0])
         media_url = f"/api/job/{job_id}/file/{media_basename}"
     return render_template('details.html', detail_table=table_html, media_url=media_url,
-                           hostname=RenderQueue.hostname, job_status=found_job.status.value.title(),
+                           hostname=server.config['HOSTNAME'], job_status=found_job.status.value.title(),
                            job=found_job, renderer_info=renderer_info())
 
 
@@ -210,23 +209,6 @@ def download_all(job_id):
         return f'Cannot find project files for job {job_id}', 500
 
 
-@server.post('/api/register_client')
-def register_client():
-    client_hostname = request.values['hostname']
-    return RenderQueue.register_client(client_hostname)
-
-
-@server.post('/api/unregister_client')
-def unregister_client():
-    client_hostname = request.values['hostname']
-    return RenderQueue.unregister_client(client_hostname)
-
-
-@server.get('/api/clients')
-def render_clients():
-    return [c.hostname for c in RenderQueue.render_clients()]
-
-
 @server.get('/api/presets')
 def presets():
     with open('config/presets.yaml') as f:
@@ -239,22 +221,10 @@ def full_status():
     full_results = {'timestamp': datetime.now().isoformat(), 'servers': {}}
 
     try:
-        for client_hostname in render_clients():
-            is_online = False
-            if client_hostname == RenderQueue.hostname:
-                snapshot_results = snapshot()
-                is_online = True
-            else:
-                snapshot_results = {}
-                try:
-                    snapshot_request = requests.get(f'http://{client_hostname}:8080/snapshot', timeout=1)
-                    snapshot_results = snapshot_request.json()
-                    is_online = snapshot_request.ok
-                except requests.ConnectionError as e:
-                    pass
-            server_data = {'status': snapshot_results.get('status', {}), 'jobs': snapshot_results.get('jobs', {}),
-                           'is_online': is_online}
-            full_results['servers'][client_hostname] = server_data
+        snapshot_results = snapshot()
+        server_data = {'status': snapshot_results.get('status', {}), 'jobs': snapshot_results.get('jobs', {}),
+                       'is_online': True}
+        full_results['servers'][server.config['HOSTNAME']] = server_data
     except Exception as e:
         logger.error(f"Exception fetching full status: {e}")
 
@@ -263,7 +233,7 @@ def full_status():
 
 @server.get('/api/snapshot')
 def snapshot():
-    server_status = RenderQueue.status()
+    server_status = status()
     server_jobs = [x.json() for x in RenderQueue.all_jobs()]
     server_data = {'status': server_status, 'jobs': server_jobs, 'timestamp': datetime.now().isoformat()}
     return server_data
@@ -385,7 +355,7 @@ def add_job_handler():
                                                                input_path=loaded_project_local_path,
                                                                output_path=job["output_path"],
                                                                args=job.get('args', {}))
-                render_job.client = job.get('client', None) or RenderQueue.hostname
+                render_job.client = server.config['HOSTNAME']
                 render_job.owner = job.get("owner", None)
                 render_job.name = job.get("name", None)
                 render_job.priority = int(job.get('priority', render_job.priority))
@@ -476,7 +446,18 @@ def clear_history():
 
 @server.route('/api/status')
 def status():
-    return RenderQueue.status()
+    return {"timestamp": datetime.now().isoformat(),
+            "platform": platform.platform(),
+            "cpu_percent": psutil.cpu_percent(percpu=False),
+            "cpu_percent_per_cpu": psutil.cpu_percent(percpu=True),
+            "cpu_count": psutil.cpu_count(),
+            "memory_total": psutil.virtual_memory().total,
+            "memory_available": psutil.virtual_memory().available,
+            "memory_percent": psutil.virtual_memory().percent,
+            "job_counts": RenderQueue.job_counts(),
+            "hostname": server.config['HOSTNAME'],
+            "port": server.config['PORT']
+            }
 
 
 @server.get('/api/renderer_info')
@@ -495,8 +476,7 @@ def renderer_info():
 
 @server.route('/upload')
 def upload_file_page():
-    return render_template('upload.html', render_clients=render_clients(),
-                           supported_renderers=RenderWorkerFactory.supported_renderers())
+    return render_template('upload.html', supported_renderers=RenderWorkerFactory.supported_renderers())
 
 
 def start_server(background_thread=False):
@@ -511,15 +491,17 @@ def start_server(background_thread=False):
     logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S',
                         level=config.get('server_log_level', 'INFO').upper())
 
+    # get hostname
+    local_hostname = socket.gethostname()
+    local_hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
+
+    # load flask settings
+    server.config['HOSTNAME'] = local_hostname
+    server.config['PORT'] = int(config.get('port_number', 8080))
     server.config['UPLOAD_FOLDER'] = os.path.expanduser(config['upload_folder'])
     server.config['THUMBS_FOLDER'] = os.path.join(os.path.expanduser(config['upload_folder']), 'thumbs')
     server.config['MAX_CONTENT_PATH'] = config['max_content_path']
 
-    # Get hostname and render clients
-    local_hostname = socket.gethostname()
-    RenderQueue.hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
-    server.config['HOSTNAME'] = RenderQueue.hostname
-
     # disable most Flask logging
     flask_log = logging.getLogger('werkzeug')
     flask_log.setLevel(config.get('flask_log_level', 'ERROR').upper())
@@ -530,19 +512,18 @@ def start_server(background_thread=False):
     thread = threading.Thread(target=eval_loop, kwargs={'delay_sec': config.get('queue_eval_seconds', 1)}, daemon=True)
     thread.start()
 
-    logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.hostname}'")
-
-    zeroconf_server = ZeroconfServer("_zordon._tcp.local.", RenderQueue.hostname, RenderQueue.port)
+    logging.info(f"Starting Zordon Render Server - Hostname: '{server.config['HOSTNAME']}:'")
+    zeroconf_server = ZeroconfServer("_zordon._tcp.local.", server.config['HOSTNAME'], server.config['PORT'])
     zeroconf_server.start()
 
     try:
         if background_thread:
             server_thread = threading.Thread(
-                target=lambda: server.run(host='0.0.0.0', port=RenderQueue.port, debug=False, use_reloader=False))
+                target=lambda: server.run(host='0.0.0.0', port=server.config['PORT'], debug=False, use_reloader=False))
             server_thread.start()
             server_thread.join()
         else:
-            server.run(host='0.0.0.0', port=RenderQueue.port, debug=config.get('flask_debug_enable', False),
+            server.run(host='0.0.0.0', port=server.config['PORT'], debug=config.get('flask_debug_enable', False),
                        use_reloader=False, threaded=True)
     finally:
         zeroconf_server.stop()

lib/server/server_proxy.py

@@ -4,7 +4,7 @@ import json
 import requests
 import time
 import threading
-from lib.render_workers.base_worker import RenderStatus
+from lib.workers.base_worker import RenderStatus
 from requests_toolbelt.multipart import MultipartEncoder, MultipartEncoderMonitor
 
 status_colors = {RenderStatus.ERROR: "red", RenderStatus.CANCELLED: 'orange1', RenderStatus.COMPLETED: 'green',
@@ -21,7 +21,7 @@ OFFLINE_MAX = 2
 class RenderServerProxy:
 
     def __init__(self, hostname, server_port="8080"):
-        self._hostname = hostname
+        self.hostname = hostname
         self.port = server_port
         self.fetched_status_data = None
         self.__jobs_cache_token = None
@@ -31,15 +31,6 @@ class RenderServerProxy:
         self.__offline_flags = 0
         self.update_cadence = 5
 
-    @property
-    def hostname(self):
-        return self._hostname
-
-    @hostname.setter
-    def hostname(self, value):
-        self._hostname = value
-        self.__jobs_cache_token = None
-
     def connect(self):
         status = self.request_data('status')
         return status

lib/utilities/ffmpeg_helper.py

@@ -1,6 +1,6 @@
 import subprocess
 import ffmpeg  # todo: remove all references to ffmpeg library and instead use direct subprocesses
-from ..render_engines.ffmpeg_engine import FFMPEG
+from ..engines.ffmpeg_engine import FFMPEG
 
 
 def file_info(path):

lib/utilities/server_helper.py

@@ -7,7 +7,7 @@ import threading
 import requests
 
 from .ffmpeg_helper import generate_thumbnail, save_first_frame
-from lib.render_workers.base_worker import RenderStatus
+from lib.workers.base_worker import RenderStatus
 
 logger = logging.getLogger()
 

lib/workers/aerender_worker.py

@@ -5,7 +5,7 @@ import re
 import time
 
 from .base_worker import *
-from ..render_engines.aerender_engine import AERender
+from ..engines.aerender_engine import AERender
 
 def aerender_path():
     paths = glob.glob('/Applications/*After Effects*/aerender')

lib/workers/blender_worker.py

@@ -6,7 +6,7 @@ try:
 except ImportError:
     from base_worker import *
 
-from ..render_engines.blender_engine import Blender
+from ..engines.blender_engine import Blender
 
 
 class BlenderRenderWorker(BaseRenderWorker):
@@ -30,9 +30,9 @@ class BlenderRenderWorker(BaseRenderWorker):
 
         # Scene Info
         self.scene_info = Blender.get_scene_info(input_path)
-        self.total_frames = (int(self.scene_info.get('frame_end', 0)) - int(self.scene_info.get('frame_start', 0)) + 1) \
+        self.total_frames = (int(self.scene_info.get('frame_end', 1)) - int(self.scene_info.get('frame_start', 1)) + 1) \
             if self.render_all_frames else 1
-        self.current_frame = int(self.scene_info.get('frame_start', 0))
+        self.current_frame = int(self.scene_info.get('frame_start', 1))
 
     def generate_worker_subprocess(self):
 
@@ -90,7 +90,7 @@ class BlenderRenderWorker(BaseRenderWorker):
                                                                             time_elapsed, time_remaining))
         elif "file doesn't exist" in line.lower():
             self.log_error(line, halt_render=True)
-        elif 'error' in line.lower():
+        elif line.lower().startswith('error'):
             self.log_error(line)
         elif 'Saved' in line or 'Saving' in line or 'quit' in line:
             match = re.match(r'Time: (.*) \(Saving', line)

lib/workers/ffmpeg_worker.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import re
 from .base_worker import *
-from ..render_engines.ffmpeg_engine import FFMPEG
+from ..engines.ffmpeg_engine import FFMPEG
 
 
 class FFMPEGRenderWorker(BaseRenderWorker):

start_server.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-from lib.server.job_server import start_server
+from lib.server.api_server import start_server
 
 if __name__ == '__main__':
     start_server()