Add RenderClient object to DB for client tracking

lib/render_queue.py

@@ -4,7 +4,7 @@ from datetime import datetime
 
 import psutil
 import requests
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, Column, String, Integer
 from sqlalchemy.orm import sessionmaker
 
 from .render_workers.render_worker import RenderStatus
@@ -19,6 +19,27 @@ class JobNotFoundError(Exception):
         self.job_id = job_id
 
 
+class RenderClient(Base):
+    __tablename__ = 'render_clients'
+    id = Column(Integer, primary_key=True)
+    hostname = Column(String)
+
+    def __init__(self, hostname):
+        self.hostname = hostname
+
+    def is_available(self, timeout=3):
+        try:
+            response = requests.get(f"http://{self.hostname}:8080/api/status", timeout=timeout)
+            if response.ok:
+                return True
+        except requests.ConnectionError as e:
+            pass
+        return False
+
+    def __repr__(self):
+        return "client stuff"
+
+
 class RenderQueue:
     engine = create_engine('sqlite:///database.db')
     Base.metadata.create_all(engine)
@@ -26,9 +47,8 @@ class RenderQueue:
     session = Session()
     ScheduledJob.register_user_events()
     job_queue = []
-    render_clients = []
     maximum_renderer_instances = {'blender': 2, 'aerender': 1, 'ffmpeg': 4}
-    host_name = None
+    hostname = None
     port = 8080
     client_mode = False
     server_hostname = None
@@ -41,9 +61,9 @@ class RenderQueue:
     @classmethod
     def add_to_render_queue(cls, render_job, force_start=False, client=None):
 
-        if not client or render_job.client == cls.host_name:
+        if not client or render_job.client == cls.hostname:
             logger.debug('Adding priority {} job to render queue: {}'.format(render_job.priority, render_job.worker_object))
-            render_job.client = cls.host_name
+            render_job.client = cls.hostname
             cls.job_queue.append(render_job)
             if force_start:
                 cls.start_job(render_job)
@@ -165,44 +185,56 @@ class RenderQueue:
                 "memory_available": psutil.virtual_memory().available,
                 "memory_percent": psutil.virtual_memory().percent,
                 "job_counts": cls.job_counts(),
-                "host_name": cls.host_name
+                "host_name": cls.hostname
                 }
 
     @classmethod
-    def register_client(cls, hostname):
+    def render_clients(cls):
+        all_clients = cls.session.query(RenderClient).all()
+        if not all_clients:
+            cls.session.add(RenderClient(hostname=cls.hostname))
+            cls.save_state()
+            all_clients = cls.session.query(RenderClient).all()
+        return all_clients
 
+    @classmethod
+    def client_with_hostname(cls, hostname):
+        return cls.session.query(RenderClient).filter(RenderClient.hostname == hostname).first()
+
+    @classmethod
+    def register_client(cls, hostname):
+        new_client = None
         err_msg = None
 
-        if hostname == cls.host_name:
+        if hostname == cls.hostname:
             err_msg = "Cannot register same hostname as server"
-        elif hostname in cls.render_clients:
+        elif cls.client_with_hostname(hostname):
             err_msg = f"Client '{hostname}' already registered"
         else:
-            try:
+            new_client = RenderClient(hostname=hostname)
-                response = requests.get(f"http://{hostname}:8080/api/status", timeout=3)
+            if not new_client.is_available():
-                if response.ok:
+                cls.session.add(new_client)
-                    cls.render_clients.append(hostname)
+                logger.info(f"Client '{hostname}' successfully registered")
-                    logger.info(f"Client '{hostname}' successfully registered")
+                cls.save_state()
-                    cls.save_state()
+            else:
-                else:
-                    err_msg = f'Response from server not ok: {response.text}'
-            except requests.ConnectionError as e:
                 err_msg = f"Cannot connect to client at hostname: {hostname}"
 
         if err_msg:
             logger.warning(err_msg)
             return err_msg, 400
         else:
-            return 'success'
+            return new_client.hostname
 
     @classmethod
     def unregister_client(cls, hostname):
         success = False
-        if hostname in cls.render_clients and hostname != cls.host_name:
+        client = cls.client_with_hostname(hostname)
-            cls.render_clients.remove(hostname)
+        if client and hostname != cls.hostname:
+            cls.session.delete(client)
+            cls.save_state()
             logger.info(f"Client '{hostname}' successfully unregistered")
             success = True
-        return success
+        return str(success)
 
     @staticmethod
     def is_client_available(client_hostname, timeout=3):

lib/server/job_server.py

@@ -50,8 +50,8 @@ def index():
         presets = yaml.load(f, Loader=yaml.FullLoader)
 
     return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.all_jobs()),
-                           hostname=RenderQueue.host_name, renderer_info=renderer_info(),
+                           hostname=RenderQueue.hostname, renderer_info=renderer_info(),
-                           render_clients=RenderQueue.render_clients, preset_list=presets)
+                           render_clients=render_clients(), preset_list=presets)
 
 
 @server.route('/ui/job/<job_id>/full_details')
@@ -64,7 +64,7 @@ def job_detail(job_id):
         media_basename = os.path.basename(found_job.file_list()[0])
         media_url = f"/api/job/{job_id}/file/{media_basename}"
     return render_template('details.html', detail_table=table_html, media_url=media_url,
-                           hostname=RenderQueue.host_name, job_status=found_job.render_status().value.title(),
+                           hostname=RenderQueue.hostname, job_status=found_job.render_status().value.title(),
                            job=found_job, renderer_info=renderer_info())
 
 
@@ -166,20 +166,18 @@ def download_all(job_id):
 @server.post('/api/register_client')
 def register_client():
     client_hostname = request.values['hostname']
-    x = RenderQueue.register_client(client_hostname)
+    return RenderQueue.register_client(client_hostname)
-    return "Success" if x else "Fail"
 
 
 @server.post('/api/unregister_client')
 def unregister_client():
     client_hostname = request.values['hostname']
-    x = RenderQueue.unregister_client(client_hostname)
+    return RenderQueue.unregister_client(client_hostname)
-    return "Success" if x else "Fail"
 
 
 @server.get('/api/clients')
 def render_clients():
-    return RenderQueue.render_clients
+    return [c.hostname for c in RenderQueue.render_clients()]
 
 
 @server.get('/api/presets')
@@ -194,9 +192,9 @@ def full_status():
     full_results = {'timestamp': datetime.now().isoformat(), 'servers': {}}
 
     try:
-        for client_hostname in RenderQueue.render_clients:
+        for client_hostname in render_clients():
             is_online = False
-            if client_hostname == RenderQueue.host_name:
+            if client_hostname == RenderQueue.hostname:
                 snapshot_results = snapshot()
                 is_online = True
             else:
@@ -301,7 +299,7 @@ def add_job(job_params, remove_job_dir_on_failure=False):
     output_path = job_params.get("output_path", None)
     priority = int(job_params.get('priority', 2))
     args = job_params.get('args', {})
-    client = job_params.get('client', None) or RenderQueue.host_name
+    client = job_params.get('client', None) or RenderQueue.hostname
     force_start = job_params.get('force_start', False)
     custom_id = None
     job_dir = None
@@ -313,7 +311,7 @@ def add_job(job_params, remove_job_dir_on_failure=False):
         return {'error': err_msg, 'code': 400}
 
     # local renders
-    if client == RenderQueue.host_name:
+    if client == RenderQueue.hostname:
         logger.info(f"Creating job locally - {name if name else input_path}")
         try:
             render_job = ScheduledJob(renderer, input_path, output_path, args, priority, job_owner, client,
@@ -327,16 +325,13 @@ def add_job(job_params, remove_job_dir_on_failure=False):
             return {'error': err_msg, 'code': 400}
 
     # client renders
-    elif client in RenderQueue.render_clients:
+    elif client in RenderQueue.render_clients():
-
+        if client.is_available():
-        # see if host is available
-        if RenderQueue.is_client_available(client):
-
             # call uploader on remote client
             try:
                 logger.info(f"Uploading file {input_path} to client {client}")
                 job_data = request.json
-                response = post_job_to_server(input_path, job_data, client)
+                response = post_job_to_server(input_path, job_data, client.hostname)
                 if response.ok:
                     logger.info("Job submitted successfully!")
                     return response.json() if response.json() else "Job ok"
@@ -382,16 +377,9 @@ def delete_job(job_id):
         if not request.args.get('confirm', False):
             return 'Confirmation required to delete job', 400
 
-        # First, remove all render files and logs
-        found_job = RenderQueue.job_with_id(job_id)
-        files_to_delete = found_job.file_list()
-        files_to_delete.append(found_job.log_path())
-        for d in files_to_delete:
-            if os.path.exists(d):
-                os.remove(d)
-
         # Check if we can remove the 'output' directory
-        output_dir = os.path.dirname(files_to_delete[0])
+        found_job = RenderQueue.job_with_id(job_id)
+        output_dir = os.path.dirname(os.path.dirname(found_job.output_path))
         if server.config['UPLOAD_FOLDER'] in output_dir and os.path.exists(output_dir):
             shutil.rmtree(output_dir)
 
@@ -416,7 +404,8 @@ def delete_job(job_id):
             return "Job deleted", 200
 
     except Exception as e:
-        return f"Unknown error: {e}", 500
+        logger.error(f"Error deleting job: {e}")
+        return f"Error deleting job: {e}", 500
 
 
 @server.get('/api/clear_history')
@@ -446,7 +435,7 @@ def renderer_info():
 
 @server.route('/upload')
 def upload_file_page():
-    return render_template('upload.html', render_clients=RenderQueue.render_clients,
+    return render_template('upload.html', render_clients=render_clients(),
                            supported_renderers=RenderWorkerFactory.supported_renderers())
 
 
@@ -467,10 +456,8 @@ def start_server(background_thread=False):
     server.config['MAX_CONTENT_PATH'] = config['max_content_path']
 
     # Get hostname and render clients
-    RenderQueue.host_name = socket.gethostname()
+    RenderQueue.hostname = socket.gethostname()
-    server.config['HOSTNAME'] = RenderQueue.host_name
+    server.config['HOSTNAME'] = RenderQueue.hostname
-    if not RenderQueue.render_clients:
-        RenderQueue.render_clients = [RenderQueue.host_name]
 
     # disable most Flask logging
     flask_log = logging.getLogger('werkzeug')
@@ -482,7 +469,7 @@ def start_server(background_thread=False):
     thread = threading.Thread(target=eval_loop, kwargs={'delay_sec': config.get('queue_eval_seconds', 1)}, daemon=True)
     thread.start()
 
-    logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.host_name}'")
+    logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.hostname}'")
 
     if background_thread:
         server_thread = threading.Thread(

lib/server/templates/details.html

@@ -12,9 +12,9 @@
     <div style="width: 100%; height: 720px; position: relative; background: black; text-align: center; color: white;">
       <img src="/static/images/gears.png" style="vertical-align: middle; width: auto; height: auto; position:absolute; margin: auto; top: 0; bottom: 0; left: 0; right: 0;">
         <span style="height: auto; position:absolute; margin: auto; top: 58%; left: 0; right: 0; color: white; width: 60%">
-          <progress class="progress is-primary" value="{{job.percent_complete() * 100}}" max="100" style="margin-top: 6px;" id="progress-bar">Rendering</progress>
+          <progress class="progress is-primary" value="{{job.worker_data()['percent_complete'] * 100}}" max="100" style="margin-top: 6px;" id="progress-bar">Rendering</progress>
-          Rendering in Progress - <span id="percent-complete">{{(job.percent_complete() * 100) | int}}%</span>
+          Rendering in Progress - <span id="percent-complete">{{(job.worker_data()['percent_complete'] * 100) | int}}%</span>
-          <br>Time Elapsed: <span id="time-elapsed">{{job.time_elapsed()}}</span>
+          <br>Time Elapsed: <span id="time-elapsed">{{job.worker_data()['time_elapsed']}}</span>
         </span>
       <script>
       var startingStatus = '{{job.render_status().value}}';

lib/utilities/server_helper.py

@@ -9,12 +9,13 @@ from .ffmpeg_helper import generate_thumbnail, save_first_frame
 from lib.render_workers.render_worker import RenderStatus
 
 
-def post_job_to_server(input_path, job_list, client, server_port=8080):
+
+def post_job_to_server(input_path, job_list, hostname, server_port=8080):
     # Pack job data and submit to server
     job_files = {'file': (os.path.basename(input_path), open(input_path, 'rb'), 'application/octet-stream'),
                  'json': (None, json.dumps(job_list), 'application/json')}
 
-    req = requests.post(f'http://{client}:{server_port}/api/add_job', files=job_files)
+    req = requests.post(f'http://{hostname}:{server_port}/api/add_job', files=job_files)
     return req