From 9ac1c44ba339e8301bbaecf2ddf55b72e72a17ec Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Wed, 24 May 2023 15:40:45 -0500 Subject: [PATCH] Add RenderClient object to DB for client tracking --- lib/render_queue.py | 76 ++++++++++++++++++++++--------- lib/server/job_server.py | 55 +++++++++------------- lib/server/templates/details.html | 6 +-- lib/utilities/server_helper.py | 5 +- 4 files changed, 81 insertions(+), 61 deletions(-) diff --git a/lib/render_queue.py b/lib/render_queue.py index 7c9009f..215c119 100755 --- a/lib/render_queue.py +++ b/lib/render_queue.py @@ -4,7 +4,7 @@ from datetime import datetime import psutil import requests -from sqlalchemy import create_engine +from sqlalchemy import create_engine, Column, String, Integer from sqlalchemy.orm import sessionmaker from .render_workers.render_worker import RenderStatus @@ -19,6 +19,27 @@ class JobNotFoundError(Exception): self.job_id = job_id +class RenderClient(Base): + __tablename__ = 'render_clients' + id = Column(Integer, primary_key=True) + hostname = Column(String) + + def __init__(self, hostname): + self.hostname = hostname + + def is_available(self, timeout=3): + try: + response = requests.get(f"http://{self.hostname}:8080/api/status", timeout=timeout) + if response.ok: + return True + except requests.ConnectionError as e: + pass + return False + + def __repr__(self): + return "client stuff" + + class RenderQueue: engine = create_engine('sqlite:///database.db') Base.metadata.create_all(engine) @@ -26,9 +47,8 @@ class RenderQueue: session = Session() ScheduledJob.register_user_events() job_queue = [] - render_clients = [] maximum_renderer_instances = {'blender': 2, 'aerender': 1, 'ffmpeg': 4} - host_name = None + hostname = None port = 8080 client_mode = False server_hostname = None @@ -41,9 +61,9 @@ class RenderQueue: @classmethod def add_to_render_queue(cls, render_job, force_start=False, client=None): - if not client or render_job.client == cls.host_name: + if not client or render_job.client == cls.hostname: logger.debug('Adding priority {} job to render queue: {}'.format(render_job.priority, render_job.worker_object)) - render_job.client = cls.host_name + render_job.client = cls.hostname cls.job_queue.append(render_job) if force_start: cls.start_job(render_job) @@ -165,44 +185,56 @@ class RenderQueue: "memory_available": psutil.virtual_memory().available, "memory_percent": psutil.virtual_memory().percent, "job_counts": cls.job_counts(), - "host_name": cls.host_name + "host_name": cls.hostname } @classmethod - def register_client(cls, hostname): + def render_clients(cls): + all_clients = cls.session.query(RenderClient).all() + if not all_clients: + cls.session.add(RenderClient(hostname=cls.hostname)) + cls.save_state() + all_clients = cls.session.query(RenderClient).all() + return all_clients + @classmethod + def client_with_hostname(cls, hostname): + return cls.session.query(RenderClient).filter(RenderClient.hostname == hostname).first() + + @classmethod + def register_client(cls, hostname): + new_client = None err_msg = None - if hostname == cls.host_name: + if hostname == cls.hostname: err_msg = "Cannot register same hostname as server" - elif hostname in cls.render_clients: + elif cls.client_with_hostname(hostname): err_msg = f"Client '{hostname}' already registered" else: - try: - response = requests.get(f"http://{hostname}:8080/api/status", timeout=3) - if response.ok: - cls.render_clients.append(hostname) - logger.info(f"Client '{hostname}' successfully registered") - cls.save_state() - else: - err_msg = f'Response from server not ok: {response.text}' - except requests.ConnectionError as e: + new_client = RenderClient(hostname=hostname) + if not new_client.is_available(): + cls.session.add(new_client) + logger.info(f"Client '{hostname}' successfully registered") + cls.save_state() + else: err_msg = f"Cannot connect to client at hostname: {hostname}" if err_msg: logger.warning(err_msg) return err_msg, 400 else: - return 'success' + return new_client.hostname @classmethod def unregister_client(cls, hostname): success = False - if hostname in cls.render_clients and hostname != cls.host_name: - cls.render_clients.remove(hostname) + client = cls.client_with_hostname(hostname) + if client and hostname != cls.hostname: + cls.session.delete(client) + cls.save_state() logger.info(f"Client '{hostname}' successfully unregistered") success = True - return success + return str(success) @staticmethod def is_client_available(client_hostname, timeout=3): diff --git a/lib/server/job_server.py b/lib/server/job_server.py index d6cf488..9a73a73 100755 --- a/lib/server/job_server.py +++ b/lib/server/job_server.py @@ -50,8 +50,8 @@ def index(): presets = yaml.load(f, Loader=yaml.FullLoader) return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.all_jobs()), - hostname=RenderQueue.host_name, renderer_info=renderer_info(), - render_clients=RenderQueue.render_clients, preset_list=presets) + hostname=RenderQueue.hostname, renderer_info=renderer_info(), + render_clients=render_clients(), preset_list=presets) @server.route('/ui/job//full_details') @@ -64,7 +64,7 @@ def job_detail(job_id): media_basename = os.path.basename(found_job.file_list()[0]) media_url = f"/api/job/{job_id}/file/{media_basename}" return render_template('details.html', detail_table=table_html, media_url=media_url, - hostname=RenderQueue.host_name, job_status=found_job.render_status().value.title(), + hostname=RenderQueue.hostname, job_status=found_job.render_status().value.title(), job=found_job, renderer_info=renderer_info()) @@ -166,20 +166,18 @@ def download_all(job_id): @server.post('/api/register_client') def register_client(): client_hostname = request.values['hostname'] - x = RenderQueue.register_client(client_hostname) - return "Success" if x else "Fail" + return RenderQueue.register_client(client_hostname) @server.post('/api/unregister_client') def unregister_client(): client_hostname = request.values['hostname'] - x = RenderQueue.unregister_client(client_hostname) - return "Success" if x else "Fail" + return RenderQueue.unregister_client(client_hostname) @server.get('/api/clients') def render_clients(): - return RenderQueue.render_clients + return [c.hostname for c in RenderQueue.render_clients()] @server.get('/api/presets') @@ -194,9 +192,9 @@ def full_status(): full_results = {'timestamp': datetime.now().isoformat(), 'servers': {}} try: - for client_hostname in RenderQueue.render_clients: + for client_hostname in render_clients(): is_online = False - if client_hostname == RenderQueue.host_name: + if client_hostname == RenderQueue.hostname: snapshot_results = snapshot() is_online = True else: @@ -301,7 +299,7 @@ def add_job(job_params, remove_job_dir_on_failure=False): output_path = job_params.get("output_path", None) priority = int(job_params.get('priority', 2)) args = job_params.get('args', {}) - client = job_params.get('client', None) or RenderQueue.host_name + client = job_params.get('client', None) or RenderQueue.hostname force_start = job_params.get('force_start', False) custom_id = None job_dir = None @@ -313,7 +311,7 @@ def add_job(job_params, remove_job_dir_on_failure=False): return {'error': err_msg, 'code': 400} # local renders - if client == RenderQueue.host_name: + if client == RenderQueue.hostname: logger.info(f"Creating job locally - {name if name else input_path}") try: render_job = ScheduledJob(renderer, input_path, output_path, args, priority, job_owner, client, @@ -327,16 +325,13 @@ def add_job(job_params, remove_job_dir_on_failure=False): return {'error': err_msg, 'code': 400} # client renders - elif client in RenderQueue.render_clients: - - # see if host is available - if RenderQueue.is_client_available(client): - + elif client in RenderQueue.render_clients(): + if client.is_available(): # call uploader on remote client try: logger.info(f"Uploading file {input_path} to client {client}") job_data = request.json - response = post_job_to_server(input_path, job_data, client) + response = post_job_to_server(input_path, job_data, client.hostname) if response.ok: logger.info("Job submitted successfully!") return response.json() if response.json() else "Job ok" @@ -382,16 +377,9 @@ def delete_job(job_id): if not request.args.get('confirm', False): return 'Confirmation required to delete job', 400 - # First, remove all render files and logs - found_job = RenderQueue.job_with_id(job_id) - files_to_delete = found_job.file_list() - files_to_delete.append(found_job.log_path()) - for d in files_to_delete: - if os.path.exists(d): - os.remove(d) - # Check if we can remove the 'output' directory - output_dir = os.path.dirname(files_to_delete[0]) + found_job = RenderQueue.job_with_id(job_id) + output_dir = os.path.dirname(os.path.dirname(found_job.output_path)) if server.config['UPLOAD_FOLDER'] in output_dir and os.path.exists(output_dir): shutil.rmtree(output_dir) @@ -416,7 +404,8 @@ def delete_job(job_id): return "Job deleted", 200 except Exception as e: - return f"Unknown error: {e}", 500 + logger.error(f"Error deleting job: {e}") + return f"Error deleting job: {e}", 500 @server.get('/api/clear_history') @@ -446,7 +435,7 @@ def renderer_info(): @server.route('/upload') def upload_file_page(): - return render_template('upload.html', render_clients=RenderQueue.render_clients, + return render_template('upload.html', render_clients=render_clients(), supported_renderers=RenderWorkerFactory.supported_renderers()) @@ -467,10 +456,8 @@ def start_server(background_thread=False): server.config['MAX_CONTENT_PATH'] = config['max_content_path'] # Get hostname and render clients - RenderQueue.host_name = socket.gethostname() - server.config['HOSTNAME'] = RenderQueue.host_name - if not RenderQueue.render_clients: - RenderQueue.render_clients = [RenderQueue.host_name] + RenderQueue.hostname = socket.gethostname() + server.config['HOSTNAME'] = RenderQueue.hostname # disable most Flask logging flask_log = logging.getLogger('werkzeug') @@ -482,7 +469,7 @@ def start_server(background_thread=False): thread = threading.Thread(target=eval_loop, kwargs={'delay_sec': config.get('queue_eval_seconds', 1)}, daemon=True) thread.start() - logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.host_name}'") + logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.hostname}'") if background_thread: server_thread = threading.Thread( diff --git a/lib/server/templates/details.html b/lib/server/templates/details.html index 9ffc691..5e19b6a 100644 --- a/lib/server/templates/details.html +++ b/lib/server/templates/details.html @@ -12,9 +12,9 @@
- Rendering - Rendering in Progress - {{(job.percent_complete() * 100) | int}}% -
Time Elapsed: {{job.time_elapsed()}} + Rendering + Rendering in Progress - {{(job.worker_data()['percent_complete'] * 100) | int}}% +
Time Elapsed: {{job.worker_data()['time_elapsed']}}