brett/Zordon
1
0
Fork 0
mirror of https://github.com/blw1138/Zordon.git synced 2025-12-17 16:58:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add RenderClient object to DB for client tracking

Browse Source
This commit is contained in:
Brett Williams
2023-05-24 15:40:45 -05:00
parent 31ef59a9d4
commit 9ac1c44ba3
4 changed files with 81 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
lib/render_queue.py
View File

@@ -4,7 +4,7 @@ from datetime import datetime
import psutil
import requests
from sqlalchemy import create_engine
from sqlalchemy import create_engine, Column, String, Integer
from sqlalchemy.orm import sessionmaker
from .render_workers.render_worker import RenderStatus
@@ -19,6 +19,27 @@ class JobNotFoundError(Exception):
self.job_id = job_id
class RenderClient(Base):
__tablename__ = 'render_clients'
id = Column(Integer, primary_key=True)
hostname = Column(String)
def __init__(self, hostname):
self.hostname = hostname
def is_available(self, timeout=3):
try:
response = requests.get(f"http://{self.hostname}:8080/api/status", timeout=timeout)
if response.ok:
return True
except requests.ConnectionError as e:
pass
return False
def __repr__(self):
return "client stuff"
class RenderQueue:
engine = create_engine('sqlite:///database.db')
Base.metadata.create_all(engine)
@@ -26,9 +47,8 @@ class RenderQueue:
session = Session()
ScheduledJob.register_user_events()
job_queue = []
render_clients = []
maximum_renderer_instances = {'blender': 2, 'aerender': 1, 'ffmpeg': 4}
host_name = None
hostname = None
port = 8080
client_mode = False
server_hostname = None
@@ -41,9 +61,9 @@ class RenderQueue:
@classmethod
def add_to_render_queue(cls, render_job, force_start=False, client=None):
if not client or render_job.client == cls.host_name:
if not client or render_job.client == cls.hostname:
logger.debug('Adding priority {} job to render queue: {}'.format(render_job.priority, render_job.worker_object))
render_job.client = cls.host_name
render_job.client = cls.hostname
cls.job_queue.append(render_job)
if force_start:
cls.start_job(render_job)
@@ -165,44 +185,56 @@ class RenderQueue:
"memory_available": psutil.virtual_memory().available,
"memory_percent": psutil.virtual_memory().percent,
"job_counts": cls.job_counts(),
"host_name": cls.host_name
"host_name": cls.hostname
}
@classmethod
def register_client(cls, hostname):
def render_clients(cls):
all_clients = cls.session.query(RenderClient).all()
if not all_clients:
cls.session.add(RenderClient(hostname=cls.hostname))
cls.save_state()
all_clients = cls.session.query(RenderClient).all()
return all_clients
@classmethod
def client_with_hostname(cls, hostname):
return cls.session.query(RenderClient).filter(RenderClient.hostname == hostname).first()
@classmethod
def register_client(cls, hostname):
new_client = None
err_msg = None
if hostname == cls.host_name:
if hostname == cls.hostname:
err_msg = "Cannot register same hostname as server"
elif hostname in cls.render_clients:
elif cls.client_with_hostname(hostname):
err_msg = f"Client '{hostname}' already registered"
else:
try:
response = requests.get(f"http://{hostname}:8080/api/status", timeout=3)
if response.ok:
cls.render_clients.append(hostname)
new_client = RenderClient(hostname=hostname)
if not new_client.is_available():
cls.session.add(new_client)
logger.info(f"Client '{hostname}' successfully registered")
cls.save_state()
else:
err_msg = f'Response from server not ok: {response.text}'
except requests.ConnectionError as e:
err_msg = f"Cannot connect to client at hostname: {hostname}"
if err_msg:
logger.warning(err_msg)
return err_msg, 400
else:
return 'success'
return new_client.hostname
@classmethod
def unregister_client(cls, hostname):
success = False
if hostname in cls.render_clients and hostname != cls.host_name:
cls.render_clients.remove(hostname)
client = cls.client_with_hostname(hostname)
if client and hostname != cls.hostname:
cls.session.delete(client)
cls.save_state()
logger.info(f"Client '{hostname}' successfully unregistered")
success = True
return success
return str(success)
@staticmethod
def is_client_available(client_hostname, timeout=3):

55
lib/server/job_server.py
View File

@@ -50,8 +50,8 @@ def index():
presets = yaml.load(f, Loader=yaml.FullLoader)
return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.all_jobs()),
hostname=RenderQueue.host_name, renderer_info=renderer_info(),
render_clients=RenderQueue.render_clients, preset_list=presets)
hostname=RenderQueue.hostname, renderer_info=renderer_info(),
render_clients=render_clients(), preset_list=presets)
@server.route('/ui/job/<job_id>/full_details')
@@ -64,7 +64,7 @@ def job_detail(job_id):
media_basename = os.path.basename(found_job.file_list()[0])
media_url = f"/api/job/{job_id}/file/{media_basename}"
return render_template('details.html', detail_table=table_html, media_url=media_url,
hostname=RenderQueue.host_name, job_status=found_job.render_status().value.title(),
hostname=RenderQueue.hostname, job_status=found_job.render_status().value.title(),
job=found_job, renderer_info=renderer_info())
@@ -166,20 +166,18 @@ def download_all(job_id):
@server.post('/api/register_client')
def register_client():
client_hostname = request.values['hostname']
x = RenderQueue.register_client(client_hostname)
return "Success" if x else "Fail"
return RenderQueue.register_client(client_hostname)
@server.post('/api/unregister_client')
def unregister_client():
client_hostname = request.values['hostname']
x = RenderQueue.unregister_client(client_hostname)
return "Success" if x else "Fail"
return RenderQueue.unregister_client(client_hostname)
@server.get('/api/clients')
def render_clients():
return RenderQueue.render_clients
return [c.hostname for c in RenderQueue.render_clients()]
@server.get('/api/presets')
@@ -194,9 +192,9 @@ def full_status():
full_results = {'timestamp': datetime.now().isoformat(), 'servers': {}}
try:
for client_hostname in RenderQueue.render_clients:
for client_hostname in render_clients():
is_online = False
if client_hostname == RenderQueue.host_name:
if client_hostname == RenderQueue.hostname:
snapshot_results = snapshot()
is_online = True
else:
@@ -301,7 +299,7 @@ def add_job(job_params, remove_job_dir_on_failure=False):
output_path = job_params.get("output_path", None)
priority = int(job_params.get('priority', 2))
args = job_params.get('args', {})
client = job_params.get('client', None) or RenderQueue.host_name
client = job_params.get('client', None) or RenderQueue.hostname
force_start = job_params.get('force_start', False)
custom_id = None
job_dir = None
@@ -313,7 +311,7 @@ def add_job(job_params, remove_job_dir_on_failure=False):
return {'error': err_msg, 'code': 400}
# local renders
if client == RenderQueue.host_name:
if client == RenderQueue.hostname:
logger.info(f"Creating job locally - {name if name else input_path}")
try:
render_job = ScheduledJob(renderer, input_path, output_path, args, priority, job_owner, client,
@@ -327,16 +325,13 @@ def add_job(job_params, remove_job_dir_on_failure=False):
return {'error': err_msg, 'code': 400}
# client renders
elif client in RenderQueue.render_clients:
# see if host is available
if RenderQueue.is_client_available(client):
elif client in RenderQueue.render_clients():
if client.is_available():
# call uploader on remote client
try:
logger.info(f"Uploading file {input_path} to client {client}")
job_data = request.json
response = post_job_to_server(input_path, job_data, client)
response = post_job_to_server(input_path, job_data, client.hostname)
if response.ok:
logger.info("Job submitted successfully!")
return response.json() if response.json() else "Job ok"
@@ -382,16 +377,9 @@ def delete_job(job_id):
if not request.args.get('confirm', False):
return 'Confirmation required to delete job', 400
# First, remove all render files and logs
found_job = RenderQueue.job_with_id(job_id)
files_to_delete = found_job.file_list()
files_to_delete.append(found_job.log_path())
for d in files_to_delete:
if os.path.exists(d):
os.remove(d)
# Check if we can remove the 'output' directory
output_dir = os.path.dirname(files_to_delete[0])
found_job = RenderQueue.job_with_id(job_id)
output_dir = os.path.dirname(os.path.dirname(found_job.output_path))
if server.config['UPLOAD_FOLDER'] in output_dir and os.path.exists(output_dir):
shutil.rmtree(output_dir)
@@ -416,7 +404,8 @@ def delete_job(job_id):
return "Job deleted", 200
except Exception as e:
return f"Unknown error: {e}", 500
logger.error(f"Error deleting job: {e}")
return f"Error deleting job: {e}", 500
@server.get('/api/clear_history')
@@ -446,7 +435,7 @@ def renderer_info():
@server.route('/upload')
def upload_file_page():
return render_template('upload.html', render_clients=RenderQueue.render_clients,
return render_template('upload.html', render_clients=render_clients(),
supported_renderers=RenderWorkerFactory.supported_renderers())
@@ -467,10 +456,8 @@ def start_server(background_thread=False):
server.config['MAX_CONTENT_PATH'] = config['max_content_path']
# Get hostname and render clients
RenderQueue.host_name = socket.gethostname()
server.config['HOSTNAME'] = RenderQueue.host_name
if not RenderQueue.render_clients:
RenderQueue.render_clients = [RenderQueue.host_name]
RenderQueue.hostname = socket.gethostname()
server.config['HOSTNAME'] = RenderQueue.hostname
# disable most Flask logging
flask_log = logging.getLogger('werkzeug')
@@ -482,7 +469,7 @@ def start_server(background_thread=False):
thread = threading.Thread(target=eval_loop, kwargs={'delay_sec': config.get('queue_eval_seconds', 1)}, daemon=True)
thread.start()
logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.host_name}'")
logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.hostname}'")
if background_thread:
server_thread = threading.Thread(

6
lib/server/templates/details.html
View File

@@ -12,9 +12,9 @@
<div style="width: 100%; height: 720px; position: relative; background: black; text-align: center; color: white;">
<img src="/static/images/gears.png" style="vertical-align: middle; width: auto; height: auto; position:absolute; margin: auto; top: 0; bottom: 0; left: 0; right: 0;">
<span style="height: auto; position:absolute; margin: auto; top: 58%; left: 0; right: 0; color: white; width: 60%">
<progress class="progress is-primary" value="{{job.percent_complete() * 100}}" max="100" style="margin-top: 6px;" id="progress-bar">Rendering</progress>
Rendering in Progress - <span id="percent-complete">{{(job.percent_complete() * 100) | int}}%</span>
<br>Time Elapsed: <span id="time-elapsed">{{job.time_elapsed()}}</span>
<progress class="progress is-primary" value="{{job.worker_data()['percent_complete'] * 100}}" max="100" style="margin-top: 6px;" id="progress-bar">Rendering</progress>
Rendering in Progress - <span id="percent-complete">{{(job.worker_data()['percent_complete'] * 100) | int}}%</span>
<br>Time Elapsed: <span id="time-elapsed">{{job.worker_data()['time_elapsed']}}</span>
</span>
<script>
var startingStatus = '{{job.render_status().value}}';

5
lib/utilities/server_helper.py
View File

@@ -9,12 +9,13 @@ from .ffmpeg_helper import generate_thumbnail, save_first_frame
from lib.render_workers.render_worker import RenderStatus
def post_job_to_server(input_path, job_list, client, server_port=8080):
def post_job_to_server(input_path, job_list, hostname, server_port=8080):
# Pack job data and submit to server
job_files = {'file': (os.path.basename(input_path), open(input_path, 'rb'), 'application/octet-stream'),
'json': (None, json.dumps(job_list), 'application/json')}
req = requests.post(f'http://{client}:{server_port}/api/add_job', files=job_files)
req = requests.post(f'http://{hostname}:{server_port}/api/add_job', files=job_files)
return req