diff --git a/config.yaml b/config/config.yaml similarity index 100% rename from config.yaml rename to config/config.yaml diff --git a/utilities/presets.yaml b/config/presets.yaml similarity index 100% rename from utilities/presets.yaml rename to config/presets.yaml diff --git a/dashboard.py b/dashboard.py index 0b05320..be20044 100755 --- a/dashboard.py +++ b/dashboard.py @@ -17,7 +17,7 @@ from rich.table import Table from rich.text import Text from rich.tree import Tree -from utilities.render_worker import RenderStatus, string_to_status +from lib.render_workers.render_worker import RenderStatus, string_to_status from start_server import start_server """ diff --git a/lib/render_job.py b/lib/render_job.py index 8e3e6fb..98affe0 100644 --- a/lib/render_job.py +++ b/lib/render_job.py @@ -6,7 +6,7 @@ import os import uuid from datetime import datetime -from utilities.render_worker import RenderStatus, RenderWorkerFactory +from .render_workers.render_worker import RenderStatus, RenderWorkerFactory logger = logging.getLogger() diff --git a/lib/render_queue.py b/lib/render_queue.py index 9fbe061..9be2757 100755 --- a/lib/render_queue.py +++ b/lib/render_queue.py @@ -7,8 +7,8 @@ from datetime import datetime import psutil import requests -from lib.render_job import RenderJob -from utilities.render_worker import RenderStatus +from .render_job import RenderJob +from .render_workers.render_worker import RenderStatus logger = logging.getLogger() diff --git a/utilities/__init__.py b/lib/render_workers/__init__.py similarity index 100% rename from utilities/__init__.py rename to lib/render_workers/__init__.py diff --git a/utilities/aerender_worker.py b/lib/render_workers/aerender_worker.py similarity index 99% rename from utilities/aerender_worker.py rename to lib/render_workers/aerender_worker.py index e46bba1..79ae63a 100644 --- a/utilities/aerender_worker.py +++ b/lib/render_workers/aerender_worker.py @@ -4,7 +4,7 @@ import json import re import time -from utilities.render_worker import * +from .render_worker import * def aerender_path(): diff --git a/utilities/blender_worker.py b/lib/render_workers/blender_worker.py similarity index 99% rename from utilities/blender_worker.py rename to lib/render_workers/blender_worker.py index ea68254..101349f 100644 --- a/utilities/blender_worker.py +++ b/lib/render_workers/blender_worker.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import json import re -from utilities.render_worker import * +from .render_worker import * class BlenderRenderWorker(BaseRenderWorker): diff --git a/utilities/ffmpeg_worker.py b/lib/render_workers/ffmpeg_worker.py similarity index 98% rename from utilities/ffmpeg_worker.py rename to lib/render_workers/ffmpeg_worker.py index 64d2cd3..a3448d9 100644 --- a/utilities/ffmpeg_worker.py +++ b/lib/render_workers/ffmpeg_worker.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import re import time -from utilities.render_worker import * +from .render_worker import * class FFMPEGRenderWorker(BaseRenderWorker): diff --git a/utilities/render_worker.py b/lib/render_workers/render_worker.py similarity index 97% rename from utilities/render_worker.py rename to lib/render_workers/render_worker.py index 42fca3c..8ab8ccf 100644 --- a/utilities/render_worker.py +++ b/lib/render_workers/render_worker.py @@ -218,9 +218,9 @@ class RenderWorkerFactory: @staticmethod def supported_classes(): # to add support for any additional RenderWorker classes, import their classes and add to list here - from utilities.blender_worker import BlenderRenderWorker - from utilities.aerender_worker import AERenderWorker - from utilities.ffmpeg_worker import FFMPEGRenderWorker + from .blender_worker import BlenderRenderWorker + from .aerender_worker import AERenderWorker + from .ffmpeg_worker import FFMPEGRenderWorker classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker] return classes diff --git a/utilities/scripts/get_blender_info.py b/lib/render_workers/scripts/get_blender_info.py similarity index 100% rename from utilities/scripts/get_blender_info.py rename to lib/render_workers/scripts/get_blender_info.py diff --git a/lib/server/__init__.py b/lib/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/job_server.py b/lib/server/job_server.py similarity index 87% rename from lib/job_server.py rename to lib/server/job_server.py index bedb010..44a22b5 100755 --- a/lib/job_server.py +++ b/lib/server/job_server.py @@ -4,10 +4,13 @@ import logging import os import pathlib import shutil -import json2html +import socket +import threading +import time from datetime import datetime from zipfile import ZipFile +import json2html import requests import yaml from flask import Flask, request, render_template, send_file, after_this_request, Response, redirect, url_for, abort @@ -15,11 +18,11 @@ from werkzeug.utils import secure_filename from lib.render_job import RenderJob from lib.render_queue import RenderQueue, JobNotFoundError -from lib.server_helper import post_job_to_server, generate_thumbnail_for_job -from utilities.render_worker import RenderWorkerFactory, string_to_status, RenderStatus +from lib.render_workers.render_worker import RenderWorkerFactory, string_to_status, RenderStatus +from lib.utilities.server_helper import post_job_to_server, generate_thumbnail_for_job logger = logging.getLogger() -server = Flask(__name__, template_folder='../templates') +server = Flask(__name__, template_folder='templates', static_folder='static') categories = [RenderStatus.RUNNING, RenderStatus.ERROR, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED, RenderStatus.COMPLETED, RenderStatus.CANCELLED] @@ -43,11 +46,12 @@ def sorted_jobs(all_jobs, sort_by_date=True): @server.route('/index') def index(): - with open('utilities/presets.yaml') as f: + with open('config/presets.yaml') as f: presets = yaml.load(f, Loader=yaml.FullLoader) - return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.job_queue), hostname=RenderQueue.host_name, - renderer_info=renderer_info(), render_clients=RenderQueue.render_clients, preset_list=presets) + return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.job_queue), + hostname=RenderQueue.host_name, renderer_info=renderer_info(), + render_clients=RenderQueue.render_clients, preset_list=presets) @server.route('/ui/job//full_details') @@ -133,11 +137,7 @@ def get_job_logs(job_id): @server.get('/api/job//file_list') def get_file_list(job_id): - found_job = RenderQueue.job_with_id(job_id) - if found_job: - return '\n'.join(found_job.file_list()) - else: - return f'Cannot find job with ID {job_id}', 400 + return RenderQueue.job_with_id(job_id) @server.route('/api/job//download_all') @@ -439,3 +439,47 @@ def renderer_info(): def upload_file_page(): return render_template('upload.html', render_clients=RenderQueue.render_clients, supported_renderers=RenderWorkerFactory.supported_renderers()) + + +def start_server(background_thread=False): + def eval_loop(delay_sec=1): + while True: + RenderQueue.evaluate_queue() + time.sleep(delay_sec) + + with open('config/config.yaml') as f: + config = yaml.load(f, Loader=yaml.FullLoader) + + logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S', + level=config.get('server_log_level', 'INFO').upper()) + + server.config['UPLOAD_FOLDER'] = os.path.expanduser(config['upload_folder']) + server.config['THUMBS_FOLDER'] = os.path.join(os.path.expanduser(config['upload_folder']), 'thumbs') + server.config['MAX_CONTENT_PATH'] = config['max_content_path'] + + # Get hostname and render clients + RenderQueue.host_name = socket.gethostname() + server.config['HOSTNAME'] = RenderQueue.host_name + if not RenderQueue.render_clients: + RenderQueue.render_clients = [RenderQueue.host_name] + + # disable most Flask logging + flask_log = logging.getLogger('werkzeug') + flask_log.setLevel(config.get('flask_log_level', 'ERROR').upper()) + + # Set up the RenderQueue object + RenderQueue.load_state() + + thread = threading.Thread(target=eval_loop, kwargs={'delay_sec': config.get('queue_eval_seconds', 1)}, daemon=True) + thread.start() + + logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.host_name}'") + + if background_thread: + server_thread = threading.Thread( + target=lambda: server.run(host='0.0.0.0', port=RenderQueue.port, debug=False, use_reloader=False)) + server_thread.start() + server_thread.join() + else: + server.run(host='0.0.0.0', port=RenderQueue.port, debug=config.get('flask_debug_enable', False), + use_reloader=False, threaded=True) \ No newline at end of file diff --git a/lib/static/images/cancelled.png b/lib/server/static/images/cancelled.png similarity index 100% rename from lib/static/images/cancelled.png rename to lib/server/static/images/cancelled.png diff --git a/lib/static/images/desktop.png b/lib/server/static/images/desktop.png similarity index 100% rename from lib/static/images/desktop.png rename to lib/server/static/images/desktop.png diff --git a/lib/static/images/error.png b/lib/server/static/images/error.png similarity index 100% rename from lib/static/images/error.png rename to lib/server/static/images/error.png diff --git a/lib/static/images/gears.png b/lib/server/static/images/gears.png similarity index 100% rename from lib/static/images/gears.png rename to lib/server/static/images/gears.png diff --git a/lib/static/images/logo.png b/lib/server/static/images/logo.png similarity index 100% rename from lib/static/images/logo.png rename to lib/server/static/images/logo.png diff --git a/lib/static/images/not_started.png b/lib/server/static/images/not_started.png similarity index 100% rename from lib/static/images/not_started.png rename to lib/server/static/images/not_started.png diff --git a/lib/static/images/scheduled.png b/lib/server/static/images/scheduled.png similarity index 100% rename from lib/static/images/scheduled.png rename to lib/server/static/images/scheduled.png diff --git a/lib/static/images/spinner.gif b/lib/server/static/images/spinner.gif similarity index 100% rename from lib/static/images/spinner.gif rename to lib/server/static/images/spinner.gif diff --git a/lib/static/js/job_table.js b/lib/server/static/js/job_table.js similarity index 94% rename from lib/static/js/job_table.js rename to lib/server/static/js/job_table.js index 2538dc6..ba2d63c 100644 --- a/lib/static/js/job_table.js +++ b/lib/server/static/js/job_table.js @@ -50,19 +50,10 @@ columns: [ `), sort: false - } + }, + { id: 'owner', name: 'Owner' } ], -style: { - table: { - 'white-space': 'nowrap' - }, - th: { - 'vertical-align': 'middle', - }, - td: { - 'vertical-align': 'middle', - } -}, +autoWidth: true, server: { url: '/api/jobs', then: results => results, diff --git a/lib/static/js/modals.js b/lib/server/static/js/modals.js similarity index 100% rename from lib/static/js/modals.js rename to lib/server/static/js/modals.js diff --git a/templates/details.html b/lib/server/templates/details.html similarity index 97% rename from templates/details.html rename to lib/server/templates/details.html index 76b200b..9ffc691 100644 --- a/templates/details.html +++ b/lib/server/templates/details.html @@ -30,7 +30,7 @@ }); } if (startingStatus == 'running'){ - var renderingTimer = setInterval(update_job, 2000); + var renderingTimer = setInterval(update_job, 1000); }; diff --git a/templates/index.html b/lib/server/templates/index.html similarity index 100% rename from templates/index.html rename to lib/server/templates/index.html diff --git a/templates/layout.html b/lib/server/templates/layout.html similarity index 100% rename from templates/layout.html rename to lib/server/templates/layout.html diff --git a/templates/upload.html b/lib/server/templates/upload.html similarity index 100% rename from templates/upload.html rename to lib/server/templates/upload.html diff --git a/lib/utilities/__init__.py b/lib/utilities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utilities/ffmpeg_presets.py b/lib/utilities/ffmpeg_helper.py similarity index 84% rename from utilities/ffmpeg_presets.py rename to lib/utilities/ffmpeg_helper.py index afbdf84..611591f 100644 --- a/utilities/ffmpeg_presets.py +++ b/lib/utilities/ffmpeg_helper.py @@ -1,4 +1,5 @@ -import ffmpeg +import subprocess +import ffmpeg # todo: remove all references to ffmpeg library and instead use direct subprocesses def file_info(path): @@ -9,6 +10,11 @@ def file_info(path): return None +def image_sequence_to_video(source_glob_pattern, output_path, framerate="24", encoder="libx264"): + subprocess.run(['ffmpeg', "-y", "-framerate", framerate, "-pattern_type", "glob", "-i", f"{source_glob_pattern}", + "-c:v", encoder, output_path]) + + def save_first_frame(source_path, dest_path, max_width=1280, run_async=False): stream = ffmpeg.input(source_path) stream = ffmpeg.output(stream, dest_path, **{'vf': f'format=yuv420p,scale={max_width}:trunc(ow/a/2)*2', diff --git a/lib/server_helper.py b/lib/utilities/server_helper.py similarity index 93% rename from lib/server_helper.py rename to lib/utilities/server_helper.py index 82b9cc5..1929278 100644 --- a/lib/server_helper.py +++ b/lib/utilities/server_helper.py @@ -1,10 +1,12 @@ -import subprocess -import requests -import os import json +import os +import subprocess import threading -from utilities.render_worker import RenderStatus -from utilities.ffmpeg_presets import generate_thumbnail, save_first_frame + +import requests + +from .ffmpeg_helper import generate_thumbnail, save_first_frame +from lib.render_workers.render_worker import RenderStatus def post_job_to_server(input_path, job_list, client, server_port=8080): diff --git a/scheduler_gui.py b/scheduler_gui.py index 080f302..26d7f09 100755 --- a/scheduler_gui.py +++ b/scheduler_gui.py @@ -6,17 +6,16 @@ import pathlib import socket from tkinter import * from tkinter import filedialog, messagebox -from tkinter.ttk import Frame, Label, Entry, Combobox from tkinter.simpledialog import askstring +from tkinter.ttk import Frame, Label, Entry, Combobox import psutil import requests - -from lib.job_server import post_job_to_server +from lib.utilities.server_helper import post_job_to_server logger = logging.getLogger() -prefs_name = '.scheduler_prefs' +prefs_name = 'config/.scheduler_prefs' label_width = 7 header_padding = 6 server_setup_timeout = 5 @@ -244,7 +243,7 @@ class ScheduleJob(Frame): # get file stats if self.chosen_file: - from utilities.blender_worker import get_scene_info + from lib.render_workers.blender_worker import get_scene_info scene_data = get_scene_info(self.chosen_file) # blender settings @@ -317,7 +316,7 @@ class ScheduleJob(Frame): temp_files = [] if renderer == 'blender': if self.blender_pack_textures.get(): - from utilities.blender_worker import pack_blender_files + from lib.render_workers.blender_worker import pack_blender_files new_path = pack_blender_files(path=input_path) if new_path: logger.info(f'Packed Blender file successfully: {new_path}') diff --git a/start_server.py b/start_server.py index 16aff08..52fd4a0 100755 --- a/start_server.py +++ b/start_server.py @@ -1,57 +1,5 @@ #!/usr/bin/env python3 -import logging -import os -import socket -import threading -import time -import yaml -from lib.job_server import server -from lib.render_queue import RenderQueue - - -def start_server(background_thread=False): - def eval_loop(delay_sec=1): - while True: - RenderQueue.evaluate_queue() - time.sleep(delay_sec) - - with open('config.yaml') as f: - config = yaml.load(f, Loader=yaml.FullLoader) - - logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S', - level=config.get('server_log_level', 'INFO').upper()) - - server.config['UPLOAD_FOLDER'] = os.path.expanduser(config['upload_folder']) - server.config['THUMBS_FOLDER'] = os.path.join(os.path.expanduser(config['upload_folder']), 'thumbs') - server.config['MAX_CONTENT_PATH'] = config['max_content_path'] - - # Get hostname and render clients - RenderQueue.host_name = socket.gethostname() - server.config['HOSTNAME'] = RenderQueue.host_name - if not RenderQueue.render_clients: - RenderQueue.render_clients = [RenderQueue.host_name] - - # disable most Flask logging - flask_log = logging.getLogger('werkzeug') - flask_log.setLevel(config.get('flask_log_level', 'ERROR').upper()) - - # Set up the RenderQueue object - RenderQueue.load_state() - - thread = threading.Thread(target=eval_loop, kwargs={'delay_sec': config.get('queue_eval_seconds', 1)}, daemon=True) - thread.start() - - logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.host_name}'") - - if background_thread: - server_thread = threading.Thread( - target=lambda: server.run(host='0.0.0.0', port=RenderQueue.port, debug=False, use_reloader=False)) - server_thread.start() - server_thread.join() - else: - server.run(host='0.0.0.0', port=RenderQueue.port, debug=config.get('flask_debug_enable', False), - use_reloader=True, threaded=True) - +from lib.server.job_server import start_server if __name__ == '__main__': start_server()