Major file reorganization

This commit is contained in:
Brett Williams
2022-12-20 11:22:04 -08:00
parent 63c866166b
commit 4ef94dd7b8
32 changed files with 90 additions and 100 deletions

View File

@@ -6,7 +6,7 @@ import os
import uuid
from datetime import datetime
from utilities.render_worker import RenderStatus, RenderWorkerFactory
from .render_workers.render_worker import RenderStatus, RenderWorkerFactory
logger = logging.getLogger()

View File

@@ -7,8 +7,8 @@ from datetime import datetime
import psutil
import requests
from lib.render_job import RenderJob
from utilities.render_worker import RenderStatus
from .render_job import RenderJob
from .render_workers.render_worker import RenderStatus
logger = logging.getLogger()

View File

View File

@@ -0,0 +1,146 @@
#!/usr/bin/env python3
import glob
import json
import re
import time
from .render_worker import *
def aerender_path():
paths = glob.glob('/Applications/*After Effects*/aerender')
if len(paths) > 1:
logging.warning('Multiple After Effects installations detected')
elif not paths:
logging.error('After Effects installation not found')
else:
return paths[0]
class AERenderWorker(BaseRenderWorker):
renderer = 'After Effects'
render_engine = 'aerender'
supported_extensions = ['.aep']
def __init__(self, input_path, output_path, args=None):
super(AERenderWorker, self).__init__(input_path=input_path, output_path=output_path, ignore_extensions=False,
args=args)
self.comp = args.get('comp', None)
self.render_settings = args.get('render_settings', None)
self.omsettings = args.get('omsettings', None)
self.progress = 0
self.progress_history = []
self.attributes = {}
@classmethod
def version(cls):
version = None
try:
render_path = cls.renderer_path()
if render_path:
ver_out = subprocess.check_output([render_path, '-version'])
version = ver_out.decode('utf-8').split(" ")[-1].strip()
except Exception as e:
logging.error(f'Failed to get {cls.renderer} version: {e}')
return version
def _generate_subprocess(self):
if os.path.exists('nexrender-cli-macos'):
logging.info('nexrender found')
# {
# "template": {
# "src": String,
# "composition": String,
#
# "frameStart": Number,
# "frameEnd": Number,
# "frameIncrement": Number,
#
# "continueOnMissing": Boolean,
# "settingsTemplate": String,
# "outputModule": String,
# "outputExt": String,
# },
# "assets": [],
# "actions": {
# "prerender": [],
# "postrender": [],
# },
# "onChange": Function,
# "onRenderProgress": Function
# }
job = {'template':
{
'src': 'file://' + self.input_path, 'composition': self.comp.replace('"', ''),
'settingsTemplate': self.render_settings.replace('"', ''),
'outputModule': self.omsettings.replace('"', ''), 'outputExt': 'mov'}
}
x = ['./nexrender-cli-macos', "'{}'".format(json.dumps(job))]
else:
logging.info('nexrender not found')
x = [aerender_path(), '-project', self.input_path, '-comp', self.comp, '-RStemplate', self.render_settings,
'-OMtemplate', self.omsettings, '-output', self.output_path]
return x
def _parse_stdout(self, line):
# print line
if line.startswith('PROGRESS:'):
# print 'progress'
trimmed = line.replace('PROGRESS:', '').strip()
if len(trimmed):
self.progress_history.append(line)
if 'Seconds' in trimmed:
self._update_progress(line)
elif ': ' in trimmed:
tmp = trimmed.split(': ')
self.attributes[tmp[0].strip()] = tmp[1].strip()
elif line.startswith('WARNING:'):
trimmed = line.replace('WARNING:', '').strip()
self.warnings.append(trimmed)
logging.warning(trimmed)
elif line.startswith('aerender ERROR') or 'ERROR:' in line:
self.errors.append(line)
logging.error(line)
def _update_progress(self, line):
if not self.total_frames:
duration_string = self.attributes.get('Duration', None)
frame_rate = self.attributes.get('Frame Rate', '0').split(' ')[0]
self.total_frames = timecode_to_frames(duration_string.split('Duration:')[-1], float(frame_rate))
match = re.match(r'PROGRESS:.*\((?P<frame>\d+)\): (?P<time>\d+)', line).groupdict()
self.last_frame = match['frame']
def average_frame_duration(self):
total_durations = 0
for line in self.progress_history:
match = re.match(r'PROGRESS:.*\((?P<frame>\d+)\): (?P<time>\d+)', line)
if match:
total_durations += int(match.group(2))
average = float(total_durations) / self.last_frame
return average
def percent_complete(self):
if self.total_frames:
return (float(self.last_frame) / float(self.total_frames)) * 100
else:
return 0
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG)
r = AERenderWorker('/Users/brett/Desktop/Youtube_Vids/Film_Formats/Frame_Animations.aep', '"Film Pan"',
'"Draft Settings"', '"ProRes"', '/Users/brett/Desktop/test_render')
r.start()
while r.is_running():
time.sleep(0.1)

View File

@@ -0,0 +1,211 @@
#!/usr/bin/env python3
import json
import re
from .render_worker import *
class BlenderRenderWorker(BaseRenderWorker):
renderer = 'Blender'
render_engine = 'blender'
supported_extensions = ['.blend']
install_paths = ['/Applications/Blender.app/Contents/MacOS/Blender']
supported_export_formats = ['TGA', 'RAWTGA', 'JPEG', 'IRIS', 'IRIZ', 'AVIRAW', 'AVIJPEG', 'PNG', 'BMP', 'HDR',
'TIFF', 'OPEN_EXR', 'OPEN_EXR_MULTILAYER', 'MPEG', 'CINEON', 'DPX', 'DDS', 'JP2']
def __init__(self, input_path, output_path, args=None):
super(BlenderRenderWorker, self).__init__(input_path=input_path, output_path=output_path,
ignore_extensions=False, args=args)
# Args
self.engine = self.args.get('engine', 'BLENDER_EEVEE').upper()
self.export_format = self.args.get('export_format', None) or 'JPEG'
self.camera = self.args.get('camera', None)
self.render_all_frames = self.args.get('render_all_frames', False)
self.frame_to_render = 0
# Stats
self.__frame_percent_complete = 0.0
# Scene Info
self.scene_info = get_scene_info(input_path)
self.total_frames = (int(self.scene_info.get('frame_end', 0)) - int(self.scene_info.get('frame_start', 0)) + 1) \
if self.render_all_frames else 1
self.current_frame = int(self.scene_info.get('frame_start', 0))
@classmethod
def version(cls):
version = None
try:
render_path = cls.renderer_path()
if render_path:
ver_out = subprocess.check_output([render_path, '-v'])
version = ver_out.decode('utf-8').splitlines()[0].replace('Blender', '').strip()
except Exception as e:
logging.error(f'Failed to get {cls.renderer} version: {e}')
return version
def _generate_subprocess(self):
cmd = [self.renderer_path()]
if self.args.get('background', True): # optionally run render not in background
cmd.append('-b')
cmd.append(self.input_path)
if self.camera:
cmd.extend(['--python-expr', f"import bpy;bpy.context.scene.camera = bpy.data.objects['{self.camera}'];"])
cmd.extend(['-E', self.engine, '-o', self.output_path, '-F', self.export_format])
# all frames or single
cmd.extend(['-a'] if self.render_all_frames else ['-f', str(self.frame_to_render)])
# Convert raw args from string if available
raw_args = self.args.get('raw', None)
if raw_args:
cmd.extend(raw_args.split(' '))
return cmd
def _parse_stdout(self, line):
pattern = re.compile(
r'Fra:(?P<frame>\d*).*Mem:(?P<memory>\S+).*Time:(?P<time>\S+)(?:.*Remaining:)?(?P<remaining>\S*)')
found = pattern.search(line)
if found:
stats = found.groupdict()
memory_use = stats['memory']
time_elapsed = stats['time']
time_remaining = stats['remaining'] or 'Unknown'
sample_string = line.split('|')[-1].strip()
if "sample" in sample_string.lower():
samples = re.sub(r'[^\d/]', '', sample_string)
self.__frame_percent_complete = int(samples.split('/')[0]) / int(samples.split('/')[-1])
# Calculate rough percent based on cycles
# EEVEE
# 10-Apr-22 22:42:06 - RENDERER: Fra:0 Mem:857.99M (Peak 928.55M) | Time:00:03.96 | Rendering 1 / 65 samples
# 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:07.92 | Rendering 26 / 64 samples
# 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:08.17 | Rendering 51 / 64 samples
# 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:08.31 | Rendering 64 / 64 samples
# CYCLES
# 10-Apr-22 22:43:22 - RENDERER: Fra:0 Mem:836.30M (Peak 1726.13M) | Time:00:01.56 | Remaining:00:30.65 | Mem:588.68M, Peak:588.68M | Scene, View Layer | Sample 1/150
# 10-Apr-22 22:43:43 - RENDERER: Fra:0 Mem:836.30M (Peak 1726.13M) | Time:00:22.01 | Remaining:00:03.36 | Mem:588.68M, Peak:588.68M | Scene, View Layer | Sample 129/150
if int(stats['frame']) > self.current_frame:
self.current_frame = int(stats['frame'])
logger.debug(
'Frame:{0} | Mem:{1} | Time:{2} | Remaining:{3}'.format(self.current_frame, memory_use,
time_elapsed, time_remaining))
elif 'error' in line.lower():
logger.error(line)
self.errors.append(line)
elif 'Saved' in line or 'Saving' in line or 'quit' in line:
match = re.match(r'Time: (.*) \(Saving', line)
if match:
time_completed = match.groups()[0]
if self.render_all_frames:
logger.debug(f'Frame {self.current_frame} completed in {time_completed}')
else:
logger.info(f'Render completed in {time_completed}')
else:
logger.debug(line)
else:
pass
# if len(line.strip()):
# logger.debug(line.strip())
def percent_complete(self):
if self.total_frames <= 1:
return self.__frame_percent_complete
else:
whole_frame_percent = (self.current_frame - 1) / self.total_frames
adjusted_frame_percent = self.__frame_percent_complete / self.total_frames
total_percent = whole_frame_percent + adjusted_frame_percent
return max(total_percent, 0)
def run_python_expression_in_blend(path, python_expression):
if os.path.exists(path):
try:
return subprocess.run([BlenderRenderWorker.renderer_path(), '-b', path, '--python-expr', python_expression],
capture_output=True)
except Exception as e:
logger.warning(f"Error running python expression in blender: {e}")
pass
else:
raise FileNotFoundError
def run_python_script_in_blend(path, python_path):
if os.path.exists(path) and os.path.exists(python_path):
try:
return subprocess.run([BlenderRenderWorker.renderer_path(), '-b', path, '--python', python_path],
capture_output=True)
except Exception as e:
logger.warning(f"Error running python expression in blender: {e}")
pass
else:
raise FileNotFoundError
def pack_blender_files(path):
# Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935
pack_script = "import bpy\n" \
"bpy.ops.file.pack_all()\n" \
"myPath = bpy.data.filepath\n" \
"myPath = str(myPath)\n" \
"bpy.ops.wm.save_as_mainfile(filepath=myPath[:-6]+'_packed'+myPath[-6:])"
try:
results = run_python_expression_in_blend(path, pack_script)
result_text = results.stdout.decode()
dir_name = os.path.dirname(path)
# report any missing textures
not_found = re.findall("(Unable to pack file, source path .*)\n", result_text)
for err in not_found:
logger.error(err)
p = re.compile('Info: Saved "(.*)"')
match = p.search(result_text)
if match:
new_path = os.path.join(dir_name, match.group(1))
logger.info(f'Blender file packed successfully to {new_path}')
return new_path
except Exception as e:
logger.error(f'Error packing .blend file: {e}')
return None
def get_scene_info(path):
scene_info = None
try:
results = run_python_script_in_blend(path, os.path.join(os.path.dirname(os.path.realpath(__file__)),
'scripts', 'get_blender_info.py'))
result_text = results.stdout.decode()
for line in result_text.splitlines():
if line.startswith('SCENE_DATA:'):
raw_data = line.split('SCENE_DATA:')[-1]
scene_info = json.loads(raw_data)
break
except Exception as e:
logger.error(f'Error getting file details for .blend file: {e}')
return scene_info
if __name__ == '__main__':
# x = pack_blender_files('/Users/brett/Blender Files/temple_animatic.blend')
# print(x)
logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG)
r = BlenderRenderWorker('/Users/brett/Blender Files/temple_animatic.blend', '/Users/brett/testing1234')
# r.engine = 'CYCLES'
r.start()
while r.is_running():
time.sleep(1)

View File

@@ -0,0 +1,78 @@
#!/usr/bin/env python3
import re
import time
from .render_worker import *
class FFMPEGRenderWorker(BaseRenderWorker):
renderer = 'ffmpeg'
render_engine = 'ffmpeg'
def __init__(self, input_path, output_path, args=None):
super(FFMPEGRenderWorker, self).__init__(input_path=input_path, output_path=output_path, ignore_extensions=True,
args=args)
stream_info = subprocess.check_output([self.renderer_path(), "-i", # https://stackoverflow.com/a/61604105
input_path, "-map", "0:v:0", "-c", "copy", "-f", "null", "-y",
"/dev/null"], stderr=subprocess.STDOUT).decode('utf-8')
found_frames = re.findall('frame=\s*(\d+)', stream_info)
self.total_frames = found_frames[-1] if found_frames else '-1'
self.frame = 0
# Stats
self.current_frame = -1
@classmethod
def version(cls):
version = None
try:
ver_out = subprocess.check_output([cls.renderer_path(), '-version']).decode('utf-8')
match = re.match(".*version\s*(\S+)\s*Copyright", ver_out)
if match:
version = match.groups()[0]
except Exception as e:
logger.error("Failed to get FFMPEG version: {}".format(e))
return version
def _generate_subprocess(self):
cmd = [self.renderer_path(), '-y', '-stats', '-i', self.input_path]
# Resize frame
if self.args.get('x_resolution', None) and self.args.get('y_resolution', None):
cmd.extend(['-vf', f"scale={self.args['x_resolution']}:{self.args['y_resolution']}"])
# Convert raw args from string if available
raw_args = self.args.get('raw', None)
if raw_args:
cmd.extend(raw_args.split(' '))
# Close with output path
cmd.append(self.output_path)
return cmd
def percent_complete(self):
return max(float(self.current_frame) / float(self.total_frames), 0.0)
def _parse_stdout(self, line):
pattern = re.compile(r'frame=\s*(?P<current_frame>\d+)\s*fps.*time=(?P<time_elapsed>\S+)')
found = pattern.search(line)
if found:
stats = found.groupdict()
self.current_frame = stats['current_frame']
time_elapsed = stats['time_elapsed']
elif "not found" in line:
self.errors.append(line)
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG)
test_movie = '/Users/brettwilliams/Desktop/dark_knight_rises.mp4'
r = FFMPEGRenderWorker(test_movie, '/Users/brettwilliams/Desktop/test-ffmpeg.mp4', args=['-c:v', 'libx265', '-vtag', 'hvc1'])
# r = FFMPEGRenderer(test_movie, '/Users/brettwilliams/Desktop/dark_knight_rises-output.mp4')
r.start()
while r.is_running():
time.sleep(1)

View File

@@ -0,0 +1,250 @@
#!/usr/bin/env python3
import io
import logging
import os
import subprocess
import threading
from datetime import datetime
from enum import Enum
import psutil
logger = logging.getLogger()
class RenderStatus(Enum):
NOT_STARTED = "not_started"
RUNNING = "running"
COMPLETED = "completed"
CANCELLED = "cancelled"
ERROR = "error"
SCHEDULED = "scheduled"
def string_to_status(string):
for stat in RenderStatus:
if stat.value == string:
return stat
return RenderStatus.ERROR
class BaseRenderWorker(object):
renderer = 'BaseRenderWorker'
render_engine = None
render_engine_version = None
supported_extensions = []
install_paths = []
supported_export_formats = []
def __init__(self, input_path, output_path, args=None, ignore_extensions=True):
if not ignore_extensions:
if not any(ext in input_path for ext in self.supported_extensions):
err_meg = f'Cannot find valid project with supported file extension for {self.renderer} renderer'
logger.error(err_meg)
raise ValueError(err_meg)
# Essential Info
self.input_path = input_path
self.output_path = output_path
self.args = args or {}
self.date_created = datetime.now()
self.renderer_version = self.version()
# Frame Ranges
self.total_frames = 0
self.current_frame = 0
# Logging
self.log_path = None
self.start_time = None
self.end_time = None
# History
self.status = RenderStatus.NOT_STARTED
self.warnings = []
self.errors = []
self.failed_attempts = 0
self.maximum_attempts = 1
# Threads and processes
self.__thread = threading.Thread(target=self.run, args=())
self.__thread.daemon = True
self.__process = None
self.is_finished = False
self.last_output = None
@classmethod
def version(cls):
raise NotImplementedError("Unknown version")
@classmethod
def renderer_path(cls):
path = None
try:
path = subprocess.check_output(['which', cls.render_engine]).decode('utf-8').strip()
except subprocess.CalledProcessError:
for p in cls.install_paths:
if os.path.exists(p):
path = p
except Exception as e:
logging.exception(e)
return path
def _generate_subprocess(self):
raise NotImplementedError("_generate_subprocess not implemented")
def start(self):
if not os.path.exists(self.input_path):
self.status = RenderStatus.ERROR
msg = 'Cannot find input path: {}'.format(self.input_path)
logger.error(msg)
self.errors.append(msg)
return
if not self.renderer_path():
self.status = RenderStatus.ERROR
msg = 'Cannot find render engine path for {}'.format(self.render_engine)
logger.error(msg)
self.errors.append(msg)
return
self.status = RenderStatus.RUNNING
logger.info('Starting {0} {1} Render for {2}'.format(self.renderer, self.version(), self.input_path))
self.__thread.start()
def run(self):
# Setup logging
try:
if not self.log_path:
log_dir = os.path.join(os.path.dirname(self.input_path), 'logs')
if not os.path.exists(log_dir):
os.makedirs(log_dir)
self.log_path = os.path.join(log_dir, os.path.basename(self.input_path)) + '.log'
logger.info('Logs saved in {}'.format(self.log_path))
except Exception as e:
logger.error("Error setting up logging: {}".format(e))
while self.failed_attempts < self.maximum_attempts and self.status is not RenderStatus.COMPLETED:
if self.failed_attempts:
logger.info('Attempt #{} failed. Starting attempt #{}'.format(self.failed_attempts, self.failed_attempts + 1))
# Start process and get updates
subprocess_cmds = self._generate_subprocess()
logger.debug("Renderer commands generated - {}".format(" ".join(subprocess_cmds)))
self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=False)
self.start_time = datetime.now()
with open(self.log_path, "a") as f:
f.write("{3} - Starting {0} {1} Render for {2}\n".format(self.renderer, self.version(), self.input_path,
self.start_time.isoformat()))
f.write(f"Running command: {' '.join(subprocess_cmds)}\n")
for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"): # or another encoding
f.write(c)
logger.debug(f"{self.renderer}Worker: {c.strip()}")
self.last_output = c.strip()
self._parse_stdout(c.strip())
f.write('\n')
# Check return codes
return_code = self.__process.wait()
self.end_time = datetime.now()
# Return early if job was cancelled
if self.status is RenderStatus.CANCELLED:
self.is_finished = True
return
duration = self.end_time - self.start_time
if return_code:
message = f"{self.renderer} render failed with return_code {return_code} after {duration}"
logger.error(message)
self.failed_attempts = self.failed_attempts + 1
else:
message = f"{self.renderer} render completed successfully in {duration}"
logger.info(message)
self.status = RenderStatus.COMPLETED
f.write(message)
if self.failed_attempts >= self.maximum_attempts and self.status is not RenderStatus.CANCELLED:
logger.error('{} Render of {} failed after {} attempts'.format(self.renderer, self.input_path, self.failed_attempts))
self.status = RenderStatus.ERROR
if not self.errors:
self.errors = [self.last_output]
self.is_finished = True
def is_running(self):
if self.__thread:
return self.__thread.is_alive()
return False
def stop(self):
if self.__process:
try:
self.status = RenderStatus.CANCELLED
self.maximum_attempts = 0
process = psutil.Process(self.__process.pid)
for proc in process.children(recursive=True):
proc.kill()
process.kill()
except Exception as e:
logger.error(f"Exception stopping the process: {e}")
def percent_complete(self):
return 0
def _parse_stdout(self, line):
raise NotImplementedError("_parse_stdout not implemented")
def elapsed_time(self):
elapsed = ""
if self.start_time:
if self.end_time:
elapsed = self.end_time - self.start_time
elif self.is_running():
elapsed = datetime.now() - self.start_time
return elapsed
class RenderWorkerFactory:
@staticmethod
def supported_classes():
# to add support for any additional RenderWorker classes, import their classes and add to list here
from .blender_worker import BlenderRenderWorker
from .aerender_worker import AERenderWorker
from .ffmpeg_worker import FFMPEGRenderWorker
classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker]
return classes
@staticmethod
def create_worker(renderer, input_path, output_path, args=None):
worker_class = RenderWorkerFactory.class_for_name(renderer)
return worker_class(input_path=input_path, output_path=output_path, args=args)
@staticmethod
def supported_renderers():
return [x.render_engine for x in RenderWorkerFactory.supported_classes()]
@staticmethod
def class_for_name(name):
name = name.lower()
for render_class in RenderWorkerFactory.supported_classes():
if render_class.render_engine == name:
return render_class
raise LookupError(f'Cannot find class for name: {name}')
def timecode_to_frames(timecode, frame_rate):
e = [int(x) for x in timecode.split(':')]
seconds = (((e[0] * 60) + e[1] * 60) + e[2])
frames = (seconds * frame_rate) + e[-1] + 1
return frames

View File

@@ -0,0 +1,20 @@
import json
import bpy
# Get all cameras
cameras = []
for cam_obj in bpy.data.cameras:
user_map = bpy.data.user_map(subset={cam_obj}, value_types={'OBJECT'})
for data_obj in user_map[cam_obj]:
cam = {'name': data_obj.name, 'lens': cam_obj.lens}
cameras.append(cam)
data = {'cameras': cameras,
'frame_start': bpy.data.scenes[0].frame_start,
'frame_end': bpy.data.scenes[0].frame_end,
'resolution_x': bpy.data.scenes[0].render.resolution_x,
'resolution_y': bpy.data.scenes[0].render.resolution_y,
'fps': bpy.data.scenes[0].render.fps}
data_string = json.dumps(data)
print("SCENE_DATA:" + data_string)

0
lib/server/__init__.py Normal file
View File

View File

@@ -4,10 +4,13 @@ import logging
import os
import pathlib
import shutil
import json2html
import socket
import threading
import time
from datetime import datetime
from zipfile import ZipFile
import json2html
import requests
import yaml
from flask import Flask, request, render_template, send_file, after_this_request, Response, redirect, url_for, abort
@@ -15,11 +18,11 @@ from werkzeug.utils import secure_filename
from lib.render_job import RenderJob
from lib.render_queue import RenderQueue, JobNotFoundError
from lib.server_helper import post_job_to_server, generate_thumbnail_for_job
from utilities.render_worker import RenderWorkerFactory, string_to_status, RenderStatus
from lib.render_workers.render_worker import RenderWorkerFactory, string_to_status, RenderStatus
from lib.utilities.server_helper import post_job_to_server, generate_thumbnail_for_job
logger = logging.getLogger()
server = Flask(__name__, template_folder='../templates')
server = Flask(__name__, template_folder='templates', static_folder='static')
categories = [RenderStatus.RUNNING, RenderStatus.ERROR, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED,
RenderStatus.COMPLETED, RenderStatus.CANCELLED]
@@ -43,11 +46,12 @@ def sorted_jobs(all_jobs, sort_by_date=True):
@server.route('/index')
def index():
with open('utilities/presets.yaml') as f:
with open('config/presets.yaml') as f:
presets = yaml.load(f, Loader=yaml.FullLoader)
return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.job_queue), hostname=RenderQueue.host_name,
renderer_info=renderer_info(), render_clients=RenderQueue.render_clients, preset_list=presets)
return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.job_queue),
hostname=RenderQueue.host_name, renderer_info=renderer_info(),
render_clients=RenderQueue.render_clients, preset_list=presets)
@server.route('/ui/job/<job_id>/full_details')
@@ -133,11 +137,7 @@ def get_job_logs(job_id):
@server.get('/api/job/<job_id>/file_list')
def get_file_list(job_id):
found_job = RenderQueue.job_with_id(job_id)
if found_job:
return '\n'.join(found_job.file_list())
else:
return f'Cannot find job with ID {job_id}', 400
return RenderQueue.job_with_id(job_id)
@server.route('/api/job/<job_id>/download_all')
@@ -439,3 +439,47 @@ def renderer_info():
def upload_file_page():
return render_template('upload.html', render_clients=RenderQueue.render_clients,
supported_renderers=RenderWorkerFactory.supported_renderers())
def start_server(background_thread=False):
def eval_loop(delay_sec=1):
while True:
RenderQueue.evaluate_queue()
time.sleep(delay_sec)
with open('config/config.yaml') as f:
config = yaml.load(f, Loader=yaml.FullLoader)
logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S',
level=config.get('server_log_level', 'INFO').upper())
server.config['UPLOAD_FOLDER'] = os.path.expanduser(config['upload_folder'])
server.config['THUMBS_FOLDER'] = os.path.join(os.path.expanduser(config['upload_folder']), 'thumbs')
server.config['MAX_CONTENT_PATH'] = config['max_content_path']
# Get hostname and render clients
RenderQueue.host_name = socket.gethostname()
server.config['HOSTNAME'] = RenderQueue.host_name
if not RenderQueue.render_clients:
RenderQueue.render_clients = [RenderQueue.host_name]
# disable most Flask logging
flask_log = logging.getLogger('werkzeug')
flask_log.setLevel(config.get('flask_log_level', 'ERROR').upper())
# Set up the RenderQueue object
RenderQueue.load_state()
thread = threading.Thread(target=eval_loop, kwargs={'delay_sec': config.get('queue_eval_seconds', 1)}, daemon=True)
thread.start()
logging.info(f"Starting Zordon Render Server - Hostname: '{RenderQueue.host_name}'")
if background_thread:
server_thread = threading.Thread(
target=lambda: server.run(host='0.0.0.0', port=RenderQueue.port, debug=False, use_reloader=False))
server_thread.start()
server_thread.join()
else:
server.run(host='0.0.0.0', port=RenderQueue.port, debug=config.get('flask_debug_enable', False),
use_reloader=False, threaded=True)

View File

Before

Width:  |  Height:  |  Size: 1.7 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

View File

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

View File

Before

Width:  |  Height:  |  Size: 995 B

After

Width:  |  Height:  |  Size: 995 B

View File

Before

Width:  |  Height:  |  Size: 81 KiB

After

Width:  |  Height:  |  Size: 81 KiB

View File

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

View File

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

View File

Before

Width:  |  Height:  |  Size: 2.1 KiB

After

Width:  |  Height:  |  Size: 2.1 KiB

View File

Before

Width:  |  Height:  |  Size: 66 KiB

After

Width:  |  Height:  |  Size: 66 KiB

View File

@@ -50,19 +50,10 @@ columns: [
</div>
`),
sort: false
}
},
{ id: 'owner', name: 'Owner' }
],
style: {
table: {
'white-space': 'nowrap'
},
th: {
'vertical-align': 'middle',
},
td: {
'vertical-align': 'middle',
}
},
autoWidth: true,
server: {
url: '/api/jobs',
then: results => results,

View File

@@ -0,0 +1,48 @@
{% extends 'layout.html' %}
{% block body %}
<div class="container" style="text-align:center; width: 100%">
<br>
{% if media_url: %}
<video width="1280" height="720" controls>
<source src="{{media_url}}" type="video/mp4">
Your browser does not support the video tag.
</video>
{% elif job_status == 'Running': %}
<div style="width: 100%; height: 720px; position: relative; background: black; text-align: center; color: white;">
<img src="/static/images/gears.png" style="vertical-align: middle; width: auto; height: auto; position:absolute; margin: auto; top: 0; bottom: 0; left: 0; right: 0;">
<span style="height: auto; position:absolute; margin: auto; top: 58%; left: 0; right: 0; color: white; width: 60%">
<progress class="progress is-primary" value="{{job.percent_complete() * 100}}" max="100" style="margin-top: 6px;" id="progress-bar">Rendering</progress>
Rendering in Progress - <span id="percent-complete">{{(job.percent_complete() * 100) | int}}%</span>
<br>Time Elapsed: <span id="time-elapsed">{{job.time_elapsed()}}</span>
</span>
<script>
var startingStatus = '{{job.render_status().value}}';
function update_job() {
$.getJSON('/api/job/{{job.id}}', function(data) {
document.getElementById('progress-bar').value = (data.percent_complete * 100);
document.getElementById('percent-complete').innerHTML = (data.percent_complete * 100).toFixed(0) + '%';
document.getElementById('time-elapsed').innerHTML = data.time_elapsed;
if (data.status != startingStatus){
clearInterval(renderingTimer);
window.location.reload(true);
};
});
}
if (startingStatus == 'running'){
var renderingTimer = setInterval(update_job, 1000);
};
</script>
</div>
{% else %}
<div style="width: 100%; height: 720px; position: relative; background: black;">
<img src="/static/images/{{job_status}}.png" style="vertical-align: middle; width: auto; height: auto; position:absolute; margin: auto; top: 0; bottom: 0; left: 0; right: 0;">
<span style="height: auto; position:absolute; margin: auto; top: 58%; left: 0; right: 0; color: white;">
{{job_status}}
</span>
</div>
{% endif %}
<br>
{{detail_table|safe}}
</div>
{% endblock %}

View File

@@ -0,0 +1,8 @@
{% extends 'layout.html' %}
{% block body %}
<div class="container is-fluid" style="padding-top: 20px;">
<div id="table" class="table"></div>
</div>
<script src="/static/js/job_table.js"></script>
{% endblock %}

View File

@@ -0,0 +1,236 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Zordon Dashboard</title>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/gridjs/dist/gridjs.umd.js"></script>
<link href="https://unpkg.com/gridjs/dist/theme/mermaid.min.css" rel="stylesheet" />
<script src="https://kit.fontawesome.com/698705d14d.js" crossorigin="anonymous"></script>
<script type="text/javascript" src="/static/js/modals.js"></script>
</head>
<body onload="rendererChanged(document.getElementById('renderer'))">
<nav class="navbar is-dark" role="navigation" aria-label="main navigation">
<div class="navbar-brand">
<a class="navbar-item" href="/">
<img src="/static/images/logo.png">
</a>
</div>
<div id="navbarBasicExample" class="navbar-menu">
<div class="navbar-start">
<a class="navbar-item" href="/">
Home
</a>
</div>
<div class="navbar-end">
<div class="navbar-item">
<button class="button is-primary js-modal-trigger" data-target="add-job-modal">
<span class="icon">
<i class="fa-solid fa-upload"></i>
</span>
<span>Submit Job</span>
</button>
</div>
</div>
</div>
</nav>
{% block body %}
{% endblock %}
<div id="add-job-modal" class="modal">
<!-- Start Add Form -->
<form id="submit_job" action="/api/add_job?redirect=True" method="POST" enctype="multipart/form-data">
<div class="modal-background"></div>
<div class="modal-card">
<header class="modal-card-head">
<p class="modal-card-title">Submit New Job</p>
<button class="delete" aria-label="close" type="button"></button>
</header>
<section class="modal-card-body">
<!-- File Uploader -->
<label class="label">Upload File</label>
<div id="file-uploader" class="file has-name is-fullwidth">
<label class="file-label">
<input class="file-input is-small" type="file" name="file">
<span class="file-cta">
<span class="file-icon">
<i class="fas fa-upload"></i>
</span>
<span class="file-label">
Choose a file…
</span>
</span>
<span class="file-name">
No File Uploaded
</span>
</label>
</div>
<br>
<script>
const fileInput = document.querySelector('#file-uploader input[type=file]');
fileInput.onchange = () => {
if (fileInput.files.length > 0) {
const fileName = document.querySelector('#file-uploader .file-name');
fileName.textContent = fileInput.files[0].name;
}
}
const presets = {
{% for preset in preset_list: %}
{{preset}}: {
name: '{{preset_list[preset]['name']}}',
renderer: '{{preset_list[preset]['renderer']}}',
args: '{{preset_list[preset]['args']}}',
},
{% endfor %}
};
function rendererChanged(ddl1) {
var renderers = {
{% for renderer in renderer_info: %}
{% if renderer_info[renderer]['supported_export_formats']: %}
{{renderer}}: [
{% for format in renderer_info[renderer]['supported_export_formats']: %}
'{{format}}',
{% endfor %}
],
{% endif %}
{% endfor %}
};
var selectedRenderer = ddl1.value;
var ddl3 = document.getElementById('preset_list');
ddl3.options.length = 0;
createOption(ddl3, '-Presets-', '');
for (var preset_name in presets) {
if (presets[preset_name]['renderer'] == selectedRenderer) {
createOption(ddl3, presets[preset_name]['name'], preset_name);
};
};
document.getElementById('raw_args').value = "";
var ddl2 = document.getElementById('export_format');
ddl2.options.length = 0;
var options = renderers[selectedRenderer];
for (i = 0; i < options.length; i++) {
createOption(ddl2, options[i], options[i]);
};
}
function createOption(ddl, text, value) {
var opt = document.createElement('option');
opt.value = value;
opt.text = text;
ddl.options.add(opt);
}
function addPresetTextToInput(presetfield, textfield) {
var p = presets[presetfield.value];
textfield.value = p['args'];
}
</script>
<!-- Renderer & Priority -->
<div class="field is-grouped">
<p class="control">
<label class="label">Renderer</label>
<span class="select">
<select id="renderer" name="renderer" onchange="rendererChanged(this)">
{% for renderer in renderer_info: %}
<option name="renderer" value="{{renderer}}">{{renderer}}</option>
{% endfor %}
</select>
</span>
</p>
<p class="control">
<label class="label">Client</label>
<span class="select">
<select name="client">
<option name="client" value="">First Available</option>
{% for client in render_clients: %}
<option name="client" value="{{client}}">{{client}}</option>
{% endfor %}
</select>
</span>
</p>
<p class="control">
<label class="label">Priority</label>
<span class="select">
<select name="priority">
<option name="priority" value="1">1</option>
<option name="priority" value="2" selected="selected">2</option>
<option name="priority" value="3">3</option>
</select>
</span>
</p>
</div>
<!-- Output Path -->
<label class="label">Output</label>
<div class="field has-addons">
<div class="control is-expanded">
<input class="input is-small" type="text" placeholder="Output Name" name="output_path" value="output.mp4">
</div>
<p class="control">
<span class="select is-small">
<select id="export_format" name="export_format">
<option value="ar">option</option>
</select>
</span>
</p>
</div>
<!-- Resolution -->
<!-- <label class="label">Resolution</label>-->
<!-- <div class="field is-grouped">-->
<!-- <p class="control">-->
<!-- <input class="input" type="text" placeholder="auto" maxlength="5" size="8" name="AnyRenderer-arg_x_resolution">-->
<!-- </p>-->
<!-- <label class="label"> x </label>-->
<!-- <p class="control">-->
<!-- <input class="input" type="text" placeholder="auto" maxlength="5" size="8" name="AnyRenderer-arg_y_resolution">-->
<!-- </p>-->
<!-- <label class="label"> @ </label>-->
<!-- <p class="control">-->
<!-- <input class="input" type="text" placeholder="auto" maxlength="3" size="5" name="AnyRenderer-arg_frame_rate">-->
<!-- </p>-->
<!-- <label class="label"> fps </label>-->
<!-- </div>-->
<label class="label">Command Line Arguments</label>
<div class="field has-addons">
<p class="control">
<span class="select is-small">
<select id="preset_list" onchange="addPresetTextToInput(this, document.getElementById('raw_args'))">
<option value="preset-placeholder">presets</option>
</select>
</span>
</p>
<p class="control is-expanded">
<input class="input is-small" type="text" placeholder="Args" id="raw_args" name="raw_args">
</p>
</div>
<!-- End Add Form -->
</section>
<footer class="modal-card-foot">
<input class="button is-link" type="submit"/>
<button class="button" type="button">Cancel</button>
</footer>
</div>
</form>
</div>
</body>
</html>

View File

@@ -0,0 +1,62 @@
<html>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
<script>
$(function() {
$('#renderer').change(function() {
$('.render_settings').hide();
$('#' + $(this).val()).show();
});
});
</script>
<body>
<h3>Upload a file</h3>
<div>
<form action="/add_job" method="POST"
enctype="multipart/form-data">
<div>
<input type="file" name="file"/><br>
</div>
<input type="hidden" id="origin" name="origin" value="html">
<div id="client">
Render Client:
<select name="client">
{% for client in render_clients %}
<option value="{{client}}">{{client}}</option>
{% endfor %}
</select>
</div>
<div id="priority">
Priority:
<select name="priority">
<option value="1">1</option>
<option value="2" selected>2</option>
<option value="3">3</option>
</select>
</div>
<div>
<label for="renderer">Renderer:</label>
<select id="renderer" name="renderer">
{% for renderer in supported_renderers %}
<option value="{{renderer}}">{{renderer}}</option>
{% endfor %}
</select>
</div>
<div id="blender" class="render_settings" style="display:none">
Engine:
<select name="blender+engine">
<option value="CYCLES">Cycles</option>
<option value="BLENDER_EEVEE">Eevee</option>
</select>
</div>
<br>
<input type="submit"/>
</form>
</div>
</body>
</html>

View File

View File

@@ -0,0 +1,57 @@
import subprocess
import ffmpeg # todo: remove all references to ffmpeg library and instead use direct subprocesses
def file_info(path):
try:
return ffmpeg.probe(path)
except Exception as e:
print('Error getting ffmpeg info: ' + str(e))
return None
def image_sequence_to_video(source_glob_pattern, output_path, framerate="24", encoder="libx264"):
subprocess.run(['ffmpeg', "-y", "-framerate", framerate, "-pattern_type", "glob", "-i", f"{source_glob_pattern}",
"-c:v", encoder, output_path])
def save_first_frame(source_path, dest_path, max_width=1280, run_async=False):
stream = ffmpeg.input(source_path)
stream = ffmpeg.output(stream, dest_path, **{'vf': f'format=yuv420p,scale={max_width}:trunc(ow/a/2)*2',
'vframes': '1'})
return _run_output(stream, run_async)
def generate_fast_preview(source_path, dest_path, max_width=1280, run_async=False):
stream = ffmpeg.input(source_path)
stream = ffmpeg.output(stream, dest_path, **{'vf': 'format=yuv420p,scale={width}:-2'.format(width=max_width),
'preset': 'ultrafast'})
return _run_output(stream, run_async)
def generate_thumbnail(source_path, dest_path, max_width=240, run_async=False):
stream = ffmpeg.input(source_path).video
stream = ffmpeg.output(stream, dest_path, **{'vf': f'scale={max_width}:trunc(ow/a/2)*2',
'preset': 'veryfast',
'r': '15',
'c:v': 'libx265',
'tag:v': 'hvc1'})
return _run_output(stream, run_async)
def generate_prores_trim(source_path, dest_path, start_frame, end_frame, handles=10, run_async=False):
stream = ffmpeg.input(source_path)
stream = stream.trim(**{'start_frame': max(start_frame-handles, 0), 'end_frame': end_frame + handles})
stream = stream.setpts('PTS-STARTPTS') # reset timecode
stream = ffmpeg.output(stream, dest_path, strict='-2', **{'c:v': 'prores_ks', 'profile:v': 4})
return _run_output(stream, run_async)
def _run_output(stream, run_async):
return ffmpeg.run_async(stream, quiet=True, overwrite_output=True) if run_async else \
ffmpeg.run(stream, quiet=True, overwrite_output=True)
if __name__ == '__main__':
x = generate_thumbnail("/Users/brett/Desktop/pexels.mp4", "/Users/brett/Desktop/test-output.mp4", max_width=320)
print(x)

View File

@@ -1,10 +1,12 @@
import subprocess
import requests
import os
import json
import os
import subprocess
import threading
from utilities.render_worker import RenderStatus
from utilities.ffmpeg_presets import generate_thumbnail, save_first_frame
import requests
from .ffmpeg_helper import generate_thumbnail, save_first_frame
from lib.render_workers.render_worker import RenderStatus
def post_job_to_server(input_path, job_list, client, server_port=8080):