From df922697084cb9ed6fa86aec2199b273471a9e59 Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Sat, 27 May 2023 14:40:34 -0500 Subject: [PATCH] Refactor Worker / Engine / Factory file layout --- lib/render_engines/__init__.py | 0 lib/render_engines/aerender_engine.py | 26 +++++ lib/render_engines/base_engine.py | 44 ++++++++ lib/render_engines/blender_engine.py | 103 ++++++++++++++++++ lib/render_engines/ffmpeg_engine.py | 38 +++++++ .../scripts/get_blender_info.py | 0 lib/render_queue.py | 2 +- lib/render_workers/aerender_worker.py | 26 +---- .../{render_worker.py => base_worker.py} | 68 ------------ lib/render_workers/blender_worker.py | 101 +---------------- lib/render_workers/ffmpeg_worker.py | 32 +----- lib/render_workers/worker_factory.py | 27 +++++ lib/scheduled_job.py | 3 +- lib/server/job_server.py | 3 +- lib/utilities/server_helper.py | 2 +- 15 files changed, 251 insertions(+), 224 deletions(-) create mode 100644 lib/render_engines/__init__.py create mode 100644 lib/render_engines/aerender_engine.py create mode 100644 lib/render_engines/base_engine.py create mode 100644 lib/render_engines/blender_engine.py create mode 100644 lib/render_engines/ffmpeg_engine.py rename lib/{render_workers => render_engines}/scripts/get_blender_info.py (100%) rename lib/render_workers/{render_worker.py => base_worker.py} (82%) create mode 100644 lib/render_workers/worker_factory.py diff --git a/lib/render_engines/__init__.py b/lib/render_engines/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/render_engines/aerender_engine.py b/lib/render_engines/aerender_engine.py new file mode 100644 index 0000000..6c95da5 --- /dev/null +++ b/lib/render_engines/aerender_engine.py @@ -0,0 +1,26 @@ +try: + from .base_engine import * +except ImportError: + from base_engine import * + + +class AERender(BaseRenderEngine): + + supported_extensions = ['.aep'] + + @classmethod + def version(cls): + version = None + try: + render_path = cls.renderer_path() + if render_path: + ver_out = subprocess.check_output([render_path, '-version']) + version = ver_out.decode('utf-8').split(" ")[-1].strip() + except Exception as e: + logger.error(f'Failed to get {cls.name()} version: {e}') + return version + + @classmethod + def get_output_formats(cls): + # todo: create implementation + return [] \ No newline at end of file diff --git a/lib/render_engines/base_engine.py b/lib/render_engines/base_engine.py new file mode 100644 index 0000000..1d36600 --- /dev/null +++ b/lib/render_engines/base_engine.py @@ -0,0 +1,44 @@ +import logging +import os +import subprocess + +logger = logging.getLogger() + + +class BaseRenderEngine(object): + + install_paths = [] + supported_extensions = [] + + @classmethod + def name(cls): + return cls.__name__.lower() + + @classmethod + def renderer_path(cls): + path = None + try: + path = subprocess.check_output(['which', cls.name()]).decode('utf-8').strip() + except subprocess.CalledProcessError: + for p in cls.install_paths: + if os.path.exists(p): + path = p + except Exception as e: + logger.exception(e) + return path + + @classmethod + def version(cls): + raise NotImplementedError("version not implemented") + + @classmethod + def get_help(cls): + path = cls.renderer_path() + if not path: + raise FileNotFoundError("renderer path not found") + help_doc = subprocess.check_output([path, '-h'], stderr=subprocess.STDOUT).decode('utf-8') + return help_doc + + @classmethod + def get_output_formats(cls): + raise NotImplementedError(f"get_output_formats not implemented for {cls.__name__}") \ No newline at end of file diff --git a/lib/render_engines/blender_engine.py b/lib/render_engines/blender_engine.py new file mode 100644 index 0000000..198ac02 --- /dev/null +++ b/lib/render_engines/blender_engine.py @@ -0,0 +1,103 @@ +try: + from .base_engine import * +except ImportError: + from base_engine import * +import json +import re + + +class Blender(BaseRenderEngine): + + install_paths = ['/Applications/Blender.app/Contents/MacOS/Blender'] + supported_extensions = ['.blend'] + + @classmethod + def version(cls): + version = None + try: + render_path = cls.renderer_path() + if render_path: + ver_out = subprocess.check_output([render_path, '-v']) + version = ver_out.decode('utf-8').splitlines()[0].replace('Blender', '').strip() + except Exception as e: + logger.error(f'Failed to get Blender version: {e}') + return version + + @classmethod + def get_output_formats(cls): + format_string = cls.get_help().split('Format Options')[-1].split('Animation Playback Options')[0] + formats = re.findall(r"'([A-Z_0-9]+)'", format_string) + return formats + + @classmethod + def run_python_expression(cls, project_path, python_expression): + if os.path.exists(project_path): + try: + return subprocess.run([cls.renderer_path(), '-b', project_path, '--python-expr', python_expression], + capture_output=True) + except Exception as e: + logger.warning(f"Error running python expression in blender: {e}") + pass + else: + raise FileNotFoundError(f'Project file not found: {project_path}') + + @classmethod + def run_python_script(cls, project_path, script_path): + if os.path.exists(project_path) and os.path.exists(script_path): + try: + return subprocess.run([cls.renderer_path(), '-b', project_path, '--python', script_path], + capture_output=True) + except Exception as e: + logger.warning(f"Error running python expression in blender: {e}") + pass + elif not os.path.exists(project_path): + raise FileNotFoundError(f'Project file not found: {project_path}') + elif not os.path.exists(script_path): + raise FileNotFoundError(f'Python script not found: {script_path}') + raise Exception("Uncaught exception") + + @classmethod + def get_scene_info(cls, project_path): + scene_info = None + try: + results = cls.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'scripts', 'get_blender_info.py')) + result_text = results.stdout.decode() + for line in result_text.splitlines(): + if line.startswith('SCENE_DATA:'): + raw_data = line.split('SCENE_DATA:')[-1] + scene_info = json.loads(raw_data) + break + except Exception as e: + logger.error(f'Error getting file details for .blend file: {e}') + return scene_info + + @classmethod + def pack_project_file(cls, project_path): + # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935 + pack_expression = "import bpy\n" \ + "bpy.ops.file.pack_all()\n" \ + "myPath = bpy.data.filepath\n" \ + "myPath = str(myPath)\n" \ + "bpy.ops.wm.save_as_mainfile(filepath=myPath[:-6]+'_packed'+myPath[-6:])" + + try: + results = Blender.run_python_expression(project_path, pack_expression) + + result_text = results.stdout.decode() + dir_name = os.path.dirname(project_path) + + # report any missing textures + not_found = re.findall("(Unable to pack file, source path .*)\n", result_text) + for err in not_found: + logger.error(err) + + p = re.compile('Info: Saved "(.*)"') + match = p.search(result_text) + if match: + new_path = os.path.join(dir_name, match.group(1)) + logger.info(f'Blender file packed successfully to {new_path}') + return new_path + except Exception as e: + logger.error(f'Error packing .blend file: {e}') + return None diff --git a/lib/render_engines/ffmpeg_engine.py b/lib/render_engines/ffmpeg_engine.py new file mode 100644 index 0000000..2d627b7 --- /dev/null +++ b/lib/render_engines/ffmpeg_engine.py @@ -0,0 +1,38 @@ +try: + from .base_engine import * +except ImportError: + from base_engine import * +import re + + +class FFMPEG(BaseRenderEngine): + + @classmethod + def version(cls): + version = None + try: + ver_out = subprocess.check_output([cls.renderer_path(), '-version']).decode('utf-8') + match = re.match(".*version\s*(\S+)\s*Copyright", ver_out) + if match: + version = match.groups()[0] + except Exception as e: + logger.error("Failed to get FFMPEG version: {}".format(e)) + return version + + @classmethod + def get_encoders(cls): + encoders_raw = subprocess.check_output([cls.renderer_path(), '-encoders'], stderr=subprocess.DEVNULL).decode('utf-8') + pattern = '(?P[VASFXBD.]{6})\s+(?P\S{2,})\s+(?P.*)' + encoders = [m.groupdict() for m in re.finditer(pattern, encoders_raw)] + return encoders + + @classmethod + def get_all_formats(cls): + formats_raw = subprocess.check_output([cls.renderer_path(), '-formats'], stderr=subprocess.DEVNULL).decode('utf-8') + pattern = '(?P[DE]{1,2})\s+(?P\S{2,})\s+(?P.*)' + formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)] + return formats + + @classmethod + def get_output_formats(cls): + return [x for x in cls.get_all_formats() if 'E' in x['type'].upper()] \ No newline at end of file diff --git a/lib/render_workers/scripts/get_blender_info.py b/lib/render_engines/scripts/get_blender_info.py similarity index 100% rename from lib/render_workers/scripts/get_blender_info.py rename to lib/render_engines/scripts/get_blender_info.py diff --git a/lib/render_queue.py b/lib/render_queue.py index 215c119..c7b857c 100755 --- a/lib/render_queue.py +++ b/lib/render_queue.py @@ -7,7 +7,7 @@ import requests from sqlalchemy import create_engine, Column, String, Integer from sqlalchemy.orm import sessionmaker -from .render_workers.render_worker import RenderStatus +from .render_workers.base_worker import RenderStatus from .scheduled_job import ScheduledJob, Base logger = logging.getLogger() diff --git a/lib/render_workers/aerender_worker.py b/lib/render_workers/aerender_worker.py index 4e586a2..5a47a8e 100644 --- a/lib/render_workers/aerender_worker.py +++ b/lib/render_workers/aerender_worker.py @@ -4,8 +4,8 @@ import json import re import time -from .render_worker import * - +from .base_worker import * +from ..render_engines.aerender_engine import AERender def aerender_path(): paths = glob.glob('/Applications/*After Effects*/aerender') @@ -17,28 +17,6 @@ def aerender_path(): return paths[0] -class AERender(BaseRenderEngine): - - supported_extensions = ['.aep'] - - @classmethod - def version(cls): - version = None - try: - render_path = cls.renderer_path() - if render_path: - ver_out = subprocess.check_output([render_path, '-version']) - version = ver_out.decode('utf-8').split(" ")[-1].strip() - except Exception as e: - logging.error(f'Failed to get {cls.name()} version: {e}') - return version - - @classmethod - def get_output_formats(cls): - # todo: create implementation - return [] - - class AERenderWorker(BaseRenderWorker): supported_extensions = ['.aep'] diff --git a/lib/render_workers/render_worker.py b/lib/render_workers/base_worker.py similarity index 82% rename from lib/render_workers/render_worker.py rename to lib/render_workers/base_worker.py index 057d499..975eea9 100644 --- a/lib/render_workers/render_worker.py +++ b/lib/render_workers/base_worker.py @@ -276,74 +276,6 @@ class BaseRenderWorker(object): return worker_json -class BaseRenderEngine(object): - - install_paths = [] - supported_extensions = [] - - @classmethod - def name(cls): - return cls.__name__.lower() - - @classmethod - def renderer_path(cls): - path = None - try: - path = subprocess.check_output(['which', cls.name()]).decode('utf-8').strip() - except subprocess.CalledProcessError: - for p in cls.install_paths: - if os.path.exists(p): - path = p - except Exception as e: - logging.exception(e) - return path - - @classmethod - def version(cls): - raise NotImplementedError("version not implemented") - - @classmethod - def get_help(cls): - path = cls.renderer_path() - if not path: - raise FileNotFoundError("renderer path not found") - help_doc = subprocess.check_output([path, '-h'], stderr=subprocess.STDOUT).decode('utf-8') - return help_doc - - @classmethod - def get_output_formats(cls): - raise NotImplementedError(f"get_output_formats not implemented for {cls.__name__}") - - -class RenderWorkerFactory: - - @staticmethod - def supported_classes(): - # to add support for any additional RenderWorker classes, import their classes and add to list here - from .blender_worker import BlenderRenderWorker - from .aerender_worker import AERenderWorker - from .ffmpeg_worker import FFMPEGRenderWorker - classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker] - return classes - - @staticmethod - def create_worker(renderer, input_path, output_path, args=None): - worker_class = RenderWorkerFactory.class_for_name(renderer) - return worker_class(input_path=input_path, output_path=output_path, args=args) - - @staticmethod - def supported_renderers(): - return [x.engine.name() for x in RenderWorkerFactory.supported_classes()] - - @staticmethod - def class_for_name(name): - name = name.lower() - for render_class in RenderWorkerFactory.supported_classes(): - if render_class.engine.name() == name: - return render_class - raise LookupError(f'Cannot find class for name: {name}') - - def timecode_to_frames(timecode, frame_rate): e = [int(x) for x in timecode.split(':')] seconds = (((e[0] * 60) + e[1] * 60) + e[2]) diff --git a/lib/render_workers/blender_worker.py b/lib/render_workers/blender_worker.py index 1e6e48b..8dbc3f1 100644 --- a/lib/render_workers/blender_worker.py +++ b/lib/render_workers/blender_worker.py @@ -2,106 +2,11 @@ import json import re try: - from .render_worker import * + from .base_worker import * except ImportError: - from render_worker import * + from base_worker import * - -class Blender(BaseRenderEngine): - - install_paths = ['/Applications/Blender.app/Contents/MacOS/Blender'] - supported_extensions = ['.blend'] - - @classmethod - def version(cls): - version = None - try: - render_path = cls.renderer_path() - if render_path: - ver_out = subprocess.check_output([render_path, '-v']) - version = ver_out.decode('utf-8').splitlines()[0].replace('Blender', '').strip() - except Exception as e: - logging.error(f'Failed to get Blender version: {e}') - return version - - @classmethod - def get_output_formats(cls): - format_string = cls.get_help().split('Format Options')[-1].split('Animation Playback Options')[0] - formats = re.findall(r"'([A-Z_0-9]+)'", format_string) - return formats - - @classmethod - def run_python_expression(cls, project_path, python_expression): - if os.path.exists(project_path): - try: - return subprocess.run([cls.renderer_path(), '-b', project_path, '--python-expr', python_expression], - capture_output=True) - except Exception as e: - logger.warning(f"Error running python expression in blender: {e}") - pass - else: - raise FileNotFoundError(f'Project file not found: {project_path}') - - @classmethod - def run_python_script(cls, project_path, script_path): - if os.path.exists(project_path) and os.path.exists(script_path): - try: - return subprocess.run([cls.renderer_path(), '-b', project_path, '--python', script_path], - capture_output=True) - except Exception as e: - logger.warning(f"Error running python expression in blender: {e}") - pass - elif not os.path.exists(project_path): - raise FileNotFoundError(f'Project file not found: {project_path}') - elif not os.path.exists(script_path): - raise FileNotFoundError(f'Python script not found: {script_path}') - raise Exception("Uncaught exception") - - @classmethod - def get_scene_info(cls, project_path): - scene_info = None - try: - results = cls.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'scripts', 'get_blender_info.py')) - result_text = results.stdout.decode() - for line in result_text.splitlines(): - if line.startswith('SCENE_DATA:'): - raw_data = line.split('SCENE_DATA:')[-1] - scene_info = json.loads(raw_data) - break - except Exception as e: - logger.error(f'Error getting file details for .blend file: {e}') - return scene_info - - @classmethod - def pack_project_file(cls, project_path): - # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935 - pack_expression = "import bpy\n" \ - "bpy.ops.file.pack_all()\n" \ - "myPath = bpy.data.filepath\n" \ - "myPath = str(myPath)\n" \ - "bpy.ops.wm.save_as_mainfile(filepath=myPath[:-6]+'_packed'+myPath[-6:])" - - try: - results = Blender.run_python_expression(project_path, pack_expression) - - result_text = results.stdout.decode() - dir_name = os.path.dirname(project_path) - - # report any missing textures - not_found = re.findall("(Unable to pack file, source path .*)\n", result_text) - for err in not_found: - logger.error(err) - - p = re.compile('Info: Saved "(.*)"') - match = p.search(result_text) - if match: - new_path = os.path.join(dir_name, match.group(1)) - logger.info(f'Blender file packed successfully to {new_path}') - return new_path - except Exception as e: - logger.error(f'Error packing .blend file: {e}') - return None +from ..render_engines.blender_engine import Blender class BlenderRenderWorker(BaseRenderWorker): diff --git a/lib/render_workers/ffmpeg_worker.py b/lib/render_workers/ffmpeg_worker.py index 2e17490..4156313 100644 --- a/lib/render_workers/ffmpeg_worker.py +++ b/lib/render_workers/ffmpeg_worker.py @@ -1,35 +1,7 @@ #!/usr/bin/env python3 import re -from .render_worker import * - - -class FFMPEG(BaseRenderEngine): - - @classmethod - def version(cls): - version = None - try: - ver_out = subprocess.check_output([cls.renderer_path(), '-version']).decode('utf-8') - match = re.match(".*version\s*(\S+)\s*Copyright", ver_out) - if match: - version = match.groups()[0] - except Exception as e: - logger.error("Failed to get FFMPEG version: {}".format(e)) - return version - - @classmethod - def get_encoders(cls): - encoders_raw = subprocess.check_output([cls.renderer_path(), '-encoders'], stderr=subprocess.DEVNULL).decode('utf-8') - pattern = '(?P[VASFXBD.]{6})\s+(?P\S{2,})\s+(?P.*)' - encoders = [m.groupdict() for m in re.finditer(pattern, encoders_raw)] - return encoders - - @classmethod - def get_output_formats(cls): - formats_raw = subprocess.check_output([cls.renderer_path(), '-formats'], stderr=subprocess.DEVNULL).decode('utf-8') - pattern = '(?P[DE]{1,2})\s+(?P\S{2,})\s+(?P.*)' - formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)] - return formats +from .base_worker import * +from ..render_engines.ffmpeg_engine import FFMPEG class FFMPEGRenderWorker(BaseRenderWorker): diff --git a/lib/render_workers/worker_factory.py b/lib/render_workers/worker_factory.py new file mode 100644 index 0000000..f2cc3eb --- /dev/null +++ b/lib/render_workers/worker_factory.py @@ -0,0 +1,27 @@ +class RenderWorkerFactory: + + @staticmethod + def supported_classes(): + # to add support for any additional RenderWorker classes, import their classes and add to list here + from .blender_worker import BlenderRenderWorker + from .aerender_worker import AERenderWorker + from .ffmpeg_worker import FFMPEGRenderWorker + classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker] + return classes + + @staticmethod + def create_worker(renderer, input_path, output_path, args=None): + worker_class = RenderWorkerFactory.class_for_name(renderer) + return worker_class(input_path=input_path, output_path=output_path, args=args) + + @staticmethod + def supported_renderers(): + return [x.engine.name() for x in RenderWorkerFactory.supported_classes()] + + @staticmethod + def class_for_name(name): + name = name.lower() + for render_class in RenderWorkerFactory.supported_classes(): + if render_class.engine.name() == name: + return render_class + raise LookupError(f'Cannot find class for name: {name}') \ No newline at end of file diff --git a/lib/scheduled_job.py b/lib/scheduled_job.py index 67ac190..7aab6f1 100644 --- a/lib/scheduled_job.py +++ b/lib/scheduled_job.py @@ -9,7 +9,8 @@ from datetime import datetime from sqlalchemy import Column, Integer, String, DateTime, JSON, event from sqlalchemy.ext.declarative import declarative_base -from .render_workers.render_worker import RenderStatus, RenderWorkerFactory +from .render_workers.base_worker import RenderStatus +from .render_workers.worker_factory import RenderWorkerFactory logger = logging.getLogger() Base = declarative_base() diff --git a/lib/server/job_server.py b/lib/server/job_server.py index 9a73a73..846aceb 100755 --- a/lib/server/job_server.py +++ b/lib/server/job_server.py @@ -18,7 +18,8 @@ from werkzeug.utils import secure_filename from lib.scheduled_job import ScheduledJob from lib.render_queue import RenderQueue, JobNotFoundError -from lib.render_workers.render_worker import RenderWorkerFactory, string_to_status, RenderStatus +from lib.render_workers.worker_factory import RenderWorkerFactory +from lib.render_workers.base_worker import string_to_status, RenderStatus from lib.utilities.server_helper import post_job_to_server, generate_thumbnail_for_job logger = logging.getLogger() diff --git a/lib/utilities/server_helper.py b/lib/utilities/server_helper.py index a58c105..05c7399 100644 --- a/lib/utilities/server_helper.py +++ b/lib/utilities/server_helper.py @@ -7,7 +7,7 @@ import threading import requests from .ffmpeg_helper import generate_thumbnail, save_first_frame -from lib.render_workers.render_worker import RenderStatus +from lib.render_workers.base_worker import RenderStatus logger = logging.getLogger()