From a475aa999a8b4780cb2777f9df1bc9962c04373a Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Fri, 30 Jun 2023 21:24:40 -0500 Subject: [PATCH] Major file reorganization (#26) * Major file reorganization * Rearrange imports * Fix default log level --- dashboard.py | 6 +-- lib/workers/__init__.py | 0 requirements.txt | 5 +-- {lib => src}/__init__.py | 0 {lib/server => src}/api_server.py | 38 +++++++++--------- {lib => src}/client/__init__.py | 0 {lib => src}/client/dashboard_window.py | 21 +++++----- {lib => src}/client/new_job_window.py | 10 ++--- {lib => src}/distributed_job_manager.py | 11 ++--- {lib => src}/render_queue.py | 18 +++++++-- {lib/server => src}/server_proxy.py | 14 ++++--- {lib/engines => src/utilities}/__init__.py | 0 {lib => src}/utilities/ffmpeg_helper.py | 2 +- {lib => src}/utilities/misc_helper.py | 0 {lib => src}/utilities/server_helper.py | 10 ++--- src/utilities/status_utils.py | 20 +++++++++ .../utilities}/zeroconf_server.py | 1 + .../web}/static/images/cancelled.png | Bin .../web}/static/images/desktop.png | Bin .../web}/static/images/error.png | Bin .../web}/static/images/gears.png | Bin .../server => src/web}/static/images/logo.png | Bin .../web}/static/images/not_started.png | Bin .../web}/static/images/scheduled.png | Bin .../web}/static/images/spinner.gif | Bin .../server => src/web}/static/js/job_table.js | 0 {lib/server => src/web}/static/js/modals.js | 0 .../server => src/web}/templates/details.html | 0 {lib/server => src/web}/templates/index.html | 0 {lib/server => src/web}/templates/layout.html | 0 {lib/server => src/web}/templates/upload.html | 0 {lib/workers => src}/worker_factory.py | 6 +-- {lib/server => src/workers}/__init__.py | 0 {lib => src}/workers/aerender_worker.py | 7 +++- {lib => src}/workers/base_worker.py | 34 ++++------------ {lib => src}/workers/blender_worker.py | 17 ++------ .../workers/engines}/__init__.py | 0 .../workers}/engines/aerender_engine.py | 0 {lib => src/workers}/engines/base_engine.py | 0 .../workers}/engines/blender_engine.py | 0 {lib => src/workers}/engines/ffmpeg_engine.py | 0 .../engines/scripts/blender/get_file_info.py | 0 .../engines/scripts/blender/pack_project.py | 0 {lib => src}/workers/ffmpeg_worker.py | 21 ++-------- start_client.py | 2 +- start_server.py | 2 +- 46 files changed, 117 insertions(+), 128 deletions(-) delete mode 100644 lib/workers/__init__.py rename {lib => src}/__init__.py (100%) rename {lib/server => src}/api_server.py (95%) rename {lib => src}/client/__init__.py (100%) rename {lib => src}/client/dashboard_window.py (97%) rename {lib => src}/client/new_job_window.py (99%) rename {lib => src}/distributed_job_manager.py (98%) rename {lib => src}/render_queue.py (86%) rename {lib/server => src}/server_proxy.py (96%) rename {lib/engines => src/utilities}/__init__.py (100%) rename {lib => src}/utilities/ffmpeg_helper.py (95%) rename {lib => src}/utilities/misc_helper.py (100%) rename {lib => src}/utilities/server_helper.py (89%) create mode 100644 src/utilities/status_utils.py rename {lib/server => src/utilities}/zeroconf_server.py (98%) rename {lib/server => src/web}/static/images/cancelled.png (100%) rename {lib/server => src/web}/static/images/desktop.png (100%) rename {lib/server => src/web}/static/images/error.png (100%) rename {lib/server => src/web}/static/images/gears.png (100%) rename {lib/server => src/web}/static/images/logo.png (100%) rename {lib/server => src/web}/static/images/not_started.png (100%) rename {lib/server => src/web}/static/images/scheduled.png (100%) rename {lib/server => src/web}/static/images/spinner.gif (100%) rename {lib/server => src/web}/static/js/job_table.js (100%) rename {lib/server => src/web}/static/js/modals.js (100%) rename {lib/server => src/web}/templates/details.html (100%) rename {lib/server => src/web}/templates/index.html (100%) rename {lib/server => src/web}/templates/layout.html (100%) rename {lib/server => src/web}/templates/upload.html (100%) rename {lib/workers => src}/worker_factory.py (83%) rename {lib/server => src/workers}/__init__.py (100%) rename {lib => src}/workers/aerender_worker.py (96%) rename {lib => src}/workers/base_worker.py (92%) rename {lib => src}/workers/blender_worker.py (85%) rename {lib/utilities => src/workers/engines}/__init__.py (100%) rename {lib => src/workers}/engines/aerender_engine.py (100%) rename {lib => src/workers}/engines/base_engine.py (100%) rename {lib => src/workers}/engines/blender_engine.py (100%) rename {lib => src/workers}/engines/ffmpeg_engine.py (100%) rename {lib => src/workers}/engines/scripts/blender/get_file_info.py (100%) rename {lib => src/workers}/engines/scripts/blender/pack_project.py (100%) rename {lib => src}/workers/ffmpeg_worker.py (76%) diff --git a/dashboard.py b/dashboard.py index 703a5d4..4d84a0e 100755 --- a/dashboard.py +++ b/dashboard.py @@ -17,9 +17,9 @@ from rich.table import Table from rich.text import Text from rich.tree import Tree -from lib.workers.base_worker import RenderStatus, string_to_status -from lib.server.server_proxy import RenderServerProxy -from lib.utilities.misc_helper import get_time_elapsed +from src.workers.base_worker import RenderStatus, string_to_status +from src.server_proxy import RenderServerProxy +from src.utilities.misc_helper import get_time_elapsed from start_server import start_server """ diff --git a/lib/workers/__init__.py b/lib/workers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/requirements.txt b/requirements.txt index 63d56a7..3a9e33c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,12 @@ -click~=8.1.3 requests==2.31.0 psutil==5.9.5 PyYAML~=6.0 Flask==2.3.2 rich==13.4.1 Werkzeug==2.3.5 -tkinterdnd2~=0.3.0 future==0.18.3 json2html~=1.3.0 SQLAlchemy~=2.0.15 Pillow==9.5.0 zeroconf==0.64.1 -requests-toolbelt~=1.0 -arrow~=1.2.3 \ No newline at end of file +Pypubsub~=4.0.3 \ No newline at end of file diff --git a/lib/__init__.py b/src/__init__.py similarity index 100% rename from lib/__init__.py rename to src/__init__.py diff --git a/lib/server/api_server.py b/src/api_server.py similarity index 95% rename from lib/server/api_server.py rename to src/api_server.py index e75b7ed..2f284af 100755 --- a/lib/server/api_server.py +++ b/src/api_server.py @@ -20,16 +20,16 @@ import yaml from flask import Flask, request, render_template, send_file, after_this_request, Response, redirect, url_for, abort from werkzeug.utils import secure_filename -from lib.distributed_job_manager import DistributedJobManager -from lib.render_queue import RenderQueue, JobNotFoundError -from lib.server.server_proxy import RenderServerProxy -from lib.server.zeroconf_server import ZeroconfServer -from lib.utilities.server_helper import generate_thumbnail_for_job -from lib.workers.base_worker import string_to_status, RenderStatus -from lib.workers.worker_factory import RenderWorkerFactory +from src.distributed_job_manager import DistributedJobManager +from src.render_queue import RenderQueue, JobNotFoundError +from src.server_proxy import RenderServerProxy +from src.utilities.server_helper import generate_thumbnail_for_job +from src.utilities.zeroconf_server import ZeroconfServer +from src.worker_factory import RenderWorkerFactory +from src.workers.base_worker import string_to_status, RenderStatus logger = logging.getLogger() -server = Flask(__name__, template_folder='templates', static_folder='static') +server = Flask(__name__, template_folder='web/templates', static_folder='web/static') ssl._create_default_https_context = ssl._create_unverified_context # disable SSL for downloads categories = [RenderStatus.RUNNING, RenderStatus.ERROR, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED, @@ -53,13 +53,12 @@ def sorted_jobs(all_jobs, sort_by_date=True): @server.route('/') @server.route('/index') def index(): - with open('config/presets.yaml') as f: - presets = yaml.load(f, Loader=yaml.FullLoader) + render_presets = yaml.load(f, Loader=yaml.FullLoader) return render_template('index.html', all_jobs=sorted_jobs(RenderQueue.all_jobs()), hostname=server.config['HOSTNAME'], renderer_info=renderer_info(), - render_clients=[server.config['HOSTNAME']], preset_list=presets) + render_clients=[server.config['HOSTNAME']], preset_list=render_presets) @server.get('/api/jobs') @@ -131,15 +130,15 @@ def job_thumbnail(job_id): # Misc status icons if found_job.status == RenderStatus.RUNNING: - return send_file('static/images/gears.png', mimetype="image/png") + return send_file('web/static/images/gears.png', mimetype="image/png") elif found_job.status == RenderStatus.CANCELLED: - return send_file('static/images/cancelled.png', mimetype="image/png") + return send_file('web/static/images/cancelled.png', mimetype="image/png") elif found_job.status == RenderStatus.SCHEDULED: - return send_file('static/images/scheduled.png', mimetype="image/png") + return send_file('web/static/images/scheduled.png', mimetype="image/png") elif found_job.status == RenderStatus.NOT_STARTED: - return send_file('static/images/not_started.png', mimetype="image/png") + return send_file('web/static/images/not_started.png', mimetype="image/png") # errors - return send_file('static/images/error.png', mimetype="image/png") + return send_file('web/static/images/error.png', mimetype="image/png") # Get job file routing @@ -396,9 +395,9 @@ def add_job_handler(): # create & configure jobs worker = RenderWorkerFactory.create_worker(renderer=job_data['renderer'], - input_path=loaded_project_local_path, - output_path=job_data["output_path"], - args=job_data.get('args', {})) + input_path=loaded_project_local_path, + output_path=job_data["output_path"], + args=job_data.get('args', {})) worker.status = job_data.get("initial_status", worker.status) worker.parent = job_data.get("parent", worker.parent) worker.name = job_data.get("name", worker.name) @@ -498,7 +497,6 @@ def clear_history(): @server.route('/api/status') def status(): - renderer_data = {} for render_class in RenderWorkerFactory.supported_classes(): if render_class.engine.renderer_path(): # only return renderers installed on host diff --git a/lib/client/__init__.py b/src/client/__init__.py similarity index 100% rename from lib/client/__init__.py rename to src/client/__init__.py diff --git a/lib/client/dashboard_window.py b/src/client/dashboard_window.py similarity index 97% rename from lib/client/dashboard_window.py rename to src/client/dashboard_window.py index 34a3268..79ff6d9 100644 --- a/lib/client/dashboard_window.py +++ b/src/client/dashboard_window.py @@ -1,17 +1,20 @@ import datetime import logging -import tkinter as tk +import os +import socket import threading import time -import socket -import os +import tkinter as tk from tkinter import ttk, messagebox, simpledialog + from PIL import Image, ImageTk -from lib.client.new_job_window import NewJobWindow -from lib.server.server_proxy import RenderServerProxy -from lib.server.zeroconf_server import ZeroconfServer -from lib.workers.base_worker import RenderStatus -from lib.utilities.misc_helper import launch_url, file_exists_in_mounts, get_time_elapsed + +from src.client.new_job_window import NewJobWindow +# from src.client.server_details import create_server_popup +from src.server_proxy import RenderServerProxy +from src.utilities.misc_helper import launch_url, file_exists_in_mounts, get_time_elapsed +from src.utilities.zeroconf_server import ZeroconfServer +from src.workers.base_worker import RenderStatus logger = logging.getLogger() @@ -31,7 +34,7 @@ def make_sortable(tree): class DashboardWindow: lib_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - image_path = os.path.join(lib_path, 'server', 'static', 'images') + image_path = os.path.join(lib_path, 'web', 'static', 'images') default_image = Image.open(os.path.join(image_path, 'desktop.png')) def __init__(self): diff --git a/lib/client/new_job_window.py b/src/client/new_job_window.py similarity index 99% rename from lib/client/new_job_window.py rename to src/client/new_job_window.py index 46c3c2c..19d80ac 100755 --- a/lib/client/new_job_window.py +++ b/src/client/new_job_window.py @@ -4,16 +4,16 @@ import logging import os.path import pathlib import socket +import threading from tkinter import * from tkinter import filedialog, messagebox from tkinter.ttk import Frame, Label, Entry, Combobox, Progressbar import psutil -import requests -import threading -from lib.workers.blender_worker import Blender -from lib.workers.ffmpeg_worker import FFMPEG -from lib.server.server_proxy import RenderServerProxy + +from src.server_proxy import RenderServerProxy +from src.workers.blender_worker import Blender +from src.workers.ffmpeg_worker import FFMPEG logger = logging.getLogger() diff --git a/lib/distributed_job_manager.py b/src/distributed_job_manager.py similarity index 98% rename from lib/distributed_job_manager.py rename to src/distributed_job_manager.py index cc13d68..c6300be 100644 --- a/lib/distributed_job_manager.py +++ b/src/distributed_job_manager.py @@ -5,11 +5,12 @@ import time import zipfile from pubsub import pub -from lib.render_queue import RenderQueue -from lib.server.server_proxy import RenderServerProxy -from lib.server.zeroconf_server import ZeroconfServer -from lib.utilities.misc_helper import get_file_size_human -from lib.workers.base_worker import RenderStatus, string_to_status + +from src.render_queue import RenderQueue +from src.server_proxy import RenderServerProxy +from src.utilities.misc_helper import get_file_size_human +from src.utilities.status_utils import RenderStatus, string_to_status +from src.utilities.zeroconf_server import ZeroconfServer logger = logging.getLogger() diff --git a/lib/render_queue.py b/src/render_queue.py similarity index 86% rename from lib/render_queue.py rename to src/render_queue.py index d710148..9e6d535 100755 --- a/lib/render_queue.py +++ b/src/render_queue.py @@ -4,8 +4,9 @@ from datetime import datetime from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from lib.workers.base_worker import RenderStatus, BaseRenderWorker, Base -from lib.workers.worker_factory import RenderWorkerFactory +from src.utilities.status_utils import RenderStatus +from src.worker_factory import RenderWorkerFactory +from src.workers.base_worker import Base logger = logging.getLogger() @@ -83,12 +84,20 @@ class RenderQueue: @classmethod def load_state(cls): + from src.workers.base_worker import BaseRenderWorker cls.job_queue = cls.session.query(BaseRenderWorker).all() @classmethod def save_state(cls): cls.session.commit() + @classmethod + def prepare_for_shutdown(cls): + running_jobs = cls.jobs_with_status(RenderStatus.RUNNING) # cancel all running jobs + for job in running_jobs: + cls.cancel_job(job) + cls.save_state() + @classmethod def is_available_for_job(cls, renderer, priority=2): if not RenderWorkerFactory.class_for_name(renderer).engine.renderer_path(): @@ -96,8 +105,9 @@ class RenderQueue: instances = cls.renderer_instances() higher_priority_jobs = [x for x in cls.running_jobs() if x.priority < priority] - max_renderers = renderer in instances.keys() and instances[renderer] >= cls.maximum_renderer_instances.get(renderer, 1) - return not max_renderers and not higher_priority_jobs + max_allowed_instances = cls.maximum_renderer_instances.get(renderer, 1) + maxed_out_instances = renderer in instances.keys() and instances[renderer] >= max_allowed_instances + return not maxed_out_instances and not higher_priority_jobs @classmethod def evaluate_queue(cls): diff --git a/lib/server/server_proxy.py b/src/server_proxy.py similarity index 96% rename from lib/server/server_proxy.py rename to src/server_proxy.py index aaff5c4..19c3cc4 100644 --- a/lib/server/server_proxy.py +++ b/src/server_proxy.py @@ -1,19 +1,21 @@ +import json import logging import os -import json -import requests import socket -import time import threading -from lib.workers.base_worker import RenderStatus +import time + +import requests from requests_toolbelt.multipart import MultipartEncoder, MultipartEncoderMonitor +from src.utilities.status_utils import RenderStatus + status_colors = {RenderStatus.ERROR: "red", RenderStatus.CANCELLED: 'orange1', RenderStatus.COMPLETED: 'green', RenderStatus.NOT_STARTED: "yellow", RenderStatus.SCHEDULED: 'purple', RenderStatus.RUNNING: 'cyan', RenderStatus.WAITING_FOR_SUBJOBS: 'blue'} -categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStatus.ERROR, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED, - RenderStatus.COMPLETED, RenderStatus.CANCELLED, RenderStatus.UNDEFINED] +categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStatus.ERROR, RenderStatus.NOT_STARTED, + RenderStatus.SCHEDULED, RenderStatus.COMPLETED, RenderStatus.CANCELLED, RenderStatus.UNDEFINED] logger = logging.getLogger() OFFLINE_MAX = 2 diff --git a/lib/engines/__init__.py b/src/utilities/__init__.py similarity index 100% rename from lib/engines/__init__.py rename to src/utilities/__init__.py diff --git a/lib/utilities/ffmpeg_helper.py b/src/utilities/ffmpeg_helper.py similarity index 95% rename from lib/utilities/ffmpeg_helper.py rename to src/utilities/ffmpeg_helper.py index da6538c..96bc2df 100644 --- a/lib/utilities/ffmpeg_helper.py +++ b/src/utilities/ffmpeg_helper.py @@ -1,5 +1,5 @@ import subprocess -from lib.engines.ffmpeg_engine import FFMPEG +from src.workers.engines.ffmpeg_engine import FFMPEG def image_sequence_to_video(source_glob_pattern, output_path, framerate=24, encoder="prores_ks", profile=4, diff --git a/lib/utilities/misc_helper.py b/src/utilities/misc_helper.py similarity index 100% rename from lib/utilities/misc_helper.py rename to src/utilities/misc_helper.py diff --git a/lib/utilities/server_helper.py b/src/utilities/server_helper.py similarity index 89% rename from lib/utilities/server_helper.py rename to src/utilities/server_helper.py index 7c8bb33..bb37bd7 100644 --- a/lib/utilities/server_helper.py +++ b/src/utilities/server_helper.py @@ -1,13 +1,9 @@ import logging import os -import socket import subprocess import threading -import psutil - -from lib.server.server_proxy import RenderServerProxy -from lib.utilities.ffmpeg_helper import generate_thumbnail, save_first_frame +from src.utilities.ffmpeg_helper import generate_thumbnail, save_first_frame logger = logging.getLogger() @@ -21,8 +17,8 @@ def generate_thumbnail_for_job(job, thumb_video_path, thumb_image_path, max_widt try: logger.debug(f"Generating video thumbnail for {source}") generate_thumbnail(source_path=source, dest_path=thumb_video_path, max_width=max_width) - except subprocess.CalledProcessError as e: - logger.error(f"Error generating video thumbnail for {source}: {e}") + except subprocess.CalledProcessError as err: + logger.error(f"Error generating video thumbnail for {source}: {err}") try: os.remove(in_progress_path) diff --git a/src/utilities/status_utils.py b/src/utilities/status_utils.py new file mode 100644 index 0000000..891c942 --- /dev/null +++ b/src/utilities/status_utils.py @@ -0,0 +1,20 @@ +from enum import Enum + + +class RenderStatus(Enum): + NOT_STARTED = "not_started" + RUNNING = "running" + COMPLETED = "completed" + CANCELLED = "cancelled" + ERROR = "error" + SCHEDULED = "scheduled" + WAITING_FOR_SUBJOBS = "waiting_for_subjobs" + CONFIGURING = "configuring" + UNDEFINED = "undefined" + + +def string_to_status(string): + for stat in RenderStatus: + if stat.value == string: + return stat + return RenderStatus.UNDEFINED diff --git a/lib/server/zeroconf_server.py b/src/utilities/zeroconf_server.py similarity index 98% rename from lib/server/zeroconf_server.py rename to src/utilities/zeroconf_server.py index 93014a4..a3f74ac 100644 --- a/lib/server/zeroconf_server.py +++ b/src/utilities/zeroconf_server.py @@ -59,6 +59,7 @@ class ZeroconfServer: @classmethod def _browse_services(cls): browser = ServiceBrowser(cls.zeroconf, cls.service_type, [cls._on_service_discovered]) + browser.is_alive() @classmethod def _on_service_discovered(cls, zeroconf, service_type, name, state_change): diff --git a/lib/server/static/images/cancelled.png b/src/web/static/images/cancelled.png similarity index 100% rename from lib/server/static/images/cancelled.png rename to src/web/static/images/cancelled.png diff --git a/lib/server/static/images/desktop.png b/src/web/static/images/desktop.png similarity index 100% rename from lib/server/static/images/desktop.png rename to src/web/static/images/desktop.png diff --git a/lib/server/static/images/error.png b/src/web/static/images/error.png similarity index 100% rename from lib/server/static/images/error.png rename to src/web/static/images/error.png diff --git a/lib/server/static/images/gears.png b/src/web/static/images/gears.png similarity index 100% rename from lib/server/static/images/gears.png rename to src/web/static/images/gears.png diff --git a/lib/server/static/images/logo.png b/src/web/static/images/logo.png similarity index 100% rename from lib/server/static/images/logo.png rename to src/web/static/images/logo.png diff --git a/lib/server/static/images/not_started.png b/src/web/static/images/not_started.png similarity index 100% rename from lib/server/static/images/not_started.png rename to src/web/static/images/not_started.png diff --git a/lib/server/static/images/scheduled.png b/src/web/static/images/scheduled.png similarity index 100% rename from lib/server/static/images/scheduled.png rename to src/web/static/images/scheduled.png diff --git a/lib/server/static/images/spinner.gif b/src/web/static/images/spinner.gif similarity index 100% rename from lib/server/static/images/spinner.gif rename to src/web/static/images/spinner.gif diff --git a/lib/server/static/js/job_table.js b/src/web/static/js/job_table.js similarity index 100% rename from lib/server/static/js/job_table.js rename to src/web/static/js/job_table.js diff --git a/lib/server/static/js/modals.js b/src/web/static/js/modals.js similarity index 100% rename from lib/server/static/js/modals.js rename to src/web/static/js/modals.js diff --git a/lib/server/templates/details.html b/src/web/templates/details.html similarity index 100% rename from lib/server/templates/details.html rename to src/web/templates/details.html diff --git a/lib/server/templates/index.html b/src/web/templates/index.html similarity index 100% rename from lib/server/templates/index.html rename to src/web/templates/index.html diff --git a/lib/server/templates/layout.html b/src/web/templates/layout.html similarity index 100% rename from lib/server/templates/layout.html rename to src/web/templates/layout.html diff --git a/lib/server/templates/upload.html b/src/web/templates/upload.html similarity index 100% rename from lib/server/templates/upload.html rename to src/web/templates/upload.html diff --git a/lib/workers/worker_factory.py b/src/worker_factory.py similarity index 83% rename from lib/workers/worker_factory.py rename to src/worker_factory.py index d4a95f7..fc14b25 100644 --- a/lib/workers/worker_factory.py +++ b/src/worker_factory.py @@ -3,9 +3,9 @@ class RenderWorkerFactory: @staticmethod def supported_classes(): # to add support for any additional RenderWorker classes, import their classes and add to list here - from .blender_worker import BlenderRenderWorker - from .aerender_worker import AERenderWorker - from .ffmpeg_worker import FFMPEGRenderWorker + from src.workers.blender_worker import BlenderRenderWorker + from src.workers.aerender_worker import AERenderWorker + from src.workers.ffmpeg_worker import FFMPEGRenderWorker classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker] return classes diff --git a/lib/server/__init__.py b/src/workers/__init__.py similarity index 100% rename from lib/server/__init__.py rename to src/workers/__init__.py diff --git a/lib/workers/aerender_worker.py b/src/workers/aerender_worker.py similarity index 96% rename from lib/workers/aerender_worker.py rename to src/workers/aerender_worker.py index 4079152..f475bb7 100644 --- a/lib/workers/aerender_worker.py +++ b/src/workers/aerender_worker.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 import glob import json +import logging +import os import re import time -from .base_worker import * -from ..engines.aerender_engine import AERender +from src.workers.base_worker import BaseRenderWorker, timecode_to_frames +from src.workers.engines.aerender_engine import AERender + def aerender_path(): paths = glob.glob('/Applications/*After Effects*/aerender') diff --git a/lib/workers/base_worker.py b/src/workers/base_worker.py similarity index 92% rename from lib/workers/base_worker.py rename to src/workers/base_worker.py index 5eb73f0..84b55a5 100644 --- a/lib/workers/base_worker.py +++ b/src/workers/base_worker.py @@ -6,38 +6,19 @@ import os import subprocess import threading from datetime import datetime -from enum import Enum import psutil from pubsub import pub from sqlalchemy import Column, Integer, String, DateTime, JSON from sqlalchemy.ext.declarative import declarative_base -from lib.utilities.misc_helper import get_time_elapsed +from src.utilities.misc_helper import get_time_elapsed +from src.utilities.status_utils import RenderStatus, string_to_status logger = logging.getLogger() Base = declarative_base() -class RenderStatus(Enum): - NOT_STARTED = "not_started" - RUNNING = "running" - COMPLETED = "completed" - CANCELLED = "cancelled" - ERROR = "error" - SCHEDULED = "scheduled" - WAITING_FOR_SUBJOBS = "waiting_for_subjobs" - CONFIGURING = "configuring" - UNDEFINED = "undefined" - - -def string_to_status(string): - for stat in RenderStatus: - if stat.value == string: - return stat - return RenderStatus.UNDEFINED - - class BaseRenderWorker(Base): __tablename__ = 'render_workers' @@ -121,7 +102,7 @@ class BaseRenderWorker(Base): @status.setter def status(self, new_status): if self._status != new_status.value: - old_status = self._status + old_status = self.status self._status = new_status.value pub.sendMessage('status_change', job_id=self.id, old_status=old_status, new_status=new_status) @@ -186,7 +167,8 @@ class BaseRenderWorker(Base): self.status = RenderStatus.RUNNING self.start_time = datetime.now() - logger.info(f'Starting {self.engine.name()} {self.engine.version()} Render for {self.input_path} | Frame Count: {self.total_frames}') + logger.info(f'Starting {self.engine.name()} {self.engine.version()} Render for {self.input_path} | ' + f'Frame Count: {self.total_frames}') self.__thread.start() def run(self): @@ -207,7 +189,7 @@ class BaseRenderWorker(Base): f.write(f"Running command: {' '.join(subprocess_cmds)}\n") for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"): # or another encoding f.write(c) - logger.debug(f"{self.engine.name()}Worker: {c.strip()}") + # logger.debug(f"{self.engine.name()}Worker: {c.strip()}") self.last_output = c.strip() self._parse_stdout(c.strip()) f.write('\n') @@ -220,7 +202,7 @@ class BaseRenderWorker(Base): return if return_code: - message = f"{self.engine.name()} render failed with return_code {return_code} after {self.time_elapsed()}" + message = f"{self.engine.name()} render failed with code {return_code} after {self.time_elapsed()}" logger.error(message) f.write(message) self.status = RenderStatus.ERROR @@ -233,7 +215,7 @@ class BaseRenderWorker(Base): f.write(message) if self.children: - from lib.distributed_job_manager import DistributedJobManager + from src.distributed_job_manager import DistributedJobManager DistributedJobManager.wait_for_subjobs(local_job=self) # Post Render Work diff --git a/lib/workers/blender_worker.py b/src/workers/blender_worker.py similarity index 85% rename from lib/workers/blender_worker.py rename to src/workers/blender_worker.py index a6771f2..c45fa11 100644 --- a/lib/workers/blender_worker.py +++ b/src/workers/blender_worker.py @@ -2,9 +2,9 @@ import re from collections import Counter -from lib.engines.blender_engine import Blender -from lib.utilities.ffmpeg_helper import image_sequence_to_video -from lib.workers.base_worker import * +from src.workers.engines.blender_engine import Blender +from src.utilities.ffmpeg_helper import image_sequence_to_video +from src.workers.base_worker import * class BlenderRenderWorker(BaseRenderWorker): @@ -69,17 +69,6 @@ class BlenderRenderWorker(BaseRenderWorker): samples = re.sub(r'[^\d/]', '', sample_string) self.__frame_percent_complete = int(samples.split('/')[0]) / int(samples.split('/')[-1]) - # Calculate rough percent based on cycles - # EEVEE - # 10-Apr-22 22:42:06 - RENDERER: Fra:0 Mem:857.99M (Peak 928.55M) | Time:00:03.96 | Rendering 1 / 65 samples - # 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:07.92 | Rendering 26 / 64 samples - # 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:08.17 | Rendering 51 / 64 samples - # 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:08.31 | Rendering 64 / 64 samples - - # CYCLES - # 10-Apr-22 22:43:22 - RENDERER: Fra:0 Mem:836.30M (Peak 1726.13M) | Time:00:01.56 | Remaining:00:30.65 | Mem:588.68M, Peak:588.68M | Scene, View Layer | Sample 1/150 - # 10-Apr-22 22:43:43 - RENDERER: Fra:0 Mem:836.30M (Peak 1726.13M) | Time:00:22.01 | Remaining:00:03.36 | Mem:588.68M, Peak:588.68M | Scene, View Layer | Sample 129/150 - if int(stats['frame']) > self.current_frame: self.current_frame = int(stats['frame']) logger.debug( diff --git a/lib/utilities/__init__.py b/src/workers/engines/__init__.py similarity index 100% rename from lib/utilities/__init__.py rename to src/workers/engines/__init__.py diff --git a/lib/engines/aerender_engine.py b/src/workers/engines/aerender_engine.py similarity index 100% rename from lib/engines/aerender_engine.py rename to src/workers/engines/aerender_engine.py diff --git a/lib/engines/base_engine.py b/src/workers/engines/base_engine.py similarity index 100% rename from lib/engines/base_engine.py rename to src/workers/engines/base_engine.py diff --git a/lib/engines/blender_engine.py b/src/workers/engines/blender_engine.py similarity index 100% rename from lib/engines/blender_engine.py rename to src/workers/engines/blender_engine.py diff --git a/lib/engines/ffmpeg_engine.py b/src/workers/engines/ffmpeg_engine.py similarity index 100% rename from lib/engines/ffmpeg_engine.py rename to src/workers/engines/ffmpeg_engine.py diff --git a/lib/engines/scripts/blender/get_file_info.py b/src/workers/engines/scripts/blender/get_file_info.py similarity index 100% rename from lib/engines/scripts/blender/get_file_info.py rename to src/workers/engines/scripts/blender/get_file_info.py diff --git a/lib/engines/scripts/blender/pack_project.py b/src/workers/engines/scripts/blender/pack_project.py similarity index 100% rename from lib/engines/scripts/blender/pack_project.py rename to src/workers/engines/scripts/blender/pack_project.py diff --git a/lib/workers/ffmpeg_worker.py b/src/workers/ffmpeg_worker.py similarity index 76% rename from lib/workers/ffmpeg_worker.py rename to src/workers/ffmpeg_worker.py index 4522dc3..2287ee1 100644 --- a/lib/workers/ffmpeg_worker.py +++ b/src/workers/ffmpeg_worker.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import re -from .base_worker import * -from ..engines.ffmpeg_engine import FFMPEG +import subprocess + +from src.workers.base_worker import BaseRenderWorker +from src.workers.engines.ffmpeg_engine import FFMPEG class FFMPEGRenderWorker(BaseRenderWorker): @@ -48,18 +50,3 @@ class FFMPEGRenderWorker(BaseRenderWorker): time_elapsed = stats['time_elapsed'] elif "not found" in line: self.log_error(line) - - -if __name__ == '__main__': - - print(FFMPEG.full_report()) - - # logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG) - # - # test_movie = '/Users/brettwilliams/Desktop/dark_knight_rises.mp4' - # - # r = FFMPEGRenderWorker(test_movie, '/Users/brettwilliams/Desktop/test-ffmpeg.mp4', args=['-c:v', 'libx265', '-vtag', 'hvc1']) - # # r = FFMPEGRenderer(test_movie, '/Users/brettwilliams/Desktop/dark_knight_rises-output.mp4') - # r.start() - # while r.is_running(): - # time.sleep(1) diff --git a/start_client.py b/start_client.py index 544ea85..0ad54ca 100644 --- a/start_client.py +++ b/start_client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from lib.client.dashboard_window import start_client +from src.client.dashboard_window import start_client if __name__ == '__main__': start_client() diff --git a/start_server.py b/start_server.py index 4f9e9ba..9935eb7 100755 --- a/start_server.py +++ b/start_server.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from lib.server.api_server import start_server +from src.api_server import start_server if __name__ == '__main__': start_server()