diff --git a/src/api/api_server.py b/src/api/api_server.py index 388ca2a..8786b77 100755 --- a/src/api/api_server.py +++ b/src/api/api_server.py @@ -12,24 +12,25 @@ import threading import time import zipfile from datetime import datetime -from urllib.request import urlretrieve from zipfile import ZipFile import json2html import psutil +import requests import yaml from flask import Flask, request, render_template, send_file, after_this_request, Response, redirect, url_for, abort +from tqdm import tqdm from werkzeug.utils import secure_filename +from src.api.server_proxy import RenderServerProxy from src.distributed_job_manager import DistributedJobManager +from src.engines.core.base_worker import string_to_status, RenderStatus +from src.engines.core.worker_factory import RenderWorkerFactory from src.engines.engine_manager import EngineManager from src.render_queue import RenderQueue, JobNotFoundError -from src.api.server_proxy import RenderServerProxy +from src.utilities.misc_helper import system_safe_path from src.utilities.server_helper import generate_thumbnail_for_job from src.utilities.zeroconf_server import ZeroconfServer -from src.utilities.misc_helper import system_safe_path -from src.engines.core.worker_factory import RenderWorkerFactory -from src.engines.core.base_worker import string_to_status, RenderStatus logger = logging.getLogger() server = Flask(__name__, template_folder='web/templates', static_folder='web/static') @@ -133,15 +134,15 @@ def job_thumbnail(job_id): # Misc status icons if found_job.status == RenderStatus.RUNNING: - return send_file('web/static/images/gears.png', mimetype="image/png") + return send_file('../web/static/images/gears.png', mimetype="image/png") elif found_job.status == RenderStatus.CANCELLED: - return send_file('web/static/images/cancelled.png', mimetype="image/png") + return send_file('../web/static/images/cancelled.png', mimetype="image/png") elif found_job.status == RenderStatus.SCHEDULED: - return send_file('web/static/images/scheduled.png', mimetype="image/png") + return send_file('../web/static/images/scheduled.png', mimetype="image/png") elif found_job.status == RenderStatus.NOT_STARTED: - return send_file('web/static/images/not_started.png', mimetype="image/png") + return send_file('../web/static/images/not_started.png', mimetype="image/png") # errors - return send_file('web/static/images/error.png', mimetype="image/png") + return send_file('../web/static/images/error.png', mimetype="image/png") # Get job file routing @@ -190,7 +191,7 @@ def get_job_status(job_id): @server.get('/api/job//logs') def get_job_logs(job_id): found_job = RenderQueue.job_with_id(job_id) - log_path = system_safe_path(found_job.log_path()), + log_path = system_safe_path(found_job.log_path()) log_data = None if log_path and os.path.exists(log_path): with open(log_path) as file: @@ -322,10 +323,30 @@ def add_job_handler(): referred_name = os.path.basename(uploaded_project.filename) elif project_url: # download and save url - have to download first to know filename due to redirects - logger.info(f"Attempting to download URL: {project_url}") + logger.info(f"Downloading project from url: {project_url}") try: - downloaded_file_url, info = urlretrieve(project_url) - referred_name = info.get_filename() or os.path.basename(project_url) + referred_name = os.path.basename(project_url) + response = requests.get(project_url, stream=True) + if response.status_code == 200: + # Get the total file size from the "Content-Length" header + file_size = int(response.headers.get("Content-Length", 0)) + + # Create a progress bar using tqdm + progress_bar = tqdm(total=file_size, unit="B", unit_scale=True) + + # Open a file for writing in binary mode + downloaded_file_url = os.path.join(tempfile.gettempdir(), referred_name) + with open(downloaded_file_url, "wb") as file: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + # Write the chunk to the file + file.write(chunk) + # Update the progress bar + progress_bar.update(len(chunk)) + + # Close the progress bar + progress_bar.close() + except Exception as e: err_msg = f"Error downloading file: {e}" logger.error(err_msg) @@ -419,6 +440,10 @@ def add_job_handler(): if not worker.parent: make_job_ready(worker.id) results.append(worker.json()) + except FileNotFoundError as e: + err_msg = f"Cannot create job: {e}" + logger.error(err_msg) + results.append({'error': err_msg}) except Exception as e: err_msg = f"Exception creating render job: {e}" logger.exception(err_msg) @@ -549,6 +574,24 @@ def renderer_info(): 'supported_export_formats': engine(install_path).get_output_formats()} return renderer_data +@server.get('/api/is_engine_available_to_download') +def is_engine_available_to_download(): + available_result = EngineManager.version_is_available_to_download(request.args.get('engine'), + request.args.get('version'), + request.args.get('system_os'), + request.args.get('cpu')) + return available_result if available_result else \ + (f"Cannot find available download for {request.args.get('engine')} {request.args.get('version')}", 500) + + +@server.get('/api/find_most_recent_version') +def find_most_recent_version(): + most_recent = EngineManager.find_most_recent_version(request.args.get('engine'), + request.args.get('system_os'), + request.args.get('cpu')) + return most_recent if most_recent else \ + (f"Error finding most recent version of {request.args.get('engine')}", 500) + @server.post('/api/download_engine') def download_engine(): @@ -556,7 +599,8 @@ def download_engine(): request.args.get('version'), request.args.get('system_os'), request.args.get('cpu')) - return download_result if download_result else ("Error downloading requested engine", 500) + return download_result if download_result else \ + (f"Error downloading {request.args.get('engine')} {request.args.get('version')}", 500) @server.post('/api/delete_engine') @@ -565,7 +609,8 @@ def delete_engine_download(): request.args.get('version'), request.args.get('system_os'), request.args.get('cpu')) - return "Success" if delete_result else ("Error deleting requested engine", 500) + return "Success" if delete_result else \ + (f"Error deleting {request.args.get('engine')} {request.args.get('version')}", 500) @server.get('/api/renderer//args') diff --git a/src/client/dashboard_window.py b/src/client/dashboard_window.py index 79ff6d9..429c0af 100644 --- a/src/client/dashboard_window.py +++ b/src/client/dashboard_window.py @@ -11,10 +11,10 @@ from PIL import Image, ImageTk from src.client.new_job_window import NewJobWindow # from src.client.server_details import create_server_popup -from src.server_proxy import RenderServerProxy +from src.api.server_proxy import RenderServerProxy from src.utilities.misc_helper import launch_url, file_exists_in_mounts, get_time_elapsed from src.utilities.zeroconf_server import ZeroconfServer -from src.workers.base_worker import RenderStatus +from src.engines.core.base_worker import RenderStatus logger = logging.getLogger() diff --git a/src/client/new_job_window.py b/src/client/new_job_window.py index 27c5c18..a1aabb3 100755 --- a/src/client/new_job_window.py +++ b/src/client/new_job_window.py @@ -11,9 +11,9 @@ from tkinter.ttk import Frame, Label, Entry, Combobox, Progressbar import psutil -from src.server_proxy import RenderServerProxy -from src.workers.blender_worker import Blender -from src.workers.ffmpeg_worker import FFMPEG +from src.api.server_proxy import RenderServerProxy +from src.engines.blender.blender_worker import Blender +from src.engines.ffmpeg.ffmpeg_worker import FFMPEG logger = logging.getLogger() diff --git a/src/engines/blender/blender_downloader.py b/src/engines/blender/blender_downloader.py index 3605ff5..b777b6f 100644 --- a/src/engines/blender/blender_downloader.py +++ b/src/engines/blender/blender_downloader.py @@ -4,7 +4,8 @@ import re import requests -from ..core.downloader_core import download_and_extract_app +from src.engines.core.downloader_core import download_and_extract_app +from src.utilities.misc_helper import current_system_os, current_system_cpu # url = "https://download.blender.org/release/" url = "https://ftp.nluug.nl/pub/graphics/blender/release/" # much faster mirror for testing @@ -35,27 +36,42 @@ class BlenderDownloader: @staticmethod def get_minor_versions(major_version, system_os=None, cpu=None): - base_url = url + 'Blender' + major_version + try: + base_url = url + 'Blender' + major_version + response = requests.get(base_url) + response.raise_for_status() - response = requests.get(base_url) - response.raise_for_status() + versions_pattern = r'blender-(?P[\d\.]+)-(?P\w+)-(?P\w+).*' + versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)] - versions_pattern = r'blender-(?P[\d\.]+)-(?P\w+)-(?P\w+).*' - versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)] + # Filter to just the supported formats + versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)] - # Filter to just the supported formats - versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)] - - if system_os: + # Filter down OS and CPU + system_os = system_os or current_system_os() + cpu = cpu or current_system_cpu() versions_data = [x for x in versions_data if x['system_os'] == system_os] - if cpu: versions_data = [x for x in versions_data if x['cpu'] == cpu] - for v in versions_data: - v['url'] = base_url + '/' + v['file'] + for v in versions_data: + v['url'] = base_url + '/' + v['file'] - versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True) - return versions_data + versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True) + return versions_data + except requests.exceptions.HTTPError as e: + logger.error(f"Invalid url: {e}") + except Exception as e: + logger.exception(e) + return [] + + @classmethod + def version_is_available_to_download(cls, version, system_os=None, cpu=None): + requested_major_version = '.'.join(version.split('.')[:2]) + minor_versions = cls.get_minor_versions(requested_major_version, system_os, cpu) + for minor in minor_versions: + if minor['version'] == version: + return minor + return None @staticmethod def find_LTS_versions(): @@ -79,7 +95,7 @@ class BlenderDownloader: logger.error("Cannot find a most recent version") @classmethod - def download_engine(cls, version, download_location, system_os=None, cpu=None): + def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120): system_os = system_os or platform.system().lower().replace('darwin', 'macos') cpu = cpu or platform.machine().lower().replace('amd64', 'x64') @@ -89,7 +105,8 @@ class BlenderDownloader: minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version] # we get the URL instead of calculating it ourselves. May change this - download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location) + download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location, + timeout=timeout) except IndexError: logger.error("Cannot find requested engine") diff --git a/src/engines/core/downloader_core.py b/src/engines/core/downloader_core.py index 65d8abc..c0cb6e4 100644 --- a/src/engines/core/downloader_core.py +++ b/src/engines/core/downloader_core.py @@ -12,7 +12,7 @@ supported_formats = ['.zip', '.tar.xz', '.dmg'] logger = logging.getLogger() -def download_and_extract_app(remote_url, download_location): +def download_and_extract_app(remote_url, download_location, timeout=120): # Create a temp download directory temp_download_dir = tempfile.mkdtemp() @@ -30,7 +30,7 @@ def download_and_extract_app(remote_url, download_location): if not os.path.exists(temp_downloaded_file_path): # Make a GET request to the URL with stream=True to enable streaming logger.info(f"Downloading {output_dir_name} from {remote_url}") - response = requests.get(remote_url, stream=True) + response = requests.get(remote_url, stream=True, timeout=timeout) # Check if the request was successful if response.status_code == 200: @@ -54,6 +54,7 @@ def download_and_extract_app(remote_url, download_location): logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}") else: logger.error(f"Failed to download the file. Status code: {response.status_code}") + return os.makedirs(download_location, exist_ok=True) diff --git a/src/engines/core/worker_factory.py b/src/engines/core/worker_factory.py index fd549c5..1c3ac2d 100644 --- a/src/engines/core/worker_factory.py +++ b/src/engines/core/worker_factory.py @@ -1,4 +1,5 @@ import logging + from src.engines.engine_manager import EngineManager logger = logging.getLogger() @@ -20,19 +21,29 @@ class RenderWorkerFactory: worker_class = RenderWorkerFactory.class_for_name(renderer) - # find correct engine version + # check to make sure we have versions installed all_versions = EngineManager.all_versions_for_engine(renderer) if not all_versions: raise FileNotFoundError(f"Cannot find any installed {renderer} engines") - engine_path = all_versions[0]['path'] + # Find the path to the requested engine version or use default + engine_path = None if engine_version else all_versions[0]['path'] if engine_version: for ver in all_versions: if ver['version'] == engine_version: engine_path = ver['path'] break + + # Download the required engine if not found locally if not engine_path: - logger.warning(f"Cannot find requested engine version {engine_version}. Using default version {all_versions[0]['version']}") + download_result = EngineManager.download_engine(renderer, engine_version) + if not download_result: + raise FileNotFoundError(f"Cannot download requested version: {renderer} {engine_version}") + engine_path = download_result['path'] + logger.info("Engine downloaded. Creating worker.") + + if not engine_path: + raise FileNotFoundError(f"Cannot find requested engine version {engine_version}") return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args, parent=parent, name=name) diff --git a/src/engines/engine_manager.py b/src/engines/engine_manager.py index 1be5cb1..327b181 100644 --- a/src/engines/engine_manager.py +++ b/src/engines/engine_manager.py @@ -15,6 +15,11 @@ logger = logging.getLogger() class EngineManager: engines_path = "~/zordon-uploads/engines" + downloader_classes = { + "blender": BlenderDownloader, + "ffmpeg": FFMPEGDownloader, + # Add more engine types and corresponding downloader classes as needed + } @classmethod def supported_engines(cls): @@ -99,6 +104,20 @@ class EngineManager: def system_cpu(): return platform.machine().lower().replace('amd64', 'x64') + @classmethod + def version_is_available_to_download(cls, engine, version, system_os=None, cpu=None): + try: + return cls.downloader_classes[engine].version_is_available_to_download(version, system_os, cpu) + except Exception as e: + return None + + @classmethod + def find_most_recent_version(cls, engine, system_os, cpu, lts_only=False): + try: + return cls.downloader_classes[engine].find_most_recent_version(system_os, cpu) + except Exception as e: + return None + @classmethod def download_engine(cls, engine, version, system_os=None, cpu=None): existing_download = cls.has_engine_version(engine, version, system_os, cpu) @@ -107,20 +126,15 @@ class EngineManager: return existing_download logger.info(f"Requesting download of {engine} {version}") - downloader_classes = { - "blender": BlenderDownloader, - "ffmpeg": FFMPEGDownloader, - # Add more engine types and corresponding downloader classes as needed - } # Check if the provided engine type is valid - if engine not in downloader_classes: + if engine not in cls.downloader_classes: logger.error("No valid engine found") return # Get the appropriate downloader class based on the engine type - downloader_classes[engine].download_engine(version, download_location=cls.engines_path, - system_os=system_os, cpu=cpu) + cls.downloader_classes[engine].download_engine(version, download_location=cls.engines_path, + system_os=system_os, cpu=cpu, timeout=300) # Check that engine was properly downloaded found_engine = cls.has_engine_version(engine, version, system_os, cpu) diff --git a/src/engines/ffmpeg/ffmpeg_downloader.py b/src/engines/ffmpeg/ffmpeg_downloader.py index 989576d..b988d9c 100644 --- a/src/engines/ffmpeg/ffmpeg_downloader.py +++ b/src/engines/ffmpeg/ffmpeg_downloader.py @@ -1,11 +1,11 @@ import logging import os -import platform import re import requests from src.engines.core.downloader_core import download_and_extract_app +from src.utilities.misc_helper import current_system_cpu, current_system_os logger = logging.getLogger() supported_formats = ['.zip', '.tar.xz', '.dmg'] @@ -24,7 +24,7 @@ class FFMPEGDownloader: windows_api_url = "https://api.github.com/repos/GyanD/codexffmpeg/releases" @classmethod - def get_macos_versions(cls): + def __get_macos_versions(cls): response = requests.get(cls.macos_url) response.raise_for_status() @@ -34,7 +34,7 @@ class FFMPEGDownloader: return [link.split('-')[-1].split('.zip')[0] for link in link_matches] @classmethod - def get_linux_versions(cls): + def __get_linux_versions(cls): # Link 1 / 2 - Current Version response = requests.get(cls.linux_url) @@ -50,7 +50,7 @@ class FFMPEGDownloader: return releases @classmethod - def get_windows_versions(cls): + def __get_windows_versions(cls): response = requests.get(cls.windows_api_url) response.raise_for_status() @@ -63,36 +63,65 @@ class FFMPEGDownloader: @classmethod def find_most_recent_version(cls, system_os, cpu, lts_only=False): - pass + return cls.all_versions(system_os, cpu)[0] @classmethod - def download_engine(cls, version, download_location, system_os=None, cpu=None): - system_os = system_os or platform.system().lower().replace('darwin', 'macos') - cpu = cpu or platform.machine().lower().replace('amd64', 'x64') - - # Verify requested version is available - remote_url = None - versions_per_os = {'linux': cls.get_linux_versions, 'macos': cls.get_macos_versions, 'windows': cls.get_windows_versions} + def all_versions(cls, system_os=None, cpu=None): + system_os = system_os or current_system_os() + cpu = cpu or current_system_cpu() + versions_per_os = {'linux': cls.__get_linux_versions, 'macos': cls.__get_macos_versions, 'windows': cls.__get_windows_versions} if not versions_per_os.get(system_os): logger.error(f"Cannot find version list for {system_os}") return - if version not in versions_per_os[system_os](): - logger.error(f"Cannot find FFMPEG version {version} for {system_os}") + results = [] + all_versions = versions_per_os[system_os]() + for version in all_versions: + remote_url = cls.__get_remote_url_for_version(version, system_os, cpu) + results.append({'cpu': cpu, 'file': os.path.basename(remote_url), 'system_os': system_os, 'url': remote_url, + 'version': version}) + return results + + @classmethod + def version_is_available_to_download(cls, version, system_os=None, cpu=None): + for ver in cls.all_versions(system_os, cpu): + if ver['version'] == version: + return ver + return None + + @classmethod + def __get_remote_url_for_version(cls, version, system_os, cpu): # Platform specific naming cleanup + remote_url = None if system_os == 'macos': remote_url = os.path.join(cls.macos_url, f"ffmpeg-{version}.zip") - download_location = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}') # override location to match linux elif system_os == 'linux': - release_dir = 'releases' if version == cls.get_linux_versions()[0] else 'old-releases' + release_dir = 'releases' if version == cls.__get_linux_versions()[0] else 'old-releases' remote_url = os.path.join(cls.linux_url, release_dir, f'ffmpeg-{version}-{cpu}-static.tar.xz') elif system_os == 'windows': remote_url = f"{cls.windows_download_url.strip('/')}/{version}/ffmpeg-{version}-full_build.zip" + else: + logger.error("Unknown system os") + return remote_url + + @classmethod + def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120): + system_os = system_os or current_system_os() + cpu = cpu or current_system_os() + + # Verify requested version is available + if version not in cls.all_versions(system_os): + logger.error(f"Cannot find FFMPEG version {version} for {system_os}") + + # Platform specific naming cleanup + remote_url = cls.__get_remote_url_for_version(version, system_os, cpu) + if system_os == 'macos': # override location to match linux + download_location = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}') # Download and extract try: logger.info(f"Requesting download of ffmpeg-{version}-{system_os}-{cpu}") - download_and_extract_app(remote_url=remote_url, download_location=download_location) + download_and_extract_app(remote_url=remote_url, download_location=download_location, timeout=timeout) # naming cleanup to match existing naming convention if system_os == 'linux': diff --git a/src/utilities/misc_helper.py b/src/utilities/misc_helper.py index 0362e31..edda562 100644 --- a/src/utilities/misc_helper.py +++ b/src/utilities/misc_helper.py @@ -110,3 +110,11 @@ def system_safe_path(path): if platform.system().lower() == "windows": return os.path.normpath(path) return path.replace("\\", "/") + + +def current_system_os(): + return platform.system().lower().replace('darwin', 'macos') + + +def current_system_cpu(): + return platform.machine().lower().replace('amd64', 'x64')