Engine downloader API for #31 (#42)

* Add is_engine_available_to_download API call

* Fix issue with worker never throwing error if engine is not found

* Add API call to get most recent engine version

* Fix some minor import issues

* Fix web urls

* Fix default server log level

* Add progress bar for project download worker_factory downloads missing engine versions

* Better error handling when invalid version is given

* Add timeouts to engine downloaders

src/api/api_server.py

@@ -12,24 +12,25 @@ import threading
 import time
 import zipfile
 from datetime import datetime
-from urllib.request import urlretrieve
 from zipfile import ZipFile
 
 import json2html
 import psutil
+import requests
 import yaml
 from flask import Flask, request, render_template, send_file, after_this_request, Response, redirect, url_for, abort
+from tqdm import tqdm
 from werkzeug.utils import secure_filename
 
+from src.api.server_proxy import RenderServerProxy
 from src.distributed_job_manager import DistributedJobManager
+from src.engines.core.base_worker import string_to_status, RenderStatus
+from src.engines.core.worker_factory import RenderWorkerFactory
 from src.engines.engine_manager import EngineManager
 from src.render_queue import RenderQueue, JobNotFoundError
-from src.api.server_proxy import RenderServerProxy
+from src.utilities.misc_helper import system_safe_path
 from src.utilities.server_helper import generate_thumbnail_for_job
 from src.utilities.zeroconf_server import ZeroconfServer
-from src.utilities.misc_helper import system_safe_path
-from src.engines.core.worker_factory import RenderWorkerFactory
-from src.engines.core.base_worker import string_to_status, RenderStatus
 
 logger = logging.getLogger()
 server = Flask(__name__, template_folder='web/templates', static_folder='web/static')
@@ -133,15 +134,15 @@ def job_thumbnail(job_id):
 
         # Misc status icons
         if found_job.status == RenderStatus.RUNNING:
-            return send_file('web/static/images/gears.png', mimetype="image/png")
+            return send_file('../web/static/images/gears.png', mimetype="image/png")
         elif found_job.status == RenderStatus.CANCELLED:
-            return send_file('web/static/images/cancelled.png', mimetype="image/png")
+            return send_file('../web/static/images/cancelled.png', mimetype="image/png")
         elif found_job.status == RenderStatus.SCHEDULED:
-            return send_file('web/static/images/scheduled.png', mimetype="image/png")
+            return send_file('../web/static/images/scheduled.png', mimetype="image/png")
         elif found_job.status == RenderStatus.NOT_STARTED:
-            return send_file('web/static/images/not_started.png', mimetype="image/png")
+            return send_file('../web/static/images/not_started.png', mimetype="image/png")
     # errors
-    return send_file('web/static/images/error.png', mimetype="image/png")
+    return send_file('../web/static/images/error.png', mimetype="image/png")
 
 
 # Get job file routing
@@ -190,7 +191,7 @@ def get_job_status(job_id):
 @server.get('/api/job/<job_id>/logs')
 def get_job_logs(job_id):
     found_job = RenderQueue.job_with_id(job_id)
-    log_path = system_safe_path(found_job.log_path()),
+    log_path = system_safe_path(found_job.log_path())
     log_data = None
     if log_path and os.path.exists(log_path):
         with open(log_path) as file:
@@ -322,10 +323,30 @@ def add_job_handler():
             referred_name = os.path.basename(uploaded_project.filename)
         elif project_url:
             # download and save url - have to download first to know filename due to redirects
-            logger.info(f"Attempting to download URL: {project_url}")
+            logger.info(f"Downloading project from url: {project_url}")
             try:
-                downloaded_file_url, info = urlretrieve(project_url)
+                referred_name = os.path.basename(project_url)
-                referred_name = info.get_filename() or os.path.basename(project_url)
+                response = requests.get(project_url, stream=True)
+                if response.status_code == 200:
+                    # Get the total file size from the "Content-Length" header
+                    file_size = int(response.headers.get("Content-Length", 0))
+
+                    # Create a progress bar using tqdm
+                    progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
+
+                    # Open a file for writing in binary mode
+                    downloaded_file_url = os.path.join(tempfile.gettempdir(), referred_name)
+                    with open(downloaded_file_url, "wb") as file:
+                        for chunk in response.iter_content(chunk_size=1024):
+                            if chunk:
+                                # Write the chunk to the file
+                                file.write(chunk)
+                                # Update the progress bar
+                                progress_bar.update(len(chunk))
+
+                    # Close the progress bar
+                    progress_bar.close()
+
             except Exception as e:
                 err_msg = f"Error downloading file: {e}"
                 logger.error(err_msg)
@@ -419,6 +440,10 @@ def add_job_handler():
                 if not worker.parent:
                     make_job_ready(worker.id)
                 results.append(worker.json())
+            except FileNotFoundError as e:
+                err_msg = f"Cannot create job: {e}"
+                logger.error(err_msg)
+                results.append({'error': err_msg})
             except Exception as e:
                 err_msg = f"Exception creating render job: {e}"
                 logger.exception(err_msg)
@@ -549,6 +574,24 @@ def renderer_info():
                                 'supported_export_formats': engine(install_path).get_output_formats()}
     return renderer_data
 
+@server.get('/api/is_engine_available_to_download')
+def is_engine_available_to_download():
+    available_result = EngineManager.version_is_available_to_download(request.args.get('engine'),
+                                                                      request.args.get('version'),
+                                                                      request.args.get('system_os'),
+                                                                      request.args.get('cpu'))
+    return available_result if available_result else \
+        (f"Cannot find available download for {request.args.get('engine')} {request.args.get('version')}", 500)
+
+
+@server.get('/api/find_most_recent_version')
+def find_most_recent_version():
+    most_recent = EngineManager.find_most_recent_version(request.args.get('engine'),
+                                                         request.args.get('system_os'),
+                                                         request.args.get('cpu'))
+    return most_recent if most_recent else \
+        (f"Error finding most recent version of {request.args.get('engine')}", 500)
+
 
 @server.post('/api/download_engine')
 def download_engine():
@@ -556,7 +599,8 @@ def download_engine():
                                                     request.args.get('version'),
                                                     request.args.get('system_os'),
                                                     request.args.get('cpu'))
-    return download_result if download_result else ("Error downloading requested engine", 500)
+    return download_result if download_result else \
+        (f"Error downloading {request.args.get('engine')} {request.args.get('version')}", 500)
 
 
 @server.post('/api/delete_engine')
@@ -565,7 +609,8 @@ def delete_engine_download():
                                                          request.args.get('version'),
                                                          request.args.get('system_os'),
                                                          request.args.get('cpu'))
-    return "Success" if delete_result else ("Error deleting requested engine", 500)
+    return "Success" if delete_result else \
+        (f"Error deleting {request.args.get('engine')} {request.args.get('version')}", 500)
 
 
 @server.get('/api/renderer/<renderer>/args')

src/client/dashboard_window.py

@@ -11,10 +11,10 @@ from PIL import Image, ImageTk
 
 from src.client.new_job_window import NewJobWindow
 # from src.client.server_details import create_server_popup
-from src.server_proxy import RenderServerProxy
+from src.api.server_proxy import RenderServerProxy
 from src.utilities.misc_helper import launch_url, file_exists_in_mounts, get_time_elapsed
 from src.utilities.zeroconf_server import ZeroconfServer
-from src.workers.base_worker import RenderStatus
+from src.engines.core.base_worker import RenderStatus
 
 logger = logging.getLogger()
 

src/client/new_job_window.py

@@ -11,9 +11,9 @@ from tkinter.ttk import Frame, Label, Entry, Combobox, Progressbar
 
 import psutil
 
-from src.server_proxy import RenderServerProxy
+from src.api.server_proxy import RenderServerProxy
-from src.workers.blender_worker import Blender
+from src.engines.blender.blender_worker import Blender
-from src.workers.ffmpeg_worker import FFMPEG
+from src.engines.ffmpeg.ffmpeg_worker import FFMPEG
 
 logger = logging.getLogger()
 

src/engines/blender/blender_downloader.py

@@ -4,7 +4,8 @@ import re
 
 import requests
 
-from ..core.downloader_core import download_and_extract_app
+from src.engines.core.downloader_core import download_and_extract_app
+from src.utilities.misc_helper import current_system_os, current_system_cpu
 
 # url = "https://download.blender.org/release/"
 url = "https://ftp.nluug.nl/pub/graphics/blender/release/"  # much faster mirror for testing
@@ -35,27 +36,42 @@ class BlenderDownloader:
     @staticmethod
     def get_minor_versions(major_version, system_os=None, cpu=None):
 
-        base_url = url + 'Blender' + major_version
+        try:
+            base_url = url + 'Blender' + major_version
+            response = requests.get(base_url)
+            response.raise_for_status()
 
-        response = requests.get(base_url)
+            versions_pattern = r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
-        response.raise_for_status()
+            versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)]
 
-        versions_pattern = r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
+            # Filter to just the supported formats
-        versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)]
+            versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)]
 
-        # Filter to just the supported formats
+            # Filter down OS and CPU
-        versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)]
+            system_os = system_os or current_system_os()
-
+            cpu = cpu or current_system_cpu()
-        if system_os:
             versions_data = [x for x in versions_data if x['system_os'] == system_os]
-        if cpu:
             versions_data = [x for x in versions_data if x['cpu'] == cpu]
 
-        for v in versions_data:
+            for v in versions_data:
-            v['url'] = base_url + '/' + v['file']
+                v['url'] = base_url + '/' + v['file']
 
-        versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True)
+            versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True)
-        return versions_data
+            return versions_data
+        except requests.exceptions.HTTPError as e:
+            logger.error(f"Invalid url: {e}")
+        except Exception as e:
+            logger.exception(e)
+        return []
+
+    @classmethod
+    def version_is_available_to_download(cls, version, system_os=None, cpu=None):
+        requested_major_version = '.'.join(version.split('.')[:2])
+        minor_versions = cls.get_minor_versions(requested_major_version, system_os, cpu)
+        for minor in minor_versions:
+            if minor['version'] == version:
+                return minor
+        return None
 
     @staticmethod
     def find_LTS_versions():
@@ -79,7 +95,7 @@ class BlenderDownloader:
             logger.error("Cannot find a most recent version")
 
     @classmethod
-    def download_engine(cls, version, download_location, system_os=None, cpu=None):
+    def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
         system_os = system_os or platform.system().lower().replace('darwin', 'macos')
         cpu = cpu or platform.machine().lower().replace('amd64', 'x64')
 
@@ -89,7 +105,8 @@ class BlenderDownloader:
             minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version]
             # we get the URL instead of calculating it ourselves. May change this
 
-            download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location)
+            download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location,
+                                     timeout=timeout)
         except IndexError:
             logger.error("Cannot find requested engine")
 

src/engines/core/downloader_core.py

@@ -12,7 +12,7 @@ supported_formats = ['.zip', '.tar.xz', '.dmg']
 logger = logging.getLogger()
 
 
-def download_and_extract_app(remote_url, download_location):
+def download_and_extract_app(remote_url, download_location, timeout=120):
 
     # Create a temp download directory
     temp_download_dir = tempfile.mkdtemp()
@@ -30,7 +30,7 @@ def download_and_extract_app(remote_url, download_location):
         if not os.path.exists(temp_downloaded_file_path):
             # Make a GET request to the URL with stream=True to enable streaming
             logger.info(f"Downloading {output_dir_name} from {remote_url}")
-            response = requests.get(remote_url, stream=True)
+            response = requests.get(remote_url, stream=True, timeout=timeout)
 
             # Check if the request was successful
             if response.status_code == 200:
@@ -54,6 +54,7 @@ def download_and_extract_app(remote_url, download_location):
                 logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}")
             else:
                 logger.error(f"Failed to download the file. Status code: {response.status_code}")
+                return
 
         os.makedirs(download_location, exist_ok=True)
 

src/engines/core/worker_factory.py

@@ -1,4 +1,5 @@
 import logging
+
 from src.engines.engine_manager import EngineManager
 
 logger = logging.getLogger()
@@ -20,19 +21,29 @@ class RenderWorkerFactory:
 
         worker_class = RenderWorkerFactory.class_for_name(renderer)
 
-        # find correct engine version
+        # check to make sure we have versions installed
         all_versions = EngineManager.all_versions_for_engine(renderer)
         if not all_versions:
             raise FileNotFoundError(f"Cannot find any installed {renderer} engines")
 
-        engine_path = all_versions[0]['path']
+        # Find the path to the requested engine version or use default
+        engine_path = None if engine_version else all_versions[0]['path']
         if engine_version:
             for ver in all_versions:
                 if ver['version'] == engine_version:
                     engine_path = ver['path']
                     break
+
+            # Download the required engine if not found locally
             if not engine_path:
-                logger.warning(f"Cannot find requested engine version {engine_version}. Using default version {all_versions[0]['version']}")
+                download_result = EngineManager.download_engine(renderer, engine_version)
+                if not download_result:
+                    raise FileNotFoundError(f"Cannot download requested version: {renderer} {engine_version}")
+                engine_path = download_result['path']
+                logger.info("Engine downloaded. Creating worker.")
+
+        if not engine_path:
+            raise FileNotFoundError(f"Cannot find requested engine version {engine_version}")
 
         return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args,
                             parent=parent, name=name)

src/engines/engine_manager.py

@@ -15,6 +15,11 @@ logger = logging.getLogger()
 class EngineManager:
 
     engines_path = "~/zordon-uploads/engines"
+    downloader_classes = {
+        "blender": BlenderDownloader,
+        "ffmpeg": FFMPEGDownloader,
+        # Add more engine types and corresponding downloader classes as needed
+    }
 
     @classmethod
     def supported_engines(cls):
@@ -99,6 +104,20 @@ class EngineManager:
     def system_cpu():
         return platform.machine().lower().replace('amd64', 'x64')
 
+    @classmethod
+    def version_is_available_to_download(cls, engine, version, system_os=None, cpu=None):
+        try:
+            return cls.downloader_classes[engine].version_is_available_to_download(version, system_os, cpu)
+        except Exception as e:
+            return None
+
+    @classmethod
+    def find_most_recent_version(cls, engine, system_os, cpu, lts_only=False):
+        try:
+            return cls.downloader_classes[engine].find_most_recent_version(system_os, cpu)
+        except Exception as e:
+            return None
+
     @classmethod
     def download_engine(cls, engine, version, system_os=None, cpu=None):
         existing_download = cls.has_engine_version(engine, version, system_os, cpu)
@@ -107,20 +126,15 @@ class EngineManager:
             return existing_download
 
         logger.info(f"Requesting download of {engine} {version}")
-        downloader_classes = {
-            "blender": BlenderDownloader,
-            "ffmpeg": FFMPEGDownloader,
-            # Add more engine types and corresponding downloader classes as needed
-        }
 
         # Check if the provided engine type is valid
-        if engine not in downloader_classes:
+        if engine not in cls.downloader_classes:
             logger.error("No valid engine found")
             return
 
         # Get the appropriate downloader class based on the engine type
-        downloader_classes[engine].download_engine(version, download_location=cls.engines_path,
+        cls.downloader_classes[engine].download_engine(version, download_location=cls.engines_path,
-                                                   system_os=system_os, cpu=cpu)
+                                                       system_os=system_os, cpu=cpu, timeout=300)
 
         # Check that engine was properly downloaded
         found_engine = cls.has_engine_version(engine, version, system_os, cpu)

src/engines/ffmpeg/ffmpeg_downloader.py

@@ -1,11 +1,11 @@
 import logging
 import os
-import platform
 import re
 
 import requests
 
 from src.engines.core.downloader_core import download_and_extract_app
+from src.utilities.misc_helper import current_system_cpu, current_system_os
 
 logger = logging.getLogger()
 supported_formats = ['.zip', '.tar.xz', '.dmg']
@@ -24,7 +24,7 @@ class FFMPEGDownloader:
     windows_api_url = "https://api.github.com/repos/GyanD/codexffmpeg/releases"
 
     @classmethod
-    def get_macos_versions(cls):
+    def __get_macos_versions(cls):
         response = requests.get(cls.macos_url)
         response.raise_for_status()
 
@@ -34,7 +34,7 @@ class FFMPEGDownloader:
         return [link.split('-')[-1].split('.zip')[0] for link in link_matches]
 
     @classmethod
-    def get_linux_versions(cls):
+    def __get_linux_versions(cls):
 
         # Link 1 / 2 - Current Version
         response = requests.get(cls.linux_url)
@@ -50,7 +50,7 @@ class FFMPEGDownloader:
         return releases
 
     @classmethod
-    def get_windows_versions(cls):
+    def __get_windows_versions(cls):
         response = requests.get(cls.windows_api_url)
         response.raise_for_status()
 
@@ -63,36 +63,65 @@ class FFMPEGDownloader:
 
     @classmethod
     def find_most_recent_version(cls, system_os, cpu, lts_only=False):
-        pass
+        return cls.all_versions(system_os, cpu)[0]
 
     @classmethod
-    def download_engine(cls, version, download_location, system_os=None, cpu=None):
+    def all_versions(cls, system_os=None, cpu=None):
-        system_os = system_os or platform.system().lower().replace('darwin', 'macos')
+        system_os = system_os or current_system_os()
-        cpu = cpu or platform.machine().lower().replace('amd64', 'x64')
+        cpu = cpu or current_system_cpu()
-
+        versions_per_os = {'linux': cls.__get_linux_versions, 'macos': cls.__get_macos_versions, 'windows': cls.__get_windows_versions}
-        # Verify requested version is available
-        remote_url = None
-        versions_per_os = {'linux': cls.get_linux_versions, 'macos': cls.get_macos_versions, 'windows': cls.get_windows_versions}
         if not versions_per_os.get(system_os):
             logger.error(f"Cannot find version list for {system_os}")
             return
-        if version not in versions_per_os[system_os]():
-            logger.error(f"Cannot find FFMPEG version {version} for {system_os}")
 
+        results = []
+        all_versions = versions_per_os[system_os]()
+        for version in all_versions:
+            remote_url = cls.__get_remote_url_for_version(version, system_os, cpu)
+            results.append({'cpu': cpu, 'file': os.path.basename(remote_url), 'system_os': system_os, 'url': remote_url,
+                    'version': version})
+        return results
+
+    @classmethod
+    def version_is_available_to_download(cls, version, system_os=None, cpu=None):
+        for ver in cls.all_versions(system_os, cpu):
+            if ver['version'] == version:
+                return ver
+        return None
+
+    @classmethod
+    def __get_remote_url_for_version(cls, version, system_os, cpu):
         # Platform specific naming cleanup
+        remote_url = None
         if system_os == 'macos':
             remote_url = os.path.join(cls.macos_url, f"ffmpeg-{version}.zip")
-            download_location = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}')  # override location to match linux
         elif system_os == 'linux':
-            release_dir = 'releases' if version == cls.get_linux_versions()[0] else 'old-releases'
+            release_dir = 'releases' if version == cls.__get_linux_versions()[0] else 'old-releases'
             remote_url = os.path.join(cls.linux_url, release_dir, f'ffmpeg-{version}-{cpu}-static.tar.xz')
         elif system_os == 'windows':
             remote_url = f"{cls.windows_download_url.strip('/')}/{version}/ffmpeg-{version}-full_build.zip"
+        else:
+            logger.error("Unknown system os")
+        return remote_url
+
+    @classmethod
+    def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
+        system_os = system_os or current_system_os()
+        cpu = cpu or current_system_os()
+
+        # Verify requested version is available
+        if version not in cls.all_versions(system_os):
+            logger.error(f"Cannot find FFMPEG version {version} for {system_os}")
+
+        # Platform specific naming cleanup
+        remote_url = cls.__get_remote_url_for_version(version, system_os, cpu)
+        if system_os == 'macos':  # override location to match linux
+            download_location = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}')
 
         # Download and extract
         try:
             logger.info(f"Requesting download of ffmpeg-{version}-{system_os}-{cpu}")
-            download_and_extract_app(remote_url=remote_url, download_location=download_location)
+            download_and_extract_app(remote_url=remote_url, download_location=download_location, timeout=timeout)
 
             # naming cleanup to match existing naming convention
             if system_os == 'linux':

src/utilities/misc_helper.py

@@ -110,3 +110,11 @@ def system_safe_path(path):
     if platform.system().lower() == "windows":
         return os.path.normpath(path)
     return path.replace("\\", "/")
+
+
+def current_system_os():
+    return platform.system().lower().replace('darwin', 'macos')
+
+
+def current_system_cpu():
+    return platform.machine().lower().replace('amd64', 'x64')