Engine downloader API for #31 (#42)

* Add is_engine_available_to_download API call

* Fix issue with worker never throwing error if engine is not found

* Add API call to get most recent engine version

* Fix some minor import issues

* Fix web urls

* Fix default server log level

* Add progress bar for project download worker_factory downloads missing engine versions

* Better error handling when invalid version is given

* Add timeouts to engine downloaders

src/engines/blender/blender_downloader.py

@@ -4,7 +4,8 @@ import re
 
 import requests
 
-from ..core.downloader_core import download_and_extract_app
+from src.engines.core.downloader_core import download_and_extract_app
+from src.utilities.misc_helper import current_system_os, current_system_cpu
 
 # url = "https://download.blender.org/release/"
 url = "https://ftp.nluug.nl/pub/graphics/blender/release/"  # much faster mirror for testing
@@ -35,27 +36,42 @@ class BlenderDownloader:
     @staticmethod
     def get_minor_versions(major_version, system_os=None, cpu=None):
 
-        base_url = url + 'Blender' + major_version
+        try:
+            base_url = url + 'Blender' + major_version
+            response = requests.get(base_url)
+            response.raise_for_status()
 
-        response = requests.get(base_url)
-        response.raise_for_status()
+            versions_pattern = r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
+            versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)]
 
-        versions_pattern = r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
-        versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)]
+            # Filter to just the supported formats
+            versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)]
 
-        # Filter to just the supported formats
-        versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)]
-
-        if system_os:
+            # Filter down OS and CPU
+            system_os = system_os or current_system_os()
+            cpu = cpu or current_system_cpu()
             versions_data = [x for x in versions_data if x['system_os'] == system_os]
-        if cpu:
             versions_data = [x for x in versions_data if x['cpu'] == cpu]
 
-        for v in versions_data:
-            v['url'] = base_url + '/' + v['file']
+            for v in versions_data:
+                v['url'] = base_url + '/' + v['file']
 
-        versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True)
-        return versions_data
+            versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True)
+            return versions_data
+        except requests.exceptions.HTTPError as e:
+            logger.error(f"Invalid url: {e}")
+        except Exception as e:
+            logger.exception(e)
+        return []
+
+    @classmethod
+    def version_is_available_to_download(cls, version, system_os=None, cpu=None):
+        requested_major_version = '.'.join(version.split('.')[:2])
+        minor_versions = cls.get_minor_versions(requested_major_version, system_os, cpu)
+        for minor in minor_versions:
+            if minor['version'] == version:
+                return minor
+        return None
 
     @staticmethod
     def find_LTS_versions():
@@ -79,7 +95,7 @@ class BlenderDownloader:
             logger.error("Cannot find a most recent version")
 
     @classmethod
-    def download_engine(cls, version, download_location, system_os=None, cpu=None):
+    def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
         system_os = system_os or platform.system().lower().replace('darwin', 'macos')
         cpu = cpu or platform.machine().lower().replace('amd64', 'x64')
 
@@ -89,7 +105,8 @@ class BlenderDownloader:
             minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version]
             # we get the URL instead of calculating it ourselves. May change this
 
-            download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location)
+            download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location,
+                                     timeout=timeout)
         except IndexError:
             logger.error("Cannot find requested engine")
 

src/engines/core/downloader_core.py

@@ -12,7 +12,7 @@ supported_formats = ['.zip', '.tar.xz', '.dmg']
 logger = logging.getLogger()
 
 
-def download_and_extract_app(remote_url, download_location):
+def download_and_extract_app(remote_url, download_location, timeout=120):
 
     # Create a temp download directory
     temp_download_dir = tempfile.mkdtemp()
@@ -30,7 +30,7 @@ def download_and_extract_app(remote_url, download_location):
         if not os.path.exists(temp_downloaded_file_path):
             # Make a GET request to the URL with stream=True to enable streaming
             logger.info(f"Downloading {output_dir_name} from {remote_url}")
-            response = requests.get(remote_url, stream=True)
+            response = requests.get(remote_url, stream=True, timeout=timeout)
 
             # Check if the request was successful
             if response.status_code == 200:
@@ -54,6 +54,7 @@ def download_and_extract_app(remote_url, download_location):
                 logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}")
             else:
                 logger.error(f"Failed to download the file. Status code: {response.status_code}")
+                return
 
         os.makedirs(download_location, exist_ok=True)
 

src/engines/core/worker_factory.py

@@ -1,4 +1,5 @@
 import logging
+
 from src.engines.engine_manager import EngineManager
 
 logger = logging.getLogger()
@@ -20,19 +21,29 @@ class RenderWorkerFactory:
 
         worker_class = RenderWorkerFactory.class_for_name(renderer)
 
-        # find correct engine version
+        # check to make sure we have versions installed
         all_versions = EngineManager.all_versions_for_engine(renderer)
         if not all_versions:
             raise FileNotFoundError(f"Cannot find any installed {renderer} engines")
 
-        engine_path = all_versions[0]['path']
+        # Find the path to the requested engine version or use default
+        engine_path = None if engine_version else all_versions[0]['path']
         if engine_version:
             for ver in all_versions:
                 if ver['version'] == engine_version:
                     engine_path = ver['path']
                     break
+
+            # Download the required engine if not found locally
             if not engine_path:
-                logger.warning(f"Cannot find requested engine version {engine_version}. Using default version {all_versions[0]['version']}")
+                download_result = EngineManager.download_engine(renderer, engine_version)
+                if not download_result:
+                    raise FileNotFoundError(f"Cannot download requested version: {renderer} {engine_version}")
+                engine_path = download_result['path']
+                logger.info("Engine downloaded. Creating worker.")
+
+        if not engine_path:
+            raise FileNotFoundError(f"Cannot find requested engine version {engine_version}")
 
         return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args,
                             parent=parent, name=name)

src/engines/engine_manager.py

@@ -15,6 +15,11 @@ logger = logging.getLogger()
 class EngineManager:
 
     engines_path = "~/zordon-uploads/engines"
+    downloader_classes = {
+        "blender": BlenderDownloader,
+        "ffmpeg": FFMPEGDownloader,
+        # Add more engine types and corresponding downloader classes as needed
+    }
 
     @classmethod
     def supported_engines(cls):
@@ -99,6 +104,20 @@ class EngineManager:
     def system_cpu():
         return platform.machine().lower().replace('amd64', 'x64')
 
+    @classmethod
+    def version_is_available_to_download(cls, engine, version, system_os=None, cpu=None):
+        try:
+            return cls.downloader_classes[engine].version_is_available_to_download(version, system_os, cpu)
+        except Exception as e:
+            return None
+
+    @classmethod
+    def find_most_recent_version(cls, engine, system_os, cpu, lts_only=False):
+        try:
+            return cls.downloader_classes[engine].find_most_recent_version(system_os, cpu)
+        except Exception as e:
+            return None
+
     @classmethod
     def download_engine(cls, engine, version, system_os=None, cpu=None):
         existing_download = cls.has_engine_version(engine, version, system_os, cpu)
@@ -107,20 +126,15 @@ class EngineManager:
             return existing_download
 
         logger.info(f"Requesting download of {engine} {version}")
-        downloader_classes = {
-            "blender": BlenderDownloader,
-            "ffmpeg": FFMPEGDownloader,
-            # Add more engine types and corresponding downloader classes as needed
-        }
 
         # Check if the provided engine type is valid
-        if engine not in downloader_classes:
+        if engine not in cls.downloader_classes:
             logger.error("No valid engine found")
             return
 
         # Get the appropriate downloader class based on the engine type
-        downloader_classes[engine].download_engine(version, download_location=cls.engines_path,
-                                                   system_os=system_os, cpu=cpu)
+        cls.downloader_classes[engine].download_engine(version, download_location=cls.engines_path,
+                                                       system_os=system_os, cpu=cpu, timeout=300)
 
         # Check that engine was properly downloaded
         found_engine = cls.has_engine_version(engine, version, system_os, cpu)

src/engines/ffmpeg/ffmpeg_downloader.py

@@ -1,11 +1,11 @@
 import logging
 import os
-import platform
 import re
 
 import requests
 
 from src.engines.core.downloader_core import download_and_extract_app
+from src.utilities.misc_helper import current_system_cpu, current_system_os
 
 logger = logging.getLogger()
 supported_formats = ['.zip', '.tar.xz', '.dmg']
@@ -24,7 +24,7 @@ class FFMPEGDownloader:
     windows_api_url = "https://api.github.com/repos/GyanD/codexffmpeg/releases"
 
     @classmethod
-    def get_macos_versions(cls):
+    def __get_macos_versions(cls):
         response = requests.get(cls.macos_url)
         response.raise_for_status()
 
@@ -34,7 +34,7 @@ class FFMPEGDownloader:
         return [link.split('-')[-1].split('.zip')[0] for link in link_matches]
 
     @classmethod
-    def get_linux_versions(cls):
+    def __get_linux_versions(cls):
 
         # Link 1 / 2 - Current Version
         response = requests.get(cls.linux_url)
@@ -50,7 +50,7 @@ class FFMPEGDownloader:
         return releases
 
     @classmethod
-    def get_windows_versions(cls):
+    def __get_windows_versions(cls):
         response = requests.get(cls.windows_api_url)
         response.raise_for_status()
 
@@ -63,36 +63,65 @@ class FFMPEGDownloader:
 
     @classmethod
     def find_most_recent_version(cls, system_os, cpu, lts_only=False):
-        pass
+        return cls.all_versions(system_os, cpu)[0]
 
     @classmethod
-    def download_engine(cls, version, download_location, system_os=None, cpu=None):
-        system_os = system_os or platform.system().lower().replace('darwin', 'macos')
-        cpu = cpu or platform.machine().lower().replace('amd64', 'x64')
-
-        # Verify requested version is available
-        remote_url = None
-        versions_per_os = {'linux': cls.get_linux_versions, 'macos': cls.get_macos_versions, 'windows': cls.get_windows_versions}
+    def all_versions(cls, system_os=None, cpu=None):
+        system_os = system_os or current_system_os()
+        cpu = cpu or current_system_cpu()
+        versions_per_os = {'linux': cls.__get_linux_versions, 'macos': cls.__get_macos_versions, 'windows': cls.__get_windows_versions}
         if not versions_per_os.get(system_os):
             logger.error(f"Cannot find version list for {system_os}")
             return
-        if version not in versions_per_os[system_os]():
-            logger.error(f"Cannot find FFMPEG version {version} for {system_os}")
 
+        results = []
+        all_versions = versions_per_os[system_os]()
+        for version in all_versions:
+            remote_url = cls.__get_remote_url_for_version(version, system_os, cpu)
+            results.append({'cpu': cpu, 'file': os.path.basename(remote_url), 'system_os': system_os, 'url': remote_url,
+                    'version': version})
+        return results
+
+    @classmethod
+    def version_is_available_to_download(cls, version, system_os=None, cpu=None):
+        for ver in cls.all_versions(system_os, cpu):
+            if ver['version'] == version:
+                return ver
+        return None
+
+    @classmethod
+    def __get_remote_url_for_version(cls, version, system_os, cpu):
         # Platform specific naming cleanup
+        remote_url = None
         if system_os == 'macos':
             remote_url = os.path.join(cls.macos_url, f"ffmpeg-{version}.zip")
-            download_location = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}')  # override location to match linux
         elif system_os == 'linux':
-            release_dir = 'releases' if version == cls.get_linux_versions()[0] else 'old-releases'
+            release_dir = 'releases' if version == cls.__get_linux_versions()[0] else 'old-releases'
             remote_url = os.path.join(cls.linux_url, release_dir, f'ffmpeg-{version}-{cpu}-static.tar.xz')
         elif system_os == 'windows':
             remote_url = f"{cls.windows_download_url.strip('/')}/{version}/ffmpeg-{version}-full_build.zip"
+        else:
+            logger.error("Unknown system os")
+        return remote_url
+
+    @classmethod
+    def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
+        system_os = system_os or current_system_os()
+        cpu = cpu or current_system_os()
+
+        # Verify requested version is available
+        if version not in cls.all_versions(system_os):
+            logger.error(f"Cannot find FFMPEG version {version} for {system_os}")
+
+        # Platform specific naming cleanup
+        remote_url = cls.__get_remote_url_for_version(version, system_os, cpu)
+        if system_os == 'macos':  # override location to match linux
+            download_location = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}')
 
         # Download and extract
         try:
             logger.info(f"Requesting download of ffmpeg-{version}-{system_os}-{cpu}")
-            download_and_extract_app(remote_url=remote_url, download_location=download_location)
+            download_and_extract_app(remote_url=remote_url, download_location=download_location, timeout=timeout)
 
             # naming cleanup to match existing naming convention
             if system_os == 'linux':