Improve performance on several API calls (#80)

* Streamline fetching renderer_info from API - use threading for performance improvements

* Use concurrent.futures instead of Threading

* Fix timeout issue with server proxy

* Minor fixes to code that handles proxy server online / offline status
This commit is contained in:
2024-08-03 11:02:40 -05:00
committed by GitHub
parent 47770c4fdd
commit 21de69ca4f
6 changed files with 88 additions and 51 deletions

View File

@@ -2,6 +2,7 @@ import logging
import os
import shutil
import threading
import concurrent.futures
from src.engines.blender.blender_engine import Blender
from src.engines.ffmpeg.ffmpeg_engine import FFMPEG
@@ -26,7 +27,7 @@ class EngineManager:
return obj
@classmethod
def all_engines(cls):
def all_engines(cls, filter_name=None):
if not cls.engines_path:
raise FileNotFoundError("Engines path must be set before requesting downloads")
@@ -36,47 +37,65 @@ class EngineManager:
try:
all_items = os.listdir(cls.engines_path)
all_directories = [item for item in all_items if os.path.isdir(os.path.join(cls.engines_path, item))]
keys = ["engine", "version", "system_os", "cpu"] # Define keys for result dictionary
for directory in all_directories:
# Split the input string by dashes to get segments
# Split directory name into segments
segments = directory.split('-')
# Create a dictionary with named keys
keys = ["engine", "version", "system_os", "cpu"]
# Create a dictionary mapping keys to corresponding segments
result_dict = {keys[i]: segments[i] for i in range(min(len(keys), len(segments)))}
result_dict['type'] = 'managed'
# Figure out the binary name for the path
# Initialize binary_name with engine name
binary_name = result_dict['engine'].lower()
# Determine the correct binary name based on the engine and system_os
for eng in cls.supported_engines():
if eng.name().lower() == result_dict['engine']:
binary_name = eng.binary_names.get(result_dict['system_os'], binary_name)
# Find path to binary
path = None
for root, _, files in os.walk(system_safe_path(os.path.join(cls.engines_path, directory))):
if binary_name in files:
path = os.path.join(root, binary_name)
break
# Find the path to the binary file
path = next(
(os.path.join(root, binary_name) for root, _, files in
os.walk(system_safe_path(os.path.join(cls.engines_path, directory))) if binary_name in files),
None
)
result_dict['path'] = path
results.append(result_dict)
# Add the result dictionary to results if it matches the filter_name or if no filter is applied
if not filter_name or filter_name == result_dict['engine']:
results.append(result_dict)
except FileNotFoundError as e:
logger.warning(f"Cannot find local engines download directory: {e}")
# add system installs to this list
for eng in cls.supported_engines():
if eng.default_renderer_path():
results.append({'engine': eng.name(), 'version': eng().version(),
'system_os': current_system_os(),
'cpu': current_system_cpu(),
'path': eng.default_renderer_path(), 'type': 'system'})
# add system installs to this list - use bg thread because it can be slow
def fetch_engine_details(eng):
return {
'engine': eng.name(),
'version': eng().version(),
'system_os': current_system_os(),
'cpu': current_system_cpu(),
'path': eng.default_renderer_path(),
'type': 'system'
}
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
executor.submit(fetch_engine_details, eng): eng.name()
for eng in cls.supported_engines()
if eng.default_renderer_path() and (not filter_name or filter_name == eng.name())
}
for future in concurrent.futures.as_completed(futures):
result = future.result()
if result:
results.append(result)
return results
@classmethod
def all_versions_for_engine(cls, engine):
versions = [x for x in cls.all_engines() if x['engine'] == engine]
def all_versions_for_engine(cls, engine_name):
versions = cls.all_engines(filter_name=engine_name)
sorted_versions = sorted(versions, key=lambda x: x['version'], reverse=True)
return sorted_versions