mirror of
https://github.com/blw1138/Zordon.git
synced 2025-12-18 17:28:12 +00:00
Engine and downloader refactoring (#50)
* Make downloaders subclass of base_downloader.py * Link engines and downloaders together for all engines * Replace / merge worker_factory.py with engine_manager.py
This commit is contained in:
GitHub
158
src/engines/core/base_downloader.py
Normal file
158
src/engines/core/base_downloader.py
Normal file
@@ -0,0 +1,158 @@
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
class EngineDownloader:
|
||||
|
||||
supported_formats = ['.zip', '.tar.xz', '.dmg']
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False):
|
||||
raise NotImplementedError # implement this method in your engine subclass
|
||||
|
||||
@classmethod
|
||||
def version_is_available_to_download(cls, version, system_os=None, cpu=None):
|
||||
raise NotImplementedError # implement this method in your engine subclass
|
||||
|
||||
@classmethod
|
||||
def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
|
||||
raise NotImplementedError # implement this method in your engine subclass
|
||||
|
||||
@classmethod
|
||||
def __download_and_extract_app(cls, remote_url, download_location, timeout=120):
|
||||
|
||||
# Create a temp download directory
|
||||
temp_download_dir = tempfile.mkdtemp()
|
||||
temp_downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url))
|
||||
|
||||
try:
|
||||
output_dir_name = os.path.basename(remote_url)
|
||||
for fmt in cls.supported_formats:
|
||||
output_dir_name = output_dir_name.split(fmt)[0]
|
||||
|
||||
if os.path.exists(os.path.join(download_location, output_dir_name)):
|
||||
logger.error(f"Engine download for {output_dir_name} already exists")
|
||||
return
|
||||
|
||||
if not os.path.exists(temp_downloaded_file_path):
|
||||
# Make a GET request to the URL with stream=True to enable streaming
|
||||
logger.info(f"Downloading {output_dir_name} from {remote_url}")
|
||||
response = requests.get(remote_url, stream=True, timeout=timeout)
|
||||
|
||||
# Check if the request was successful
|
||||
if response.status_code == 200:
|
||||
# Get the total file size from the "Content-Length" header
|
||||
file_size = int(response.headers.get("Content-Length", 0))
|
||||
|
||||
# Create a progress bar using tqdm
|
||||
progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
|
||||
|
||||
# Open a file for writing in binary mode
|
||||
with open(temp_downloaded_file_path, "wb") as file:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
# Write the chunk to the file
|
||||
file.write(chunk)
|
||||
# Update the progress bar
|
||||
progress_bar.update(len(chunk))
|
||||
|
||||
# Close the progress bar
|
||||
progress_bar.close()
|
||||
logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}")
|
||||
else:
|
||||
logger.error(f"Failed to download the file. Status code: {response.status_code}")
|
||||
return
|
||||
|
||||
os.makedirs(download_location, exist_ok=True)
|
||||
|
||||
# Extract the downloaded file
|
||||
# Process .tar.xz files
|
||||
if temp_downloaded_file_path.lower().endswith('.tar.xz'):
|
||||
try:
|
||||
with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar:
|
||||
tar.extractall(path=download_location)
|
||||
|
||||
logger.info(
|
||||
f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
|
||||
except tarfile.TarError as e:
|
||||
logger.error(f'Error extracting {temp_downloaded_file_path}: {e}')
|
||||
except FileNotFoundError:
|
||||
logger.error(f'File not found: {temp_downloaded_file_path}')
|
||||
|
||||
# Process .zip files
|
||||
elif temp_downloaded_file_path.lower().endswith('.zip'):
|
||||
try:
|
||||
with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref:
|
||||
zip_ref.extractall(download_location)
|
||||
logger.info(
|
||||
f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
|
||||
except zipfile.BadZipFile as e:
|
||||
logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.')
|
||||
except FileNotFoundError:
|
||||
logger.error(f'File not found: {temp_downloaded_file_path}')
|
||||
|
||||
# Process .dmg files (macOS only)
|
||||
elif temp_downloaded_file_path.lower().endswith('.dmg'):
|
||||
import dmglib
|
||||
dmg = dmglib.DiskImage(temp_downloaded_file_path)
|
||||
for mount_point in dmg.attach():
|
||||
try:
|
||||
copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name))
|
||||
logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}')
|
||||
except FileNotFoundError:
|
||||
logger.error(f'Error: The source .app bundle does not exist.')
|
||||
except PermissionError:
|
||||
logger.error(f'Error: Permission denied to copy {download_location}.')
|
||||
except Exception as e:
|
||||
logger.error(f'An error occurred: {e}')
|
||||
dmg.detach()
|
||||
|
||||
else:
|
||||
logger.error("Unknown file. Unable to extract binary.")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
|
||||
# remove downloaded file on completion
|
||||
shutil.rmtree(temp_download_dir)
|
||||
return download_location
|
||||
|
||||
|
||||
# Function to copy directory contents but ignore symbolic links and hidden files
|
||||
def copy_directory_contents(src_dir, dest_dir):
|
||||
try:
|
||||
# Create the destination directory if it doesn't exist
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
|
||||
for item in os.listdir(src_dir):
|
||||
item_path = os.path.join(src_dir, item)
|
||||
|
||||
# Ignore symbolic links
|
||||
if os.path.islink(item_path):
|
||||
continue
|
||||
|
||||
# Ignore hidden files or directories (those starting with a dot)
|
||||
if not item.startswith('.'):
|
||||
dest_item_path = os.path.join(dest_dir, item)
|
||||
|
||||
# If it's a directory, recursively copy its contents
|
||||
if os.path.isdir(item_path):
|
||||
copy_directory_contents(item_path, dest_item_path)
|
||||
else:
|
||||
# Otherwise, copy the file
|
||||
shutil.copy2(item_path, dest_item_path)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Error copying directory contents: {e}")
|
||||
@@ -21,7 +21,7 @@ class BaseRenderEngine(object):
|
||||
|
||||
@classmethod
|
||||
def name(cls):
|
||||
return cls.__name__.lower()
|
||||
return str(cls.__name__).lower()
|
||||
|
||||
@classmethod
|
||||
def default_renderer_path(cls):
|
||||
@@ -39,6 +39,14 @@ class BaseRenderEngine(object):
|
||||
def version(self):
|
||||
raise NotImplementedError("version not implemented")
|
||||
|
||||
@staticmethod
|
||||
def downloader(): # override when subclassing if using a downloader class
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def worker_class(): # override when subclassing to link worker class
|
||||
raise NotImplementedError("Worker class not implemented")
|
||||
|
||||
def get_help(self):
|
||||
path = self.renderer_path()
|
||||
if not path:
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
supported_formats = ['.zip', '.tar.xz', '.dmg']
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
def download_and_extract_app(remote_url, download_location, timeout=120):
|
||||
|
||||
# Create a temp download directory
|
||||
temp_download_dir = tempfile.mkdtemp()
|
||||
temp_downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url))
|
||||
|
||||
try:
|
||||
output_dir_name = os.path.basename(remote_url)
|
||||
for fmt in supported_formats:
|
||||
output_dir_name = output_dir_name.split(fmt)[0]
|
||||
|
||||
if os.path.exists(os.path.join(download_location, output_dir_name)):
|
||||
logger.error(f"Engine download for {output_dir_name} already exists")
|
||||
return
|
||||
|
||||
if not os.path.exists(temp_downloaded_file_path):
|
||||
# Make a GET request to the URL with stream=True to enable streaming
|
||||
logger.info(f"Downloading {output_dir_name} from {remote_url}")
|
||||
response = requests.get(remote_url, stream=True, timeout=timeout)
|
||||
|
||||
# Check if the request was successful
|
||||
if response.status_code == 200:
|
||||
# Get the total file size from the "Content-Length" header
|
||||
file_size = int(response.headers.get("Content-Length", 0))
|
||||
|
||||
# Create a progress bar using tqdm
|
||||
progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
|
||||
|
||||
# Open a file for writing in binary mode
|
||||
with open(temp_downloaded_file_path, "wb") as file:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
# Write the chunk to the file
|
||||
file.write(chunk)
|
||||
# Update the progress bar
|
||||
progress_bar.update(len(chunk))
|
||||
|
||||
# Close the progress bar
|
||||
progress_bar.close()
|
||||
logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}")
|
||||
else:
|
||||
logger.error(f"Failed to download the file. Status code: {response.status_code}")
|
||||
return
|
||||
|
||||
os.makedirs(download_location, exist_ok=True)
|
||||
|
||||
# Extract the downloaded file
|
||||
# Process .tar.xz files
|
||||
if temp_downloaded_file_path.lower().endswith('.tar.xz'):
|
||||
try:
|
||||
with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar:
|
||||
tar.extractall(path=download_location)
|
||||
|
||||
logger.info(
|
||||
f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
|
||||
except tarfile.TarError as e:
|
||||
logger.error(f'Error extracting {temp_downloaded_file_path}: {e}')
|
||||
except FileNotFoundError:
|
||||
logger.error(f'File not found: {temp_downloaded_file_path}')
|
||||
|
||||
# Process .zip files
|
||||
elif temp_downloaded_file_path.lower().endswith('.zip'):
|
||||
try:
|
||||
with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref:
|
||||
zip_ref.extractall(download_location)
|
||||
logger.info(
|
||||
f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
|
||||
except zipfile.BadZipFile as e:
|
||||
logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.')
|
||||
except FileNotFoundError:
|
||||
logger.error(f'File not found: {temp_downloaded_file_path}')
|
||||
|
||||
# Process .dmg files (macOS only)
|
||||
elif temp_downloaded_file_path.lower().endswith('.dmg'):
|
||||
import dmglib
|
||||
dmg = dmglib.DiskImage(temp_downloaded_file_path)
|
||||
for mount_point in dmg.attach():
|
||||
try:
|
||||
copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name))
|
||||
logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}')
|
||||
except FileNotFoundError:
|
||||
logger.error(f'Error: The source .app bundle does not exist.')
|
||||
except PermissionError:
|
||||
logger.error(f'Error: Permission denied to copy {download_location}.')
|
||||
except Exception as e:
|
||||
logger.error(f'An error occurred: {e}')
|
||||
dmg.detach()
|
||||
|
||||
else:
|
||||
logger.error("Unknown file. Unable to extract binary.")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
|
||||
# remove downloaded file on completion
|
||||
shutil.rmtree(temp_download_dir)
|
||||
return download_location
|
||||
|
||||
|
||||
# Function to copy directory contents but ignore symbolic links and hidden files
|
||||
def copy_directory_contents(src_dir, dest_dir):
|
||||
try:
|
||||
# Create the destination directory if it doesn't exist
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
|
||||
for item in os.listdir(src_dir):
|
||||
item_path = os.path.join(src_dir, item)
|
||||
|
||||
# Ignore symbolic links
|
||||
if os.path.islink(item_path):
|
||||
continue
|
||||
|
||||
# Ignore hidden files or directories (those starting with a dot)
|
||||
if not item.startswith('.'):
|
||||
dest_item_path = os.path.join(dest_dir, item)
|
||||
|
||||
# If it's a directory, recursively copy its contents
|
||||
if os.path.isdir(item_path):
|
||||
copy_directory_contents(item_path, dest_item_path)
|
||||
else:
|
||||
# Otherwise, copy the file
|
||||
shutil.copy2(item_path, dest_item_path)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Error copying directory contents: {e}")
|
||||
@@ -1,61 +0,0 @@
|
||||
import logging
|
||||
|
||||
from src.engines.engine_manager import EngineManager
|
||||
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
class RenderWorkerFactory:
|
||||
|
||||
@staticmethod
|
||||
def supported_classes():
|
||||
# to add support for any additional RenderWorker classes, import their classes and add to list here
|
||||
from src.engines.blender.blender_worker import BlenderRenderWorker
|
||||
from src.engines.aerender.aerender_worker import AERenderWorker
|
||||
from src.engines.ffmpeg.ffmpeg_worker import FFMPEGRenderWorker
|
||||
classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker]
|
||||
return classes
|
||||
|
||||
@staticmethod
|
||||
def create_worker(renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None):
|
||||
|
||||
worker_class = RenderWorkerFactory.class_for_name(renderer)
|
||||
|
||||
# check to make sure we have versions installed
|
||||
all_versions = EngineManager.all_versions_for_engine(renderer)
|
||||
if not all_versions:
|
||||
raise FileNotFoundError(f"Cannot find any installed {renderer} engines")
|
||||
|
||||
# Find the path to the requested engine version or use default
|
||||
engine_path = None if engine_version else all_versions[0]['path']
|
||||
if engine_version:
|
||||
for ver in all_versions:
|
||||
if ver['version'] == engine_version:
|
||||
engine_path = ver['path']
|
||||
break
|
||||
|
||||
# Download the required engine if not found locally
|
||||
if not engine_path:
|
||||
download_result = EngineManager.download_engine(renderer, engine_version)
|
||||
if not download_result:
|
||||
raise FileNotFoundError(f"Cannot download requested version: {renderer} {engine_version}")
|
||||
engine_path = download_result['path']
|
||||
logger.info("Engine downloaded. Creating worker.")
|
||||
|
||||
if not engine_path:
|
||||
raise FileNotFoundError(f"Cannot find requested engine version {engine_version}")
|
||||
|
||||
return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args,
|
||||
parent=parent, name=name)
|
||||
|
||||
@staticmethod
|
||||
def supported_renderers():
|
||||
return [x.engine.name() for x in RenderWorkerFactory.supported_classes()]
|
||||
|
||||
@staticmethod
|
||||
def class_for_name(name):
|
||||
name = name.lower()
|
||||
for render_class in RenderWorkerFactory.supported_classes():
|
||||
if render_class.engine.name() == name:
|
||||
return render_class
|
||||
raise LookupError(f'Cannot find class for name: {name}')
|
||||
Reference in New Issue
Block a user