brett/Zordon
1
0
Fork 0
mirror of https://github.com/blw1138/Zordon.git synced 2026-04-25 13:54:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Document all the things! (#117)

Browse Source 
Add lots of docstrings everywhere
This commit is contained in:
Brett
2024-08-23 19:26:05 -05:00
committed by GitHub
parent 8bbf19cb30
commit af6d6e1525
10 changed files with 699 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engines/core/base_downloader.py
+129 -5
View File
@@ -1,9 +1,7 @@
import logging
import os
import shutil
import tarfile
import tempfile
import zipfile
import requests
from tqdm import tqdm
@@ -12,26 +10,150 @@ logger = logging.getLogger()
class EngineDownloader:
"""A class responsible for downloading and extracting rendering engines from publicly available URLs.
Attributes:
supported_formats (list[str]): A list of file formats supported by the downloader.
"""
supported_formats = ['.zip', '.tar.xz', '.dmg']
def __init__(self):
pass
# --------------------------------------------
# Required Overrides for Subclasses:
# --------------------------------------------
@classmethod
def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False):
raise NotImplementedError # implement this method in your engine subclass
"""
Finds the most recent version of the rendering engine available for download.
This method should be overridden in a subclass to implement the logic for determining
the most recent version of the rendering engine, optionally filtering by long-term
support (LTS) versions, the operating system, and CPU architecture.
Args:
system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
lts_only (bool, optional): Limit the search to LTS (long-term support) versions only. Default is False.
Returns:
dict: A dict with the following keys:
- 'cpu' (str): The CPU architecture.
- 'system_os' (str): The operating system.
- 'file' (str): The filename of the version's download file.
- 'url' (str): The remote URL for downloading the version.
- 'version' (str): The version number.
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"find_most_recent_version not implemented for {cls.__class__.__name__}")
@classmethod
def version_is_available_to_download(cls, version, system_os=None, cpu=None):
raise NotImplementedError # implement this method in your engine subclass
"""Checks if a requested version of the rendering engine is available for download.
This method should be overridden in a subclass to implement the logic for determining
whether a given version of the rendering engine is available for download, based on the
operating system and CPU architecture.
Args:
version (str): The requested renderer version to download.
system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
Returns:
bool: True if the version is available for download, False otherwise.
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"version_is_available_to_download not implemented for {cls.__class__.__name__}")
@classmethod
def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
raise NotImplementedError # implement this method in your engine subclass
"""Downloads the requested version of the rendering engine to the given download location.
This method should be overridden in a subclass to implement the logic for downloading
a specific version of the rendering engine. The method is intended to handle the
downloading process based on the version, operating system, CPU architecture, and
timeout parameters.
Args:
version (str): The requested renderer version to download.
download_location (str): The directory where the engine should be downloaded.
system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
timeout (int, optional): The maximum time in seconds to wait for the download. Default is 120 seconds.
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"download_engine not implemented for {cls.__class__.__name__}")
# --------------------------------------------
# Optional Overrides for Subclasses:
# --------------------------------------------
@classmethod
def all_versions(cls, system_os=None, cpu=None):
"""Retrieves a list of available versions of the software for a specific operating system and CPU architecture.
This method fetches all available versions for the given operating system and CPU type, constructing
a list of dictionaries containing details such as the version, CPU architecture, system OS, and the
remote URL for downloading each version.
Args:
system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
Returns:
list[dict]: A list of dictionaries, each containing:
- 'cpu' (str): The CPU architecture.
- 'file' (str): The filename of the version's download file.
- 'system_os' (str): The operating system.
- 'url' (str): The remote URL for downloading the version.
- 'version' (str): The version number.
"""
return []
# --------------------------------------------
# Do Not Override These Methods:
# --------------------------------------------
@classmethod
def download_and_extract_app(cls, remote_url, download_location, timeout=120):
"""Downloads an application from the given remote URL and extracts it to the specified location.
This method handles the downloading of the application, supports multiple archive formats,
and extracts the contents to the specified `download_location`. It also manages temporary
files and logs progress throughout the process.
Args:
remote_url (str): The URL of the application to download.
download_location (str): The directory where the application should be extracted.
timeout (int, optional): The maximum time in seconds to wait for the download. Default is 120 seconds.
Returns:
str: The path to the directory where the application was extracted.
Raises:
Exception: Catches and logs any exceptions that occur during the download or extraction process.
Supported Formats:
- `.tar.xz`: Extracted using the `tarfile` module.
- `.zip`: Extracted using the `zipfile` module.
- `.dmg`: macOS disk image files, handled using the `dmglib` library.
- Other formats will result in an error being logged.
Notes:
- If the application already exists in the `download_location`, the method will log an error
and return without downloading or extracting.
- Temporary files created during the download process are cleaned up after completion.
"""
# Create a temp download directory
temp_download_dir = tempfile.mkdtemp()
@@ -80,6 +202,7 @@ class EngineDownloader:
# Extract the downloaded file
# Process .tar.xz files
if temp_downloaded_file_path.lower().endswith('.tar.xz'):
import tarfile
try:
with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar:
tar.extractall(path=download_location)
@@ -93,6 +216,7 @@ class EngineDownloader:
# Process .zip files
elif temp_downloaded_file_path.lower().endswith('.zip'):
import zipfile
try:
with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref:
zip_ref.extractall(download_location)
src/engines/core/base_engine.py
+121 -43
View File
@@ -8,9 +8,21 @@ SUBPROCESS_TIMEOUT = 5
class BaseRenderEngine(object):
"""Base class for render engines. This class provides common functionality and structure for various rendering
engines. Create subclasses and override the methods marked below to add additional renderers
Attributes:
install_paths (list): A list of default installation paths where the render engine
might be found. This list can be populated with common paths to help locate the
executable on different operating systems or environments.
"""
install_paths = []
# --------------------------------------------
# Required Overrides for Subclasses:
# --------------------------------------------
def __init__(self, custom_path=None):
self.custom_renderer_path = custom_path
if not self.renderer_path() or not os.path.exists(self.renderer_path()):
@@ -20,6 +32,115 @@ class BaseRenderEngine(object):
logger.warning(f"Path is not executable. Setting permissions to 755 for {self.renderer_path()}")
os.chmod(self.renderer_path(), 0o755)
def version(self):
"""Return the version number as a string.
Returns:
str: Version number.
Raises:
NotImplementedError: If not overridden.
"""
raise NotImplementedError(f"version not implemented for {self.__class__.__name__}")
def get_project_info(self, project_path, timeout=10):
"""Extracts detailed project information from the given project path.
Args:
project_path (str): The path to the project file.
timeout (int, optional): The maximum time (in seconds) to wait for the operation. Default is 10 seconds.
Returns:
dict: A dictionary containing project information (subclasses should define the structure).
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"get_project_info not implemented for {self.__class__.__name__}")
@classmethod
def get_output_formats(cls):
"""Returns a list of available output formats supported by the renderer.
Returns:
list[str]: A list of strings representing the available output formats.
"""
raise NotImplementedError(f"get_output_formats not implemented for {cls.__name__}")
@staticmethod
def worker_class(): # override when subclassing to link worker class
raise NotImplementedError("Worker class not implemented")
# --------------------------------------------
# Optional Overrides for Subclasses:
# --------------------------------------------
def supported_extensions(self):
"""
Returns:
list[str]: list of supported extensions
"""
return []
def get_help(self):
"""Retrieves the help documentation for the renderer.
This method runs the renderer's help command (default: '-h') and captures the output.
Override this method if the renderer uses a different help flag.
Returns:
str: The help documentation as a string.
Raises:
FileNotFoundError: If the renderer path is not found.
"""
path = self.renderer_path()
if not path:
raise FileNotFoundError("renderer path not found")
creationflags = subprocess.CREATE_NO_WINDOW if platform.system() == 'Windows' else 0
help_doc = subprocess.check_output([path, '-h'], stderr=subprocess.STDOUT,
timeout=SUBPROCESS_TIMEOUT, creationflags=creationflags).decode('utf-8')
return help_doc
def system_info(self):
"""Return additional information about the system specfic to the engine (configured GPUs, render engines, etc)
Returns:
dict: A dictionary with engine-specific system information
"""
return {}
def perform_presubmission_tasks(self, project_path):
"""Perform any pre-submission tasks on a project file before uploading it to a server (pack textures, etc.)
Override this method to:
1. Copy the project file to a temporary location (DO NOT MODIFY ORIGINAL PATH).
2. Perform additional modifications or tasks.
3. Return the path to the modified project file.
Args:
project_path (str): The original project file path.
Returns:
str: The path to the modified project file.
"""
return project_path
def get_arguments(self):
pass
@staticmethod
def downloader(): # override when subclassing if using a downloader class
return None
@staticmethod
def ui_options(system_info): # override to return options for ui
return {}
# --------------------------------------------
# Do Not Override These Methods:
# --------------------------------------------
def renderer_path(self):
return self.custom_renderer_path or self.default_renderer_path()
@@ -39,46 +160,3 @@ class BaseRenderEngine(object):
except Exception as e:
logger.exception(e)
return path
def version(self):
raise NotImplementedError("version not implemented")
def supported_extensions(self):
return []
@staticmethod
def downloader(): # override when subclassing if using a downloader class
return None
@staticmethod
def worker_class(): # override when subclassing to link worker class
raise NotImplementedError("Worker class not implemented")
@staticmethod
def ui_options(system_info): # override to return options for ui
return {}
def get_help(self): # override if renderer uses different help flag
path = self.renderer_path()
if not path:
raise FileNotFoundError("renderer path not found")
creationflags = subprocess.CREATE_NO_WINDOW if platform.system() == 'Windows' else 0
help_doc = subprocess.check_output([path, '-h'], stderr=subprocess.STDOUT,
timeout=SUBPROCESS_TIMEOUT, creationflags=creationflags).decode('utf-8')
return help_doc
def get_project_info(self, project_path, timeout=10):
raise NotImplementedError(f"get_project_info not implemented for {self.__name__}")
@classmethod
def get_output_formats(cls):
raise NotImplementedError(f"get_output_formats not implemented for {cls.__name__}")
def get_arguments(self):
pass
def system_info(self):
pass
def perform_presubmission_tasks(self, project_path):
return project_path
src/engines/core/base_worker.py
+50 -16
View File
@@ -48,6 +48,10 @@ class BaseRenderWorker(Base):
engine = None
# --------------------------------------------
# Required Overrides for Subclasses:
# --------------------------------------------
def __init__(self, input_path, output_path, engine_path, priority=2, args=None, ignore_extensions=True, parent=None,
name=None):
@@ -57,7 +61,7 @@ class BaseRenderWorker(Base):
logger.error(err_meg)
raise ValueError(err_meg)
if not self.engine:
raise NotImplementedError("Engine not defined")
raise NotImplementedError(f"Engine not defined for {self.__class__.__name__}")
def generate_id():
import uuid
@@ -103,6 +107,50 @@ class BaseRenderWorker(Base):
self.__last_output_time = None
self.watchdog_timeout = 120
def generate_worker_subprocess(self):
"""Generate a return a list of the command line arguments necessary to perform requested job
Returns:
list[str]: list of command line arguments
"""
raise NotImplementedError("generate_worker_subprocess not implemented")
def _parse_stdout(self, line):
"""Parses a line of standard output from the renderer.
This method should be overridden in a subclass to implement the logic for processing
and interpreting a single line of output from the renderer's standard output stream.
On frame completion, the subclass should:
1. Update value of self.current_frame
2. Call self._send_frame_complete_notification()
Args:
line (str): A line of text from the renderer's standard output.
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"_parse_stdout not implemented for {self.__class__.__name__}")
# --------------------------------------------
# Optional Overrides for Subclasses:
# --------------------------------------------
def percent_complete(self):
# todo: fix this
if self.status == RenderStatus.COMPLETED:
return 1.0
return 0
def post_processing(self):
"""Override to perform any engine-specific postprocessing"""
pass
# --------------------------------------------
# Do Not Override These Methods:
# --------------------------------------------
def __repr__(self):
return f"<Job id:{self.id} p{self.priority} {self.renderer}-{self.renderer_version} '{self.name}' status:{self.status.value}>"
@@ -142,16 +190,13 @@ class BaseRenderWorker(Base):
return generated_args
def get_raw_args(self):
raw_args_string = self.args.get('raw', None)
raw_args_string = self.args.get('raw', '')
raw_args = None
if raw_args_string:
import shlex
raw_args = shlex.split(raw_args_string)
return raw_args
def generate_worker_subprocess(self):
raise NotImplementedError("generate_worker_subprocess not implemented")
def log_path(self):
filename = (self.name or os.path.basename(self.input_path)) + '_' + \
self.date_created.strftime("%Y.%m.%d_%H.%M.%S") + '.log'
@@ -387,9 +432,6 @@ class BaseRenderWorker(Base):
except Exception as e:
logger.error(f"Error stopping the process: {e}")
def post_processing(self):
pass
def is_running(self):
if hasattr(self, '__thread'):
return self.__thread.is_alive()
@@ -418,14 +460,6 @@ class BaseRenderWorker(Base):
if self.is_running(): # allow the log files to close
self.__thread.join(timeout=5)
def percent_complete(self):
if self.status == RenderStatus.COMPLETED:
return 1.0
return 0
def _parse_stdout(self, line):
raise NotImplementedError("_parse_stdout not implemented")
def time_elapsed(self):
return get_time_elapsed(self.start_time, self.end_time)
src/engines/engine_manager.py
+14
View File
@@ -12,6 +12,9 @@ logger = logging.getLogger()
class EngineManager:
"""Class that manages different versions of installed renderers and handles fetching and downloading new versions,
if possible.
"""
engines_path = None
download_tasks = []
@@ -283,6 +286,17 @@ class EngineManager:
class EngineDownloadWorker(threading.Thread):
"""A thread worker for downloading a specific version of a rendering engine.
This class handles the process of downloading a rendering engine in a separate thread,
ensuring that the download process does not block the main application.
Attributes:
engine (str): The name of the rendering engine to download.
version (str): The version of the rendering engine to download.
system_os (str, optional): The operating system for which to download the engine. Defaults to current OS type.
cpu (str, optional): Requested CPU architecture. Defaults to system CPU type.
"""
def __init__(self, engine, version, system_os=None, cpu=None):
super().__init__()
self.engine = engine