diff --git a/src/engines/downloaders/__init__.py b/src/engines/downloaders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/engines/downloaders/blender_downloader.py b/src/engines/downloaders/blender_downloader.py new file mode 100644 index 0000000..b7d5b94 --- /dev/null +++ b/src/engines/downloaders/blender_downloader.py @@ -0,0 +1,102 @@ +import logging +import os +import platform +import re + +import requests + +from downloader_core import download_and_extract_app + +# url = "https://download.blender.org/release/" +url = "https://ftp.nluug.nl/pub/graphics/blender/release/" # much faster mirror for testing + +logger = logging.getLogger() +supported_formats = ['.zip', '.tar.xz', '.dmg'] + + +class BlenderDownloader: + + @staticmethod + def get_major_versions(): + try: + response = requests.get(url) + response.raise_for_status() + + # Use regex to find all the tags and extract the href attribute + link_pattern = r'Blender(\d+[^<]+)' + link_matches = re.findall(link_pattern, response.text) + + major_versions = [link[-1].strip('/') for link in link_matches] + major_versions.sort(reverse=True) + return major_versions + except requests.exceptions.RequestException as e: + logger.error(f"Error: {e}") + return None + + @staticmethod + def get_minor_versions(major_version, system_os=None, cpu=None): + + base_url = url + 'Blender' + major_version + + response = requests.get(base_url) + response.raise_for_status() + + versions_pattern = r'blender-(?P[\d\.]+)-(?P\w+)-(?P\w+).*' + versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)] + + # Filter to just the supported formats + versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)] + + if system_os: + versions_data = [x for x in versions_data if x['system_os'] == system_os] + if cpu: + versions_data = [x for x in versions_data if x['cpu'] == cpu] + + for v in versions_data: + v['url'] = os.path.join(base_url, v['file']) + + versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True) + return versions_data + + @staticmethod + def find_LTS_versions(): + response = requests.get('https://www.blender.org/download/lts/') + response.raise_for_status() + + lts_pattern = r'https://www.blender.org/download/lts/(\d+-\d+)/' + lts_matches = re.findall(lts_pattern, response.text) + lts_versions = [ver.replace('-', '.') for ver in list(set(lts_matches))] + lts_versions.sort(reverse=True) + + return lts_versions + + @classmethod + def find_most_recent_version(cls, system_os, cpu, lts_only=False): + try: + major_version = cls.find_LTS_versions()[0] if lts_only else cls.get_major_versions()[0] + most_recent = cls.get_minor_versions(major_version, system_os, cpu)[0] + return most_recent + except IndexError: + logger.error("Cannot find a most recent version") + + @classmethod + def download_engine(cls, version, download_location, system_os=None, cpu=None): + system_os = system_os or platform.system().lower().replace('darwin', 'macos') + cpu = cpu or platform.machine().lower().replace('amd64', 'x64') + + try: + logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}") + major_version = '.'.join(version.split('.')[:2]) + minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version] + # we get the URL instead of calculating it ourselves. May change this + + download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location) + except IndexError: + logger.error("Cannot find requested engine") + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + print(BlenderDownloader.get_major_versions()) + diff --git a/src/engines/downloaders/downloader_core.py b/src/engines/downloaders/downloader_core.py new file mode 100644 index 0000000..65d8abc --- /dev/null +++ b/src/engines/downloaders/downloader_core.py @@ -0,0 +1,138 @@ +import logging +import os +import shutil +import tarfile +import tempfile +import zipfile + +import requests +from tqdm import tqdm + +supported_formats = ['.zip', '.tar.xz', '.dmg'] +logger = logging.getLogger() + + +def download_and_extract_app(remote_url, download_location): + + # Create a temp download directory + temp_download_dir = tempfile.mkdtemp() + temp_downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url)) + + try: + output_dir_name = os.path.basename(remote_url) + for fmt in supported_formats: + output_dir_name = output_dir_name.split(fmt)[0] + + if os.path.exists(os.path.join(download_location, output_dir_name)): + logger.error(f"Engine download for {output_dir_name} already exists") + return + + if not os.path.exists(temp_downloaded_file_path): + # Make a GET request to the URL with stream=True to enable streaming + logger.info(f"Downloading {output_dir_name} from {remote_url}") + response = requests.get(remote_url, stream=True) + + # Check if the request was successful + if response.status_code == 200: + # Get the total file size from the "Content-Length" header + file_size = int(response.headers.get("Content-Length", 0)) + + # Create a progress bar using tqdm + progress_bar = tqdm(total=file_size, unit="B", unit_scale=True) + + # Open a file for writing in binary mode + with open(temp_downloaded_file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=1024): + if chunk: + # Write the chunk to the file + file.write(chunk) + # Update the progress bar + progress_bar.update(len(chunk)) + + # Close the progress bar + progress_bar.close() + logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}") + else: + logger.error(f"Failed to download the file. Status code: {response.status_code}") + + os.makedirs(download_location, exist_ok=True) + + # Extract the downloaded file + # Process .tar.xz files + if temp_downloaded_file_path.lower().endswith('.tar.xz'): + try: + with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar: + tar.extractall(path=download_location) + + logger.info( + f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}') + except tarfile.TarError as e: + logger.error(f'Error extracting {temp_downloaded_file_path}: {e}') + except FileNotFoundError: + logger.error(f'File not found: {temp_downloaded_file_path}') + + # Process .zip files + elif temp_downloaded_file_path.lower().endswith('.zip'): + try: + with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref: + zip_ref.extractall(download_location) + logger.info( + f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}') + except zipfile.BadZipFile as e: + logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.') + except FileNotFoundError: + logger.error(f'File not found: {temp_downloaded_file_path}') + + # Process .dmg files (macOS only) + elif temp_downloaded_file_path.lower().endswith('.dmg'): + import dmglib + dmg = dmglib.DiskImage(temp_downloaded_file_path) + for mount_point in dmg.attach(): + try: + copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name)) + logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}') + except FileNotFoundError: + logger.error(f'Error: The source .app bundle does not exist.') + except PermissionError: + logger.error(f'Error: Permission denied to copy {download_location}.') + except Exception as e: + logger.error(f'An error occurred: {e}') + dmg.detach() + + else: + logger.error("Unknown file. Unable to extract binary.") + + except Exception as e: + logger.exception(e) + + # remove downloaded file on completion + shutil.rmtree(temp_download_dir) + return download_location + + +# Function to copy directory contents but ignore symbolic links and hidden files +def copy_directory_contents(src_dir, dest_dir): + try: + # Create the destination directory if it doesn't exist + os.makedirs(dest_dir, exist_ok=True) + + for item in os.listdir(src_dir): + item_path = os.path.join(src_dir, item) + + # Ignore symbolic links + if os.path.islink(item_path): + continue + + # Ignore hidden files or directories (those starting with a dot) + if not item.startswith('.'): + dest_item_path = os.path.join(dest_dir, item) + + # If it's a directory, recursively copy its contents + if os.path.isdir(item_path): + copy_directory_contents(item_path, dest_item_path) + else: + # Otherwise, copy the file + shutil.copy2(item_path, dest_item_path) + + except Exception as e: + logger.exception(f"Error copying directory contents: {e}") diff --git a/src/engines/downloaders/ffmpeg_downloader.py b/src/engines/downloaders/ffmpeg_downloader.py new file mode 100644 index 0000000..23b0c76 --- /dev/null +++ b/src/engines/downloaders/ffmpeg_downloader.py @@ -0,0 +1,112 @@ +import logging +import os +import platform +import re + +import requests + +from downloader_core import download_and_extract_app + +logger = logging.getLogger() +supported_formats = ['.zip', '.tar.xz', '.dmg'] + + +class FFMPEGDownloader: + + # macOS FFMPEG mirror maintained by Evermeet - https://evermeet.cx/ffmpeg/ + macos_url = "https://evermeet.cx/pub/ffmpeg/" + + # Linux FFMPEG mirror maintained by John van Sickle - https://johnvansickle.com/ffmpeg/ + linux_url = "https://johnvansickle.com/ffmpeg/" + + # macOS FFMPEG mirror maintained by GyanD - https://www.gyan.dev/ffmpeg/builds/ + windows_download_url = "https://github.com/GyanD/codexffmpeg/releases/download/" + windows_api_url = "https://api.github.com/repos/GyanD/codexffmpeg/releases" + + @classmethod + def get_macos_versions(cls): + response = requests.get(cls.macos_url) + response.raise_for_status() + + link_pattern = r'>(.*\.zip)[^\.]' + link_matches = re.findall(link_pattern, response.text) + + return [link.split('-')[-1].split('.zip')[0] for link in link_matches] + + @classmethod + def get_linux_versions(cls): + + # Link 1 / 2 - Current Version + response = requests.get(cls.linux_url) + response.raise_for_status() + current_release = re.findall(r'release: ([\w\.]+)', response.text)[0] + + # Link 2 / 2 - Previous Versions + response = requests.get(os.path.join(cls.linux_url, 'old-releases')) + response.raise_for_status() + releases = list(set(re.findall(r'href="ffmpeg-([\w\.]+)-.*">ffmpeg', response.text))) + releases.sort(reverse=True) + releases.insert(0, current_release) + return releases + + @classmethod + def get_windows_versions(cls): + response = requests.get(cls.windows_api_url) + response.raise_for_status() + + versions = [] + all_git_releases = response.json() + for item in all_git_releases: + if re.match(r'^[0-9.]+$', item['tag_name']): + versions.append(item['tag_name']) + return versions + + @classmethod + def find_most_recent_version(cls, system_os, cpu, lts_only=False): + pass + + @classmethod + def download_engine(cls, version, download_location, system_os=None, cpu=None): + system_os = system_os or platform.system().lower().replace('darwin', 'macos') + cpu = cpu or platform.machine().lower().replace('amd64', 'x64') + + # Verify requested version is available + remote_url = None + versions_per_os = {'linux': cls.get_linux_versions, 'macos': cls.get_macos_versions, 'windows': cls.get_windows_versions} + if not versions_per_os.get(system_os): + logger.error(f"Cannot find version list for {system_os}") + return + if version not in versions_per_os[system_os](): + logger.error(f"Cannot find FFMPEG version {version} for {system_os}") + + # Platform specific naming cleanup + if system_os == 'macos': + remote_url = os.path.join(cls.macos_url, f"ffmpeg-{version}.zip") + download_location = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}') # override location to match linux + elif system_os == 'linux': + release_dir = 'releases' if version == cls.get_linux_versions()[0] else 'old-releases' + remote_url = os.path.join(cls.linux_url, release_dir, f'ffmpeg-{version}-{cpu}-static.tar.xz') + elif system_os == 'windows': + remote_url = os.path.join(cls.windows_download_url, version, f'ffmpeg-{version}-full_build.zip') + + # Download and extract + try: + logger.info(f"Requesting download of ffmpeg-{version}-{system_os}-{cpu}") + download_and_extract_app(remote_url=remote_url, download_location=download_location) + + # naming cleanup to match existing naming convention + if system_os == 'linux': + os.rename(os.path.join(download_location, f'ffmpeg-{version}-{cpu}-static'), + os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}')) + elif system_os == 'windows': + os.rename(os.path.join(download_location, f'ffmpeg-{version}-full_build'), + os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}')) + + except IndexError: + logger.error("Cannot download requested engine") + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + # print(FFMPEGDownloader.download_engine('6.0', '/Users/brett/zordon-uploads/engines/')) + print(FFMPEGDownloader.download_engine(version='6.0', download_location='/Users/brett/zordon-uploads/engines/')) \ No newline at end of file diff --git a/src/engines/engine_manager.py b/src/engines/engine_manager.py index dcd2f49..c999747 100644 --- a/src/engines/engine_manager.py +++ b/src/engines/engine_manager.py @@ -2,6 +2,8 @@ import os import logging import platform import shutil +from .downloaders.blender_downloader import BlenderDownloader +from .downloaders.ffmpeg_downloader import FFMPEGDownloader try: from .blender_engine import Blender @@ -41,7 +43,7 @@ class EngineManager: # Create a dictionary with named keys executable_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender.app'} result_dict = {keys[i]: segments[i] for i in range(min(len(keys), len(segments)))} - result_dict['path'] = os.path.join(cls.engines_path, directory, executable_names[result_dict['system_os']]) + result_dict['path'] = os.path.join(cls.engines_path, directory, executable_names.get(result_dict['system_os'], 'unknown')) result_dict['type'] = 'managed' results.append(result_dict) except FileNotFoundError: @@ -98,15 +100,24 @@ class EngineManager: logger.info(f"Requested download of {engine} {version}, but local copy already exists") return existing_download - if engine == "blender": - from .scripts.blender.blender_downloader import BlenderDownloader - logger.info(f"Requesting download of {engine} {version}") - if BlenderDownloader.download_engine(version, download_location=cls.engines_path, system_os=system_os, cpu=cpu): - return cls.has_engine_version(engine, version, system_os, cpu) - else: - logger.error("Error downloading Engine") + logger.info(f"Requesting download of {engine} {version}") + downloader_classes = { + "blender": BlenderDownloader, + "ffmpeg": FFMPEGDownloader, + # Add more engine types and corresponding downloader classes as needed + } - return None # Return None to indicate an error + # Check if the provided engine type is valid + if engine not in downloader_classes: + logger.error("No valid engine found") + return + + # Get the appropriate downloader class based on the engine type + downloader = downloader_classes[engine] + if downloader.download_engine(version, download_location=cls.engines_path, system_os=system_os, cpu=cpu): + return cls.has_engine_version(engine, version, system_os, cpu) + else: + logger.error(f"Error downloading {engine}") @classmethod def delete_engine_download(cls, engine, version, system_os=None, cpu=None): @@ -125,4 +136,4 @@ if __name__ == '__main__': logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') # print(EngineManager.newest_engine_version('blender', 'macos', 'arm64')) - EngineManager.delete_engine_download('blender', '3.2.1', 'windows', 'x64') + EngineManager.delete_engine_download('blender', '3.2.1', 'macos', 'a') diff --git a/src/engines/scripts/blender/blender_downloader.py b/src/engines/scripts/blender/blender_downloader.py deleted file mode 100644 index 832fca4..0000000 --- a/src/engines/scripts/blender/blender_downloader.py +++ /dev/null @@ -1,206 +0,0 @@ -import logging -import os -import re -import platform -import shutil -import tarfile -import tempfile -import zipfile - -import requests -from tqdm import tqdm - -# url = "https://download.blender.org/release/" -url = "https://ftp.nluug.nl/pub/graphics/blender/release/" # much faster mirror for testing - -logger = logging.getLogger() -supported_formats = ['.zip', '.tar.xz', '.dmg'] - - -class BlenderDownloader: - - @staticmethod - def get_major_versions(): - try: - response = requests.get(url) - response.raise_for_status() - - # Use regex to find all the tags and extract the href attribute - link_pattern = r'Blender(\d+[^<]+)' - link_matches = re.findall(link_pattern, response.text) - - major_versions = [link[-1].strip('/') for link in link_matches] - major_versions.sort(reverse=True) - return major_versions - except requests.exceptions.RequestException as e: - logger.error(f"Error: {e}") - return None - - @staticmethod - def get_minor_versions(major_version, system_os=None, cpu=None): - - base_url = url + 'Blender' + major_version - - response = requests.get(base_url) - response.raise_for_status() - - versions_pattern = r'blender-(?P[\d\.]+)-(?P\w+)-(?P\w+).*' - versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)] - - # Filter to just the supported formats - versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)] - - if system_os: - versions_data = [x for x in versions_data if x['system_os'] == system_os] - if cpu: - versions_data = [x for x in versions_data if x['cpu'] == cpu] - - for v in versions_data: - v['url'] = os.path.join(base_url, v['file']) - - versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True) - return versions_data - - @staticmethod - def find_LTS_versions(): - response = requests.get('https://www.blender.org/download/lts/') - response.raise_for_status() - - lts_pattern = r'https://www.blender.org/download/lts/(\d+-\d+)/' - lts_matches = re.findall(lts_pattern, response.text) - lts_versions = [ver.replace('-', '.') for ver in list(set(lts_matches))] - lts_versions.sort(reverse=True) - - return lts_versions - - @classmethod - def find_most_recent_version(cls, system_os, cpu, lts_only=False): - try: - major_version = cls.find_LTS_versions()[0] if lts_only else cls.get_major_versions()[0] - most_recent = cls.get_minor_versions(major_version, system_os, cpu)[0] - return most_recent - except IndexError: - logger.error("Cannot find a most recent version") - - @classmethod - def download_engine(cls, version, download_location, system_os=None, cpu=None): - system_os = system_os or platform.system().lower().replace('darwin', 'macos') - cpu = cpu or platform.machine().lower().replace('amd64', 'x64') - - try: - logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}") - major_version = '.'.join(version.split('.')[:2]) - minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version] - # we get the URL instead of calculating it ourselves. May change this - - cls.download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location) - except IndexError: - logger.error("Cannot find requested engine") - - @classmethod - def download_and_extract_app(cls, remote_url, download_location): - - binary_path = None - - # Create a temp download directory - temp_download_dir = tempfile.mkdtemp() - downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url)) - - try: - output_dir_name = os.path.basename(remote_url) - for fmt in supported_formats: - output_dir_name = output_dir_name.split(fmt)[0] - - if os.path.exists(os.path.join(download_location, output_dir_name)): - logger.error(f"Engine download for {output_dir_name} already exists") - return - - if not os.path.exists(downloaded_file_path): - # Make a GET request to the URL with stream=True to enable streaming - logger.info(f"Downloading {output_dir_name} from {remote_url}") - response = requests.get(remote_url, stream=True) - - # Check if the request was successful - if response.status_code == 200: - # Get the total file size from the "Content-Length" header - file_size = int(response.headers.get("Content-Length", 0)) - - # Create a progress bar using tqdm - progress_bar = tqdm(total=file_size, unit="B", unit_scale=True) - - # Open a file for writing in binary mode - with open(downloaded_file_path, "wb") as file: - for chunk in response.iter_content(chunk_size=1024): - if chunk: - # Write the chunk to the file - file.write(chunk) - # Update the progress bar - progress_bar.update(len(chunk)) - - # Close the progress bar - progress_bar.close() - logger.info(f"Successfully downloaded {os.path.basename(downloaded_file_path)}") - else: - logger.error(f"Failed to download the file. Status code: {response.status_code}") - - os.makedirs(download_location, exist_ok=True) - - # Extract the downloaded Blender file - # Linux - Process .tar.xz files - if downloaded_file_path.lower().endswith('.tar.xz'): - try: - with tarfile.open(downloaded_file_path, 'r:xz') as tar: - tar.extractall(path=download_location) - os.path.join(download_location, output_dir_name, 'blender') - logger.info(f'Successfully extracted {os.path.basename(downloaded_file_path)} to {download_location}') - except tarfile.TarError as e: - logger.error(f'Error extracting {downloaded_file_path}: {e}') - except FileNotFoundError: - logger.error(f'File not found: {downloaded_file_path}') - - # Windows - Process .zip files - elif downloaded_file_path.lower().endswith('.zip'): - try: - with zipfile.ZipFile(downloaded_file_path, 'r') as zip_ref: - zip_ref.extractall(download_location) - logger.info(f'Successfully extracted {os.path.basename(downloaded_file_path)} to {download_location}') - except zipfile.BadZipFile as e: - logger.error(f'Error: {downloaded_file_path} is not a valid ZIP file.') - except FileNotFoundError: - logger.error(f'File not found: {downloaded_file_path}') - - # macOS - Process .dmg files - elif downloaded_file_path.lower().endswith('.dmg'): - import dmglib - dmg = dmglib.DiskImage(downloaded_file_path) - for mount_point in dmg.attach(): - try: - # Copy the entire .app bundle to the destination directory - shutil.copytree(os.path.join(mount_point, 'Blender.app'), - os.path.join(download_location, output_dir_name, 'Blender.app')) - binary_path = os.path.join(download_location, output_dir_name, 'Blender.app') - logger.info(f'Successfully copied {os.path.basename(downloaded_file_path)} to {download_location}') - except FileNotFoundError: - logger.error(f'Error: The source .app bundle does not exist.') - except PermissionError: - logger.error(f'Error: Permission denied to copy {download_location}.') - except Exception as e: - logger.error(f'An error occurred: {e}') - dmg.detach() - - else: - logger.error("Unknown file. Unable to extract binary.") - - except Exception as e: - logger.exception(e) - - # remove downloaded file on completion - shutil.rmtree(temp_download_dir) - return binary_path - - -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') - - BlenderDownloader.download_engine('3.3.1', download_location="/Users/brett/Desktop/test/releases", system_os='linux', cpu='x64') -