FFMPEG downloader (#36)

* Add ffmpeg_downloader.py

* Move shared download logic to downloader_core.py

* Added Windows support and some misc cleanup

* Fix issue with copying contents of .dmg files

* Add FFMPEGDownloader to engine_manager.py
This commit is contained in:
2023-10-20 18:46:54 -05:00
committed by GitHub
parent 7d1ecf1fa5
commit af5eda97c5
6 changed files with 373 additions and 216 deletions

View File

View File

@@ -0,0 +1,102 @@
import logging
import os
import platform
import re
import requests
from downloader_core import download_and_extract_app
# url = "https://download.blender.org/release/"
url = "https://ftp.nluug.nl/pub/graphics/blender/release/" # much faster mirror for testing
logger = logging.getLogger()
supported_formats = ['.zip', '.tar.xz', '.dmg']
class BlenderDownloader:
@staticmethod
def get_major_versions():
try:
response = requests.get(url)
response.raise_for_status()
# Use regex to find all the <a> tags and extract the href attribute
link_pattern = r'<a href="([^"]+)">Blender(\d+[^<]+)</a>'
link_matches = re.findall(link_pattern, response.text)
major_versions = [link[-1].strip('/') for link in link_matches]
major_versions.sort(reverse=True)
return major_versions
except requests.exceptions.RequestException as e:
logger.error(f"Error: {e}")
return None
@staticmethod
def get_minor_versions(major_version, system_os=None, cpu=None):
base_url = url + 'Blender' + major_version
response = requests.get(base_url)
response.raise_for_status()
versions_pattern = r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)]
# Filter to just the supported formats
versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)]
if system_os:
versions_data = [x for x in versions_data if x['system_os'] == system_os]
if cpu:
versions_data = [x for x in versions_data if x['cpu'] == cpu]
for v in versions_data:
v['url'] = os.path.join(base_url, v['file'])
versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True)
return versions_data
@staticmethod
def find_LTS_versions():
response = requests.get('https://www.blender.org/download/lts/')
response.raise_for_status()
lts_pattern = r'https://www.blender.org/download/lts/(\d+-\d+)/'
lts_matches = re.findall(lts_pattern, response.text)
lts_versions = [ver.replace('-', '.') for ver in list(set(lts_matches))]
lts_versions.sort(reverse=True)
return lts_versions
@classmethod
def find_most_recent_version(cls, system_os, cpu, lts_only=False):
try:
major_version = cls.find_LTS_versions()[0] if lts_only else cls.get_major_versions()[0]
most_recent = cls.get_minor_versions(major_version, system_os, cpu)[0]
return most_recent
except IndexError:
logger.error("Cannot find a most recent version")
@classmethod
def download_engine(cls, version, download_location, system_os=None, cpu=None):
system_os = system_os or platform.system().lower().replace('darwin', 'macos')
cpu = cpu or platform.machine().lower().replace('amd64', 'x64')
try:
logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}")
major_version = '.'.join(version.split('.')[:2])
minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version]
# we get the URL instead of calculating it ourselves. May change this
download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location)
except IndexError:
logger.error("Cannot find requested engine")
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
print(BlenderDownloader.get_major_versions())

View File

@@ -0,0 +1,138 @@
import logging
import os
import shutil
import tarfile
import tempfile
import zipfile
import requests
from tqdm import tqdm
supported_formats = ['.zip', '.tar.xz', '.dmg']
logger = logging.getLogger()
def download_and_extract_app(remote_url, download_location):
# Create a temp download directory
temp_download_dir = tempfile.mkdtemp()
temp_downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url))
try:
output_dir_name = os.path.basename(remote_url)
for fmt in supported_formats:
output_dir_name = output_dir_name.split(fmt)[0]
if os.path.exists(os.path.join(download_location, output_dir_name)):
logger.error(f"Engine download for {output_dir_name} already exists")
return
if not os.path.exists(temp_downloaded_file_path):
# Make a GET request to the URL with stream=True to enable streaming
logger.info(f"Downloading {output_dir_name} from {remote_url}")
response = requests.get(remote_url, stream=True)
# Check if the request was successful
if response.status_code == 200:
# Get the total file size from the "Content-Length" header
file_size = int(response.headers.get("Content-Length", 0))
# Create a progress bar using tqdm
progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
# Open a file for writing in binary mode
with open(temp_downloaded_file_path, "wb") as file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
# Write the chunk to the file
file.write(chunk)
# Update the progress bar
progress_bar.update(len(chunk))
# Close the progress bar
progress_bar.close()
logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}")
else:
logger.error(f"Failed to download the file. Status code: {response.status_code}")
os.makedirs(download_location, exist_ok=True)
# Extract the downloaded file
# Process .tar.xz files
if temp_downloaded_file_path.lower().endswith('.tar.xz'):
try:
with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar:
tar.extractall(path=download_location)
logger.info(
f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
except tarfile.TarError as e:
logger.error(f'Error extracting {temp_downloaded_file_path}: {e}')
except FileNotFoundError:
logger.error(f'File not found: {temp_downloaded_file_path}')
# Process .zip files
elif temp_downloaded_file_path.lower().endswith('.zip'):
try:
with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref:
zip_ref.extractall(download_location)
logger.info(
f'Successfully extracted {os.path.basename(temp_downloaded_file_path)} to {download_location}')
except zipfile.BadZipFile as e:
logger.error(f'Error: {temp_downloaded_file_path} is not a valid ZIP file.')
except FileNotFoundError:
logger.error(f'File not found: {temp_downloaded_file_path}')
# Process .dmg files (macOS only)
elif temp_downloaded_file_path.lower().endswith('.dmg'):
import dmglib
dmg = dmglib.DiskImage(temp_downloaded_file_path)
for mount_point in dmg.attach():
try:
copy_directory_contents(mount_point, os.path.join(download_location, output_dir_name))
logger.info(f'Successfully copied {os.path.basename(temp_downloaded_file_path)} to {download_location}')
except FileNotFoundError:
logger.error(f'Error: The source .app bundle does not exist.')
except PermissionError:
logger.error(f'Error: Permission denied to copy {download_location}.')
except Exception as e:
logger.error(f'An error occurred: {e}')
dmg.detach()
else:
logger.error("Unknown file. Unable to extract binary.")
except Exception as e:
logger.exception(e)
# remove downloaded file on completion
shutil.rmtree(temp_download_dir)
return download_location
# Function to copy directory contents but ignore symbolic links and hidden files
def copy_directory_contents(src_dir, dest_dir):
try:
# Create the destination directory if it doesn't exist
os.makedirs(dest_dir, exist_ok=True)
for item in os.listdir(src_dir):
item_path = os.path.join(src_dir, item)
# Ignore symbolic links
if os.path.islink(item_path):
continue
# Ignore hidden files or directories (those starting with a dot)
if not item.startswith('.'):
dest_item_path = os.path.join(dest_dir, item)
# If it's a directory, recursively copy its contents
if os.path.isdir(item_path):
copy_directory_contents(item_path, dest_item_path)
else:
# Otherwise, copy the file
shutil.copy2(item_path, dest_item_path)
except Exception as e:
logger.exception(f"Error copying directory contents: {e}")

View File

@@ -0,0 +1,112 @@
import logging
import os
import platform
import re
import requests
from downloader_core import download_and_extract_app
logger = logging.getLogger()
supported_formats = ['.zip', '.tar.xz', '.dmg']
class FFMPEGDownloader:
# macOS FFMPEG mirror maintained by Evermeet - https://evermeet.cx/ffmpeg/
macos_url = "https://evermeet.cx/pub/ffmpeg/"
# Linux FFMPEG mirror maintained by John van Sickle - https://johnvansickle.com/ffmpeg/
linux_url = "https://johnvansickle.com/ffmpeg/"
# macOS FFMPEG mirror maintained by GyanD - https://www.gyan.dev/ffmpeg/builds/
windows_download_url = "https://github.com/GyanD/codexffmpeg/releases/download/"
windows_api_url = "https://api.github.com/repos/GyanD/codexffmpeg/releases"
@classmethod
def get_macos_versions(cls):
response = requests.get(cls.macos_url)
response.raise_for_status()
link_pattern = r'>(.*\.zip)[^\.]'
link_matches = re.findall(link_pattern, response.text)
return [link.split('-')[-1].split('.zip')[0] for link in link_matches]
@classmethod
def get_linux_versions(cls):
# Link 1 / 2 - Current Version
response = requests.get(cls.linux_url)
response.raise_for_status()
current_release = re.findall(r'release: ([\w\.]+)', response.text)[0]
# Link 2 / 2 - Previous Versions
response = requests.get(os.path.join(cls.linux_url, 'old-releases'))
response.raise_for_status()
releases = list(set(re.findall(r'href="ffmpeg-([\w\.]+)-.*">ffmpeg', response.text)))
releases.sort(reverse=True)
releases.insert(0, current_release)
return releases
@classmethod
def get_windows_versions(cls):
response = requests.get(cls.windows_api_url)
response.raise_for_status()
versions = []
all_git_releases = response.json()
for item in all_git_releases:
if re.match(r'^[0-9.]+$', item['tag_name']):
versions.append(item['tag_name'])
return versions
@classmethod
def find_most_recent_version(cls, system_os, cpu, lts_only=False):
pass
@classmethod
def download_engine(cls, version, download_location, system_os=None, cpu=None):
system_os = system_os or platform.system().lower().replace('darwin', 'macos')
cpu = cpu or platform.machine().lower().replace('amd64', 'x64')
# Verify requested version is available
remote_url = None
versions_per_os = {'linux': cls.get_linux_versions, 'macos': cls.get_macos_versions, 'windows': cls.get_windows_versions}
if not versions_per_os.get(system_os):
logger.error(f"Cannot find version list for {system_os}")
return
if version not in versions_per_os[system_os]():
logger.error(f"Cannot find FFMPEG version {version} for {system_os}")
# Platform specific naming cleanup
if system_os == 'macos':
remote_url = os.path.join(cls.macos_url, f"ffmpeg-{version}.zip")
download_location = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}') # override location to match linux
elif system_os == 'linux':
release_dir = 'releases' if version == cls.get_linux_versions()[0] else 'old-releases'
remote_url = os.path.join(cls.linux_url, release_dir, f'ffmpeg-{version}-{cpu}-static.tar.xz')
elif system_os == 'windows':
remote_url = os.path.join(cls.windows_download_url, version, f'ffmpeg-{version}-full_build.zip')
# Download and extract
try:
logger.info(f"Requesting download of ffmpeg-{version}-{system_os}-{cpu}")
download_and_extract_app(remote_url=remote_url, download_location=download_location)
# naming cleanup to match existing naming convention
if system_os == 'linux':
os.rename(os.path.join(download_location, f'ffmpeg-{version}-{cpu}-static'),
os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}'))
elif system_os == 'windows':
os.rename(os.path.join(download_location, f'ffmpeg-{version}-full_build'),
os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}'))
except IndexError:
logger.error("Cannot download requested engine")
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# print(FFMPEGDownloader.download_engine('6.0', '/Users/brett/zordon-uploads/engines/'))
print(FFMPEGDownloader.download_engine(version='6.0', download_location='/Users/brett/zordon-uploads/engines/'))

View File

@@ -2,6 +2,8 @@ import os
import logging
import platform
import shutil
from .downloaders.blender_downloader import BlenderDownloader
from .downloaders.ffmpeg_downloader import FFMPEGDownloader
try:
from .blender_engine import Blender
@@ -41,7 +43,7 @@ class EngineManager:
# Create a dictionary with named keys
executable_names = {'linux': 'blender', 'windows': 'blender.exe', 'macos': 'Blender.app'}
result_dict = {keys[i]: segments[i] for i in range(min(len(keys), len(segments)))}
result_dict['path'] = os.path.join(cls.engines_path, directory, executable_names[result_dict['system_os']])
result_dict['path'] = os.path.join(cls.engines_path, directory, executable_names.get(result_dict['system_os'], 'unknown'))
result_dict['type'] = 'managed'
results.append(result_dict)
except FileNotFoundError:
@@ -98,15 +100,24 @@ class EngineManager:
logger.info(f"Requested download of {engine} {version}, but local copy already exists")
return existing_download
if engine == "blender":
from .scripts.blender.blender_downloader import BlenderDownloader
logger.info(f"Requesting download of {engine} {version}")
if BlenderDownloader.download_engine(version, download_location=cls.engines_path, system_os=system_os, cpu=cpu):
downloader_classes = {
"blender": BlenderDownloader,
"ffmpeg": FFMPEGDownloader,
# Add more engine types and corresponding downloader classes as needed
}
# Check if the provided engine type is valid
if engine not in downloader_classes:
logger.error("No valid engine found")
return
# Get the appropriate downloader class based on the engine type
downloader = downloader_classes[engine]
if downloader.download_engine(version, download_location=cls.engines_path, system_os=system_os, cpu=cpu):
return cls.has_engine_version(engine, version, system_os, cpu)
else:
logger.error("Error downloading Engine")
return None # Return None to indicate an error
logger.error(f"Error downloading {engine}")
@classmethod
def delete_engine_download(cls, engine, version, system_os=None, cpu=None):
@@ -125,4 +136,4 @@ if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# print(EngineManager.newest_engine_version('blender', 'macos', 'arm64'))
EngineManager.delete_engine_download('blender', '3.2.1', 'windows', 'x64')
EngineManager.delete_engine_download('blender', '3.2.1', 'macos', 'a')

View File

@@ -1,206 +0,0 @@
import logging
import os
import re
import platform
import shutil
import tarfile
import tempfile
import zipfile
import requests
from tqdm import tqdm
# url = "https://download.blender.org/release/"
url = "https://ftp.nluug.nl/pub/graphics/blender/release/" # much faster mirror for testing
logger = logging.getLogger()
supported_formats = ['.zip', '.tar.xz', '.dmg']
class BlenderDownloader:
@staticmethod
def get_major_versions():
try:
response = requests.get(url)
response.raise_for_status()
# Use regex to find all the <a> tags and extract the href attribute
link_pattern = r'<a href="([^"]+)">Blender(\d+[^<]+)</a>'
link_matches = re.findall(link_pattern, response.text)
major_versions = [link[-1].strip('/') for link in link_matches]
major_versions.sort(reverse=True)
return major_versions
except requests.exceptions.RequestException as e:
logger.error(f"Error: {e}")
return None
@staticmethod
def get_minor_versions(major_version, system_os=None, cpu=None):
base_url = url + 'Blender' + major_version
response = requests.get(base_url)
response.raise_for_status()
versions_pattern = r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)]
# Filter to just the supported formats
versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)]
if system_os:
versions_data = [x for x in versions_data if x['system_os'] == system_os]
if cpu:
versions_data = [x for x in versions_data if x['cpu'] == cpu]
for v in versions_data:
v['url'] = os.path.join(base_url, v['file'])
versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True)
return versions_data
@staticmethod
def find_LTS_versions():
response = requests.get('https://www.blender.org/download/lts/')
response.raise_for_status()
lts_pattern = r'https://www.blender.org/download/lts/(\d+-\d+)/'
lts_matches = re.findall(lts_pattern, response.text)
lts_versions = [ver.replace('-', '.') for ver in list(set(lts_matches))]
lts_versions.sort(reverse=True)
return lts_versions
@classmethod
def find_most_recent_version(cls, system_os, cpu, lts_only=False):
try:
major_version = cls.find_LTS_versions()[0] if lts_only else cls.get_major_versions()[0]
most_recent = cls.get_minor_versions(major_version, system_os, cpu)[0]
return most_recent
except IndexError:
logger.error("Cannot find a most recent version")
@classmethod
def download_engine(cls, version, download_location, system_os=None, cpu=None):
system_os = system_os or platform.system().lower().replace('darwin', 'macos')
cpu = cpu or platform.machine().lower().replace('amd64', 'x64')
try:
logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}")
major_version = '.'.join(version.split('.')[:2])
minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version]
# we get the URL instead of calculating it ourselves. May change this
cls.download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location)
except IndexError:
logger.error("Cannot find requested engine")
@classmethod
def download_and_extract_app(cls, remote_url, download_location):
binary_path = None
# Create a temp download directory
temp_download_dir = tempfile.mkdtemp()
downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url))
try:
output_dir_name = os.path.basename(remote_url)
for fmt in supported_formats:
output_dir_name = output_dir_name.split(fmt)[0]
if os.path.exists(os.path.join(download_location, output_dir_name)):
logger.error(f"Engine download for {output_dir_name} already exists")
return
if not os.path.exists(downloaded_file_path):
# Make a GET request to the URL with stream=True to enable streaming
logger.info(f"Downloading {output_dir_name} from {remote_url}")
response = requests.get(remote_url, stream=True)
# Check if the request was successful
if response.status_code == 200:
# Get the total file size from the "Content-Length" header
file_size = int(response.headers.get("Content-Length", 0))
# Create a progress bar using tqdm
progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
# Open a file for writing in binary mode
with open(downloaded_file_path, "wb") as file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
# Write the chunk to the file
file.write(chunk)
# Update the progress bar
progress_bar.update(len(chunk))
# Close the progress bar
progress_bar.close()
logger.info(f"Successfully downloaded {os.path.basename(downloaded_file_path)}")
else:
logger.error(f"Failed to download the file. Status code: {response.status_code}")
os.makedirs(download_location, exist_ok=True)
# Extract the downloaded Blender file
# Linux - Process .tar.xz files
if downloaded_file_path.lower().endswith('.tar.xz'):
try:
with tarfile.open(downloaded_file_path, 'r:xz') as tar:
tar.extractall(path=download_location)
os.path.join(download_location, output_dir_name, 'blender')
logger.info(f'Successfully extracted {os.path.basename(downloaded_file_path)} to {download_location}')
except tarfile.TarError as e:
logger.error(f'Error extracting {downloaded_file_path}: {e}')
except FileNotFoundError:
logger.error(f'File not found: {downloaded_file_path}')
# Windows - Process .zip files
elif downloaded_file_path.lower().endswith('.zip'):
try:
with zipfile.ZipFile(downloaded_file_path, 'r') as zip_ref:
zip_ref.extractall(download_location)
logger.info(f'Successfully extracted {os.path.basename(downloaded_file_path)} to {download_location}')
except zipfile.BadZipFile as e:
logger.error(f'Error: {downloaded_file_path} is not a valid ZIP file.')
except FileNotFoundError:
logger.error(f'File not found: {downloaded_file_path}')
# macOS - Process .dmg files
elif downloaded_file_path.lower().endswith('.dmg'):
import dmglib
dmg = dmglib.DiskImage(downloaded_file_path)
for mount_point in dmg.attach():
try:
# Copy the entire .app bundle to the destination directory
shutil.copytree(os.path.join(mount_point, 'Blender.app'),
os.path.join(download_location, output_dir_name, 'Blender.app'))
binary_path = os.path.join(download_location, output_dir_name, 'Blender.app')
logger.info(f'Successfully copied {os.path.basename(downloaded_file_path)} to {download_location}')
except FileNotFoundError:
logger.error(f'Error: The source .app bundle does not exist.')
except PermissionError:
logger.error(f'Error: Permission denied to copy {download_location}.')
except Exception as e:
logger.error(f'An error occurred: {e}')
dmg.detach()
else:
logger.error("Unknown file. Unable to extract binary.")
except Exception as e:
logger.exception(e)
# remove downloaded file on completion
shutil.rmtree(temp_download_dir)
return binary_path
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
BlenderDownloader.download_engine('3.3.1', download_location="/Users/brett/Desktop/test/releases", system_os='linux', cpu='x64')