brett/Zordon
1
0
Fork 0
mirror of https://github.com/blw1138/Zordon.git synced 2025-12-19 01:38:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Downloadable engines (#34)

Browse Source 
* Add blender_downloader.py

* Add engine_manager.py

* Add additional methods to engine_manager.py

* Refactor file layout to make engines on par with workers

* Add system platform info to status response

* Default to using system platform / cpu if none are provided

* Add API to download an engine and some general cleanup

* Add method to delete downloaded engine

* Add API calls to download engines and delete downloads

* Misc fixes
This commit is contained in:
Brett
2023-10-20 15:05:29 -05:00
committed by GitHub
parent 4563dcb255
commit 7d1ecf1fa5
21 changed files with 439 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
src/engines/scripts/blender/__init__.py Normal file
View File

206
src/engines/scripts/blender/blender_downloader.py Normal file
View File

@@ -0,0 +1,206 @@
import logging
import os
import re
import platform
import shutil
import tarfile
import tempfile
import zipfile
import requests
from tqdm import tqdm
# url = "https://download.blender.org/release/"
url = "https://ftp.nluug.nl/pub/graphics/blender/release/" # much faster mirror for testing
logger = logging.getLogger()
supported_formats = ['.zip', '.tar.xz', '.dmg']
class BlenderDownloader:
@staticmethod
def get_major_versions():
try:
response = requests.get(url)
response.raise_for_status()
# Use regex to find all the <a> tags and extract the href attribute
link_pattern = r'<a href="([^"]+)">Blender(\d+[^<]+)</a>'
link_matches = re.findall(link_pattern, response.text)
major_versions = [link[-1].strip('/') for link in link_matches]
major_versions.sort(reverse=True)
return major_versions
except requests.exceptions.RequestException as e:
logger.error(f"Error: {e}")
return None
@staticmethod
def get_minor_versions(major_version, system_os=None, cpu=None):
base_url = url + 'Blender' + major_version
response = requests.get(base_url)
response.raise_for_status()
versions_pattern = r'<a href="(?P<file>[^"]+)">blender-(?P<version>[\d\.]+)-(?P<system_os>\w+)-(?P<cpu>\w+).*</a>'
versions_data = [match.groupdict() for match in re.finditer(versions_pattern, response.text)]
# Filter to just the supported formats
versions_data = [item for item in versions_data if any(item["file"].endswith(ext) for ext in supported_formats)]
if system_os:
versions_data = [x for x in versions_data if x['system_os'] == system_os]
if cpu:
versions_data = [x for x in versions_data if x['cpu'] == cpu]
for v in versions_data:
v['url'] = os.path.join(base_url, v['file'])
versions_data = sorted(versions_data, key=lambda x: x['version'], reverse=True)
return versions_data
@staticmethod
def find_LTS_versions():
response = requests.get('https://www.blender.org/download/lts/')
response.raise_for_status()
lts_pattern = r'https://www.blender.org/download/lts/(\d+-\d+)/'
lts_matches = re.findall(lts_pattern, response.text)
lts_versions = [ver.replace('-', '.') for ver in list(set(lts_matches))]
lts_versions.sort(reverse=True)
return lts_versions
@classmethod
def find_most_recent_version(cls, system_os, cpu, lts_only=False):
try:
major_version = cls.find_LTS_versions()[0] if lts_only else cls.get_major_versions()[0]
most_recent = cls.get_minor_versions(major_version, system_os, cpu)[0]
return most_recent
except IndexError:
logger.error("Cannot find a most recent version")
@classmethod
def download_engine(cls, version, download_location, system_os=None, cpu=None):
system_os = system_os or platform.system().lower().replace('darwin', 'macos')
cpu = cpu or platform.machine().lower().replace('amd64', 'x64')
try:
logger.info(f"Requesting download of blender-{version}-{system_os}-{cpu}")
major_version = '.'.join(version.split('.')[:2])
minor_versions = [x for x in cls.get_minor_versions(major_version, system_os, cpu) if x['version'] == version]
# we get the URL instead of calculating it ourselves. May change this
cls.download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location)
except IndexError:
logger.error("Cannot find requested engine")
@classmethod
def download_and_extract_app(cls, remote_url, download_location):
binary_path = None
# Create a temp download directory
temp_download_dir = tempfile.mkdtemp()
downloaded_file_path = os.path.join(temp_download_dir, os.path.basename(remote_url))
try:
output_dir_name = os.path.basename(remote_url)
for fmt in supported_formats:
output_dir_name = output_dir_name.split(fmt)[0]
if os.path.exists(os.path.join(download_location, output_dir_name)):
logger.error(f"Engine download for {output_dir_name} already exists")
return
if not os.path.exists(downloaded_file_path):
# Make a GET request to the URL with stream=True to enable streaming
logger.info(f"Downloading {output_dir_name} from {remote_url}")
response = requests.get(remote_url, stream=True)
# Check if the request was successful
if response.status_code == 200:
# Get the total file size from the "Content-Length" header
file_size = int(response.headers.get("Content-Length", 0))
# Create a progress bar using tqdm
progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
# Open a file for writing in binary mode
with open(downloaded_file_path, "wb") as file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
# Write the chunk to the file
file.write(chunk)
# Update the progress bar
progress_bar.update(len(chunk))
# Close the progress bar
progress_bar.close()
logger.info(f"Successfully downloaded {os.path.basename(downloaded_file_path)}")
else:
logger.error(f"Failed to download the file. Status code: {response.status_code}")
os.makedirs(download_location, exist_ok=True)
# Extract the downloaded Blender file
# Linux - Process .tar.xz files
if downloaded_file_path.lower().endswith('.tar.xz'):
try:
with tarfile.open(downloaded_file_path, 'r:xz') as tar:
tar.extractall(path=download_location)
os.path.join(download_location, output_dir_name, 'blender')
logger.info(f'Successfully extracted {os.path.basename(downloaded_file_path)} to {download_location}')
except tarfile.TarError as e:
logger.error(f'Error extracting {downloaded_file_path}: {e}')
except FileNotFoundError:
logger.error(f'File not found: {downloaded_file_path}')
# Windows - Process .zip files
elif downloaded_file_path.lower().endswith('.zip'):
try:
with zipfile.ZipFile(downloaded_file_path, 'r') as zip_ref:
zip_ref.extractall(download_location)
logger.info(f'Successfully extracted {os.path.basename(downloaded_file_path)} to {download_location}')
except zipfile.BadZipFile as e:
logger.error(f'Error: {downloaded_file_path} is not a valid ZIP file.')
except FileNotFoundError:
logger.error(f'File not found: {downloaded_file_path}')
# macOS - Process .dmg files
elif downloaded_file_path.lower().endswith('.dmg'):
import dmglib
dmg = dmglib.DiskImage(downloaded_file_path)
for mount_point in dmg.attach():
try:
# Copy the entire .app bundle to the destination directory
shutil.copytree(os.path.join(mount_point, 'Blender.app'),
os.path.join(download_location, output_dir_name, 'Blender.app'))
binary_path = os.path.join(download_location, output_dir_name, 'Blender.app')
logger.info(f'Successfully copied {os.path.basename(downloaded_file_path)} to {download_location}')
except FileNotFoundError:
logger.error(f'Error: The source .app bundle does not exist.')
except PermissionError:
logger.error(f'Error: Permission denied to copy {download_location}.')
except Exception as e:
logger.error(f'An error occurred: {e}')
dmg.detach()
else:
logger.error("Unknown file. Unable to extract binary.")
except Exception as e:
logger.exception(e)
# remove downloaded file on completion
shutil.rmtree(temp_download_dir)
return binary_path
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
BlenderDownloader.download_engine('3.3.1', download_location="/Users/brett/Desktop/test/releases", system_os='linux', cpu='x64')

29
src/engines/scripts/blender/get_file_info.py Normal file
View File

@@ -0,0 +1,29 @@
import json
import bpy
# Get all cameras
cameras = []
for cam_obj in bpy.data.cameras:
user_map = bpy.data.user_map(subset={cam_obj}, value_types={'OBJECT'})
for data_obj in user_map[cam_obj]:
cam = {'name': data_obj.name,
'cam_name': cam_obj.name,
'cam_name_full': cam_obj.name_full,
'lens': cam_obj.lens,
'lens_unit': cam_obj.lens_unit,
'sensor_height': cam_obj.sensor_height,
'sensor_width': cam_obj.sensor_width}
cameras.append(cam)
scene = bpy.data.scenes[0]
data = {'cameras': cameras,
'engine': scene.render.engine,
'frame_start': scene.frame_start,
'frame_end': scene.frame_end,
'resolution_x': scene.render.resolution_x,
'resolution_y': scene.render.resolution_y,
'resolution_percentage': scene.render.resolution_percentage,
'fps': scene.render.fps}
data_string = json.dumps(data)
print("SCENE_DATA:" + data_string)

53
src/engines/scripts/blender/pack_project.py Normal file
View File

@@ -0,0 +1,53 @@
import bpy
import os
import shutil
import zipfile
def zip_files(paths, output_zip_path):
with zipfile.ZipFile(output_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
for path in paths:
if os.path.isfile(path):
# If the path is a file, add it to the zip
zipf.write(path, arcname=os.path.basename(path))
elif os.path.isdir(path):
# If the path is a directory, add all its files and subdirectories
for root, dirs, files in os.walk(path):
for file in files:
full_path = os.path.join(root, file)
zipf.write(full_path, arcname=os.path.join(os.path.basename(path), os.path.relpath(full_path, path)))
# Get File path
project_path = str(bpy.data.filepath)
# Pack Files
bpy.ops.file.pack_all()
bpy.ops.file.make_paths_absolute()
# Temp dir
tmp_dir = os.path.join(os.path.dirname(project_path), 'tmp')
asset_dir = os.path.join(tmp_dir, 'assets')
os.makedirs(tmp_dir, exist_ok=True)
# Find images we could not pack - usually videos
for img in bpy.data.images:
if not img.packed_file and img.filepath and img.users:
os.makedirs(asset_dir, exist_ok=True)
shutil.copy2(img.filepath, os.path.join(asset_dir, os.path.basename(img.filepath)))
print(f"Copied {os.path.basename(img.filepath)} to tmp directory")
img.filepath = '//' + os.path.join('assets', os.path.basename(img.filepath))
# Save Output
bpy.ops.wm.save_as_mainfile(filepath=os.path.join(tmp_dir, os.path.basename(project_path)), compress=True, relative_remap=False)
# Save Zip
zip_path = os.path.join(os.path.dirname(project_path), f"{os.path.basename(project_path).split('.')[0]}.zip")
zip_files([os.path.join(tmp_dir, os.path.basename(project_path)), asset_dir], zip_path)
if os.path.exists(zip_path):
print(f'Saved to: {zip_path}')
else:
print("Error saving zip file!")
# Cleanup
shutil.rmtree(tmp_dir, ignore_errors=True)