From fab96619480429a4f01ad004343d5fff8c30220f Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Mon, 5 Jun 2023 13:11:18 -0500 Subject: [PATCH 1/4] Update pack_project script for Blender --- lib/render_engines/blender_engine.py | 12 ++---- .../get_file_info.py} | 0 .../scripts/blender/pack_project.py | 41 +++++++++++++++++++ 3 files changed, 44 insertions(+), 9 deletions(-) rename lib/render_engines/scripts/{get_blender_info.py => blender/get_file_info.py} (100%) create mode 100644 lib/render_engines/scripts/blender/pack_project.py diff --git a/lib/render_engines/blender_engine.py b/lib/render_engines/blender_engine.py index c217168..adbf9dd 100644 --- a/lib/render_engines/blender_engine.py +++ b/lib/render_engines/blender_engine.py @@ -61,7 +61,7 @@ class Blender(BaseRenderEngine): scene_info = None try: results = cls.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'scripts', 'get_blender_info.py'), timeout=timeout) + 'scripts', 'blender', 'get_file_info.py'), timeout=timeout) result_text = results.stdout.decode() for line in result_text.splitlines(): if line.startswith('SCENE_DATA:'): @@ -75,15 +75,9 @@ class Blender(BaseRenderEngine): @classmethod def pack_project_file(cls, project_path, timeout=30): # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935 - pack_expression = "import bpy\n" \ - "bpy.ops.file.pack_all()\n" \ - "bpy.ops.file.make_paths_absolute()\n" \ - "myPath = bpy.data.filepath\n" \ - "myPath = str(myPath)\n" \ - "bpy.ops.wm.save_as_mainfile(filepath=myPath[:-6]+'_packed'+myPath[-6:])" - try: - results = Blender.run_python_expression(project_path, pack_expression, timeout=timeout) + results = cls.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'scripts', 'blender', 'pack_project.py'), timeout=timeout) result_text = results.stdout.decode() dir_name = os.path.dirname(project_path) diff --git a/lib/render_engines/scripts/get_blender_info.py b/lib/render_engines/scripts/blender/get_file_info.py similarity index 100% rename from lib/render_engines/scripts/get_blender_info.py rename to lib/render_engines/scripts/blender/get_file_info.py diff --git a/lib/render_engines/scripts/blender/pack_project.py b/lib/render_engines/scripts/blender/pack_project.py new file mode 100644 index 0000000..d4e155f --- /dev/null +++ b/lib/render_engines/scripts/blender/pack_project.py @@ -0,0 +1,41 @@ +import bpy +import os +import shutil +import zipfile + + +def zip_files(file_paths, output_zip_path): + # Create a new Zip file + with zipfile.ZipFile(output_zip_path, 'w') as myzip: + for file_path in file_paths: + # Add each file to the Zip file + myzip.write(file_path) + + +# Get File path +project_path = str(bpy.data.filepath) + +# Pack Files +bpy.ops.file.pack_all() +bpy.ops.file.make_paths_absolute() + +# Temp dir +tmp_dir = os.path.join(os.path.dirname(project_path), 'tmp') +asset_dir = os.path.join(tmp_dir, 'assets') +os.makedirs(tmp_dir, exist_ok=True) + +try: + # Find images we could not pack - usually videos + for img in bpy.data.images: + if not img.packed_file and img.filepath and img.users: + os.makedirs(asset_dir, exist_ok=True) + shutil.copy2(img.filepath, os.path.join(asset_dir, os.path.basename(img.filepath))) + img.filepath = '//' + os.path.join('assets', os.path.basename(img.filepath)) + + # Save Output + bpy.ops.wm.save_as_mainfile(filepath=os.path.join(tmp_dir, os.path.basename(project_path)), compress=True) + + zip_files([os.path.join(tmp_dir, os.path.basename(project_path)), asset_dir], + os.path.join(os.path.dirname(project_path), 'output.zip')) +finally: + os.remove(tmp_dir) From 5b54a1178820cd99adb33a2bd734302c812302f2 Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Mon, 5 Jun 2023 14:46:51 -0500 Subject: [PATCH 2/4] Packing Blender file now creates a zip --- lib/render_engines/blender_engine.py | 8 +++- .../scripts/blender/pack_project.py | 47 +++++++++++-------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/lib/render_engines/blender_engine.py b/lib/render_engines/blender_engine.py index adbf9dd..ea4b08f 100644 --- a/lib/render_engines/blender_engine.py +++ b/lib/render_engines/blender_engine.py @@ -4,6 +4,9 @@ except ImportError: from base_engine import * import json import re +import logging + +logger = logging.getLogger() class Blender(BaseRenderEngine): @@ -76,6 +79,7 @@ class Blender(BaseRenderEngine): def pack_project_file(cls, project_path, timeout=30): # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935 try: + logger.info(f"Starting to pack Blender file: {project_path}") results = cls.run_python_script(project_path, os.path.join(os.path.dirname(os.path.realpath(__file__)), 'scripts', 'blender', 'pack_project.py'), timeout=timeout) @@ -87,10 +91,10 @@ class Blender(BaseRenderEngine): for err in not_found: logger.error(err) - p = re.compile('Info: Saved "(.*)"') + p = re.compile('Saved to: (.*)\n') match = p.search(result_text) if match: - new_path = os.path.join(dir_name, match.group(1)) + new_path = os.path.join(dir_name, match.group(1).strip()) logger.info(f'Blender file packed successfully to {new_path}') return new_path except Exception as e: diff --git a/lib/render_engines/scripts/blender/pack_project.py b/lib/render_engines/scripts/blender/pack_project.py index d4e155f..3849dbe 100644 --- a/lib/render_engines/scripts/blender/pack_project.py +++ b/lib/render_engines/scripts/blender/pack_project.py @@ -4,12 +4,18 @@ import shutil import zipfile -def zip_files(file_paths, output_zip_path): - # Create a new Zip file - with zipfile.ZipFile(output_zip_path, 'w') as myzip: - for file_path in file_paths: - # Add each file to the Zip file - myzip.write(file_path) +def zip_files(paths, output_zip_path): + with zipfile.ZipFile(output_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + for path in paths: + if os.path.isfile(path): + # If the path is a file, add it to the zip + zipf.write(path, arcname=os.path.basename(path)) + elif os.path.isdir(path): + # If the path is a directory, add all its files and subdirectories + for root, dirs, files in os.walk(path): + for file in files: + full_path = os.path.join(root, file) + zipf.write(full_path, arcname=os.path.join(os.path.basename(path), os.path.relpath(full_path, path))) # Get File path @@ -24,18 +30,21 @@ tmp_dir = os.path.join(os.path.dirname(project_path), 'tmp') asset_dir = os.path.join(tmp_dir, 'assets') os.makedirs(tmp_dir, exist_ok=True) -try: - # Find images we could not pack - usually videos - for img in bpy.data.images: - if not img.packed_file and img.filepath and img.users: - os.makedirs(asset_dir, exist_ok=True) - shutil.copy2(img.filepath, os.path.join(asset_dir, os.path.basename(img.filepath))) - img.filepath = '//' + os.path.join('assets', os.path.basename(img.filepath)) +# Find images we could not pack - usually videos +for img in bpy.data.images: + if not img.packed_file and img.filepath and img.users: + os.makedirs(asset_dir, exist_ok=True) + shutil.copy2(img.filepath, os.path.join(asset_dir, os.path.basename(img.filepath))) + print(f"Copied {os.path.basename(img.filepath)} to tmp directory") + img.filepath = '//' + os.path.join('assets', os.path.basename(img.filepath)) - # Save Output - bpy.ops.wm.save_as_mainfile(filepath=os.path.join(tmp_dir, os.path.basename(project_path)), compress=True) +# Save Output +bpy.ops.wm.save_as_mainfile(filepath=os.path.join(tmp_dir, os.path.basename(project_path)), compress=True, relative_remap=False) - zip_files([os.path.join(tmp_dir, os.path.basename(project_path)), asset_dir], - os.path.join(os.path.dirname(project_path), 'output.zip')) -finally: - os.remove(tmp_dir) +# Save Zip +zip_path = os.path.join(os.path.dirname(project_path), f"{os.path.basename(project_path).split('.')[0]}.zip") +zip_files([os.path.join(tmp_dir, os.path.basename(project_path)), asset_dir], zip_path) +print(f'Saved to: {zip_path}') + +# Cleanup +shutil.rmtree(tmp_dir, ignore_errors=True) From 30bd679de8e7efdbeae234613f8cf5c8c48cf5d4 Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Mon, 5 Jun 2023 15:52:49 -0500 Subject: [PATCH 3/4] Process and unzip uploaded zip files --- lib/server/job_server.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/lib/server/job_server.py b/lib/server/job_server.py index d99d7bb..f506a90 100755 --- a/lib/server/job_server.py +++ b/lib/server/job_server.py @@ -7,6 +7,7 @@ import shutil import socket import threading import time +import zipfile from datetime import datetime from zipfile import ZipFile @@ -297,7 +298,7 @@ def add_job_handler(): job_dir = os.path.join(server.config['UPLOAD_FOLDER'], '_'.join( [datetime.now().strftime("%Y.%m.%d_%H.%M.%S"), jobs_list[0]['renderer'], - uploaded_file.filename])) + os.path.splitext(uploaded_file.filename)[0]])) os.makedirs(job_dir, exist_ok=True) upload_dir = os.path.join(job_dir, 'source') @@ -350,8 +351,7 @@ def add_job(job_params, remove_job_dir_on_failure=False): args = job_params.get('args', {}) client = job_params.get('client', None) or RenderQueue.hostname force_start = job_params.get('force_start', False) - custom_id = None - job_dir = None + job_dir = None # todo: I dont think this gets set anywhere # check for minimum render requirements if None in [renderer, input_path, output_path]: @@ -359,6 +359,34 @@ def add_job(job_params, remove_job_dir_on_failure=False): logger.error(err_msg) return {'error': err_msg, 'code': 400} + # process uploaded zip files + if '.zip' in input_path: + zip_path = input_path + work_path = os.path.dirname(zip_path) + try: + with zipfile.ZipFile(zip_path, 'r') as myzip: + myzip.extractall(os.path.dirname(zip_path)) + + project_files = [x for x in os.listdir(work_path) if os.path.isfile(os.path.join(work_path, x))] + project_files = [x for x in project_files if '.zip' not in x] + supported_exts = RenderWorkerFactory.class_for_name(renderer).engine.supported_extensions + if supported_exts: + project_files = [file for file in project_files if any(file.endswith(ext) for ext in supported_exts)] + + if len(project_files) != 1: # we have to narrow down to 1 main project file, otherwise error + return {'error': f'Cannot find valid project file in {os.path.basename(zip_path)}'}, 400 + + extracted_project_path = os.path.join(work_path, project_files[0]) + logger.info(f"Extracted zip file to {extracted_project_path}") + input_path = extracted_project_path + except (zipfile.BadZipFile, zipfile.LargeZipFile) as e: + err_msg = f"Error processing zip file: {e}" + logger.error(err_msg) + return {'error': err_msg, 'code': 400} + finally: + os.remove(zip_path) + logger.info(f"Removed zip: {zip_path}") + # local renders if client == RenderQueue.hostname: logger.info(f"Creating job locally - {name if name else input_path}") From f7bda499349aa3c93eabbc9c7b857e7158e55ffa Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Mon, 5 Jun 2023 16:45:13 -0500 Subject: [PATCH 4/4] Minor addition to pack_project.py script --- lib/render_engines/scripts/blender/pack_project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/render_engines/scripts/blender/pack_project.py b/lib/render_engines/scripts/blender/pack_project.py index 3849dbe..bc52094 100644 --- a/lib/render_engines/scripts/blender/pack_project.py +++ b/lib/render_engines/scripts/blender/pack_project.py @@ -44,7 +44,10 @@ bpy.ops.wm.save_as_mainfile(filepath=os.path.join(tmp_dir, os.path.basename(proj # Save Zip zip_path = os.path.join(os.path.dirname(project_path), f"{os.path.basename(project_path).split('.')[0]}.zip") zip_files([os.path.join(tmp_dir, os.path.basename(project_path)), asset_dir], zip_path) -print(f'Saved to: {zip_path}') +if os.path.exists(zip_path): + print(f'Saved to: {zip_path}') +else: + print("Error saving zip file!") # Cleanup shutil.rmtree(tmp_dir, ignore_errors=True)