From 8ae50e2431a237654baca6691bbbac68832f7cdf Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Wed, 12 Oct 2022 00:09:27 -0700 Subject: [PATCH] Add local to client transfer and Blender texture packing --- utilities/blender_worker.py | 40 ++++++++++++++--- zordon_server.py | 85 +++++++++++++++++++++++++++++-------- 2 files changed, 101 insertions(+), 24 deletions(-) diff --git a/utilities/blender_worker.py b/utilities/blender_worker.py index ebfa503..75d335a 100644 --- a/utilities/blender_worker.py +++ b/utilities/blender_worker.py @@ -1,7 +1,9 @@ #! /usr/bin/python -from utilities.render_worker import * +import re import time +from utilities.render_worker import * + SUPPORTED_FORMATS = ['TGA', 'RAWTGA', 'JPEG', 'IRIS', 'IRIZ', 'AVIRAW', 'AVIJPEG', 'PNG', 'BMP', 'HDR', 'TIFF', 'OPEN_EXR', 'OPEN_EXR_MULTILAYER', 'MPEG', 'CINEON', 'DPX', 'DDS', 'JP2'] @@ -48,10 +50,10 @@ class BlenderRenderWorker(BaseRenderWorker): if self.render_all_frames: cmd = [self.renderer_path(), '-b', self.input, '-E', self.engine, '-o', self.output, - '-F', self.format, '-a'] + '-F', self.format, '-a'] else: cmd = [self.renderer_path(), '-b', self.input, '-E', self.engine, '-o', self.output, - '-F', self.format, '-f', str(self.frame)] + '-F', self.format, '-f', str(self.frame)] return cmd def _parse_stdout(self, line): @@ -67,7 +69,7 @@ class BlenderRenderWorker(BaseRenderWorker): sample_string = line.split('|')[-1].strip() if "sample" in sample_string.lower(): - samples = re.sub(r'[^/d/]', '', sample_string) + samples = re.sub(r'[^\d/]', '', sample_string) self.frame_percent_complete = int(samples.split('/')[0]) / int(samples.split('/')[-1]) @@ -106,14 +108,38 @@ class BlenderRenderWorker(BaseRenderWorker): if self.total_frames <= 1: return self.frame_percent_complete else: - return (self.current_frame / self.total_frames) +\ + return (self.current_frame / self.total_frames) + \ (self.frame_percent_complete * (self.current_frame / self.total_frames)) +def pack_blender_files(path): + # Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935 + pack_script = "import bpy\nbpy.ops.file.pack_all()\nmyPath = bpy.data.filepath\nmyPath = str(myPath)\n" \ + "bpy.ops.wm.save_as_mainfile(filepath=myPath[:-6]+'_packed'+myPath[-6:])" + + if os.path.exists(path): + try: + results = subprocess.check_output(['blender', '-b', path, '--python-expr', pack_script]) + result_text = results.decode() + dir_name = os.path.dirname(path) + + p = re.compile('Info: Saved "(.*)"') + match = p.search(result_text) + new_path = os.path.join(dir_name, match.group(1)) + return new_path + except Exception as e: + logger.error(f'Error packing .blend file: {e}') + return None + + if __name__ == '__main__': + + # x = pack_blender_files('/Users/brett/Blender Files/temple_animatic.blend') + # print(x) + logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG) - r = BlenderRenderWorker('/Users/brett/Blender Files/Ian Hubert/CyberExtras.blend', '/Users/brett/testing1234', render_all_frames=False, engine="CYCLES") + r = BlenderRenderWorker('/Users/brett/Blender Files/temple_animatic.blend', '/Users/brett/testing1234') # r.engine = 'CYCLES' r.start() while r.is_running(): - time.sleep(1) \ No newline at end of file + time.sleep(1) diff --git a/zordon_server.py b/zordon_server.py index db248a5..24351bb 100755 --- a/zordon_server.py +++ b/zordon_server.py @@ -294,7 +294,7 @@ class RenderServer: return success try: - response = requests.get(f"http://{hostname}:8080/status", timeout=1) + response = requests.get(f"http://{hostname}:8080/status", timeout=3) if response.ok: cls.render_clients.append(hostname) logger.info(f"Client '{hostname}' successfully registered") @@ -313,6 +313,16 @@ class RenderServer: success = True return success + @staticmethod + def is_client_available(client_hostname, timeout=3): + try: + response = requests.get(f"http://{client_hostname}:8080/status", timeout=timeout) + if response.ok: + return True + except requests.ConnectionError as e: + pass + return False + @classmethod def start(cls, background_thread=False): @@ -439,21 +449,66 @@ def add_job(): output_path = request.json.get("output", None) priority = request.json.get('priority', 2) args = request.json.get('args', None) + client = request.json.get('client', RenderServer.host_name) force_start = request.json.get('force_start', False) if None in [renderer, input_path, output_path]: return {'error': 'missing required parameters'}, 400 - try: - render_job = RenderWorkerFactory.create_worker(renderer, input_path, output_path, args) - except ValueError as e: - logger.exception(e) - return {'error': str(e)}, 400 + if client == RenderServer.host_name: + # Local Renders + try: + render_job = RenderWorkerFactory.create_worker(renderer, input_path, output_path, args) + except ValueError as e: + logger.exception(e) + return {'error': str(e)}, 400 - new_job = RenderJob(render_job, priority=priority) - RenderServer.add_to_render_queue(new_job, force_start=force_start) + new_job = RenderJob(render_job, priority=priority) + RenderServer.add_to_render_queue(new_job, force_start=force_start) - return new_job.json() + return new_job.json() + + elif client in RenderServer.render_clients: + + # see if host is available + if RenderServer.is_client_available(client): + + if args and renderer == 'blender' and args.get('pack_files', False): + from utilities.blender_worker import pack_blender_files + new_path = pack_blender_files(path=input_path) + if new_path: + logger.info(f'Packed Blender file successfully: {new_path}') + input_path = new_path + else: + err_msg = f'Failed to pack Blender file: {input_path}' + logger.error(err_msg) + return {'error': err_msg}, 400 + + # call uploader on remote client + try: + job_files = {'file': open(input_path, 'rb')} + job_data = request.json + job_data['input'] = input_path + response = requests.post(f"http://{client}:8080/uploader", files=job_files, data=job_data) + if response.ok: + logger.info("Job submitted successfully!") + return response.json() if response.json() else "Job ok" + else: + return {'error', 'Job rejected by client'}, 400 + except requests.ConnectionError as e: + err_msg = f"Error submitting job to client: {client}" + logger.error(err_msg) + return {'error', err_msg}, 400 + else: + # client is not available + err_msg = f"Render client '{client}' is unreachable" + logger.error(err_msg) + return {'error', err_msg}, 400 + + else: + err_msg = f"Unknown render client: '{client}'" + logger.error(err_msg) + return {'error', err_msg}, 400 @app.get('/cancel_job') @@ -496,7 +551,7 @@ def upload_file_page(): return render_template('upload.html') -@app.route('/uploader', methods=['GET', 'POST']) +@app.route('/uploader', methods=['POST']) def upload_file(): if request.method == 'POST': @@ -517,16 +572,12 @@ def upload_file(): uploaded_file.save(local_path) renderer = request.values['renderer'] - # todo: finish output_path and args - currently placeholder data + # todo: finish output_path - currently placeholder data output_path = os.path.join(job_dir, uploaded_file.filename + "-output.mp4") - if renderer == 'blender': - args = {'engine': request.values['blender_engine']} - else: - args = None - try: - render_job = RenderWorkerFactory.create_worker(renderer, local_path, output_path, args=args) + render_job = RenderWorkerFactory.create_worker(renderer, local_path, output_path, + args=request.values.get('args', None)) new_job = RenderJob(render_job, custom_id=new_id) RenderServer.add_to_render_queue(new_job) return new_job.json()