From 9ec7ef48c2efb03491a04d06e22dbf0cda69700d Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Mon, 5 Jun 2023 19:30:47 -0500 Subject: [PATCH] See progress of uploads in client --- lib/client/new_job_window.py | 157 +++++++++++++++++++++++------------ lib/server/server_proxy.py | 24 ++++-- requirements.txt | 3 +- 3 files changed, 126 insertions(+), 58 deletions(-) diff --git a/lib/client/new_job_window.py b/lib/client/new_job_window.py index 979313e..bf8e56d 100755 --- a/lib/client/new_job_window.py +++ b/lib/client/new_job_window.py @@ -6,13 +6,11 @@ import pathlib import socket from tkinter import * from tkinter import filedialog, messagebox -from tkinter.simpledialog import askstring -from tkinter.ttk import Frame, Label, Entry, Combobox +from tkinter.ttk import Frame, Label, Entry, Combobox, Progressbar import psutil import requests -import sys -sys.path.append('../../') +import threading from lib.render_workers.blender_worker import Blender from lib.server.server_proxy import RenderServerProxy @@ -67,6 +65,7 @@ class NewJobWindow(Frame): def __init__(self, parent=None, clients=None): super().__init__(parent) + self.root = parent self.clients = clients or [] self.server_proxy = RenderServerProxy(hostname=clients[0] if clients else None) self.chosen_file = None @@ -158,6 +157,11 @@ class NewJobWindow(Frame): self.custom_args_entry = None self.submit_frame = None + self.progress_frame = None + self.progress_label = None + self.progress_bar = None + self.upload_status = None + self.fetch_server_data() def client_picked(self, event=None): @@ -249,10 +253,20 @@ class NewJobWindow(Frame): self.submit_frame.forget() self.submit_frame = Frame(self) self.submit_frame.pack(fill=BOTH, expand=True) - Label(self.submit_frame, text="").pack(fill=BOTH, expand=True) + # Label(self.submit_frame, text="").pack(fill=BOTH, expand=True) submit_button = Button(self.submit_frame, text="Submit", command=self.submit_job) submit_button.pack(fill=Y, anchor="s", pady=header_padding) + def draw_progress_frame(self): + if hasattr(self, 'progress_frame') and self.progress_frame: + self.progress_frame.forget() + self.progress_frame = LabelFrame(self, text="Job Submission") + self.progress_frame.pack(side=TOP, fill=X, expand=False, padx=5, pady=5) + self.progress_bar = Progressbar(self.progress_frame, length=300, mode="determinate") + self.progress_bar.pack() + self.progress_label = Label(self.progress_frame, text="Starting Up") + self.progress_label.pack(pady=5, padx=5) + def draw_blender_settings(self): scene_data = None @@ -263,8 +277,7 @@ class NewJobWindow(Frame): # blender settings self.blender_frame = LabelFrame(self, text="Blender Settings") - self.blender_frame.pack(fill=X, padx=5, pady=5) - + self.blender_frame.pack(fill=X, padx=5) blender_engine_frame = Frame(self.blender_frame) blender_engine_frame.pack(fill=X) @@ -318,56 +331,98 @@ class NewJobWindow(Frame): def submit_job(self): - client = self.client_combo.get() + def submit_job_worker(): - renderer = self.renderer_combo.get() - job_json = {'owner': psutil.Process().username() + '@' + socket.gethostname(), - 'renderer': renderer, - 'client': client, - 'output_path': os.path.join(os.path.dirname(self.chosen_file), self.output_entry.get()), - 'args': {'raw': self.custom_args_entry.get()}, - 'name': None} - job_list = [] + self.draw_progress_frame() + self.progress_bar['value'] = 0 + self.progress_bar.configure(mode='determinate') + self.progress_bar.start() + self.progress_label.configure(text="Preparing files...") - input_path = self.chosen_file + # start the progress UI + client = self.client_combo.get() - temp_files = [] - if renderer == 'blender': - if self.blender_pack_textures.get(): - new_path = Blender.pack_project_file(project_path=input_path) - if new_path: - logger.info(f'Packed Blender file successfully: {new_path}') - input_path = new_path - temp_files.append(new_path) + renderer = self.renderer_combo.get() + job_json = {'owner': psutil.Process().username() + '@' + socket.gethostname(), + 'renderer': renderer, + 'client': client, + 'output_path': os.path.join(os.path.dirname(self.chosen_file), self.output_entry.get()), + 'args': {'raw': self.custom_args_entry.get()}, + 'name': None} + job_list = [] + + input_path = self.chosen_file + + temp_files = [] + if renderer == 'blender': + if self.blender_pack_textures.get(): + self.progress_label.configure(text="Packing Blender file...") + new_path = Blender.pack_project_file(project_path=input_path) + if new_path: + logger.info(f"New Path is now {new_path}") + input_path = new_path + temp_files.append(new_path) + else: + err_msg = f'Failed to pack Blender file: {input_path}' + messagebox.showinfo("Error", err_msg) + return + # add all Blender args + job_json['args']['engine'] = self.blender_engine.get() + job_json['args']['render_all_frames'] = self.blender_render_all_frames.get() + job_json['args']['export_format'] = self.output_format.get() + + # multiple camera rendering + if self.blender_cameras_list and self.blender_multiple_cameras.get(): + selected_cameras = self.blender_cameras_list.getCheckedItems() + for cam in selected_cameras: + job_copy = copy.deepcopy(job_json) + job_copy['args']['camera'] = cam.rsplit('-', 1)[0].strip() + job_copy['name'] = pathlib.Path(input_path).stem.replace(' ', '_') + "-" + cam.replace(' ', '') + job_list.append(job_copy) + + # Submit to server + job_list = job_list or [job_json] + self.progress_label.configure(text="Posting to server...") + self.progress_bar.stop() + self.progress_bar.configure(mode='determinate') + self.progress_bar.start() + + def create_callback(encoder): + encoder_len = encoder.len + + def callback(monitor): + percent = f"{monitor.bytes_read / encoder_len * 100:.0f}" + self.progress_label.configure(text=f"Transferring to {client} - {percent}%") + self.progress_bar['value'] = int(percent) + return callback + + result = self.server_proxy.post_job_to_server(input_path=input_path, job_list=job_list, + callback=create_callback) + + self.progress_bar.stop() + # clean up + for temp in temp_files: + os.remove(temp) + + def finish_on_main(): + if result.ok: + message = "Job successfully submitted to server." + self.progress_label.configure(text=message) + messagebox.showinfo("Success", message) + logger.info(message) else: - err_msg = f'Failed to pack Blender file: {input_path}' - messagebox.showinfo("Error", err_msg) - return - # add all Blender args - job_json['args']['engine'] = self.blender_engine.get() - job_json['args']['render_all_frames'] = self.blender_render_all_frames.get() - job_json['args']['export_format'] = self.output_format.get() + message = result.text or "Unknown error" + self.progress_label.configure(text=message) + logger.warning(message) + messagebox.showinfo("Error", message) + self.progress_label.configure(text="") + self.progress_frame.forget() - # multiple camera rendering - if self.blender_cameras_list and self.blender_multiple_cameras.get(): - selected_cameras = self.blender_cameras_list.getCheckedItems() - for cam in selected_cameras: - job_copy = copy.deepcopy(job_json) - job_copy['args']['camera'] = cam.rsplit('-', 1)[0].strip() - job_copy['name'] = pathlib.Path(input_path).stem.replace(' ', '_') + "-" + cam.replace(' ', '') - job_list.append(job_copy) + self.root.after(0, finish_on_main) - # Submit to server - job_list = job_list or [job_json] - result = self.server_proxy.post_job_to_server(input_path=input_path, job_list=job_list) - if result.ok: - messagebox.showinfo("Success", "Job successfully submitted to server.") - else: - messagebox.showinfo("Error", result.text or "Unknown error") - - # clean up - for temp in temp_files: - os.remove(temp) + # Start the job submit task as a bg thread + bg_thread = threading.Thread(target=submit_job_worker) + bg_thread.start() def main(): diff --git a/lib/server/server_proxy.py b/lib/server/server_proxy.py index 4969444..18db6b5 100644 --- a/lib/server/server_proxy.py +++ b/lib/server/server_proxy.py @@ -3,6 +3,7 @@ import os import json import requests from lib.render_workers.base_worker import RenderStatus +from requests_toolbelt.multipart import MultipartEncoder, MultipartEncoderMonitor status_colors = {RenderStatus.ERROR: "red", RenderStatus.CANCELLED: 'orange1', RenderStatus.COMPLETED: 'green', RenderStatus.NOT_STARTED: "yellow", RenderStatus.SCHEDULED: 'purple', @@ -69,10 +70,21 @@ class RenderServerProxy: all_data = self.request_data('full_status', timeout=timeout) return all_data - def post_job_to_server(self, input_path, job_list): - # Pack job data and submit to server - job_files = {'file': (os.path.basename(input_path), open(input_path, 'rb'), 'application/octet-stream'), - 'json': (None, json.dumps(job_list), 'application/json')} + def post_job_to_server(self, input_path, job_list, callback=None): + # Prepare the form data + encoder = MultipartEncoder({ + 'file': (os.path.basename(input_path), open(input_path, 'rb'), 'application/octet-stream'), + 'json': (None, json.dumps(job_list), 'application/json'), + }) - req = requests.post(f'http://{self.hostname}:{self.port}/api/add_job', files=job_files) - return req \ No newline at end of file + # Create a monitor that will track the upload progress + if callback: + monitor = MultipartEncoderMonitor(encoder, callback(encoder)) + else: + monitor = MultipartEncoderMonitor(encoder) + + # Send the request + headers = {'Content-Type': monitor.content_type} + response = requests.post(f'http://{self.hostname}:{self.port}/api/add_job', data=monitor, headers=headers) + + return response diff --git a/requirements.txt b/requirements.txt index 48d1a88..633b60e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ future==0.18.3 json2html~=1.3.0 SQLAlchemy~=2.0.15 Pillow~=9.3.0 -zeroconf~=0.63.0 \ No newline at end of file +zeroconf~=0.63.0 +requests-toolbelt~=1.0 \ No newline at end of file