diff --git a/src/api/api_server.py b/src/api/api_server.py index f3930b6..1a8c694 100755 --- a/src/api/api_server.py +++ b/src/api/api_server.py @@ -322,19 +322,11 @@ def delete_job(job_id): RenderQueue.delete_job(found_job) - # Delete output directory if we own it - if output_dir.exists() and output_dir.is_relative_to(upload_root): - shutil.rmtree(output_dir) - - # Delete project directory if we own it and it's unused - try: - if project_dir.exists() and project_dir.is_relative_to(upload_root): - project_dir_files = [p for p in project_dir.iterdir() if not p.name.startswith(".")] - if not project_dir_files or (len(project_dir_files) == 1 and "source" in project_dir_files[0].name): - logger.info(f"Removing project directory: {project_dir}") - shutil.rmtree(project_dir) - except Exception as e: - logger.error(f"Error removing project files: {e}") + if _has_remaining_jobs_for_project(project_dir): + _delete_job_output(output_path, project_dir, upload_root) + elif project_dir.exists() and project_dir.is_relative_to(upload_root): + logger.info(f"Removing project directory: {project_dir}") + shutil.rmtree(project_dir) return "Job deleted", 200 @@ -343,6 +335,41 @@ def delete_job(job_id): return f"Error deleting job: {e}", 500 +def _has_remaining_jobs_for_project(project_dir): + for job in RenderQueue.all_jobs(): + try: + if Path(job.input_path).parent.parent == project_dir: + return True + except (AttributeError, TypeError): + continue + return False + + +def _delete_job_output(output_path, project_dir, upload_root): + output_dir = output_path.parent + project_output_dir = project_dir / 'output' + + if not output_dir.exists() or not output_dir.is_relative_to(upload_root): + return + + if output_dir != project_output_dir: + shutil.rmtree(output_dir) + return + + output_prefix = output_path.stem + for output_file in output_dir.iterdir(): + if _output_file_matches_prefix(output_file.name, output_prefix) and output_file.is_file(): + output_file.unlink() + + +def _output_file_matches_prefix(filename, output_prefix): + return ( + filename == output_prefix or + filename.startswith(f'{output_prefix}_') or + filename.startswith(f'{output_prefix}.') + ) + + # -------------------------------------------- # Engine Info and Management: # -------------------------------------------- diff --git a/src/api/server_proxy.py b/src/api/server_proxy.py index c0873b4..60dcc4e 100644 --- a/src/api/server_proxy.py +++ b/src/api/server_proxy.py @@ -22,6 +22,8 @@ categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStat logger = logging.getLogger() OFFLINE_MAX = 4 +JOB_UPLOAD_TIMEOUT = (10, 1800) +FILE_DOWNLOAD_TIMEOUT = (10, 1800) class RenderServerProxy: @@ -161,7 +163,7 @@ class RenderServerProxy: def get_jobs(self, timeout=5, ignore_token=False): if not self.__update_in_background or ignore_token: self.__update_job_cache(timeout, ignore_token) - return self.__jobs_cache.copy() if self.__jobs_cache else None + return self.__jobs_cache.copy() def get_status(self): status = self.request_data('status') @@ -187,7 +189,7 @@ class RenderServerProxy: # Job Lifecycle: # -------------------------------------------- - def create_job(self, file_path: Path, job_data, callback=None): + def create_job(self, file_path: Path, job_data, callback=None, timeout=JOB_UPLOAD_TIMEOUT): """ Posts a job to the server. @@ -195,6 +197,8 @@ class RenderServerProxy: file_path (Path): The path to the file to upload. job_data (dict): A dict of jobs data. callback (function, optional): A callback function to call during the upload. Defaults to None. + timeout (float | tuple, optional): Requests timeout. Defaults to a 10-second connect timeout and + 30-minute read timeout for large uploads. Returns: Response: The response from the server. @@ -208,7 +212,7 @@ class RenderServerProxy: job_data['local_path'] = str(file_path) url = urljoin(f'http://{self.hostname}:{self.port}', '/api/jobs') headers = {'Content-Type': 'application/json'} - return requests.post(url, data=json.dumps(job_data), headers=headers) + return requests.post(url, data=json.dumps(job_data), headers=headers, timeout=timeout) # Prepare the form data for remote host with open(file_path, 'rb') as file: @@ -223,16 +227,22 @@ class RenderServerProxy: url = urljoin(f'http://{self.hostname}:{self.port}', '/api/jobs') # Send the request with proper resource management - with requests.post(url, data=monitor, headers=headers) as response: + with requests.post(url, data=monitor, headers=headers, timeout=timeout) as response: return response def cancel_job(self, job_id, confirm=False): - return self._post(f'jobs/{job_id}/cancel', params={'confirm': confirm}) + response = self._post(f'jobs/{job_id}/cancel', params={'confirm': confirm}) + if response.ok: + self.__update_job_cache(timeout=5, ignore_token=True) + return response def delete_job(self, job_id, confirm=False): - return self._post(f'jobs/{job_id}/delete', params={'confirm': confirm}) + response = self._post(f'jobs/{job_id}/delete', params={'confirm': confirm}) + if response.ok: + self.__update_job_cache(timeout=5, ignore_token=True) + return response - def send_subjob_update_notification(self, parent_id, subjob): + def send_subjob_update_notification(self, parent_id, subjob, timeout=5): """ Notifies the parent job of an update in a subjob. @@ -244,7 +254,7 @@ class RenderServerProxy: Response: The response from the server. """ return requests.post(f'http://{self.hostname}:{self.port}/api/jobs/{parent_id}/subjob_update', - json=subjob.json()) + json=subjob.json(), timeout=timeout) # -------------------------------------------- # Engines: @@ -308,17 +318,17 @@ class RenderServerProxy: # Download Files: # -------------------------------------------- - def download_all_job_files(self, job_id, save_path): + def download_all_job_files(self, job_id, save_path, timeout=FILE_DOWNLOAD_TIMEOUT): url = f'http://{self.hostname}:{self.port}/api/jobs/{job_id}/download_all' - return self.__download_file_from_url(url, output_filepath=save_path) + return self.__download_file_from_url(url, output_filepath=save_path, timeout=timeout) - def download_job_file(self, job_id, job_filename, save_path): + def download_job_file(self, job_id, job_filename, save_path, timeout=FILE_DOWNLOAD_TIMEOUT): url = f'http://{self.hostname}:{self.port}/api/jobs/{job_id}/download?filename={job_filename}' - return self.__download_file_from_url(url, output_filepath=save_path) + return self.__download_file_from_url(url, output_filepath=save_path, timeout=timeout) @staticmethod - def __download_file_from_url(url, output_filepath): - with requests.get(url, stream=True) as r: + def __download_file_from_url(url, output_filepath, timeout=FILE_DOWNLOAD_TIMEOUT): + with requests.get(url, stream=True, timeout=timeout) as r: r.raise_for_status() with open(output_filepath, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): diff --git a/src/engines/blender/blender_downloader.py b/src/engines/blender/blender_downloader.py index c3b141b..bc65649 100644 --- a/src/engines/blender/blender_downloader.py +++ b/src/engines/blender/blender_downloader.py @@ -70,7 +70,7 @@ class BlenderDownloader(EngineDownloader): @staticmethod def __find_LTS_versions(): - response = requests.get('https://www.blender.org/download/lts/') + response = requests.get('https://www.blender.org/download/lts/', timeout=5) response.raise_for_status() lts_pattern = r'https://www.blender.org/download/lts/(\d+-\d+)/' diff --git a/src/ui/add_job_window.py b/src/ui/add_job_window.py index 36ebcb5..122696b 100644 --- a/src/ui/add_job_window.py +++ b/src/ui/add_job_window.py @@ -601,7 +601,7 @@ class SubmitWorker(QThread): # determine if any cameras are checked selected_cameras = [] - if self.window.cameras_list.count() and not self.window.cameras_group.isHidden(): + if self.window.cameras_list.count() and self.window.cameras_group.isEnabled(): for index in range(self.window.cameras_list.count()): item = self.window.cameras_list.item(index) if item.checkState() == Qt.CheckState.Checked: @@ -613,7 +613,7 @@ class SubmitWorker(QThread): children_jobs = [] for cam in selected_cameras: child_job_data = dict() - child_job_data['args'] = {} + child_job_data['args'] = dict(job_json['args']) child_job_data['args']['camera'] = cam child_job_data['name'] = job_json['name'].replace(' ', '-') + "_" + cam.replace(' ', '') child_job_data['output_path'] = child_job_data['name'] diff --git a/src/ui/engine_help_window.py b/src/ui/engine_help_window.py index bf76976..f55e047 100644 --- a/src/ui/engine_help_window.py +++ b/src/ui/engine_help_window.py @@ -26,5 +26,5 @@ class EngineHelpViewer(QMainWindow): self.fetch_help() def fetch_help(self): - result = requests.get(self.help_path) + result = requests.get(self.help_path, timeout=10) self.text_edit.setPlainText(result.text) diff --git a/src/ui/log_window.py b/src/ui/log_window.py index 9338fd8..fa40385 100644 --- a/src/ui/log_window.py +++ b/src/ui/log_window.py @@ -26,5 +26,5 @@ class LogViewer(QMainWindow): self.fetch_logs() def fetch_logs(self): - result = requests.get(self.log_path) + result = requests.get(self.log_path, timeout=10) self.text_edit.setPlainText(result.text) diff --git a/src/ui/main_window.py b/src/ui/main_window.py index 1d7fa27..a33c6a8 100644 --- a/src/ui/main_window.py +++ b/src/ui/main_window.py @@ -293,34 +293,36 @@ class MainWindow(QMainWindow): return server_job_data = self.job_data.get(self.current_server_proxy.hostname) - if server_job_data: - num_jobs = len(server_job_data) - self.job_list_view.setRowCount(num_jobs) + if server_job_data is None: + return - for row, job in enumerate(server_job_data): + num_jobs = len(server_job_data) + self.job_list_view.setRowCount(num_jobs) - display_status = job['status'] if job['status'] != RenderStatus.RUNNING.value else \ - ('%.0f%%' % (job['percent_complete'] * 100)) # if running, show percent, otherwise just show status - tags = (job['status'],) - start_time = datetime.datetime.fromisoformat(job['start_time']) if job['start_time'] else None - end_time = datetime.datetime.fromisoformat(job['end_time']) if job['end_time'] else None + for row, job in enumerate(server_job_data): - time_elapsed = "" if (job['status'] != RenderStatus.RUNNING.value and not end_time) else \ - get_time_elapsed(start_time, end_time) + display_status = job['status'] if job['status'] != RenderStatus.RUNNING.value else \ + ('%.0f%%' % (job['percent_complete'] * 100)) # if running, show percent, otherwise just show status + tags = (job['status'],) + start_time = datetime.datetime.fromisoformat(job['start_time']) if job['start_time'] else None + end_time = datetime.datetime.fromisoformat(job['end_time']) if job['end_time'] else None - name = job.get('name') or os.path.basename(job.get('input_path', '')) - engine_name = f"{job.get('engine', '')}-{job.get('engine_version')}" - priority = str(job.get('priority', '')) - total_frames = str(job.get('total_frames', '')) - converted_time = datetime.datetime.fromisoformat(job['date_created']) - humanized_time = humanize.naturaltime(converted_time) + time_elapsed = "" if (job['status'] != RenderStatus.RUNNING.value and not end_time) else \ + get_time_elapsed(start_time, end_time) - items = [QTableWidgetItem(job['id']), QTableWidgetItem(name), QTableWidgetItem(engine_name), - QTableWidgetItem(priority), QTableWidgetItem(display_status), QTableWidgetItem(time_elapsed), - QTableWidgetItem(total_frames), QTableWidgetItem(humanized_time)] + name = job.get('name') or os.path.basename(job.get('input_path', '')) + engine_name = f"{job.get('engine', '')}-{job.get('engine_version')}" + priority = str(job.get('priority', '')) + total_frames = str(job.get('total_frames', '')) + converted_time = datetime.datetime.fromisoformat(job['date_created']) + humanized_time = humanize.naturaltime(converted_time) - for col, item in enumerate(items): - self.job_list_view.setItem(row, col, item) + items = [QTableWidgetItem(job['id']), QTableWidgetItem(name), QTableWidgetItem(engine_name), + QTableWidgetItem(priority), QTableWidgetItem(display_status), QTableWidgetItem(time_elapsed), + QTableWidgetItem(total_frames), QTableWidgetItem(humanized_time)] + + for col, item in enumerate(items): + self.job_list_view.setItem(row, col, item) # -- Job Code -- # def job_picked(self): diff --git a/src/utilities/server_helper.py b/src/utilities/server_helper.py index e417ead..4d1e7a4 100644 --- a/src/utilities/server_helper.py +++ b/src/utilities/server_helper.py @@ -127,7 +127,7 @@ def distribute_server_work(start_frame, end_frame, available_servers, method='ev def fetch_benchmark(server): try: benchmark = requests.get(f'http://{server["hostname"]}:{ZeroconfServer.server_port}' - f'/api/cpu_benchmark').text + f'/api/cpu_benchmark', timeout=15).text server['cpu_benchmark'] = benchmark logger.debug(f'Benchmark for {server["hostname"]}: {benchmark}') except requests.exceptions.RequestException as e: