brett/Zordon
1
0
Fork 0
mirror of https://github.com/blw1138/Zordon.git synced 2026-06-09 13:39:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: brett/Zordon:24eb7b5616f66f22291e38ebcc38e18f8e09f581
Branches Tags
brett/Zordon:master brett/Zordon:feature/blender-report-render-engines brett/Zordon:feature/84-after-effects-support brett/Zordon:dynamic_render_options brett/Zordon:#24_generate_new_subjobs_on_error
brett/Zordon:v0.8.0
..
compare: brett/Zordon:feature/blender-report-render-engines
Branches Tags
brett/Zordon:master brett/Zordon:feature/blender-report-render-engines brett/Zordon:feature/84-after-effects-support brett/Zordon:dynamic_render_options brett/Zordon:#24_generate_new_subjobs_on_error
brett/Zordon:v0.8.0

2 Commits
24eb7b5616 .. feature/blender-report-render-engines

Author SHA1 Message Date
Brett Williams a464910426 Always add render engine to Blender args 2024-08-08 23:20:38 -05:00
Brett Williams a4ff36ac56 Add render engines to system_info 2024-08-08 23:13:31 -05:00
68 changed files with 2640 additions and 6854 deletions
Expand all files Collapse all files
.flake8
-11
View File
@@ -1,11 +0,0 @@
[flake8]
exclude =
src/engines/aerender
.git
build
dist
*.egg
venv
.venv
max-complexity = 10
max-line-length = 127
.github/workflows/ci.yml
-45
View File
@@ -1,45 +0,0 @@
name: CI
on:
push:
branches: [master]
pull_request:
branches: [master]
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install system deps
run: sudo apt-get update && sudo apt-get install -y libxcb-cursor0 libxcb-xinerama0 blender
- name: Install Python deps
run: |
pip install -r requirements.txt
pip install pytest flake8 pylint
- name: Test
run: python -m pytest tests/ --ignore=tests/job_creation_tests.py -v
- name: Lint — bugs (flake8)
# Fails on syntax errors only. Ignored: ~100+ pre-existing
# style issues (E/W), star-import false positives (F403/F405),
# unused imports (F401), unused vars (F841), f-string debt (F541).
run: flake8 --select=E9 --exclude=src/engines/aerender,build,dist,.git,__pycache__,venv --max-line-length=127 --max-complexity=35
- name: Lint — style (flake8)
# Reports style issues but never fails the build.
# TODO: resolve ~100+ pre-existing style issues
run: flake8 --exit-zero --exclude=src/engines/aerender,build,dist,.git,__pycache__,venv --max-line-length=127 --max-complexity=35
- name: Lint (pylint)
# Reports all issues but never fails the build.
# TODO: resolve pre-existing debt (current score 7.73/10)
run: pylint src/ --max-line-length=120 --ignore-paths=^src/engines/aerender/ --fail-under=0
.github/workflows/create-executables.yml
-38
View File
@@ -1,38 +0,0 @@
name: Create Executables
on:
workflow_dispatch:
release:
- types: [created]
jobs:
pyinstaller-build-windows:
runs-on: windows-latest
steps:
- name: Create Executables (Windows)
uses: sayyid5416/pyinstaller@v1
with:
python_ver: '3.11'
spec: 'client.spec'
requirements: 'requirements.txt'
upload_exe_with_name: 'Zordon'
pyinstaller-build-linux:
runs-on: ubuntu-latest
steps:
- name: Create Executables (Linux)
uses: sayyid5416/pyinstaller@v1
with:
python_ver: '3.11'
spec: 'client.spec'
requirements: 'requirements.txt'
upload_exe_with_name: 'Zordon'
pyinstaller-build-macos:
runs-on: macos-latest
steps:
- name: Create Executables (macOS)
uses: sayyid5416/pyinstaller@v1
with:
python_ver: '3.11'
spec: 'client.spec'
requirements: 'requirements.txt'
upload_exe_with_name: 'Zordon'
.github/workflows/pylint.yml
+23
View File
@@ -0,0 +1,23 @@
name: Pylint
on: [push]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pylint
- name: Analysing the code with pylint
run: |
pylint $(git ls-files '*.py')
.github/workflows/python-app.yml
+39
View File
@@ -0,0 +1,39 @@
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
name: Python application
on:
push:
branches: [ "master" ]
pull_request:
branches: [ "master" ]
permissions:
contents: read
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install flake8 pytest
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Test with pytest
run: |
pytest
.gitignore
-8
View File
@@ -6,11 +6,3 @@
/dist/
/build/
/.github/
*.idea
.DS_Store
/venv/
.env
venv/
/.eggs/
/.ai/
/.github/
.pylintrc
-1
View File
@@ -1,5 +1,4 @@
[MASTER]
max-line-length = 120
ignore-paths=^src/engines/aerender/
[MESSAGES CONTROL]
disable = missing-docstring, invalid-name, import-error, logging-fstring-interpolation
README.md
+7 -189
View File
@@ -1,196 +1,14 @@
![Zordon Screenshot](docs/screenshot.png)
# 🎬 Zordon - Render Management Tools
---
Welcome to Zordon! It's a local network render farm manager, aiming to streamline and simplify the rendering process across multiple home computers.
# Zordon
## 📦 Installation
A Python-based distributed rendering management tool that supports Blender, FFmpeg, and other render engines. Zordon efficiently manages render jobs across multiple machines, making it ideal for small render farms in home studios or small businesses.
Install the necessary dependencies: `pip3 install -r requirements.txt`
## What is Zordon?
## 🎨 Supported Renderers
Zordon is a tool designed for small render farms, such as those used in home studios or small businesses, to efficiently manage and run render jobs for Blender, FFmpeg, and other video renderers. It simplifies the process of distributing rendering tasks across multiple available machines, optimizing the rendering workflow for artists, animators, and video professionals.
The system works by:
- **Server**: Central coordinator that manages job queues and distributes tasks to available workers
- **Clients**: Lightweight workers that run on rendering machines and execute assigned jobs
- **API**: RESTful endpoints for programmatic job submission and monitoring
## Features
- **Distributed Rendering**: Queue and distribute render jobs across multiple machines
- **Multi-Engine Support**: Compatible with Blender, FFmpeg, and extensible to other render engines
- **Desktop UI**: PyQt6 interface for job management and monitoring
- **REST API**: Flask-based API for programmatic access
- **Cross-Platform**: Runs on Windows, macOS, and Linux
- **Zero-Install Clients**: Lightweight client executables for worker machines
## Installation
### Prerequisites
- Python 3.11 or later
- Git
### Setup
1. Clone the repository:
```bash
git clone https://github.com/blw1138/Zordon.git
cd Zordon
```
2. Install dependencies:
```bash
pip install -r requirements.txt
```
3. (Optional) Install PyInstaller for building executables:
```bash
pip install pyinstaller pyinstaller_versionfile
```
## Usage
### Quick Start
1. **Start the Server**: Run the central server to coordinate jobs.
```bash
python server.py
```
2. **Launch Clients**: On each rendering machine, run the client to connect to the server.
```bash
python client.py
```
### Detailed Workflow
#### Setting Up a Render Farm
1. Choose one machine as the server (preferably a dedicated machine with good network connectivity).
2. Build and distribute client executables to worker machines:
```bash
pyinstaller client.spec
```
Copy the generated executable to each worker machine.
3. Ensure all machines can communicate via network (same subnet recommended).
#### Submitting Render Jobs
Jobs can be submitted via the desktop UI or programmatically via the API:
- **Via UI**: Use the desktop interface to upload project files, specify render settings, and queue jobs.
- **Via API**: Send POST requests to `/api/jobs` with job configuration in JSON format.
Example API request:
```bash
curl -X POST http://localhost:5000/api/jobs \
-H "Content-Type: application/json" \
-d '{
"engine": "blender",
"project_path": "/path/to/project.blend",
"output_path": "/path/to/output",
"frames": "1-100",
"settings": {"resolution": "1920x1080"}
}'
```
#### Monitoring and Managing Jobs
- **UI**: View job status, progress, logs, and worker availability in real-time.
- **API Endpoints**:
- `GET /api/jobs`: List all jobs
- `GET /api/jobs/{id}`: Get job details
- `DELETE /api/jobs/{id}`: Cancel a job
- `GET /api/workers`: List connected workers
#### Worker Management
Workers automatically connect to the server when started. You can:
- View worker status and capabilities in the dashboard
- Configure worker priorities and resource limits
- Monitor CPU/GPU usage per worker
### Development Mode
For development and testing:
Run the server:
```bash
python server.py
```
Run a client (can run multiple for testing):
```bash
python client.py
```
### Building Executables
Build server executable:
```bash
pyinstaller server.spec
```
Build client executable:
```bash
pyinstaller client.spec
```
## Configuration
Settings are stored in `src/utilities/config.py`. Supports YAML/JSON for data serialization and environment-specific configurations.
## Architecture
Zordon follows a modular architecture with the following key components:
- **API Server** (`src/api/`): Flask-based REST API
- **Engine System** (`src/engines/`): Pluggable render engines (Blender, FFmpeg, etc.)
- **UI** (`src/ui/`): PyQt6-based interface
- **Job Management** (`src/render_queue.py`): Distributed job queue
Design patterns include Factory Pattern for engine creation, Observer Pattern for status updates, and Strategy Pattern for different worker implementations.
## Contributing
1. Fork the repository
2. Create a feature branch: `git checkout -b feature/your-feature`
3. Follow the code style guidelines in `AGENTS.md`
4. Test the build: `pyinstaller server.spec`
5. Submit a pull request
### Commit Message Format
```
feat: add support for new render engine
fix: resolve crash when engine path is invalid
docs: update API documentation
refactor: simplify job status handling
```
## Supported Render Engines
Zordon currently supports the following renderers:
- **Blender**
- **FFmpeg**
- **Adobe After Effects** (planned)
- **Cinema 4D** (planned)
- **Autodesk Maya** (planned)
## System Requirements
- Windows 10 or later
- macOS Ventura (13.0) or later
- Linux (supported versions TBD)
## License
Zordon is licensed under the MIT License. See the [LICENSE](LICENSE.txt) file for more details.
## Notice
This software is in beta and intended for casual/hobbyist use. Not recommended for mission-critical environments.
- **FFMPEG**
add_job.py
-131
View File
@@ -1,131 +0,0 @@
#!/usr/bin/env python3
import argparse
import logging
import os
import socket
import sys
import time
from server import ZordonServer
from src.api.serverproxy_manager import ServerProxyManager
logger = logging.getLogger()
def main():
parser = argparse.ArgumentParser(
description="Zordon CLI tool for preparing/submitting a render job",
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
# Required arguments
parser.add_argument("scene_file", help="Path to the scene file (e.g., .blend, .max, .mp4)")
parser.add_argument("engine", help="Desired render engine", choices=['blender', 'ffmpeg'])
# Frame range
parser.add_argument("--start", type=int, default=1, help="Start frame")
parser.add_argument("--end", type=int, default=1, help="End frame")
# Job metadata
parser.add_argument("--name", default=None, help="Job name")
# Output
parser.add_argument("--output", default="", help="Output path/pattern (e.g., /renders/frame_####.exr)")
# Target OS and Engine Version
parser.add_argument(
"--os",
choices=["any", "windows", "linux", "macos"],
default="any",
help="Target operating system for render workers"
)
parser.add_argument(
"--engine-version",
default="latest",
help="Required renderer/engine version number (e.g., '4.2', '5.0')"
)
# Optional flags
parser.add_argument("--dry-run", action="store_true", help="Print job details without submitting")
args = parser.parse_args()
# Basic validation
if not os.path.exists(args.scene_file):
print(f"Error: Scene file '{args.scene_file}' not found!", file=sys.stderr)
sys.exit(1)
if args.start > args.end:
print("Error: Start frame cannot be greater than end frame!", file=sys.stderr)
sys.exit(1)
# Calculate total frames
total_frames = len(range(args.start, args.end + 1))
job_name = args.name or os.path.basename(args.scene_file)
file_path = os.path.abspath(args.scene_file)
# Print job summary
print("Render Job Summary:")
print(f" Job Name : {job_name}")
print(f" Scene File : {file_path}")
print(f" Engine : {args.engine}")
print(f" Frames : {args.start}-{args.end} → {total_frames} frames")
print(f" Output Path : {args.output or '(default from scene)'}")
print(f" Target OS : {args.os}")
print(f" Engine Version : {args.engine_version}")
if args.dry_run:
print("\nDry run complete (no submission performed).")
return
local_hostname = socket.gethostname()
local_hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
found_proxy = ServerProxyManager.get_proxy_for_hostname(local_hostname)
is_connected = found_proxy.check_connection()
adhoc_server = None
if not is_connected:
adhoc_server = ZordonServer()
adhoc_server.start_server()
found_proxy = ServerProxyManager.get_proxy_for_hostname(adhoc_server.server_hostname)
while not is_connected:
# todo: add timeout
is_connected = found_proxy.check_connection()
time.sleep(1)
new_job = {"name": job_name, "engine_name": args.engine}
try:
response = found_proxy.post_job_to_server(file_path, new_job)
except Exception as e:
print(f"Error creating job: {e}")
exit(1)
if response and response.ok:
print(f"Uploaded to {found_proxy.hostname} successfully!")
running_job_data = response.json()
job_id = running_job_data.get('id')
print(f"Job {job_id} Summary:")
print(f" Status : {running_job_data.get('status')}")
print(f" Engine : {running_job_data.get('engine_name')}-{running_job_data.get('engine_version')}")
print("\nWaiting for render to complete...")
percent_complete = 0.0
while percent_complete < 1.0:
# add checks for errors
time.sleep(1)
running_job_data = found_proxy.get_job_info(job_id)
percent_complete = running_job_data['percent_complete']
sys.stdout.write("\x1b[1A") # Move up 1
sys.stdout.write("\x1b[0J") # Clear from cursor to end of screen (optional)
print(f"Percent Complete: {percent_complete:.2%}")
sys.stdout.flush()
print("Finished rendering successfully!")
else:
print(f"Failed to upload job. {response.text} !")
if adhoc_server:
adhoc_server.stop_server()
if __name__ == "__main__":
main()
client.py
-77
View File
@@ -1,77 +0,0 @@
#!/usr/bin/env python3
import logging
import threading
from collections import deque
from server import ZordonServer
logger = logging.getLogger()
def __setup_buffer_handler():
# lazy load GUI frameworks
from PyQt6.QtCore import QObject, pyqtSignal
class BufferingHandler(logging.Handler, QObject):
new_record = pyqtSignal(str)
flushOnClose = True
def __init__(self, capacity=100):
logging.Handler.__init__(self)
QObject.__init__(self)
self.buffer = deque(maxlen=capacity) # Define a buffer with a fixed capacity
def emit(self, record):
try:
msg = self.format(record)
self.buffer.append(msg) # Add message to the buffer
self.new_record.emit(msg) # Emit signal
except RuntimeError:
pass
def get_buffer(self):
return list(self.buffer) # Return a copy of the buffer
buffer_handler = BufferingHandler()
buffer_handler.setFormatter(logging.getLogger().handlers[0].formatter)
new_logger = logging.getLogger()
new_logger.addHandler(buffer_handler)
return buffer_handler
def __show_gui(buffer_handler):
# lazy load GUI frameworks
from PyQt6.QtWidgets import QApplication
# load application
app: QApplication = QApplication(sys.argv)
if app.style().objectName() != 'macos':
app.setStyle('Fusion')
# configure main window
from src.ui.main_window import MainWindow
window: MainWindow = MainWindow()
window.buffer_handler = buffer_handler
window.show()
exit_code = app.exec()
# cleanup: remove and close the GUI logging handler before interpreter shutdown
root_logger = logging.getLogger()
if buffer_handler in root_logger.handlers:
root_logger.removeHandler(buffer_handler)
try:
buffer_handler.close()
except Exception:
# never let logging cleanup throw during shutdown
pass
return exit_code
if __name__ == '__main__':
import sys
server = ZordonServer()
server.start_server()
__show_gui(__setup_buffer_handler())
server.stop_server()
sys.exit()
client.spec
-158
View File
@@ -1,158 +0,0 @@
# -*- mode: python ; coding: utf-8 -*-
from PyInstaller.utils.hooks import collect_all
from pathlib import Path
import os
import sys
import platform
# ------------------------------------------------------------
# Project paths
# ------------------------------------------------------------
project_root = Path(SPECPATH).resolve()
src_dir = project_root / "src"
# Ensure `src.*` imports work during analysis
sys.path.insert(0, str(project_root))
# ------------------------------------------------------------
# Import version info
# ------------------------------------------------------------
from src.version import APP_NAME, APP_VERSION, APP_AUTHOR
APP_NAME = f"{APP_NAME}-client"
# ------------------------------------------------------------
# PyInstaller data / imports
# ------------------------------------------------------------
datas = [
("resources", "resources"),
("src/engines/blender/scripts", "src/engines/blender/scripts"),
]
binaries = []
hiddenimports = ["zeroconf", "src.version"]
tmp_ret = collect_all("zeroconf")
datas += tmp_ret[0]
binaries += tmp_ret[1]
hiddenimports += tmp_ret[2]
# ------------------------------------------------------------
# Analysis
# ------------------------------------------------------------
a = Analysis(
["client.py"],
pathex=[str(project_root)],
binaries=binaries,
datas=datas,
hiddenimports=hiddenimports,
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
noarchive=False,
optimize=1, # optimize=2 breaks Windows builds
)
pyz = PYZ(a.pure)
# ------------------------------------------------------------
# Platform targets
# ------------------------------------------------------------
if platform.system() == "Darwin": # macOS
exe = EXE(
pyz,
a.scripts,
[],
exclude_binaries=True,
name=APP_NAME,
debug=False,
bootloader_ignore_signals=False,
strip=True,
upx=True,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)
app = BUNDLE(
exe,
a.binaries,
a.datas,
strip=True,
name=f"{APP_NAME}.app",
icon="resources/Server.png",
bundle_identifier=None,
version=APP_VERSION,
)
elif platform.system() == "Windows":
import pyinstaller_versionfile
import tempfile
version_file_path = os.path.join(
tempfile.gettempdir(), "versionfile.txt"
)
pyinstaller_versionfile.create_versionfile(
output_file=version_file_path,
version=APP_VERSION,
company_name=APP_AUTHOR,
file_description=APP_NAME,
internal_name=APP_NAME,
legal_copyright=f"© {APP_AUTHOR}",
original_filename=f"{APP_NAME}.exe",
product_name=APP_NAME,
)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.datas,
[],
name=APP_NAME,
debug=False,
bootloader_ignore_signals=False,
strip=True,
upx=True,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
version=version_file_path,
)
else: # Linux
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.datas,
[],
name=APP_NAME,
debug=False,
bootloader_ignore_signals=False,
strip=True,
upx=True,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)
docs/screenshot.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 838 KiB

main.py
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env python3
from src import init
if __name__ == '__main__':
import sys
sys.exit(init.run())
pytest.ini
-4
View File
@@ -1,4 +0,0 @@
[pytest]
norecursedirs = src/engines/aerender .git build dist *.egg venv .venv env .env __pycache__ .pytest_cache
testpaths = tests
python_files = test_*.py job_creation_tests.py
requirements.txt
+33 -16
View File
@@ -1,20 +1,37 @@
PyQt6>=6.7.0
PyQt6>=6.6.1
psutil>=5.9.8
requests>=2.32.2
Pillow>=10.3.0
requests>=2.31.0
Pillow>=10.2.0
PyYAML>=6.0.1
flask>=3.0.3
tqdm>=4.66.4
werkzeug>=3.0.3
flask>=3.0.2
tqdm>=4.66.2
werkzeug>=3.0.1
Pypubsub>=4.0.3
zeroconf>=0.132.2
SQLAlchemy>=2.0.30
zeroconf>=0.131.0
SQLAlchemy>=2.0.25
plyer>=2.1.0
rich>=13.7.1
setuptools>=70.0.0
py-cpuinfo>=9.0.0
requests-toolbelt>=1.0.0
PyQt6-sip>=13.6.0
humanize>=4.12.1
macholib>=1.16.3
altgraph>=0.17.4
pytz>=2023.3.post1
future>=0.18.3
rich>=13.7.0
pytest>=8.0.0
numpy>=1.26.3
setuptools>=69.0.3
pandas>=2.2.0
matplotlib>=3.8.2
MarkupSafe>=2.1.4
dmglib>=0.9.5; sys_platform == 'darwin'
python-dateutil>=2.8.2
certifi>=2023.11.17
shiboken6>=6.6.1
Pygments>=2.17.2
cycler>=0.12.1
contourpy>=1.2.0
packaging>=23.2
fonttools>=4.47.2
Jinja2>=3.1.3
pyparsing>=3.1.1
kiwisolver>=1.4.5
attrs>=23.2.0
lxml>=5.1.0
click>=8.1.7
requests_toolbelt>=1.0.0
server.py
+3 -153
View File
@@ -1,155 +1,5 @@
import logging
import multiprocessing
import os
import socket
import threading
from pathlib import Path
import psutil
from src.api.api_server import API_VERSION
from src.api.api_server import start_api_server
from src.api.preview_manager import PreviewManager
from src.api.serverproxy_manager import ServerProxyManager
from src.application_context import ApplicationContext
from src.distributed_job_manager import DistributedJobManager
from src.engines.engine_manager import EngineManager
from src.render_queue import RenderQueue
from src.utilities.config import Config
from src.utilities.misc_helper import (get_gpu_info, current_system_cpu, current_system_os,
current_system_os_version, current_system_cpu_brand)
from src.utilities.zeroconf_server import ZeroconfServer
from src.version import APP_NAME, APP_VERSION
logger = logging.getLogger()
class ZordonServer:
def __init__(self):
self.ctx = ApplicationContext()
# setup logging
logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S',
level=Config.server_log_level.upper())
logging.getLogger("requests").setLevel(logging.WARNING) # suppress noisy requests/urllib3 logging
logging.getLogger("urllib3").setLevel(logging.WARNING)
# ---- Bootstrap Config ----
Config.setup_config_dir()
config_path = Path(Config.config_dir()) / "config.yaml"
self.ctx.config = Config()
self.ctx.config.load(config_path)
Config._default_instance = self.ctx.config
Config._sync_class()
# ---- Engine Manager ----
self.ctx.engine_manager = EngineManager()
self.ctx.engine_manager.engines_path = Path(Config.upload_folder).expanduser() / "engines"
os.makedirs(self.ctx.engine_manager.engines_path, exist_ok=True)
EngineManager._default_instance = self.ctx.engine_manager
EngineManager._sync_class()
# ---- Preview Manager ----
self.ctx.preview_manager = PreviewManager()
self.ctx.preview_manager.storage_path = Path(Config.upload_folder).expanduser() / "previews"
PreviewManager._default_instance = self.ctx.preview_manager
PreviewManager._sync_class()
# ---- Render Queue ----
self.ctx.render_queue = RenderQueue()
self.ctx.render_queue.load_state(database_directory=Path(Config.upload_folder).expanduser())
RenderQueue._default_instance = self.ctx.render_queue
RenderQueue._sync_class()
# ---- Distributed Job Manager ----
self.ctx.distributed_job_manager = DistributedJobManager()
self.ctx.distributed_job_manager.subscribe_to_listener()
DistributedJobManager._default_instance = self.ctx.distributed_job_manager
DistributedJobManager._sync_class()
self.api_server = None
self.server_hostname: str = socket.gethostname()
def start_server(self):
def existing_process(process_name):
current_pid = os.getpid()
current_process = psutil.Process(current_pid)
for proc in psutil.process_iter(['pid', 'name', 'ppid']):
proc_name = proc.info['name'].lower().rstrip('.exe')
if proc_name == process_name.lower() and proc.info['pid'] != current_pid:
if proc.info['pid'] == current_process.ppid():
continue # parent process
elif proc.info['ppid'] == current_pid:
continue # child process
else:
return proc # unrelated process
return None
# check for existing instance
existing_proc = existing_process(APP_NAME)
if existing_proc:
err_msg = f"Another instance of {APP_NAME} is already running (pid: {existing_proc.pid})"
logger.fatal(err_msg)
raise ProcessLookupError(err_msg)
# main start
logger.info(f"Starting {APP_NAME} Render Server ({APP_VERSION})")
logger.debug(f"Upload directory: {Path(Config.upload_folder).expanduser()}")
logger.debug(f"Thumbs directory: {PreviewManager.storage_path}")
logger.debug(f"Engines directory: {EngineManager.engines_path}")
ServerProxyManager.subscribe_to_listener()
# update hostname
self.server_hostname = socket.gethostname()
# configure and start API server
self.api_server = threading.Thread(target=start_api_server, args=(self.server_hostname,))
self.api_server.daemon = True
self.api_server.start()
# start zeroconf server
ctx = self.ctx
ctx.zeroconf_server = ZeroconfServer()
ctx.zeroconf_server._configure(f"_{APP_NAME.lower()}._tcp.local.", self.server_hostname, Config.port_number)
ctx.zeroconf_server.properties = {'system_cpu': current_system_cpu(),
'system_cpu_brand': current_system_cpu_brand(),
'system_cpu_cores': multiprocessing.cpu_count(),
'system_os': current_system_os(),
'system_os_version': current_system_os_version(),
'system_memory': round(psutil.virtual_memory().total / (1024**3)),
'gpu_info': get_gpu_info(),
'api_version': API_VERSION}
ZeroconfServer._default_instance = ctx.zeroconf_server
ZeroconfServer._sync_class()
ctx.zeroconf_server._start()
logger.info(f"{APP_NAME} Render Server started - Hostname: {self.server_hostname}")
RenderQueue.start() # Start evaluating the render queue
def is_running(self):
return self.api_server and self.api_server.is_alive()
def stop_server(self):
logger.info(f"{APP_NAME} Render Server is preparing to stop")
try:
ZeroconfServer.stop()
RenderQueue.prepare_for_shutdown()
except Exception as e:
logger.exception(f"Exception during prepare for shutdown: {e}")
logger.info(f"{APP_NAME} Render Server has shut down")
#!/usr/bin/env python3
from init import run
if __name__ == '__main__':
server = ZordonServer()
try:
server.start_server()
server.api_server.join()
except KeyboardInterrupt:
pass
except Exception as e:
logger.fatal(f"Unhandled exception: {e}")
finally:
server.stop_server()
run(server_only=True)
server.spec
-158
View File
@@ -1,158 +0,0 @@
# -*- mode: python ; coding: utf-8 -*-
from PyInstaller.utils.hooks import collect_all
from pathlib import Path
import os
import sys
import platform
# ------------------------------------------------------------
# Project paths
# ------------------------------------------------------------
project_root = Path(SPECPATH).resolve()
src_dir = project_root / "src"
# Ensure `src.*` imports work during analysis
sys.path.insert(0, str(project_root))
# ------------------------------------------------------------
# Import version info
# ------------------------------------------------------------
from src.version import APP_NAME, APP_VERSION, APP_AUTHOR
APP_NAME = f"{APP_NAME}-server"
# ------------------------------------------------------------
# PyInstaller data / imports
# ------------------------------------------------------------
datas = [
("resources", "resources"),
("src/engines/blender/scripts", "src/engines/blender/scripts"),
]
binaries = []
hiddenimports = ["zeroconf", "src.version"]
tmp_ret = collect_all("zeroconf")
datas += tmp_ret[0]
binaries += tmp_ret[1]
hiddenimports += tmp_ret[2]
# ------------------------------------------------------------
# Analysis
# ------------------------------------------------------------
a = Analysis(
["server.py"],
pathex=[str(project_root)],
binaries=binaries,
datas=datas,
hiddenimports=hiddenimports,
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
noarchive=False,
optimize=1, # optimize=2 breaks Windows builds
)
pyz = PYZ(a.pure)
# ------------------------------------------------------------
# Platform targets
# ------------------------------------------------------------
if platform.system() == "Darwin": # macOS
exe = EXE(
pyz,
a.scripts,
[],
exclude_binaries=True,
name=APP_NAME,
debug=False,
bootloader_ignore_signals=False,
strip=True,
upx=True,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)
app = BUNDLE(
exe,
a.binaries,
a.datas,
strip=True,
name=f"{APP_NAME}.app",
icon="resources/Server.png",
bundle_identifier=None,
version=APP_VERSION,
)
elif platform.system() == "Windows":
import pyinstaller_versionfile
import tempfile
version_file_path = os.path.join(
tempfile.gettempdir(), "versionfile.txt"
)
pyinstaller_versionfile.create_versionfile(
output_file=version_file_path,
version=APP_VERSION,
company_name=APP_AUTHOR,
file_description=APP_NAME,
internal_name=APP_NAME,
legal_copyright=f"© {APP_AUTHOR}",
original_filename=f"{APP_NAME}.exe",
product_name=APP_NAME,
)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.datas,
[],
name=APP_NAME,
debug=False,
bootloader_ignore_signals=False,
strip=True,
upx=True,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
version=version_file_path,
)
else: # Linux
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.datas,
[],
name=APP_NAME,
debug=False,
bootloader_ignore_signals=False,
strip=True,
upx=True,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)
setup.py
+22
View File
@@ -0,0 +1,22 @@
"""
This is a setup.py script generated by py2applet
Usage:
python setup.py py2app
"""
import glob
from setuptools import setup
APP = ['main.py']
DATA_FILES = [('config', glob.glob('config/*.*')),
('resources', glob.glob('resources/*.*'))]
OPTIONS = {}
setup(
app=APP,
data_files=DATA_FILES,
options={'py2app': OPTIONS},
setup_requires=['py2app'],
name='Zordon'
)
src/api/add_job_helpers.py
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
import logging
import os
import shutil
import tempfile
import zipfile
from datetime import datetime
import requests
from tqdm import tqdm
from werkzeug.utils import secure_filename
logger = logging.getLogger()
def handle_uploaded_project_files(request, jobs_list, upload_directory):
"""
Handles the uploaded project files.
This method takes a request with a file, a list of jobs, and an upload directory. It checks if the file was uploaded
directly, if it needs to be downloaded from a URL, or if it's already present on the local file system. It then
moves the file to the appropriate directory and returns the local path to the file and its name.
Args:
request (Request): The request object containing the file.
jobs_list (list): A list of jobs. The first job in the list is used to get the file's URL and local path.
upload_directory (str): The directory where the file should be uploaded.
Raises:
ValueError: If no valid project paths are found.
Returns:
tuple: A tuple containing the local path to the loaded project file and its name.
"""
# Initialize default values
loaded_project_local_path = None
uploaded_project = request.files.get('file', None)
project_url = jobs_list[0].get('url', None)
local_path = jobs_list[0].get('local_path', None)
renderer = jobs_list[0].get('renderer')
downloaded_file_url = None
if uploaded_project and uploaded_project.filename:
referred_name = os.path.basename(uploaded_project.filename)
elif project_url:
referred_name, downloaded_file_url = download_project_from_url(project_url)
if not referred_name:
raise ValueError(f"Error downloading file from URL: {project_url}")
elif local_path and os.path.exists(local_path):
referred_name = os.path.basename(local_path)
else:
raise ValueError("Cannot find any valid project paths")
# Prepare the local filepath
cleaned_path_name = jobs_list[0].get('name', os.path.splitext(referred_name)[0]).replace(' ', '-')
job_dir = os.path.join(upload_directory, '-'.join(
[datetime.now().strftime("%Y.%m.%d_%H.%M.%S"), renderer, cleaned_path_name]))
os.makedirs(job_dir, exist_ok=True)
project_source_dir = os.path.join(job_dir, 'source')
os.makedirs(project_source_dir, exist_ok=True)
# Move projects to their work directories
if uploaded_project and uploaded_project.filename:
loaded_project_local_path = os.path.join(project_source_dir, secure_filename(uploaded_project.filename))
uploaded_project.save(loaded_project_local_path)
logger.info(f"Transfer complete for {loaded_project_local_path.split(upload_directory)[-1]}")
elif project_url:
loaded_project_local_path = os.path.join(project_source_dir, referred_name)
shutil.move(downloaded_file_url, loaded_project_local_path)
logger.info(f"Download complete for {loaded_project_local_path.split(upload_directory)[-1]}")
elif local_path:
loaded_project_local_path = os.path.join(project_source_dir, referred_name)
shutil.copy(local_path, loaded_project_local_path)
logger.info(f"Import complete for {loaded_project_local_path.split(upload_directory)[-1]}")
return loaded_project_local_path, referred_name
def download_project_from_url(project_url):
# This nested function is to handle downloading from a URL
logger.info(f"Downloading project from url: {project_url}")
referred_name = os.path.basename(project_url)
try:
response = requests.get(project_url, stream=True)
if response.status_code == 200:
# Get the total file size from the "Content-Length" header
file_size = int(response.headers.get("Content-Length", 0))
# Create a progress bar using tqdm
progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
# Open a file for writing in binary mode
downloaded_file_url = os.path.join(tempfile.gettempdir(), referred_name)
with open(downloaded_file_url, "wb") as file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
# Write the chunk to the file
file.write(chunk)
# Update the progress bar
progress_bar.update(len(chunk))
# Close the progress bar
progress_bar.close()
return referred_name, downloaded_file_url
except Exception as e:
logger.error(f"Error downloading file: {e}")
return None, None
def process_zipped_project(zip_path):
"""
Processes a zipped project.
This method takes a path to a zip file, extracts its contents, and returns the path to the extracted project file.
If the zip file contains more than one project file or none, an error is raised.
Args:
zip_path (str): The path to the zip file.
Raises:
ValueError: If there's more than 1 project file or none in the zip file.
Returns:
str: The path to the main project file.
"""
work_path = os.path.dirname(zip_path)
try:
with zipfile.ZipFile(zip_path, 'r') as myzip:
myzip.extractall(work_path)
project_files = [x for x in os.listdir(work_path) if os.path.isfile(os.path.join(work_path, x))]
project_files = [x for x in project_files if '.zip' not in x]
logger.debug(f"Zip files: {project_files}")
# supported_exts = RenderWorkerFactory.class_for_name(renderer).engine.supported_extensions
# if supported_exts:
# project_files = [file for file in project_files if any(file.endswith(ext) for ext in supported_exts)]
# If there's more than 1 project file or none, raise an error
if len(project_files) != 1:
raise ValueError(f'Cannot find a valid project file in {os.path.basename(zip_path)}')
extracted_project_path = os.path.join(work_path, project_files[0])
logger.info(f"Extracted zip file to {extracted_project_path}")
except (zipfile.BadZipFile, zipfile.LargeZipFile) as e:
logger.error(f"Error processing zip file: {e}")
raise ValueError(f"Error processing zip file: {e}")
return extracted_project_path
src/api/api_server.py
+279 -403
View File
@@ -2,99 +2,151 @@
import concurrent.futures
import json
import logging
import os
import pathlib
import shutil
import socket
import ssl
import tempfile
import time
import traceback
from datetime import datetime
from pathlib import Path
from typing import Dict, Any, Optional
from zipfile import ZipFile
import cpuinfo
import psutil
import yaml
from flask import Flask, request, send_file, after_this_request, Response, redirect, url_for
from flask import Flask, request, send_file, after_this_request, Response, redirect, url_for, abort
from sqlalchemy.orm.exc import DetachedInstanceError
from src.api.job_import_handler import JobImportHandler
from src.api.add_job_helpers import handle_uploaded_project_files, process_zipped_project
from src.api.preview_manager import PreviewManager
from src.distributed_job_manager import DistributedJobManager
from src.engines.core.base_worker import string_to_status, RenderStatus
from src.engines.engine_manager import EngineManager
from src.render_queue import RenderQueue, JobNotFoundError
from src.utilities.benchmark import cpu_benchmark, disk_io_benchmark
from src.utilities.config import Config
from src.utilities.misc_helper import current_system_os, current_system_cpu, \
current_system_os_version, num_to_alphanumeric, get_gpu_info
from src.utilities.status_utils import string_to_status
from src.version import APP_VERSION
from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu, \
current_system_os_version, num_to_alphanumeric
from src.utilities.zeroconf_server import ZeroconfServer
logger = logging.getLogger()
server = Flask(__name__)
ssl._create_default_https_context = ssl._create_unverified_context # disable SSL for downloads
API_VERSION = "0.1"
def start_api_server(hostname: Optional[str] = None) -> None:
# get hostname
if not hostname:
local_hostname = socket.gethostname()
hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
# load flask settings
server.config['HOSTNAME'] = hostname
server.config['PORT'] = int(Config.port_number)
server.config['UPLOAD_FOLDER'] = str(Path(Config.upload_folder).expanduser())
server.config['MAX_CONTENT_PATH'] = Config.max_content_path
server.config['enable_split_jobs'] = Config.enable_split_jobs
# disable most Flask logging
flask_log = logging.getLogger('werkzeug')
flask_log.setLevel(Config.flask_log_level.upper())
logger.debug('Starting API server')
try:
server.run(host=hostname, port=server.config['PORT'], debug=Config.flask_debug_enable, use_reloader=False,
threaded=True)
finally:
logger.debug('Stopping API server')
categories = [RenderStatus.RUNNING, RenderStatus.ERROR, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED,
RenderStatus.COMPLETED, RenderStatus.CANCELLED]
# --------------------------------------------
# Get All Jobs
# --------------------------------------------
# -- Error Handlers --
@server.errorhandler(JobNotFoundError)
def handle_job_not_found(job_error):
return str(job_error), 400
@server.errorhandler(DetachedInstanceError)
def handle_detached_instance(error):
# logger.debug(f"detached instance: {error}")
return "Unavailable", 503
@server.errorhandler(Exception)
def handle_general_error(general_error):
err_msg = f"Server error: {general_error}"
logger.error(err_msg)
return err_msg, 500
# -- Jobs --
def sorted_jobs(all_jobs, sort_by_date=True):
if not sort_by_date:
sorted_job_list = []
if all_jobs:
for status_category in categories:
found_jobs = [x for x in all_jobs if x.status == status_category.value]
if found_jobs:
sorted_found_jobs = sorted(found_jobs, key=lambda d: d.date_created, reverse=True)
sorted_job_list.extend(sorted_found_jobs)
else:
sorted_job_list = sorted(all_jobs, key=lambda d: d.date_created, reverse=True)
return sorted_job_list
@server.get('/api/jobs')
def jobs_json() -> Dict[str, Any]:
"""Retrieves all jobs from the render queue in JSON format.
This endpoint fetches all jobs currently in the render queue, converts them to JSON format,
and returns them along with a cache token that represents the current state of the job list.
Returns:
dict: A dictionary containing:
- 'jobs' (list[dict]): A list of job dictionaries, each representing a job in the queue.
- 'token' (str): A cache token generated from the hash of the job list.
"""
all_jobs = [x.json() for x in RenderQueue.all_jobs()]
job_cache_int = int(json.dumps(all_jobs).__hash__())
job_cache_token = num_to_alphanumeric(job_cache_int)
return {'jobs': all_jobs, 'token': job_cache_token}
def jobs_json():
try:
all_jobs = [x.json() for x in RenderQueue.all_jobs()]
job_cache_int = int(json.dumps(all_jobs).__hash__())
job_cache_token = num_to_alphanumeric(job_cache_int)
return {'jobs': all_jobs, 'token': job_cache_token}
except DetachedInstanceError as e:
raise e
except Exception as e:
logger.error(f"Error fetching jobs_json: {e}")
raise e
@server.get('/api/jobs_long_poll')
def long_polling_jobs():
hash_token = request.args.get('token', None)
start_time = time.time()
while True:
all_jobs = jobs_json()
if all_jobs['token'] != hash_token:
return all_jobs
# Break after 30 seconds to avoid gateway timeout
if time.time() - start_time > 30:
return {}, 204
time.sleep(1)
try:
hash_token = request.args.get('token', None)
start_time = time.time()
while True:
all_jobs = jobs_json()
if all_jobs['token'] != hash_token:
return all_jobs
# Break after 30 seconds to avoid gateway timeout
if time.time() - start_time > 30:
return {}, 204
time.sleep(1)
except DetachedInstanceError as e:
raise e
except Exception as e:
logger.error(f"Error fetching long_polling_jobs: {e}")
raise e
@server.route('/api/job/<job_id>/thumbnail')
def job_thumbnail(job_id):
try:
big_thumb = request.args.get('size', False) == "big"
video_ok = request.args.get('video_ok', False)
found_job = RenderQueue.job_with_id(job_id, none_ok=False)
# trigger a thumbnail update - just in case
PreviewManager.update_previews_for_job(found_job, wait_until_completion=True, timeout=60)
previews = PreviewManager.get_previews_for_job(found_job)
all_previews_list = previews.get('output', previews.get('input', []))
video_previews = [x for x in all_previews_list if x['kind'] == 'video']
image_previews = [x for x in all_previews_list if x['kind'] == 'image']
filtered_list = video_previews if video_previews and video_ok else image_previews
# todo - sort by size or other metrics here
if filtered_list:
preview_to_send = filtered_list[0]
mime_types = {'image': 'image/jpeg', 'video': 'video/mp4'}
file_mime_type = mime_types.get(preview_to_send['kind'], 'unknown')
return send_file(preview_to_send['filename'], mimetype=file_mime_type)
except Exception as e:
logger.error(f'Error getting thumbnail: {e}')
return f'Error getting thumbnail: {e}', 500
return "No thumbnail available", 404
# Get job file routing
@server.route('/api/job/<job_id>/file/<filename>', methods=['GET'])
def get_job_file(job_id, filename):
found_job = RenderQueue.job_with_id(job_id)
try:
for full_path in found_job.file_list():
if filename in full_path:
return send_file(path_or_file=full_path)
except FileNotFoundError:
abort(404)
@server.get('/api/jobs/<status_val>')
@@ -107,37 +159,28 @@ def filtered_jobs_json(status_val):
return f'Cannot find jobs with status {status_val}', 400
# --------------------------------------------
# Job Details / File Handling
# --------------------------------------------
@server.post('/api/job/<job_id>/send_subjob_update_notification')
def subjob_update_notification(job_id):
try:
subjob_details = request.json
logger.info(f"Subjob to job id: {job_id} is now {subjob_details['status']}")
DistributedJobManager.handle_subjob_update_notification(RenderQueue.job_with_id(job_id), subjob_data=subjob_details)
return Response(status=200)
except JobNotFoundError:
return "Job not found", 404
@server.get('/api/job/<job_id>')
def get_job_details(job_id):
"""Retrieves the details of a requested job in JSON format
Args:
job_id (str): The ID of the render job.
Returns:
dict: A JSON representation of the job's details.
"""
def get_job_status(job_id):
return RenderQueue.job_with_id(job_id).json()
@server.get('/api/job/<job_id>/logs')
def get_job_logs(job_id):
"""Retrieves the log file for a specific render job.
Args:
job_id (str): The ID of the render job.
Returns:
Response: The log file's content as plain text, or an empty response if the log file is not found.
"""
found_job = RenderQueue.job_with_id(job_id)
log_path = Path(found_job.log_path())
log_path = system_safe_path(found_job.log_path())
log_data = None
if log_path and log_path.exists():
if log_path and os.path.exists(log_path):
with open(log_path) as file:
log_data = file.read()
return Response(log_data, mimetype='text/plain')
@@ -145,61 +188,54 @@ def get_job_logs(job_id):
@server.get('/api/job/<job_id>/file_list')
def get_file_list(job_id):
return [Path(p).name for p in RenderQueue.job_with_id(job_id).file_list()]
return [os.path.basename(x) for x in RenderQueue.job_with_id(job_id).file_list()]
@server.route('/api/job/<job_id>/download')
def download_requested_file(job_id):
requested_filename = request.args.get("filename")
def download_file(job_id):
requested_filename = request.args.get('filename')
if not requested_filename:
return "Filename required", 400
return 'Filename required', 400
found_job = RenderQueue.job_with_id(job_id)
for job_file in found_job.file_list():
p = Path(job_file)
if p.name.lower() == requested_filename.lower():
return send_file(str(p), as_attachment=True)
for job_filename in found_job.file_list():
if os.path.basename(job_filename).lower() == requested_filename.lower():
return send_file(job_filename, as_attachment=True, )
return f"File '{requested_filename}' not found", 404
@server.route('/api/job/<job_id>/download_all')
def download_all_files(job_id):
def download_all(job_id):
zip_filename = None
@after_this_request
def clear_zip(response):
if zip_filename and zip_filename.exists():
if zip_filename and os.path.exists(zip_filename):
try:
zip_filename.unlink()
os.remove(zip_filename)
except Exception as e:
logger.warning(f"Error removing zip file '{zip_filename}': {e}")
return response
found_job = RenderQueue.job_with_id(job_id)
output_dir = os.path.dirname(found_job.output_path)
if os.path.exists(output_dir):
zip_filename = system_safe_path(os.path.join(tempfile.gettempdir(),
pathlib.Path(found_job.input_path).stem + '.zip'))
with ZipFile(zip_filename, 'w') as zipObj:
for f in os.listdir(output_dir):
zipObj.write(filename=system_safe_path(os.path.join(output_dir, f)),
arcname=os.path.basename(f))
return send_file(zip_filename, mimetype="zip", as_attachment=True, )
else:
return f'Cannot find project files for job {job_id}', 500
output_dir = Path(found_job.output_path).parent
if not output_dir.exists():
return f"Cannot find project files for job {job_id}", 500
zip_filename = Path(tempfile.gettempdir()) / f"{Path(found_job.input_path).stem}.zip"
from zipfile import ZipFile
with ZipFile(zip_filename, "w") as zipObj:
for f in output_dir.iterdir():
if f.is_file():
zipObj.write(f, arcname=f.name)
return send_file(str(zip_filename), mimetype="zip", as_attachment=True)
# --------------------------------------------
# System Environment / Status
# --------------------------------------------
@server.get('/api/presets')
def presets() -> Dict[str, Any]:
presets_path = Path('config/presets.yaml')
def presets():
presets_path = system_safe_path('config/presets.yaml')
with open(presets_path) as f:
loaded_presets = yaml.load(f, Loader=yaml.FullLoader)
return loaded_presets
@@ -228,81 +264,41 @@ def snapshot():
return server_data
@server.route('/api/status')
def status():
return {"timestamp": datetime.now().isoformat(),
"system_os": current_system_os(),
"system_os_version": current_system_os_version(),
"system_cpu": current_system_cpu(),
"system_cpu_brand": cpuinfo.get_cpu_info()['brand_raw'],
"cpu_percent": psutil.cpu_percent(percpu=False),
"cpu_percent_per_cpu": psutil.cpu_percent(percpu=True),
"cpu_count": psutil.cpu_count(logical=False),
"memory_total": psutil.virtual_memory().total,
"memory_available": psutil.virtual_memory().available,
"memory_percent": psutil.virtual_memory().percent,
"job_counts": RenderQueue.job_counts(),
"hostname": server.config['HOSTNAME'],
"port": server.config['PORT'],
"app_version": APP_VERSION,
"api_version": API_VERSION
}
@server.get('/api/_detected_clients')
def detected_clients():
# todo: dev/debug only. Should not ship this - probably.
return ZeroconfServer.found_hostnames()
# --------------------------------------------
# Job Lifecyle (Create, Cancel, Delete)
# --------------------------------------------
# New version
@server.post('/api/add_job')
def add_job_handler():
"""
POST /api/add_job
Add a render job to the queue.
**Request Formats**
- JSON body:
{
"name": "example.blend",
"engine": "blender",
"frame_start": 1,
"frame_end": 100,
"render_settings": {...}
"child_jobs"; [...]
}
**Responses**
200 Success
400 Invalid or missing input
500 Internal server error while parsing or creating jobs
"""
# Process request data
try:
if request.is_json:
new_job_data = request.get_json()
jobs_list = [request.json] if not isinstance(request.json, list) else request.json
elif request.form.get('json', None):
new_job_data = json.loads(request.form['json'])
jobs_list = json.loads(request.form['json'])
else:
return "Cannot find valid job data", 400
return "Invalid data", 400
except Exception as e:
err_msg = f"Error processing job data: {e}"
logger.error(err_msg)
return err_msg, 500
# Validate Job Data - check for required values and download or unzip project files
try:
processed_job_data = JobImportHandler.validate_job_data(new_job_data, server.config['UPLOAD_FOLDER'],
uploaded_file=request.files.get('file'))
except (KeyError, FileNotFoundError) as e:
err_msg = f"Error processing job data: {e}"
return err_msg, 400
except Exception as e:
traceback.print_exception(e)
err_msg = f"Unknown error processing data: {e}"
return err_msg, 500
loaded_project_local_path, referred_name = handle_uploaded_project_files(request, jobs_list,
server.config['UPLOAD_FOLDER'])
if loaded_project_local_path.lower().endswith('.zip'):
loaded_project_local_path = process_zipped_project(loaded_project_local_path)
try:
return JobImportHandler.create_jobs_from_processed_data(processed_job_data)
results = []
for new_job_data in jobs_list:
new_job = DistributedJobManager.create_render_job(new_job_data, loaded_project_local_path)
results.append(new_job.json())
return results, 200
except Exception as e:
logger.exception(f"Error creating render job: {e}")
logger.exception(f"Error adding job: {e}")
return 'unknown error', 500
@@ -323,190 +319,126 @@ def cancel_job(job_id):
@server.route('/api/job/<job_id>/delete', methods=['POST', 'GET'])
def delete_job(job_id):
try:
if not request.args.get("confirm", False):
return "Confirmation required to delete job", 400
if not request.args.get('confirm', False):
return 'Confirmation required to delete job', 400
# Check if we can remove the 'output' directory
found_job = RenderQueue.job_with_id(job_id)
input_path = Path(found_job.input_path)
output_path = Path(found_job.output_path)
upload_root = Path(server.config["UPLOAD_FOLDER"])
project_dir = input_path.parent.parent
output_dir = output_path.parent
found_job.stop()
output_dir = os.path.dirname(found_job.output_path)
if server.config['UPLOAD_FOLDER'] in output_dir and os.path.exists(output_dir):
shutil.rmtree(output_dir)
try:
PreviewManager.delete_previews_for_job(found_job)
except Exception as e:
logger.error(f"Error deleting previews for {found_job}: {e}")
# See if we own the project_dir (i.e. was it uploaded)
project_dir = os.path.dirname(os.path.dirname(found_job.input_path))
if server.config['UPLOAD_FOLDER'] in project_dir and os.path.exists(project_dir):
# check to see if any other projects are sharing the same project file
project_dir_files = [f for f in os.listdir(project_dir) if not f.startswith('.')]
if len(project_dir_files) == 0 or (len(project_dir_files) == 1 and 'source' in project_dir_files[0]):
logger.info(f"Removing project directory: {project_dir}")
shutil.rmtree(project_dir)
RenderQueue.delete_job(found_job)
# Delete output directory if we own it
if output_dir.exists() and output_dir.is_relative_to(upload_root):
shutil.rmtree(output_dir)
# Delete project directory if we own it and it's unused
try:
if project_dir.exists() and project_dir.is_relative_to(upload_root):
project_dir_files = [p for p in project_dir.iterdir() if not p.name.startswith(".")]
if not project_dir_files or (len(project_dir_files) == 1 and "source" in project_dir_files[0].name):
logger.info(f"Removing project directory: {project_dir}")
shutil.rmtree(project_dir)
except Exception as e:
logger.error(f"Error removing project files: {e}")
return "Job deleted", 200
if request.args.get('redirect', False):
return redirect(url_for('index'))
else:
return "Job deleted", 200
except Exception as e:
logger.error(f"Error deleting job: {e}")
return f"Error deleting job: {e}", 500
# --------------------------------------------
# Engine Info and Management:
# --------------------------------------------
@server.get('/api/engine_for_filename')
def get_engine_for_filename():
filename = request.args.get("filename")
if not filename:
return "Error: filename is required", 400
found_engine = EngineManager.engine_class_for_project_path(filename)
if not found_engine:
return f"Error: cannot find a suitable engine for '{filename}'", 400
return found_engine.name()
@server.get('/api/installed_engines')
def get_installed_engines():
result = {}
for engine_class in EngineManager.supported_engines():
data = EngineManager.all_version_data_for_engine(engine_class.name())
if data:
result[engine_class.name()] = data
return result
@server.get('/api/clear_history')
def clear_history():
RenderQueue.clear_history()
return 'success'
@server.get('/api/engine_info')
def engine_info():
@server.route('/api/status')
def status():
# Get system info
return {"timestamp": datetime.now().isoformat(),
"system_os": current_system_os(),
"system_os_version": current_system_os_version(),
"system_cpu": current_system_cpu(),
"cpu_percent": psutil.cpu_percent(percpu=False),
"cpu_percent_per_cpu": psutil.cpu_percent(percpu=True),
"cpu_count": psutil.cpu_count(logical=False),
"memory_total": psutil.virtual_memory().total,
"memory_available": psutil.virtual_memory().available,
"memory_percent": psutil.virtual_memory().percent,
"job_counts": RenderQueue.job_counts(),
"hostname": server.config['HOSTNAME'],
"port": server.config['PORT']
}
@server.get('/api/renderer_info')
def renderer_info():
response_type = request.args.get('response_type', 'standard')
if response_type not in ['full', 'standard']:
raise ValueError(f"Invalid response_type: {response_type}")
def process_engine(engine):
try:
# Get all installed versions of the engine
installed_versions = EngineManager.all_version_data_for_engine(engine.name())
if not installed_versions:
return None
installed_versions = EngineManager.all_versions_for_engine(engine.name())
if installed_versions:
# Use system-installed versions to avoid permission issues
system_installed_versions = [x for x in installed_versions if x['type'] == 'system']
install_path = system_installed_versions[0]['path'] if system_installed_versions else \
installed_versions[0]['path']
system_installed_versions = [v for v in installed_versions if v['type'] == 'system']
install_path = system_installed_versions[0]['path'] if system_installed_versions else installed_versions[0]['path']
en = engine(install_path)
en = engine(install_path)
engine_name = en.name()
result = {
engine_name: {
'is_available': RenderQueue.is_available_for_job(engine_name),
'versions': installed_versions
}
}
if response_type == 'full':
with concurrent.futures.ThreadPoolExecutor() as executor:
future_results = {
'supported_extensions': executor.submit(en.supported_extensions),
'supported_export_formats': executor.submit(en.get_output_formats),
'system_info': executor.submit(en.system_info)
if response_type == 'full': # Full dataset - Can be slow
return {
en.name(): {
'is_available': RenderQueue.is_available_for_job(en.name()),
'versions': installed_versions,
'supported_extensions': engine.supported_extensions(),
'supported_export_formats': en.get_output_formats(),
'system_info': en.system_info()
}
}
for key, future in future_results.items():
result[engine_name][key] = future.result()
return result
elif response_type == 'standard': # Simpler dataset to reduce response times
return {
en.name(): {
'is_available': RenderQueue.is_available_for_job(en.name()),
'versions': installed_versions,
}
}
else:
raise AttributeError(f"Invalid response_type: {response_type}")
except Exception as e:
traceback.print_exc(e)
logger.error(f"Error fetching details for engine '{engine.name()}': {e}")
logger.error(f'Error fetching details for {engine.name()} renderer: {e}')
return {}
engine_data = {}
renderer_data = {}
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {executor.submit(process_engine, engine): engine.name() for engine in EngineManager.supported_engines()}
for future in concurrent.futures.as_completed(futures):
result = future.result()
if result:
engine_data.update(result)
renderer_data.update(result)
return engine_data
@server.get('/api/<engine_name>/info')
def get_engine_info(engine_name):
try:
response_type = request.args.get('response_type', 'standard')
# Get all installed versions of the engine
installed_versions = EngineManager.all_version_data_for_engine(engine_name)
if not installed_versions:
return {}
result = { 'is_available': RenderQueue.is_available_for_job(engine_name),
'versions': installed_versions
}
if response_type == 'full':
with concurrent.futures.ThreadPoolExecutor() as executor:
engine_class = EngineManager.engine_class_with_name(engine_name)
en = EngineManager.get_latest_engine_instance(engine_class)
future_results = {
'supported_extensions': executor.submit(en.supported_extensions),
'supported_export_formats': executor.submit(en.get_output_formats),
'system_info': executor.submit(en.system_info),
'options': executor.submit(en.ui_options)
}
for key, future in future_results.items():
result[key] = future.result()
return result
except Exception as e:
logger.error(f"Error fetching details for engine '{engine_name}': {e}")
return {}
return renderer_data
@server.get('/api/<engine_name>/is_available')
def is_engine_available(engine_name):
return {'engine': engine_name, 'available': RenderQueue.is_available_for_job(engine_name),
'cpu_count': int(psutil.cpu_count(logical=False)),
'versions': EngineManager.all_version_data_for_engine(engine_name),
'versions': EngineManager.all_versions_for_engine(engine_name),
'hostname': server.config['HOSTNAME']}
@server.get('/api/engine/<engine_name>/args')
def get_engine_args(engine_name):
try:
engine_class = EngineManager.engine_class_with_name(engine_name)
return engine_class().get_arguments()
except LookupError:
return f"Cannot find engine '{engine_name}'", 400
@server.get('/api/engine/<engine_name>/help')
def get_engine_help(engine_name):
try:
engine_class = EngineManager.engine_class_with_name(engine_name)
return engine_class().get_help()
except LookupError:
return f"Cannot find engine '{engine_name}'", 400
# --------------------------------------------
# Engine Downloads and Updates:
# --------------------------------------------
@server.get('/api/is_engine_available_to_download')
def is_engine_available_to_download():
available_result = EngineManager.version_is_available_to_download(request.args.get('engine'),
@@ -547,109 +479,53 @@ def delete_engine_download():
(f"Error deleting {json_data.get('engine')} {json_data.get('version')}", 500)
# --------------------------------------------
# Miscellaneous:
# --------------------------------------------
@server.get('/api/heartbeat')
def heartbeat():
return datetime.now().isoformat(), 200
@server.post('/api/job/<job_id>/send_subjob_update_notification')
def subjob_update_notification(job_id):
subjob_details = request.json
DistributedJobManager.handle_subjob_update_notification(RenderQueue.job_with_id(job_id), subjob_data=subjob_details)
return Response(status=200)
@server.route('/api/job/<job_id>/thumbnail')
def job_thumbnail(job_id):
@server.get('/api/renderer/<renderer>/args')
def get_renderer_args(renderer):
try:
big_thumb = request.args.get('size', False) == "big"
video_ok = request.args.get('video_ok', False)
found_job = RenderQueue.job_with_id(job_id, none_ok=False)
# trigger a thumbnail update - just in case
PreviewManager.update_previews_for_job(found_job, wait_until_completion=True, timeout=60)
previews = PreviewManager.get_previews_for_job(found_job)
all_previews_list = previews.get('output', previews.get('input', []))
video_previews = [x for x in all_previews_list if x['kind'] == 'video']
image_previews = [x for x in all_previews_list if x['kind'] == 'image']
filtered_list = video_previews if video_previews and video_ok else image_previews
# todo - sort by size or other metrics here
if filtered_list:
preview_to_send = filtered_list[0]
mime_types = {'image': 'image/jpeg', 'video': 'video/mp4'}
file_mime_type = mime_types.get(preview_to_send['kind'], 'unknown')
return send_file(preview_to_send['filename'], mimetype=file_mime_type)
except Exception as e:
logger.error(f'Error getting thumbnail: {e}')
return f'Error getting thumbnail: {e}', 500
return "No thumbnail available", 404
renderer_engine_class = EngineManager.engine_with_name(renderer)
return renderer_engine_class().get_arguments()
except LookupError:
return f"Cannot find renderer '{renderer}'", 400
# --------------------------------------------
# System Benchmarks:
# --------------------------------------------
@server.get('/api/renderer/<renderer>/help')
def get_renderer_help(renderer):
try:
renderer_engine_class = EngineManager.engine_with_name(renderer)
return renderer_engine_class().get_help()
except LookupError:
return f"Cannot find renderer '{renderer}'", 400
@server.get('/api/cpu_benchmark')
def get_cpu_benchmark_score():
from src.utilities.benchmark import cpu_benchmark
return str(cpu_benchmark(10))
@server.get('/api/disk_benchmark')
def get_disk_benchmark():
from src.utilities.benchmark import disk_io_benchmark
results = disk_io_benchmark()
return {'write_speed': results[0], 'read_speed': results[-1]}
# --------------------------------------------
# Error Handlers:
# --------------------------------------------
def start_server(hostname=None):
@server.errorhandler(JobNotFoundError)
def handle_job_not_found(job_error):
return str(job_error), 400
# get hostname
if not hostname:
local_hostname = socket.gethostname()
hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
# load flask settings
server.config['HOSTNAME'] = hostname
server.config['PORT'] = int(Config.port_number)
server.config['UPLOAD_FOLDER'] = system_safe_path(os.path.expanduser(Config.upload_folder))
server.config['MAX_CONTENT_PATH'] = Config.max_content_path
server.config['enable_split_jobs'] = Config.enable_split_jobs
@server.errorhandler(DetachedInstanceError)
def handle_detached_instance(_):
return "Unavailable", 503
# disable most Flask logging
flask_log = logging.getLogger('werkzeug')
flask_log.setLevel(Config.flask_log_level.upper())
@server.errorhandler(404)
def handle_404(error):
url = request.url
err_msg = f"404 Not Found: {url}"
if 'favicon' not in url:
logger.warning(err_msg)
return err_msg, 404
@server.errorhandler(Exception)
def handle_general_error(general_error):
traceback.print_exception(type(general_error), general_error, general_error.__traceback__)
err_msg = f"Server error: {general_error}"
logger.error(err_msg)
return err_msg, 500
# --------------------------------------------
# Debug / Development Only:
# --------------------------------------------
@server.get('/api/_debug/detected_clients')
def detected_clients():
# todo: dev/debug only. Should not ship this - probably.
from src.utilities.zeroconf_server import ZeroconfServer
return ZeroconfServer.found_hostnames()
@server.get('/api/_debug/clear_history')
def clear_history():
RenderQueue.clear_history()
return 'success'
logger.debug('Starting API server')
server.run(host='0.0.0.0', port=server.config['PORT'], debug=Config.flask_debug_enable, use_reloader=False,
threaded=True)
src/api/job_import_handler.py
-221
View File
@@ -1,221 +0,0 @@
#!/usr/bin/env python3
import logging
import os
import shutil
import tempfile
import zipfile
from datetime import datetime
from pathlib import Path
import requests
from tqdm import tqdm
from werkzeug.utils import secure_filename
from src.distributed_job_manager import DistributedJobManager
logger = logging.getLogger()
class JobImportHandler:
"""Handles job import operations for rendering projects.
This class provides functionality to validate, download, and process
job data and project files for the rendering queue system.
"""
@classmethod
def create_jobs_from_processed_data(cls, processed_job_data: dict) -> list[dict]:
""" Takes processed job data and creates new jobs
Args: processed_job_data: Dictionary containing job information"""
loaded_project_local_path = processed_job_data['__loaded_project_local_path']
# prepare child job data
job_data_to_create = []
if processed_job_data.get("child_jobs"):
for child_job_diffs in processed_job_data["child_jobs"]:
processed_child_job_data = processed_job_data.copy()
processed_child_job_data.pop("child_jobs")
processed_child_job_data.update(child_job_diffs)
job_data_to_create.append(processed_child_job_data)
else:
job_data_to_create.append(processed_job_data)
# create the jobs
created_jobs = []
for job_data in job_data_to_create:
new_job = DistributedJobManager.create_render_job(job_data, loaded_project_local_path)
created_jobs.append(new_job)
# Save notes to .txt
if processed_job_data.get("notes"):
parent_dir = Path(loaded_project_local_path).parent.parent
notes_name = processed_job_data['name'] + "-notes.txt"
with (Path(parent_dir) / notes_name).open("w") as f:
f.write(processed_job_data["notes"])
return [x.json() for x in created_jobs]
@classmethod
def validate_job_data(cls, new_job_data: dict, upload_directory: Path, uploaded_file=None) -> dict:
"""Validates and prepares job data for import.
This method validates the job data dictionary, handles project file
acquisition (upload, download, or local copy), and prepares the job
directory structure.
Args:
new_job_data: Dictionary containing job information including
'name', 'engine_name', and optionally 'url' or 'local_path'.
upload_directory: Base directory for storing uploaded jobs.
uploaded_file: Optional uploaded file object from the request.
Returns:
The validated job data dictionary with additional metadata.
Raises:
KeyError: If required fields 'name' or 'engine_name' are missing.
FileNotFoundError: If no valid project file can be found.
"""
loaded_project_local_path = None
# check for required keys
job_name = new_job_data.get('name')
engine_name = new_job_data.get('engine_name')
if not job_name:
raise KeyError("Missing job name")
if not engine_name:
raise KeyError("Missing engine name")
project_url = new_job_data.get('url', None)
local_path = new_job_data.get('local_path', None)
downloaded_file_url = None
if uploaded_file and uploaded_file.filename:
referred_name = os.path.basename(uploaded_file.filename)
elif project_url:
referred_name, downloaded_file_url = cls.download_project_from_url(project_url)
if not referred_name:
raise FileNotFoundError(f"Error downloading file from URL: {project_url}")
elif local_path and os.path.exists(local_path):
referred_name = os.path.basename(local_path)
else:
raise FileNotFoundError("Cannot find any valid project paths")
# Prepare the local filepath
cleaned_path_name = job_name.replace(' ', '-')
folder_name = f"{cleaned_path_name}-{engine_name}-{datetime.now().strftime('%Y.%m.%d_%H.%M.%S')}"
job_dir = Path(upload_directory) / folder_name
os.makedirs(job_dir, exist_ok=True)
project_source_dir = Path(job_dir) / 'source'
os.makedirs(project_source_dir, exist_ok=True)
# Move projects to their work directories
if uploaded_file and uploaded_file.filename:
# Handle file uploading
filename = secure_filename(uploaded_file.filename)
loaded_project_local_path = Path(project_source_dir) / filename
uploaded_file.save(str(loaded_project_local_path))
logger.info(f"Transfer complete for {loaded_project_local_path.relative_to(upload_directory)}")
elif project_url:
# Handle downloading project from a URL
loaded_project_local_path = Path(project_source_dir) / referred_name
shutil.move(str(downloaded_file_url), str(loaded_project_local_path))
logger.info(f"Download complete for {loaded_project_local_path.relative_to(upload_directory)}")
elif local_path:
# Handle local files
loaded_project_local_path = Path(project_source_dir) / referred_name
shutil.copy(str(local_path), str(loaded_project_local_path))
logger.info(f"Import complete for {loaded_project_local_path.relative_to(upload_directory)}")
if loaded_project_local_path.suffix == ".zip":
loaded_project_local_path = cls.process_zipped_project(loaded_project_local_path)
new_job_data["__loaded_project_local_path"] = loaded_project_local_path
return new_job_data
@staticmethod
def download_project_from_url(project_url: str):
"""Downloads a project file from the given URL.
Downloads the file from the specified URL to a temporary directory
with progress tracking. Returns the filename and temporary path.
Args:
project_url: The URL to download the project file from.
Returns:
A tuple of (filename, temp_file_path) if successful,
(None, None) if download fails.
"""
# This nested function is to handle downloading from a URL
logger.info(f"Downloading project from url: {project_url}")
referred_name = os.path.basename(project_url)
try:
response = requests.get(project_url, stream=True, timeout=300)
if response.status_code == 200:
# Get the total file size from the "Content-Length" header
file_size = int(response.headers.get("Content-Length", 0))
# Create a progress bar using tqdm
progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
# Open a file for writing in binary mode
downloaded_file_url = os.path.join(tempfile.gettempdir(), referred_name)
with open(downloaded_file_url, "wb") as file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
# Write the chunk to the file
file.write(chunk)
# Update the progress bar
progress_bar.update(len(chunk))
# Close the progress bar
progress_bar.close()
return referred_name, downloaded_file_url
except Exception as e:
logger.error(f"Error downloading file: {e}")
return None, None
@staticmethod
def process_zipped_project(zip_path: Path) -> Path:
"""
Processes a zipped project.
This method takes a path to a zip file, extracts its contents, and returns the path to the extracted project
file. If the zip file contains more than one project file or none, an error is raised.
Args:
zip_path (Path): The path to the zip file.
Raises:
ValueError: If there's more than 1 project file or none in the zip file.
Returns:
Path: The path to the main project file.
"""
work_path = zip_path.parent
try:
with zipfile.ZipFile(zip_path, 'r') as myzip:
myzip.extractall(str(work_path))
project_files = [p for p in work_path.iterdir() if p.is_file() and p.suffix.lower() != ".zip"]
logger.debug(f"Zip files: {project_files}")
# supported_exts = RenderWorkerFactory.class_for_name(engine).engine.supported_extensions
# if supported_exts:
# project_files = [file for file in project_files if any(file.endswith(ext) for ext in supported_exts)]
# If there's more than 1 project file or none, raise an error
if len(project_files) != 1:
raise ValueError(f'Cannot find a valid project file in {zip_path.name}')
extracted_project_path = work_path / project_files[0]
logger.info(f"Extracted zip file to {extracted_project_path}")
except (zipfile.BadZipFile, zipfile.LargeZipFile) as e:
logger.error(f"Error processing zip file: {e}")
raise ValueError(f'Error processing zip file: {e}') from e
return extracted_project_path
src/api/preview_manager.py
+18 -42
View File
@@ -3,30 +3,21 @@ import os
import subprocess
import threading
from pathlib import Path
from typing import Dict, Optional
from src.utilities.ffmpeg_helper import generate_thumbnail, save_first_frame
logger = logging.getLogger()
supported_video_formats = ['.mp4', '.mov', '.avi', '.mpg', '.mpeg', '.mxf', '.m4v', '.mkv', '.webm']
supported_image_formats = ['.jpg', '.png', '.exr', '.tif', '.tga', '.bmp', '.webp']
supported_video_formats = ['.mp4', '.mov', '.avi', '.mpg', '.mpeg', '.mxf', '.m4v', 'mkv']
supported_image_formats = ['.jpg', '.png', '.exr', '.tif']
class PreviewManager:
_default_instance: Optional['PreviewManager'] = None
storage_path: Optional[str] = None
def __init__(self) -> None:
self.storage_path = None
self._running_jobs: Dict = {}
storage_path = None
_running_jobs = {}
@classmethod
def _sync_class(cls) -> None:
if cls._default_instance is not None:
cls.storage_path = cls._default_instance.storage_path
def _generate_job_preview_worker(self, job, replace_existing=False, max_width=480):
def __generate_job_preview_worker(cls, job, replace_existing=False, max_width=320):
# Determine best source file to use for thumbs
job_file_list = job.file_list()
@@ -42,8 +33,8 @@ class PreviewManager:
logger.warning(f"No valid image or video files found in files from job: {job}")
return
os.makedirs(self.storage_path, exist_ok=True)
base_path = os.path.join(self.storage_path, f"{job.id}-{preview_label}-{max_width}")
os.makedirs(cls.storage_path, exist_ok=True)
base_path = os.path.join(cls.storage_path, f"{job.id}-{preview_label}-{max_width}")
preview_video_path = base_path + '.mp4'
preview_image_path = base_path + '.jpg'
@@ -74,23 +65,25 @@ class PreviewManager:
except subprocess.CalledProcessError as e:
logger.error(f"Error generating video preview for {job}: {e}")
def _update_previews_for_job(self, job, replace_existing=False, wait_until_completion=False, timeout=None):
job_thread = self._running_jobs.get(job.id)
@classmethod
def update_previews_for_job(cls, job, replace_existing=False, wait_until_completion=False, timeout=None):
job_thread = cls._running_jobs.get(job.id)
if job_thread and job_thread.is_alive():
logger.debug(f'Preview generation job already running for {job}')
else:
job_thread = threading.Thread(target=self._generate_job_preview_worker, args=(job, replace_existing,))
job_thread = threading.Thread(target=cls.__generate_job_preview_worker, args=(job, replace_existing,))
job_thread.start()
self._running_jobs[job.id] = job_thread
cls._running_jobs[job.id] = job_thread
if wait_until_completion:
job_thread.join(timeout=timeout)
def _get_previews_for_job(self, job):
@classmethod
def get_previews_for_job(cls, job):
results = {}
try:
directory_path = Path(self.storage_path)
directory_path = Path(cls.storage_path)
preview_files_for_job = [f for f in directory_path.iterdir() if f.is_file() and f.name.startswith(job.id)]
for preview_filename in preview_files_for_job:
@@ -108,8 +101,9 @@ class PreviewManager:
pass
return results
def _delete_previews_for_job(self, job):
all_previews = self.get_previews_for_job(job)
@classmethod
def delete_previews_for_job(cls, job):
all_previews = cls.get_previews_for_job(job)
flattened_list = [item for sublist in all_previews.values() for item in sublist]
for preview in flattened_list:
try:
@@ -117,21 +111,3 @@ class PreviewManager:
os.remove(preview['filename'])
except OSError as e:
logger.error(f"Error removing preview '{preview.get('filename')}': {e}")
# --- Forwarders for backward compatibility ---
@classmethod
def update_previews_for_job(cls, job, replace_existing=False, wait_until_completion=False, timeout=None):
if cls._default_instance is not None:
cls._default_instance._update_previews_for_job(job, replace_existing, wait_until_completion, timeout)
@classmethod
def get_previews_for_job(cls, job):
if cls._default_instance is not None:
return cls._default_instance._get_previews_for_job(job)
return {}
@classmethod
def delete_previews_for_job(cls, job):
if cls._default_instance is not None:
cls._default_instance._delete_previews_for_job(job)
src/api/server_proxy.py
+115 -151
View File
@@ -3,7 +3,6 @@ import logging
import os
import threading
import time
from pathlib import Path
import requests
from requests_toolbelt.multipart import MultipartEncoder, MultipartEncoderMonitor
@@ -11,6 +10,7 @@ from urllib.parse import urljoin
from src.utilities.misc_helper import is_localhost
from src.utilities.status_utils import RenderStatus
from src.utilities.zeroconf_server import ZeroconfServer
status_colors = {RenderStatus.ERROR: "red", RenderStatus.CANCELLED: 'orange1', RenderStatus.COMPLETED: 'green',
RenderStatus.NOT_STARTED: "yellow", RenderStatus.SCHEDULED: 'purple',
@@ -22,11 +22,19 @@ categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStat
logger = logging.getLogger()
OFFLINE_MAX = 4
LOOPBACK = '127.0.0.1'
class RenderServerProxy:
"""The ServerProxy class is responsible for interacting with a remote server.
It provides convenience methods to request data from the server and store the status of the server.
"""
The ServerProxy class is responsible for interacting with a remote server.
It provides methods to request data from the server and store the status of the server.
Attributes:
system_cpu (str): The CPU type of the system.
system_cpu_count (int): The number of CPUs in the system.
system_os (str): The operating system of the system.
system_os_version (str): The version of the operating system.
"""
def __init__(self, hostname, server_port="8080"):
@@ -46,27 +54,18 @@ class RenderServerProxy:
self.system_cpu_count = None
self.system_os = None
self.system_os_version = None
self.system_api_version = None
# --------------------------------------------
# Basics / Connection:
# --------------------------------------------
def __repr__(self):
return f"<RenderServerProxy - {self.hostname}>"
def check_connection(self):
try:
return self.request("heartbeat").ok
except Exception:
pass
return False
def connect(self):
return self.status()
def is_online(self):
if self.__update_in_background:
return self.__offline_flags < OFFLINE_MAX
else:
return self.check_connection()
return self.get_status() is not None
def status(self):
if not self.is_online():
@@ -74,10 +73,6 @@ class RenderServerProxy:
running_jobs = [x for x in self.__jobs_cache if x['status'] == 'running'] if self.__jobs_cache else []
return f"{len(running_jobs)} running" if running_jobs else "Ready"
# --------------------------------------------
# Requests:
# --------------------------------------------
def request_data(self, payload, timeout=5):
try:
req = self.request(payload, timeout)
@@ -105,13 +100,8 @@ class RenderServerProxy:
return None
def request(self, payload, timeout=5):
from src.api.api_server import API_VERSION
return requests.get(f'http://{self.hostname}:{self.port}/api/{payload}', timeout=timeout,
headers={"X-API-Version": str(API_VERSION)})
# --------------------------------------------
# Background Updates:
# --------------------------------------------
hostname = LOOPBACK if self.is_localhost else self.hostname
return requests.get(f'http://{hostname}:{self.port}/api/{payload}', timeout=timeout)
def start_background_update(self):
if self.__update_in_background:
@@ -129,6 +119,17 @@ class RenderServerProxy:
self.__background_thread.daemon = True
self.__background_thread.start()
def stop_background_update(self):
self.__update_in_background = False
def get_job_info(self, job_id, timeout=5):
return self.request_data(f'job/{job_id}', timeout=timeout)
def get_all_jobs(self, timeout=5, ignore_token=False):
if not self.__update_in_background or ignore_token:
self.__update_job_cache(timeout, ignore_token)
return self.__jobs_cache.copy() if self.__jobs_cache else None
def __update_job_cache(self, timeout=40, ignore_token=False):
if self.__offline_flags: # if we're offline, don't bother with the long poll
@@ -146,21 +147,15 @@ class RenderServerProxy:
self.__jobs_cache = sorted_jobs
self.__jobs_cache_token = status_result['token']
def stop_background_update(self):
self.__update_in_background = False
# --------------------------------------------
# Get System Info:
# --------------------------------------------
def get_all_jobs(self, timeout=5, ignore_token=False):
if not self.__update_in_background or ignore_token:
self.__update_job_cache(timeout, ignore_token)
return self.__jobs_cache.copy() if self.__jobs_cache else None
def get_data(self, timeout=5):
return self.request_data('full_status', timeout=timeout)
def cancel_job(self, job_id, confirm=False):
return self.request_data(f'job/{job_id}/cancel?confirm={confirm}')
def delete_job(self, job_id, confirm=False):
return self.request_data(f'job/{job_id}/delete?confirm={confirm}')
def get_status(self):
status = self.request_data('status')
if status and not self.system_cpu:
@@ -168,67 +163,13 @@ class RenderServerProxy:
self.system_cpu_count = status['cpu_count']
self.system_os = status['system_os']
self.system_os_version = status['system_os_version']
self.system_api_version = status['api_version']
return status
# --------------------------------------------
# Get Job Info:
# --------------------------------------------
def is_engine_available(self, engine_name):
return self.request_data(f'{engine_name}/is_available')
def get_job_info(self, job_id, timeout=5):
return self.request_data(f'job/{job_id}', timeout=timeout)
def get_job_files_list(self, job_id):
return self.request_data(f"job/{job_id}/file_list")
# --------------------------------------------
# Job Lifecycle:
# --------------------------------------------
def post_job_to_server(self, file_path: Path, job_data, callback=None):
"""
Posts a job to the server.
Args:
file_path (Path): The path to the file to upload.
job_data (dict): A dict of jobs data.
callback (function, optional): A callback function to call during the upload. Defaults to None.
Returns:
Response: The response from the server.
"""
# Check if file exists
if not file_path.exists():
raise FileNotFoundError(f"File not found: {file_path}")
# Bypass uploading file if posting to localhost
if self.is_localhost:
job_data['local_path'] = str(file_path)
url = urljoin(f'http://{self.hostname}:{self.port}', '/api/add_job')
headers = {'Content-Type': 'application/json'}
return requests.post(url, data=json.dumps(job_data), headers=headers)
# Prepare the form data for remote host
with open(file_path, 'rb') as file:
encoder = MultipartEncoder({
'file': (file_path.name, file, 'application/octet-stream'),
'json': (None, json.dumps(job_data), 'application/json'),
})
# Create a monitor that will track the upload progress
monitor = MultipartEncoderMonitor(encoder, callback) if callback else MultipartEncoderMonitor(encoder)
headers = {'Content-Type': monitor.content_type}
url = urljoin(f'http://{self.hostname}:{self.port}', '/api/add_job')
# Send the request with proper resource management
with requests.post(url, data=monitor, headers=headers) as response:
return response
def cancel_job(self, job_id, confirm=False):
return self.request_data(f'job/{job_id}/cancel?confirm={confirm}')
def delete_job(self, job_id, confirm=False):
return self.request_data(f'job/{job_id}/delete?confirm={confirm}')
def get_all_engines(self):
return self.request_data('all_engines')
def send_subjob_update_notification(self, parent_id, subjob):
"""
@@ -241,76 +182,67 @@ class RenderServerProxy:
Returns:
Response: The response from the server.
"""
return requests.post(f'http://{self.hostname}:{self.port}/api/job/{parent_id}/send_subjob_update_notification',
hostname = LOOPBACK if self.is_localhost else self.hostname
return requests.post(f'http://{hostname}:{self.port}/api/job/{parent_id}/send_subjob_update_notification',
json=subjob.json())
# --------------------------------------------
# Engines:
# --------------------------------------------
def get_engine_for_filename(self, filename:str, timeout=5):
response = self.request(f'engine_for_filename?filename={os.path.basename(filename)}', timeout)
return response.text
def get_installed_engines(self, timeout=5):
return self.request_data(f'installed_engines', timeout)
def is_engine_available(self, engine_name:str, timeout=5):
return self.request_data(f'{engine_name}/is_available', timeout)
def get_all_engine_info(self, response_type='standard', timeout=5):
def post_job_to_server(self, file_path, job_list, callback=None):
"""
Fetches all engine information from the server.
Posts a job to the server.
Args:
response_type (str, optional): Returns standard or full version of engine info
timeout (int, optional): The number of seconds to wait for a response from the server. Defaults to 5.
Returns:
dict: A dictionary containing the engine information.
"""
all_data = self.request_data(f"engine_info?response_type={response_type}", timeout=timeout)
return all_data
def get_engine_info(self, engine_name:str, response_type='standard', timeout=5):
"""
Fetches specific engine information from the server.
Args:
engine_name (str): Name of the engine
response_type (str, optional): Returns standard or full version of engine info
timeout (int, optional): The number of seconds to wait for a response from the server. Defaults to 5.
Returns:
dict: A dictionary containing the engine information.
"""
return self.request_data(f'{engine_name}/info?response_type={response_type}', timeout)
def delete_engine(self, engine_name:str, version:str, system_cpu=None):
"""
Sends a request to the server to delete a specific engine.
Args:
engine_name (str): The name of the engine to delete.
version (str): The version of the engine to delete.
system_cpu (str, optional): The system CPU type. Defaults to None.
file_path (str): The path to the file to upload.
job_list (list): A list of jobs to post.
callback (function, optional): A callback function to call during the upload. Defaults to None.
Returns:
Response: The response from the server.
"""
form_data = {'engine': engine_name, 'version': version, 'system_cpu': system_cpu}
return requests.post(f'http://{self.hostname}:{self.port}/api/delete_engine', json=form_data)
try:
# Check if file exists
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
# --------------------------------------------
# Download Files:
# --------------------------------------------
# Bypass uploading file if posting to localhost
if self.is_localhost:
jobs_with_path = [{'local_path': file_path, **item} for item in job_list]
job_data = json.dumps(jobs_with_path)
url = urljoin(f'http://{LOOPBACK}:{self.port}', '/api/add_job')
headers = {'Content-Type': 'application/json'}
return requests.post(url, data=job_data, headers=headers)
# Prepare the form data for remote host
with open(file_path, 'rb') as file:
encoder = MultipartEncoder({
'file': (os.path.basename(file_path), file, 'application/octet-stream'),
'json': (None, json.dumps(job_list), 'application/json'),
})
# Create a monitor that will track the upload progress
monitor = MultipartEncoderMonitor(encoder, callback) if callback else MultipartEncoderMonitor(encoder)
headers = {'Content-Type': monitor.content_type}
url = urljoin(f'http://{self.hostname}:{self.port}', '/api/add_job')
# Send the request with proper resource management
with requests.post(url, data=monitor, headers=headers) as response:
return response
except requests.ConnectionError as e:
logger.error(f"Connection error: {e}")
except Exception as e:
logger.error(f"An error occurred: {e}")
def get_job_files_list(self, job_id):
return self.request_data(f"job/{job_id}/file_list")
def download_all_job_files(self, job_id, save_path):
url = f"http://{self.hostname}:{self.port}/api/job/{job_id}/download_all"
hostname = LOOPBACK if self.is_localhost else self.hostname
url = f"http://{hostname}:{self.port}/api/job/{job_id}/download_all"
return self.__download_file_from_url(url, output_filepath=save_path)
def download_job_file(self, job_id, job_filename, save_path):
url = f"http://{self.hostname}:{self.port}/api/job/{job_id}/download?filename={job_filename}"
hostname = LOOPBACK if self.is_localhost else self.hostname
url = f"http://{hostname}:{self.port}/api/job/{job_id}/download?filename={job_filename}"
return self.__download_file_from_url(url, output_filepath=save_path)
@staticmethod
@@ -321,3 +253,35 @@ class RenderServerProxy:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
return output_filepath
# --- Renderer --- #
def get_renderer_info(self, response_type='standard', timeout=5):
"""
Fetches renderer information from the server.
Args:
response_type (str, optional): Returns standard or full version of renderer info
timeout (int, optional): The number of seconds to wait for a response from the server. Defaults to 5.
Returns:
dict: A dictionary containing the renderer information.
"""
all_data = self.request_data(f"renderer_info?response_type={response_type}", timeout=timeout)
return all_data
def delete_engine(self, engine, version, system_cpu=None):
"""
Sends a request to the server to delete a specific engine.
Args:
engine (str): The name of the engine to delete.
version (str): The version of the engine to delete.
system_cpu (str, optional): The system CPU type. Defaults to None.
Returns:
Response: The response from the server.
"""
form_data = {'engine': engine, 'version': version, 'system_cpu': system_cpu}
hostname = LOOPBACK if self.is_localhost else self.hostname
return requests.post(f'http://{hostname}:{self.port}/api/delete_engine', json=form_data)
src/application_context.py
-23
View File
@@ -1,23 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from src.api.preview_manager import PreviewManager
from src.distributed_job_manager import DistributedJobManager
from src.engines.engine_manager import EngineManager
from src.render_queue import RenderQueue
from src.utilities.config import Config
from src.utilities.zeroconf_server import ZeroconfServer
class ApplicationContext:
"""Holds all service instances. Single source of truth for wiring."""
def __init__(self) -> None:
self.config: Optional[Config] = None
self.engine_manager: Optional[EngineManager] = None
self.preview_manager: Optional[PreviewManager] = None
self.zeroconf_server: Optional[ZeroconfServer] = None
self.render_queue: Optional[RenderQueue] = None
self.distributed_job_manager: Optional[DistributedJobManager] = None
src/distributed_job_manager.py
+399 -150
View File
@@ -3,9 +3,10 @@ import os
import socket
import threading
import time
from typing import Optional
from pathlib import Path
import zipfile
from concurrent.futures import ThreadPoolExecutor
import requests
from plyer import notification
from pubsub import pub
@@ -14,7 +15,7 @@ from src.api.server_proxy import RenderServerProxy
from src.engines.engine_manager import EngineManager
from src.render_queue import RenderQueue
from src.utilities.config import Config
from src.utilities.server_helper import download_missing_frames_from_subjob, distribute_server_work
from src.utilities.misc_helper import get_file_size_human
from src.utilities.status_utils import RenderStatus, string_to_status
from src.utilities.zeroconf_server import ZeroconfServer
@@ -22,32 +23,47 @@ logger = logging.getLogger()
class DistributedJobManager:
_default_instance: Optional['DistributedJobManager'] = None
def __init__(self):
pass
@classmethod
def _sync_class(cls) -> None:
if cls._default_instance is not None:
pass # no class-level attributes to sync
def subscribe_to_listener(cls):
"""
Subscribes the private class method '__local_job_status_changed' to the 'status_change' pubsub message.
This should be called once, typically during the initialization phase.
"""
pub.subscribe(cls.__local_job_status_changed, 'status_change')
pub.subscribe(cls.__local_job_frame_complete, 'frame_complete')
def __init__(self) -> None:
self.background_worker: Optional[threading.Thread] = None
@classmethod
def __local_job_frame_complete(cls, job_id, frame_number, update_interval=5):
def _subscribe_to_listener(self) -> None:
pub.subscribe(self._local_job_status_changed, 'status_change')
pub.subscribe(self._local_job_frame_complete, 'frame_complete')
"""
Responds to the 'frame_complete' pubsub message for local jobs.
Parameters:
job_id (str): The ID of the job that has changed status.
old_status (str): The previous status of the job.
new_status (str): The new (current) status of the job.
Note: Do not call directly. Instead, call via the 'frame_complete' pubsub message.
"""
def _local_job_frame_complete(self, job_id, frame_number, update_interval=5) -> None:
render_job = RenderQueue.job_with_id(job_id, none_ok=True)
if not render_job:
if not render_job: # ignore jobs not in the queue
return
logger.debug(f"Job {job_id} has completed frame #{frame_number}")
replace_existing_previews = (frame_number % update_interval) == 0
self._job_update_shared(render_job, replace_existing_previews)
cls.__job_update_shared(render_job, replace_existing_previews)
def _job_update_shared(self, render_job, replace_existing_previews=False) -> None:
@classmethod
def __job_update_shared(cls, render_job, replace_existing_previews=False):
# update previews
PreviewManager.update_previews_for_job(job=render_job, replace_existing=replace_existing_previews)
# notify parent to allow individual frames to be copied instead of waiting until the end
if render_job.parent:
parent_id, parent_hostname = render_job.parent.split('@')[0], render_job.parent.split('@')[-1]
try:
@@ -56,41 +72,57 @@ class DistributedJobManager:
except Exception as e:
logger.error(f"Error notifying parent {parent_hostname} about update in subjob {render_job.id}: {e}")
def _local_job_status_changed(self, job_id: str, old_status: str, new_status: str) -> None:
@classmethod
def __local_job_status_changed(cls, job_id, old_status, new_status):
"""
Responds to the 'status_change' pubsub message for local jobs.
If it's a child job, it notifies the parent job about the status change.
Parameters:
job_id (str): The ID of the job that has changed status.
old_status (str): The previous status of the job.
new_status (str): The new (current) status of the job.
Note: Do not call directly. Instead, call via the 'status_change' pubsub message.
"""
render_job = RenderQueue.job_with_id(job_id, none_ok=True)
if not render_job:
if not render_job: # ignore jobs created but not yet added to queue
return
logger.debug(f"Job {job_id} status change: {old_status} -> {new_status}")
self._job_update_shared(render_job, replace_existing_previews=(render_job.status == RenderStatus.COMPLETED))
cls.__job_update_shared(render_job, replace_existing_previews=(render_job.status == RenderStatus.COMPLETED))
# Handle children
if render_job.children:
if new_status in (RenderStatus.CANCELLED, RenderStatus.ERROR):
if new_status in [RenderStatus.CANCELLED, RenderStatus.ERROR]: # Cancel children if necessary
for child in render_job.children:
child_id, child_hostname = child.split('@')
RenderServerProxy(child_hostname).cancel_job(child_id, confirm=True)
# UI Notifications
try:
if new_status == RenderStatus.COMPLETED:
logger.debug("Show render complete notification")
notification.notify(
title='Render Job Complete',
message=f'{render_job.name} completed succesfully',
timeout=10
timeout=10 # Display time in seconds
)
elif new_status == RenderStatus.ERROR:
logger.debug("Show render error notification")
notification.notify(
title='Render Job Failed',
message=f'{render_job.name} failed rendering',
timeout=10
timeout=10 # Display time in seconds
)
elif new_status == RenderStatus.RUNNING:
logger.debug("Show render started notification")
notification.notify(
title='Render Job Started',
message=f'{render_job.name} started rendering',
timeout=10
timeout=10 # Display time in seconds
)
except Exception as e:
logger.debug(f"Unable to show UI notification: {e}")
@@ -99,35 +131,57 @@ class DistributedJobManager:
# Create Job
# --------------------------------------------
def _create_render_job(self, new_job_attributes: dict, loaded_project_local_path: Path):
output_path = new_job_attributes.get('output_path')
output_filename = loaded_project_local_path.name if output_path else loaded_project_local_path.stem
@classmethod
def create_render_job(cls, job_data, loaded_project_local_path):
"""
Creates render jobs.
output_dir = loaded_project_local_path.parent.parent / "output"
output_path = output_dir / output_filename
This method job data and a local path to a loaded project. It creates and returns new a render job.
Args:
job_data (dict): Job data.
loaded_project_local_path (str): The local path to the loaded project.
Returns:
worker: Created job worker
"""
# get new output path in output_dir
output_path = job_data.get('output_path')
if not output_path:
loaded_project_filename = os.path.basename(loaded_project_local_path)
output_filename = os.path.splitext(loaded_project_filename)[0]
else:
output_filename = os.path.basename(output_path)
# Prepare output path
output_dir = os.path.join(os.path.dirname(os.path.dirname(loaded_project_local_path)), 'output')
output_path = os.path.join(output_dir, output_filename)
os.makedirs(output_dir, exist_ok=True)
logger.debug(f"New job output path: {output_path}")
worker = EngineManager.create_worker(engine_name=new_job_attributes['engine_name'],
# create & configure jobs
worker = EngineManager.create_worker(renderer=job_data['renderer'],
input_path=loaded_project_local_path,
output_path=output_path,
engine_version=new_job_attributes.get('engine_version'),
args=new_job_attributes.get('args', {}),
parent=new_job_attributes.get('parent'),
name=new_job_attributes.get('name'))
worker.status = new_job_attributes.get("initial_status", worker.status)
worker.priority = int(new_job_attributes.get('priority', worker.priority))
worker.start_frame = int(new_job_attributes.get("start_frame", worker.start_frame))
worker.end_frame = int(new_job_attributes.get("end_frame", worker.end_frame))
engine_version=job_data.get('engine_version'),
args=job_data.get('args', {}),
parent=job_data.get('parent'),
name=job_data.get('name'))
worker.status = job_data.get("initial_status", worker.status) # todo: is this necessary?
worker.priority = int(job_data.get('priority', worker.priority))
worker.start_frame = int(job_data.get("start_frame", worker.start_frame))
worker.end_frame = int(job_data.get("end_frame", worker.end_frame))
worker.watchdog_timeout = Config.worker_process_timeout
worker.hostname = socket.gethostname()
if new_job_attributes.get("enable_split_jobs", False) and (worker.total_frames > 1) and not worker.parent:
self.split_into_subjobs_async(worker, new_job_attributes, loaded_project_local_path)
# determine if we can / should split the job
if job_data.get("enable_split_jobs", False) and (worker.total_frames > 1) and not worker.parent:
cls.split_into_subjobs_async(worker, job_data, loaded_project_local_path)
else:
worker.status = RenderStatus.NOT_STARTED
logger.debug("Not splitting into subjobs")
RenderQueue.add_to_render_queue(worker, force_start=new_job_attributes.get('force_start', False))
RenderQueue.add_to_render_queue(worker, force_start=job_data.get('force_start', False))
PreviewManager.update_previews_for_job(worker)
return worker
@@ -136,7 +190,16 @@ class DistributedJobManager:
# Handling Subjobs
# --------------------------------------------
def _handle_subjob_update_notification(self, local_job, subjob_data: dict) -> None:
@classmethod
def handle_subjob_update_notification(cls, local_job, subjob_data):
"""
Responds to a notification from a remote subjob and the host requests any subsequent updates from the subjob.
Args:
local_job (BaseRenderWorker): The local parent job worker.
subjob_data (dict): Subjob data sent from the remote server.
"""
subjob_status = string_to_status(subjob_data['status'])
subjob_id = subjob_data['id']
subjob_hostname = subjob_data['hostname']
@@ -148,120 +211,209 @@ class DistributedJobManager:
if old_status != subjob_status.value:
logger.debug(f"Subjob status changed: {logname} -> {subjob_status.value}")
download_success = download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname)
if subjob_data['status'] == 'completed' and download_success:
local_job.children[subjob_key]['download_status'] = 'completed'
cls.download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname)
def _wait_for_subjobs(self, parent_job) -> None:
logger.debug(f"Waiting for subjobs for job {parent_job}")
parent_job.status = RenderStatus.WAITING_FOR_SUBJOBS
statuses_to_download = (RenderStatus.CANCELLED, RenderStatus.ERROR, RenderStatus.COMPLETED)
@staticmethod
def download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname):
try:
local_files = [os.path.basename(x) for x in local_job.file_list()]
subjob_proxy = RenderServerProxy(subjob_hostname)
subjob_files = subjob_proxy.get_job_files_list(job_id=subjob_id) or []
for subjob_filename in subjob_files:
if subjob_filename not in local_files:
try:
logger.debug(f"Downloading new file '{subjob_filename}' from {subjob_hostname}")
local_save_path = os.path.join(os.path.dirname(local_job.output_path), subjob_filename)
subjob_proxy.download_job_file(job_id=subjob_id, job_filename=subjob_filename,
save_path=local_save_path)
logger.debug(f'Downloaded successfully - {local_save_path}')
except Exception as e:
logger.error(f"Error downloading file '{subjob_filename}' from {subjob_hostname}: {e}")
except Exception as e:
logger.exception(f'Uncaught exception while trying to download from subjob: {e}')
@staticmethod
def download_all_from_subjob(local_job, subjob_id, subjob_hostname):
"""
Downloads and extracts files from a completed subjob on a remote server.
Parameters:
local_job (BaseRenderWorker): The local parent job worker.
subjob_id (str or int): The ID of the subjob.
subjob_hostname (str): The hostname of the remote server where the subjob is located.
Returns:
bool: True if the files have been downloaded and extracted successfully, False otherwise.
"""
child_key = f'{subjob_id}@{subjob_hostname}'
logname = f"{local_job.id}:{child_key}"
zip_file_path = local_job.output_path + f'_{subjob_hostname}_{subjob_id}.zip'
# download zip file from server
try:
local_job.children[child_key]['download_status'] = 'working'
logger.info(f"Downloading completed subjob files from {subjob_hostname} to localhost")
RenderServerProxy(subjob_hostname).download_all_job_files(subjob_id, zip_file_path)
logger.info(f"File transfer complete for {logname} - Transferred {get_file_size_human(zip_file_path)}")
except Exception as e:
logger.error(f"Error downloading files from remote server: {e}")
local_job.children[child_key]['download_status'] = 'failed'
return False
# extract zip
try:
logger.debug(f"Extracting zip file: {zip_file_path}")
extract_path = os.path.dirname(zip_file_path)
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
zip_ref.extractall(extract_path)
logger.info(f"Successfully extracted zip to: {extract_path}")
os.remove(zip_file_path)
local_job.children[child_key]['download_status'] = 'complete'
except Exception as e:
logger.exception(f"Exception extracting zip file: {e}")
local_job.children[child_key]['download_status'] = 'failed'
return local_job.children[child_key].get('download_status', None) == 'complete'
@classmethod
def wait_for_subjobs(cls, local_job):
# todo: rewrite this method
logger.debug(f"Waiting for subjobs for job {local_job}")
local_job.status = RenderStatus.WAITING_FOR_SUBJOBS
statuses_to_download = [RenderStatus.CANCELLED, RenderStatus.ERROR, RenderStatus.COMPLETED]
def subjobs_not_downloaded():
return {k: v for k, v in parent_job.children.items() if 'download_status' not in v or
return {k: v for k, v in local_job.children.items() if 'download_status' not in v or
v['download_status'] == 'working' or v['download_status'] is None}
logger.info(f'Waiting on {len(subjobs_not_downloaded())} subjobs for {parent_job.id}')
logger.info(f'Waiting on {len(subjobs_not_downloaded())} subjobs for {local_job.id}')
server_delay = 10
sleep_counter = 0
while parent_job.status == RenderStatus.WAITING_FOR_SUBJOBS:
while len(subjobs_not_downloaded()):
for child_key, subjob_cached_data in subjobs_not_downloaded().items():
if sleep_counter % server_delay == 0:
for child_key in subjobs_not_downloaded():
subjob_id = child_key.split('@')[0]
subjob_hostname = child_key.split('@')[-1]
subjob_id = child_key.split('@')[0]
subjob_hostname = child_key.split('@')[-1]
subjob_data = RenderServerProxy(subjob_hostname).get_job_info(subjob_id)
if not subjob_data:
logger.warning(f"No response from {subjob_hostname}")
parent_job.children[child_key]['download_status'] = f'error: No response from {subjob_hostname}'
continue
# Fetch info from server and handle failing case
subjob_data = RenderServerProxy(subjob_hostname).get_job_info(subjob_id)
if not subjob_data:
logger.warning(f"No response from: {subjob_hostname}")
# todo: handle timeout / missing server situations
continue
download_status = parent_job.children[child_key].get('download_status', None)
parent_job.children[child_key] = subjob_data
parent_job.children[child_key]['download_status'] = download_status
# Update parent job cache but keep the download status
download_status = local_job.children[child_key].get('download_status', None)
local_job.children[child_key] = subjob_data
local_job.children[child_key]['download_status'] = download_status
status = string_to_status(subjob_data.get('status', ''))
status_msg = f"Subjob {child_key} | {status} | " \
f"{float(subjob_data.get('percent_complete')) * 100.0}%"
logger.debug(status_msg)
status = string_to_status(subjob_data.get('status', ''))
status_msg = f"Subjob {child_key} | {status} | " \
f"{float(subjob_data.get('percent_complete')) * 100.0}%"
logger.debug(status_msg)
if download_status is None and subjob_data.get('file_count') and status in statuses_to_download:
try:
download_missing_frames_from_subjob(parent_job, subjob_id, subjob_hostname)
parent_job.children[child_key]['download_status'] = 'complete'
except Exception as e:
logger.error(f"Error downloading missing frames from subjob: {e}")
parent_job.children[child_key]['download_status'] = 'error: {}'
# Still working in another thread - keep waiting
if download_status == 'working':
continue
if parent_job.children[child_key].get('download_status', None) is None and \
status in statuses_to_download:
logger.warning(f"Skipping waiting on downloading from subjob: {child_key}")
parent_job.children[child_key]['download_status'] = 'skipped'
# Check if job is finished, but has not had files copied yet over yet
if download_status is None and subjob_data['file_count'] and status in statuses_to_download:
try:
cls.download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname)
except Exception as e:
logger.error(f"Error downloading missing frames from subjob: {e}")
# Any finished jobs not successfully downloaded at this point are skipped
if local_job.children[child_key].get('download_status', None) is None and \
status in statuses_to_download:
logger.warning(f"Skipping waiting on downloading from subjob: {child_key}")
local_job.children[child_key]['download_status'] = 'skipped'
if subjobs_not_downloaded():
logger.debug(f"Waiting on {len(subjobs_not_downloaded())} subjobs on "
f"{', '.join(list(subjobs_not_downloaded().keys()))}")
time.sleep(1)
sleep_counter += 1
else:
parent_job.status = RenderStatus.RUNNING
time.sleep(5)
# --------------------------------------------
# Creating Subjobs
# --------------------------------------------
def _split_into_subjobs_async(self, parent_worker, new_job_attributes, project_path, system_os=None) -> None:
@classmethod
def split_into_subjobs_async(cls, parent_worker, job_data, project_path, system_os=None):
# todo: I don't love this
parent_worker.status = RenderStatus.CONFIGURING
self.background_worker = threading.Thread(target=self.split_into_subjobs, args=(
parent_worker, new_job_attributes, project_path, system_os))
self.background_worker.start()
cls.background_worker = threading.Thread(target=cls.split_into_subjobs, args=(parent_worker, job_data,
project_path, system_os))
cls.background_worker.start()
def split_into_subjobs(self, parent_worker, new_job_attributes, project_path, system_os=None, specific_servers=None) -> None:
available_servers = specific_servers if specific_servers else self.find_available_servers(
parent_worker.engine_name, system_os)
external_servers = [x for x in available_servers if x['hostname'] != parent_worker.hostname]
if not external_servers:
parent_worker.status = RenderStatus.NOT_STARTED
return
@classmethod
def split_into_subjobs(cls, parent_worker, job_data, project_path, system_os=None, specific_servers=None):
"""
Splits a job into subjobs and distributes them among available servers.
logger.debug(f"Splitting into subjobs - Available servers: {[x['hostname'] for x in available_servers]}")
all_subjob_server_data = distribute_server_work(parent_worker.start_frame, parent_worker.end_frame, available_servers)
This method checks the availability of servers, distributes the work among them, and creates subjobs on each
server. If a server is the local host, it adjusts the frame range of the parent job instead of creating a
subjob.
Args:
parent_worker (Worker): The worker that is handling the job.
job_data (dict): The data for the job to be split.
project_path (str): The path to the project associated with the job.
system_os (str, optional): The operating system of the servers. Default is any OS.
specific_servers (list, optional): List of specific servers to split work between. Defaults to all found.
"""
# Check availability
parent_worker.status = RenderStatus.CONFIGURING
available_servers = specific_servers if specific_servers else cls.find_available_servers(parent_worker.renderer, system_os)
logger.debug(f"Splitting into subjobs - Available servers: {available_servers}")
all_subjob_server_data = cls.distribute_server_work(parent_worker.start_frame, parent_worker.end_frame, available_servers)
# Prep and submit these sub-jobs
logger.info(f"Job {parent_worker.id} split plan: {all_subjob_server_data}")
try:
for subjob_data in all_subjob_server_data:
subjob_hostname = subjob_data['hostname']
post_results = self._create_subjob(new_job_attributes, project_path, subjob_data, subjob_hostname,
parent_worker)
if not post_results.ok:
ValueError(f"Failed to create subjob on {subjob_hostname}")
if subjob_hostname != parent_worker.hostname:
post_results = cls.__create_subjob(job_data, project_path, subjob_data, subjob_hostname,
parent_worker)
if not post_results.ok:
ValueError(f"Failed to create subjob on {subjob_hostname}")
submission_results = post_results.json()[0]
child_key = f"{submission_results['id']}@{subjob_hostname}"
parent_worker.children[child_key] = submission_results
# save child info
submission_results = post_results.json()[0]
child_key = f"{submission_results['id']}@{subjob_hostname}"
parent_worker.children[child_key] = submission_results
else:
# truncate parent render_job
parent_worker.start_frame = max(subjob_data['frame_range'][0], parent_worker.start_frame)
parent_worker.end_frame = min(subjob_data['frame_range'][-1], parent_worker.end_frame)
logger.info(f"Local job now rendering from {parent_worker.start_frame} to {parent_worker.end_frame}")
logger.debug(f"Created {len(all_subjob_server_data)} subjobs successfully")
parent_worker.name = f"{parent_worker.name} (Parent)"
parent_worker.status = RenderStatus.NOT_STARTED
# start subjobs
logger.debug(f"Created {len(all_subjob_server_data) - 1} subjobs successfully")
parent_worker.name = f"{parent_worker.name}[{parent_worker.start_frame}-{parent_worker.end_frame}]"
parent_worker.status = RenderStatus.NOT_STARTED # todo: this won't work with scheduled starts
except Exception as e:
# cancel all the subjobs
logger.error(f"Failed to split job into subjobs: {e}")
logger.debug(f"Cancelling {len(all_subjob_server_data) - 1} attempted subjobs")
RenderServerProxy(parent_worker.hostname).cancel_job(parent_worker.id, confirm=True)
@staticmethod
def _create_subjob(new_job_attributes: dict, project_path, server_data, server_hostname, parent_worker):
subjob = new_job_attributes.copy()
def __create_subjob(job_data, project_path, server_data, server_hostname, parent_worker):
subjob = job_data.copy()
subjob['name'] = f"{parent_worker.name}[{server_data['frame_range'][0]}-{server_data['frame_range'][-1]}]"
subjob['parent'] = f"{parent_worker.id}@{parent_worker.hostname}"
subjob['start_frame'] = server_data['frame_range'][0]
subjob['end_frame'] = server_data['frame_range'][-1]
subjob['engine_version'] = parent_worker.engine_version
subjob['engine_version'] = parent_worker.renderer_version
logger.debug(f"Posting subjob with frames {subjob['start_frame']}-"
f"{subjob['end_frame']} to {server_hostname}")
post_results = RenderServerProxy(server_hostname).post_job_to_server(
file_path=project_path, job_data=subjob)
file_path=project_path, job_list=[subjob])
return post_results
# --------------------------------------------
@@ -269,46 +421,143 @@ class DistributedJobManager:
# --------------------------------------------
@staticmethod
def find_available_servers(engine_name: str, system_os=None):
from src.api.api_server import API_VERSION
found_available_servers = []
def distribute_server_work(start_frame, end_frame, available_servers, method='cpu_benchmark'):
"""
Splits the frame range among available servers proportionally based on their performance (CPU count).
Args:
start_frame (int): The start frame number of the animation to be rendered.
end_frame (int): The end frame number of the animation to be rendered.
available_servers (list): A list of available server dictionaries. Each server dictionary should include
'hostname' and 'cpu_count' keys (see find_available_servers).
method (str, optional): Specifies the distribution method. Possible values are 'cpu_benchmark', 'cpu_count'
and 'evenly'.
Defaults to 'cpu_benchmark'.
Returns:
list: A list of server dictionaries where each dictionary includes the frame range and total number of
frames to be rendered by the server.
"""
# Calculate respective frames for each server
def divide_frames_by_cpu_count(frame_start, frame_end, servers):
total_frames = frame_end - frame_start + 1
total_cpus = sum(server['cpu_count'] for server in servers)
frame_ranges = {}
current_frame = frame_start
allocated_frames = 0
for i, server in enumerate(servers):
if i == len(servers) - 1: # if it's the last server
# Give all remaining frames to the last server
num_frames = total_frames - allocated_frames
else:
num_frames = round((server['cpu_count'] / total_cpus) * total_frames)
allocated_frames += num_frames
frame_end_for_server = current_frame + num_frames - 1
if current_frame <= frame_end_for_server:
frame_ranges[server['hostname']] = (current_frame, frame_end_for_server)
current_frame = frame_end_for_server + 1
return frame_ranges
def divide_frames_by_benchmark(frame_start, frame_end, servers):
def fetch_benchmark(server):
try:
benchmark = requests.get(f'http://{server["hostname"]}:{ZeroconfServer.server_port}'
f'/api/cpu_benchmark').text
server['cpu_benchmark'] = benchmark
logger.debug(f'Benchmark for {server["hostname"]}: {benchmark}')
except requests.exceptions.RequestException as e:
logger.error(f'Error fetching benchmark for {server["hostname"]}: {e}')
# Number of threads to use (can adjust based on your needs or number of servers)
threads = len(servers)
with ThreadPoolExecutor(max_workers=threads) as executor:
executor.map(fetch_benchmark, servers)
total_frames = frame_end - frame_start + 1
total_performance = sum(int(server['cpu_benchmark']) for server in servers)
frame_ranges = {}
current_frame = frame_start
allocated_frames = 0
for i, server in enumerate(servers):
if i == len(servers) - 1: # if it's the last server
# Give all remaining frames to the last server
num_frames = total_frames - allocated_frames
else:
num_frames = round((int(server['cpu_benchmark']) / total_performance) * total_frames)
allocated_frames += num_frames
frame_end_for_server = current_frame + num_frames - 1
if current_frame <= frame_end_for_server:
frame_ranges[server['hostname']] = (current_frame, frame_end_for_server)
current_frame = frame_end_for_server + 1
return frame_ranges
def divide_frames_equally(frame_start, frame_end, servers):
frame_range = frame_end - frame_start + 1
frames_per_server = frame_range // len(servers)
leftover_frames = frame_range % len(servers)
frame_ranges = {}
current_start = frame_start
for i, server in enumerate(servers):
current_end = current_start + frames_per_server - 1
if leftover_frames > 0:
current_end += 1
leftover_frames -= 1
if current_start <= current_end:
frame_ranges[server['hostname']] = (current_start, current_end)
current_start = current_end + 1
return frame_ranges
if len(available_servers) == 1:
breakdown = {available_servers[0]['hostname']: (start_frame, end_frame)}
else:
logger.debug(f'Splitting between {len(available_servers)} servers by {method} method')
if method == 'evenly':
breakdown = divide_frames_equally(start_frame, end_frame, available_servers)
elif method == 'cpu_benchmark':
breakdown = divide_frames_by_benchmark(start_frame, end_frame, available_servers)
elif method == 'cpu_count':
breakdown = divide_frames_by_cpu_count(start_frame, end_frame, available_servers)
else:
raise ValueError(f"Invalid distribution method: {method}")
server_breakdown = [server for server in available_servers if breakdown.get(server['hostname']) is not None]
for server in server_breakdown:
server['frame_range'] = breakdown[server['hostname']]
server['total_frames'] = breakdown[server['hostname']][-1] - breakdown[server['hostname']][0] + 1
return server_breakdown
@staticmethod
def find_available_servers(engine_name, system_os=None):
"""
Scan the Zeroconf network for currently available render servers supporting a specific engine.
:param engine_name: str, The engine type to search for
:param system_os: str, Restrict results to servers running a specific OS
:return: A list of dictionaries with each dict containing hostname and cpu_count of available servers
"""
available_servers = []
for hostname in ZeroconfServer.found_hostnames():
host_properties = ZeroconfServer.get_hostname_properties(hostname)
if host_properties.get('api_version') == API_VERSION:
if not system_os or (system_os and system_os == host_properties.get('system_os')):
response = RenderServerProxy(hostname).is_engine_available(engine_name)
if response and response.get('available', False):
found_available_servers.append(response)
if not system_os or (system_os and system_os == host_properties.get('system_os')):
response = RenderServerProxy(hostname).is_engine_available(engine_name)
if response and response.get('available', False):
available_servers.append(response)
return found_available_servers
# --- Forwarders for backward compatibility ---
@classmethod
def subscribe_to_listener(cls):
if cls._default_instance is not None:
cls._default_instance._subscribe_to_listener()
@classmethod
def create_render_job(cls, new_job_attributes, loaded_project_local_path):
if cls._default_instance is not None:
return cls._default_instance._create_render_job(new_job_attributes, loaded_project_local_path)
raise RuntimeError("DistributedJobManager is not initialized")
@classmethod
def handle_subjob_update_notification(cls, local_job, subjob_data):
if cls._default_instance is not None:
cls._default_instance._handle_subjob_update_notification(local_job, subjob_data)
@classmethod
def wait_for_subjobs(cls, parent_job):
if cls._default_instance is not None:
cls._default_instance._wait_for_subjobs(parent_job)
@classmethod
def split_into_subjobs_async(cls, parent_worker, new_job_attributes, project_path, system_os=None):
if cls._default_instance is not None:
cls._default_instance._split_into_subjobs_async(parent_worker, new_job_attributes, project_path, system_os)
return available_servers
if __name__ == '__main__':
@@ -320,6 +569,6 @@ if __name__ == '__main__':
time.sleep(2)
available_servers = DistributedJobManager.find_available_servers('blender')
print(f"AVAILABLE SERVERS ({len(available_servers)}): {available_servers}")
# results = distribute_server_work(1, 100, available_servers)
# results = DistributedJobManager.distribute_server_work(1, 100, available_servers)
# print(f"RESULTS: {results}")
ZeroconfServer.stop()
src/engines/aerender/aerender_engine.py
+1 -1
View File
@@ -8,7 +8,7 @@ class AERender(BaseRenderEngine):
def version(self):
version = None
try:
render_path = self.engine_path()
render_path = self.renderer_path()
if render_path:
ver_out = subprocess.check_output([render_path, '-version'], timeout=SUBPROCESS_TIMEOUT)
version = ver_out.decode('utf-8').split(" ")[-1].strip()
src/engines/blender/blender_downloader.py
+6 -5
View File
@@ -8,7 +8,8 @@ from src.engines.blender.blender_engine import Blender
from src.engines.core.base_downloader import EngineDownloader
from src.utilities.misc_helper import current_system_os, current_system_cpu
url = "https://download.blender.org/release/"
# url = "https://download.blender.org/release/"
url = "https://ftp.nluug.nl/pub/graphics/blender/release/" # much faster mirror for testing
logger = logging.getLogger()
supported_formats = ['.zip', '.tar.xz', '.dmg']
@@ -87,8 +88,8 @@ class BlenderDownloader(EngineDownloader):
threads = []
results = [[] for _ in majors]
def thread_function(major_version, index, system_os_t, cpu_t):
results[index] = cls.__get_minor_versions(major_version, system_os_t, cpu_t)
def thread_function(major_version, index, system_os, cpu):
results[index] = cls.__get_minor_versions(major_version, system_os, cpu)
for i, m in enumerate(majors):
thread = threading.Thread(target=thread_function, args=(m, i, system_os, cpu))
@@ -125,7 +126,7 @@ class BlenderDownloader(EngineDownloader):
return None
@classmethod
def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120, progress_callback=None):
def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
system_os = system_os or current_system_os()
cpu = cpu or current_system_cpu()
@@ -135,7 +136,7 @@ class BlenderDownloader(EngineDownloader):
minor_versions = [x for x in cls.__get_minor_versions(major_version, system_os, cpu) if
x['version'] == version]
cls.download_and_extract_app(remote_url=minor_versions[0]['url'], download_location=download_location,
timeout=timeout, progress_callback=progress_callback)
timeout=timeout)
except IndexError:
logger.error("Cannot find requested engine")
src/engines/blender/blender_engine.py
+33 -45
View File
@@ -1,13 +1,12 @@
import json
import re
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor
from src.engines.core.base_engine import *
from src.utilities.misc_helper import system_safe_path
logger = logging.getLogger()
_creationflags = subprocess.CREATE_NO_WINDOW if platform.system() == 'Windows' else 0
class Blender(BaseRenderEngine):
@@ -25,22 +24,19 @@ class Blender(BaseRenderEngine):
return BlenderRenderWorker
def ui_options(self):
options = [
{'name': 'engine', 'options': self.supported_render_engines()},
{'name': 'render_device', 'options': ['Any', 'GPU', 'CPU']},
]
return options
from src.engines.blender.blender_ui import BlenderUI
return BlenderUI.get_options(self)
def supported_extensions(self):
@staticmethod
def supported_extensions():
return ['blend']
def version(self):
version = None
try:
render_path = self.engine_path()
render_path = self.renderer_path()
if render_path:
ver_out = subprocess.check_output([render_path, '-v'], timeout=SUBPROCESS_TIMEOUT,
creationflags=_creationflags)
ver_out = subprocess.check_output([render_path, '-v'], timeout=SUBPROCESS_TIMEOUT)
version = ver_out.decode('utf-8').splitlines()[0].replace('Blender', '').strip()
except Exception as e:
logger.error(f'Failed to get Blender version: {e}')
@@ -54,12 +50,10 @@ class Blender(BaseRenderEngine):
def run_python_expression(self, project_path, python_expression, timeout=None):
if os.path.exists(project_path):
try:
return subprocess.run([self.engine_path(), '-b', project_path, '--python-expr', python_expression],
capture_output=True, timeout=timeout, creationflags=_creationflags)
return subprocess.run([self.renderer_path(), '-b', project_path, '--python-expr', python_expression],
capture_output=True, timeout=timeout)
except Exception as e:
err_msg = f"Error running python expression in blender: {e}"
logger.error(err_msg)
raise ChildProcessError(err_msg)
logger.error(f"Error running python expression in blender: {e}")
else:
raise FileNotFoundError(f'Project file not found: {project_path}')
@@ -71,25 +65,18 @@ class Blender(BaseRenderEngine):
raise FileNotFoundError(f'Python script not found: {script_path}')
try:
command = [self.engine_path(), '-b', '--python', script_path]
command = [self.renderer_path(), '-b', '--python', script_path]
if project_path:
command.insert(2, project_path)
result = subprocess.run(command, capture_output=True, timeout=timeout, creationflags=_creationflags)
return result
except subprocess.TimeoutExpired:
err_msg = f"Timed out after {timeout}s while running python script in blender: {script_path}"
logger.error(err_msg)
raise TimeoutError(err_msg)
return subprocess.run(command, capture_output=True, timeout=timeout)
except Exception as e:
err_msg = f"Error running python script in blender: {e}"
logger.error(err_msg)
raise ChildProcessError(err_msg)
logger.exception(f"Error running python script in blender: {e}")
def get_project_info(self, project_path, timeout=10):
scene_info = {}
try:
script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'scripts', 'get_file_info.py')
results = self.run_python_script(project_path=project_path, script_path=Path(script_path),
results = self.run_python_script(project_path=project_path, script_path=system_safe_path(script_path),
timeout=timeout)
result_text = results.stdout.decode()
for line in result_text.splitlines():
@@ -100,17 +87,15 @@ class Blender(BaseRenderEngine):
elif line.startswith('Error'):
logger.error(f"get_scene_info error: {line.strip()}")
except Exception as e:
msg = f'Error getting file details for .blend file: {e}'
logger.error(msg)
raise ChildProcessError(msg)
logger.error(f'Error getting file details for .blend file: {e}')
return scene_info
def pack_project_file(self, project_path, timeout=None):
def pack_project_file(self, project_path, timeout=30):
# Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935
try:
logger.info(f"Starting to pack Blender file: {project_path}")
script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'scripts', 'pack_project.py')
results = self.run_python_script(project_path=project_path, script_path=Path(script_path),
results = self.run_python_script(project_path=project_path, script_path=system_safe_path(script_path),
timeout=timeout)
result_text = results.stdout.decode()
@@ -119,7 +104,7 @@ class Blender(BaseRenderEngine):
# report any missing textures
not_found = re.findall("(Unable to pack file, source path .*)\n", result_text)
for err in not_found:
raise ChildProcessError(err)
logger.error(err)
p = re.compile('Saved to: (.*)\n')
match = p.search(result_text)
@@ -127,15 +112,12 @@ class Blender(BaseRenderEngine):
new_path = os.path.join(dir_name, match.group(1).strip())
logger.info(f'Blender file packed successfully to {new_path}')
return new_path
return project_path
except Exception as e:
msg = f'Error packing .blend file: {e}'
logger.error(msg)
raise ChildProcessError(msg)
logger.error(f'Error packing .blend file: {e}')
return None
def get_arguments(self):
help_text = subprocess.check_output([self.engine_path(), '-h'], creationflags=_creationflags).decode('utf-8')
help_text = subprocess.check_output([self.renderer_path(), '-h']).decode('utf-8')
lines = help_text.splitlines()
options = {}
@@ -167,13 +149,19 @@ class Blender(BaseRenderEngine):
return options
def system_info(self):
return {'render_devices': self.get_render_devices(), 'engines': self.supported_render_engines()}
with ThreadPoolExecutor() as executor:
future_render_devices = executor.submit(self.get_render_devices)
future_engines = executor.submit(self.supported_render_engines)
render_devices = future_render_devices.result()
engines = future_engines.result()
return {'render_devices': render_devices, 'engines': engines}
def get_render_devices(self):
script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'scripts', 'get_system_info.py')
results = self.run_python_script(script_path=script_path)
output = results.stdout.decode()
match = re.search(r"GPU DATA:(\[.*?\])", output, re.DOTALL)
match = re.search(r"GPU DATA:(\[[\s\S]*\])", output)
if match:
gpu_data_json = match.group(1)
gpus_info = json.loads(gpu_data_json)
@@ -182,16 +170,16 @@ class Blender(BaseRenderEngine):
logger.error("GPU data not found in the output.")
def supported_render_engines(self):
engine_output = subprocess.run([self.engine_path(), '-b', '-E', 'help'], timeout=SUBPROCESS_TIMEOUT,
capture_output=True, creationflags=_creationflags).stdout.decode('utf-8').strip()
engine_output = subprocess.run([self.renderer_path(), '-E', 'help'], timeout=SUBPROCESS_TIMEOUT,
capture_output=True).stdout.decode('utf-8').strip()
render_engines = [x.strip() for x in engine_output.split('Blender Engine Listing:')[-1].strip().splitlines()]
return render_engines
def perform_presubmission_tasks(self, project_path):
packed_path = self.pack_project_file(project_path, timeout=120)
packed_path = self.pack_project_file(project_path, timeout=30)
return packed_path
if __name__ == "__main__":
x = Blender().system_info()
x = Blender().get_render_devices()
print(x)
src/engines/blender/blender_ui.py
+9
View File
@@ -0,0 +1,9 @@
class BlenderUI:
@staticmethod
def get_options(instance):
options = [
{'name': 'engine', 'options': instance.supported_render_engines()},
{'name': 'render_device', 'options': ['Any', 'GPU', 'CPU']},
]
return options
src/engines/blender/blender_worker.py
+17 -34
View File
@@ -18,43 +18,35 @@ class BlenderRenderWorker(BaseRenderWorker):
self.__frame_percent_complete = 0.0
# Scene Info
self.scene_info = {}
self.scene_info = Blender(engine_path).get_project_info(input_path)
self.start_frame = int(self.scene_info.get('start_frame', 1))
self.end_frame = int(self.scene_info.get('end_frame', self.start_frame))
self.project_length = (self.end_frame - self.start_frame) + 1
self.current_frame = -1
def generate_worker_subprocess(self):
self.scene_info = Blender(self.engine_path).get_project_info(self.input_path)
cmd = [self.engine_path]
cmd = [self.renderer_path]
if self.args.get('background', True): # optionally run render not in background
cmd.append('-b')
cmd.append(self.input_path)
# Set Render Engine
blender_engine = self.args.get('engine')
if blender_engine:
blender_engine = blender_engine.upper()
cmd.extend(['-E', blender_engine])
# Start Python expressions - # todo: investigate splitting into separate 'setup' script
cmd.append('--python-expr')
python_exp = 'import bpy; bpy.context.scene.render.use_overwrite = False;'
# Setup Custom Resolution
if self.args.get('resolution'):
res = self.args.get('resolution')
python_exp += 'bpy.context.scene.render.resolution_percentage = 100;'
python_exp += f'bpy.context.scene.render.resolution_x={res[0]}; bpy.context.scene.render.resolution_y={res[1]};'
# Setup Custom Camera
custom_camera = self.args.get('camera', None)
if custom_camera:
python_exp += f"bpy.context.scene.camera = bpy.data.objects['{custom_camera}'];"
python_exp = python_exp + f"bpy.context.scene.camera = bpy.data.objects['{custom_camera}'];"
# Set Render Device for Cycles (gpu/cpu/any)
if blender_engine == 'CYCLES':
# Setup Render Engines
self.args['engine'] = self.args.get('engine', 'CYCLES').upper() # set default render engine
# Configure Cycles
if self.args['engine'] == 'CYCLES':
# Set Render Device (gpu/cpu/any)
render_device = self.args.get('render_device', 'any').lower()
if render_device not in {'any', 'gpu', 'cpu'}:
if render_device not in ['any', 'gpu', 'cpu']:
raise AttributeError(f"Invalid Cycles render device: {render_device}")
use_gpu = render_device in {'any', 'gpu'}
@@ -76,7 +68,7 @@ class BlenderRenderWorker(BaseRenderWorker):
# Remove the extension only if it is not composed entirely of digits
path_without_ext = main_part if not ext[1:].isdigit() else self.output_path
path_without_ext += "_"
cmd.extend(['-o', path_without_ext, '-F', export_format])
cmd.extend(['-E', blender_engine, '-o', path_without_ext, '-F', export_format])
# set frame range
cmd.extend(['-s', self.start_frame, '-e', self.end_frame, '-a'])
@@ -90,15 +82,11 @@ class BlenderRenderWorker(BaseRenderWorker):
def _parse_stdout(self, line):
cycles_pattern = re.compile(
pattern = re.compile(
r'Fra:(?P<frame>\d*).*Mem:(?P<memory>\S+).*Time:(?P<time>\S+)(?:.*Remaining:)?(?P<remaining>\S*)')
cycles_match = cycles_pattern.search(line)
eevee_pattern = re.compile(
r"Rendering\s+(?P<current>\d+)\s*/\s*(?P<total>\d+)\s+samples"
)
eevee_match = eevee_pattern.search(line)
if cycles_match:
stats = cycles_match.groupdict()
found = pattern.search(line)
if found:
stats = found.groupdict()
memory_use = stats['memory']
time_elapsed = stats['time']
time_remaining = stats['remaining'] or 'Unknown'
@@ -113,11 +101,6 @@ class BlenderRenderWorker(BaseRenderWorker):
logger.debug(
'Frame:{0} | Mem:{1} | Time:{2} | Remaining:{3}'.format(self.current_frame, memory_use,
time_elapsed, time_remaining))
elif eevee_match:
self.__frame_percent_complete = int(eevee_match.groups()[0]) / int(eevee_match.groups()[1])
logger.debug(f'Frame:{self.current_frame} | Samples:{eevee_match.groups()[0]}/{eevee_match.groups()[1]}')
elif "Rendering frame" in line: # used for eevee
self.current_frame = int(line.split("Rendering frame")[-1].strip())
elif "file doesn't exist" in line.lower():
self.log_error(line, halt_render=True)
elif line.lower().startswith('error'):
src/engines/blender/scripts/get_system_info.py
-4
View File
@@ -1,10 +1,6 @@
import bpy
import json
# Force CPU rendering
bpy.context.preferences.addons["cycles"].preferences.compute_device_type = "NONE"
bpy.context.scene.cycles.device = "CPU"
# Ensure Cycles is available
bpy.context.preferences.addons['cycles'].preferences.get_devices()
src/engines/core/base_downloader.py
+9 -143
View File
@@ -1,7 +1,9 @@
import logging
import os
import shutil
import tarfile
import tempfile
import zipfile
import requests
from tqdm import tqdm
@@ -10,154 +12,26 @@ logger = logging.getLogger()
class EngineDownloader:
"""A class responsible for downloading and extracting rendering engines from publicly available URLs.
Attributes:
supported_formats (list[str]): A list of file formats supported by the downloader.
"""
supported_formats = ['.zip', '.tar.xz', '.dmg']
def __init__(self):
pass
# --------------------------------------------
# Required Overrides for Subclasses:
# --------------------------------------------
@classmethod
def find_most_recent_version(cls, system_os=None, cpu=None, lts_only=False):
"""
Finds the most recent version of the rendering engine available for download.
This method should be overridden in a subclass to implement the logic for determining
the most recent version of the rendering engine, optionally filtering by long-term
support (LTS) versions, the operating system, and CPU architecture.
Args:
system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
lts_only (bool, optional): Limit the search to LTS (long-term support) versions only. Default is False.
Returns:
dict: A dict with the following keys:
- 'cpu' (str): The CPU architecture.
- 'system_os' (str): The operating system.
- 'file' (str): The filename of the version's download file.
- 'url' (str): The remote URL for downloading the version.
- 'version' (str): The version number.
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"find_most_recent_version not implemented for {cls.__class__.__name__}")
raise NotImplementedError # implement this method in your engine subclass
@classmethod
def version_is_available_to_download(cls, version, system_os=None, cpu=None):
"""Checks if a requested version of the rendering engine is available for download.
This method should be overridden in a subclass to implement the logic for determining
whether a given version of the rendering engine is available for download, based on the
operating system and CPU architecture.
Args:
version (str): The requested renderer version to download.
system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
Returns:
bool: True if the version is available for download, False otherwise.
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"version_is_available_to_download not implemented for {cls.__class__.__name__}")
raise NotImplementedError # implement this method in your engine subclass
@classmethod
def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120, progress_callback=None):
"""Downloads the requested version of the rendering engine to the given download location.
This method should be overridden in a subclass to implement the logic for downloading
a specific version of the rendering engine. The method is intended to handle the
downloading process based on the version, operating system, CPU architecture, and
timeout parameters.
Args:
version (str): The requested renderer version to download.
download_location (str): The directory where the engine should be downloaded.
system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
timeout (int, optional): The maximum time in seconds to wait for the download. Default is 120 seconds.
progress_callback (callable, optional): A callback function that is called periodically with the current download progress.
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"download_engine not implemented for {cls.__class__.__name__}")
# --------------------------------------------
# Optional Overrides for Subclasses:
# --------------------------------------------
def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
raise NotImplementedError # implement this method in your engine subclass
@classmethod
def all_versions(cls, system_os=None, cpu=None):
"""Retrieves a list of available versions of the software for a specific operating system and CPU architecture.
This method fetches all available versions for the given operating system and CPU type, constructing
a list of dictionaries containing details such as the version, CPU architecture, system OS, and the
remote URL for downloading each version.
Args:
system_os (str, optional): Desired OS ('linux', 'macos', 'windows'). Defaults to system os.
cpu (str, optional): The CPU architecture for which to download the engine. Default is system cpu.
Returns:
list[dict]: A list of dictionaries, each containing:
- 'cpu' (str): The CPU architecture.
- 'file' (str): The filename of the version's download file.
- 'system_os' (str): The operating system.
- 'url' (str): The remote URL for downloading the version.
- 'version' (str): The version number.
"""
return []
# --------------------------------------------
# Do Not Override These Methods:
# --------------------------------------------
@classmethod
def download_and_extract_app(cls, remote_url, download_location, timeout=120, progress_callback=None):
"""Downloads an application from the given remote URL and extracts it to the specified location.
This method handles the downloading of the application, supports multiple archive formats,
and extracts the contents to the specified `download_location`. It also manages temporary
files and logs progress throughout the process.
Args:
remote_url (str): The URL of the application to download.
download_location (str): The directory where the application should be extracted.
timeout (int, optional): The maximum time in seconds to wait for the download. Default is 120 seconds.
progress_callback (callable, optional): A callback function that is called periodically with the current download progress.
Returns:
str: The path to the directory where the application was extracted.
Raises:
Exception: Catches and logs any exceptions that occur during the download or extraction process.
Supported Formats:
- `.tar.xz`: Extracted using the `tarfile` module.
- `.zip`: Extracted using the `zipfile` module.
- `.dmg`: macOS disk image files, handled using the `dmglib` library.
- Other formats will result in an error being logged.
Notes:
- If the application already exists in the `download_location`, the method will log an error
and return without downloading or extracting.
- Temporary files created during the download process are cleaned up after completion.
"""
if progress_callback:
progress_callback(0)
def download_and_extract_app(cls, remote_url, download_location, timeout=120):
# Create a temp download directory
temp_download_dir = tempfile.mkdtemp()
@@ -170,7 +44,7 @@ class EngineDownloader:
if os.path.exists(os.path.join(download_location, output_dir_name)):
logger.error(f"Engine download for {output_dir_name} already exists")
return None
return
if not os.path.exists(temp_downloaded_file_path):
# Make a GET request to the URL with stream=True to enable streaming
@@ -186,33 +60,26 @@ class EngineDownloader:
progress_bar = tqdm(total=file_size, unit="B", unit_scale=True)
# Open a file for writing in binary mode
total_saved = 0
with open(temp_downloaded_file_path, "wb") as file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
# Write the chunk to the file
file.write(chunk)
total_saved += len(chunk)
# Update the progress bar
progress_bar.update(len(chunk))
if progress_callback:
percent = float(total_saved) / float(file_size)
progress_callback(percent)
# Close the progress bar
progress_callback(1.0)
progress_bar.close()
logger.info(f"Successfully downloaded {os.path.basename(temp_downloaded_file_path)}")
else:
logger.error(f"Failed to download the file. Status code: {response.status_code}")
return None
return
os.makedirs(download_location, exist_ok=True)
# Extract the downloaded file
# Process .tar.xz files
if temp_downloaded_file_path.lower().endswith('.tar.xz'):
import tarfile
try:
with tarfile.open(temp_downloaded_file_path, 'r:xz') as tar:
tar.extractall(path=download_location)
@@ -226,7 +93,6 @@ class EngineDownloader:
# Process .zip files
elif temp_downloaded_file_path.lower().endswith('.zip'):
import zipfile
try:
with zipfile.ZipFile(temp_downloaded_file_path, 'r') as zip_ref:
zip_ref.extractall(download_location)
src/engines/core/base_engine.py
+53 -176
View File
@@ -1,196 +1,35 @@
import logging
import os
import platform
import subprocess
from typing import Optional, List, Dict, Any, Type
logger = logging.getLogger()
SUBPROCESS_TIMEOUT = 10
SUBPROCESS_TIMEOUT = 5
class BaseRenderEngine(object):
"""Base class for render engines. This class provides common functionality and structure for various rendering
engines. Create subclasses and override the methods marked below to add additional engines
Attributes:
install_paths (list): A list of default installation paths where the render engine
might be found. This list can be populated with common paths to help locate the
executable on different operating systems or environments.
"""
install_paths = []
supported_extensions = []
install_paths: List[str] = []
binary_names: Dict[str, str] = {}
def __init__(self, custom_path=None):
self.custom_renderer_path = custom_path
if not self.renderer_path() or not os.path.exists(self.renderer_path()):
raise FileNotFoundError(f"Cannot find path to renderer for {self.name()} instance")
# --------------------------------------------
# Required Overrides for Subclasses:
# --------------------------------------------
if not os.access(self.renderer_path(), os.X_OK):
logger.warning(f"Path is not executable. Setting permissions to 755 for {self.renderer_path()}")
os.chmod(self.renderer_path(), 0o755)
def __init__(self, custom_path: Optional[str] = None) -> None:
"""Initialize the render engine.
Args:
custom_path: Optional custom path to the engine executable.
Raises:
FileNotFoundError: If the engine executable cannot be found.
"""
self.custom_engine_path: Optional[str] = custom_path
if not self.engine_path() or not os.path.exists(self.engine_path()):
raise FileNotFoundError(f"Cannot find path to engine for {self.name()} instance: {self.engine_path()}")
if not os.access(self.engine_path(), os.X_OK):
logger.warning(f"Path is not executable. Setting permissions to 755 for {self.engine_path()}")
os.chmod(self.engine_path(), 0o755)
def version(self) -> str:
"""Return the version number as a string.
Returns:
str: Version number.
Raises:
NotImplementedError: If not overridden.
"""
raise NotImplementedError(f"version not implemented for {self.__class__.__name__}")
def get_project_info(self, project_path: str, timeout: int = 10) -> Dict[str, Any]:
"""Extracts detailed project information from the given project path.
Args:
project_path (str): The path to the project file.
timeout (int, optional): The maximum time (in seconds) to wait for the operation. Default is 10 seconds.
Returns:
dict: A dictionary containing project information (subclasses should define the structure).
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"get_project_info not implemented for {self.__class__.__name__}")
def renderer_path(self):
return self.custom_renderer_path or self.default_renderer_path()
@classmethod
def get_output_formats(cls) -> List[str]:
"""Returns a list of available output formats supported by the engine.
Returns:
list[str]: A list of strings representing the available output formats.
"""
raise NotImplementedError(f"get_output_formats not implemented for {cls.__name__}")
@staticmethod
def worker_class() -> Type[Any]: # override when subclassing to link worker class
raise NotImplementedError("Worker class not implemented")
# --------------------------------------------
# Optional Overrides for Subclasses:
# --------------------------------------------
def supported_extensions(self) -> List[str]:
"""Return a list of file extensions supported by this engine.
Returns:
List[str]: List of supported file extensions (e.g., ['.blend', '.mp4']).
"""
return []
def get_help(self) -> str:
"""Retrieves the help documentation for the engine.
This method runs the engine's help command (default: '-h') and captures the output.
Override this method if the engine uses a different help flag.
Returns:
str: The help documentation as a string.
Raises:
FileNotFoundError: If the engine path is not found.
"""
path = self.engine_path()
if not path:
raise FileNotFoundError(f"Engine path not found: {path}")
creationflags = subprocess.CREATE_NO_WINDOW if platform.system() == 'Windows' else 0
help_doc = subprocess.check_output([path, '-h'], stderr=subprocess.STDOUT,
timeout=SUBPROCESS_TIMEOUT, creationflags=creationflags).decode('utf-8')
return help_doc
def system_info(self) -> Dict[str, Any]:
"""Return additional information about the system specfic to the engine (configured GPUs, render engines, etc)
Returns:
dict: A dictionary with engine-specific system information
"""
return {}
def perform_presubmission_tasks(self, project_path: str) -> str:
"""Perform any pre-submission tasks on a project file before uploading it to a server (pack textures, etc.)
Override this method to:
1. Copy the project file to a temporary location (DO NOT MODIFY ORIGINAL PATH).
2. Perform additional modifications or tasks.
3. Return the path to the modified project file.
Args:
project_path (str): The original project file path.
Returns:
str: The path to the modified project file.
"""
return project_path
def get_arguments(self) -> Dict[str, Any]:
"""Return command-line arguments for this engine.
Returns:
Dict[str, Any]: Dictionary of argument specifications.
"""
return {}
@staticmethod
def downloader() -> Optional[Any]:
"""Return the downloader class for this engine.
Returns:
Optional[Any]: Downloader class instance, or None if no downloader is used.
"""
return None
def ui_options(self) -> Dict[str, Any]:
"""Return UI configuration options for this engine.
Returns:
Dict[str, Any]: Dictionary of UI options and their configurations.
"""
return {}
# --------------------------------------------
# Do Not Override These Methods:
# --------------------------------------------
def engine_path(self) -> Optional[str]:
"""Return the path to the engine executable.
Returns:
Optional[str]: Path to the engine executable, or None if not found.
"""
return self.custom_engine_path or self.default_engine_path()
@classmethod
def name(cls) -> str:
"""Return the name of this engine.
Returns:
str: Engine name in lowercase.
"""
def name(cls):
return str(cls.__name__).lower()
@classmethod
def default_engine_path(cls) -> Optional[str]:
"""Find the default path to the engine executable.
Returns:
Optional[str]: Default path to the engine executable, or None if not found.
"""
path: Optional[str] = None
def default_renderer_path(cls):
path = None
try: # Linux and macOS
path = subprocess.check_output(['which', cls.name()], timeout=SUBPROCESS_TIMEOUT).decode('utf-8').strip()
except (subprocess.CalledProcessError, FileNotFoundError):
@@ -200,3 +39,41 @@ class BaseRenderEngine(object):
except Exception as e:
logger.exception(e)
return path
def version(self):
raise NotImplementedError("version not implemented")
@staticmethod
def downloader(): # override when subclassing if using a downloader class
return None
@staticmethod
def worker_class(): # override when subclassing to link worker class
raise NotImplementedError("Worker class not implemented")
def ui_options(self): # override to return options for ui
return {}
def get_help(self): # override if renderer uses different help flag
path = self.renderer_path()
if not path:
raise FileNotFoundError("renderer path not found")
help_doc = subprocess.check_output([path, '-h'], stderr=subprocess.STDOUT,
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
return help_doc
def get_project_info(self, project_path, timeout=10):
raise NotImplementedError(f"get_project_info not implemented for {self.__name__}")
@classmethod
def get_output_formats(cls):
raise NotImplementedError(f"get_output_formats not implemented for {cls.__name__}")
def get_arguments(self):
pass
def system_info(self):
pass
def perform_presubmission_tasks(self, project_path):
return project_path
src/engines/core/base_worker.py
+117 -309
View File
@@ -3,12 +3,10 @@ import io
import json
import logging
import os
import signal
import subprocess
import threading
import time
from datetime import datetime
from typing import Optional, Dict, Any, List, Union
import psutil
from pubsub import pub
@@ -33,10 +31,11 @@ class BaseRenderWorker(Base):
date_created = Column(DateTime)
start_time = Column(DateTime, nullable=True)
end_time = Column(DateTime, nullable=True)
engine_name = Column(String)
engine_version = Column(String)
engine_path = Column(String)
renderer = Column(String)
renderer_version = Column(String)
renderer_path = Column(String)
priority = Column(Integer)
project_length = Column(Integer)
start_frame = Column(Integer)
end_frame = Column(Integer, nullable=True)
parent = Column(String, nullable=True)
@@ -46,39 +45,20 @@ class BaseRenderWorker(Base):
file_hash = Column(String)
_status = Column(String)
# --------------------------------------------
# Required Overrides for Subclasses:
# --------------------------------------------
engine = None
def __init__(self, input_path: str, output_path: str, engine_path: str, priority: int = 2,
args: Optional[Dict[str, Any]] = None, ignore_extensions: bool = True,
parent: Optional[str] = None, name: Optional[str] = None) -> None:
"""Initialize a render worker.
def __init__(self, input_path, output_path, engine_path, priority=2, args=None, ignore_extensions=True, parent=None,
name=None):
Args:
input_path: Path to the input project file.
output_path: Path where output files will be saved.
engine_path: Path to the render engine executable.
priority: Job priority level (default: 2).
args: Additional arguments for the render job.
ignore_extensions: Whether to ignore file extension validation.
parent: Parent job ID for distributed jobs.
name: Custom name for the job.
Raises:
ValueError: If file extension is not supported.
NotImplementedError: If engine is not defined.
"""
if not ignore_extensions:
if not any(ext in input_path for ext in self.engine.supported_extensions()):
err_meg = f"Cannot find valid project with supported file extension for '{self.engine.name()}'"
err_meg = f'Cannot find valid project with supported file extension for {self.engine.name()} renderer'
logger.error(err_meg)
raise ValueError(err_meg)
if not self.engine:
raise NotImplementedError(f"Engine not defined for {self.__class__.__name__}")
raise NotImplementedError("Engine not defined")
def generate_id():
"""Generate a unique job ID."""
import uuid
return str(uuid.uuid4()).split('-')[0]
@@ -89,10 +69,10 @@ class BaseRenderWorker(Base):
self.output_path = output_path
self.args = args or {}
self.date_created = datetime.now()
self.engine_name = self.engine.name()
self.engine_path = engine_path
self.engine_version = self.engine(engine_path).version()
self.custom_engine_path = None
self.renderer = self.engine.name()
self.renderer_path = engine_path
self.renderer_version = self.engine(engine_path).version()
self.custom_renderer_path = None
self.priority = priority
self.parent = parent
self.children = {}
@@ -100,6 +80,7 @@ class BaseRenderWorker(Base):
self.maximum_attempts = 3
# Frame Ranges
self.project_length = 0 # is this necessary?
self.current_frame = 0
self.start_frame = 0
self.end_frame = None
@@ -109,7 +90,7 @@ class BaseRenderWorker(Base):
self.end_time = None
# History
self.status = RenderStatus.CONFIGURING
self.status = RenderStatus.NOT_STARTED
self.warnings = []
self.errors = []
@@ -121,66 +102,12 @@ class BaseRenderWorker(Base):
self.__last_output_time = None
self.watchdog_timeout = 120
def generate_worker_subprocess(self) -> List[str]:
"""Generate a return a list of the command line arguments necessary to perform requested job
Returns:
list[str]: list of command line arguments
"""
raise NotImplementedError("generate_worker_subprocess not implemented")
def _parse_stdout(self, line: str) -> None:
"""Parses a line of standard output from the engine.
This method should be overridden in a subclass to implement the logic for processing
and interpreting a single line of output from the engine's standard output stream.
On frame completion, the subclass should:
1. Update value of self.current_frame
2. Call self._send_frame_complete_notification()
Args:
line (str): A line of text from the engine's standard output.
Raises:
NotImplementedError: If the method is not overridden in a subclass.
"""
raise NotImplementedError(f"_parse_stdout not implemented for {self.__class__.__name__}")
# --------------------------------------------
# Optional Overrides for Subclasses:
# --------------------------------------------
def percent_complete(self) -> float:
"""Return the completion percentage of this job.
Returns:
float: Completion percentage between 0.0 and 1.0.
"""
# todo: fix this
if self.status == RenderStatus.COMPLETED:
return 1.0
return 0
def post_processing(self) -> None:
"""Override to perform any engine-specific postprocessing"""
pass
# --------------------------------------------
# Do Not Override These Methods:
# --------------------------------------------
def __repr__(self):
"""Return string representation of the worker.
Returns:
str: String representation showing job details.
"""
return f"<Job id:{self.id} p{self.priority} {self.engine_name}-{self.engine_version} '{self.name}' status:{self.status.value}>"
return f"<Job id:{self.id} p{self.priority} {self.renderer}-{self.renderer_version} '{self.name}' status:{self.status.value}>"
@property
def total_frames(self):
return max(self.end_frame, 1) - self.start_frame + 1
return (self.end_frame or self.project_length) - self.start_frame + 1
@property
def status(self):
@@ -204,14 +131,6 @@ class BaseRenderWorker(Base):
pub.sendMessage('frame_complete', job_id=self.id, frame_number=self.current_frame)
def generate_subprocess(self):
"""Generate subprocess command arguments.
Returns:
List[str]: List of command line arguments for the subprocess.
Raises:
ValueError: If argument conflicts are detected.
"""
# Convert raw args from string if available and catch conflicts
generated_args = [str(x) for x in self.generate_worker_subprocess()]
generated_args_flags = [x for x in generated_args if x.startswith('-')]
@@ -222,34 +141,24 @@ class BaseRenderWorker(Base):
return generated_args
def get_raw_args(self):
"""Parse raw command line arguments from args dictionary.
Returns:
Optional[List[str]]: Parsed raw arguments, or None if no raw args.
"""
raw_args_string = self.args.get('raw', '')
raw_args_string = self.args.get('raw', None)
raw_args = None
if raw_args_string:
import shlex
raw_args = shlex.split(raw_args_string)
return raw_args
def log_path(self):
"""Generate the log file path for this job.
def generate_worker_subprocess(self):
raise NotImplementedError("generate_worker_subprocess not implemented")
Returns:
str: Path to the log file.
"""
def log_path(self):
filename = (self.name or os.path.basename(self.input_path)) + '_' + \
self.date_created.strftime("%Y.%m.%d_%H.%M.%S") + '.log'
return os.path.join(os.path.dirname(self.input_path), filename)
def start(self):
"""Start the render job.
Validates input paths and engine availability, then starts the render thread.
"""
if self.status not in [RenderStatus.SCHEDULED, RenderStatus.NOT_STARTED, RenderStatus.CONFIGURING]:
if self.status not in [RenderStatus.SCHEDULED, RenderStatus.NOT_STARTED]:
logger.error(f"Trying to start job with status: {self.status}")
return
@@ -260,136 +169,97 @@ class BaseRenderWorker(Base):
self.errors.append(msg)
return
if not os.path.exists(self.engine_path):
if not os.path.exists(self.renderer_path):
self.status = RenderStatus.ERROR
msg = f'Cannot find render engine path for {self.engine.name()}'
logger.error(msg)
self.errors.append(msg)
return
self.status = RenderStatus.RUNNING if not self.children else RenderStatus.WAITING_FOR_SUBJOBS
self.status = RenderStatus.RUNNING
self.start_time = datetime.now()
self.__thread.start()
# handle multiple attempts at running subprocess
def __run__subprocess_cycle(self, log_file):
subprocess_cmds = self.generate_subprocess()
initial_file_count = len(self.file_list())
failed_attempts = 0
log_file.write(f"Running command: {subprocess_cmds}\n")
log_file.write('=' * 80 + '\n\n')
while True:
# Log attempt #
if failed_attempts:
if failed_attempts >= self.maximum_attempts:
err_msg = f"Maximum attempts exceeded ({self.maximum_attempts})"
logger.error(err_msg)
self.status = RenderStatus.ERROR
self.errors.append(err_msg)
return
else:
log_file.write(f'\n{"=" * 20} Attempt #{failed_attempts + 1} {"=" * 20}\n\n')
logger.warning(f"Restarting render - Attempt #{failed_attempts + 1}")
self.status = RenderStatus.RUNNING
return_code = self.__setup_and_run_process(log_file, subprocess_cmds)
message = f"{'=' * 50}\n\n{self.engine.name()} render ended with code {return_code} " \
f"after {self.time_elapsed()}\n\n"
log_file.write(message)
# don't try again if we've been cancelled
if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]:
return
# if file output hasn't increased, return as error, otherwise restart process.
file_count_has_increased = len(self.file_list()) > initial_file_count
if (self.status == RenderStatus.RUNNING) and file_count_has_increased and not return_code:
break
if return_code:
err_msg = f"{self.engine.name()} render failed with code {return_code}"
logger.error(err_msg)
self.errors.append(err_msg)
# handle instances where engine exits ok but doesnt generate files
if not return_code and not file_count_has_increased:
err_msg = (f"{self.engine.name()} render exited ok, but file count has not increased. "
f"Count is still {len(self.file_list())}")
log_file.write(f'Error: {err_msg}\n\n')
self.errors.append(err_msg)
# only count the attempt as failed if engine creates no output - reset counter on successful output
failed_attempts = 0 if file_count_has_increased else failed_attempts + 1
def __run__wait_for_subjobs(self, logfile):
from src.distributed_job_manager import DistributedJobManager
DistributedJobManager.wait_for_subjobs(parent_job=self)
@staticmethod
def log_and_print(message, log_file, level='info'):
if level == 'debug':
logger.debug(message)
elif level == 'error':
logger.error(message)
else:
logger.info(message)
log_file.write(f"{message}\n")
def __run(self):
logger.info(f'Starting {self.engine.name()} {self.renderer_version} Render for {self.input_path} | '
f'Frame Count: {self.total_frames}')
# Setup logging
log_dir = os.path.dirname(self.log_path())
os.makedirs(log_dir, exist_ok=True)
with open(self.log_path(), "a") as log_file:
subprocess_cmds = self.generate_subprocess()
initial_file_count = len(self.file_list())
failed_attempts = 0
self.log_and_print(f"{self.start_time.isoformat()} - Starting "
f"{self.engine.name()} {self.engine_version} render job for {self.name} "
f"({self.input_path})", log_file)
log_file.write(f"\n")
if not self.children:
self.__run__subprocess_cycle(log_file)
else:
self.__run__wait_for_subjobs(log_file)
with open(self.log_path(), "a") as f:
# Validate Output - End if missing frames
if self.status == RenderStatus.RUNNING:
file_list_length = len(self.file_list())
expected_list_length = (self.end_frame - self.start_frame + 1) if self.end_frame else 1
f.write(f"{self.start_time.isoformat()} - Starting {self.engine.name()} {self.renderer_version} "
f"render for {self.input_path}\n\n")
f.write(f"Running command: {subprocess_cmds}\n")
f.write('=' * 80 + '\n\n')
msg = f"Frames: Expected ({expected_list_length}) vs actual ({file_list_length}) for {self}"
self.log_and_print(msg, log_file, 'debug')
while True:
# Log attempt #
if failed_attempts:
if failed_attempts >= self.maximum_attempts:
err_msg = f"Maximum attempts exceeded ({self.maximum_attempts})"
logger.error(err_msg)
self.status = RenderStatus.ERROR
self.errors.append(err_msg)
return
else:
f.write(f'\n{"=" * 20} Attempt #{failed_attempts + 1} {"=" * 20}\n\n')
logger.warning(f"Restarting render - Attempt #{failed_attempts + 1}")
self.status = RenderStatus.RUNNING
if file_list_length not in (expected_list_length, 1):
msg = f"Missing frames: Expected ({expected_list_length}) vs actual ({file_list_length})"
self.log_and_print(msg, log_file, 'error')
self.errors.append(msg)
self.status = RenderStatus.ERROR
# todo: create new subjob to generate missing frames
# cleanup and close if cancelled / error
if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]:
return_code = self.__setup_and_run_process(f, subprocess_cmds)
self.end_time = datetime.now()
message = f"{self.engine.name()} render ended with status '{self.status.value}' " \
f"after {self.time_elapsed()}"
self.log_and_print(message, log_file)
log_file.close()
return
# Post Render Work
if not self.parent:
logger.debug(f"Starting post-processing work for {self}")
self.log_and_print(f"Starting post-processing work for {self}", log_file, 'debug')
self.post_processing()
self.log_and_print(f"Completed post-processing work for {self}", log_file, 'debug')
message = f"{'=' * 50}\n\n{self.engine.name()} render ended with code {return_code} " \
f"after {self.time_elapsed()}\n\n"
f.write(message)
self.status = RenderStatus.COMPLETED
self.end_time = datetime.now()
message = f"Render {self.name} completed successfully after {self.time_elapsed()}"
self.log_and_print(message, log_file)
# Teardown
if self.status in [RenderStatus.CANCELLED, RenderStatus.ERROR]:
message = f"{self.engine.name()} render ended with status '{self.status}' " \
f"after {self.time_elapsed()}"
f.write(message)
return
# if file output hasn't increased, return as error, otherwise restart process.
file_count_has_increased = len(self.file_list()) > initial_file_count
if (self.status == RenderStatus.RUNNING) and file_count_has_increased and not return_code:
message = (f"{'=' * 50}\n\n{self.engine.name()} render completed successfully in "
f"{self.time_elapsed()}\n")
f.write(message)
break
if return_code:
err_msg = f"{self.engine.name()} render failed with code {return_code}"
logger.error(err_msg)
self.errors.append(err_msg)
# handle instances where renderer exits ok but doesnt generate files
if not return_code and not file_count_has_increased:
err_msg = (f"{self.engine.name()} render exited ok, but file count has not increased. "
f"Count is still {len(self.file_list())}")
f.write(f'Error: {err_msg}\n\n')
self.errors.append(err_msg)
# only count the attempt as failed if renderer creates no output - ignore error codes for now
if not file_count_has_increased:
failed_attempts += 1
if self.children:
from src.distributed_job_manager import DistributedJobManager
DistributedJobManager.wait_for_subjobs(local_job=self)
# Post Render Work
logger.debug("Starting post-processing work")
self.post_processing()
self.status = RenderStatus.COMPLETED
logger.info(f"Render {self.id}-{self.name} completed successfully after {self.time_elapsed()}")
def __setup_and_run_process(self, f, subprocess_cmds):
@@ -399,7 +269,7 @@ class BaseRenderWorker(Base):
time_since_last_update = time.time() - self.__last_output_time
if time_since_last_update > self.watchdog_timeout:
logger.error(f"Process for {self} terminated due to exceeding timeout ({self.watchdog_timeout}s)")
self.__kill_process()
self.__process.kill()
break
# logger.debug(f'Watchdog for {self} - Time since last update: {time_since_last_update}')
time.sleep(1)
@@ -412,14 +282,8 @@ class BaseRenderWorker(Base):
try:
# Start process and get updates
if os.name == 'posix': # linux / mac
self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=False, preexec_fn=os.setsid)
else: # windows
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW
self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=False,
creationflags=creationflags)
self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=False)
# Start watchdog
self.__last_output_time = time.time()
@@ -448,7 +312,7 @@ class BaseRenderWorker(Base):
message = f'Uncaught error running render process: {e}'
f.write(message)
logger.exception(message)
self.__kill_process()
self.__process.kill()
# let watchdog end before continuing - prevents multiple watchdogs running when process restarts
if watchdog_thread.is_alive():
@@ -456,60 +320,30 @@ class BaseRenderWorker(Base):
return return_code
def __kill_process(self):
try:
if self.__process.poll():
return
logger.debug(f"Trying to kill process {self.__process}")
self.__process.terminate()
self.__process.kill()
if os.name == 'posix': # linux / macos
os.killpg(os.getpgid(self.__process.pid), signal.SIGTERM)
os.killpg(os.getpgid(self.__process.pid), signal.SIGKILL)
else: # windows
parent = psutil.Process(self.__process.pid)
for child in parent.children(recursive=True):
child.kill()
self.__process.wait(timeout=5)
logger.debug(f"Process ended with status {self.__process.poll()}")
except (ProcessLookupError, AttributeError, psutil.NoSuchProcess):
pass
except Exception as e:
logger.error(f"Error stopping the process: {e}")
def post_processing(self):
pass
def is_running(self):
"""Check if the render job is currently running.
Returns:
bool: True if the job is running, False otherwise.
"""
if hasattr(self, '__thread'):
if self.__thread:
return self.__thread.is_alive()
return False
def log_error(self, error_line, halt_render=False):
"""Log an error and optionally halt the render.
Args:
error_line: Error message to log.
halt_render: Whether to stop the render due to this error.
"""
logger.error(error_line)
self.errors.append(error_line)
if halt_render:
self.stop(is_error=True)
def stop(self, is_error=False):
"""Stop the render job.
Args:
is_error: Whether this stop is due to an error.
"""
logger.debug(f"Stopping {self}")
# cleanup status
if self.status in [RenderStatus.RUNNING, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED,
RenderStatus.CONFIGURING]:
if hasattr(self, '__process'):
try:
process = psutil.Process(self.__process.pid)
for proc in process.children(recursive=True):
proc.kill()
process.kill()
except Exception as e:
logger.debug(f"Error stopping the process: {e}")
if self.status in [RenderStatus.RUNNING, RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED]:
if is_error:
err_message = self.errors[-1] if self.errors else 'Unknown error'
logger.error(f"Halting render due to error: {err_message}")
@@ -517,42 +351,27 @@ class BaseRenderWorker(Base):
else:
self.status = RenderStatus.CANCELLED
self.__kill_process()
if self.is_running(): # allow the log files to close
self.__thread.join(timeout=5)
def percent_complete(self):
if self.status == RenderStatus.COMPLETED:
return 1.0
return 0
def _parse_stdout(self, line):
raise NotImplementedError("_parse_stdout not implemented")
def time_elapsed(self):
"""Get elapsed time for this job.
Returns:
str: Formatted time elapsed string.
"""
return get_time_elapsed(self.start_time, self.end_time)
def file_list(self):
"""Get list of output files for this job.
Returns:
List[str]: List of output file paths.
"""
try:
job_dir = os.path.dirname(self.output_path)
file_list = [
os.path.join(job_dir, file)
for file in os.listdir(job_dir)
if not file.startswith('.') # Ignore hidden files
]
file_list = [os.path.join(job_dir, file) for file in os.listdir(job_dir)]
file_list.sort()
return file_list
except FileNotFoundError:
return []
def json(self):
"""Convert worker to JSON-serializable dictionary.
Returns:
Dict[str, Any]: Dictionary representation of worker data.
"""
job_dict = {
'id': self.id,
'name': self.name,
@@ -569,8 +388,8 @@ class BaseRenderWorker(Base):
'file_hash': self.file_hash,
'percent_complete': self.percent_complete(),
'file_count': len(self.file_list()),
'engine': self.engine_name,
'engine_version': self.engine_version,
'renderer': self.renderer,
'renderer_version': self.renderer_version,
'errors': getattr(self, 'errors', None),
'start_frame': self.start_frame,
'end_frame': self.end_frame,
@@ -582,10 +401,8 @@ class BaseRenderWorker(Base):
# convert to json and back to auto-convert dates to iso format
def date_serializer(o):
"""Serialize datetime objects to ISO format."""
if isinstance(o, datetime):
return o.isoformat()
return None
json_convert = json.dumps(job_dict, default=date_serializer)
worker_json = json.loads(json_convert)
@@ -593,15 +410,6 @@ class BaseRenderWorker(Base):
def timecode_to_frames(timecode, frame_rate):
"""Convert timecode string to frame number.
Args:
timecode: Timecode in format HH:MM:SS:FF.
frame_rate: Frames per second.
Returns:
int: Frame number corresponding to timecode.
"""
e = [int(x) for x in timecode.split(':')]
seconds = (((e[0] * 60) + e[1] * 60) + e[2])
frames = (seconds * frame_rate) + e[-1] + 1
src/engines/engine_manager.py
+151 -271
View File
@@ -3,261 +3,240 @@ import os
import shutil
import threading
import concurrent.futures
from pathlib import Path
from typing import Type, List, Dict, Any, Optional
from src.engines.core.base_engine import BaseRenderEngine
from src.engines.blender.blender_engine import Blender
from src.engines.ffmpeg.ffmpeg_engine import FFMPEG
from src.utilities.misc_helper import current_system_os, current_system_cpu
from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu
logger = logging.getLogger()
ENGINE_CLASSES = [Blender, FFMPEG]
class EngineManager:
"""Class that manages different versions of installed render engines and handles fetching and downloading new versions,
if possible.
"""
_default_instance: Optional['EngineManager'] = None
engines_path: Optional[str] = None
download_tasks: List[Any] = []
def __init__(self) -> None:
self.engines_path: Optional[str] = None
self.download_tasks: List[Any] = []
@classmethod
def _sync_class(cls) -> None:
if cls._default_instance is not None:
cls.engines_path = cls._default_instance.engines_path
cls.download_tasks = cls._default_instance.download_tasks
engines_path = None
download_tasks = []
@staticmethod
def supported_engines() -> list[type[BaseRenderEngine]]:
return ENGINE_CLASSES
def supported_engines():
return [Blender, FFMPEG]
# --- Installed Engines ---
def _engine_class_for_project_path(self, path: str) -> Type[BaseRenderEngine]:
_, extension = os.path.splitext(path)
extension = extension.lower().strip('.')
for engine_class in self.supported_engines():
engine = self.get_latest_engine_instance(engine_class)
if extension in engine.supported_extensions():
return engine_class
undefined_renderer_support = [x for x in self.supported_engines() if not self.get_latest_engine_instance(x).supported_extensions()]
return undefined_renderer_support[0]
def _engine_class_with_name(self, engine_name: str) -> Optional[Type[BaseRenderEngine]]:
for obj in self.supported_engines():
@classmethod
def engine_with_name(cls, engine_name):
for obj in cls.supported_engines():
if obj.name().lower() == engine_name.lower():
return obj
return None
def _get_latest_engine_instance(self, engine_class: Type[BaseRenderEngine]) -> BaseRenderEngine:
newest = self.newest_installed_engine_data(engine_class.name())
engine = engine_class(newest["path"])
return engine
@classmethod
def get_engines(cls, filter_name=None):
def _get_installed_engine_data(self, filter_name: Optional[str] = None, include_corrupt: bool = False,
ignore_system: bool = False) -> List[Dict[str, Any]]:
if not self.engines_path:
if not cls.engines_path:
raise FileNotFoundError("Engine path is not set")
# Parse downloaded engine directory
results = []
try:
all_items = os.listdir(self.engines_path)
all_directories = [item for item in all_items if os.path.isdir(os.path.join(self.engines_path, item))]
keys = ["engine", "version", "system_os", "cpu"]
all_items = os.listdir(cls.engines_path)
all_directories = [item for item in all_items if os.path.isdir(os.path.join(cls.engines_path, item))]
keys = ["engine", "version", "system_os", "cpu"] # Define keys for result dictionary
for directory in all_directories:
# Split directory name into segments
segments = directory.split('-')
# Create a dictionary mapping keys to corresponding segments
result_dict = {keys[i]: segments[i] for i in range(min(len(keys), len(segments)))}
result_dict['type'] = 'managed'
# Initialize binary_name with engine name
binary_name = result_dict['engine'].lower()
eng = self.engine_class_with_name(result_dict['engine'])
if eng:
binary_name = eng.binary_names.get(result_dict['system_os'], binary_name)
# Determine the correct binary name based on the engine and system_os
for eng in cls.supported_engines():
if eng.name().lower() == result_dict['engine']:
binary_name = eng.binary_names.get(result_dict['system_os'], binary_name)
break
# Find the path to the binary file
path = next(
(os.path.join(root, binary_name) for root, _, files in
os.walk(system_safe_path(os.path.join(cls.engines_path, directory))) if binary_name in files),
None
)
search_root = Path(self.engines_path) / directory
match = next((p for p in search_root.rglob(binary_name) if p.is_file()), None)
path = str(match) if match else None
result_dict['path'] = path
# Add the result dictionary to results if it matches the filter_name or if no filter is applied
if not filter_name or filter_name == result_dict['engine']:
results.append(result_dict)
except FileNotFoundError as e:
logger.warning(f"Cannot find local engines download directory: {e}")
def fetch_engine_details(eng, include_corrupt=False):
version = eng().version()
if not version and not include_corrupt:
return
# add system installs to this list - use bg thread because it can be slow
def fetch_engine_details(eng):
return {
'engine': eng.name(),
'version': version or 'error',
'version': eng().version(),
'system_os': current_system_os(),
'cpu': current_system_cpu(),
'path': eng.default_engine_path(),
'path': eng.default_renderer_path(),
'type': 'system'
}
if not ignore_system:
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
executor.submit(fetch_engine_details, eng, include_corrupt): eng.name()
for eng in self.supported_engines()
if eng.default_engine_path() and (not filter_name or filter_name == eng.name())
}
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
executor.submit(fetch_engine_details, eng): eng.name()
for eng in cls.supported_engines()
if eng.default_renderer_path() and (not filter_name or filter_name == eng.name())
}
for future in concurrent.futures.as_completed(futures):
result = future.result()
if result:
results.append(result)
for future in concurrent.futures.as_completed(futures):
result = future.result()
if result:
results.append(result)
return results
# --- Check for Updates ---
def _update_all_engines(self) -> None:
for engine in self.downloadable_engines():
update_available = self.is_engine_update_available(engine)
if update_available:
update_available['name'] = engine.name()
self.download_engine(engine.name(), update_available['version'], background=True)
def _all_version_data_for_engine(self, engine_name: str, include_corrupt=False, ignore_system=False) -> list:
versions = self.get_installed_engine_data(filter_name=engine_name, include_corrupt=include_corrupt, ignore_system=ignore_system)
@classmethod
def all_versions_for_engine(cls, engine_name):
versions = cls.get_engines(filter_name=engine_name)
sorted_versions = sorted(versions, key=lambda x: x['version'], reverse=True)
return sorted_versions
def _newest_installed_engine_data(self, engine_name: str, system_os=None, cpu=None, ignore_system=None) -> list:
@classmethod
def newest_engine_version(cls, engine, system_os=None, cpu=None):
system_os = system_os or current_system_os()
cpu = cpu or current_system_cpu()
try:
filtered = [x for x in self.all_version_data_for_engine(engine_name, ignore_system=ignore_system)
if x['system_os'] == system_os and x['cpu'] == cpu]
filtered = [x for x in cls.all_versions_for_engine(engine) if x['system_os'] == system_os and
x['cpu'] == cpu]
return filtered[0]
except IndexError:
logger.error(f"Cannot find newest engine version for {engine_name}-{system_os}-{cpu}")
return []
logger.error(f"Cannot find newest engine version for {engine}-{system_os}-{cpu}")
return None
def _is_version_installed(self, engine_name: str, version: str, system_os=None, cpu=None, ignore_system=False):
@classmethod
def is_version_downloaded(cls, engine, version, system_os=None, cpu=None):
system_os = system_os or current_system_os()
cpu = cpu or current_system_cpu()
filtered = [x for x in self.get_installed_engine_data(filter_name=engine_name, ignore_system=ignore_system) if
x['system_os'] == system_os and x['cpu'] == cpu and x['version'] == version]
filtered = [x for x in cls.get_engines(filter_name=engine) if x['system_os'] == system_os and
x['cpu'] == cpu and x['version'] == version]
return filtered[0] if filtered else False
def _version_is_available_to_download(self, engine_name: str, version, system_os=None, cpu=None):
@classmethod
def version_is_available_to_download(cls, engine, version, system_os=None, cpu=None):
try:
downloader = self.engine_class_with_name(engine_name).downloader()
downloader = cls.engine_with_name(engine).downloader()
return downloader.version_is_available_to_download(version=version, system_os=system_os, cpu=cpu)
except Exception as e:
logger.debug(f"Exception in version_is_available_to_download: {e}")
return None
def _find_most_recent_version(self, engine_name: str, system_os=None, cpu=None, lts_only=False) -> dict:
@classmethod
def find_most_recent_version(cls, engine=None, system_os=None, cpu=None, lts_only=False):
try:
downloader = self.engine_class_with_name(engine_name).downloader()
downloader = cls.engine_with_name(engine).downloader()
return downloader.find_most_recent_version(system_os=system_os, cpu=cpu)
except Exception as e:
logger.debug(f"Exception in find_most_recent_version: {e}")
return {}
def _is_engine_update_available(self, engine_class: Type[BaseRenderEngine], ignore_system_installs=False):
logger.debug(f"Checking for updates to {engine_class.name()}")
latest_version = engine_class.downloader().find_most_recent_version()
if not latest_version:
logger.warning(f"Could not find most recent version of {engine_class.name()} to download")
return None
version_num = latest_version.get('version')
if self.is_version_installed(engine_class.name(), version_num, ignore_system=ignore_system_installs):
logger.debug(f"Latest version of {engine_class.name()} ({version_num}) already downloaded")
return None
return latest_version
# --- Downloads ---
def _downloadable_engines(self):
return [engine for engine in self.supported_engines() if hasattr(engine, "downloader") and engine.downloader()]
def _get_existing_download_task(self, engine_name, version, system_os=None, cpu=None):
for task in self.download_tasks:
@classmethod
def get_existing_download_task(cls, engine, version, system_os=None, cpu=None):
for task in cls.download_tasks:
task_parts = task.name.split('-')
task_engine, task_version, task_system_os, task_cpu = task_parts[:4]
if engine_name == task_engine and version == task_version:
if engine == task_engine and version == task_version:
if system_os in (task_system_os, None) and cpu in (task_cpu, None):
return task
return None
def _download_engine(self, engine_name, version, system_os=None, cpu=None, background=False, ignore_system=False):
engine_to_download = self.engine_class_with_name(engine_name)
existing_task = self.get_existing_download_task(engine_name, version, system_os, cpu)
@classmethod
def download_engine(cls, engine, version, system_os=None, cpu=None, background=False):
engine_to_download = cls.engine_with_name(engine)
existing_task = cls.get_existing_download_task(engine, version, system_os, cpu)
if existing_task:
logger.debug(f"Already downloading {engine_name} {version}")
logger.debug(f"Already downloading {engine} {version}")
if not background:
existing_task.join()
return None
existing_task.join() # If download task exists, wait until it's done downloading
return
elif not engine_to_download.downloader():
logger.warning("No valid downloader for this engine. Please update this software manually.")
return None
elif not self.engines_path:
return
elif not cls.engines_path:
raise FileNotFoundError("Engines path must be set before requesting downloads")
thread = EngineDownloadWorker(engine_name, version, system_os, cpu)
self.download_tasks.append(thread)
thread = EngineDownloadWorker(engine, version, system_os, cpu)
cls.download_tasks.append(thread)
thread.start()
if background:
return thread
thread.join()
found_engine = self.is_version_installed(engine_name, version, system_os, cpu, ignore_system)
found_engine = cls.is_version_downloaded(engine, version, system_os, cpu) # Check that engine downloaded
if not found_engine:
logger.error(f"Error downloading {engine_name}")
logger.error(f"Error downloading {engine}")
return found_engine
def _delete_engine_download(self, engine_name, version, system_os=None, cpu=None):
logger.info(f"Requested deletion of engine: {engine_name}-{version}")
@classmethod
def delete_engine_download(cls, engine, version, system_os=None, cpu=None):
logger.info(f"Requested deletion of engine: {engine}-{version}")
found = self.is_version_installed(engine_name, version, system_os, cpu)
if found and found['type'] == 'managed':
root_dir_name = '-'.join([engine_name, version, found['system_os'], found['cpu']])
found = cls.is_version_downloaded(engine, version, system_os, cpu)
if found and found['type'] == 'managed': # don't delete system installs
# find the root directory of the engine executable
root_dir_name = '-'.join([engine, version, found['system_os'], found['cpu']])
remove_path = os.path.join(found['path'].split(root_dir_name)[0], root_dir_name)
# delete the file path
logger.info(f"Deleting engine at path: {remove_path}")
shutil.rmtree(remove_path, ignore_errors=False)
logger.info(f"Engine {engine_name}-{version}-{found['system_os']}-{found['cpu']} successfully deleted")
logger.info(f"Engine {engine}-{version}-{found['system_os']}-{found['cpu']} successfully deleted")
return True
elif found:
logger.error(f'Cannot delete requested {engine_name} {version}. Managed externally.')
elif found: # these are managed by the system / user. Don't delete these.
logger.error(f'Cannot delete requested {engine} {version}. Managed externally.')
else:
logger.error(f"Cannot find engine: {engine_name}-{version}")
logger.error(f"Cannot find engine: {engine}-{version}")
return False
# --- Background Tasks ---
@classmethod
def update_all_engines(cls):
def engine_update_task(engine_class):
logger.debug(f"Checking for updates to {engine_class.name()}")
latest_version = engine_class.downloader().find_most_recent_version()
def _active_downloads(self) -> list:
return [x for x in self.download_tasks if x.is_alive()]
if not latest_version:
logger.warning(f"Could not find most recent version of {engine.name()} to download")
return
def _create_worker(self, engine_name: str, input_path: Path, output_path: Path, engine_version=None, args=None, parent=None, name=None):
worker_class = self.engine_class_with_name(engine_name).worker_class()
version_num = latest_version.get('version')
if cls.is_version_downloaded(engine_class.name(), version_num):
logger.debug(f"Latest version of {engine_class.name()} ({version_num}) already downloaded")
return
all_versions = self.all_version_data_for_engine(engine_name)
# download the engine
logger.info(f"Downloading latest version of {engine_class.name()} ({version_num})...")
cls.download_engine(engine=engine_class.name(), version=version_num, background=True)
logger.info(f"Checking for updates for render engines...")
threads = []
for engine in cls.supported_engines():
if engine.downloader():
thread = threading.Thread(target=engine_update_task, args=(engine,))
threads.append(thread)
thread.start()
@classmethod
def create_worker(cls, renderer, input_path, output_path, engine_version=None, args=None, parent=None, name=None):
worker_class = cls.engine_with_name(renderer).worker_class()
# check to make sure we have versions installed
all_versions = cls.all_versions_for_engine(renderer)
if not all_versions:
raise FileNotFoundError(f"Cannot find any installed '{engine_name}' engines")
raise FileNotFoundError(f"Cannot find any installed {renderer} engines")
# Find the path to the requested engine version or use default
engine_path = None
if engine_version and engine_version != 'latest':
for ver in all_versions:
@@ -265,10 +244,11 @@ class EngineManager:
engine_path = ver['path']
break
# Download the required engine if not found locally
if not engine_path:
download_result = self.download_engine(engine_name, engine_version)
download_result = cls.download_engine(renderer, engine_version)
if not download_result:
raise FileNotFoundError(f"Cannot download requested version: {engine_name} {engine_version}")
raise FileNotFoundError(f"Cannot download requested version: {renderer} {engine_version}")
engine_path = download_result['path']
logger.info("Engine downloaded. Creating worker.")
else:
@@ -278,108 +258,18 @@ class EngineManager:
if not engine_path:
raise FileNotFoundError(f"Cannot find requested engine version {engine_version}")
return worker_class(input_path=str(input_path), output_path=str(output_path), engine_path=engine_path, args=args,
return worker_class(input_path=input_path, output_path=output_path, engine_path=engine_path, args=args,
parent=parent, name=name)
# --- Forwarders for backward compatibility ---
@classmethod
def engine_class_for_project_path(cls, path):
if cls._default_instance is not None:
return cls._default_instance._engine_class_for_project_path(path)
@classmethod
def engine_class_with_name(cls, engine_name):
if cls._default_instance is not None:
return cls._default_instance._engine_class_with_name(engine_name)
@classmethod
def get_latest_engine_instance(cls, engine_class):
if cls._default_instance is not None:
return cls._default_instance._get_latest_engine_instance(engine_class)
@classmethod
def get_installed_engine_data(cls, filter_name=None, include_corrupt=False, ignore_system=False):
if cls._default_instance is not None:
return cls._default_instance._get_installed_engine_data(filter_name, include_corrupt, ignore_system)
return []
@classmethod
def update_all_engines(cls):
if cls._default_instance is not None:
cls._default_instance._update_all_engines()
@classmethod
def all_version_data_for_engine(cls, engine_name, include_corrupt=False, ignore_system=False):
if cls._default_instance is not None:
return cls._default_instance._all_version_data_for_engine(engine_name, include_corrupt, ignore_system)
return []
@classmethod
def newest_installed_engine_data(cls, engine_name, system_os=None, cpu=None, ignore_system=None):
if cls._default_instance is not None:
return cls._default_instance._newest_installed_engine_data(engine_name, system_os, cpu, ignore_system)
return []
@classmethod
def is_version_installed(cls, engine_name, version, system_os=None, cpu=None, ignore_system=False):
if cls._default_instance is not None:
return cls._default_instance._is_version_installed(engine_name, version, system_os, cpu, ignore_system)
return False
@classmethod
def version_is_available_to_download(cls, engine_name, version, system_os=None, cpu=None):
if cls._default_instance is not None:
return cls._default_instance._version_is_available_to_download(engine_name, version, system_os, cpu)
return None
@classmethod
def find_most_recent_version(cls, engine_name, system_os=None, cpu=None, lts_only=False):
if cls._default_instance is not None:
return cls._default_instance._find_most_recent_version(engine_name, system_os, cpu, lts_only)
return {}
@classmethod
def is_engine_update_available(cls, engine_class, ignore_system_installs=False):
if cls._default_instance is not None:
return cls._default_instance._is_engine_update_available(engine_class, ignore_system_installs)
return None
@classmethod
def downloadable_engines(cls):
if cls._default_instance is not None:
return cls._default_instance._downloadable_engines()
return []
@classmethod
def get_existing_download_task(cls, engine_name, version, system_os=None, cpu=None):
if cls._default_instance is not None:
return cls._default_instance._get_existing_download_task(engine_name, version, system_os, cpu)
return None
@classmethod
def download_engine(cls, engine_name, version, system_os=None, cpu=None, background=False, ignore_system=False):
if cls._default_instance is not None:
return cls._default_instance._download_engine(engine_name, version, system_os, cpu, background, ignore_system)
return None
@classmethod
def delete_engine_download(cls, engine_name, version, system_os=None, cpu=None):
if cls._default_instance is not None:
return cls._default_instance._delete_engine_download(engine_name, version, system_os, cpu)
return False
@classmethod
def active_downloads(cls):
if cls._default_instance is not None:
return cls._default_instance._active_downloads()
return []
@classmethod
def create_worker(cls, engine_name, input_path, output_path, engine_version=None, args=None, parent=None, name=None):
if cls._default_instance is not None:
return cls._default_instance._create_worker(engine_name, input_path, output_path, engine_version, args, parent, name)
raise RuntimeError("EngineManager is not initialized")
def engine_for_project_path(cls, path):
_, extension = os.path.splitext(path)
extension = extension.lower().strip('.')
for engine in cls.supported_engines():
if extension in engine.supported_extensions():
return engine
undefined_renderer_support = [x for x in cls.supported_engines() if not x.supported_extensions()]
return undefined_renderer_support[0]
class EngineDownloadWorker(threading.Thread):
@@ -389,30 +279,20 @@ class EngineDownloadWorker(threading.Thread):
self.version = version
self.system_os = system_os
self.cpu = cpu
self.percent_complete = 0
def _update_progress(self, current_progress):
self.percent_complete = current_progress
def run(self):
try:
existing_download = EngineManager.is_version_installed(self.engine, self.version, self.system_os, self.cpu,
ignore_system=True)
if existing_download:
logger.info(f"Requested download of {self.engine} {self.version}, but local copy already exists")
return existing_download
existing_download = EngineManager.is_version_downloaded(self.engine, self.version, self.system_os, self.cpu)
if existing_download:
logger.info(f"Requested download of {self.engine} {self.version}, but local copy already exists")
return existing_download
downloader = EngineManager.engine_class_with_name(self.engine).downloader()
downloader.download_engine(self.version, download_location=EngineManager.engines_path,
system_os=self.system_os, cpu=self.cpu, timeout=300, progress_callback=self._update_progress)
except Exception as e:
logger.error(f"Error in download worker: {e}")
finally:
try:
if EngineManager._default_instance is not None:
EngineManager._default_instance.download_tasks.remove(self)
except ValueError:
pass
# Get the appropriate downloader class based on the engine type
EngineManager.engine_with_name(self.engine).downloader().download_engine(
self.version, download_location=EngineManager.engines_path, system_os=self.system_os, cpu=self.cpu,
timeout=300)
# remove itself from the downloader list
EngineManager.download_tasks.remove(self)
if __name__ == '__main__':
@@ -422,4 +302,4 @@ if __name__ == '__main__':
# EngineManager.delete_engine_download('blender', '3.2.1', 'macos', 'a')
EngineManager.engines_path = "/Users/brettwilliams/zordon-uploads/engines"
# print(EngineManager.is_version_downloaded("ffmpeg", "6.0"))
print(EngineManager.get_installed_engine_data())
print(EngineManager.get_engines())
src/engines/ffmpeg/ffmpeg_downloader.py
+4 -5
View File
@@ -97,7 +97,7 @@ class FFMPEGDownloader(EngineDownloader):
'windows': cls.__get_windows_versions}
if not versions_per_os.get(system_os):
logger.error(f"Cannot find version list for {system_os}")
return None
return
results = []
all_versions = versions_per_os[system_os]()
@@ -144,7 +144,7 @@ class FFMPEGDownloader(EngineDownloader):
return None
@classmethod
def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120, progress_callback=None):
def download_engine(cls, version, download_location, system_os=None, cpu=None, timeout=120):
system_os = system_os or current_system_os()
cpu = cpu or current_system_cpu()
@@ -152,7 +152,7 @@ class FFMPEGDownloader(EngineDownloader):
found_version = [item for item in cls.all_versions(system_os, cpu) if item['version'] == version]
if not found_version:
logger.error(f"Cannot find FFMPEG version {version} for {system_os} and {cpu}")
return None
return
# Platform specific naming cleanup
remote_url = cls.__get_remote_url_for_version(version=version, system_os=system_os, cpu=cpu)
@@ -162,8 +162,7 @@ class FFMPEGDownloader(EngineDownloader):
# Download and extract
try:
logger.info(f"Requesting download of ffmpeg-{version}-{system_os}-{cpu}")
cls.download_and_extract_app(remote_url=remote_url, download_location=download_location, timeout=timeout,
progress_callback=progress_callback)
cls.download_and_extract_app(remote_url=remote_url, download_location=download_location, timeout=timeout)
# naming cleanup to match existing naming convention
output_path = os.path.join(download_location, f'ffmpeg-{version}-{system_os}-{cpu}')
src/engines/ffmpeg/ffmpeg_engine.py
+21 -34
View File
@@ -1,11 +1,8 @@
import json
import os
import re
from src.engines.core.base_engine import *
_creationflags = subprocess.CREATE_NO_WINDOW if platform.system() == 'Windows' else 0
class FFMPEG(BaseRenderEngine):
binary_names = {'linux': 'ffmpeg', 'windows': 'ffmpeg.exe', 'macos': 'ffmpeg'}
@@ -21,11 +18,13 @@ class FFMPEG(BaseRenderEngine):
return FFMPEGRenderWorker
def ui_options(self):
return []
from src.engines.ffmpeg.ffmpeg_ui import FFMPEGUI
return FFMPEGUI.get_options(self)
def supported_extensions(self):
help_text = (subprocess.check_output([self.engine_path(), '-h', 'full'], stderr=subprocess.STDOUT,
creationflags=_creationflags).decode('utf-8'))
@classmethod
def supported_extensions(cls):
help_text = (subprocess.check_output([cls().renderer_path(), '-h', 'full'], stderr=subprocess.STDOUT)
.decode('utf-8'))
found = re.findall(r'extensions that .* is allowed to access \(default "(.*)"', help_text)
found_extensions = set()
for match in found:
@@ -35,8 +34,8 @@ class FFMPEG(BaseRenderEngine):
def version(self):
version = None
try:
ver_out = subprocess.check_output([self.engine_path(), '-version'], timeout=SUBPROCESS_TIMEOUT,
creationflags=_creationflags).decode('utf-8')
ver_out = subprocess.check_output([self.renderer_path(), '-version'],
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
match = re.match(r".*version\s*([\w.*]+)\W*", ver_out)
if match:
version = match.groups()[0]
@@ -45,23 +44,13 @@ class FFMPEG(BaseRenderEngine):
return version
def get_project_info(self, project_path, timeout=10):
"""Run ffprobe and parse the output as JSON"""
try:
# resolve ffprobe path
engine_dir = os.path.dirname(self.engine_path())
ffprobe_path = os.path.join(engine_dir, 'ffprobe')
if self.engine_path().endswith('.exe'):
ffprobe_path += '.exe'
if not os.path.exists(ffprobe_path): # fallback to system install (if available)
ffprobe_path = 'ffprobe'
# run ffprobe
# Run ffprobe and parse the output as JSON
cmd = [
ffprobe_path, '-v', 'quiet', '-print_format', 'json',
'ffprobe', '-v', 'quiet', '-print_format', 'json',
'-show_streams', '-select_streams', 'v', project_path
]
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True,
creationflags=_creationflags)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, text=True)
video_info = json.loads(output)
# Extract the necessary information
@@ -87,12 +76,12 @@ class FFMPEG(BaseRenderEngine):
}
except Exception as e:
print(f"Failed to get FFMPEG project info: {e}")
print(f"An error occurred: {e}")
return None
def get_encoders(self):
raw_stdout = subprocess.check_output([self.engine_path(), '-encoders'], stderr=subprocess.DEVNULL,
timeout=SUBPROCESS_TIMEOUT, creationflags=_creationflags).decode('utf-8')
raw_stdout = subprocess.check_output([self.renderer_path(), '-encoders'], stderr=subprocess.DEVNULL,
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
pattern = r'(?P<type>[VASFXBD.]{6})\s+(?P<name>\S{2,})\s+(?P<description>.*)'
encoders = [m.groupdict() for m in re.finditer(pattern, raw_stdout)]
return encoders
@@ -103,9 +92,8 @@ class FFMPEG(BaseRenderEngine):
def get_all_formats(self):
try:
formats_raw = subprocess.check_output([self.engine_path(), '-formats'], stderr=subprocess.DEVNULL,
timeout=SUBPROCESS_TIMEOUT,
creationflags=_creationflags).decode('utf-8')
formats_raw = subprocess.check_output([self.renderer_path(), '-formats'], stderr=subprocess.DEVNULL,
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
pattern = r'(?P<type>[DE]{1,2})\s+(?P<id>\S{2,})\s+(?P<name>.*)'
all_formats = [m.groupdict() for m in re.finditer(pattern, formats_raw)]
return all_formats
@@ -117,8 +105,7 @@ class FFMPEG(BaseRenderEngine):
# Extract the common extension using regex
muxer_flag = 'muxer' if 'E' in ffmpeg_format['type'] else 'demuxer'
format_detail_raw = subprocess.check_output(
[self.engine_path(), '-hide_banner', '-h', f"{muxer_flag}={ffmpeg_format['id']}"],
creationflags=_creationflags).decode('utf-8')
[self.renderer_path(), '-hide_banner', '-h', f"{muxer_flag}={ffmpeg_format['id']}"]).decode('utf-8')
pattern = r"Common extensions: (\w+)"
common_extensions = re.findall(pattern, format_detail_raw)
found_extensions = []
@@ -130,9 +117,9 @@ class FFMPEG(BaseRenderEngine):
return [x['id'] for x in self.get_all_formats() if 'E' in x['type'].upper()]
def get_frame_count(self, path_to_file):
raw_stdout = subprocess.check_output([self.engine_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy',
raw_stdout = subprocess.check_output([self.renderer_path(), '-i', path_to_file, '-map', '0:v:0', '-c', 'copy',
'-f', 'null', '-'], stderr=subprocess.STDOUT,
timeout=SUBPROCESS_TIMEOUT, creationflags=_creationflags).decode('utf-8')
timeout=SUBPROCESS_TIMEOUT).decode('utf-8')
match = re.findall(r'frame=\s*(\d+)', raw_stdout)
if match:
frame_number = int(match[-1])
@@ -140,8 +127,8 @@ class FFMPEG(BaseRenderEngine):
return -1
def get_arguments(self):
help_text = (subprocess.check_output([self.engine_path(), '-h', 'long'], stderr=subprocess.STDOUT,
creationflags=_creationflags).decode('utf-8'))
help_text = (subprocess.check_output([self.renderer_path(), '-h', 'long'], stderr=subprocess.STDOUT)
.decode('utf-8'))
lines = help_text.splitlines()
options = {}
src/engines/ffmpeg/ffmpeg_ui.py
+5
View File
@@ -0,0 +1,5 @@
class FFMPEGUI:
@staticmethod
def get_options(instance):
options = []
return options
src/engines/ffmpeg/ffmpeg_worker.py
+3 -3
View File
@@ -16,11 +16,11 @@ class FFMPEGRenderWorker(BaseRenderWorker):
def generate_worker_subprocess(self):
cmd = [self.engine_path, '-y', '-stats', '-i', self.input_path]
cmd = [self.renderer_path, '-y', '-stats', '-i', self.input_path]
# Resize frame
if self.args.get('resolution', None):
cmd.extend(['-vf', f"scale={self.args['resolution'][0]}:{self.args['resolution'][1]}"])
if self.args.get('x_resolution', None) and self.args.get('y_resolution', None):
cmd.extend(['-vf', f"scale={self.args['x_resolution']}:{self.args['y_resolution']}"])
# Convert raw args from string if available
raw_args = self.args.get('raw', None)
src/init.py
+158
View File
@@ -0,0 +1,158 @@
''' app/init.py '''
import logging
import multiprocessing
import os
import socket
import sys
import threading
import time
from collections import deque
from src.api.api_server import start_server
from src.api.preview_manager import PreviewManager
from src.api.serverproxy_manager import ServerProxyManager
from src.distributed_job_manager import DistributedJobManager
from src.engines.engine_manager import EngineManager
from src.render_queue import RenderQueue
from src.utilities.config import Config
from src.utilities.misc_helper import system_safe_path, current_system_cpu, current_system_os, current_system_os_version
from src.utilities.zeroconf_server import ZeroconfServer
logger = logging.getLogger()
def run(server_only=False) -> int:
"""
Initializes the application and runs it.
Returns:
int: The exit status code.
"""
# setup logging
logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S',
level=Config.server_log_level.upper())
logging.getLogger("requests").setLevel(logging.WARNING) # suppress noisy requests/urllib3 logging
logging.getLogger("urllib3").setLevel(logging.WARNING)
# Setup logging for console ui
buffer_handler = __setup_buffer_handler() if not server_only else None
logger.info(f"Starting Zordon Render Server")
return_code = 0
try:
# Load Config YAML
Config.setup_config_dir()
Config.load_config(system_safe_path(os.path.join(Config.config_dir(), 'config.yaml')))
# configure default paths
EngineManager.engines_path = system_safe_path(
os.path.join(os.path.join(os.path.expanduser(Config.upload_folder),
'engines')))
os.makedirs(EngineManager.engines_path, exist_ok=True)
PreviewManager.storage_path = system_safe_path(
os.path.join(os.path.expanduser(Config.upload_folder), 'previews'))
# Debug info
logger.debug(f"Upload directory: {os.path.expanduser(Config.upload_folder)}")
logger.debug(f"Thumbs directory: {PreviewManager.storage_path}")
logger.debug(f"Engines directory: {EngineManager.engines_path}")
# Set up the RenderQueue object
RenderQueue.load_state(database_directory=system_safe_path(os.path.expanduser(Config.upload_folder)))
ServerProxyManager.subscribe_to_listener()
DistributedJobManager.subscribe_to_listener()
# check for updates for render engines if configured or on first launch
if Config.update_engines_on_launch or not EngineManager.get_engines():
EngineManager.update_all_engines()
# get hostname
local_hostname = socket.gethostname()
local_hostname = local_hostname + (".local" if not local_hostname.endswith(".local") else "")
# configure and start API server
api_server = threading.Thread(target=start_server, args=(local_hostname,))
api_server.daemon = True
api_server.start()
# start zeroconf server
ZeroconfServer.configure("_zordon._tcp.local.", local_hostname, Config.port_number)
ZeroconfServer.properties = {'system_cpu': current_system_cpu(),
'system_cpu_cores': multiprocessing.cpu_count(),
'system_os': current_system_os(),
'system_os_version': current_system_os_version()}
ZeroconfServer.start()
logger.info(f"Zordon Render Server started - Hostname: {local_hostname}")
RenderQueue.evaluation_inverval = Config.queue_eval_seconds
RenderQueue.start()
# start in gui or server only (cli) mode
logger.debug(f"Launching in {'server only' if server_only else 'GUI'} mode")
if server_only: # CLI only
api_server.join()
else: # GUI
return_code = __show_gui(buffer_handler)
except KeyboardInterrupt:
pass
except Exception as e:
logging.error(f"Unhandled exception: {e}")
return_code = 1
finally:
# shut down gracefully
logger.info(f"Zordon Render Server is preparing to shut down")
try:
RenderQueue.prepare_for_shutdown()
except Exception as e:
logger.exception(f"Exception during prepare for shutdown: {e}")
ZeroconfServer.stop()
logger.info(f"Zordon Render Server has shut down")
return sys.exit(return_code)
def __setup_buffer_handler():
# lazy load GUI frameworks
from PyQt6.QtCore import QObject, pyqtSignal
class BufferingHandler(logging.Handler, QObject):
new_record = pyqtSignal(str)
def __init__(self, capacity=100):
logging.Handler.__init__(self)
QObject.__init__(self)
self.buffer = deque(maxlen=capacity) # Define a buffer with a fixed capacity
def emit(self, record):
try:
msg = self.format(record)
self.buffer.append(msg) # Add message to the buffer
self.new_record.emit(msg) # Emit signal
except RuntimeError:
pass
def get_buffer(self):
return list(self.buffer) # Return a copy of the buffer
buffer_handler = BufferingHandler()
buffer_handler.setFormatter(logging.getLogger().handlers[0].formatter)
logger = logging.getLogger()
logger.addHandler(buffer_handler)
return buffer_handler
def __show_gui(buffer_handler):
# lazy load GUI frameworks
from PyQt6.QtWidgets import QApplication
# load application
app: QApplication = QApplication(sys.argv)
# configure main window
from src.ui.main_window import MainWindow
window: MainWindow = MainWindow()
window.buffer_handler = buffer_handler
window.show()
return app.exec()
src/render_queue.py
+124 -232
View File
@@ -1,299 +1,191 @@
import logging
import threading
from collections import Counter
import os
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
from pubsub import pub
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.exc import DetachedInstanceError
from src.engines.core.base_worker import Base, BaseRenderWorker
from src.engines.core.base_worker import Base
from src.utilities.status_utils import RenderStatus
logger = logging.getLogger()
class JobNotFoundError(Exception):
def __init__(self, job_id: str, *args: Any) -> None:
def __init__(self, job_id, *args):
super().__init__(args)
self.job_id = job_id
def __str__(self) -> str:
def __str__(self):
return f"Cannot find job with ID: {self.job_id}"
class RenderQueue:
_default_instance: Optional['RenderQueue'] = None
engine = None
session = None
job_queue = []
maximum_renderer_instances = {'blender': 1, 'aerender': 1, 'ffmpeg': 4}
last_saved_counts = {}
is_running = False
__eval_thread = None
evaluation_inverval = 1
# --------------------------------------------
# Start / Stop Background Updates
# --------------------------------------------
@classmethod
def _sync_class(cls) -> None:
if cls._default_instance is not None:
pass # no class-level attributes to sync
def __init__(self) -> None:
self.engine: Optional[create_engine] = None
self.session: Optional[Session] = None
self.job_queue: List[BaseRenderWorker] = []
self.maximum_renderer_instances: Dict[str, int] = {'blender': 1, 'aerender': 1, 'ffmpeg': 4}
self.last_saved_counts: Dict[str, int] = {}
self.is_running: bool = False
self._lock = threading.Lock()
# --------------------------------------------
# Render Queue Evaluation:
# --------------------------------------------
def _start(self) -> None:
def start(cls):
logger.debug("Starting render queue updates")
self.is_running = True
self.evaluate_queue()
cls.is_running = True
cls.evaluate_queue()
def _evaluate_queue(self) -> None:
try:
not_started = self.jobs_with_status(RenderStatus.NOT_STARTED, priority_sorted=True)
for job in not_started:
if self.is_available_for_job(job.engine_name, job.priority):
self.start_job(job)
scheduled = self.jobs_with_status(RenderStatus.SCHEDULED, priority_sorted=True)
for job in scheduled:
if job.scheduled_start <= datetime.now():
logger.debug(f"Starting scheduled job: {job}")
self.start_job(job)
if self.last_saved_counts != self.job_counts():
self.save_state()
except DetachedInstanceError:
pass
def _local_job_status_changed(self, job_id: str, old_status: str, new_status: str) -> None:
render_job = self.job_with_id(job_id, none_ok=True)
if render_job and self.is_running:
@classmethod
def __local_job_status_changed(cls, job_id, old_status, new_status):
render_job = RenderQueue.job_with_id(job_id, none_ok=True)
if render_job and cls.is_running: # ignore changes from render jobs not in the queue yet
logger.debug(f"RenderQueue detected job {job_id} has changed from {old_status} -> {new_status}")
self.evaluate_queue()
RenderQueue.evaluate_queue()
def _stop(self) -> None:
@classmethod
def stop(cls):
logger.debug("Stopping render queue updates")
self.is_running = False
cls.is_running = False
# --------------------------------------------
# Fetch Jobs:
# Queue Management
# --------------------------------------------
def _all_jobs(self) -> List[BaseRenderWorker]:
return self.job_queue
@classmethod
def add_to_render_queue(cls, render_job, force_start=False):
logger.info(f"Adding job to render queue: {render_job}")
cls.job_queue.append(render_job)
if cls.is_running and force_start and render_job.status in (RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED):
cls.start_job(render_job)
cls.session.add(render_job)
cls.save_state()
if cls.is_running:
cls.evaluate_queue()
def _running_jobs(self) -> List[BaseRenderWorker]:
return self.jobs_with_status(RenderStatus.RUNNING)
@classmethod
def all_jobs(cls):
return cls.job_queue
def _pending_jobs(self) -> List[BaseRenderWorker]:
pending = self.jobs_with_status(RenderStatus.NOT_STARTED)
pending.extend(self.jobs_with_status(RenderStatus.SCHEDULED))
return pending
@classmethod
def running_jobs(cls):
return cls.jobs_with_status(RenderStatus.RUNNING)
def _jobs_with_status(self, status: RenderStatus, priority_sorted: bool = False) -> List[BaseRenderWorker]:
found_jobs = [x for x in self.all_jobs() if x.status == status]
@classmethod
def pending_jobs(cls):
pending_jobs = cls.jobs_with_status(RenderStatus.NOT_STARTED)
pending_jobs.extend(cls.jobs_with_status(RenderStatus.SCHEDULED))
return pending_jobs
@classmethod
def jobs_with_status(cls, status, priority_sorted=False):
found_jobs = [x for x in cls.all_jobs() if x.status == status]
if priority_sorted:
found_jobs = sorted(found_jobs, key=lambda a: a.priority, reverse=False)
return found_jobs
def _job_with_id(self, job_id: str, none_ok: bool = False) -> Optional[BaseRenderWorker]:
found_job = next((x for x in self.all_jobs() if x.id == job_id), None)
@classmethod
def job_with_id(cls, job_id, none_ok=False):
found_job = next((x for x in cls.all_jobs() if x.id == job_id), None)
if not found_job and not none_ok:
raise JobNotFoundError(job_id)
return found_job
def _job_counts(self) -> Dict[str, int]:
counts = Counter(x.status for x in self.all_jobs())
return {s.value: counts.get(s, 0) for s in RenderStatus}
@classmethod
def clear_history(cls):
to_remove = [x for x in cls.all_jobs() if x.status in [RenderStatus.CANCELLED,
RenderStatus.COMPLETED, RenderStatus.ERROR]]
for job_to_remove in to_remove:
cls.delete_job(job_to_remove)
cls.save_state()
# --------------------------------------------
# Startup / Shutdown:
# --------------------------------------------
def _load_state(self, database_directory: Path) -> None:
self.engine = create_engine(f"sqlite:///{database_directory / 'database.db'}")
Base.metadata.create_all(self.engine)
self.session = sessionmaker(bind=self.engine)()
@classmethod
def load_state(cls, database_directory):
if not cls.engine:
cls.engine = create_engine(f"sqlite:///{os.path.join(database_directory, 'database.db')}")
Base.metadata.create_all(cls.engine)
cls.session = sessionmaker(bind=cls.engine)()
from src.engines.core.base_worker import BaseRenderWorker
self.job_queue = self.session.query(BaseRenderWorker).all()
pub.subscribe(self._local_job_status_changed, 'status_change')
cls.job_queue = cls.session.query(BaseRenderWorker).all()
pub.subscribe(cls.__local_job_status_changed, 'status_change')
def _save_state(self) -> None:
if self.session:
self.session.commit()
@classmethod
def save_state(cls):
cls.session.commit()
def _prepare_for_shutdown(self) -> None:
@classmethod
def prepare_for_shutdown(cls):
logger.debug("Closing session")
self.stop()
running_jobs = self.jobs_with_status(RenderStatus.RUNNING)
for job in running_jobs:
self.cancel_job(job)
self.save_state()
if self.session:
self.session.close()
cls.stop()
running_jobs = cls.jobs_with_status(RenderStatus.RUNNING) # cancel all running jobs
[cls.cancel_job(job) for job in running_jobs]
cls.save_state()
cls.session.close()
# --------------------------------------------
# Renderer Availability:
# --------------------------------------------
@classmethod
def is_available_for_job(cls, renderer, priority=2):
def renderer_instances(self) -> Counter:
all_instances = [x.engine_name for x in self.running_jobs()]
return Counter(all_instances)
def _is_available_for_job(self, renderer: str, priority: int = 2) -> bool:
instances = self.renderer_instances()
higher_priority_jobs = [x for x in self.running_jobs() if x.priority < priority]
max_allowed_instances = self.maximum_renderer_instances.get(renderer, 1)
maxed_out_instances = renderer in instances and instances[renderer] >= max_allowed_instances
instances = cls.renderer_instances()
higher_priority_jobs = [x for x in cls.running_jobs() if x.priority < priority]
max_allowed_instances = cls.maximum_renderer_instances.get(renderer, 1)
maxed_out_instances = renderer in instances.keys() and instances[renderer] >= max_allowed_instances
return not maxed_out_instances and not higher_priority_jobs
# --------------------------------------------
# Job Lifecycle Management:
# --------------------------------------------
@classmethod
def evaluate_queue(cls):
try:
not_started = cls.jobs_with_status(RenderStatus.NOT_STARTED, priority_sorted=True)
for job in not_started:
if cls.is_available_for_job(job.renderer, job.priority):
cls.start_job(job)
def _add_to_render_queue(self, render_job: BaseRenderWorker, force_start: bool = False) -> None:
logger.info(f"Adding job to render queue: {render_job}")
with self._lock:
self.job_queue.append(render_job)
if force_start and render_job.status in (RenderStatus.NOT_STARTED, RenderStatus.SCHEDULED):
self.start_job(render_job)
self.session.add(render_job)
self.save_state()
if self.is_running:
self.evaluate_queue()
scheduled = cls.jobs_with_status(RenderStatus.SCHEDULED, priority_sorted=True)
for job in scheduled:
if job.scheduled_start <= datetime.now():
logger.debug(f"Starting scheduled job: {job}")
cls.start_job(job)
def _start_job(self, job: BaseRenderWorker) -> None:
if cls.last_saved_counts != cls.job_counts():
cls.save_state()
except DetachedInstanceError:
pass
@classmethod
def start_job(cls, job):
logger.info(f'Starting job: {job}')
job.start()
self.save_state()
cls.save_state()
def _cancel_job(self, job: BaseRenderWorker) -> bool:
@classmethod
def cancel_job(cls, job):
logger.info(f'Cancelling job: {job}')
job.stop()
return job.status == RenderStatus.CANCELLED
def _delete_job(self, job: BaseRenderWorker) -> bool:
@classmethod
def delete_job(cls, job):
logger.info(f"Deleting job: {job}")
with self._lock:
job.stop()
self.job_queue.remove(job)
self.session.delete(job)
self.save_state()
job.stop()
cls.job_queue.remove(job)
cls.session.delete(job)
cls.save_state()
return True
# --------------------------------------------
# Miscellaneous:
# --------------------------------------------
def _clear_history(self) -> None:
for job in list(self.all_jobs()):
if job.status in (RenderStatus.CANCELLED, RenderStatus.COMPLETED, RenderStatus.ERROR):
self.delete_job(job)
self.save_state()
# --- Forwarders for backward compatibility ---
@classmethod
def start(cls):
if cls._default_instance is not None:
cls._default_instance._start()
@classmethod
def evaluate_queue(cls):
if cls._default_instance is not None:
cls._default_instance._evaluate_queue()
@classmethod
def stop(cls):
if cls._default_instance is not None:
cls._default_instance._stop()
@classmethod
def all_jobs(cls):
if cls._default_instance is not None:
return cls._default_instance.job_queue
return []
@classmethod
def running_jobs(cls):
if cls._default_instance is not None:
return cls._default_instance._running_jobs()
return []
@classmethod
def pending_jobs(cls):
if cls._default_instance is not None:
return cls._default_instance._pending_jobs()
return []
@classmethod
def jobs_with_status(cls, status, priority_sorted=False):
if cls._default_instance is not None:
return cls._default_instance._jobs_with_status(status, priority_sorted)
return []
@classmethod
def job_with_id(cls, job_id, none_ok=False):
if cls._default_instance is not None:
return cls._default_instance._job_with_id(job_id, none_ok)
if not none_ok:
raise JobNotFoundError(job_id)
return None
def renderer_instances(cls):
from collections import Counter
all_instances = [x.renderer for x in cls.running_jobs()]
return Counter(all_instances)
@classmethod
def job_counts(cls):
if cls._default_instance is not None:
return cls._default_instance._job_counts()
return {}
@classmethod
def load_state(cls, database_directory):
if cls._default_instance is not None:
cls._default_instance._load_state(database_directory)
@classmethod
def save_state(cls):
if cls._default_instance is not None:
cls._default_instance._save_state()
@classmethod
def prepare_for_shutdown(cls):
if cls._default_instance is not None:
cls._default_instance._prepare_for_shutdown()
@classmethod
def is_available_for_job(cls, renderer, priority=2):
if cls._default_instance is not None:
return cls._default_instance._is_available_for_job(renderer, priority)
return True
@classmethod
def add_to_render_queue(cls, render_job, force_start=False):
if cls._default_instance is not None:
cls._default_instance._add_to_render_queue(render_job, force_start)
@classmethod
def start_job(cls, job):
if cls._default_instance is not None:
cls._default_instance._start_job(job)
@classmethod
def cancel_job(cls, job):
if cls._default_instance is not None:
return cls._default_instance._cancel_job(job)
return False
@classmethod
def delete_job(cls, job):
if cls._default_instance is not None:
return cls._default_instance._delete_job(job)
return False
@classmethod
def clear_history(cls):
if cls._default_instance is not None:
cls._default_instance._clear_history()
job_counts = {}
for job_status in RenderStatus:
job_counts[job_status.value] = len(cls.jobs_with_status(job_status))
return job_counts
src/ui/about_window.py
-74
View File
@@ -1,74 +0,0 @@
import os
import sys
from PyQt6.QtCore import Qt
from PyQt6.QtGui import QPixmap
from PyQt6.QtWidgets import QDialog, QVBoxLayout, QLabel, QDialogButtonBox, QHBoxLayout
from src.version import *
class AboutDialog(QDialog):
def __init__(self):
super().__init__()
self.setWindowTitle(f"About {APP_NAME}")
# Create the layout
layout = QVBoxLayout()
# App Icon
icon_name = 'Server.png' # todo: temp icon - replace with final later
icon_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
'resources', icon_name)
icon_label = QLabel(self)
icon_pixmap = QPixmap(icon_path)
icon_label.setPixmap(icon_pixmap)
icon_layout = QHBoxLayout()
icon_layout.addStretch()
icon_layout.addWidget(icon_label)
icon_layout.addStretch()
layout.addLayout(icon_layout)
# Application name
name_label = QLabel(f"<h2>{APP_NAME}</h2>")
layout.addWidget(name_label)
# Description
description_label = QLabel(APP_DESCRIPTION)
layout.addWidget(description_label)
# Version
version_label = QLabel(f"<strong>Version:</strong> {APP_VERSION}")
layout.addWidget(version_label)
# Contributors
contributors_label = QLabel(f"Copyright © {APP_COPYRIGHT_YEAR} {APP_AUTHOR}")
layout.addWidget(contributors_label)
# License
license_label = QLabel(f"Released under {APP_LICENSE}")
layout.addWidget(license_label)
# Add an "OK" button to close the dialog
button_box = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok)
button_box.accepted.connect(self.accept)
layout.addWidget(button_box)
# Set the layout for the dialog
self.setLayout(layout)
# Make the dialog non-resizable
self.setWindowFlags(self.windowFlags() & ~Qt.WindowType.WindowContextHelpButtonHint)
self.setFixedSize(self.sizeHint())
if __name__ == '__main__':
# lazy load GUI frameworks
from PyQt6.QtWidgets import QApplication
# load application
app: QApplication = QApplication(sys.argv)
window: AboutDialog = AboutDialog()
window.show()
app.exec()
src/ui/add_job.py
+544
View File
@@ -0,0 +1,544 @@
import copy
import os.path
import pathlib
import socket
import threading
import psutil
from PyQt6.QtCore import QThread, pyqtSignal, Qt, pyqtSlot
from PyQt6.QtWidgets import (
QApplication, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QPushButton, QFileDialog, QSpinBox, QComboBox,
QGroupBox, QCheckBox, QProgressBar, QPlainTextEdit, QDoubleSpinBox, QMessageBox, QListWidget, QListWidgetItem
)
from requests import Response
from src.api.server_proxy import RenderServerProxy
from src.engines.engine_manager import EngineManager
from src.ui.engine_help_viewer import EngineHelpViewer
from src.utilities.zeroconf_server import ZeroconfServer
class NewRenderJobForm(QWidget):
def __init__(self, project_path=None):
super().__init__()
self.notes_group = None
self.frame_rate_input = None
self.resolution_x_input = None
self.renderer_group = None
self.output_settings_group = None
self.resolution_y_input = None
self.project_path = project_path
# UI
self.project_group = None
self.load_file_group = None
self.current_engine_options = None
self.file_format_combo = None
self.renderer_options_layout = None
self.cameras_list = None
self.cameras_group = None
self.renderer_version_combo = None
self.worker_thread = None
self.msg_box = None
self.engine_help_viewer = None
self.raw_args = None
self.submit_progress_label = None
self.submit_progress = None
self.renderer_type = None
self.process_label = None
self.process_progress_bar = None
self.splitjobs_same_os = None
self.enable_splitjobs = None
self.server_input = None
self.submit_button = None
self.notes_input = None
self.priority_input = None
self.end_frame_input = None
self.start_frame_input = None
self.render_name_input = None
self.scene_file_input = None
self.scene_file_browse_button = None
self.job_name_input = None
# Job / Server Data
self.server_proxy = RenderServerProxy(socket.gethostname())
self.renderer_info = None
self.project_info = None
# Setup
self.setWindowTitle("New Job")
self.setup_ui()
self.setup_project()
# get renderer info in bg thread
t = threading.Thread(target=self.update_renderer_info)
t.start()
self.show()
def setup_ui(self):
# Main Layout
main_layout = QVBoxLayout(self)
# Loading File Group
self.load_file_group = QGroupBox("Loading")
load_file_layout = QVBoxLayout(self.load_file_group)
# progress bar
progress_layout = QHBoxLayout()
self.process_progress_bar = QProgressBar()
self.process_progress_bar.setMinimum(0)
self.process_progress_bar.setMaximum(0)
self.process_label = QLabel("Processing")
progress_layout.addWidget(self.process_label)
progress_layout.addWidget(self.process_progress_bar)
load_file_layout.addLayout(progress_layout)
main_layout.addWidget(self.load_file_group)
# Project Group
self.project_group = QGroupBox("Project")
server_layout = QVBoxLayout(self.project_group)
# File Path
scene_file_picker_layout = QHBoxLayout()
self.scene_file_input = QLineEdit()
self.scene_file_input.setText(self.project_path)
self.scene_file_browse_button = QPushButton("Browse...")
self.scene_file_browse_button.clicked.connect(self.browse_scene_file)
scene_file_picker_layout.addWidget(QLabel("File:"))
scene_file_picker_layout.addWidget(self.scene_file_input)
scene_file_picker_layout.addWidget(self.scene_file_browse_button)
server_layout.addLayout(scene_file_picker_layout)
# Server List
server_list_layout = QHBoxLayout()
server_list_layout.setSpacing(0)
self.server_input = QComboBox()
server_list_layout.addWidget(QLabel("Hostname:"), 1)
server_list_layout.addWidget(self.server_input, 3)
server_layout.addLayout(server_list_layout)
main_layout.addWidget(self.project_group)
self.update_server_list()
# Priority
priority_layout = QHBoxLayout()
priority_layout.addWidget(QLabel("Priority:"), 1)
self.priority_input = QComboBox()
self.priority_input.addItems(["High", "Medium", "Low"])
self.priority_input.setCurrentIndex(1)
priority_layout.addWidget(self.priority_input, 3)
server_layout.addLayout(priority_layout)
# Splitjobs
self.enable_splitjobs = QCheckBox("Automatically split render across multiple servers")
self.enable_splitjobs.setEnabled(True)
server_layout.addWidget(self.enable_splitjobs)
self.splitjobs_same_os = QCheckBox("Only render on same OS")
self.splitjobs_same_os.setEnabled(True)
server_layout.addWidget(self.splitjobs_same_os)
# Output Settings Group
self.output_settings_group = QGroupBox("Output Settings")
output_settings_layout = QVBoxLayout(self.output_settings_group)
# output path
render_name_layout = QHBoxLayout()
render_name_layout.addWidget(QLabel("Render name:"))
self.render_name_input = QLineEdit()
render_name_layout.addWidget(self.render_name_input)
output_settings_layout.addLayout(render_name_layout)
# file format
file_format_layout = QHBoxLayout()
file_format_layout.addWidget(QLabel("Format:"))
self.file_format_combo = QComboBox()
file_format_layout.addWidget(self.file_format_combo)
output_settings_layout.addLayout(file_format_layout)
# frame range
frame_range_layout = QHBoxLayout(self.output_settings_group)
self.start_frame_input = QSpinBox()
self.start_frame_input.setRange(1, 99999)
self.end_frame_input = QSpinBox()
self.end_frame_input.setRange(1, 99999)
frame_range_layout.addWidget(QLabel("Frames:"))
frame_range_layout.addWidget(self.start_frame_input)
frame_range_layout.addWidget(QLabel("to"))
frame_range_layout.addWidget(self.end_frame_input)
output_settings_layout.addLayout(frame_range_layout)
# resolution
resolution_layout = QHBoxLayout(self.output_settings_group)
self.resolution_x_input = QSpinBox()
self.resolution_x_input.setRange(1, 9999) # Assuming max resolution width 9999
self.resolution_x_input.setValue(1920)
self.resolution_y_input = QSpinBox()
self.resolution_y_input.setRange(1, 9999) # Assuming max resolution height 9999
self.resolution_y_input.setValue(1080)
self.frame_rate_input = QDoubleSpinBox()
self.frame_rate_input.setRange(1, 9999) # Assuming max resolution width 9999
self.frame_rate_input.setDecimals(3)
self.frame_rate_input.setValue(23.976)
resolution_layout.addWidget(QLabel("Resolution:"))
resolution_layout.addWidget(self.resolution_x_input)
resolution_layout.addWidget(QLabel("x"))
resolution_layout.addWidget(self.resolution_y_input)
resolution_layout.addWidget(QLabel("@"))
resolution_layout.addWidget(self.frame_rate_input)
resolution_layout.addWidget(QLabel("fps"))
output_settings_layout.addLayout(resolution_layout)
# add group to layout
main_layout.addWidget(self.output_settings_group)
# Renderer Group
self.renderer_group = QGroupBox("Renderer Settings")
renderer_group_layout = QVBoxLayout(self.renderer_group)
renderer_layout = QHBoxLayout()
renderer_layout.addWidget(QLabel("Renderer:"))
self.renderer_type = QComboBox()
self.renderer_type.currentIndexChanged.connect(self.renderer_changed)
renderer_layout.addWidget(self.renderer_type)
# Version
renderer_layout.addWidget(QLabel("Version:"))
self.renderer_version_combo = QComboBox()
self.renderer_version_combo.addItem('latest')
renderer_layout.addWidget(self.renderer_version_combo)
renderer_group_layout.addLayout(renderer_layout)
# dynamic options
self.renderer_options_layout = QVBoxLayout()
renderer_group_layout.addLayout(self.renderer_options_layout)
# Raw Args
raw_args_layout = QHBoxLayout(self.renderer_group)
raw_args_layout.addWidget(QLabel("Raw Args:"))
self.raw_args = QLineEdit()
raw_args_layout.addWidget(self.raw_args)
args_help_button = QPushButton("?")
args_help_button.clicked.connect(self.args_help_button_clicked)
raw_args_layout.addWidget(args_help_button)
renderer_group_layout.addLayout(raw_args_layout)
main_layout.addWidget(self.renderer_group)
# Cameras Group
self.cameras_group = QGroupBox("Cameras")
cameras_layout = QVBoxLayout(self.cameras_group)
self.cameras_list = QListWidget()
self.cameras_group.setHidden(True)
cameras_layout.addWidget(self.cameras_list)
main_layout.addWidget(self.cameras_group)
# Notes Group
self.notes_group = QGroupBox("Additional Notes")
notes_layout = QVBoxLayout(self.notes_group)
self.notes_input = QPlainTextEdit()
notes_layout.addWidget(self.notes_input)
main_layout.addWidget(self.notes_group)
# Submit Button
self.submit_button = QPushButton("Submit Job")
self.submit_button.clicked.connect(self.submit_job)
main_layout.addWidget(self.submit_button)
self.submit_progress = QProgressBar()
self.submit_progress.setMinimum(0)
self.submit_progress.setMaximum(0)
self.submit_progress.setHidden(True)
main_layout.addWidget(self.submit_progress)
self.submit_progress_label = QLabel("Submitting...")
self.submit_progress_label.setHidden(True)
main_layout.addWidget(self.submit_progress_label)
self.toggle_renderer_enablement(False)
def update_renderer_info(self):
# get the renderer info and add them all to the ui
self.renderer_info = self.server_proxy.get_renderer_info(response_type='full')
self.renderer_type.addItems(self.renderer_info.keys())
# select the best renderer for the file type
engine = EngineManager.engine_for_project_path(self.project_path)
self.renderer_type.setCurrentText(engine.name().lower())
# refresh ui
self.renderer_changed()
def renderer_changed(self):
# load the version numbers
current_renderer = self.renderer_type.currentText().lower() or self.renderer_type.itemText(0)
self.renderer_version_combo.clear()
self.renderer_version_combo.addItem('latest')
self.file_format_combo.clear()
if current_renderer:
renderer_vers = [version_info['version'] for version_info in self.renderer_info[current_renderer]['versions']]
self.renderer_version_combo.addItems(renderer_vers)
self.file_format_combo.addItems(self.renderer_info[current_renderer]['supported_export_formats'])
def update_server_list(self):
clients = ZeroconfServer.found_hostnames()
self.server_input.clear()
self.server_input.addItems(clients)
def browse_scene_file(self):
file_name, _ = QFileDialog.getOpenFileName(self, "Select Scene File")
if file_name:
self.scene_file_input.setText(file_name)
self.setup_project()
def setup_project(self):
# UI stuff on main thread
self.process_progress_bar.setHidden(False)
self.process_label.setHidden(False)
self.toggle_renderer_enablement(False)
output_name, _ = os.path.splitext(os.path.basename(self.scene_file_input.text()))
output_name = output_name.replace(' ', '_')
self.render_name_input.setText(output_name)
file_name = self.scene_file_input.text()
# setup bg worker
self.worker_thread = GetProjectInfoWorker(window=self, project_path=file_name)
self.worker_thread.message_signal.connect(self.post_get_project_info_update)
self.worker_thread.start()
def browse_output_path(self):
directory = QFileDialog.getExistingDirectory(self, "Select Output Directory")
if directory:
self.render_name_input.setText(directory)
def args_help_button_clicked(self):
url = (f'http://{self.server_proxy.hostname}:{self.server_proxy.port}/api/renderer/'
f'{self.renderer_type.currentText()}/help')
self.engine_help_viewer = EngineHelpViewer(url)
self.engine_help_viewer.show()
# -------- Update --------
def post_get_project_info_update(self):
"""Called by the GetProjectInfoWorker - Do not call directly."""
try:
# Set the best renderer we can find
input_path = self.scene_file_input.text()
engine = EngineManager.engine_for_project_path(input_path)
engine_index = self.renderer_type.findText(engine.name().lower())
if engine_index >= 0:
self.renderer_type.setCurrentIndex(engine_index)
else:
self.renderer_type.setCurrentIndex(0) #todo: find out why we don't have renderer info yet
# not ideal but if we don't have the renderer info we have to pick something
# cleanup progress UI
self.load_file_group.setHidden(True)
self.toggle_renderer_enablement(True)
# Load scene data
self.start_frame_input.setValue(self.project_info.get('frame_start'))
self.end_frame_input.setValue(self.project_info.get('frame_end'))
self.resolution_x_input.setValue(self.project_info.get('resolution_x'))
self.resolution_y_input.setValue(self.project_info.get('resolution_y'))
self.frame_rate_input.setValue(self.project_info.get('fps'))
# Cameras
self.cameras_list.clear()
if self.project_info.get('cameras'):
self.cameras_group.setHidden(False)
found_active = False
for camera in self.project_info['cameras']:
# create the list items and make them checkable
item = QListWidgetItem(f"{camera['name']} - {camera['lens']}mm")
item.setFlags(item.flags() | Qt.ItemFlag.ItemIsUserCheckable)
is_checked = camera['is_active'] or len(self.project_info['cameras']) == 1
found_active = found_active or is_checked
item.setCheckState(Qt.CheckState.Checked if is_checked else Qt.CheckState.Unchecked)
self.cameras_list.addItem(item)
if not found_active:
self.cameras_list.item(0).setCheckState(Qt.CheckState.Checked)
else:
self.cameras_group.setHidden(True)
# Dynamic Engine Options
clear_layout(self.renderer_options_layout) # clear old options
# dynamically populate option list
self.current_engine_options = engine().ui_options()
for option in self.current_engine_options:
h_layout = QHBoxLayout()
label = QLabel(option['name'].replace('_', ' ').capitalize() + ':')
h_layout.addWidget(label)
if option.get('options'):
combo_box = QComboBox()
for opt in option['options']:
combo_box.addItem(opt)
h_layout.addWidget(combo_box)
else:
text_box = QLineEdit()
h_layout.addWidget(text_box)
self.renderer_options_layout.addLayout(h_layout)
except AttributeError:
pass
def toggle_renderer_enablement(self, enabled=False):
"""Toggle on/off all the render settings"""
self.project_group.setHidden(not enabled)
self.output_settings_group.setHidden(not enabled)
self.renderer_group.setHidden(not enabled)
self.notes_group.setHidden(not enabled)
if not enabled:
self.cameras_group.setHidden(True)
self.submit_button.setEnabled(enabled)
def after_job_submission(self, result):
# UI cleanup
self.submit_progress.setMaximum(0)
self.submit_button.setHidden(False)
self.submit_progress.setHidden(True)
self.submit_progress_label.setHidden(True)
self.process_progress_bar.setHidden(True)
self.process_label.setHidden(True)
self.toggle_renderer_enablement(True)
self.msg_box = QMessageBox()
if result.ok:
self.msg_box.setStandardButtons(QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No)
self.msg_box.setIcon(QMessageBox.Icon.Information)
self.msg_box.setText("Job successfully submitted to server. Submit another?")
self.msg_box.setWindowTitle("Success")
x = self.msg_box.exec()
if x == QMessageBox.StandardButton.No:
self.close()
else:
self.msg_box.setStandardButtons(QMessageBox.StandardButton.Ok)
self.msg_box.setIcon(QMessageBox.Icon.Critical)
self.msg_box.setText(result.text or "Unknown error")
self.msg_box.setWindowTitle("Error")
self.msg_box.exec()
# -------- Submit Job Calls --------
def submit_job(self):
# Pre-worker UI
self.submit_progress.setHidden(False)
self.submit_progress_label.setHidden(False)
self.submit_button.setHidden(True)
self.submit_progress.setMaximum(0)
# submit job in background thread
self.worker_thread = SubmitWorker(window=self)
self.worker_thread.update_ui_signal.connect(self.update_submit_progress)
self.worker_thread.message_signal.connect(self.after_job_submission)
self.worker_thread.start()
@pyqtSlot(str, str)
def update_submit_progress(self, hostname, percent):
# Update the UI here. This slot will be executed in the main thread
self.submit_progress_label.setText(f"Transferring to {hostname} - {percent}%")
self.submit_progress.setMaximum(100)
self.submit_progress.setValue(int(percent))
class SubmitWorker(QThread):
"""Worker class called to submit all the jobs to the server and update the UI accordingly"""
message_signal = pyqtSignal(Response)
update_ui_signal = pyqtSignal(str, str)
def __init__(self, window):
super().__init__()
self.window = window
def run(self):
def create_callback(encoder):
encoder_len = encoder.len
def callback(monitor):
percent = f"{monitor.bytes_read / encoder_len * 100:.0f}"
self.update_ui_signal.emit(hostname, percent)
return callback
hostname = self.window.server_input.currentText()
job_json = {'owner': psutil.Process().username() + '@' + socket.gethostname(),
'renderer': self.window.renderer_type.currentText().lower(),
'engine_version': self.window.renderer_version_combo.currentText(),
'args': {'raw': self.window.raw_args.text(),
'export_format': self.window.file_format_combo.currentText()},
'output_path': self.window.render_name_input.text(),
'start_frame': self.window.start_frame_input.value(),
'end_frame': self.window.end_frame_input.value(),
'priority': self.window.priority_input.currentIndex() + 1,
'notes': self.window.notes_input.toPlainText(),
'enable_split_jobs': self.window.enable_splitjobs.isChecked(),
'split_jobs_same_os': self.window.splitjobs_same_os.isChecked(),
'name': self.window.render_name_input.text()}
# get the dynamic args
for i in range(self.window.renderer_options_layout.count()):
item = self.window.renderer_options_layout.itemAt(i)
layout = item.layout() # get the layout
for x in range(layout.count()):
z = layout.itemAt(x)
widget = z.widget()
if isinstance(widget, QComboBox):
job_json['args'][self.window.current_engine_options[i]['name']] = widget.currentText()
elif isinstance(widget, QLineEdit):
job_json['args'][self.window.current_engine_options[i]['name']] = widget.text()
# determine if any cameras are checked
selected_cameras = []
if self.window.cameras_list.count() and not self.window.cameras_group.isHidden():
for index in range(self.window.cameras_list.count()):
item = self.window.cameras_list.item(index)
if item.checkState() == Qt.CheckState.Checked:
selected_cameras.append(item.text().rsplit('-', 1)[0].strip()) # cleanup to just camera name
# process cameras into nested format
input_path = self.window.scene_file_input.text()
if selected_cameras:
job_list = []
for cam in selected_cameras:
job_copy = copy.deepcopy(job_json)
job_copy['args']['camera'] = cam
job_copy['name'] = job_copy['name'].replace(' ', '-') + "_" + cam.replace(' ', '')
job_copy['output_path'] = job_copy['name']
job_list.append(job_copy)
else:
job_list = [job_json]
# presubmission tasks
engine = EngineManager.engine_with_name(self.window.renderer_type.currentText().lower())
input_path = engine().perform_presubmission_tasks(input_path)
# submit
result = self.window.server_proxy.post_job_to_server(file_path=input_path, job_list=job_list,
callback=create_callback)
self.message_signal.emit(result)
class GetProjectInfoWorker(QThread):
"""Worker class called to retrieve information about a project file on a background thread and update the UI"""
message_signal = pyqtSignal()
def __init__(self, window, project_path):
super().__init__()
self.window = window
self.project_path = project_path
def run(self):
engine = EngineManager.engine_for_project_path(self.project_path)
self.window.project_info = engine().get_project_info(self.project_path)
self.message_signal.emit()
def clear_layout(layout):
if layout is not None:
# Go through the layout's items in reverse order
for i in reversed(range(layout.count())):
# Take the item at the current position
item = layout.takeAt(i)
# Check if the item is a widget
if item.widget():
# Remove the widget and delete it
widget_to_remove = item.widget()
widget_to_remove.setParent(None)
widget_to_remove.deleteLater()
elif item.layout():
# If the item is a sub-layout, clear its contents recursively
clear_layout(item.layout())
# Then delete the layout
item.layout().deleteLater()
# Run the application
if __name__ == '__main__':
app = QApplication([])
window = NewRenderJobForm()
app.exec()
src/ui/add_job_window.py
-672
View File
@@ -1,672 +0,0 @@
import logging
import socket
from pathlib import Path
import psutil
from PyQt6.QtCore import QThread, pyqtSignal, Qt, pyqtSlot
logger = logging.getLogger(__name__)
from PyQt6.QtWidgets import (
QApplication, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QPushButton, QFileDialog, QSpinBox, QComboBox,
QGroupBox, QCheckBox, QProgressBar, QPlainTextEdit, QDoubleSpinBox, QMessageBox, QListWidget, QListWidgetItem,
QTabWidget
)
from src.api.server_proxy import RenderServerProxy
from src.engines.engine_manager import EngineManager
from src.ui.engine_help_window import EngineHelpViewer
from src.utilities.misc_helper import COMMON_RESOLUTIONS, COMMON_FRAME_RATES
from src.utilities.zeroconf_server import ZeroconfServer
class NewRenderJobForm(QWidget):
def __init__(self, project_path=None):
super().__init__()
self.resolution_options_list = None
self.resolution_x_input = None
self.resolution_y_input = None
self.fps_options_list = None
self.fps_input = None
self.engine_group = None
self.notes_group = None
self.output_settings_group = None
self.project_path = project_path
# UI
self.project_group = None
self.load_file_group = None
self.current_engine_options = None
self.file_format_combo = None
self.engine_options_layout = None
self.cameras_list = None
self.cameras_group = None
self.engine_version_combo = None
self.worker_thread = None
self.msg_box = None
self.engine_help_viewer = None
self.raw_args = None
self.submit_progress_label = None
self.submit_progress = None
self.engine_type = None
self.process_label = None
self.process_progress_bar = None
self.splitjobs_same_os = None
self.enable_splitjobs = None
self.server_input = None
self.submit_button = None
self.notes_input = None
self.priority_input = None
self.end_frame_input = None
self.start_frame_input = None
self.job_name_input = None
self.scene_file_input = None
self.scene_file_browse_button = None
self.tabs = None
# Job / Server Data
self.server_proxy = RenderServerProxy(socket.gethostname())
self.project_info = None
self.installed_engines = {}
self.preferred_engine = None
# Setup
self.setWindowTitle("New Job")
self.setup_ui()
self.setup_project()
self.show()
def setup_ui(self):
# Main widget layout
main_layout = QVBoxLayout(self)
# Tabs
self.tabs = QTabWidget()
# ==================== Loading Section (outside tabs) ====================
self.load_file_group = QGroupBox("Loading")
load_file_layout = QVBoxLayout(self.load_file_group)
progress_layout = QHBoxLayout()
self.process_label = QLabel("Processing")
self.process_progress_bar = QProgressBar()
self.process_progress_bar.setMinimum(0)
self.process_progress_bar.setMaximum(0) # Indeterminate
progress_layout.addWidget(self.process_label)
progress_layout.addWidget(self.process_progress_bar)
load_file_layout.addLayout(progress_layout)
# Scene File
job_overview_group = QGroupBox("Project File")
file_group_layout = QVBoxLayout(job_overview_group)
# Job Name
job_name_layout = QHBoxLayout()
job_name_layout.addWidget(QLabel("Job name:"))
self.job_name_input = QLineEdit()
job_name_layout.addWidget(self.job_name_input)
self.engine_type = QComboBox()
job_name_layout.addWidget(self.engine_type)
file_group_layout.addLayout(job_name_layout)
# Job File
scene_file_picker_layout = QHBoxLayout()
scene_file_picker_layout.addWidget(QLabel("File:"))
self.scene_file_input = QLineEdit()
self.scene_file_input.setText(self.project_path)
self.scene_file_browse_button = QPushButton("Browse...")
self.scene_file_browse_button.clicked.connect(self.browse_scene_file)
scene_file_picker_layout.addWidget(self.scene_file_input)
scene_file_picker_layout.addWidget(self.scene_file_browse_button)
file_group_layout.addLayout(scene_file_picker_layout)
main_layout.addWidget(job_overview_group)
main_layout.addWidget(self.load_file_group)
main_layout.addWidget(self.tabs)
# ==================== Tab 1: Job Settings ====================
self.project_group = QWidget()
project_layout = QVBoxLayout(self.project_group) # Fixed: proper parent
# Server / Hostname
server_list_layout = QHBoxLayout()
server_list_layout.addWidget(QLabel("Render Target:"))
self.server_input = QComboBox()
server_list_layout.addWidget(self.server_input)
project_layout.addLayout(server_list_layout)
# Priority
priority_layout = QHBoxLayout()
priority_layout.addWidget(QLabel("Priority:"))
self.priority_input = QComboBox()
self.priority_input.addItems(["High", "Medium", "Low"])
self.priority_input.setCurrentIndex(1)
priority_layout.addWidget(self.priority_input)
project_layout.addLayout(priority_layout)
# Split Jobs Options
self.enable_splitjobs = QCheckBox("Automatically split render across multiple servers")
project_layout.addWidget(self.enable_splitjobs)
self.splitjobs_same_os = QCheckBox("Only render on same OS")
project_layout.addWidget(self.splitjobs_same_os)
project_layout.addStretch() # Push everything up
# ==================== Tab 2: Output Settings ====================
self.output_settings_group = QWidget()
output_settings_layout = QVBoxLayout(self.output_settings_group)
# File Format
format_group = QGroupBox("Format / Range")
output_settings_layout.addWidget(format_group)
format_group_layout = QVBoxLayout()
format_group.setLayout(format_group_layout)
file_format_layout = QHBoxLayout()
file_format_layout.addWidget(QLabel("Format:"))
self.file_format_combo = QComboBox()
self.file_format_combo.setFixedWidth(200)
file_format_layout.addWidget(self.file_format_combo)
file_format_layout.addStretch()
format_group_layout.addLayout(file_format_layout)
# Frame Range
frame_range_layout = QHBoxLayout()
frame_range_layout.addWidget(QLabel("Frames:"))
self.start_frame_input = QSpinBox()
self.start_frame_input.setRange(1, 99999)
self.start_frame_input.setFixedWidth(80)
self.end_frame_input = QSpinBox()
self.end_frame_input.setRange(1, 99999)
self.end_frame_input.setFixedWidth(80)
frame_range_layout.addWidget(self.start_frame_input)
frame_range_layout.addWidget(QLabel("to"))
frame_range_layout.addWidget(self.end_frame_input)
frame_range_layout.addStretch()
format_group_layout.addLayout(frame_range_layout)
# --- Resolution & FPS Group ---
resolution_group = QGroupBox("Resolution / Frame Rate")
output_settings_layout.addWidget(resolution_group)
resolution_group_layout = QVBoxLayout()
resolution_group.setLayout(resolution_group_layout)
# Resolution
resolution_layout = QHBoxLayout()
self.resolution_options_list = QComboBox()
self.resolution_options_list.setFixedWidth(200)
self.resolution_options_list.addItem("Original Size")
for res in COMMON_RESOLUTIONS:
self.resolution_options_list.addItem(res)
self.resolution_options_list.currentIndexChanged.connect(self._resolution_preset_changed)
resolution_layout.addWidget(self.resolution_options_list)
resolution_group_layout.addLayout(resolution_layout)
self.resolution_x_input = QSpinBox()
self.resolution_x_input.setRange(1, 9999)
self.resolution_x_input.setValue(1920)
self.resolution_x_input.setFixedWidth(80)
resolution_layout.addWidget(self.resolution_x_input)
self.resolution_y_input = QSpinBox()
self.resolution_y_input.setRange(1, 9999)
self.resolution_y_input.setValue(1080)
self.resolution_y_input.setFixedWidth(80)
resolution_layout.addWidget(QLabel("x"))
resolution_layout.addWidget(self.resolution_y_input)
resolution_layout.addStretch()
fps_layout = QHBoxLayout()
self.fps_options_list = QComboBox()
self.fps_options_list.setFixedWidth(200)
self.fps_options_list.addItem("Original FPS")
for fps_option in COMMON_FRAME_RATES:
self.fps_options_list.addItem(fps_option)
self.fps_options_list.currentIndexChanged.connect(self._fps_preset_changed)
fps_layout.addWidget(self.fps_options_list)
self.fps_input = QDoubleSpinBox()
self.fps_input.setDecimals(3)
self.fps_input.setRange(1.0, 999.0)
self.fps_input.setValue(23.976)
self.fps_input.setFixedWidth(80)
fps_layout.addWidget(self.fps_input)
fps_layout.addWidget(QLabel("fps"))
fps_layout.addStretch()
resolution_group_layout.addLayout(fps_layout)
output_settings_layout.addStretch()
# ==================== Tab 3: Engine Settings ====================
self.engine_group = QWidget()
engine_group_layout = QVBoxLayout(self.engine_group)
engine_layout = QHBoxLayout()
engine_layout.addWidget(QLabel("Engine Version:"))
self.engine_version_combo = QComboBox()
self.engine_version_combo.addItem('latest')
engine_layout.addWidget(self.engine_version_combo)
engine_group_layout.addLayout(engine_layout)
# Dynamic engine options
self.engine_options_layout = QVBoxLayout()
engine_group_layout.addLayout(self.engine_options_layout)
# Raw Args
raw_args_layout = QHBoxLayout()
raw_args_layout.addWidget(QLabel("Raw Args:"))
self.raw_args = QLineEdit()
raw_args_layout.addWidget(self.raw_args)
args_help_button = QPushButton("?")
args_help_button.clicked.connect(self.args_help_button_clicked)
raw_args_layout.addWidget(args_help_button)
engine_group_layout.addLayout(raw_args_layout)
engine_group_layout.addStretch()
# ==================== Tab 4: Cameras ====================
self.cameras_group = QWidget()
cameras_layout = QVBoxLayout(self.cameras_group)
self.cameras_list = QListWidget()
self.cameras_list.itemChanged.connect(self.update_job_count)
cameras_layout.addWidget(self.cameras_list)
# ==================== Tab 5: Misc / Notes ====================
self.notes_group = QWidget()
notes_layout = QVBoxLayout(self.notes_group)
self.notes_input = QPlainTextEdit()
notes_layout.addWidget(self.notes_input)
# == Create Tabs
self.tabs.addTab(self.project_group, "Job Settings")
self.tabs.addTab(self.output_settings_group, "Output Settings")
self.tabs.addTab(self.engine_group, "Engine Settings")
self.tabs.addTab(self.cameras_group, "Cameras")
self.tabs.addTab(self.notes_group, "Notes")
self.update_server_list()
index = self.tabs.indexOf(self.cameras_group)
if index != -1:
self.tabs.setTabEnabled(index, False)
# ==================== Submit Section (outside tabs) ====================
self.submit_button = QPushButton("Submit Job")
self.submit_button.clicked.connect(self.submit_job)
main_layout.addWidget(self.submit_button)
self.submit_progress = QProgressBar()
self.submit_progress.setMinimum(0)
self.submit_progress.setMaximum(0)
self.submit_progress.setHidden(True)
main_layout.addWidget(self.submit_progress)
self.submit_progress_label = QLabel("Submitting...")
self.submit_progress_label.setHidden(True)
main_layout.addWidget(self.submit_progress_label)
# Initial engine state
self.toggle_engine_enablement(False)
self.tabs.setCurrentIndex(0)
def update_job_count(self, changed_item=None):
checked = 1
if self.cameras_group.isEnabled():
checked = 0
total = self.cameras_list.count()
for i in range(total):
item = self.cameras_list.item(i)
if item.checkState() == Qt.CheckState.Checked:
checked += 1
message = f"Submit {checked} Jobs" if checked > 1 else "Submit Job"
self.submit_button.setText(message)
self.submit_button.setEnabled(bool(checked))
def _resolution_preset_changed(self, index):
selected_res = COMMON_RESOLUTIONS.get(self.resolution_options_list.currentText())
if selected_res:
self.resolution_x_input.setValue(selected_res[0])
self.resolution_y_input.setValue(selected_res[1])
elif index == 0:
self.resolution_x_input.setValue(self.project_info.get('resolution_x'))
self.resolution_y_input.setValue(self.project_info.get('resolution_y'))
def _fps_preset_changed(self, index):
selected_fps = COMMON_FRAME_RATES.get(self.fps_options_list.currentText())
if selected_fps:
self.fps_input.setValue(selected_fps)
elif index == 0:
self.fps_input.setValue(self.project_info.get('fps'))
def engine_changed(self):
# load the version numbers
current_engine = self.engine_type.currentText().lower() or self.engine_type.itemText(0)
self.engine_version_combo.clear()
self.engine_version_combo.addItem('latest')
self.file_format_combo.clear()
if current_engine:
engine_info = self.server_proxy.get_engine_info(current_engine, 'full', timeout=10)
self.current_engine_options = engine_info.get('options', [])
if not engine_info:
raise FileNotFoundError(f"Cannot get information about engine '{current_engine}'")
engine_vers = [v['version'] for v in engine_info['versions']]
self.engine_version_combo.addItems(engine_vers)
self.file_format_combo.addItems(engine_info.get('supported_export_formats'))
def update_server_list(self):
clients = ZeroconfServer.found_hostnames()
self.server_input.clear()
self.server_input.addItems(clients)
def browse_scene_file(self):
file_name, _ = QFileDialog.getOpenFileName(self, "Select Scene File")
if file_name:
self.scene_file_input.setText(file_name)
self.setup_project()
def setup_project(self):
# UI stuff on main thread
self.process_progress_bar.setHidden(False)
self.process_label.setHidden(False)
self.toggle_engine_enablement(False)
output_name = Path(self.scene_file_input.text()).stem.replace(' ', '_')
self.job_name_input.setText(output_name)
file_name = self.scene_file_input.text()
# setup bg worker
self.worker_thread = GetProjectInfoWorker(window=self, project_path=file_name)
self.worker_thread.message_signal.connect(self.post_get_project_info_update)
self.worker_thread.error_signal.connect(self.show_error_message)
self.worker_thread.start()
def browse_output_path(self):
directory = QFileDialog.getExistingDirectory(self, "Select Output Directory")
if directory:
self.job_name_input.setText(directory)
def args_help_button_clicked(self):
url = (f'http://{self.server_proxy.hostname}:{self.server_proxy.port}/api/engine/'
f'{self.engine_type.currentText()}/help')
self.engine_help_viewer = EngineHelpViewer(url)
self.engine_help_viewer.show()
def show_error_message(self, message):
msg = QMessageBox(self)
msg.setIcon(QMessageBox.Icon.Critical)
msg.setWindowTitle("Error")
msg.setText(message)
msg.exec()
# -------- Update --------
def post_get_project_info_update(self):
"""Called by the GetProjectInfoWorker - Do not call directly."""
try:
self.engine_type.addItems(self.installed_engines.keys())
self.engine_type.setCurrentText(self.preferred_engine)
self.engine_changed()
# Set the best engine we can find
input_path = self.scene_file_input.text()
engine = EngineManager.engine_class_for_project_path(input_path)
engine_index = self.engine_type.findText(engine.name().lower())
if engine_index >= 0:
self.engine_type.setCurrentIndex(engine_index)
else:
self.engine_type.setCurrentIndex(0) #todo: find out why we don't have engine info yet
# not ideal but if we don't have the engine info we have to pick something
# cleanup progress UI
self.load_file_group.setHidden(True)
self.toggle_engine_enablement(True)
# Load scene data
self.start_frame_input.setValue(self.project_info.get('frame_start'))
self.end_frame_input.setValue(self.project_info.get('frame_end'))
self.resolution_x_input.setValue(self.project_info.get('resolution_x'))
self.resolution_y_input.setValue(self.project_info.get('resolution_y'))
self.fps_input.setValue(self.project_info.get('fps'))
# Cameras
self.cameras_list.clear()
index = self.tabs.indexOf(self.cameras_group)
if self.project_info.get('cameras'):
self.tabs.setTabEnabled(index, True)
found_active = False
for camera in self.project_info['cameras']:
# create the list items and make them checkable
item = QListWidgetItem(f"{camera['name']} - {camera['lens']}mm")
item.setFlags(item.flags() | Qt.ItemFlag.ItemIsUserCheckable)
is_checked = camera['is_active'] or len(self.project_info['cameras']) == 1
found_active = found_active or is_checked
item.setCheckState(Qt.CheckState.Checked if is_checked else Qt.CheckState.Unchecked)
self.cameras_list.addItem(item)
if not found_active:
self.cameras_list.item(0).setCheckState(Qt.CheckState.Checked)
else:
self.tabs.setTabEnabled(index, False)
self.update_job_count()
# Dynamic Engine Options
clear_layout(self.engine_options_layout) # clear old options
# dynamically populate option list
for option in self.current_engine_options:
h_layout = QHBoxLayout()
label = QLabel(option['name'].replace('_', ' ').capitalize() + ':')
h_layout.addWidget(label)
if option.get('options'):
combo_box = QComboBox()
for opt in option['options']:
combo_box.addItem(opt)
h_layout.addWidget(combo_box)
else:
text_box = QLineEdit()
h_layout.addWidget(text_box)
self.engine_options_layout.addLayout(h_layout)
except AttributeError as e:
logger.error(f"AttributeError in post_get_project_info_update: {e}")
def toggle_engine_enablement(self, enabled=False):
"""Toggle on/off all the render settings"""
indexes = [self.tabs.indexOf(self.project_group),
self.tabs.indexOf(self.output_settings_group),
self.tabs.indexOf(self.engine_group),
self.tabs.indexOf(self.cameras_group),
self.tabs.indexOf(self.notes_group)]
for idx in indexes:
self.tabs.setTabEnabled(idx, enabled)
self.submit_button.setEnabled(enabled)
def after_job_submission(self, error_string):
# UI cleanup
self.submit_progress.setMaximum(0)
self.submit_button.setHidden(False)
self.submit_progress.setHidden(True)
self.submit_progress_label.setHidden(True)
self.process_progress_bar.setHidden(True)
self.process_label.setHidden(True)
self.toggle_engine_enablement(True)
self.msg_box = QMessageBox()
if not error_string:
self.msg_box.setStandardButtons(QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No)
self.msg_box.setIcon(QMessageBox.Icon.Information)
self.msg_box.setText("Job successfully submitted to server. Submit another?")
self.msg_box.setWindowTitle("Success")
x = self.msg_box.exec()
if x == QMessageBox.StandardButton.No:
self.close()
else:
self.msg_box.setStandardButtons(QMessageBox.StandardButton.Ok)
self.msg_box.setIcon(QMessageBox.Icon.Critical)
self.msg_box.setText(error_string)
self.msg_box.setWindowTitle("Error")
self.msg_box.exec()
# -------- Submit Job Calls --------
def submit_job(self):
# Pre-worker UI
self.submit_progress.setHidden(False)
self.submit_progress_label.setHidden(False)
self.submit_button.setHidden(True)
self.submit_progress.setMaximum(0)
# submit job in background thread
self.worker_thread = SubmitWorker(window=self)
self.worker_thread.update_ui_signal.connect(self.update_submit_progress)
self.worker_thread.message_signal.connect(self.after_job_submission)
self.worker_thread.start()
@pyqtSlot(str, str)
def update_submit_progress(self, hostname, percent):
# Update the UI here. This slot will be executed in the main thread
self.submit_progress_label.setText(f"Transferring to {hostname} - {percent}%")
self.submit_progress.setMaximum(100)
self.submit_progress.setValue(int(percent))
class SubmitWorker(QThread):
"""Worker class called to submit all the jobs to the server and update the UI accordingly"""
message_signal = pyqtSignal(str)
update_ui_signal = pyqtSignal(str, str)
def __init__(self, window):
super().__init__()
self.window = window
def run(self):
def create_callback(encoder):
encoder_len = encoder.len
def callback(monitor):
percent = f"{monitor.bytes_read / encoder_len * 100:.0f}"
self.update_ui_signal.emit(hostname, percent)
return callback
try:
hostname = self.window.server_input.currentText()
resolution = (self.window.resolution_x_input.text(), self.window.resolution_y_input.text())
job_json = {'owner': psutil.Process().username() + '@' + socket.gethostname(),
'engine_name': self.window.engine_type.currentText().lower(),
'engine_version': self.window.engine_version_combo.currentText(),
'args': {'raw': self.window.raw_args.text(),
'export_format': self.window.file_format_combo.currentText(),
'resolution': resolution,
'fps': self.window.fps_input.text(),},
'output_path': self.window.job_name_input.text(),
'start_frame': self.window.start_frame_input.value(),
'end_frame': self.window.end_frame_input.value(),
'priority': self.window.priority_input.currentIndex() + 1,
'notes': self.window.notes_input.toPlainText(),
'enable_split_jobs': self.window.enable_splitjobs.isChecked(),
'split_jobs_same_os': self.window.splitjobs_same_os.isChecked(),
'name': self.window.job_name_input.text()}
# get the dynamic args
for i in range(self.window.engine_options_layout.count()):
item = self.window.engine_options_layout.itemAt(i)
layout = item.layout() # get the layout
for x in range(layout.count()):
z = layout.itemAt(x)
widget = z.widget()
if isinstance(widget, QComboBox):
job_json['args'][self.window.current_engine_options[i]['name']] = widget.currentText()
elif isinstance(widget, QLineEdit):
job_json['args'][self.window.current_engine_options[i]['name']] = widget.text()
# determine if any cameras are checked
selected_cameras = []
if self.window.cameras_list.count() and not self.window.cameras_group.isHidden():
for index in range(self.window.cameras_list.count()):
item = self.window.cameras_list.item(index)
if item.checkState() == Qt.CheckState.Checked:
selected_cameras.append(item.text().rsplit('-', 1)[0].strip()) # cleanup to just camera name
# process cameras into nested format
input_path = self.window.scene_file_input.text()
if selected_cameras and self.window.cameras_list.count() > 1:
children_jobs = []
for cam in selected_cameras:
child_job_data = dict()
child_job_data['args'] = {}
child_job_data['args']['camera'] = cam
child_job_data['name'] = job_json['name'].replace(' ', '-') + "_" + cam.replace(' ', '')
child_job_data['output_path'] = child_job_data['name']
children_jobs.append(child_job_data)
job_json['child_jobs'] = children_jobs
# presubmission tasks - use local installs
engine_class = EngineManager.engine_class_with_name(self.window.engine_type.currentText().lower())
latest_engine = EngineManager.get_latest_engine_instance(engine_class)
input_path = Path(latest_engine.perform_presubmission_tasks(input_path))
# submit
err_msg = ""
result = self.window.server_proxy.post_job_to_server(file_path=input_path, job_data=job_json,
callback=create_callback)
if not (result and result.ok):
err_msg = f"Error posting job to server: {result.text}"
self.message_signal.emit(err_msg)
except Exception as e:
self.message_signal.emit(str(e))
class GetProjectInfoWorker(QThread):
"""Worker class called to retrieve information about a project file on a background thread and update the UI"""
message_signal = pyqtSignal()
error_signal = pyqtSignal(str)
def __init__(self, window, project_path):
super().__init__()
self.window = window
self.project_path = project_path
def run(self):
try:
# get the engine info and add them all to the ui
self.window.installed_engines = self.window.server_proxy.get_installed_engines()
# select the best engine for the file type
self.window.preferred_engine = self.window.server_proxy.get_engine_for_filename(self.project_path)
# this should be the only time we use a local engine instead of using the proxy besides submitting
engine_class = EngineManager.engine_class_for_project_path(self.project_path)
engine = EngineManager.get_latest_engine_instance(engine_class)
self.window.project_info = engine.get_project_info(self.project_path)
self.message_signal.emit()
except Exception as e:
self.error_signal.emit(str(e))
def clear_layout(layout):
if layout is not None:
# Go through the layout's items in reverse order
for i in reversed(range(layout.count())):
# Take the item at the current position
item = layout.takeAt(i)
# Check if the item is a widget
if item.widget():
# Remove the widget and delete it
widget_to_remove = item.widget()
widget_to_remove.setParent(None)
widget_to_remove.deleteLater()
elif item.layout():
# If the item is a sub-layout, clear its contents recursively
clear_layout(item.layout())
# Then delete the layout
item.layout().deleteLater()
# Run the application
if __name__ == '__main__':
app = QApplication([])
window = NewRenderJobForm()
app.exec()
src/ui/console_window.py → src/ui/console.py
View File
src/ui/engine_browser.py
+6 -6
View File
@@ -93,7 +93,7 @@ class EngineBrowserWindow(QMainWindow):
def update_table(self):
def update_table_worker():
raw_server_data = RenderServerProxy(self.hostname).get_all_engine_info()
raw_server_data = RenderServerProxy(self.hostname).get_renderer_info()
if not raw_server_data:
return
@@ -128,18 +128,18 @@ class EngineBrowserWindow(QMainWindow):
self.launch_button.setEnabled(is_localhost(self.hostname))
def update_download_status(self):
running_tasks = EngineManager.active_downloads()
running_tasks = [x for x in EngineManager.download_tasks if x.is_alive()]
hide_progress = not bool(running_tasks)
self.progress_bar.setHidden(hide_progress)
self.progress_label.setHidden(hide_progress)
# Update the status labels
if len(running_tasks) == 0:
if len(EngineManager.download_tasks) == 0:
new_status = ""
elif len(running_tasks) == 1:
task = running_tasks[0]
elif len(EngineManager.download_tasks) == 1:
task = EngineManager.download_tasks[0]
new_status = f"Downloading {task.engine.capitalize()} {task.version}..."
else:
new_status = f"Downloading {len(running_tasks)} engines..."
new_status = f"Downloading {len(EngineManager.download_tasks)} engines..."
self.progress_label.setText(new_status)
def launch_button_click(self):
src/ui/engine_help_window.py → src/ui/engine_help_viewer.py
View File
src/ui/log_window.py → src/ui/log_viewer.py
View File
src/ui/main_window.py
+108 -214
View File
@@ -1,39 +1,35 @@
''' app/ui/main_window.py '''
import ast
import datetime
import io
import logging
import os
import subprocess
import sys
import threading
import time
from typing import List, Dict, Any, Optional
import PIL
import humanize
from PIL import Image
from PyQt6.QtCore import Qt, QByteArray, QBuffer, QIODevice, QThread, pyqtSignal
from PyQt6.QtCore import Qt, QByteArray, QBuffer, QIODevice, QThread
from PyQt6.QtGui import QPixmap, QImage, QFont, QIcon
from PyQt6.QtWidgets import QMainWindow, QWidget, QHBoxLayout, QListWidget, QTableWidget, QAbstractItemView, \
QTableWidgetItem, QLabel, QVBoxLayout, QHeaderView, QMessageBox, QGroupBox, QPushButton, QListWidgetItem, \
QFileDialog
from src.api.api_server import API_VERSION
from src.api.serverproxy_manager import ServerProxyManager
from src.render_queue import RenderQueue
from src.ui.add_job_window import NewRenderJobForm
from src.ui.console_window import ConsoleWindow
from src.ui.engine_browser import EngineBrowserWindow
from src.ui.log_window import LogViewer
from src.ui.widgets.menubar import MenuBar
from src.ui.widgets.proportional_image_label import ProportionalImageLabel
from src.ui.widgets.statusbar import StatusBar
from src.ui.widgets.toolbar import ToolBar
from src.utilities.misc_helper import get_time_elapsed, resources_dir, is_localhost
from src.utilities.misc_helper import launch_url
from src.utilities.status_utils import RenderStatus
from src.utilities.zeroconf_server import ZeroconfServer
from src.version import APP_NAME
from .add_job import NewRenderJobForm
from .console import ConsoleWindow
from .engine_browser import EngineBrowserWindow
from .log_viewer import LogViewer
from .widgets.menubar import MenuBar
from .widgets.proportional_image_label import ProportionalImageLabel
from .widgets.statusbar import StatusBar
from .widgets.toolbar import ToolBar
from src.api.serverproxy_manager import ServerProxyManager
from src.utilities.misc_helper import launch_url
logger = logging.getLogger()
@@ -53,12 +49,11 @@ class MainWindow(QMainWindow):
super().__init__()
# Load the queue
self.job_list_view: Optional[QTableWidget] = None
self.server_info_ram: Optional[str] = None
self.server_info_cpu: Optional[str] = None
self.server_info_os: Optional[str] = None
self.server_info_gpu: Optional[List[Dict[str, Any]]] = None
self.server_info_hostname: Optional[str] = None
self.job_list_view = None
self.server_info_ram = None
self.server_info_cpu = None
self.server_info_os = None
self.server_info_hostname = None
self.engine_browser_window = None
self.server_info_group = None
self.current_hostname = None
@@ -68,7 +63,7 @@ class MainWindow(QMainWindow):
self.buffer_handler = None
# Window-Settings
self.setWindowTitle(APP_NAME)
self.setWindowTitle("Zordon")
self.setGeometry(100, 100, 900, 800)
central_widget = QWidget(self)
self.setCentralWidget(central_widget)
@@ -92,18 +87,15 @@ class MainWindow(QMainWindow):
self.setup_ui(main_layout)
self.create_toolbars()
# Add Widgets to Window
# self.custom_menu_bar =
self.setMenuBar(MenuBar(self))
self.setStatusBar(StatusBar(self))
self.create_toolbars()
# start background update
self.found_servers = []
self.job_data = {}
self.bg_update_thread = BackgroundUpdater(window=self)
self.bg_update_thread.updated_signal.connect(self.update_ui_data)
self.bg_update_thread = QThread()
self.bg_update_thread.run = self.__background_update
self.bg_update_thread.start()
# Setup other windows
@@ -114,12 +106,7 @@ class MainWindow(QMainWindow):
# Pick default job
self.job_picked()
def setup_ui(self, main_layout: QVBoxLayout) -> None:
"""Setup the main user interface layout.
Args:
main_layout: The main layout container for the UI widgets.
"""
def setup_ui(self, main_layout):
# Servers
server_list_group = QGroupBox("Available Servers")
@@ -134,7 +121,6 @@ class MainWindow(QMainWindow):
self.server_info_os = QLabel()
self.server_info_cpu = QLabel()
self.server_info_ram = QLabel()
self.server_info_gpu = QLabel()
server_info_engines_button = QPushButton("Render Engines")
server_info_engines_button.clicked.connect(self.engine_browser)
server_info_layout = QVBoxLayout()
@@ -142,7 +128,6 @@ class MainWindow(QMainWindow):
server_info_layout.addWidget(self.server_info_os)
server_info_layout.addWidget(self.server_info_cpu)
server_info_layout.addWidget(self.server_info_ram)
server_info_layout.addWidget(self.server_info_gpu)
server_info_layout.addWidget(server_info_engines_button)
server_info_group.setLayout(server_info_layout)
@@ -169,41 +154,45 @@ class MainWindow(QMainWindow):
self.job_list_view.verticalHeader().setVisible(False)
self.job_list_view.itemSelectionChanged.connect(self.job_picked)
self.job_list_view.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers)
self.refresh_job_headers()
# Setup Job Headers
self.job_list_view.setHorizontalHeaderLabels(["ID", "Name", "Engine", "Priority", "Status",
"Time Elapsed", "Frames", "Date Created"])
self.job_list_view.setColumnHidden(0, True)
# Image Layout
image_group = QGroupBox("Job Preview")
image_layout = QVBoxLayout(image_group)
image_layout.setContentsMargins(0, 0, 0, 0)
image_center_layout = QHBoxLayout()
image_center_layout.addWidget(self.image_label)
image_layout.addWidget(self.image_label)
# image_layout.addLayout(image_center_layout)
self.job_list_view.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch)
self.job_list_view.horizontalHeader().setSectionResizeMode(2, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(3, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(4, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(5, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(6, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(7, QHeaderView.ResizeMode.ResizeToContents)
# Job List Layout
job_list_group = QGroupBox("Job Preview")
# Job Layout
job_list_group = QGroupBox("Render Jobs")
job_list_layout = QVBoxLayout(job_list_group)
job_list_layout.setContentsMargins(0, 0, 0, 0)
job_list_layout.addWidget(self.image_label)
job_list_layout.addWidget(self.job_list_view, stretch=True)
image_layout.addWidget(self.job_list_view, stretch=True)
image_layout.addLayout(job_list_layout)
# Add them all to the window
main_layout.addLayout(info_layout)
right_layout = QVBoxLayout()
right_layout.setContentsMargins(0, 0, 0, 0)
right_layout.addWidget(job_list_group)
right_layout.addWidget(image_group)
# right_layout.addWidget(job_list_group)
main_layout.addLayout(right_layout)
def closeEvent(self, event):
"""Handle window close event with job running confirmation.
def __background_update(self):
while True:
try:
self.update_servers()
self.fetch_jobs()
except RuntimeError:
pass
except Exception as e:
logger.error(f"Uncaught exception in background update: {e}")
time.sleep(0.5)
Args:
event: The close event triggered by user.
"""
def closeEvent(self, event):
running_jobs = len(RenderQueue.running_jobs())
if running_jobs:
reply = QMessageBox.question(self, "Running Jobs",
@@ -216,12 +205,7 @@ class MainWindow(QMainWindow):
else:
event.ignore()
# -- Server Code -- #
def refresh_job_list(self):
"""Refresh the job list display."""
self.job_list_view.clearContents()
self.bg_update_thread.needs_update = True
# -- Server Code -- #
@property
def current_server_proxy(self):
@@ -238,7 +222,7 @@ class MainWindow(QMainWindow):
# Update the current hostname and clear the job list
self.current_hostname = new_hostname
self.job_list_view.setRowCount(0)
self.refresh_job_list()
self.fetch_jobs(clear_table=True)
# Select the first row if there are jobs listed
if self.job_list_view.rowCount():
@@ -247,61 +231,37 @@ class MainWindow(QMainWindow):
# Update server information display
self.update_server_info_display(new_hostname)
except AttributeError as e:
logger.error(f"AttributeError in server_picked: {e}")
except AttributeError:
# Handle cases where the server list view might not be properly initialized
pass
def update_server_info_display(self, hostname):
"""Updates the server information section of the UI."""
self.server_info_hostname.setText(f"Name: {hostname}")
self.server_info_hostname.setText(hostname or "unknown")
server_info = ZeroconfServer.get_hostname_properties(hostname)
# Use the get method with defaults to avoid KeyError
os_info = f"OS: {server_info.get('system_os', 'Unknown')} {server_info.get('system_os_version', '')}"
cleaned_cpu_name = server_info.get('system_cpu_brand', 'Unknown').replace(' CPU','').replace('(TM)','').replace('(R)', '')
cpu_info = f"CPU: {cleaned_cpu_name} ({server_info.get('system_cpu_cores', 'Unknown')} cores)"
memory_info = f"RAM: {server_info.get('system_memory', 'Unknown')} GB"
# Get and format GPU info
try:
gpu_list = ast.literal_eval(server_info.get('gpu_info', []))
# Format all GPUs
gpu_info_parts = []
for gpu in gpu_list:
gpu_name = gpu.get('name', 'Unknown').replace('(TM)','').replace('(R)', '')
gpu_memory = gpu.get('memory', 'Unknown')
# Add " GB" suffix if memory is a number
if isinstance(gpu_memory, (int, float)) or (isinstance(gpu_memory, str) and gpu_memory.isdigit()):
gpu_memory_str = f"{gpu_memory} GB"
else:
gpu_memory_str = str(gpu_memory)
gpu_info_parts.append(f"{gpu_name} ({gpu_memory_str})")
gpu_info = f"GPU: {', '.join(gpu_info_parts)}" if gpu_info_parts else "GPU: Unknown"
except Exception as e:
logger.error(f"Error parsing GPU info: {e}")
gpu_info = "GPU: Unknown"
cpu_info = f"CPU: {server_info.get('system_cpu', 'Unknown')} - {server_info.get('system_cpu_cores', 'Unknown')} cores"
self.server_info_os.setText(os_info.strip())
self.server_info_cpu.setText(cpu_info)
self.server_info_ram.setText(memory_info)
self.server_info_gpu.setText(gpu_info)
def update_ui_data(self):
"""Update UI data with current server and job information."""
self.update_servers()
def fetch_jobs(self, clear_table=False):
if not self.current_server_proxy:
return
server_job_data = self.job_data.get(self.current_server_proxy.hostname)
if server_job_data:
num_jobs = len(server_job_data)
if clear_table:
self.job_list_view.clear()
self.refresh_job_headers()
job_fetch = self.current_server_proxy.get_all_jobs(ignore_token=clear_table)
if job_fetch:
num_jobs = len(job_fetch)
self.job_list_view.setRowCount(num_jobs)
for row, job in enumerate(server_job_data):
for row, job in enumerate(job_fetch):
display_status = job['status'] if job['status'] != RenderStatus.RUNNING.value else \
('%.0f%%' % (job['percent_complete'] * 100)) # if running, show percent, otherwise just show status
@@ -313,20 +273,18 @@ class MainWindow(QMainWindow):
get_time_elapsed(start_time, end_time)
name = job.get('name') or os.path.basename(job.get('input_path', ''))
engine_name = f"{job.get('engine', '')}-{job.get('engine_version')}"
renderer = f"{job.get('renderer', '')}-{job.get('renderer_version')}"
priority = str(job.get('priority', ''))
total_frames = str(job.get('total_frames', ''))
converted_time = datetime.datetime.fromisoformat(job['date_created'])
humanized_time = humanize.naturaltime(converted_time)
items = [QTableWidgetItem(job['id']), QTableWidgetItem(name), QTableWidgetItem(engine_name),
items = [QTableWidgetItem(job['id']), QTableWidgetItem(name), QTableWidgetItem(renderer),
QTableWidgetItem(priority), QTableWidgetItem(display_status), QTableWidgetItem(time_elapsed),
QTableWidgetItem(total_frames), QTableWidgetItem(humanized_time)]
QTableWidgetItem(total_frames), QTableWidgetItem(job['date_created'])]
for col, item in enumerate(items):
self.job_list_view.setItem(row, col, item)
# -- Job Code -- #
# -- Job Code -- #
def job_picked(self):
def fetch_preview(job_id):
@@ -367,7 +325,7 @@ class MainWindow(QMainWindow):
current_status = self.job_list_view.item(selected_row.row(), 4).text()
# show / hide the stop button
show_stop_button = "%" in current_status
show_stop_button = current_status.lower() == 'running'
self.topbar.actions_call['Stop Job'].setEnabled(show_stop_button)
self.topbar.actions_call['Stop Job'].setVisible(show_stop_button)
self.topbar.actions_call['Delete Job'].setEnabled(not show_stop_button)
@@ -392,11 +350,6 @@ class MainWindow(QMainWindow):
self.topbar.actions_call['Open Files'].setVisible(False)
def selected_job_ids(self):
"""Get list of selected job IDs from the job list.
Returns:
List[str]: List of selected job ID strings.
"""
try:
selected_rows = self.job_list_view.selectionModel().selectedRows()
job_ids = []
@@ -404,20 +357,26 @@ class MainWindow(QMainWindow):
id_item = self.job_list_view.item(selected_row.row(), 0)
job_ids.append(id_item.text())
return job_ids
except AttributeError as e:
logger.error(f"AttributeError in selected_job_ids: {e}")
except AttributeError:
return []
def refresh_job_headers(self):
self.job_list_view.setHorizontalHeaderLabels(["ID", "Name", "Renderer", "Priority", "Status",
"Time Elapsed", "Frames", "Date Created"])
self.job_list_view.setColumnHidden(0, True)
# -- Image Code -- #
self.job_list_view.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch)
self.job_list_view.horizontalHeader().setSectionResizeMode(2, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(3, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(4, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(5, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(6, QHeaderView.ResizeMode.ResizeToContents)
self.job_list_view.horizontalHeader().setSectionResizeMode(7, QHeaderView.ResizeMode.ResizeToContents)
# -- Image Code -- #
def load_image_path(self, image_path):
"""Load and display an image from file path.
Args:
image_path: Path to the image file to load.
"""
# Load and set image using QPixmap
# Load and set the image using QPixmap
try:
pixmap = QPixmap(image_path)
if not pixmap:
@@ -450,25 +409,26 @@ class MainWindow(QMainWindow):
logger.error(f"Error loading image data: {e}")
def update_servers(self):
found_servers = list(set(ZeroconfServer.found_hostnames() + self.added_hostnames))
# Always make sure local hostname is first
if self.found_servers and not is_localhost(self.found_servers[0]):
for hostname in self.found_servers:
if found_servers and not is_localhost(found_servers[0]):
for hostname in found_servers:
if is_localhost(hostname):
self.found_servers.remove(hostname)
self.found_servers.insert(0, hostname)
found_servers.remove(hostname)
found_servers.insert(0, hostname)
break
old_count = self.server_list_view.count()
# Update proxys
for hostname in self.found_servers:
for hostname in found_servers:
ServerProxyManager.get_proxy_for_hostname(hostname) # setup background updates
# Add in all the missing servers
current_server_list = []
for i in range(self.server_list_view.count()):
current_server_list.append(self.server_list_view.item(i).text())
for hostname in self.found_servers:
for hostname in found_servers:
if hostname not in current_server_list:
properties = ZeroconfServer.get_hostname_properties(hostname)
image_path = os.path.join(resources_dir(), f"{properties.get('system_os', 'Monitor')}.png")
@@ -479,7 +439,7 @@ class MainWindow(QMainWindow):
servers_to_remove = []
for i in range(self.server_list_view.count()):
name = self.server_list_view.item(i).text()
if name not in self.found_servers:
if name not in found_servers:
servers_to_remove.append(name)
# remove any servers that shouldn't be shown any longer
@@ -507,10 +467,10 @@ class MainWindow(QMainWindow):
resources_directory = resources_dir()
# Top Toolbar Buttons
self.topbar.add_button(
"Settings", f"{resources_directory}/Gear.png", self.menuBar().show_settings)
self.topbar.add_button(
"Console", f"{resources_directory}/Console.png", self.open_console_window)
self.topbar.add_button(
"Engines", f"{resources_directory}/SoftwareInstaller.png", self.engine_browser)
self.topbar.add_separator()
self.topbar.add_button(
"Stop Job", f"{resources_directory}/StopSign.png", self.stop_job)
@@ -526,7 +486,7 @@ class MainWindow(QMainWindow):
"New Job", f"{resources_directory}/AddProduct.png", self.new_job)
self.addToolBar(Qt.ToolBarArea.TopToolBarArea, self.topbar)
# -- Toolbar Buttons -- #
# -- Toolbar Buttons -- #
def open_console_window(self) -> None:
"""
@@ -541,9 +501,8 @@ class MainWindow(QMainWindow):
self.engine_browser_window.show()
def job_logs(self) -> None:
"""Open log viewer for selected job.
Opens a log viewer window showing the logs for the currently selected job.
"""
Event handler for the "Logs" button.
"""
selected_job_ids = self.selected_job_ids()
if selected_job_ids:
@@ -552,10 +511,8 @@ class MainWindow(QMainWindow):
self.log_viewer_window.show()
def stop_job(self, event):
"""Stop selected render jobs with user confirmation.
Args:
event: The button click event.
"""
Event handler for the "Exit" button. Closes the application.
"""
job_ids = self.selected_job_ids()
if not job_ids:
@@ -565,26 +522,24 @@ class MainWindow(QMainWindow):
job = next((job for job in self.current_server_proxy.get_all_jobs() if job.get('id') == job_ids[0]), None)
if job:
display_name = job.get('name', os.path.basename(job.get('input_path', '')))
message = f"Are you sure you want to stop job: {display_name}?"
message = f"Are you sure you want to delete the job:\n{display_name}?"
else:
return # Job not found, handle this case as needed
else:
message = f"Are you sure you want to stop these {len(job_ids)} jobs?"
message = f"Are you sure you want to delete these {len(job_ids)} jobs?"
# Display the message box and check the response in one go
msg_box = QMessageBox(QMessageBox.Icon.Warning, "Stop Job", message,
msg_box = QMessageBox(QMessageBox.Icon.Warning, "Delete Job", message,
QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, self)
if msg_box.exec() == QMessageBox.StandardButton.Yes:
for job_id in job_ids:
self.current_server_proxy.cancel_job(job_id, confirm=True)
self.refresh_job_list()
self.fetch_jobs(clear_table=True)
def delete_job(self, event):
"""Delete selected render jobs with user confirmation.
Args:
event: The button click event.
"""
Event handler for the Delete Job button
"""
job_ids = self.selected_job_ids()
if not job_ids:
@@ -607,18 +562,10 @@ class MainWindow(QMainWindow):
if msg_box.exec() == QMessageBox.StandardButton.Yes:
for job_id in job_ids:
self.current_server_proxy.delete_job(job_id, confirm=True)
self.refresh_job_list()
self.fetch_jobs(clear_table=True)
def download_files(self, event):
job_ids = self.selected_job_ids()
if not job_ids:
return
import webbrowser
download_url = (f"http://{self.current_server_proxy.hostname}:{self.current_server_proxy.port}"
f"/api/job/{job_ids[0]}/download_all")
webbrowser.open(download_url)
pass
def open_files(self, event):
job_ids = self.selected_job_ids()
@@ -636,56 +583,3 @@ class MainWindow(QMainWindow):
if file_name:
self.new_job_window = NewRenderJobForm(file_name)
self.new_job_window.show()
class BackgroundUpdater(QThread):
"""Worker class to fetch job and server information and update the UI"""
updated_signal = pyqtSignal()
error_signal = pyqtSignal(str)
def __init__(self, window):
super().__init__()
self.window = window
self.needs_update = True
def run(self):
"""Main background thread execution loop.
Continuously fetches server and job data, updating the main UI
every second or when updates are needed.
"""
try:
last_run = 0
while True:
now = time.monotonic()
if now - last_run >= 1.0 or self.needs_update:
self.window.found_servers = list(set(ZeroconfServer.found_hostnames() + self.window.added_hostnames))
self.window.found_servers = [x for x in self.window.found_servers if
ZeroconfServer.get_hostname_properties(x)['api_version'] == API_VERSION]
if self.window.current_server_proxy:
self.window.job_data[self.window.current_server_proxy.hostname] = \
self.window.current_server_proxy.get_all_jobs(ignore_token=False)
self.needs_update = False
self.updated_signal.emit()
time.sleep(0.05)
except Exception as e:
print(f"ERROR: {e}")
self.error_signal.emit(str(e))
if __name__ == "__main__":
# lazy load GUI frameworks
from PyQt6.QtWidgets import QApplication
# load application
# QtCore.QCoreApplication.setAttribute(QtCore.Qt.ApplicationAttribute.AA_MacDontSwapCtrlAndMeta)
app: QApplication = QApplication(sys.argv)
# configure main main_window
main_window = MainWindow()
# main_window.buffer_handler = buffer_handler
app.setActiveWindow(main_window)
main_window.show()
sys.exit(app.exec())
src/ui/settings_window.py
-552
View File
@@ -1,552 +0,0 @@
import os
import socket
from datetime import datetime
from pathlib import Path
import humanize
from PyQt6 import QtCore
from PyQt6.QtCore import Qt, QSettings, pyqtSignal as Signal, QThread, pyqtSignal, QTimer
from PyQt6.QtGui import QIcon
from PyQt6.QtWidgets import QApplication, QMainWindow, QListWidget, QListWidgetItem, QStackedWidget, QVBoxLayout, \
QWidget, QLabel, QCheckBox, QLineEdit, \
QPushButton, QHBoxLayout, QGroupBox, QTableWidget, QAbstractItemView, QTableWidgetItem, QHeaderView, \
QMessageBox, QProgressBar
from src.api.server_proxy import RenderServerProxy
from src.engines.engine_manager import EngineManager
from src.utilities.config import Config
from src.utilities.misc_helper import launch_url
from src.version import APP_AUTHOR, APP_NAME
settings = QSettings(APP_AUTHOR, APP_NAME)
class GetEngineInfoWorker(QThread):
"""
The GetEngineInfoWorker class fetches engine information from a server in a background thread.
Attributes:
done: A signal emitted when the engine information is retrieved.
Methods:
run(self): Fetches engine information from the server.
"""
done = pyqtSignal(object) # emits the result when finished
def __init__(self, parent=None):
super().__init__(parent)
self.parent = parent
def run(self):
data = RenderServerProxy(socket.gethostname()).get_all_engine_info()
self.done.emit(data)
class SettingsWindow(QMainWindow):
"""
The SettingsWindow class provides a user interface for managing engine settings.
"""
def __init__(self):
super().__init__()
self.engine_download_progress_bar = None
self.engines_last_update_label = None
self.check_for_engine_updates_checkbox = None
self.delete_engine_button = None
self.launch_engine_button = None
self.show_password_button = None
self.network_password_line = None
self.enable_network_password_checkbox = None
self.check_for_new_engines_button = None
if not EngineManager.engines_path: # fix issue where sometimes path was not set
EngineManager.engines_path = Path(Config.upload_folder).expanduser() / "engines"
self.installed_engines_table = None
self.setWindowTitle("Settings")
# Create the main layout
main_layout = QVBoxLayout()
# Create the sidebar (QListWidget) for navigation
self.sidebar = QListWidget()
self.sidebar.setFixedWidth(150)
# Set the icon size
self.sidebar.setIconSize(QtCore.QSize(32, 32)) # Increase the icon size to 32x32 pixels
# Adjust the font size for the sidebar items
font = self.sidebar.font()
font.setPointSize(12) # Increase the font size
self.sidebar.setFont(font)
# Add items with icons to the sidebar
resources_dir = os.path.join(Path(__file__).resolve().parent.parent.parent, 'resources')
self.add_sidebar_item("General", os.path.join(resources_dir, "Gear.png"))
self.add_sidebar_item("Server", os.path.join(resources_dir, "Server.png"))
self.add_sidebar_item("Engines", os.path.join(resources_dir, "Blender.png"))
self.sidebar.setCurrentRow(0)
# Create the stacked widget to hold different settings pages
self.stacked_widget = QStackedWidget()
# Create pages for each section
general_page = self.create_general_page()
network_page = self.create_network_page()
engines_page = self.create_engines_page()
# Add pages to the stacked widget
self.stacked_widget.addWidget(general_page)
self.stacked_widget.addWidget(network_page)
self.stacked_widget.addWidget(engines_page)
# Connect the sidebar to the stacked widget
self.sidebar.currentRowChanged.connect(self.stacked_widget.setCurrentIndex)
# Create a horizontal layout to hold the sidebar and stacked widget
content_layout = QHBoxLayout()
content_layout.addWidget(self.sidebar)
content_layout.addWidget(self.stacked_widget)
# Add the content layout to the main layout
main_layout.addLayout(content_layout)
# Add the "OK" button at the bottom
ok_button = QPushButton("OK")
ok_button.clicked.connect(self.close)
ok_button.setFixedWidth(80)
ok_button.setDefault(True)
main_layout.addWidget(ok_button, alignment=Qt.AlignmentFlag.AlignRight)
# Create a central widget and set the layout
central_widget = QWidget()
central_widget.setLayout(main_layout)
self.setCentralWidget(central_widget)
self.setMinimumSize(700, 400)
# timers for background download UI updates
self.timer = QTimer(self)
self.timer.timeout.connect(self.update_engine_download_status)
def add_sidebar_item(self, name, icon_path):
"""Add an item with an icon to the sidebar."""
item = QListWidgetItem(QIcon(icon_path), name)
self.sidebar.addItem(item)
def create_general_page(self):
"""Create the General settings page."""
page = QWidget()
layout = QVBoxLayout()
# Startup Settings Group
startup_group = QGroupBox("Startup Settings")
startup_layout = QVBoxLayout()
# startup_layout.addWidget(QCheckBox("Start application on system startup"))
check_for_updates_checkbox = QCheckBox("Check for updates automatically")
check_for_updates_checkbox.setChecked(settings.value("auto_check_for_updates", True, type=bool))
check_for_updates_checkbox.stateChanged.connect(lambda state: settings.setValue("auto_check_for_updates", bool(state)))
startup_layout.addWidget(check_for_updates_checkbox)
startup_group.setLayout(startup_layout)
# Local Files Group
data_path = Path(Config.upload_folder).expanduser()
path_size = sum(f.stat().st_size for f in Path(data_path).rglob('*') if f.is_file())
database_group = QGroupBox("Local Files")
database_layout = QVBoxLayout()
database_layout.addWidget(QLabel(f"Local Directory: {data_path}"))
database_layout.addWidget(QLabel(f"Size: {humanize.naturalsize(path_size, binary=True)}"))
open_database_path_button = QPushButton("Open Directory")
open_database_path_button.clicked.connect(lambda: launch_url(data_path))
open_database_path_button.setFixedWidth(200)
database_layout.addWidget(open_database_path_button)
database_group.setLayout(database_layout)
# Render Settings Group
render_settings_group = QGroupBox("Render Engine Settings")
render_settings_layout = QVBoxLayout()
render_settings_layout.addWidget(QLabel("Restrict to render nodes with same:"))
require_same_engine_checkbox = QCheckBox("Renderer Version")
require_same_engine_checkbox.setChecked(settings.value("render_require_same_engine_version", False, type=bool))
require_same_engine_checkbox.stateChanged.connect(lambda state: settings.setValue("render_require_same_engine_version", bool(state)))
render_settings_layout.addWidget(require_same_engine_checkbox)
require_same_cpu_checkbox = QCheckBox("CPU Architecture")
require_same_cpu_checkbox.setChecked(settings.value("render_require_same_cpu_type", False, type=bool))
require_same_cpu_checkbox.stateChanged.connect(lambda state: settings.setValue("render_require_same_cpu_type", bool(state)))
render_settings_layout.addWidget(require_same_cpu_checkbox)
require_same_os_checkbox = QCheckBox("Operating System")
require_same_os_checkbox.setChecked(settings.value("render_require_same_os", False, type=bool))
require_same_os_checkbox.stateChanged.connect(lambda state: settings.setValue("render_require_same_os", bool(state)))
render_settings_layout.addWidget(require_same_os_checkbox)
render_settings_group.setLayout(render_settings_layout)
layout.addWidget(startup_group)
layout.addWidget(database_group)
layout.addWidget(render_settings_group)
layout.addStretch() # Add a stretch to push content to the top
page.setLayout(layout)
return page
def create_network_page(self):
"""Create the Network settings page."""
page = QWidget()
layout = QVBoxLayout()
# Sharing Settings Group
sharing_group = QGroupBox("Sharing Settings")
sharing_layout = QVBoxLayout()
enable_sharing_checkbox = QCheckBox("Enable other computers on the network to render to this machine")
enable_sharing_checkbox.setChecked(settings.value("enable_network_sharing", False, type=bool))
enable_sharing_checkbox.stateChanged.connect(self.toggle_render_sharing)
sharing_layout.addWidget(enable_sharing_checkbox)
password_enabled = (settings.value("enable_network_sharing", False, type=bool) and
settings.value("enable_network_password", False, type=bool))
password_layout = QHBoxLayout()
password_layout.setContentsMargins(0, 0, 0, 0)
self.enable_network_password_checkbox = QCheckBox("Enable network password:")
self.enable_network_password_checkbox.setChecked(settings.value("enable_network_password", False, type=bool))
self.enable_network_password_checkbox.stateChanged.connect(self.enable_network_password_changed)
self.enable_network_password_checkbox.setEnabled(settings.value("enable_network_sharing", False, type=bool))
sharing_layout.addWidget(self.enable_network_password_checkbox)
self.network_password_line = QLineEdit()
self.network_password_line.setPlaceholderText("Enter a password")
self.network_password_line.setEchoMode(QLineEdit.EchoMode.Password)
self.network_password_line.setEnabled(password_enabled)
password_layout.addWidget(self.network_password_line)
self.show_password_button = QPushButton("Show")
self.show_password_button.setEnabled(password_enabled)
self.show_password_button.clicked.connect(self.show_password_button_pressed)
password_layout.addWidget(self.show_password_button)
sharing_layout.addLayout(password_layout)
sharing_group.setLayout(sharing_layout)
layout.addWidget(sharing_group)
layout.addStretch() # Add a stretch to push content to the top
page.setLayout(layout)
return page
def toggle_render_sharing(self, enable_sharing):
settings.setValue("enable_network_sharing", enable_sharing)
self.enable_network_password_checkbox.setEnabled(enable_sharing)
enable_password = enable_sharing and settings.value("enable_network_password", False, type=bool)
self.network_password_line.setEnabled(enable_password)
self.show_password_button.setEnabled(enable_password)
def enable_network_password_changed(self, new_value):
settings.setValue("enable_network_password", new_value)
self.network_password_line.setEnabled(new_value)
self.show_password_button.setEnabled(new_value)
def show_password_button_pressed(self):
# toggle showing / hiding the password
show_pass = self.show_password_button.text() == "Show"
self.show_password_button.setText("Hide" if show_pass else "Show")
self.network_password_line.setEchoMode(QLineEdit.EchoMode.Normal if show_pass else QLineEdit.EchoMode.Password)
def create_engines_page(self):
"""Create the Engines settings page."""
page = QWidget()
layout = QVBoxLayout()
# Installed Engines Group
installed_group = QGroupBox("Installed Engines")
installed_layout = QVBoxLayout()
# Setup table
self.installed_engines_table = EngineTableWidget()
self.installed_engines_table.row_selected.connect(self.engine_table_selected)
installed_layout.addWidget(self.installed_engines_table)
# Ignore system installs
engine_ignore_system_installs_checkbox = QCheckBox("Ignore system installs")
engine_ignore_system_installs_checkbox.setChecked(settings.value("engines_ignore_system_installs", False, type=bool))
engine_ignore_system_installs_checkbox.stateChanged.connect(self.change_ignore_system_installs)
installed_layout.addWidget(engine_ignore_system_installs_checkbox)
# Engine Launch / Delete buttons
installed_buttons_layout = QHBoxLayout()
self.launch_engine_button = QPushButton("Launch")
self.launch_engine_button.setEnabled(False)
self.launch_engine_button.clicked.connect(self.launch_selected_engine)
self.delete_engine_button = QPushButton("Delete")
self.delete_engine_button.setEnabled(False)
self.delete_engine_button.clicked.connect(self.delete_selected_engine)
installed_buttons_layout.addWidget(self.launch_engine_button)
installed_buttons_layout.addWidget(self.delete_engine_button)
installed_layout.addLayout(installed_buttons_layout)
installed_group.setLayout(installed_layout)
# Engine Updates Group
engine_updates_group = QGroupBox("Auto-Install")
engine_updates_layout = QVBoxLayout()
engine_download_layout = QHBoxLayout()
engine_download_layout.addWidget(QLabel("Enable Downloads for:"))
at_least_one_downloadable = False
for engine in EngineManager.downloadable_engines():
engine_download_check = QCheckBox(engine.name())
is_checked = settings.value(f"engine_download-{engine.name()}", False, type=bool)
at_least_one_downloadable |= is_checked
engine_download_check.setChecked(is_checked)
# Capture the checkbox correctly using a default argument in lambda
engine_download_check.clicked.connect(
lambda state, checkbox=engine_download_check: self.engine_download_settings_changed(state, checkbox.text())
)
engine_download_layout.addWidget(engine_download_check)
engine_updates_layout.addLayout(engine_download_layout)
self.check_for_engine_updates_checkbox = QCheckBox("Check for new versions on launch")
self.check_for_engine_updates_checkbox.setChecked(settings.value('check_for_engine_updates_on_launch', True, type=bool))
self.check_for_engine_updates_checkbox.setEnabled(at_least_one_downloadable)
self.check_for_engine_updates_checkbox.stateChanged.connect(
lambda state: settings.setValue("check_for_engine_updates_on_launch", bool(state)))
engine_updates_layout.addWidget(self.check_for_engine_updates_checkbox)
self.engines_last_update_label = QLabel()
self.update_last_checked_label()
self.engines_last_update_label.setEnabled(at_least_one_downloadable)
engine_updates_layout.addWidget(self.engines_last_update_label)
self.engine_download_progress_bar = QProgressBar()
engine_updates_layout.addWidget(self.engine_download_progress_bar)
self.engine_download_progress_bar.setHidden(True)
self.check_for_new_engines_button = QPushButton("Check for New Versions...")
self.check_for_new_engines_button.setEnabled(at_least_one_downloadable)
self.check_for_new_engines_button.clicked.connect(self.check_for_new_engines)
engine_updates_layout.addWidget(self.check_for_new_engines_button)
engine_updates_group.setLayout(engine_updates_layout)
layout.addWidget(installed_group)
layout.addWidget(engine_updates_group)
layout.addStretch() # Add a stretch to push content to the top
page.setLayout(layout)
return page
def change_ignore_system_installs(self, value):
settings.setValue("engines_ignore_system_installs", bool(value))
self.installed_engines_table.update_engines_table()
def update_last_checked_label(self):
"""Retrieve the last check timestamp and return a human-friendly string."""
last_checked_str = settings.value("engines_last_update_time", None)
if not last_checked_str:
time_string = "Never"
else:
last_checked_dt = datetime.fromisoformat(last_checked_str)
now = datetime.now()
time_string = humanize.naturaltime(now - last_checked_dt)
self.engines_last_update_label.setText(f"Last Updated: {time_string}")
def engine_download_settings_changed(self, state, engine_name):
settings.setValue(f"engine_download-{engine_name}", state)
at_least_one_downloadable = False
for engine in EngineManager.downloadable_engines():
at_least_one_downloadable |= settings.value(f"engine_download-{engine.name()}", False, type=bool)
self.check_for_new_engines_button.setEnabled(at_least_one_downloadable)
self.check_for_engine_updates_checkbox.setEnabled(at_least_one_downloadable)
self.engines_last_update_label.setEnabled(at_least_one_downloadable)
def delete_selected_engine(self):
engine_info = self.installed_engines_table.selected_engine_data()
reply = QMessageBox.question(self, f"Delete {engine_info['engine']} {engine_info['version']}?",
f"Do you want to delete {engine_info['engine']} {engine_info['version']}?",
QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No)
if reply is not QMessageBox.StandardButton.Yes:
return
delete_result = EngineManager.delete_engine_download(engine_info.get('engine'),
engine_info.get('version'),
engine_info.get('system_os'),
engine_info.get('cpu'))
self.installed_engines_table.update_engines_table(use_cached=False)
if delete_result:
QMessageBox.information(self, f"{engine_info['engine']} {engine_info['version']} Deleted",
f"{engine_info['engine']} {engine_info['version']} deleted successfully",
QMessageBox.StandardButton.Ok)
else:
QMessageBox.warning(self, f"Unknown Error",
f"Unknown error while deleting {engine_info['engine']} {engine_info['version']}.",
QMessageBox.StandardButton.Ok)
def launch_selected_engine(self):
engine_info = self.installed_engines_table.selected_engine_data()
if engine_info:
launch_url(engine_info['path'])
def engine_table_selected(self):
engine_data = self.installed_engines_table.selected_engine_data()
if engine_data:
self.launch_engine_button.setEnabled(bool(engine_data.get('path') or True))
self.delete_engine_button.setEnabled(engine_data.get('type') == 'managed')
else:
self.launch_engine_button.setEnabled(False)
self.delete_engine_button.setEnabled(False)
def check_for_new_engines(self):
ignore_system = settings.value("engines_ignore_system_installs", False, type=bool)
messagebox_shown = False
for engine in EngineManager.downloadable_engines():
if settings.value(f'engine_download-{engine.name()}', False, type=bool):
result = EngineManager.is_engine_update_available(engine, ignore_system_installs=ignore_system)
if result:
result['name'] = engine.name()
msg_box = QMessageBox()
msg_box.setWindowTitle(f"{result['name']} ({result['version']}) Available")
msg_box.setText(f"A new version of {result['name']} is available ({result['version']}).\n\n"
f"Would you like to download it now?")
msg_box.setIcon(QMessageBox.Icon.Question)
msg_box.setStandardButtons(QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No)
msg_result = msg_box.exec()
messagebox_shown = True
if msg_result == QMessageBox.StandardButton.Yes:
EngineManager.download_engine(engine_name=engine.name(), version=result['version'], background=True,
ignore_system=ignore_system)
self.engine_download_progress_bar.setHidden(False)
self.engine_download_progress_bar.setValue(0)
self.engine_download_progress_bar.setMaximum(100)
self.check_for_new_engines_button.setEnabled(False)
self.timer.start(1000)
if not messagebox_shown:
msg_box = QMessageBox()
msg_box.setWindowTitle("No Updates Available")
msg_box.setText("No Updates Available.")
msg_box.setIcon(QMessageBox.Icon.Information)
msg_box.setStandardButtons(QMessageBox.StandardButton.Ok)
msg_box.exec()
settings.setValue("engines_last_update_time", datetime.now().isoformat())
self.update_engine_download_status()
def update_engine_download_status(self):
running_tasks = EngineManager.active_downloads()
if not running_tasks:
self.timer.stop()
self.engine_download_progress_bar.setHidden(True)
self.installed_engines_table.update_engines_table(use_cached=False)
self.update_last_checked_label()
self.check_for_new_engines_button.setEnabled(True)
return
percent_complete = int(running_tasks[0].percent_complete * 100)
self.engine_download_progress_bar.setValue(percent_complete)
if percent_complete == 100:
status_update = f"Installing {running_tasks[0].engine.capitalize()} {running_tasks[0].version}..."
else:
status_update = f"Downloading {running_tasks[0].engine.capitalize()} {running_tasks[0].version}..."
self.engines_last_update_label.setText(status_update)
class EngineTableWidget(QWidget):
"""
The EngineTableWidget class displays a table of installed engines.
Attributes:
table: A table widget displaying engine information.
Methods:
on_selection_changed(self): Emits a signal when the user selects a different row in the table.
"""
row_selected = Signal()
def __init__(self):
super().__init__()
self.__get_engine_info_worker = None
self.table = QTableWidget(0, 4)
self.table.setHorizontalHeaderLabels(["Engine", "Version", "Type", "Path"])
self.table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows)
self.table.verticalHeader().setVisible(False)
# self.table_widget.itemSelectionChanged.connect(self.engine_picked)
self.table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers)
self.table.selectionModel().selectionChanged.connect(self.on_selection_changed)
layout = QVBoxLayout(self)
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
layout.addWidget(self.table)
self.raw_server_data = None
def showEvent(self, event):
"""Runs when the widget is about to be shown."""
self.update_engines_table()
super().showEvent(event) # Ensure normal event processing
def engine_data_ready(self, raw_server_data):
self.raw_server_data = raw_server_data
self.update_engines_table()
def update_engines_table(self, use_cached=True):
if not self.raw_server_data or not use_cached:
self.__get_engine_info_worker = GetEngineInfoWorker(self)
self.__get_engine_info_worker.done.connect(self.engine_data_ready)
self.__get_engine_info_worker.start()
if not self.raw_server_data:
return
table_data = [] # convert the data into a flat list
for _, engine_data in self.raw_server_data.items():
table_data.extend(engine_data['versions'])
if settings.value("engines_ignore_system_installs", False, type=bool):
table_data = [x for x in table_data if x['type'] != 'system']
self.table.clear()
self.table.setRowCount(len(table_data))
self.table.setColumnCount(4)
self.table.setHorizontalHeaderLabels(['Engine', 'Version', 'Type', 'Path'])
self.table.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.Fixed)
self.table.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Fixed)
self.table.horizontalHeader().setSectionResizeMode(2, QHeaderView.ResizeMode.Fixed)
self.table.horizontalHeader().setSectionResizeMode(3, QHeaderView.ResizeMode.Stretch)
for row, engine in enumerate(table_data):
self.table.setItem(row, 0, QTableWidgetItem(engine['engine']))
self.table.setItem(row, 1, QTableWidgetItem(engine['version']))
self.table.setItem(row, 2, QTableWidgetItem(engine['type']))
self.table.setItem(row, 3, QTableWidgetItem(engine['path']))
self.table.selectRow(0)
def selected_engine_data(self):
"""Returns the data from the selected row as a dictionary."""
row = self.table.currentRow() # Get the selected row index
if row < 0 or not len(self.table.selectedItems()): # No row selected
return None
data = {
"engine": self.table.item(row, 0).text(),
"version": self.table.item(row, 1).text(),
"type": self.table.item(row, 2).text(),
"path": self.table.item(row, 3).text(),
}
return data
def on_selection_changed(self):
self.row_selected.emit()
if __name__ == "__main__":
app = QApplication([])
window = SettingsWindow()
window.show()
app.exec()
src/ui/widgets/menubar.py
+6 -88
View File
@@ -1,6 +1,5 @@
''' app/ui/widgets/menubar.py '''
from PyQt6.QtGui import QAction
from PyQt6.QtWidgets import QMenuBar, QApplication, QMessageBox, QDialog, QVBoxLayout, QLabel, QPushButton
from PyQt6.QtWidgets import QMenuBar
class MenuBar(QMenuBar):
@@ -13,93 +12,12 @@ class MenuBar(QMenuBar):
def __init__(self, parent=None) -> None:
super().__init__(parent)
self.settings_window = None
# setup menus
file_menu = self.addMenu("File")
# edit_menu = self.addMenu("Edit")
# view_menu = self.addMenu("View")
help_menu = self.addMenu("Help")
# help_menu = self.addMenu("Help")
# --file menu--
# new job
new_job_action = QAction("New Job...", self)
new_job_action.setShortcut(f'Ctrl+N')
new_job_action.triggered.connect(self.new_job)
file_menu.addAction(new_job_action)
# settings
settings_action = QAction("Settings...", self)
settings_action.triggered.connect(self.show_settings)
settings_action.setShortcut(f'Ctrl+,')
file_menu.addAction(settings_action)
# exit
exit_action = QAction('&Exit', self)
exit_action.setShortcut('Ctrl+Q')
exit_action.triggered.connect(QApplication.instance().quit)
file_menu.addAction(exit_action)
# --help menu--
about_action = QAction("About", self)
about_action.triggered.connect(self.show_about)
help_menu.addAction(about_action)
update_action = QAction("Check for Updates...", self)
update_action.triggered.connect(self.check_for_updates)
help_menu.addAction(update_action)
def new_job(self):
self.parent().new_job()
def show_settings(self):
from src.ui.settings_window import SettingsWindow
self.settings_window = SettingsWindow()
self.settings_window.show()
@staticmethod
def show_about():
from src.ui.about_window import AboutDialog
dialog = AboutDialog()
dialog.exec()
@staticmethod
def check_for_updates():
from src.utilities.misc_helper import check_for_updates
from src.version import APP_NAME, APP_VERSION, APP_REPO_NAME, APP_REPO_OWNER
found_update = check_for_updates(APP_REPO_NAME, APP_REPO_OWNER, APP_NAME, APP_VERSION)
if found_update:
dialog = UpdateDialog(found_update, APP_VERSION)
dialog.exec()
else:
QMessageBox.information(None, "No Update", "No updates available.")
class UpdateDialog(QDialog):
def __init__(self, release_info, current_version, parent=None):
super().__init__(parent)
self.setWindowTitle(f"Update Available ({current_version} -> {release_info['tag_name']})")
layout = QVBoxLayout()
label = QLabel(f"A new version ({release_info['tag_name']}) is available! Current version: {current_version}")
layout.addWidget(label)
# Label to show the release notes
description = QLabel(release_info["body"])
layout.addWidget(description)
# Button to download the latest version
download_button = QPushButton(f"Download Latest Version ({release_info['tag_name']})")
download_button.clicked.connect(lambda: self.open_url(release_info["html_url"]))
layout.addWidget(download_button)
# OK button to dismiss the dialog
ok_button = QPushButton("Dismiss")
ok_button.clicked.connect(self.accept) # Close the dialog when clicked
layout.addWidget(ok_button)
self.setLayout(layout)
def open_url(self, url):
from PyQt6.QtCore import QUrl
from PyQt6.QtGui import QDesktopServices
QDesktopServices.openUrl(QUrl(url))
self.accept()
# Add actions to the menus
# file_menu.addAction(self.parent().topbar.actions_call["Open"]) # type: ignore
# file_menu.addAction(self.parent().topbar.actions_call["Save"]) # type: ignore
# file_menu.addAction(self.parent().topbar.actions_call["Exit"]) # type: ignore
src/ui/widgets/statusbar.py
+4 -5
View File
@@ -35,13 +35,12 @@ class StatusBar(QStatusBar):
try:
# update status label - get download status
new_status = proxy.status()
active_downloads = EngineManager.active_downloads()
if active_downloads:
if len(active_downloads) == 1:
task = active_downloads[0]
if EngineManager.download_tasks:
if len(EngineManager.download_tasks) == 1:
task = EngineManager.download_tasks[0]
new_status = f"{new_status} | Downloading {task.engine.capitalize()} {task.version}..."
else:
new_status = f"{new_status} | Downloading {len(active_downloads)} engines"
new_status = f"{new_status} | Downloading {len(EngineManager.download_tasks)} engines"
self.messageLabel.setText(new_status)
# update status image
src/utilities/config.py
+23 -43
View File
@@ -1,24 +1,10 @@
import os
from pathlib import Path
from typing import Optional
import yaml
from src.utilities.misc_helper import current_system_os, copy_directory_contents
_CONFIG_ATTRS = [
'upload_folder', 'update_engines_on_launch', 'max_content_path',
'server_log_level', 'log_buffer_length', 'worker_process_timeout',
'flask_log_level', 'flask_debug_enable', 'queue_eval_seconds',
'port_number', 'enable_split_jobs', 'download_timeout_seconds',
]
class Config:
_default_instance: Optional['Config'] = None
# Class-level defaults — mutated by _sync_class() so existing
# callers (Config.upload_folder) continue to work during the
# migration to instance-based access.
# Initialize class variables with default values
upload_folder = "~/zordon-uploads/"
update_engines_on_launch = True
max_content_path = 100000000
@@ -32,40 +18,33 @@ class Config:
enable_split_jobs = True
download_timeout_seconds = 120
def __init__(self) -> None:
for attr in _CONFIG_ATTRS:
setattr(self, attr, getattr(Config, attr))
def load(self, config_path: Path) -> None:
@classmethod
def load_config(cls, config_path):
with open(config_path, 'r') as ymlfile:
cfg = yaml.safe_load(ymlfile)
for attr in _CONFIG_ATTRS:
if attr in cfg:
setattr(self, attr, cfg[attr])
self.upload_folder = str(Path(self.upload_folder).expanduser())
cls.upload_folder = os.path.expanduser(cfg.get('upload_folder', cls.upload_folder))
cls.update_engines_on_launch = cfg.get('update_engines_on_launch', cls.update_engines_on_launch)
cls.max_content_path = cfg.get('max_content_path', cls.max_content_path)
cls.server_log_level = cfg.get('server_log_level', cls.server_log_level)
cls.log_buffer_length = cfg.get('log_buffer_length', cls.log_buffer_length)
cls.worker_process_timeout = cfg.get('worker_process_timeout', cls.worker_process_timeout)
cls.flask_log_level = cfg.get('flask_log_level', cls.flask_log_level)
cls.flask_debug_enable = cfg.get('flask_debug_enable', cls.flask_debug_enable)
cls.queue_eval_seconds = cfg.get('queue_eval_seconds', cls.queue_eval_seconds)
cls.port_number = cfg.get('port_number', cls.port_number)
cls.enable_split_jobs = cfg.get('enable_split_jobs', cls.enable_split_jobs)
cls.download_timeout_seconds = cfg.get('download_timeout_seconds', cls.download_timeout_seconds)
@classmethod
def _sync_class(cls) -> None:
if cls._default_instance is not None:
for attr in _CONFIG_ATTRS:
setattr(cls, attr, getattr(cls._default_instance, attr))
@classmethod
def load_config(cls, config_path: Path) -> None:
instance = Config()
instance.load(config_path)
cls._default_instance = instance
cls._sync_class()
@classmethod
def config_dir(cls) -> Path:
def config_dir(cls):
# Set up the config path
if current_system_os() == 'macos':
local_config_path = Path('~/Library/Application Support/Zordon').expanduser()
local_config_path = os.path.expanduser('~/Library/Application Support/Zordon')
elif current_system_os() == 'windows':
local_config_path = Path(os.environ['APPDATA']) / 'Zordon'
local_config_path = os.path.join(os.environ['APPDATA'], 'Zordon')
else:
local_config_path = Path('~/.config/Zordon').expanduser()
local_config_path = os.path.expanduser('~/.config/Zordon')
return local_config_path
@classmethod
@@ -82,9 +61,10 @@ class Config:
# Determine the template path
resource_environment_path = os.environ.get('RESOURCEPATH')
if resource_environment_path:
template_path = Path(resource_environment_path) / 'config'
template_path = os.path.join(resource_environment_path, 'config')
else:
template_path = Path(__file__).resolve().parents[2] / 'config'
template_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'config')
# Copy contents from the template to the local configuration directory
copy_directory_contents(template_path, local_config_dir)
src/utilities/ffmpeg_helper.py
+3 -3
View File
@@ -4,18 +4,18 @@ from src.engines.ffmpeg.ffmpeg_engine import FFMPEG
def image_sequence_to_video(source_glob_pattern, output_path, framerate=24, encoder="prores_ks", profile=4,
start_frame=1):
subprocess.run([FFMPEG.default_engine_path(), "-framerate", str(framerate), "-start_number",
subprocess.run([FFMPEG.default_renderer_path(), "-framerate", str(framerate), "-start_number",
str(start_frame), "-i", f"{source_glob_pattern}", "-c:v", encoder, "-profile:v", str(profile),
'-pix_fmt', 'yuva444p10le', output_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
check=True)
def save_first_frame(source_path, dest_path, max_width=1280):
subprocess.run([FFMPEG.default_engine_path(), '-i', source_path, '-vf', f'scale={max_width}:-1',
subprocess.run([FFMPEG.default_renderer_path(), '-i', source_path, '-vf', f'scale={max_width}:-1',
'-vframes', '1', dest_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
def generate_thumbnail(source_path, dest_path, max_width=240, fps=12):
subprocess.run([FFMPEG.default_engine_path(), '-i', source_path, '-vf',
subprocess.run([FFMPEG.default_renderer_path(), '-i', source_path, '-vf',
f"scale={max_width}:trunc(ow/a/2)*2,format=yuv420p", '-r', str(fps), '-c:v', 'libx264', '-preset',
'ultrafast', '-an', dest_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
src/utilities/misc_helper.py
+47 -257
View File
@@ -1,20 +1,16 @@
import logging
import json
import os
import platform
import re
import shutil
import socket
import string
import subprocess
import sys
from datetime import datetime
from typing import Optional, List, Dict, Any
logger = logging.getLogger()
def launch_url(url: str) -> None:
def launch_url(url):
logger = logging.getLogger(__name__)
if shutil.which('xdg-open'):
@@ -38,7 +34,7 @@ def launch_url(url: str) -> None:
logger.error(f"Failed to launch URL: {url}. Error: {e}")
def file_exists_in_mounts(filepath: str) -> Optional[str]:
def file_exists_in_mounts(filepath):
"""
Check if a file exists in any mounted directory.
It searches for the file in common mount points like '/Volumes', '/mnt', and '/media'.
@@ -79,7 +75,7 @@ def file_exists_in_mounts(filepath: str) -> Optional[str]:
return possible_mount_path
def get_time_elapsed(start_time: Optional[datetime] = None, end_time: Optional[datetime] = None) -> str:
def get_time_elapsed(start_time=None, end_time=None):
def strfdelta(tdelta, fmt='%H:%M:%S'):
days = tdelta.days
@@ -106,7 +102,7 @@ def get_time_elapsed(start_time: Optional[datetime] = None, end_time: Optional[d
return elapsed_time_string
def get_file_size_human(file_path: str) -> str:
def get_file_size_human(file_path):
size_in_bytes = os.path.getsize(file_path)
# Convert size to a human-readable format
@@ -122,274 +118,68 @@ def get_file_size_human(file_path: str) -> str:
return f"{size_in_bytes / 1024 ** 4:.2f} TB"
def current_system_os() -> str:
# Convert path to the appropriate format for the current platform
def system_safe_path(path):
if platform.system().lower() == "windows":
return os.path.normpath(path)
return path.replace("\\", "/")
def current_system_os():
return platform.system().lower().replace('darwin', 'macos')
def current_system_os_version() -> str:
return platform.release()
def current_system_os_version():
return platform.mac_ver()[0] if current_system_os() == 'macos' else platform.release().lower()
def current_system_cpu() -> str:
return platform.machine().lower().replace('amd64', 'x64')
def current_system_cpu():
# convert all x86 64 to "x64"
return platform.machine().lower().replace('amd64', 'x64').replace('x86_64', 'x64')
def current_system_cpu_brand() -> str:
"""Fast cross-platform CPU brand string"""
if sys.platform.startswith('darwin'): # macOS
try:
return subprocess.check_output(['sysctl', '-n', 'machdep.cpu.brand_string']).decode().strip()
except Exception:
pass
elif sys.platform.startswith('win'): # Windows
from winreg import HKEY_LOCAL_MACHINE, OpenKey, QueryValueEx
try:
# Open the registry key where Windows stores the CPU name
key = OpenKey(HKEY_LOCAL_MACHINE, r"HARDWARE\DESCRIPTION\System\CentralProcessor\0")
# The value name is "ProcessorNameString"
value, _ = QueryValueEx(key, "ProcessorNameString")
return value.strip() # Usually perfect, with full marketing name
except Exception:
# Fallback: sometimes the key is under a different index, try 1
try:
key = OpenKey(HKEY_LOCAL_MACHINE, r"HARDWARE\DESCRIPTION\System\CentralProcessor\1")
value, _ = QueryValueEx(key, "ProcessorNameString")
return value.strip()
except Exception:
return "Unknown CPU"
elif sys.platform.startswith('linux'):
try:
with open('/proc/cpuinfo') as f:
for line in f:
if line.startswith('model name'):
return line.split(':', 1)[1].strip()
except Exception:
pass
# Ultimate fallback
return platform.processor() or 'Unknown CPU'
def resources_dir() -> str:
return os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
def resources_dir():
resource_environment_path = os.environ.get('RESOURCEPATH', None)
if resource_environment_path: # running inside resource bundle
return os.path.join(resource_environment_path, 'resources')
else:
return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'resources')
def copy_directory_contents(src_dir: str, dst_dir: str) -> None:
def copy_directory_contents(src_dir, dst_dir):
"""
Copy the contents of the source directory (src_dir) to the destination directory (dst_dir).
"""
for item in os.listdir(src_dir):
src_path = os.path.join(src_dir, item)
dst_path = os.path.join(dst_dir, item)
if os.path.isdir(src_path):
shutil.copytree(src_path, dst_path)
shutil.copytree(src_path, dst_path, dirs_exist_ok=True)
else:
shutil.copy2(src_path, dst_path)
def check_for_updates(repo_name: str, repo_owner: str, app_name: str, current_version: str) -> Optional[Dict[str, Any]]:
def get_github_releases(owner, repo):
import requests
url = f"https://api.github.com/repos/{owner}/{repo}/releases"
try:
response = requests.get(url, timeout=3)
response.raise_for_status()
releases = response.json()
return releases
except Exception as e:
logger.error(f"Error checking for updates: {e}")
return []
releases = get_github_releases(repo_owner, repo_name)
if not releases:
return None
latest_version = releases[0]
latest_version_tag = latest_version['tag_name']
from packaging import version
if version.parse(latest_version_tag) > version.parse(current_version):
logger.info(f"Newer version of {app_name} available. "
f"Latest: {latest_version_tag}, Current: {current_version}")
return latest_version
return None
def is_localhost(comparison_hostname: str) -> bool:
return comparison_hostname in ['localhost', '127.0.0.1', socket.gethostname()]
def is_localhost(comparison_hostname):
# this is necessary because socket.gethostname() does not always include '.local' - This is a sanitized comparison
try:
comparison_hostname = comparison_hostname.lower().replace('.local', '')
local_hostname = socket.gethostname().lower().replace('.local', '')
return comparison_hostname == local_hostname
except AttributeError:
return False
def num_to_alphanumeric(num: int) -> str:
return string.ascii_letters[num % 26] + str(num // 26)
def num_to_alphanumeric(num):
# List of possible alphanumeric characters
characters = string.ascii_letters + string.digits
# Make sure number is positive
num = abs(num)
def get_gpu_info() -> List[Dict[str, Any]]:
"""Cross-platform GPU information retrieval"""
# Convert number to alphanumeric
result = ""
while num > 0:
num, remainder = divmod(num, len(characters))
result += characters[remainder]
def get_windows_gpu_info():
"""Get GPU info on Windows"""
try:
result = subprocess.run(
['wmic', 'path', 'win32_videocontroller', 'get', 'name,AdapterRAM', '/format:list'],
capture_output=True, text=True, timeout=5
)
# Virtual adapters to exclude
virtual_adapters = [
'virtual', 'rdp', 'hyper-v', 'microsoft basic', 'basic display',
'vga compatible', 'dummy', 'nvfbc', 'nvencode'
]
gpus = []
current_gpu = None
for line in result.stdout.strip().split('\n'):
line = line.strip()
if not line:
continue
if line.startswith('Name='):
if current_gpu and current_gpu.get('name'):
gpus.append(current_gpu)
gpu_name = line.replace('Name=', '').strip()
# Skip virtual adapters
if any(virtual in gpu_name.lower() for virtual in virtual_adapters):
current_gpu = None
else:
current_gpu = {'name': gpu_name, 'memory': 'Integrated'}
elif line.startswith('AdapterRAM=') and current_gpu:
vram_bytes_str = line.replace('AdapterRAM=', '').strip()
if vram_bytes_str and vram_bytes_str != '0':
try:
vram_gb = int(vram_bytes_str) / (1024**3)
current_gpu['memory'] = round(vram_gb, 2)
except:
pass
if current_gpu and current_gpu.get('name'):
gpus.append(current_gpu)
return gpus if gpus else [{'name': 'Unknown GPU', 'memory': 'Unknown'}]
except Exception as e:
logger.error(f"Failed to get Windows GPU info: {e}")
return [{'name': 'Unknown GPU', 'memory': 'Unknown'}]
def get_macos_gpu_info():
"""Get GPU info on macOS (works with Apple Silicon)"""
try:
if current_system_cpu() == "arm64":
# don't bother with system_profiler with Apple ARM - we know its integrated
return [{'name': current_system_cpu_brand(), 'memory': 'Integrated'}]
result = subprocess.run(['system_profiler', 'SPDisplaysDataType', '-detailLevel', 'mini', '-json'],
capture_output=True, text=True, timeout=5)
data = json.loads(result.stdout)
gpus = []
displays = data.get('SPDisplaysDataType', [])
for display in displays:
if 'sppci_model' in display:
gpus.append({
'name': display.get('sppci_model', 'Unknown GPU'),
'memory': display.get('sppci_vram', 'Integrated'),
})
return gpus if gpus else [{'name': 'Apple GPU', 'memory': 'Integrated'}]
except Exception as e:
print(f"Failed to get macOS GPU info: {e}")
return [{'name': 'Unknown GPU', 'memory': 'Unknown'}]
def get_linux_gpu_info():
gpus = []
try:
# Run plain lspci and filter for GPU-related lines
output = subprocess.check_output(
["lspci"], universal_newlines=True, stderr=subprocess.DEVNULL
)
for line in output.splitlines():
if any(keyword in line.lower() for keyword in ["vga", "3d", "display"]):
# Extract the part after the colon (vendor + model)
if ":" in line:
name_part = line.split(":", 1)[1].strip()
# Clean up common extras like (rev xx) or (prog-if ...)
name = name_part.split("(")[0].split("controller:")[-1].strip()
vendor = "Unknown"
if "nvidia" in name.lower():
vendor = "NVIDIA"
elif "amd" in name.lower() or "ati" in name.lower():
vendor = "AMD"
elif "intel" in name.lower():
vendor = "Intel"
gpus.append({
"name": name,
"vendor": vendor,
"memory": "Unknown"
})
except FileNotFoundError:
print("lspci not found. Install pciutils: sudo apt install pciutils")
return []
except Exception as e:
print(f"Error running lspci: {e}")
return []
return gpus
system = platform.system()
if system == 'Darwin': # macOS
return get_macos_gpu_info()
elif system == 'Windows':
return get_windows_gpu_info()
else: # Assume Linux or other
return get_linux_gpu_info()
COMMON_RESOLUTIONS = {
# SD
"SD_480p": (640, 480),
"NTSC_DVD": (720, 480),
"PAL_DVD": (720, 576),
# HD
"HD_720p": (1280, 720),
"HD_900p": (1600, 900),
"HD_1080p": (1920, 1080),
# Cinema / Film
"2K_DCI": (2048, 1080),
"4K_DCI": (4096, 2160),
# UHD / Consumer
"UHD_4K": (3840, 2160),
"UHD_5K": (5120, 2880),
"UHD_8K": (7680, 4320),
# Ultrawide / Aspect Variants
"UW_1080p": (2560, 1080),
"UW_1440p": (3440, 1440),
"UW_5K": (5120, 2160),
# Mobile / Social
"VERTICAL_1080x1920": (1080, 1920),
"SQUARE_1080": (1080, 1080),
# Classic / Legacy
"VGA": (640, 480),
"SVGA": (800, 600),
"XGA": (1024, 768),
"WXGA": (1280, 800),
}
COMMON_FRAME_RATES = {
"23.976 (NTSC Film)": 23.976,
"24 (Cinema)": 24.0,
"25 (PAL)": 25.0,
"29.97 (NTSC)": 29.97,
"30": 30.0,
"48 (HFR Film)": 48.0,
"50 (PAL HFR)": 50.0,
"59.94": 59.94,
"60": 60.0,
"72": 72.0,
"90 (VR)": 90.0,
"120": 120.0,
"144 (Gaming)": 144.0,
"240 (HFR)": 240.0,
}
return result[::-1] # Reverse the result to get the correct alphanumeric string
src/utilities/server_helper.py
+33 -186
View File
@@ -1,200 +1,47 @@
import logging
import os
import zipfile
from concurrent.futures import ThreadPoolExecutor
import subprocess
import threading
import requests
from src.api.server_proxy import RenderServerProxy
from src.utilities.misc_helper import get_file_size_human
from src.utilities.zeroconf_server import ZeroconfServer
from src.utilities.ffmpeg_helper import generate_thumbnail, save_first_frame
logger = logging.getLogger()
def download_missing_frames_from_subjob(local_job, subjob_id, subjob_hostname):
success = True
try:
local_files = [os.path.basename(x) for x in local_job.file_list()]
subjob_proxy = RenderServerProxy(subjob_hostname)
subjob_files = subjob_proxy.get_job_files_list(job_id=subjob_id) or []
def generate_thumbnail_for_job(job, thumb_video_path, thumb_image_path, max_width=320):
for subjob_filename in subjob_files:
if subjob_filename not in local_files:
try:
logger.debug(f"Downloading new file '{subjob_filename}' from {subjob_hostname}")
local_save_path = os.path.join(os.path.dirname(local_job.output_path), subjob_filename)
subjob_proxy.download_job_file(job_id=subjob_id, job_filename=subjob_filename,
save_path=local_save_path)
logger.debug(f'Downloaded successfully - {local_save_path}')
except Exception as e:
logger.error(f"Error downloading file '{subjob_filename}' from {subjob_hostname}: {e}")
success = False
except Exception as e:
logger.exception(f'Uncaught exception while trying to download from subjob: {e}')
success = False
return success
# Simple thread to generate thumbs in background
def generate_thumb_thread(source):
in_progress_path = thumb_video_path + '_IN-PROGRESS'
subprocess.run(['touch', in_progress_path])
try:
logger.debug(f"Generating video thumbnail for {source}")
generate_thumbnail(source_path=source, dest_path=thumb_video_path, max_width=max_width)
except subprocess.CalledProcessError as err:
logger.error(f"Error generating video thumbnail for {source}: {err}")
try:
os.remove(in_progress_path)
except FileNotFoundError:
pass
def download_all_from_subjob(local_job, subjob_id, subjob_hostname):
"""
Downloads and extracts files from a completed subjob on a remote server.
# Determine best source file to use for thumbs
source_files = job.file_list() or [job.input_path]
if source_files:
video_formats = ['.mp4', '.mov', '.avi', '.mpg', '.mpeg', '.mxf', '.m4v', 'mkv']
image_formats = ['.jpg', '.png', '.exr']
Parameters:
local_job (BaseRenderWorker): The local parent job worker.
subjob_id (str or int): The ID of the subjob.
subjob_hostname (str): The hostname of the remote server where the subjob is located.
image_files = [f for f in source_files if os.path.splitext(f)[-1].lower() in image_formats]
video_files = [f for f in source_files if os.path.splitext(f)[-1].lower() in video_formats]
Returns:
bool: True if the files have been downloaded and extracted successfully, False otherwise.
"""
child_key = f'{subjob_id}@{subjob_hostname}'
logname = f"{local_job.id}:{child_key}"
zip_file_path = local_job.output_path + f'_{subjob_hostname}_{subjob_id}.zip'
# download zip file from server
try:
local_job.children[child_key]['download_status'] = 'working'
logger.info(f"Downloading completed subjob files from {subjob_hostname} to localhost")
RenderServerProxy(subjob_hostname).download_all_job_files(subjob_id, zip_file_path)
logger.info(f"File transfer complete for {logname} - Transferred {get_file_size_human(zip_file_path)}")
except Exception as e:
logger.error(f"Error downloading files from remote server: {e}")
local_job.children[child_key]['download_status'] = 'failed'
return False
# extract zip
try:
logger.debug(f"Extracting zip file: {zip_file_path}")
extract_path = os.path.dirname(zip_file_path)
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
zip_ref.extractall(extract_path)
logger.info(f"Successfully extracted zip to: {extract_path}")
os.remove(zip_file_path)
local_job.children[child_key]['download_status'] = 'complete'
except Exception as e:
logger.exception(f"Exception extracting zip file: {e}")
local_job.children[child_key]['download_status'] = 'failed'
return local_job.children[child_key].get('download_status', None) == 'complete'
def distribute_server_work(start_frame, end_frame, available_servers, method='evenly'):
"""
Splits the frame range among available servers proportionally based on their performance (CPU count).
Args:
start_frame (int): The start frame number of the animation to be rendered.
end_frame (int): The end frame number of the animation to be rendered.
available_servers (list): A list of available server dictionaries. Each server dictionary should include
'hostname' and 'cpu_count' keys (see find_available_servers).
method (str, optional): Specifies the distribution method. Possible values are 'cpu_benchmark', 'cpu_count'
and 'evenly'.
Defaults to 'cpu_benchmark'.
Returns:
list: A list of server dictionaries where each dictionary includes the frame range and total number of
frames to be rendered by the server.
"""
# Calculate respective frames for each server
def divide_frames_by_cpu_count(frame_start, frame_end, servers):
total_frames = frame_end - frame_start + 1
total_cpus = sum(server['cpu_count'] for server in servers)
frame_ranges = {}
current_frame = frame_start
allocated_frames = 0
for i, server in enumerate(servers):
if i == len(servers) - 1: # if it's the last server
# Give all remaining frames to the last server
num_frames = total_frames - allocated_frames
else:
num_frames = round((server['cpu_count'] / total_cpus) * total_frames)
allocated_frames += num_frames
frame_end_for_server = current_frame + num_frames - 1
if current_frame <= frame_end_for_server:
frame_ranges[server['hostname']] = (current_frame, frame_end_for_server)
current_frame = frame_end_for_server + 1
return frame_ranges
def divide_frames_by_benchmark(frame_start, frame_end, servers):
def fetch_benchmark(server):
if (video_files or image_files) and not os.path.exists(thumb_image_path):
try:
benchmark = requests.get(f'http://{server["hostname"]}:{ZeroconfServer.server_port}'
f'/api/cpu_benchmark').text
server['cpu_benchmark'] = benchmark
logger.debug(f'Benchmark for {server["hostname"]}: {benchmark}')
except requests.exceptions.RequestException as e:
logger.error(f'Error fetching benchmark for {server["hostname"]}: {e}')
path_of_source = image_files[0] if image_files else video_files[0]
logger.debug(f"Generating image thumbnail for {path_of_source}")
save_first_frame(source_path=path_of_source, dest_path=thumb_image_path, max_width=max_width)
except Exception as e:
logger.error(f"Exception saving first frame: {e}")
# Number of threads to use (can adjust based on your needs or number of servers)
threads = len(servers)
with ThreadPoolExecutor(max_workers=threads) as executor:
executor.map(fetch_benchmark, servers)
total_frames = frame_end - frame_start + 1
total_performance = sum(int(server['cpu_benchmark']) for server in servers)
frame_ranges = {}
current_frame = frame_start
allocated_frames = 0
for i, server in enumerate(servers):
if i == len(servers) - 1: # if it's the last server
# Give all remaining frames to the last server
num_frames = total_frames - allocated_frames
else:
num_frames = round((int(server['cpu_benchmark']) / total_performance) * total_frames)
allocated_frames += num_frames
frame_end_for_server = current_frame + num_frames - 1
if current_frame <= frame_end_for_server:
frame_ranges[server['hostname']] = (current_frame, frame_end_for_server)
current_frame = frame_end_for_server + 1
return frame_ranges
def divide_frames_equally(frame_start, frame_end, servers):
frame_range = frame_end - frame_start + 1
frames_per_server = frame_range // len(servers)
leftover_frames = frame_range % len(servers)
frame_ranges = {}
current_start = frame_start
for i, server in enumerate(servers):
current_end = current_start + frames_per_server - 1
if leftover_frames > 0:
current_end += 1
leftover_frames -= 1
if current_start <= current_end:
frame_ranges[server['hostname']] = (current_start, current_end)
current_start = current_end + 1
return frame_ranges
if len(available_servers) == 1:
breakdown = {available_servers[0]['hostname']: (start_frame, end_frame)}
else:
logger.debug(f'Splitting between {len(available_servers)} servers by {method} method')
if method == 'evenly':
breakdown = divide_frames_equally(start_frame, end_frame, available_servers)
elif method == 'cpu_benchmark':
breakdown = divide_frames_by_benchmark(start_frame, end_frame, available_servers)
elif method == 'cpu_count':
breakdown = divide_frames_by_cpu_count(start_frame, end_frame, available_servers)
else:
raise ValueError(f"Invalid distribution method: {method}")
server_breakdown = [server for server in available_servers if breakdown.get(server['hostname']) is not None]
for server in server_breakdown:
server['frame_range'] = breakdown[server['hostname']]
server['total_frames'] = breakdown[server['hostname']][-1] - breakdown[server['hostname']][0] + 1
return server_breakdown
if video_files and not os.path.exists(thumb_video_path):
x = threading.Thread(target=generate_thumb_thread, args=(video_files[0],))
x.start()
src/utilities/zeroconf_server.py
+60 -102
View File
@@ -1,6 +1,5 @@
import logging
import socket
from typing import Dict, List, Optional
from pubsub import pub
from zeroconf import Zeroconf, ServiceInfo, ServiceBrowser, ServiceStateChange, NonUniqueNameException, \
@@ -10,144 +9,103 @@ logger = logging.getLogger()
class ZeroconfServer:
_default_instance: Optional['ZeroconfServer'] = None
service_type = None
server_name = None
server_port = None
server_ip = None
zeroconf = Zeroconf()
service_info = None
client_cache = {}
properties = {}
service_type: Optional[str] = None
server_name: Optional[str] = None
server_port: Optional[int] = None
properties: Dict = {}
def __init__(self) -> None:
self.service_type: Optional[str] = None
self.server_name: Optional[str] = None
self.server_port: Optional[int] = None
self.server_ip: Optional[str] = None
self.zeroconf: Zeroconf = Zeroconf()
self.service_info: Optional[ServiceInfo] = None
self.client_cache: Dict = {}
self.properties: Dict = {}
def _configure(self, service_type: str, server_name: str, server_port: int) -> None:
self.service_type = service_type
self.server_name = server_name
self.server_port = server_port
try:
@classmethod
def configure(cls, service_type, server_name, server_port):
cls.service_type = service_type
cls.server_name = server_name
cls.server_port = server_port
try: # Stop any previously running instances
socket.gethostbyname(socket.gethostname())
except socket.gaierror:
self.stop()
cls.stop()
def _start(self, listen_only: bool = False) -> None:
if not self.service_type:
@classmethod
def start(cls, listen_only=False):
if not cls.service_type:
raise RuntimeError("The 'configure' method must be run before starting the zeroconf server")
logger.debug("Starting zeroconf service")
if not listen_only:
logger.debug("Starting zeroconf service")
self._register_service()
else:
logger.debug("Starting zeroconf service - Listen only mode")
self._browse_services()
cls._register_service()
cls._browse_services()
def _stop(self) -> None:
@classmethod
def stop(cls):
logger.debug("Stopping zeroconf service")
self._unregister_service()
if self.zeroconf:
self.zeroconf.close()
cls._unregister_service()
cls.zeroconf.close()
def _register_service(self) -> None:
@classmethod
def _register_service(cls):
try:
self.server_ip = socket.gethostbyname(socket.gethostname())
cls.server_ip = socket.gethostbyname(socket.gethostname())
info = ServiceInfo(
self.service_type,
f"{self.server_name}.{self.service_type}",
addresses=[socket.inet_aton(self.server_ip)],
port=self.server_port,
properties=self.properties,
cls.service_type,
f"{cls.server_name}.{cls.service_type}",
addresses=[socket.inet_aton(cls.server_ip)],
port=cls.server_port,
properties=cls.properties,
)
self.service_info = info
self.zeroconf.register_service(info)
logger.info(f"Registered zeroconf service: {self.service_info.name}")
cls.service_info = info
cls.zeroconf.register_service(info)
logger.info(f"Registered zeroconf service: {cls.service_info.name}")
except (NonUniqueNameException, socket.gaierror) as e:
logger.error(f"Error establishing zeroconf: {e}")
def _unregister_service(self) -> None:
if self.service_info:
self.zeroconf.unregister_service(self.service_info)
logger.info(f"Unregistered zeroconf service: {self.service_info.name}")
self.service_info = None
@classmethod
def _unregister_service(cls):
if cls.service_info:
cls.zeroconf.unregister_service(cls.service_info)
logger.info(f"Unregistered zeroconf service: {cls.service_info.name}")
cls.service_info = None
def _browse_services(self) -> None:
ServiceBrowser(self.zeroconf, self.service_type, [self._on_service_discovered])
@classmethod
def _browse_services(cls):
browser = ServiceBrowser(cls.zeroconf, cls.service_type, [cls._on_service_discovered])
browser.is_alive()
def _on_service_discovered(self, zeroconf, service_type, name, state_change) -> None:
@classmethod
def _on_service_discovered(cls, zeroconf, service_type, name, state_change):
try:
info = zeroconf.get_service_info(service_type, name)
hostname = name.split(f'.{self.service_type}')[0]
hostname = name.split(f'.{cls.service_type}')[0]
logger.debug(f"Zeroconf: {hostname} {state_change}")
if service_type == self.service_type:
if state_change in (ServiceStateChange.Added, ServiceStateChange.Updated):
self.client_cache[hostname] = info
if service_type == cls.service_type:
if state_change == ServiceStateChange.Added or state_change == ServiceStateChange.Updated:
cls.client_cache[hostname] = info
else:
self.client_cache.pop(hostname, None)
cls.client_cache.pop(hostname)
pub.sendMessage('zeroconf_state_change', hostname=hostname, state_change=state_change)
except NotRunningException:
pass
@classmethod
def _sync_class(cls) -> None:
if cls._default_instance is not None:
inst = cls._default_instance
cls.service_type = inst.service_type
cls.server_name = inst.server_name
cls.server_port = inst.server_port
cls.server_ip = inst.server_ip
cls.properties = inst.properties
def _found_hostnames(self) -> List[str]:
def found_hostnames(cls):
local_hostname = socket.gethostname()
def sort_key(hostname):
# Return 0 if it's the local hostname so it comes first, else return 1
return False if hostname == local_hostname else True
sorted_hostnames = sorted(self.client_cache.keys(), key=sort_key)
# Sort the list with the local hostname first
sorted_hostnames = sorted(cls.client_cache.keys(), key=sort_key)
return sorted_hostnames
def _get_hostname_properties(self, hostname: str) -> Dict:
server_info = self.client_cache.get(hostname)
if server_info is None:
return {}
decoded_server_info = {key.decode('utf-8'): value.decode('utf-8') for key, value in server_info.properties.items()}
return decoded_server_info
# --- Forwarders for backward compatibility ---
@classmethod
def configure(cls, service_type, server_name, server_port):
if cls._default_instance is not None:
cls._default_instance._configure(service_type, server_name, server_port)
cls._sync_class()
@classmethod
def start(cls, listen_only=False):
if cls._default_instance is not None:
cls._default_instance._start(listen_only)
@classmethod
def stop(cls):
if cls._default_instance is not None:
cls._default_instance._stop()
@classmethod
def found_hostnames(cls):
if cls._default_instance is not None:
return cls._default_instance._found_hostnames()
return []
@classmethod
def get_hostname_properties(cls, hostname):
if cls._default_instance is not None:
return cls._default_instance._get_hostname_properties(hostname)
return {}
server_info = cls.client_cache.get(hostname).properties
decoded_server_info = {key.decode('utf-8'): value.decode('utf-8') for key, value in server_info.items()}
return decoded_server_info
# Example usage:
src/version.py
-8
View File
@@ -1,8 +0,0 @@
APP_NAME = "Zordon"
APP_VERSION = "0.0.1"
APP_AUTHOR = "Brett Williams"
APP_DESCRIPTION = "Distributed Render Farm Tools"
APP_COPYRIGHT_YEAR = "2024"
APP_LICENSE = "MIT License"
APP_REPO_NAME = APP_NAME
APP_REPO_OWNER = "blw1138"
tests/__init__.py
View File
tests/conftest.py
-134
View File
@@ -1,134 +0,0 @@
from unittest.mock import patch
from pathlib import Path
import pytest
from src.render_queue import RenderQueue
from src.engines.engine_manager import EngineManager
from src.api.preview_manager import PreviewManager
from src.utilities.config import Config
from src.distributed_job_manager import DistributedJobManager
from src.utilities.zeroconf_server import ZeroconfServer
# ---------------------------------------------------------------------------
# Global pubsub patch – real pubsub can fire callbacks across tests.
# Each service module does `from pubsub import pub` at import time, so we
# must patch each module-level reference individually.
# ---------------------------------------------------------------------------
_PUBSUB_TARGETS = [
'pubsub.pub',
'src.render_queue.pub',
'src.distributed_job_manager.pub',
'src.utilities.zeroconf_server.pub',
'src.engines.core.base_worker.pub',
]
@pytest.fixture(autouse=True)
def _patch_pubsub():
mocks = {}
patchers = []
for target in _PUBSUB_TARGETS:
p = patch(target)
patchers.append(p)
mocks[target] = p.start()
yield mocks.get('pubsub.pub')
for p in reversed(patchers):
p.stop()
# ---------------------------------------------------------------------------
# Temp directory for file-system-dependent tests
# ---------------------------------------------------------------------------
@pytest.fixture
def tmp_workspace(tmp_path: Path) -> Path:
ws = tmp_path / 'zordon_test'
ws.mkdir()
return ws
# ---------------------------------------------------------------------------
# Config fixture
# ---------------------------------------------------------------------------
@pytest.fixture
def config_instance(tmp_workspace: Path) -> Config:
orig = Config._default_instance
cfg = Config()
cfg.upload_folder = str(tmp_workspace / 'uploads')
Config._default_instance = cfg
Config._sync_class()
yield cfg
Config._default_instance = orig
Config._sync_class()
# ---------------------------------------------------------------------------
# EngineManager fixture
# ---------------------------------------------------------------------------
@pytest.fixture
def engine_manager_instance(tmp_workspace: Path) -> EngineManager:
orig = EngineManager._default_instance
em = EngineManager()
em.engines_path = str(tmp_workspace / 'engines')
EngineManager._default_instance = em
EngineManager._sync_class()
yield em
EngineManager._default_instance = orig
EngineManager._sync_class()
# ---------------------------------------------------------------------------
# RenderQueue fixture
# ---------------------------------------------------------------------------
@pytest.fixture
def render_queue_instance() -> RenderQueue:
orig = RenderQueue._default_instance
rq = RenderQueue()
RenderQueue._default_instance = rq
RenderQueue._sync_class()
yield rq
RenderQueue._default_instance = orig
RenderQueue._sync_class()
# ---------------------------------------------------------------------------
# DistributedJobManager fixture
# ---------------------------------------------------------------------------
@pytest.fixture
def distributed_job_manager_instance() -> DistributedJobManager:
orig = DistributedJobManager._default_instance
djm = DistributedJobManager()
DistributedJobManager._default_instance = djm
DistributedJobManager._sync_class()
yield djm
DistributedJobManager._default_instance = orig
DistributedJobManager._sync_class()
# ---------------------------------------------------------------------------
# PreviewManager fixture
# ---------------------------------------------------------------------------
@pytest.fixture
def preview_manager_instance(tmp_workspace: Path) -> PreviewManager:
orig = PreviewManager._default_instance
pm = PreviewManager()
pm.storage_path = str(tmp_workspace / 'previews')
PreviewManager._default_instance = pm
PreviewManager._sync_class()
yield pm
PreviewManager._default_instance = orig
PreviewManager._sync_class()
# ---------------------------------------------------------------------------
# ZeroconfServer fixture
# ---------------------------------------------------------------------------
@pytest.fixture
def zeroconf_server_instance() -> ZeroconfServer:
orig = ZeroconfServer._default_instance
zs = ZeroconfServer()
ZeroconfServer._default_instance = zs
ZeroconfServer._sync_class()
yield zs
ZeroconfServer._default_instance = orig
ZeroconfServer._sync_class()
tests/job_creation_tests.py
-85
View File
@@ -1,85 +0,0 @@
import logging
import os
import time
import unittest
from src.api.server_proxy import RenderServerProxy
logger = logging.getLogger(__name__)
SERVER_HOST = os.environ.get('ZORDON_TEST_HOST', '127.0.0.1')
SERVER_PORT = os.environ.get('ZORDON_TEST_PORT', '8080')
@unittest.skipIf(os.environ.get('ZORDON_SKIP_INTEGRATION'),
'set ZORDON_SKIP_INTEGRATION to skip integration tests')
class SubmissionTestCase(unittest.TestCase):
"""Integration tests requiring a running Zordon server.
Start the server: python server.py
Run tests: ZORDON_TEST_HOST=127.0.0.1 python -m pytest tests/job_creation_tests.py
Override host/port via ZORDON_TEST_HOST and ZORDON_TEST_PORT env vars.
"""
render_server = None
test_job_id = None
@classmethod
def setUpClass(cls):
cls.render_server = RenderServerProxy(SERVER_HOST, SERVER_PORT)
@classmethod
def tearDownClass(cls):
cls.render_server = None
def test_connection_ok(self):
self.assertTrue(self.render_server.is_online(),
msg=f'Server not reachable at {SERVER_HOST}:{SERVER_PORT}')
def test_submit_job(self):
sample_file_path = os.path.join(os.path.dirname(__file__), 'resources',
'batman_sample.blend')
self.assertTrue(os.path.exists(sample_file_path),
msg=f'Test file not found: {sample_file_path}')
sample_job = {
'name': 'sample_blender_job',
'renderer': 'blender',
'start_frame': 1,
'end_frame': 5,
'args': {'engine': 'CYCLES'},
'enable_split_jobs': False,
}
response = self.render_server.post_job_to_server(
file_path=sample_file_path, job_list=[sample_job])
self.assertIsNotNone(response, msg='No response from server')
self.assertTrue(response.ok, msg=f'Server returned {response.status_code}')
logger.info('Submitted to server ok!')
job_data = response.json()
self.__class__.test_job_id = job_data[0]['id']
def test_wait_for_job_to_complete(self):
if not self.__class__.test_job_id:
self.skipTest('No job was submitted in test_submit_job')
file_count = 0
while True:
update_response = self.render_server.get_job_info(self.__class__.test_job_id)
if update_response:
print(f"Status: {update_response['status']}")
if update_response['status'] not in [
'not_started', 'running', 'waiting_for_subjobs', 'configuring'
]:
self.assertEqual(update_response['status'], 'completed',
msg=f"Job ended with status: {update_response['status']}")
break
if update_response['file_count'] > file_count:
file_count = update_response['file_count']
print(f"File count is now {file_count}")
time.sleep(1)
tests/resources/batman_sample.blend
BIN
View File
Binary file not shown.
tests/test_config.py
-172
View File
@@ -1,172 +0,0 @@
from pathlib import Path
from unittest.mock import patch
from src.utilities.config import Config, _CONFIG_ATTRS
class TestConfigDefaults:
"""Default attribute values."""
def test_default_upload_folder(self):
assert Config.upload_folder == '~/zordon-uploads/'
def test_default_port(self):
assert Config.port_number == 8080
def test_default_server_log_level(self):
assert Config.server_log_level == 'debug'
def test_default_enable_split_jobs(self):
assert Config.enable_split_jobs is True
def test_default_worker_timeout(self):
assert Config.worker_process_timeout == 120
class TestConfigInstance:
"""Instance creation and attribute initialisation."""
def test_init_copies_class_attrs(self):
cfg = Config()
assert cfg.upload_folder == '~/zordon-uploads/'
assert cfg.port_number == 8080
def test_init_has_all_attrs(self):
cfg = Config()
for attr in _CONFIG_ATTRS:
assert hasattr(cfg, attr), f'missing attr: {attr}'
class TestConfigLoad:
"""Loading configuration from YAML."""
def test_load_sets_attributes(self, tmp_path):
config_file = tmp_path / 'config.yaml'
config_file.write_text('port_number: 9090\nserver_log_level: info\n')
cfg = Config()
cfg.load(config_file)
assert cfg.port_number == 9090
assert cfg.server_log_level == 'info'
def test_load_expands_upload_folder(self, tmp_path):
config_file = tmp_path / 'config.yaml'
config_file.write_text('upload_folder: ~/custom-uploads/\n')
cfg = Config()
cfg.load(config_file)
# expanduser strips trailing slash
assert cfg.upload_folder.endswith('/custom-uploads')
def test_load_ignores_unknown_keys(self, tmp_path):
config_file = tmp_path / 'config.yaml'
config_file.write_text('nonexistent_key: value\nport_number: 7070\n')
cfg = Config()
cfg.load(config_file)
assert cfg.port_number == 7070
def test_load_raises_on_missing_file(self):
cfg = Config()
try:
cfg.load(Path('/nonexistent/path.yaml'))
assert False, 'expected FileNotFoundError'
except FileNotFoundError:
pass
class TestConfigSyncClass:
"""_sync_class propagates instance attrs to class level."""
def test_sync_class_propagates_attrs(self):
orig = Config._default_instance
try:
cfg = Config()
cfg.port_number = 7777
Config._default_instance = cfg
Config._sync_class()
assert Config.port_number == 7777
finally:
Config._default_instance = orig
Config._sync_class()
def test_sync_class_noop_when_no_instance(self):
orig = Config._default_instance
try:
Config._default_instance = None
Config._sync_class()
finally:
Config._default_instance = orig
Config._sync_class()
class TestConfigLoadConfig:
"""Classmethod load_config — full bootstrap."""
def test_load_config_sets_default_instance(self, tmp_path):
orig = Config._default_instance
try:
config_file = tmp_path / 'config.yaml'
config_file.write_text('port_number: 9999\n')
Config._default_instance = None
Config.load_config(config_file)
assert Config._default_instance is not None
assert Config.port_number == 9999
finally:
Config._default_instance = orig
Config._sync_class()
class TestConfigDir:
"""config_dir() returns OS-specific path."""
@patch('src.utilities.config.current_system_os')
def test_config_dir_macos(self, mock_os):
mock_os.return_value = 'macos'
result = Config.config_dir()
assert 'Library/Application Support/Zordon' in str(result)
@patch('src.utilities.config.current_system_os')
def test_config_dir_windows(self, mock_os):
mock_os.return_value = 'windows'
with patch.dict('os.environ', {'APPDATA': 'C:\\Users\\Test\\AppData\\Roaming'}):
result = Config.config_dir()
assert 'Zordon' in str(result)
@patch('src.utilities.config.current_system_os')
def test_config_dir_linux(self, mock_os):
mock_os.return_value = 'linux'
result = Config.config_dir()
assert '.config/Zordon' in str(result)
class TestSetupConfigDir:
"""setup_config_dir creates dir and copies template."""
@patch('src.utilities.config.copy_directory_contents')
@patch('src.utilities.config.os.makedirs')
@patch('src.utilities.config.os.path.exists')
def test_creates_dir_when_missing(self, mock_exists, mock_makedirs, mock_copy):
mock_exists.return_value = False
Config.setup_config_dir()
mock_makedirs.assert_called_once()
@patch('src.utilities.config.copy_directory_contents')
@patch('src.utilities.config.os.makedirs')
@patch('src.utilities.config.os.path.exists')
def test_skips_when_dir_exists(self, mock_exists, mock_makedirs, mock_copy):
mock_exists.return_value = True
Config.setup_config_dir()
mock_makedirs.assert_not_called()
mock_copy.assert_not_called()
tests/test_distributed_job_manager.py
-138
View File
@@ -1,138 +0,0 @@
from unittest.mock import MagicMock, patch
import src.distributed_job_manager as djm_module
from src.distributed_job_manager import DistributedJobManager
from src.utilities.status_utils import RenderStatus
class TestSubscribeToListener:
"""PubSub subscription."""
def test_subscribes_to_status_change(self, distributed_job_manager_instance):
distributed_job_manager_instance._subscribe_to_listener()
# Check via the module-level reference (the one _subscribe_to_listener uses)
djm_module.pub.subscribe.assert_any_call(
distributed_job_manager_instance._local_job_status_changed,
'status_change',
)
djm_module.pub.subscribe.assert_any_call(
distributed_job_manager_instance._local_job_frame_complete,
'frame_complete',
)
class TestCreateRenderJob:
"""Creating a render job."""
@patch('src.distributed_job_manager.os.makedirs')
@patch('src.distributed_job_manager.EngineManager.create_worker')
def test_creates_worker_and_adds_to_queue(
self, mock_create_worker, mock_makedirs, distributed_job_manager_instance,
config_instance, tmp_path,
):
worker = MagicMock()
worker.total_frames = 10
worker.parent = None
worker.id = 'job-1'
mock_create_worker.return_value = worker
project_path = tmp_path / 'test_project.blend'
project_path.write_text('fake')
attrs = {
'engine_name': 'blender',
'args': {'engine': 'CYCLES'},
'name': 'Test Job',
'start_frame': 1,
'end_frame': 10,
'priority': 2,
'enable_split_jobs': False,
}
with patch('src.distributed_job_manager.RenderQueue.add_to_render_queue') as mock_add:
with patch('src.distributed_job_manager.PreviewManager.update_previews_for_job'):
result = DistributedJobManager.create_render_job(attrs, project_path)
assert result == worker
assert worker.status == RenderStatus.NOT_STARTED
mock_add.assert_called_once_with(worker, force_start=False)
@patch('src.distributed_job_manager.os.makedirs')
@patch('src.distributed_job_manager.EngineManager.create_worker')
def test_split_jobs_enabled_calls_split_async(
self, mock_create_worker, mock_makedirs, distributed_job_manager_instance,
config_instance, tmp_path,
):
worker = MagicMock()
worker.total_frames = 10
worker.parent = None
mock_create_worker.return_value = worker
project_path = tmp_path / 'test_project.blend'
project_path.write_text('fake')
attrs = {
'engine_name': 'blender',
'args': {},
'name': 'Split Job',
'start_frame': 1,
'end_frame': 10,
'priority': 2,
'enable_split_jobs': True,
}
# The forwarder passes system_os=None by default
with patch.object(distributed_job_manager_instance, '_split_into_subjobs_async') as mock_split:
with patch('src.distributed_job_manager.RenderQueue.add_to_render_queue'):
with patch('src.distributed_job_manager.PreviewManager.update_previews_for_job'):
DistributedJobManager.create_render_job(attrs, project_path)
mock_split.assert_called_once_with(worker, attrs, project_path, None)
class TestHandleSubjobUpdate:
"""Processing subjob update notifications."""
def test_updates_child_info(self, distributed_job_manager_instance):
parent_job = MagicMock()
parent_job.children = {}
subjob_data = {
'id': 'sub-1',
'hostname': 'worker-1',
'status': 'completed',
'percent_complete': 1.0,
'file_count': 5,
}
with patch('src.distributed_job_manager.download_missing_frames_from_subjob',
return_value=True):
DistributedJobManager.handle_subjob_update_notification(parent_job, subjob_data)
assert 'sub-1@worker-1' in parent_job.children
assert parent_job.children['sub-1@worker-1']['download_status'] == 'completed'
class TestFindAvailableServers:
"""Discovering remote servers."""
@patch('src.distributed_job_manager.ZeroconfServer.found_hostnames')
@patch('src.distributed_job_manager.ZeroconfServer.get_hostname_properties')
@patch('src.distributed_job_manager.RenderServerProxy')
def test_finds_matching_server(
self, mock_proxy_class, mock_get_props, mock_found_hostnames,
):
mock_found_hostnames.return_value = ['server-1.local']
mock_get_props.return_value = {'api_version': '0.1', 'system_os': 'macos'}
mock_proxy = MagicMock()
mock_proxy.is_engine_available.return_value = {
'available': True,
'hostname': 'server-1.local',
}
mock_proxy_class.return_value = mock_proxy
result = DistributedJobManager.find_available_servers('blender')
assert len(result) == 1
assert result[0]['hostname'] == 'server-1.local'
tests/test_engine_manager.py
-239
View File
@@ -1,239 +0,0 @@
from unittest.mock import MagicMock, patch
import pytest
from src.engines.engine_manager import EngineManager, EngineDownloadWorker
class TestEngineManagerSyncClass:
"""_sync_class propagates instance attrs to class level."""
def test_sync_class_propagates_engines_path(self, engine_manager_instance):
assert EngineManager.engines_path == engine_manager_instance.engines_path
def test_sync_class_noop_when_no_instance(self):
orig = EngineManager._default_instance
try:
EngineManager._default_instance = None
EngineManager.engines_path = 'original'
EngineManager._sync_class()
assert EngineManager.engines_path == 'original'
finally:
EngineManager._default_instance = orig
EngineManager._sync_class()
def test_supported_engines_returns_list(self):
engines = EngineManager.supported_engines()
assert len(engines) >= 2
class TestEngineClassMapping:
"""Mapping file extensions and names to engine classes."""
def test_engine_class_with_name_finds_blender(self, engine_manager_instance):
cls = EngineManager.engine_class_with_name('blender')
assert cls is not None
assert cls.__name__ == 'Blender'
def test_engine_class_with_name_finds_ffmpeg(self, engine_manager_instance):
cls = EngineManager.engine_class_with_name('ffmpeg')
assert cls is not None
assert cls.__name__ == 'FFMPEG'
def test_engine_class_with_name_returns_none_for_unknown(self, engine_manager_instance):
cls = EngineManager.engine_class_with_name('nonexistent')
assert cls is None
def test_engine_class_with_name_case_insensitive(self, engine_manager_instance):
cls = EngineManager.engine_class_with_name('BLENDER')
assert cls is not None
assert cls.__name__ == 'Blender'
def test_engine_class_for_project_path_no_engines_path(self, engine_manager_instance):
engine_manager_instance.engines_path = None
with pytest.raises(FileNotFoundError):
EngineManager.engine_class_for_project_path('test.blend')
class TestGetInstalledEngineData:
"""Parsing directory listings for managed engines."""
def test_get_installed_engine_data_no_path(self, engine_manager_instance):
engine_manager_instance.engines_path = None
with pytest.raises(FileNotFoundError):
EngineManager.get_installed_engine_data()
def test_get_installed_engine_data_empty_dir(self, engine_manager_instance, tmp_path):
engine_manager_instance.engines_path = str(tmp_path / 'engines')
result = EngineManager.get_installed_engine_data(ignore_system=True)
assert result == []
@patch('src.engines.engine_manager.os.listdir')
@patch('src.engines.engine_manager.os.path.isdir')
def test_parse_managed_engine_directory(
self, mock_isdir, mock_listdir, engine_manager_instance, tmp_path,
):
engine_manager_instance.engines_path = str(tmp_path / 'engines')
mock_listdir.return_value = ['blender-3.6.0-macos-arm64']
mock_isdir.return_value = True
result = EngineManager.get_installed_engine_data(ignore_system=True)
assert len(result) == 1
assert result[0]['engine'] == 'blender'
assert result[0]['version'] == '3.6.0'
assert result[0]['system_os'] == 'macos'
assert result[0]['cpu'] == 'arm64'
assert result[0]['type'] == 'managed'
@patch('src.engines.engine_manager.os.listdir')
@patch('src.engines.engine_manager.os.path.isdir')
def test_filter_by_engine_name(
self, mock_isdir, mock_listdir, engine_manager_instance, tmp_path,
):
engine_manager_instance.engines_path = str(tmp_path / 'engines')
mock_listdir.return_value = ['blender-3.6.0-macos-arm64', 'ffmpeg-6.0-macos-arm64']
mock_isdir.return_value = True
blender_only = EngineManager.get_installed_engine_data(
filter_name='blender', ignore_system=True)
assert len(blender_only) == 1
assert blender_only[0]['engine'] == 'blender'
class TestNewestInstalledEngineData:
"""Filtering by system and CPU."""
@patch('src.engines.engine_manager.current_system_os')
@patch('src.engines.engine_manager.current_system_cpu')
def test_newest_filters_by_system_and_cpu(
self, mock_cpu, mock_os, engine_manager_instance, tmp_path,
):
mock_os.return_value = 'macos'
mock_cpu.return_value = 'arm64'
engine_manager_instance.engines_path = str(tmp_path / 'engines')
dirs = [
'blender-3.6.0-macos-arm64',
'blender-4.0.0-linux-x86_64',
'blender-4.1.0-macos-arm64',
]
with (patch('src.engines.engine_manager.os.listdir', return_value=dirs),
patch('src.engines.engine_manager.os.path.isdir', return_value=True)):
result = EngineManager.newest_installed_engine_data(
'blender', ignore_system=True)
assert result['version'] == '4.1.0'
assert result['system_os'] == 'macos'
assert result['cpu'] == 'arm64'
def test_newest_returns_empty_on_no_match(self, engine_manager_instance, tmp_path):
engine_manager_instance.engines_path = str(tmp_path / 'engines')
result = EngineManager.newest_installed_engine_data('nonexistent')
assert result == []
class TestIsVersionInstalled:
"""Checking whether a specific version is installed."""
@patch('src.engines.engine_manager.current_system_os')
@patch('src.engines.engine_manager.current_system_cpu')
def test_finds_matching_version(
self, mock_cpu, mock_os, engine_manager_instance, tmp_path,
):
mock_os.return_value = 'macos'
mock_cpu.return_value = 'arm64'
engine_manager_instance.engines_path = str(tmp_path / 'engines')
dirs = ['blender-3.6.0-macos-arm64', 'blender-4.1.0-macos-arm64']
with (patch('src.engines.engine_manager.os.listdir', return_value=dirs),
patch('src.engines.engine_manager.os.path.isdir', return_value=True)):
result = EngineManager.is_version_installed('blender', '3.6.0')
assert result is not False
@patch('src.engines.engine_manager.current_system_os')
@patch('src.engines.engine_manager.current_system_cpu')
def test_returns_false_on_no_match(
self, mock_cpu, mock_os, engine_manager_instance, tmp_path,
):
mock_os.return_value = 'macos'
mock_cpu.return_value = 'arm64'
engine_manager_instance.engines_path = str(tmp_path / 'engines')
result = EngineManager.is_version_installed('blender', '99.0.0')
assert result is False
class TestDeleteEngineDownload:
"""Deleting a managed engine directory."""
@patch('src.engines.engine_manager.shutil.rmtree')
@patch('src.engines.engine_manager.current_system_os', return_value='macos')
@patch('src.engines.engine_manager.current_system_cpu', return_value='arm64')
def test_delete_managed_engine(
self, mock_cpu, mock_os, mock_rmtree, engine_manager_instance, tmp_path,
):
engine_manager_instance.engines_path = str(tmp_path / 'engines')
engines_dir = tmp_path / 'engines' / 'blender-3.6.0-macos-arm64'
engines_dir.mkdir(parents=True)
(engines_dir / 'Blender').write_text('fake binary')
result = EngineManager.delete_engine_download('blender', '3.6.0')
assert result is True
mock_rmtree.assert_called_once_with(str(engines_dir), ignore_errors=False)
def test_delete_nonexistent_engine(self, engine_manager_instance, tmp_path):
engine_manager_instance.engines_path = str(tmp_path / 'engines')
result = EngineManager.delete_engine_download('nonexistent', '1.0.0')
assert result is False
class TestActiveDownloads:
"""Background download tracking."""
def test_active_downloads_empty_initially(self, engine_manager_instance):
assert EngineManager.active_downloads() == []
def test_download_tasks_tracked(self, engine_manager_instance):
task = MagicMock(spec=EngineDownloadWorker)
task.is_alive.return_value = True
task.name = 'blender-4.0.0-macos-arm64'
engine_manager_instance.download_tasks.append(task)
active = EngineManager.active_downloads()
assert len(active) == 1
class TestCreateWorker:
"""Creating worker instances for render jobs."""
def test_create_worker_raises_when_no_engines_installed(self, engine_manager_instance, tmp_path):
engine_manager_instance.engines_path = str(tmp_path / 'engines')
with patch.object(engine_manager_instance, '_get_installed_engine_data', return_value=[]):
with pytest.raises(FileNotFoundError, match='Cannot find any installed'):
EngineManager.create_worker(
'blender',
input_path=tmp_path / 'test.blend',
output_path=tmp_path / 'output',
)
def test_create_worker_raises_for_unknown_engine(self, engine_manager_instance):
with pytest.raises(AttributeError):
EngineManager.create_worker(
'nonexistent',
input_path='/tmp/test.blend',
output_path='/tmp/output',
)
class TestDownloadableEngines:
"""Engines with a downloader."""
def test_downloadable_engines_returns_list(self, engine_manager_instance):
engines = EngineManager.downloadable_engines()
assert isinstance(engines, list)
tests/test_preview_manager.py
-169
View File
@@ -1,169 +0,0 @@
from unittest.mock import MagicMock, patch
from src.api.preview_manager import PreviewManager
def make_mock_job(job_id='test-job-1', input_path='/tmp/test.blend',
file_list=None, **kwargs):
job = MagicMock()
job.id = job_id
job.input_path = input_path
job.file_list.return_value = file_list or []
for k, v in kwargs.items():
setattr(job, k, v)
return job
class TestPreviewManagerDefaults:
"""Default state."""
def test_storage_path_none_initially(self, preview_manager_instance):
assert preview_manager_instance.storage_path is not None
def test_running_jobs_empty_initially(self, preview_manager_instance):
assert preview_manager_instance._running_jobs == {}
class TestGeneratePreviewWorker:
"""Core preview generation logic."""
def test_skips_when_no_supported_files(self, preview_manager_instance, tmp_path):
preview_manager_instance.storage_path = str(tmp_path)
job = make_mock_job(file_list=[str(tmp_path / 'output.txt')])
with patch.object(preview_manager_instance, '_generate_job_preview_worker') as mock_gen:
preview_manager_instance._update_previews_for_job(job)
mock_gen.assert_called_once_with(job, False)
def test_uses_input_path_when_no_file_list(self, preview_manager_instance, tmp_path):
"""When file_list is empty, falls back to input_path."""
job = make_mock_job(input_path=str(tmp_path / 'output.mp4'))
with patch.object(preview_manager_instance, '_generate_job_preview_worker') as mock_gen:
preview_manager_instance._update_previews_for_job(job)
mock_gen.assert_called_once()
def test_generate_preview_checks_existing_files(self, preview_manager_instance, tmp_path):
preview_manager_instance.storage_path = str(tmp_path)
job = make_mock_job(input_path=str(tmp_path / 'test.jpg'))
with patch('src.api.preview_manager.save_first_frame') as mock_save:
with patch('src.api.preview_manager.os.path.exists', return_value=False):
preview_manager_instance._generate_job_preview_worker(job)
# No file_list → falls back to input_path → label is "input"
expected_img = str(tmp_path / f'{job.id}-input-480.jpg')
mock_save.assert_called_once_with(
source_path=job.input_path,
dest_path=expected_img,
max_width=480,
)
class TestUpdatePreviewsForJob:
"""Dispatch of preview generation."""
def test_starts_new_thread(self, preview_manager_instance, tmp_path):
preview_manager_instance.storage_path = str(tmp_path)
job = make_mock_job(file_list=[str(tmp_path / 'test.mp4')])
with patch('src.api.preview_manager.threading.Thread') as mock_thread:
thread_instance = MagicMock()
mock_thread.return_value = thread_instance
preview_manager_instance._update_previews_for_job(job)
mock_thread.assert_called_once()
thread_instance.start.assert_called_once()
assert preview_manager_instance._running_jobs[job.id] == thread_instance
def test_reuses_existing_thread(self, preview_manager_instance, tmp_path):
preview_manager_instance.storage_path = str(tmp_path)
job = make_mock_job()
existing_thread = MagicMock()
existing_thread.is_alive.return_value = True
preview_manager_instance._running_jobs[job.id] = existing_thread
with patch('src.api.preview_manager.threading.Thread') as mock_thread:
preview_manager_instance._update_previews_for_job(job)
mock_thread.assert_not_called()
def test_join_when_wait_requested(self, preview_manager_instance, tmp_path):
preview_manager_instance.storage_path = str(tmp_path)
job = make_mock_job()
with patch('src.api.preview_manager.threading.Thread') as mock_thread:
thread_instance = MagicMock()
mock_thread.return_value = thread_instance
preview_manager_instance._update_previews_for_job(
job, wait_until_completion=True, timeout=30,
)
thread_instance.join.assert_called_once_with(timeout=30)
class TestGetPreviewsForJob:
"""Reading preview files."""
def test_returns_empty_when_no_previews(self, preview_manager_instance, tmp_path):
preview_manager_instance.storage_path = str(tmp_path)
job = make_mock_job()
result = preview_manager_instance._get_previews_for_job(job)
assert result == {}
def test_returns_preview_files(self, preview_manager_instance, tmp_path):
preview_manager_instance.storage_path = str(tmp_path)
job = make_mock_job(job_id='abc')
# Create a fake preview file
(tmp_path / 'abc-output-480.jpg').write_text('preview')
result = preview_manager_instance._get_previews_for_job(job)
assert 'output' in result
assert len(result['output']) == 1
assert result['output'][0]['kind'] == 'image'
class TestDeletePreviewsForJob:
"""Cleaning up preview files."""
def test_deletes_existing_previews(self, preview_manager_instance, tmp_path):
preview_manager_instance.storage_path = str(tmp_path)
job = make_mock_job(job_id='abc')
preview_file = tmp_path / 'abc-output-480.jpg'
preview_file.write_text('preview')
preview_manager_instance._delete_previews_for_job(job)
assert not preview_file.exists()
def test_no_error_when_no_previews(self, preview_manager_instance, tmp_path):
preview_manager_instance.storage_path = str(tmp_path)
job = make_mock_job()
preview_manager_instance._delete_previews_for_job(job)
class TestForwarders:
"""Classmethod forwarders delegate to instance."""
def test_update_previews_for_job_forwarder(self, preview_manager_instance):
job = make_mock_job()
with patch.object(preview_manager_instance, '_update_previews_for_job') as mock_method:
PreviewManager.update_previews_for_job(job)
mock_method.assert_called_once_with(job, False, False, None)
def test_get_previews_for_job_forwarder(self, preview_manager_instance):
job = make_mock_job()
with patch.object(preview_manager_instance, '_get_previews_for_job',
return_value={'input': []}) as mock_method:
result = PreviewManager.get_previews_for_job(job)
assert result == {'input': []}
mock_method.assert_called_once_with(job)
def test_delete_previews_for_job_forwarder(self, preview_manager_instance):
job = make_mock_job()
with patch.object(preview_manager_instance, '_delete_previews_for_job') as mock_method:
PreviewManager.delete_previews_for_job(job)
mock_method.assert_called_once_with(job)
tests/test_render_queue.py
-307
View File
@@ -1,307 +0,0 @@
from unittest.mock import MagicMock, patch
from datetime import datetime
import pytest
from src.render_queue import RenderQueue, JobNotFoundError
from src.utilities.status_utils import RenderStatus
def make_mock_job(job_id='test-1', status=RenderStatus.NOT_STARTED,
engine_name='blender', priority=2, **kwargs):
job = MagicMock()
job.id = job_id
job.status = status
job.engine_name = engine_name
job.priority = priority
job.name = kwargs.get('name', 'Test Job')
job.scheduled_start = kwargs.get('scheduled_start', None)
for k, v in kwargs.items():
setattr(job, k, v)
return job
class TestRenderQueueDefaults:
"""Default state."""
def test_init_sets_empty_queue(self, render_queue_instance):
assert render_queue_instance.job_queue == []
def test_init_sets_max_instances(self, render_queue_instance):
assert render_queue_instance.maximum_renderer_instances == {'blender': 1, 'aerender': 1, 'ffmpeg': 4}
def test_init_is_not_running(self, render_queue_instance):
assert render_queue_instance.is_running is False
class TestAllJobs:
"""Returning all jobs."""
def test_all_jobs_empty_initially(self, render_queue_instance):
assert RenderQueue.all_jobs() == []
def test_all_jobs_returns_job_list(self, render_queue_instance):
job = make_mock_job()
render_queue_instance.job_queue.append(job)
assert len(RenderQueue.all_jobs()) == 1
assert RenderQueue.all_jobs()[0] == job
class TestJobsWithStatus:
"""Filtering jobs by status."""
def test_jobs_with_status_returns_matching(self, render_queue_instance):
running = make_mock_job('job-1', RenderStatus.RUNNING)
pending = make_mock_job('job-2', RenderStatus.NOT_STARTED)
render_queue_instance.job_queue.extend([running, pending])
result = RenderQueue.jobs_with_status(RenderStatus.RUNNING)
assert len(result) == 1
assert result[0].id == 'job-1'
def test_jobs_with_status_empty_when_no_match(self, render_queue_instance):
job = make_mock_job('job-1', RenderStatus.COMPLETED)
render_queue_instance.job_queue.append(job)
result = RenderQueue.jobs_with_status(RenderStatus.RUNNING)
assert result == []
def test_jobs_with_status_sorts_by_priority(self, render_queue_instance):
high = make_mock_job('job-1', RenderStatus.NOT_STARTED, priority=1)
low = make_mock_job('job-2', RenderStatus.NOT_STARTED, priority=5)
render_queue_instance.job_queue.extend([low, high])
result = RenderQueue.jobs_with_status(RenderStatus.NOT_STARTED, priority_sorted=True)
assert result[0].id == 'job-1'
class TestRunningAndPending:
"""Convenience methods for running/pending."""
def test_running_jobs(self, render_queue_instance):
running = make_mock_job('job-1', RenderStatus.RUNNING)
pending = make_mock_job('job-2', RenderStatus.NOT_STARTED)
render_queue_instance.job_queue.extend([running, pending])
result = RenderQueue.running_jobs()
assert len(result) == 1
assert result[0].id == 'job-1'
def test_pending_jobs_includes_not_started_and_scheduled(self, render_queue_instance):
ns = make_mock_job('job-1', RenderStatus.NOT_STARTED)
sched = make_mock_job('job-2', RenderStatus.SCHEDULED)
running = make_mock_job('job-3', RenderStatus.RUNNING)
render_queue_instance.job_queue.extend([ns, sched, running])
result = RenderQueue.pending_jobs()
assert len(result) == 2
class TestJobWithId:
"""Lookup by job ID."""
def test_finds_job(self, render_queue_instance):
job = make_mock_job('abc-123')
render_queue_instance.job_queue.append(job)
found = RenderQueue.job_with_id('abc-123')
assert found == job
def test_raises_when_not_found(self, render_queue_instance):
with pytest.raises(JobNotFoundError, match='abc-123'):
RenderQueue.job_with_id('abc-123')
def test_returns_none_when_not_found_with_none_ok(self, render_queue_instance):
result = RenderQueue.job_with_id('abc-123', none_ok=True)
assert result is None
class TestJobCounts:
"""Counting jobs by status."""
def test_job_counts_returns_all_statuses(self, render_queue_instance):
render_queue_instance.job_queue.extend([
make_mock_job('j1', RenderStatus.RUNNING),
make_mock_job('j2', RenderStatus.COMPLETED),
make_mock_job('j3', RenderStatus.NOT_STARTED),
])
counts = RenderQueue.job_counts()
assert counts[RenderStatus.RUNNING.value] == 1
assert counts[RenderStatus.COMPLETED.value] == 1
assert counts[RenderStatus.NOT_STARTED.value] == 1
assert counts[RenderStatus.ERROR.value] == 0
assert counts[RenderStatus.CANCELLED.value] == 0
class TestIsAvailableForJob:
"""Renderer availability checks."""
def test_available_when_no_running_jobs(self, render_queue_instance):
assert RenderQueue.is_available_for_job('blender') is True
def test_not_available_when_maxed_out(self, render_queue_instance):
render_queue_instance.job_queue.append(
make_mock_job('j1', RenderStatus.RUNNING, engine_name='blender')
)
assert RenderQueue.is_available_for_job('blender') is False
def test_available_for_different_renderer(self, render_queue_instance):
render_queue_instance.job_queue.append(
make_mock_job('j1', RenderStatus.RUNNING, engine_name='blender')
)
assert RenderQueue.is_available_for_job('ffmpeg') is True
def test_blocked_by_higher_priority(self, render_queue_instance):
render_queue_instance.job_queue.append(
make_mock_job('j1', RenderStatus.RUNNING, engine_name='blender', priority=0)
)
assert RenderQueue.is_available_for_job('blender', priority=2) is False
def test_ffmpeg_allows_multiple_instances(self, render_queue_instance):
for i in range(4):
render_queue_instance.job_queue.append(
make_mock_job(f'j{i}', RenderStatus.RUNNING, engine_name='ffmpeg')
)
assert RenderQueue.is_available_for_job('ffmpeg') is False
class TestAddToRenderQueue:
"""Adding jobs."""
def test_add_job_appends_to_queue(self, render_queue_instance):
render_queue_instance.session = MagicMock()
job = make_mock_job()
RenderQueue.add_to_render_queue(job)
assert job in render_queue_instance.job_queue
def test_add_job_saves_state(self, render_queue_instance):
render_queue_instance.session = MagicMock()
job = make_mock_job()
RenderQueue.add_to_render_queue(job)
render_queue_instance.session.add.assert_called_once_with(job)
def test_add_job_force_start_calls_start(self, render_queue_instance):
render_queue_instance.session = MagicMock()
job = make_mock_job(status=RenderStatus.NOT_STARTED)
with patch.object(render_queue_instance, '_start_job') as mock_start:
RenderQueue.add_to_render_queue(job, force_start=True)
mock_start.assert_called_once_with(job)
def test_add_job_force_start_skips_completed(self, render_queue_instance):
render_queue_instance.session = MagicMock()
job = make_mock_job(status=RenderStatus.COMPLETED)
with patch.object(render_queue_instance, '_start_job') as mock_start:
RenderQueue.add_to_render_queue(job, force_start=True)
mock_start.assert_not_called()
def test_add_job_evaluates_queue_when_running(self, render_queue_instance):
render_queue_instance.session = MagicMock()
render_queue_instance.is_running = True
job = make_mock_job()
with patch.object(render_queue_instance, '_evaluate_queue') as mock_eval:
RenderQueue.add_to_render_queue(job)
mock_eval.assert_called_once()
class TestStartCancelDelete:
"""Job lifecycle."""
def test_start_job_calls_job_start(self, render_queue_instance):
job = make_mock_job()
with patch.object(render_queue_instance, '_save_state'):
RenderQueue.start_job(job)
job.start.assert_called_once()
def test_cancel_job_calls_job_stop(self, render_queue_instance):
job = make_mock_job()
job.stop.return_value = None
job.status = RenderStatus.CANCELLED
result = RenderQueue.cancel_job(job)
assert result is True
job.stop.assert_called_once()
def test_delete_job_removes_from_queue(self, render_queue_instance):
job = make_mock_job()
render_queue_instance.job_queue.append(job)
render_queue_instance.session = MagicMock()
result = RenderQueue.delete_job(job)
assert result is True
assert job not in render_queue_instance.job_queue
render_queue_instance.session.delete.assert_called_once_with(job)
class TestStartStop:
"""Starting and stopping the queue evaluation loop."""
def test_start_sets_running_and_evaluates(self, render_queue_instance):
with patch.object(render_queue_instance, '_evaluate_queue') as mock_eval:
RenderQueue.start()
assert render_queue_instance.is_running is True
mock_eval.assert_called_once()
def test_stop_clears_running(self, render_queue_instance):
render_queue_instance.is_running = True
RenderQueue.stop()
assert render_queue_instance.is_running is False
class TestEvaluateQueue:
"""Queue evaluation dispatches jobs."""
def test_evaluate_starts_not_started_jobs(self, render_queue_instance):
job = make_mock_job()
render_queue_instance.job_queue.append(job)
with patch.object(render_queue_instance, '_start_job') as mock_start:
with patch.object(render_queue_instance, '_save_state'):
RenderQueue.evaluate_queue()
mock_start.assert_called_once_with(job)
def test_evaluate_respects_max_instances(self, render_queue_instance):
# One already running, only 1 slot for blender
render_queue_instance.job_queue.append(
make_mock_job('running-1', RenderStatus.RUNNING, engine_name='blender')
)
waiting = make_mock_job('waiting-1', RenderStatus.NOT_STARTED, engine_name='blender')
render_queue_instance.job_queue.append(waiting)
with patch.object(render_queue_instance, '_start_job') as mock_start:
with patch.object(render_queue_instance, '_save_state'):
RenderQueue.evaluate_queue()
mock_start.assert_not_called()
def test_evaluate_starts_scheduled_jobs(self, render_queue_instance):
past = datetime(2020, 1, 1)
job = make_mock_job(status=RenderStatus.SCHEDULED, scheduled_start=past)
render_queue_instance.job_queue.append(job)
with patch.object(render_queue_instance, '_start_job') as mock_start:
with patch.object(render_queue_instance, '_save_state'):
RenderQueue.evaluate_queue()
mock_start.assert_called_once_with(job)
class TestClearHistory:
"""Clearing completed/cancelled/error jobs."""
def test_clear_history_removes_finished_jobs(self, render_queue_instance):
render_queue_instance.session = MagicMock()
jobs = [
make_mock_job('c', RenderStatus.COMPLETED),
make_mock_job('e', RenderStatus.ERROR),
make_mock_job('x', RenderStatus.CANCELLED),
make_mock_job('r', RenderStatus.RUNNING),
]
render_queue_instance.job_queue.extend(jobs)
RenderQueue.clear_history()
assert len(render_queue_instance.job_queue) == 1
assert render_queue_instance.job_queue[0].id == 'r'
tests/test_zeroconf_server.py
-159
View File
@@ -1,159 +0,0 @@
from unittest.mock import MagicMock, patch
import pytest
from src.utilities.zeroconf_server import ZeroconfServer
class TestConfigure:
"""Configuring service parameters."""
def test_configure_sets_attributes(self, zeroconf_server_instance):
with patch('socket.gethostbyname', return_value='192.168.1.1'):
ZeroconfServer.configure('_zordon._tcp.local.', 'test-server', 8080)
assert zeroconf_server_instance.service_type == '_zordon._tcp.local.'
assert zeroconf_server_instance.server_name == 'test-server'
assert zeroconf_server_instance.server_port == 8080
def test_configure_calls_sync_class(self, zeroconf_server_instance):
with patch('socket.gethostbyname', return_value='192.168.1.1'):
ZeroconfServer.configure('_zordon._tcp.local.', 'test-server', 8080)
assert ZeroconfServer.service_type == '_zordon._tcp.local.'
assert ZeroconfServer.server_port == 8080
def test_configure_stops_on_gaierror(self, zeroconf_server_instance):
import socket
with patch('socket.gethostbyname', side_effect=socket.gaierror):
with patch.object(zeroconf_server_instance, '_stop') as mock_stop:
ZeroconfServer.configure('_zordon._tcp.local.', 'test-server', 8080)
mock_stop.assert_called_once()
class TestSyncClass:
"""_sync_class propagates instance attrs to class level."""
def test_sync_class_propagates_all_attrs(self, zeroconf_server_instance):
zeroconf_server_instance.service_type = '_test._tcp.'
zeroconf_server_instance.server_name = 'foo'
zeroconf_server_instance.server_port = 9999
zeroconf_server_instance.server_ip = '10.0.0.1'
zeroconf_server_instance.properties = {'key': 'val'}
ZeroconfServer._sync_class()
assert ZeroconfServer.service_type == '_test._tcp.'
assert ZeroconfServer.server_name == 'foo'
assert ZeroconfServer.server_port == 9999
assert ZeroconfServer.server_ip == '10.0.0.1'
assert ZeroconfServer.properties == {'key': 'val'}
class TestStartStop:
"""Service lifecycle."""
def test_start_raises_without_configure(self, zeroconf_server_instance):
with pytest.raises(RuntimeError, match='configure'):
ZeroconfServer.start()
@patch('src.utilities.zeroconf_server.ServiceBrowser')
def test_start_listen_only_skips_register(self, mock_browser, zeroconf_server_instance):
with patch('socket.gethostbyname', return_value='192.168.1.1'):
ZeroconfServer.configure('_zordon._tcp.local.', 'test', 8080)
with patch.object(zeroconf_server_instance, '_register_service') as mock_register:
ZeroconfServer.start(listen_only=True)
mock_register.assert_not_called()
@patch('src.utilities.zeroconf_server.ServiceBrowser')
def test_start_registers_service(self, mock_browser, zeroconf_server_instance):
with patch('socket.gethostbyname', return_value='192.168.1.1'):
ZeroconfServer.configure('_zordon._tcp.local.', 'test', 8080)
with patch.object(zeroconf_server_instance, '_register_service') as mock_register:
ZeroconfServer.start(listen_only=False)
mock_register.assert_called_once()
def test_stop_unregisters_and_closes(self, zeroconf_server_instance):
zeroconf_server_instance.service_info = MagicMock()
with patch.object(zeroconf_server_instance, '_unregister_service') as mock_unreg:
with patch.object(zeroconf_server_instance.zeroconf, 'close') as mock_close:
ZeroconfServer.stop()
mock_unreg.assert_called_once()
mock_close.assert_called_once()
class TestRegisterService:
"""Service registration with Zeroconf."""
@patch('src.utilities.zeroconf_server.ServiceInfo')
@patch('socket.gethostbyname')
def test_registers_service_info(self, mock_gethostbyname, mock_service_info,
zeroconf_server_instance):
mock_gethostbyname.return_value = '192.168.1.1'
zeroconf_server_instance.service_type = '_zordon._tcp.local.'
zeroconf_server_instance.server_name = 'test'
zeroconf_server_instance.server_port = 8080
zeroconf_server_instance.properties = {}
# Replace real Zeroconf with a mock so we don't actually register
zeroconf_server_instance.zeroconf = MagicMock()
mock_info = MagicMock()
mock_service_info.return_value = mock_info
with patch('socket.inet_aton', return_value=b'\xc0\xa8\x01\x01'):
zeroconf_server_instance._register_service()
zeroconf_server_instance.zeroconf.register_service.assert_called_once_with(mock_info)
assert zeroconf_server_instance.service_info == mock_info
class TestFoundHostnames:
"""Discovery cache."""
def test_found_hostnames_empty_initially(self, zeroconf_server_instance):
result = zeroconf_server_instance._found_hostnames()
assert result == []
@patch('socket.gethostname', return_value='my-machine')
def test_hostnames_sorted_with_local_first(self, mock_hostname, zeroconf_server_instance):
zeroconf_server_instance.client_cache = {
'other-machine': MagicMock(),
'my-machine': MagicMock(),
}
result = zeroconf_server_instance._found_hostnames()
# sort_key returns False (0) for local → sorted first
assert result[0] == 'my-machine'
def test_get_hostname_properties_returns_decoded(self, zeroconf_server_instance):
info = MagicMock()
info.properties = {b'key': b'value', b'num': b'42'}
zeroconf_server_instance.client_cache['server-1'] = info
result = zeroconf_server_instance._get_hostname_properties('server-1')
assert result == {'key': 'value', 'num': '42'}
def test_get_hostname_properties_returns_empty_for_unknown(self, zeroconf_server_instance):
result = zeroconf_server_instance._get_hostname_properties('unknown')
assert result == {}
class TestForwarders:
"""Classmethod forwarders delegate to instance."""
def test_found_hostnames_forwarder(self, zeroconf_server_instance):
zeroconf_server_instance.client_cache['svr'] = MagicMock()
result = ZeroconfServer.found_hostnames()
assert 'svr' in result
def test_get_hostname_properties_forwarder(self, zeroconf_server_instance):
info = MagicMock()
info.properties = {}
zeroconf_server_instance.client_cache['svr'] = info
result = ZeroconfServer.get_hostname_properties('svr')
assert result == {}