brett/Zordon
1
0
Fork 0
mirror of https://github.com/blw1138/Zordon.git synced 2026-06-10 14:09:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

Windows path fixes (#129)

Browse Source 
* Change uses of os.path to use Pathlib

* Add return types and type hints

* Add more docstrings

* Add missing import to api_server
This commit is contained in:
Brett
2026-01-18 00:18:43 -06:00
committed by GitHub
parent ee9f44e4c4
commit 74dce5cc3d
15 changed files with 536 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/api_server.py
+54 -46
View File
@@ -2,14 +2,15 @@
import concurrent.futures
import json
import logging
import os
import pathlib
import shutil
import socket
import ssl
import tempfile
import time
import traceback
from datetime import datetime
from pathlib import Path
from typing import Dict, Any, Optional
import cpuinfo
import psutil
@@ -23,7 +24,7 @@ from src.distributed_job_manager import DistributedJobManager
from src.engines.engine_manager import EngineManager
from src.render_queue import RenderQueue, JobNotFoundError
from src.utilities.config import Config
from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu, \
from src.utilities.misc_helper import current_system_os, current_system_cpu, \
current_system_os_version, num_to_alphanumeric, get_gpu_info
from src.utilities.status_utils import string_to_status
from src.version import APP_VERSION
@@ -34,7 +35,7 @@ ssl._create_default_https_context = ssl._create_unverified_context # disable SS
API_VERSION = "0.1"
def start_api_server(hostname=None):
def start_api_server(hostname: Optional[str] = None) -> None:
# get hostname
if not hostname:
@@ -44,7 +45,7 @@ def start_api_server(hostname=None):
# load flask settings
server.config['HOSTNAME'] = hostname
server.config['PORT'] = int(Config.port_number)
server.config['UPLOAD_FOLDER'] = system_safe_path(os.path.expanduser(Config.upload_folder))
server.config['UPLOAD_FOLDER'] = str(Path(Config.upload_folder).expanduser())
server.config['MAX_CONTENT_PATH'] = Config.max_content_path
server.config['enable_split_jobs'] = Config.enable_split_jobs
@@ -65,7 +66,7 @@ def start_api_server(hostname=None):
# --------------------------------------------
@server.get('/api/jobs')
def jobs_json():
def jobs_json() -> Dict[str, Any]:
"""Retrieves all jobs from the render queue in JSON format.
This endpoint fetches all jobs currently in the render queue, converts them to JSON format,
@@ -134,9 +135,9 @@ def get_job_logs(job_id):
Response: The log file's content as plain text, or an empty response if the log file is not found.
"""
found_job = RenderQueue.job_with_id(job_id)
log_path = system_safe_path(found_job.log_path())
log_path = Path(found_job.log_path())
log_data = None
if log_path and os.path.exists(log_path):
if log_path and log_path.exists():
with open(log_path) as file:
log_data = file.read()
return Response(log_data, mimetype='text/plain')
@@ -144,20 +145,21 @@ def get_job_logs(job_id):
@server.get('/api/job/<job_id>/file_list')
def get_file_list(job_id):
return [os.path.basename(x) for x in RenderQueue.job_with_id(job_id).file_list()]
return [Path(p).name for p in RenderQueue.job_with_id(job_id).file_list()]
@server.route('/api/job/<job_id>/download')
def download_requested_file(job_id):
requested_filename = request.args.get('filename')
requested_filename = request.args.get("filename")
if not requested_filename:
return 'Filename required', 400
return "Filename required", 400
found_job = RenderQueue.job_with_id(job_id)
for job_filename in found_job.file_list():
if os.path.basename(job_filename).lower() == requested_filename.lower():
return send_file(job_filename, as_attachment=True, )
for job_file in found_job.file_list():
p = Path(job_file)
if p.name.lower() == requested_filename.lower():
return send_file(str(p), as_attachment=True)
return f"File '{requested_filename}' not found", 404
@@ -168,26 +170,27 @@ def download_all_files(job_id):
@after_this_request
def clear_zip(response):
if zip_filename and os.path.exists(zip_filename):
if zip_filename and zip_filename.exists():
try:
os.remove(zip_filename)
zip_filename.unlink()
except Exception as e:
logger.warning(f"Error removing zip file '{zip_filename}': {e}")
return response
found_job = RenderQueue.job_with_id(job_id)
output_dir = os.path.dirname(found_job.output_path)
if os.path.exists(output_dir):
from zipfile import ZipFile
zip_filename = system_safe_path(os.path.join(tempfile.gettempdir(),
pathlib.Path(found_job.input_path).stem + '.zip'))
with ZipFile(zip_filename, 'w') as zipObj:
for f in os.listdir(output_dir):
zipObj.write(filename=system_safe_path(os.path.join(output_dir, f)),
arcname=os.path.basename(f))
return send_file(zip_filename, mimetype="zip", as_attachment=True, )
else:
return f'Cannot find project files for job {job_id}', 500
output_dir = Path(found_job.output_path).parent
if not output_dir.exists():
return f"Cannot find project files for job {job_id}", 500
zip_filename = Path(tempfile.gettempdir()) / f"{Path(found_job.input_path).stem}.zip"
from zipfile import ZipFile
with ZipFile(zip_filename, "w") as zipObj:
for f in output_dir.iterdir():
if f.is_file():
zipObj.write(f, arcname=f.name)
return send_file(str(zip_filename), mimetype="zip", as_attachment=True)
# --------------------------------------------
@@ -195,8 +198,8 @@ def download_all_files(job_id):
# --------------------------------------------
@server.get('/api/presets')
def presets():
presets_path = system_safe_path('config/presets.yaml')
def presets() -> Dict[str, Any]:
presets_path = Path('config/presets.yaml')
with open(presets_path) as f:
loaded_presets = yaml.load(f, Loader=yaml.FullLoader)
return loaded_presets
@@ -292,6 +295,7 @@ def add_job_handler():
err_msg = f"Error processing job data: {e}"
return err_msg, 400
except Exception as e:
traceback.print_exception(e)
err_msg = f"Unknown error processing data: {e}"
return err_msg, 500
@@ -311,9 +315,9 @@ def add_job_handler():
# Save notes to .txt
if processed_job_data.get("notes"):
parent_dir = os.path.dirname(os.path.dirname(loaded_project_local_path))
parent_dir = Path(loaded_project_local_path).parent.parent
notes_name = processed_job_data['name'] + "-notes.txt"
with open(os.path.join(parent_dir, notes_name), "w") as f:
with (Path(parent_dir) / notes_name).open("w") as f:
f.write(processed_job_data["notes"])
return [x.json() for x in created_jobs]
except Exception as e:
@@ -338,13 +342,18 @@ def cancel_job(job_id):
@server.route('/api/job/<job_id>/delete', methods=['POST', 'GET'])
def delete_job(job_id):
try:
if not request.args.get('confirm', False):
return 'Confirmation required to delete job', 400
if not request.args.get("confirm", False):
return "Confirmation required to delete job", 400
# Check if we can remove the 'output' directory
found_job = RenderQueue.job_with_id(job_id)
project_dir = os.path.dirname(os.path.dirname(found_job.input_path))
output_dir = os.path.dirname(found_job.output_path)
input_path = Path(found_job.input_path)
output_path = Path(found_job.output_path)
upload_root = Path(server.config["UPLOAD_FOLDER"])
project_dir = input_path.parent.parent
output_dir = output_path.parent
found_job.stop()
try:
@@ -352,25 +361,24 @@ def delete_job(job_id):
except Exception as e:
logger.error(f"Error deleting previews for {found_job}: {e}")
# finally delete the job
RenderQueue.delete_job(found_job)
# delete the output_dir
if server.config['UPLOAD_FOLDER'] in output_dir and os.path.exists(output_dir):
# Delete output directory if we own it
if output_dir.exists() and output_dir.is_relative_to(upload_root):
shutil.rmtree(output_dir)
# See if we own the project_dir (i.e. was it uploaded) - if so delete the directory
# Delete project directory if we own it and it's unused
try:
if server.config['UPLOAD_FOLDER'] in project_dir and os.path.exists(project_dir):
# check to see if any other projects are sharing the same project file
project_dir_files = [f for f in os.listdir(project_dir) if not f.startswith('.')]
if len(project_dir_files) == 0 or (len(project_dir_files) == 1 and 'source' in project_dir_files[0]):
if project_dir.exists() and project_dir.is_relative_to(upload_root):
project_dir_files = [p for p in project_dir.iterdir() if not p.name.startswith(".")]
if not project_dir_files or (len(project_dir_files) == 1 and "source" in project_dir_files[0].name):
logger.info(f"Removing project directory: {project_dir}")
shutil.rmtree(project_dir)
except Exception as e:
logger.error(f"Error removing project files: {e}")
return "Job deleted", 200
except Exception as e:
logger.error(f"Error deleting job: {e}")
return f"Error deleting job: {e}", 500
src/api/job_import_handler.py
+33 -26
View File
@@ -7,6 +7,7 @@ import zipfile
from datetime import datetime
import requests
from pathlib import Path
from tqdm import tqdm
from werkzeug.utils import secure_filename
@@ -16,7 +17,7 @@ logger = logging.getLogger()
class JobImportHandler:
@classmethod
def validate_job_data(cls, new_job_data, upload_directory, uploaded_file=None):
def validate_job_data(cls, new_job_data: dict, upload_directory: Path, uploaded_file=None) -> dict:
loaded_project_local_path = None
# check for required keys
@@ -44,27 +45,34 @@ class JobImportHandler:
# Prepare the local filepath
cleaned_path_name = job_name.replace(' ', '-')
job_dir = os.path.join(upload_directory, '-'.join(
[cleaned_path_name, engine_name, datetime.now().strftime("%Y.%m.%d_%H.%M.%S")]))
timestamp = datetime.now().strftime("%Y.%m.%d_%H.%M.%S")
folder_name = f"{cleaned_path_name}-{engine_name}-{timestamp}"
job_dir = Path(upload_directory) / folder_name
os.makedirs(job_dir, exist_ok=True)
project_source_dir = os.path.join(job_dir, 'source')
project_source_dir = Path(job_dir) / 'source'
os.makedirs(project_source_dir, exist_ok=True)
# Move projects to their work directories
if uploaded_file and uploaded_file.filename:
loaded_project_local_path = os.path.join(project_source_dir, secure_filename(uploaded_file.filename))
uploaded_file.save(loaded_project_local_path)
logger.info(f"Transfer complete for {loaded_project_local_path.split(upload_directory)[-1]}")
elif project_url:
loaded_project_local_path = os.path.join(project_source_dir, referred_name)
shutil.move(downloaded_file_url, loaded_project_local_path)
logger.info(f"Download complete for {loaded_project_local_path.split(upload_directory)[-1]}")
elif local_path:
loaded_project_local_path = os.path.join(project_source_dir, referred_name)
shutil.copy(local_path, loaded_project_local_path)
logger.info(f"Import complete for {loaded_project_local_path.split(upload_directory)[-1]}")
# Handle file uploading
filename = secure_filename(uploaded_file.filename)
loaded_project_local_path = Path(project_source_dir) / filename
uploaded_file.save(str(loaded_project_local_path))
logger.info(f"Transfer complete for {loaded_project_local_path.relative_to(upload_directory)}")
if loaded_project_local_path.lower().endswith('.zip'):
elif project_url:
# Handle downloading project from a URL
loaded_project_local_path = Path(project_source_dir) / referred_name
shutil.move(str(downloaded_file_url), str(loaded_project_local_path))
logger.info(f"Download complete for {loaded_project_local_path.relative_to(upload_directory)}")
elif local_path:
# Handle local files
loaded_project_local_path = Path(project_source_dir) / referred_name
shutil.copy(str(local_path), str(loaded_project_local_path))
logger.info(f"Import complete for {loaded_project_local_path.relative_to(upload_directory)}")
if loaded_project_local_path.suffix == ".zip":
loaded_project_local_path = cls.process_zipped_project(loaded_project_local_path)
new_job_data["__loaded_project_local_path"] = loaded_project_local_path
@@ -72,7 +80,7 @@ class JobImportHandler:
return new_job_data
@staticmethod
def download_project_from_url(project_url):
def download_project_from_url(project_url: str):
# This nested function is to handle downloading from a URL
logger.info(f"Downloading project from url: {project_url}")
referred_name = os.path.basename(project_url)
@@ -101,7 +109,7 @@ class JobImportHandler:
return None, None
@staticmethod
def process_zipped_project(zip_path):
def process_zipped_project(zip_path: Path) -> Path:
"""
Processes a zipped project.
@@ -109,22 +117,21 @@ class JobImportHandler:
If the zip file contains more than one project file or none, an error is raised.
Args:
zip_path (str): The path to the zip file.
zip_path (Path): The path to the zip file.
Raises:
ValueError: If there's more than 1 project file or none in the zip file.
Returns:
str: The path to the main project file.
Path: The path to the main project file.
"""
work_path = os.path.dirname(zip_path)
work_path = zip_path.parent
try:
with zipfile.ZipFile(zip_path, 'r') as myzip:
myzip.extractall(work_path)
myzip.extractall(str(work_path))
project_files = [x for x in os.listdir(work_path) if os.path.isfile(os.path.join(work_path, x))]
project_files = [x for x in project_files if '.zip' not in x]
project_files = [p for p in work_path.iterdir() if p.is_file() and p.suffix.lower() != ".zip"]
logger.debug(f"Zip files: {project_files}")
@@ -134,9 +141,9 @@ class JobImportHandler:
# If there's more than 1 project file or none, raise an error
if len(project_files) != 1:
raise ValueError(f'Cannot find a valid project file in {os.path.basename(zip_path)}')
raise ValueError(f'Cannot find a valid project file in {zip_path.name}')
extracted_project_path = os.path.join(work_path, project_files[0])
extracted_project_path = work_path / project_files[0]
logger.info(f"Extracted zip file to {extracted_project_path}")
except (zipfile.BadZipFile, zipfile.LargeZipFile) as e:
src/api/server_proxy.py
+7 -6
View File
@@ -3,6 +3,7 @@ import logging
import os
import threading
import time
from pathlib import Path
import requests
from requests_toolbelt.multipart import MultipartEncoder, MultipartEncoderMonitor
@@ -184,12 +185,12 @@ class RenderServerProxy:
# Job Lifecycle:
# --------------------------------------------
def post_job_to_server(self, file_path, job_data, callback=None):
def post_job_to_server(self, file_path: Path, job_data, callback=None):
"""
Posts a job to the server.
Args:
file_path (str): The path to the file to upload.
file_path (Path): The path to the file to upload.
job_data (dict): A dict of jobs data.
callback (function, optional): A callback function to call during the upload. Defaults to None.
@@ -197,12 +198,12 @@ class RenderServerProxy:
Response: The response from the server.
"""
# Check if file exists
if not os.path.exists(file_path):
if not file_path.exists():
raise FileNotFoundError(f"File not found: {file_path}")
# Bypass uploading file if posting to localhost
if self.is_localhost:
job_data['local_path'] = file_path
job_data['local_path'] = str(file_path)
url = urljoin(f'http://{self.hostname}:{self.port}', '/api/add_job')
headers = {'Content-Type': 'application/json'}
return requests.post(url, data=json.dumps(job_data), headers=headers)
@@ -210,7 +211,7 @@ class RenderServerProxy:
# Prepare the form data for remote host
with open(file_path, 'rb') as file:
encoder = MultipartEncoder({
'file': (os.path.basename(file_path), file, 'application/octet-stream'),
'file': (file_path.name, file, 'application/octet-stream'),
'json': (None, json.dumps(job_data), 'application/json'),
})
@@ -247,7 +248,7 @@ class RenderServerProxy:
# Engines:
# --------------------------------------------
def get_engine_for_filename(self, filename, timeout=5):
def get_engine_for_filename(self, filename:str, timeout=5):
response = self.request(f'engine_for_filename?filename={os.path.basename(filename)}', timeout)
return response.text