Split add job helper (#45)

* Fix issue with engine not being available to download

* Add version caches to ffmpeg downloader

* Remove some test parameters

* "releases" should be "release" in linux ffmpeg url

* CPU was incorrectly reported as OS

* Fix naming structure for FFMPEG downloads for linux

* More linux ffmpeg work

* Improved error handling

* WIP

* Consolidate platform reporting to not use platform directly

* Fix missing folder name

* Fix project output naming

* Undo config.yaml commit

* Add is_engine_available API call

* Fix issue where subjobs would not find servers

src/api/add_job_helpers.py

@@ -44,22 +44,22 @@ def handle_uploaded_project_files(request, jobs_list, upload_directory):
     job_dir = os.path.join(upload_directory, '-'.join(
         [datetime.now().strftime("%Y.%m.%d_%H.%M.%S"), renderer, cleaned_path_name]))
     os.makedirs(job_dir, exist_ok=True)
-    upload_dir = os.path.join(job_dir, 'source')
-    os.makedirs(upload_dir, exist_ok=True)
+    project_source_dir = os.path.join(job_dir, 'source')
+    os.makedirs(project_source_dir, exist_ok=True)
 
     # Move projects to their work directories
     if uploaded_project and uploaded_project.filename:
-        loaded_project_local_path = os.path.join(upload_dir, secure_filename(uploaded_project.filename))
+        loaded_project_local_path = os.path.join(project_source_dir, secure_filename(uploaded_project.filename))
         uploaded_project.save(loaded_project_local_path)
-        logger.info(f"Transfer complete for {loaded_project_local_path.split(server.config['UPLOAD_FOLDER'])[-1]}")
+        logger.info(f"Transfer complete for {loaded_project_local_path.split(upload_directory)[-1]}")
     elif project_url:
-        loaded_project_local_path = os.path.join(upload_dir, referred_name)
+        loaded_project_local_path = os.path.join(project_source_dir, referred_name)
         shutil.move(downloaded_file_url, loaded_project_local_path)
-        logger.info(f"Download complete for {loaded_project_local_path.split(server.config['UPLOAD_FOLDER'])[-1]}")
+        logger.info(f"Download complete for {loaded_project_local_path.split(upload_directory)[-1]}")
     elif local_path:
-        loaded_project_local_path = os.path.join(upload_dir, referred_name)
+        loaded_project_local_path = os.path.join(project_source_dir, referred_name)
         shutil.copy(local_path, loaded_project_local_path)
-        logger.info(f"Import complete for {loaded_project_local_path.split(server.config['UPLOAD_FOLDER'])[-1]}")
+        logger.info(f"Import complete for {loaded_project_local_path.split(upload_directory)[-1]}")
 
     return loaded_project_local_path, referred_name
 
@@ -137,14 +137,21 @@ def create_render_jobs(jobs_list, loaded_project_local_path, job_dir, enable_spl
             os.makedirs(output_dir, exist_ok=True)
 
             # get new output path in output_dir
-            job_data['output_path'] = os.path.join(output_dir, os.path.basename(
-                job_data.get('output_path', None) or loaded_project_local_path
-            ))
+            output_path = job_data.get('output_path')
+            if not output_path:
+                loaded_project_filename = os.path.basename(loaded_project_local_path)
+                output_filename = os.path.splitext(loaded_project_filename)[0]
+            else:
+                output_filename = os.path.basename(output_path)
+
+            output_path = os.path.join(os.path.dirname(os.path.dirname(loaded_project_local_path)), 'output',
+                                       output_filename)
+            logger.debug(f"New job output path: {output_path}")
 
             # create & configure jobs
             worker = RenderWorkerFactory.create_worker(renderer=job_data['renderer'],
                                                        input_path=loaded_project_local_path,
-                                                       output_path=job_data["output_path"],
+                                                       output_path=output_path,
                                                        engine_version=job_data.get('engine_version'),
                                                        args=job_data.get('args', {}))
             worker.status = job_data.get("initial_status", worker.status)
@@ -157,6 +164,8 @@ def create_render_jobs(jobs_list, loaded_project_local_path, job_dir, enable_spl
             # determine if we can / should split the job
             if enable_split_jobs and (worker.total_frames > 1) and not worker.parent:
                 DistributedJobManager.split_into_subjobs(worker, job_data, loaded_project_local_path)
+            else:
+                logger.debug("Not splitting into subjobs")
 
             RenderQueue.add_to_render_queue(worker, force_start=job_data.get('force_start', False))
             if not worker.parent:

src/api/api_server.py

@@ -3,7 +3,6 @@ import json
 import logging
 import os
 import pathlib
-import platform
 import shutil
 import socket
 import ssl
@@ -25,7 +24,7 @@ from src.engines.core.base_worker import string_to_status, RenderStatus
 from src.engines.core.worker_factory import RenderWorkerFactory
 from src.engines.engine_manager import EngineManager
 from src.render_queue import RenderQueue, JobNotFoundError
-from src.utilities.misc_helper import system_safe_path
+from src.utilities.misc_helper import system_safe_path, current_system_os, current_system_cpu, current_system_os_version
 from src.utilities.server_helper import generate_thumbnail_for_job
 from src.utilities.zeroconf_server import ZeroconfServer
 
@@ -399,14 +398,10 @@ def status():
                  }
 
     # Get system info
-    system_platform = platform.system().lower().replace('darwin', 'macos')
-    system_platform_version = platform.mac_ver()[0] if system_platform == 'macos' else platform.release().lower()
-    system_cpu = platform.machine().lower().replace('amd64', 'x64')
-
     return {"timestamp": datetime.now().isoformat(),
-            "system_platform": system_platform,
-            "system_platform_version": system_platform_version,
-            "system_cpu": system_cpu,
+            "system_os": current_system_os(),
+            "system_os_version": current_system_os_version(),
+            "system_cpu": current_system_cpu(),
             "cpu_percent": psutil.cpu_percent(percpu=False),
             "cpu_percent_per_cpu": psutil.cpu_percent(percpu=True),
             "cpu_count": psutil.cpu_count(logical=False),
@@ -437,6 +432,14 @@ def renderer_info():
     return renderer_data
 
 
+@server.get('/api/<engine_name>/is_available')
+def is_engine_available(engine_name):
+    return {'engine': engine_name, 'available': RenderQueue.is_available_for_job(engine_name),
+            'cpu_count': int(psutil.cpu_count(logical=False)),
+            'versions': EngineManager.all_versions_for_engine(engine_name),
+            'hostname': server.config['HOSTNAME']}
+
+
 @server.get('/api/is_engine_available_to_download')
 def is_engine_available_to_download():
     available_result = EngineManager.version_is_available_to_download(request.args.get('engine'),

src/api/server_proxy.py

@@ -116,6 +116,9 @@ class RenderServerProxy:
     def get_status(self):
         return self.request_data('status')
 
+    def is_engine_available(self, engine_name):
+        return self.request_data(f'{engine_name}/is_available')
+
     def notify_parent_of_status_change(self, parent_id, subjob):
         return requests.post(f'http://{self.hostname}:{self.port}/api/job/{parent_id}/notify_parent_of_status_change',
                              json=subjob.json())