Major file reorganization

This commit is contained in:
Brett Williams
2022-12-20 11:22:04 -08:00
parent 63c866166b
commit 4ef94dd7b8
32 changed files with 90 additions and 100 deletions

View File

View File

@@ -0,0 +1,146 @@
#!/usr/bin/env python3
import glob
import json
import re
import time
from .render_worker import *
def aerender_path():
paths = glob.glob('/Applications/*After Effects*/aerender')
if len(paths) > 1:
logging.warning('Multiple After Effects installations detected')
elif not paths:
logging.error('After Effects installation not found')
else:
return paths[0]
class AERenderWorker(BaseRenderWorker):
renderer = 'After Effects'
render_engine = 'aerender'
supported_extensions = ['.aep']
def __init__(self, input_path, output_path, args=None):
super(AERenderWorker, self).__init__(input_path=input_path, output_path=output_path, ignore_extensions=False,
args=args)
self.comp = args.get('comp', None)
self.render_settings = args.get('render_settings', None)
self.omsettings = args.get('omsettings', None)
self.progress = 0
self.progress_history = []
self.attributes = {}
@classmethod
def version(cls):
version = None
try:
render_path = cls.renderer_path()
if render_path:
ver_out = subprocess.check_output([render_path, '-version'])
version = ver_out.decode('utf-8').split(" ")[-1].strip()
except Exception as e:
logging.error(f'Failed to get {cls.renderer} version: {e}')
return version
def _generate_subprocess(self):
if os.path.exists('nexrender-cli-macos'):
logging.info('nexrender found')
# {
# "template": {
# "src": String,
# "composition": String,
#
# "frameStart": Number,
# "frameEnd": Number,
# "frameIncrement": Number,
#
# "continueOnMissing": Boolean,
# "settingsTemplate": String,
# "outputModule": String,
# "outputExt": String,
# },
# "assets": [],
# "actions": {
# "prerender": [],
# "postrender": [],
# },
# "onChange": Function,
# "onRenderProgress": Function
# }
job = {'template':
{
'src': 'file://' + self.input_path, 'composition': self.comp.replace('"', ''),
'settingsTemplate': self.render_settings.replace('"', ''),
'outputModule': self.omsettings.replace('"', ''), 'outputExt': 'mov'}
}
x = ['./nexrender-cli-macos', "'{}'".format(json.dumps(job))]
else:
logging.info('nexrender not found')
x = [aerender_path(), '-project', self.input_path, '-comp', self.comp, '-RStemplate', self.render_settings,
'-OMtemplate', self.omsettings, '-output', self.output_path]
return x
def _parse_stdout(self, line):
# print line
if line.startswith('PROGRESS:'):
# print 'progress'
trimmed = line.replace('PROGRESS:', '').strip()
if len(trimmed):
self.progress_history.append(line)
if 'Seconds' in trimmed:
self._update_progress(line)
elif ': ' in trimmed:
tmp = trimmed.split(': ')
self.attributes[tmp[0].strip()] = tmp[1].strip()
elif line.startswith('WARNING:'):
trimmed = line.replace('WARNING:', '').strip()
self.warnings.append(trimmed)
logging.warning(trimmed)
elif line.startswith('aerender ERROR') or 'ERROR:' in line:
self.errors.append(line)
logging.error(line)
def _update_progress(self, line):
if not self.total_frames:
duration_string = self.attributes.get('Duration', None)
frame_rate = self.attributes.get('Frame Rate', '0').split(' ')[0]
self.total_frames = timecode_to_frames(duration_string.split('Duration:')[-1], float(frame_rate))
match = re.match(r'PROGRESS:.*\((?P<frame>\d+)\): (?P<time>\d+)', line).groupdict()
self.last_frame = match['frame']
def average_frame_duration(self):
total_durations = 0
for line in self.progress_history:
match = re.match(r'PROGRESS:.*\((?P<frame>\d+)\): (?P<time>\d+)', line)
if match:
total_durations += int(match.group(2))
average = float(total_durations) / self.last_frame
return average
def percent_complete(self):
if self.total_frames:
return (float(self.last_frame) / float(self.total_frames)) * 100
else:
return 0
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG)
r = AERenderWorker('/Users/brett/Desktop/Youtube_Vids/Film_Formats/Frame_Animations.aep', '"Film Pan"',
'"Draft Settings"', '"ProRes"', '/Users/brett/Desktop/test_render')
r.start()
while r.is_running():
time.sleep(0.1)

View File

@@ -0,0 +1,211 @@
#!/usr/bin/env python3
import json
import re
from .render_worker import *
class BlenderRenderWorker(BaseRenderWorker):
renderer = 'Blender'
render_engine = 'blender'
supported_extensions = ['.blend']
install_paths = ['/Applications/Blender.app/Contents/MacOS/Blender']
supported_export_formats = ['TGA', 'RAWTGA', 'JPEG', 'IRIS', 'IRIZ', 'AVIRAW', 'AVIJPEG', 'PNG', 'BMP', 'HDR',
'TIFF', 'OPEN_EXR', 'OPEN_EXR_MULTILAYER', 'MPEG', 'CINEON', 'DPX', 'DDS', 'JP2']
def __init__(self, input_path, output_path, args=None):
super(BlenderRenderWorker, self).__init__(input_path=input_path, output_path=output_path,
ignore_extensions=False, args=args)
# Args
self.engine = self.args.get('engine', 'BLENDER_EEVEE').upper()
self.export_format = self.args.get('export_format', None) or 'JPEG'
self.camera = self.args.get('camera', None)
self.render_all_frames = self.args.get('render_all_frames', False)
self.frame_to_render = 0
# Stats
self.__frame_percent_complete = 0.0
# Scene Info
self.scene_info = get_scene_info(input_path)
self.total_frames = (int(self.scene_info.get('frame_end', 0)) - int(self.scene_info.get('frame_start', 0)) + 1) \
if self.render_all_frames else 1
self.current_frame = int(self.scene_info.get('frame_start', 0))
@classmethod
def version(cls):
version = None
try:
render_path = cls.renderer_path()
if render_path:
ver_out = subprocess.check_output([render_path, '-v'])
version = ver_out.decode('utf-8').splitlines()[0].replace('Blender', '').strip()
except Exception as e:
logging.error(f'Failed to get {cls.renderer} version: {e}')
return version
def _generate_subprocess(self):
cmd = [self.renderer_path()]
if self.args.get('background', True): # optionally run render not in background
cmd.append('-b')
cmd.append(self.input_path)
if self.camera:
cmd.extend(['--python-expr', f"import bpy;bpy.context.scene.camera = bpy.data.objects['{self.camera}'];"])
cmd.extend(['-E', self.engine, '-o', self.output_path, '-F', self.export_format])
# all frames or single
cmd.extend(['-a'] if self.render_all_frames else ['-f', str(self.frame_to_render)])
# Convert raw args from string if available
raw_args = self.args.get('raw', None)
if raw_args:
cmd.extend(raw_args.split(' '))
return cmd
def _parse_stdout(self, line):
pattern = re.compile(
r'Fra:(?P<frame>\d*).*Mem:(?P<memory>\S+).*Time:(?P<time>\S+)(?:.*Remaining:)?(?P<remaining>\S*)')
found = pattern.search(line)
if found:
stats = found.groupdict()
memory_use = stats['memory']
time_elapsed = stats['time']
time_remaining = stats['remaining'] or 'Unknown'
sample_string = line.split('|')[-1].strip()
if "sample" in sample_string.lower():
samples = re.sub(r'[^\d/]', '', sample_string)
self.__frame_percent_complete = int(samples.split('/')[0]) / int(samples.split('/')[-1])
# Calculate rough percent based on cycles
# EEVEE
# 10-Apr-22 22:42:06 - RENDERER: Fra:0 Mem:857.99M (Peak 928.55M) | Time:00:03.96 | Rendering 1 / 65 samples
# 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:07.92 | Rendering 26 / 64 samples
# 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:08.17 | Rendering 51 / 64 samples
# 10-Apr-22 22:42:10 - RENDERER: Fra:0 Mem:827.09M (Peak 928.55M) | Time:00:08.31 | Rendering 64 / 64 samples
# CYCLES
# 10-Apr-22 22:43:22 - RENDERER: Fra:0 Mem:836.30M (Peak 1726.13M) | Time:00:01.56 | Remaining:00:30.65 | Mem:588.68M, Peak:588.68M | Scene, View Layer | Sample 1/150
# 10-Apr-22 22:43:43 - RENDERER: Fra:0 Mem:836.30M (Peak 1726.13M) | Time:00:22.01 | Remaining:00:03.36 | Mem:588.68M, Peak:588.68M | Scene, View Layer | Sample 129/150
if int(stats['frame']) > self.current_frame:
self.current_frame = int(stats['frame'])
logger.debug(
'Frame:{0} | Mem:{1} | Time:{2} | Remaining:{3}'.format(self.current_frame, memory_use,
time_elapsed, time_remaining))
elif 'error' in line.lower():
logger.error(line)
self.errors.append(line)
elif 'Saved' in line or 'Saving' in line or 'quit' in line:
match = re.match(r'Time: (.*) \(Saving', line)
if match:
time_completed = match.groups()[0]
if self.render_all_frames:
logger.debug(f'Frame {self.current_frame} completed in {time_completed}')
else:
logger.info(f'Render completed in {time_completed}')
else:
logger.debug(line)
else:
pass
# if len(line.strip()):
# logger.debug(line.strip())
def percent_complete(self):
if self.total_frames <= 1:
return self.__frame_percent_complete
else:
whole_frame_percent = (self.current_frame - 1) / self.total_frames
adjusted_frame_percent = self.__frame_percent_complete / self.total_frames
total_percent = whole_frame_percent + adjusted_frame_percent
return max(total_percent, 0)
def run_python_expression_in_blend(path, python_expression):
if os.path.exists(path):
try:
return subprocess.run([BlenderRenderWorker.renderer_path(), '-b', path, '--python-expr', python_expression],
capture_output=True)
except Exception as e:
logger.warning(f"Error running python expression in blender: {e}")
pass
else:
raise FileNotFoundError
def run_python_script_in_blend(path, python_path):
if os.path.exists(path) and os.path.exists(python_path):
try:
return subprocess.run([BlenderRenderWorker.renderer_path(), '-b', path, '--python', python_path],
capture_output=True)
except Exception as e:
logger.warning(f"Error running python expression in blender: {e}")
pass
else:
raise FileNotFoundError
def pack_blender_files(path):
# Credit to L0Lock for pack script - https://blender.stackexchange.com/a/243935
pack_script = "import bpy\n" \
"bpy.ops.file.pack_all()\n" \
"myPath = bpy.data.filepath\n" \
"myPath = str(myPath)\n" \
"bpy.ops.wm.save_as_mainfile(filepath=myPath[:-6]+'_packed'+myPath[-6:])"
try:
results = run_python_expression_in_blend(path, pack_script)
result_text = results.stdout.decode()
dir_name = os.path.dirname(path)
# report any missing textures
not_found = re.findall("(Unable to pack file, source path .*)\n", result_text)
for err in not_found:
logger.error(err)
p = re.compile('Info: Saved "(.*)"')
match = p.search(result_text)
if match:
new_path = os.path.join(dir_name, match.group(1))
logger.info(f'Blender file packed successfully to {new_path}')
return new_path
except Exception as e:
logger.error(f'Error packing .blend file: {e}')
return None
def get_scene_info(path):
scene_info = None
try:
results = run_python_script_in_blend(path, os.path.join(os.path.dirname(os.path.realpath(__file__)),
'scripts', 'get_blender_info.py'))
result_text = results.stdout.decode()
for line in result_text.splitlines():
if line.startswith('SCENE_DATA:'):
raw_data = line.split('SCENE_DATA:')[-1]
scene_info = json.loads(raw_data)
break
except Exception as e:
logger.error(f'Error getting file details for .blend file: {e}')
return scene_info
if __name__ == '__main__':
# x = pack_blender_files('/Users/brett/Blender Files/temple_animatic.blend')
# print(x)
logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG)
r = BlenderRenderWorker('/Users/brett/Blender Files/temple_animatic.blend', '/Users/brett/testing1234')
# r.engine = 'CYCLES'
r.start()
while r.is_running():
time.sleep(1)

View File

@@ -0,0 +1,78 @@
#!/usr/bin/env python3
import re
import time
from .render_worker import *
class FFMPEGRenderWorker(BaseRenderWorker):
renderer = 'ffmpeg'
render_engine = 'ffmpeg'
def __init__(self, input_path, output_path, args=None):
super(FFMPEGRenderWorker, self).__init__(input_path=input_path, output_path=output_path, ignore_extensions=True,
args=args)
stream_info = subprocess.check_output([self.renderer_path(), "-i", # https://stackoverflow.com/a/61604105
input_path, "-map", "0:v:0", "-c", "copy", "-f", "null", "-y",
"/dev/null"], stderr=subprocess.STDOUT).decode('utf-8')
found_frames = re.findall('frame=\s*(\d+)', stream_info)
self.total_frames = found_frames[-1] if found_frames else '-1'
self.frame = 0
# Stats
self.current_frame = -1
@classmethod
def version(cls):
version = None
try:
ver_out = subprocess.check_output([cls.renderer_path(), '-version']).decode('utf-8')
match = re.match(".*version\s*(\S+)\s*Copyright", ver_out)
if match:
version = match.groups()[0]
except Exception as e:
logger.error("Failed to get FFMPEG version: {}".format(e))
return version
def _generate_subprocess(self):
cmd = [self.renderer_path(), '-y', '-stats', '-i', self.input_path]
# Resize frame
if self.args.get('x_resolution', None) and self.args.get('y_resolution', None):
cmd.extend(['-vf', f"scale={self.args['x_resolution']}:{self.args['y_resolution']}"])
# Convert raw args from string if available
raw_args = self.args.get('raw', None)
if raw_args:
cmd.extend(raw_args.split(' '))
# Close with output path
cmd.append(self.output_path)
return cmd
def percent_complete(self):
return max(float(self.current_frame) / float(self.total_frames), 0.0)
def _parse_stdout(self, line):
pattern = re.compile(r'frame=\s*(?P<current_frame>\d+)\s*fps.*time=(?P<time_elapsed>\S+)')
found = pattern.search(line)
if found:
stats = found.groupdict()
self.current_frame = stats['current_frame']
time_elapsed = stats['time_elapsed']
elif "not found" in line:
self.errors.append(line)
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG)
test_movie = '/Users/brettwilliams/Desktop/dark_knight_rises.mp4'
r = FFMPEGRenderWorker(test_movie, '/Users/brettwilliams/Desktop/test-ffmpeg.mp4', args=['-c:v', 'libx265', '-vtag', 'hvc1'])
# r = FFMPEGRenderer(test_movie, '/Users/brettwilliams/Desktop/dark_knight_rises-output.mp4')
r.start()
while r.is_running():
time.sleep(1)

View File

@@ -0,0 +1,250 @@
#!/usr/bin/env python3
import io
import logging
import os
import subprocess
import threading
from datetime import datetime
from enum import Enum
import psutil
logger = logging.getLogger()
class RenderStatus(Enum):
NOT_STARTED = "not_started"
RUNNING = "running"
COMPLETED = "completed"
CANCELLED = "cancelled"
ERROR = "error"
SCHEDULED = "scheduled"
def string_to_status(string):
for stat in RenderStatus:
if stat.value == string:
return stat
return RenderStatus.ERROR
class BaseRenderWorker(object):
renderer = 'BaseRenderWorker'
render_engine = None
render_engine_version = None
supported_extensions = []
install_paths = []
supported_export_formats = []
def __init__(self, input_path, output_path, args=None, ignore_extensions=True):
if not ignore_extensions:
if not any(ext in input_path for ext in self.supported_extensions):
err_meg = f'Cannot find valid project with supported file extension for {self.renderer} renderer'
logger.error(err_meg)
raise ValueError(err_meg)
# Essential Info
self.input_path = input_path
self.output_path = output_path
self.args = args or {}
self.date_created = datetime.now()
self.renderer_version = self.version()
# Frame Ranges
self.total_frames = 0
self.current_frame = 0
# Logging
self.log_path = None
self.start_time = None
self.end_time = None
# History
self.status = RenderStatus.NOT_STARTED
self.warnings = []
self.errors = []
self.failed_attempts = 0
self.maximum_attempts = 1
# Threads and processes
self.__thread = threading.Thread(target=self.run, args=())
self.__thread.daemon = True
self.__process = None
self.is_finished = False
self.last_output = None
@classmethod
def version(cls):
raise NotImplementedError("Unknown version")
@classmethod
def renderer_path(cls):
path = None
try:
path = subprocess.check_output(['which', cls.render_engine]).decode('utf-8').strip()
except subprocess.CalledProcessError:
for p in cls.install_paths:
if os.path.exists(p):
path = p
except Exception as e:
logging.exception(e)
return path
def _generate_subprocess(self):
raise NotImplementedError("_generate_subprocess not implemented")
def start(self):
if not os.path.exists(self.input_path):
self.status = RenderStatus.ERROR
msg = 'Cannot find input path: {}'.format(self.input_path)
logger.error(msg)
self.errors.append(msg)
return
if not self.renderer_path():
self.status = RenderStatus.ERROR
msg = 'Cannot find render engine path for {}'.format(self.render_engine)
logger.error(msg)
self.errors.append(msg)
return
self.status = RenderStatus.RUNNING
logger.info('Starting {0} {1} Render for {2}'.format(self.renderer, self.version(), self.input_path))
self.__thread.start()
def run(self):
# Setup logging
try:
if not self.log_path:
log_dir = os.path.join(os.path.dirname(self.input_path), 'logs')
if not os.path.exists(log_dir):
os.makedirs(log_dir)
self.log_path = os.path.join(log_dir, os.path.basename(self.input_path)) + '.log'
logger.info('Logs saved in {}'.format(self.log_path))
except Exception as e:
logger.error("Error setting up logging: {}".format(e))
while self.failed_attempts < self.maximum_attempts and self.status is not RenderStatus.COMPLETED:
if self.failed_attempts:
logger.info('Attempt #{} failed. Starting attempt #{}'.format(self.failed_attempts, self.failed_attempts + 1))
# Start process and get updates
subprocess_cmds = self._generate_subprocess()
logger.debug("Renderer commands generated - {}".format(" ".join(subprocess_cmds)))
self.__process = subprocess.Popen(subprocess_cmds, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=False)
self.start_time = datetime.now()
with open(self.log_path, "a") as f:
f.write("{3} - Starting {0} {1} Render for {2}\n".format(self.renderer, self.version(), self.input_path,
self.start_time.isoformat()))
f.write(f"Running command: {' '.join(subprocess_cmds)}\n")
for c in io.TextIOWrapper(self.__process.stdout, encoding="utf-8"): # or another encoding
f.write(c)
logger.debug(f"{self.renderer}Worker: {c.strip()}")
self.last_output = c.strip()
self._parse_stdout(c.strip())
f.write('\n')
# Check return codes
return_code = self.__process.wait()
self.end_time = datetime.now()
# Return early if job was cancelled
if self.status is RenderStatus.CANCELLED:
self.is_finished = True
return
duration = self.end_time - self.start_time
if return_code:
message = f"{self.renderer} render failed with return_code {return_code} after {duration}"
logger.error(message)
self.failed_attempts = self.failed_attempts + 1
else:
message = f"{self.renderer} render completed successfully in {duration}"
logger.info(message)
self.status = RenderStatus.COMPLETED
f.write(message)
if self.failed_attempts >= self.maximum_attempts and self.status is not RenderStatus.CANCELLED:
logger.error('{} Render of {} failed after {} attempts'.format(self.renderer, self.input_path, self.failed_attempts))
self.status = RenderStatus.ERROR
if not self.errors:
self.errors = [self.last_output]
self.is_finished = True
def is_running(self):
if self.__thread:
return self.__thread.is_alive()
return False
def stop(self):
if self.__process:
try:
self.status = RenderStatus.CANCELLED
self.maximum_attempts = 0
process = psutil.Process(self.__process.pid)
for proc in process.children(recursive=True):
proc.kill()
process.kill()
except Exception as e:
logger.error(f"Exception stopping the process: {e}")
def percent_complete(self):
return 0
def _parse_stdout(self, line):
raise NotImplementedError("_parse_stdout not implemented")
def elapsed_time(self):
elapsed = ""
if self.start_time:
if self.end_time:
elapsed = self.end_time - self.start_time
elif self.is_running():
elapsed = datetime.now() - self.start_time
return elapsed
class RenderWorkerFactory:
@staticmethod
def supported_classes():
# to add support for any additional RenderWorker classes, import their classes and add to list here
from .blender_worker import BlenderRenderWorker
from .aerender_worker import AERenderWorker
from .ffmpeg_worker import FFMPEGRenderWorker
classes = [BlenderRenderWorker, AERenderWorker, FFMPEGRenderWorker]
return classes
@staticmethod
def create_worker(renderer, input_path, output_path, args=None):
worker_class = RenderWorkerFactory.class_for_name(renderer)
return worker_class(input_path=input_path, output_path=output_path, args=args)
@staticmethod
def supported_renderers():
return [x.render_engine for x in RenderWorkerFactory.supported_classes()]
@staticmethod
def class_for_name(name):
name = name.lower()
for render_class in RenderWorkerFactory.supported_classes():
if render_class.render_engine == name:
return render_class
raise LookupError(f'Cannot find class for name: {name}')
def timecode_to_frames(timecode, frame_rate):
e = [int(x) for x in timecode.split(':')]
seconds = (((e[0] * 60) + e[1] * 60) + e[2])
frames = (seconds * frame_rate) + e[-1] + 1
return frames

View File

@@ -0,0 +1,20 @@
import json
import bpy
# Get all cameras
cameras = []
for cam_obj in bpy.data.cameras:
user_map = bpy.data.user_map(subset={cam_obj}, value_types={'OBJECT'})
for data_obj in user_map[cam_obj]:
cam = {'name': data_obj.name, 'lens': cam_obj.lens}
cameras.append(cam)
data = {'cameras': cameras,
'frame_start': bpy.data.scenes[0].frame_start,
'frame_end': bpy.data.scenes[0].frame_end,
'resolution_x': bpy.data.scenes[0].render.resolution_x,
'resolution_y': bpy.data.scenes[0].render.resolution_y,
'fps': bpy.data.scenes[0].render.fps}
data_string = json.dumps(data)
print("SCENE_DATA:" + data_string)