diff --git a/.gitignore b/.gitignore index 6cb190f..1c53fff 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.icloud *.fcpxml /uploads -*.pyc \ No newline at end of file +*.pyc +/server_state.json diff --git a/config.yaml b/config.yaml index 564f5ed..114e647 100644 --- a/config.yaml +++ b/config.yaml @@ -1,4 +1,4 @@ -upload_folder: "/Users/brett/Desktop/zordon-uploads/" +upload_folder: "~/zordon-uploads/" max_content_path: 100000000 server_log_level: info flask_log_level: error diff --git a/server.py b/lib/job_server.py similarity index 81% rename from server.py rename to lib/job_server.py index af666d1..09712a8 100755 --- a/server.py +++ b/lib/job_server.py @@ -5,14 +5,10 @@ import logging import os import pathlib import shutil -import socket -import threading -import time from datetime import datetime from zipfile import ZipFile import requests -import yaml from flask import Flask, request, render_template, send_file, after_this_request from werkzeug.utils import secure_filename @@ -21,15 +17,15 @@ from lib.render_queue import RenderQueue from utilities.render_worker import RenderWorkerFactory, string_to_status logger = logging.getLogger() -app = Flask(__name__) +server = Flask(__name__) -@app.get('/jobs') +@server.get('/jobs') def jobs_json(): return [json.loads(x.json()) for x in RenderQueue.job_queue if not x.archived] -@app.get('/jobs/') +@server.get('/jobs/') def filtered_jobs_json(status_val): state = string_to_status(status_val) jobs = [json.loads(x.json()) for x in RenderQueue.jobs_with_status(state)] @@ -39,7 +35,7 @@ def filtered_jobs_json(status_val): return f'Cannot find jobs with status {status_val}', 400 -@app.get('/job_status/') +@server.get('/job_status/') def get_job_status(job_id): found_job = RenderQueue.job_with_id(job_id) if found_job: @@ -48,7 +44,7 @@ def get_job_status(job_id): return f'Cannot find job with ID {job_id}', 400 -@app.get('/file_list/') +@server.get('/file_list/') def get_file_list(job_id): found_job = RenderQueue.job_with_id(job_id) if found_job: @@ -58,7 +54,7 @@ def get_file_list(job_id): return f'Cannot find job with ID {job_id}', 400 -@app.route('/download_all/') +@server.route('/download_all/') def download_all(job_id): zip_filename = None @@ -85,26 +81,26 @@ def download_all(job_id): return f'Cannot find job with ID {job_id}', 400 -@app.post('/register_client') +@server.post('/register_client') def register_client(): client_hostname = request.values['hostname'] x = RenderQueue.register_client(client_hostname) return "Success" if x else "Fail" -@app.post('/unregister_client') +@server.post('/unregister_client') def unregister_client(): client_hostname = request.values['hostname'] x = RenderQueue.unregister_client(client_hostname) return "Success" if x else "Fail" -@app.get('/clients') +@server.get('/clients') def render_clients(): return RenderQueue.render_clients -@app.get('/full_status') +@server.get('/full_status') def full_status(): full_results = {'timestamp': datetime.now().isoformat(), 'servers': {}} @@ -131,7 +127,7 @@ def full_status(): return full_results -@app.get('/snapshot') +@server.get('/snapshot') def snapshot(): server_status = RenderQueue.status() server_jobs = [json.loads(x.json()) for x in RenderQueue.job_queue if not x.archived] @@ -139,7 +135,7 @@ def snapshot(): return server_data -@app.post('/add_job') +@server.post('/add_job') def add_job(): def remove_job_dir(): if job_dir and os.path.exists(job_dir): @@ -176,7 +172,7 @@ def add_job(): if uploaded_file and uploaded_file.filename: logger.info(f"Receiving uploaded file {uploaded_file.filename}") new_id = RenderJob.generate_id() - job_dir = os.path.join(app.config['UPLOAD_FOLDER'], new_id + "-" + uploaded_file.filename) + job_dir = os.path.join(server.config['UPLOAD_FOLDER'], new_id + "-" + uploaded_file.filename) if not os.path.exists(job_dir): os.makedirs(job_dir) @@ -245,7 +241,7 @@ def add_job(): return 'unknown error', 500 -@app.get('/cancel_job') +@server.get('/cancel_job') def cancel_job(): job_id = request.args.get('id', None) confirm = request.args.get('confirm', False) @@ -264,18 +260,18 @@ def cancel_job(): return 'job not found', 400 -@app.get('/clear_history') +@server.get('/clear_history') def clear_history(): RenderQueue.clear_history() return 'success' -@app.route('/status') +@server.route('/status') def status(): return RenderQueue.status() -@app.get('/renderer_info') +@server.get('/renderer_info') def renderer_info(): renderer_data = {} for r in RenderWorkerFactory.supported_renderers(): @@ -287,12 +283,12 @@ def renderer_info(): return renderer_data -@app.route('/') +@server.route('/') def default(): return "Server running" -@app.route('/upload') +@server.route('/upload') def upload_file_page(): return render_template('upload.html', render_clients=RenderQueue.render_clients, supported_renderers=RenderWorkerFactory.supported_renderers()) @@ -305,49 +301,3 @@ def post_job_to_server(input_path, job_json, client, server_port=8080): req = requests.post(f'http://{client}:{server_port}/add_job', files=job_files) return req - - -def start_server(background_thread=False): - def eval_loop(delay_sec=1): - while True: - RenderQueue.evaluate_queue() - time.sleep(delay_sec) - - with open('config.yaml') as f: - config = yaml.load(f, Loader=yaml.FullLoader) - - logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S', - level=config.get('server_log_level', 'INFO').upper()) - - app.config['UPLOAD_FOLDER'] = config['upload_folder'] - app.config['MAX_CONTENT_PATH'] = config['max_content_path'] - # app.config['RESULT_STATIC_PATH'] = 'static' - - # Get hostname and render clients - RenderQueue.host_name = socket.gethostname() - app.config['HOSTNAME'] = RenderQueue.host_name - if not RenderQueue.render_clients: - RenderQueue.render_clients = [RenderQueue.host_name] - - # disable most Flask logging - flask_log = logging.getLogger('werkzeug') - flask_log.setLevel(config.get('flask_log_level', 'ERROR').upper()) - - # Setup the RenderManager object - RenderQueue.load_state() - - thread = threading.Thread(target=eval_loop, kwargs={'delay_sec': config.get('queue_eval_seconds', 1)}, daemon=True) - thread.start() - - if background_thread: - server_thread = threading.Thread( - target=lambda: app.run(host='0.0.0.0', port=RenderQueue.port, debug=False, use_reloader=False)) - server_thread.start() - server_thread.join() - else: - app.run(host='0.0.0.0', port=RenderQueue.port, debug=config.get('flask_debug_enable', False), - use_reloader=False) - - -if __name__ == '__main__': - start_server() diff --git a/start_server.py b/start_server.py new file mode 100644 index 0000000..2096c07 --- /dev/null +++ b/start_server.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import logging +import os +import socket +import threading +import time +import yaml +from lib.job_server import server +from lib.render_queue import RenderQueue + + +def start_server(background_thread=False): + def eval_loop(delay_sec=1): + while True: + RenderQueue.evaluate_queue() + time.sleep(delay_sec) + + with open('config.yaml') as f: + config = yaml.load(f, Loader=yaml.FullLoader) + + logging.basicConfig(format='%(asctime)s: %(levelname)s: %(module)s: %(message)s', datefmt='%d-%b-%y %H:%M:%S', + level=config.get('server_log_level', 'INFO').upper()) + + server.config['UPLOAD_FOLDER'] = os.path.expanduser(config['upload_folder']) + server.config['MAX_CONTENT_PATH'] = config['max_content_path'] + + # Get hostname and render clients + RenderQueue.host_name = socket.gethostname() + server.config['HOSTNAME'] = RenderQueue.host_name + if not RenderQueue.render_clients: + RenderQueue.render_clients = [RenderQueue.host_name] + + # disable most Flask logging + flask_log = logging.getLogger('werkzeug') + flask_log.setLevel(config.get('flask_log_level', 'ERROR').upper()) + + # Set up the RenderQueue object + RenderQueue.load_state() + + thread = threading.Thread(target=eval_loop, kwargs={'delay_sec': config.get('queue_eval_seconds', 1)}, daemon=True) + thread.start() + + logging.info("Starting Zordon Render Server") + + if background_thread: + server_thread = threading.Thread( + target=lambda: server.run(host='0.0.0.0', port=RenderQueue.port, debug=False, use_reloader=False)) + server_thread.start() + server_thread.join() + else: + server.run(host='0.0.0.0', port=RenderQueue.port, debug=config.get('flask_debug_enable', False), + use_reloader=False) + + +if __name__ == '__main__': + start_server() diff --git a/subway.png b/subway.png deleted file mode 100644 index 0a588dc..0000000 Binary files a/subway.png and /dev/null differ diff --git a/whiteboard.jpeg b/whiteboard.jpeg deleted file mode 100644 index 0bb1dd6..0000000 Binary files a/whiteboard.jpeg and /dev/null differ