brett/Zordon
1
0
Fork 0
mirror of https://github.com/blw1138/Zordon.git synced 2025-12-17 16:58:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Convert render_queue and scheduledjob to use sql instead of json

Browse Source
This commit is contained in:
Brett Williams
2023-05-24 09:58:02 -05:00
parent dd2ae2d71a
commit e11c5e7e58
4 changed files with 76 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
lib/render_queue.py
View File

@@ -1,22 +1,17 @@
import json
import logging
import os
import platform
from datetime import datetime
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import psutil
import requests
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from .scheduled_job import ScheduledJob, Base
from .render_workers.render_worker import RenderStatus
from .scheduled_job import ScheduledJob, Base
logger = logging.getLogger()
JSON_FILE = 'server_state.json'
#todo: move history to sqlite db
class JobNotFoundError(Exception):
def __init__(self, job_id, *args):
@@ -25,7 +20,6 @@ class JobNotFoundError(Exception):
class RenderQueue:
engine = create_engine('sqlite:///database.db')
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
@@ -47,17 +41,21 @@ class RenderQueue:
def add_to_render_queue(cls, render_job, force_start=False, client=None):
if not client or render_job.client == cls.host_name:
logger.debug('Adding priority {} job to render queue: {}'.format(render_job.priority, render_job.worker))
logger.debug('Adding priority {} job to render queue: {}'.format(render_job.priority, render_job.worker()))
render_job.client = cls.host_name
cls.job_queue.append(render_job)
if force_start:
cls.start_job(render_job)
cls.session.add(render_job)
cls.session.commit()
cls.save_state()
else:
# todo: implement client rendering
logger.warning('remote client rendering not implemented yet')
@classmethod
def all_jobs(cls):
return cls.job_queue
@classmethod
def running_jobs(cls):
return cls.jobs_with_status(RenderStatus.RUNNING)
@@ -70,89 +68,33 @@ class RenderQueue:
@classmethod
def jobs_with_status(cls, status, priority_sorted=False):
found_jobs = [x for x in cls.job_queue if x.render_status() == status]
found_jobs = [x for x in cls.all_jobs() if x.render_status() == status]
if priority_sorted:
found_jobs = sorted(found_jobs, key=lambda a: a.priority, reverse=False)
return found_jobs
@classmethod
def job_with_id(cls, job_id, none_ok=False):
found_job = next((x for x in cls.job_queue if x.id == job_id), None)
found_job = next((x for x in cls.all_jobs() if x.id == job_id), None)
if not found_job and not none_ok:
raise JobNotFoundError(job_id)
return found_job
@classmethod
def clear_history(cls):
to_remove = [x for x in cls.job_queue if x.render_status() in [RenderStatus.CANCELLED,
RenderStatus.COMPLETED, RenderStatus.ERROR]]
to_remove = [x for x in cls.all_jobs() if x.render_status() in [RenderStatus.CANCELLED,
RenderStatus.COMPLETED, RenderStatus.ERROR]]
for job_to_remove in to_remove:
cls.job_queue.remove(job_to_remove)
cls.save_state()
@classmethod
def load_state(cls, json_path=None):
"""Load state history from JSON file"""
input_path = json_path or JSON_FILE
if os.path.exists(input_path):
with open(input_path) as f:
# load saved data
saved_state = json.load(f)
cls.render_clients = saved_state.get('clients', {})
for job in saved_state.get('jobs', []):
try:
render_job = ScheduledJob(renderer=job['renderer'], input_path=job['worker']['input_path'],
output_path=job['worker']['output_path'], args=job['worker']['args'],
priority=job['priority'], client=job['client'])
# Load Worker values
for key, val in job['worker'].items():
if val and key in ['start_time', 'end_time']: # convert date strings back into date objects
render_job.worker.__dict__[key] = datetime.fromisoformat(val)
else:
render_job.worker.__dict__[key] = val
render_job.worker.status = RenderStatus[job['status'].upper()]
job.pop('worker', None)
# Create RenderJob with re-created Renderer object
for key, val in job.items():
if key in ['date_created']: # convert date strings back to datetime objects
render_job.__dict__[key] = datetime.fromisoformat(val)
else:
import types
if hasattr(render_job, key):
if getattr(render_job, key) and not isinstance(getattr(render_job, key), types.MethodType):
render_job.__dict__[key] = val
# Handle older loaded jobs that were cancelled before closing
if render_job.render_status() == RenderStatus.RUNNING:
render_job.worker.status = RenderStatus.CANCELLED
# finally add back to render queue
cls.job_queue.append(render_job)
except Exception as e:
logger.exception(f"Unable to load job: {job['id']} - {e}")
cls.last_saved_counts = cls.job_counts()
def load_state(cls):
cls.job_queue = cls.session.query(ScheduledJob).all()
@classmethod
def save_state(cls, json_path=None):
"""Save state history to JSON file"""
try:
logger.debug("Saving Render History")
output = {'timestamp': datetime.now().isoformat(),
'jobs': [j.json() for j in cls.job_queue],
'clients': cls.render_clients}
output_path = json_path or JSON_FILE
with open(output_path, 'w') as f:
json.dump(output, f, indent=4)
cls.last_saved_counts = cls.job_counts()
except Exception as e:
logger.error("Error saving state JSON: {}".format(e))
def save_state(cls):
cls.session.commit()
@classmethod
def evaluate_queue(cls):
@@ -162,7 +104,7 @@ class RenderQueue:
not_started = cls.jobs_with_status(RenderStatus.NOT_STARTED, priority_sorted=True)
if not_started:
for job in not_started:
renderer = job.worker.engine.name()
renderer = job.worker().engine.name()
higher_priority_jobs = [x for x in cls.running_jobs() if x.priority < job.priority]
max_renderers = renderer in instances.keys() and instances[
renderer] >= cls.maximum_renderer_instances.get(renderer, 1)
@@ -181,7 +123,7 @@ class RenderQueue:
@classmethod
def start_job(cls, job):
logger.info('Starting {}render: {} - Priority {}'.format('scheduled ' if job.scheduled_start else '', job.name,
job.priority))
job.priority))
job.start()
@classmethod
@@ -195,12 +137,14 @@ class RenderQueue:
logger.info(f"Deleting job ID: {job.id}")
job.stop()
cls.job_queue.remove(job)
cls.session.delete(job)
cls.save_state()
return True
@classmethod
def renderer_instances(cls):
from collections import Counter
all_instances = [x.worker.engine.name() for x in cls.running_jobs()]
all_instances = [x.worker().engine.name() for x in cls.running_jobs()]
return Counter(all_instances)
@classmethod
@@ -208,25 +152,21 @@ class RenderQueue:
job_counts = {}
for job_status in RenderStatus:
job_counts[job_status.value] = len(cls.jobs_with_status(job_status))
return job_counts
@classmethod
def status(cls):
stats = {"timestamp": datetime.now().isoformat(),
"platform": platform.platform(),
"cpu_percent": psutil.cpu_percent(percpu=False),
"cpu_percent_per_cpu": psutil.cpu_percent(percpu=True),
"cpu_count": psutil.cpu_count(),
"memory_total": psutil.virtual_memory().total,
"memory_available": psutil.virtual_memory().available,
"memory_percent": psutil.virtual_memory().percent,
"job_counts": cls.job_counts(),
"host_name": cls.host_name
}
return stats
return {"timestamp": datetime.now().isoformat(),
"platform": platform.platform(),
"cpu_percent": psutil.cpu_percent(percpu=False),
"cpu_percent_per_cpu": psutil.cpu_percent(percpu=True),
"cpu_count": psutil.cpu_count(),
"memory_total": psutil.virtual_memory().total,
"memory_available": psutil.virtual_memory().available,
"memory_percent": psutil.virtual_memory().percent,
"job_counts": cls.job_counts(),
"host_name": cls.host_name
}
@classmethod
def register_client(cls, hostname):
@@ -273,4 +213,3 @@ class RenderQueue:
except requests.ConnectionError as e:
pass
return False