Add controller logging and return readable errors

The controller ran silently and surfaced errors as opaque/empty bodies,
making it impossible to diagnose failures from either the UI or the server.
- Configure Python logging (timestamps, levels) and log job lifecycle events
  (created / dispatched / done / failed) plus startup and dispatch warnings.
- Add a catch-all error handler so unhandled exceptions return a JSON error
  with the exception type+message instead of an empty response.
- Remove the frontend submit bug that hand-set multipart Content-Type without
  a boundary (and built an opts object it never used); post the FormData
  directly so the browser sets the correct content type + boundary.
This commit is contained in:
Brett Williams
2026-08-30 22:39:16 -05:00
parent 6f38beaf92
commit 8a6f02b7e9
3 changed files with 31 additions and 2 deletions
+23
View File
@@ -1,5 +1,6 @@
import datetime
import json
import logging
import os
import queue
@@ -10,6 +11,8 @@ import ctrl.settings as settings
import ctrl.workers as wrk
from ctrl.scheduler import make_scheduler
log = logging.getLogger("ctrl")
app = Flask(__name__, static_folder="static", static_url_path="")
scheduler = make_scheduler()
@@ -75,6 +78,7 @@ def api_create_job():
job_id = db.create_job("git", repo_url, os_req, cpu_req)
scheduler.start()
log.info("Job %s created: type=%s os=%s cpu=%s", job_id, "upload" if "file" in request.files else "git", os_req, cpu_req)
return jsonify({"id": job_id}), 201
@@ -144,9 +148,17 @@ def api_stream(job_id):
return Response(gen(), mimetype="text/event-stream")
@app.errorhandler(Exception)
def _handle_error(err):
log.exception("Unhandled error serving %s %s", request.method, request.path)
return jsonify({"error": f"{type(err).__name__}: {err}"}), 500
def main():
_ensure_ready()
scheduler.start()
_configure_logging()
log.info("Controller listening on 0.0.0.0:%s (%d workers configured)", settings.PORT, len(settings.WORKERS))
if settings.DEBUG:
app.run(host="0.0.0.0", port=settings.PORT, threaded=True)
return
@@ -154,5 +166,16 @@ def main():
serve(app, host="0.0.0.0", port=settings.PORT, threads=8)
def _configure_logging():
level = logging.DEBUG if settings.DEBUG else logging.INFO
logging.basicConfig(
level=level,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
# waitress's own request access log to stderr by default; keep ours separate.
logging.getLogger("waitress").setLevel(logging.INFO)
if __name__ == "__main__":
main()
+8
View File
@@ -1,4 +1,5 @@
import datetime
import logging
import os
import threading
import time
@@ -9,6 +10,8 @@ import ctrl.db as db
import ctrl.settings as settings
import ctrl.workers as wrk
log = logging.getLogger("ctrl.scheduler")
class Scheduler:
def __init__(self):
@@ -78,6 +81,7 @@ class Scheduler:
job_id = job["id"]
candidates = [w for w in wrk.probe_all() if wrk.matches(job, w)]
if not candidates:
log.warning("Job %s: no worker matched (os=%r cpu=%r)", job_id, job.get("os_req"), job.get("cpu_req"))
self._append(job_id, f"[scheduler] No worker matched requirements (os='{job['os_req']}', cpu='{job['cpu_req']}').\n")
return
@@ -94,6 +98,7 @@ class Scheduler:
job_id = job["id"]
db.update_job(job_id, status="dispatching", worker_host=worker["url"],
started_at=datetime.datetime.now().isoformat())
log.info("Job %s: dispatching to %s (%s %s)", job_id, worker["url"], worker.get("os"), worker.get("cpu"))
self._append(job_id, f"[scheduler] Dispatching to {worker['url']} ({worker.get('os')} {worker.get('cpu')})\n")
try:
@@ -102,6 +107,7 @@ class Scheduler:
else:
worker_job_id = self._request_upload(job, worker)
except Exception as e:
log.error("Job %s: dispatch failed: %s", job_id, e)
db.update_job(job_id, status="failed", finished_at=datetime.datetime.now().isoformat(),
error=str(e))
self._append(job_id, f"[scheduler] FAILED: {e}\n")
@@ -116,8 +122,10 @@ class Scheduler:
self._wait_for_build(job, worker, worker_job_id)
self._fetch_artifacts(job, worker, worker_job_id)
db.update_job(job_id, status="done", finished_at=datetime.datetime.now().isoformat())
log.info("Job %s: done, artifacts saved", job_id)
self._append(job_id, "[scheduler] Build complete, artifacts saved.\n")
except Exception as e:
log.error("Job %s: failed: %s", job_id, e)
db.update_job(job_id, status="failed", finished_at=datetime.datetime.now().isoformat(),
error=str(e))
self._append(job_id, f"[scheduler] FAILED: {e}\n")
-2
View File
@@ -176,8 +176,6 @@ $("#job-form").addEventListener("submit", async (e) => {
body.append("file", file);
}
try {
const opts = { method: "POST", body };
if (!git) opts.headers = { "Content-Type": "multipart/form-data" }; // let browser set boundary
const res = await fetch("/api/jobs", { method: "POST", body });
const data = await res.json().catch(() => ({}));
if (!res.ok) throw new Error(data.error || res.statusText);