From 8a6f02b7e9ce3a4de1b46891363aad63e66844f0 Mon Sep 17 00:00:00 2001 From: Brett Williams Date: Sun, 30 Aug 2026 22:39:16 -0500 Subject: [PATCH] Add controller logging and return readable errors The controller ran silently and surfaced errors as opaque/empty bodies, making it impossible to diagnose failures from either the UI or the server. - Configure Python logging (timestamps, levels) and log job lifecycle events (created / dispatched / done / failed) plus startup and dispatch warnings. - Add a catch-all error handler so unhandled exceptions return a JSON error with the exception type+message instead of an empty response. - Remove the frontend submit bug that hand-set multipart Content-Type without a boundary (and built an opts object it never used); post the FormData directly so the browser sets the correct content type + boundary. --- ctrl/app.py | 23 +++++++++++++++++++++++ ctrl/scheduler.py | 8 ++++++++ ctrl/static/app.js | 2 -- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/ctrl/app.py b/ctrl/app.py index b7e59be..58c7c13 100644 --- a/ctrl/app.py +++ b/ctrl/app.py @@ -1,5 +1,6 @@ import datetime import json +import logging import os import queue @@ -10,6 +11,8 @@ import ctrl.settings as settings import ctrl.workers as wrk from ctrl.scheduler import make_scheduler +log = logging.getLogger("ctrl") + app = Flask(__name__, static_folder="static", static_url_path="") scheduler = make_scheduler() @@ -75,6 +78,7 @@ def api_create_job(): job_id = db.create_job("git", repo_url, os_req, cpu_req) scheduler.start() + log.info("Job %s created: type=%s os=%s cpu=%s", job_id, "upload" if "file" in request.files else "git", os_req, cpu_req) return jsonify({"id": job_id}), 201 @@ -144,9 +148,17 @@ def api_stream(job_id): return Response(gen(), mimetype="text/event-stream") +@app.errorhandler(Exception) +def _handle_error(err): + log.exception("Unhandled error serving %s %s", request.method, request.path) + return jsonify({"error": f"{type(err).__name__}: {err}"}), 500 + + def main(): _ensure_ready() scheduler.start() + _configure_logging() + log.info("Controller listening on 0.0.0.0:%s (%d workers configured)", settings.PORT, len(settings.WORKERS)) if settings.DEBUG: app.run(host="0.0.0.0", port=settings.PORT, threaded=True) return @@ -154,5 +166,16 @@ def main(): serve(app, host="0.0.0.0", port=settings.PORT, threads=8) +def _configure_logging(): + level = logging.DEBUG if settings.DEBUG else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + # waitress's own request access log to stderr by default; keep ours separate. + logging.getLogger("waitress").setLevel(logging.INFO) + + if __name__ == "__main__": main() diff --git a/ctrl/scheduler.py b/ctrl/scheduler.py index 57dfb60..780888d 100644 --- a/ctrl/scheduler.py +++ b/ctrl/scheduler.py @@ -1,4 +1,5 @@ import datetime +import logging import os import threading import time @@ -9,6 +10,8 @@ import ctrl.db as db import ctrl.settings as settings import ctrl.workers as wrk +log = logging.getLogger("ctrl.scheduler") + class Scheduler: def __init__(self): @@ -78,6 +81,7 @@ class Scheduler: job_id = job["id"] candidates = [w for w in wrk.probe_all() if wrk.matches(job, w)] if not candidates: + log.warning("Job %s: no worker matched (os=%r cpu=%r)", job_id, job.get("os_req"), job.get("cpu_req")) self._append(job_id, f"[scheduler] No worker matched requirements (os='{job['os_req']}', cpu='{job['cpu_req']}').\n") return @@ -94,6 +98,7 @@ class Scheduler: job_id = job["id"] db.update_job(job_id, status="dispatching", worker_host=worker["url"], started_at=datetime.datetime.now().isoformat()) + log.info("Job %s: dispatching to %s (%s %s)", job_id, worker["url"], worker.get("os"), worker.get("cpu")) self._append(job_id, f"[scheduler] Dispatching to {worker['url']} ({worker.get('os')} {worker.get('cpu')})\n") try: @@ -102,6 +107,7 @@ class Scheduler: else: worker_job_id = self._request_upload(job, worker) except Exception as e: + log.error("Job %s: dispatch failed: %s", job_id, e) db.update_job(job_id, status="failed", finished_at=datetime.datetime.now().isoformat(), error=str(e)) self._append(job_id, f"[scheduler] FAILED: {e}\n") @@ -116,8 +122,10 @@ class Scheduler: self._wait_for_build(job, worker, worker_job_id) self._fetch_artifacts(job, worker, worker_job_id) db.update_job(job_id, status="done", finished_at=datetime.datetime.now().isoformat()) + log.info("Job %s: done, artifacts saved", job_id) self._append(job_id, "[scheduler] Build complete, artifacts saved.\n") except Exception as e: + log.error("Job %s: failed: %s", job_id, e) db.update_job(job_id, status="failed", finished_at=datetime.datetime.now().isoformat(), error=str(e)) self._append(job_id, f"[scheduler] FAILED: {e}\n") diff --git a/ctrl/static/app.js b/ctrl/static/app.js index d8cdebf..d031d0c 100644 --- a/ctrl/static/app.js +++ b/ctrl/static/app.js @@ -176,8 +176,6 @@ $("#job-form").addEventListener("submit", async (e) => { body.append("file", file); } try { - const opts = { method: "POST", body }; - if (!git) opts.headers = { "Content-Type": "multipart/form-data" }; // let browser set boundary const res = await fetch("/api/jobs", { method: "POST", body }); const data = await res.json().catch(() => ({})); if (!res.ok) throw new Error(data.error || res.statusText);