debug

2026-04-17 16:17:19 +08:00
parent 0e096947fc
commit bd6bcb240f
17 changed files with 863 additions and 23 deletions
--- a/domain-api/app/api/routes/detect.py
+++ b/domain-api/app/api/routes/detect.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from uuid import uuid4

 from fastapi import APIRouter
--- a/domain-api/app/services/dashboard.py
+++ b/domain-api/app/services/dashboard.py
@@ -28,11 +28,13 @@ def fetch_overview() -> dict:
    online_worker_nodes = int(cluster_summary.get("online_worker_nodes", 0) or 0)
    dedicated_online_worker_nodes = int(cluster_summary.get("dedicated_online_worker_nodes", 0) or 0)
    result["worker_status"] = "online" if runtime["worker"]["running"] else "offline"
-    result["local_worker_status"] = result["worker_status"]
+    result["local_worker_status"] = "not-applicable" if not runtime["worker"].get("expected_on_this_node", True) else result["worker_status"]
    result["cluster_worker_status"] = "online" if online_worker_nodes > 0 else "offline"
    result["cluster_online_worker_nodes"] = online_worker_nodes
    result["cluster_dedicated_online_worker_nodes"] = dedicated_online_worker_nodes
    result["cluster_online_control_nodes"] = int(cluster_summary.get("online_control_nodes", 0) or 0)
    result["api_status"] = "online"
    result["worker_mode"] = runtime["worker"]["mode"]
+    result["node_region"] = runtime["node"]["region"]
+    result["node_role"] = runtime["node"]["role"]
    return result
--- a/domain-api/app/services/runtime_status_service.py
+++ b/domain-api/app/services/runtime_status_service.py
@@ -270,6 +270,7 @@ def get_runtime_status() -> dict:
    runtime_settings = get_runtime_settings()
    worker_runtime = detect_worker_runtime()
    sync_agent_runtime = detect_sync_agent_runtime()
+    worker_expected_on_this_node = not (settings.node_region == "overseas" and settings.node_role == "control")
    api_pid = os.getpid()
    detect_snapshot = get_detect_status()
    latest_run = (detect_snapshot.get("runs") or [None])[0] or {}
@@ -314,6 +315,35 @@ def get_runtime_status() -> dict:
        "queue_health": queue_health,
        "capacity_plan": capacity_plan,
    }
+    if not worker_expected_on_this_node:
+        detect_payload.update(
+            {
+                "phase_label": "当前节点不承载",
+                "phase_detail": "当前节点为海外控制面，仅承载 API 控制与同步接收，不执行本机检测任务。",
+                "recent_event": "",
+                "recent_warning": "",
+                "progress_percent": 0,
+                "active_thread_count": 0,
+                "max_thread_count": 0,
+                "available_proxy_count": 0,
+                "proxy_pool_count": 0,
+                "proxy_runtime_label": "不适用",
+                "proxy_runtime_detail": "当前节点不承载本机 Worker，代理、线程与检测阶段信息在此节点上不适用。",
+                "proxy_runtime_reason": "not_applicable",
+                "proxy_supplier_empty": False,
+                "proxy_last_refresh_status": "",
+                "proxy_last_refresh_time": "",
+                "proxy_last_refresh_source_count": 0,
+                "proxy_last_refresh_total_items": 0,
+                "proxy_last_validated_count": 0,
+                "proxy_last_available_count": 0,
+                "proxy_source_stats": [],
+                "dependency_alerts": [],
+                "active_job": None,
+                "runs_count": 0,
+                "worker_online": False,
+            }
+        )
    detect_payload["participating_nodes"] = _build_participating_detect_nodes(
        cluster_snapshot=cluster_snapshot,
        detect_snapshot=detect_payload,
@@ -351,6 +381,7 @@ def get_runtime_status() -> dict:
            "mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
            "service_name": runtime_settings.get("worker_service_name", settings.worker_service_name),
            "running": worker_runtime.get("running", False),
+            "expected_on_this_node": worker_expected_on_this_node,
            "process_count": worker_runtime.get("process_count", 0),
            "latest_start_time": worker_runtime.get("latest_start_time", ""),
            "message": worker_runtime.get("message", ""),
--- a/domain-api/deploy/multi-region/README.md
+++ b/domain-api/deploy/multi-region/README.md
@@ -25,6 +25,11 @@
 - `deploy/multi-region/bootstrap_mainland.sh`
 - `deploy/multi-region/check_cluster.sh`
 - `deploy/multi-region/check_mainland_controller.sh`
+- `deploy/multi-region/check_mainland_worker.sh`
+- `deploy/multi-region/check_worker_participation.sh`
+- `deploy/multi-region/check_temp_overseas_control.sh`
+- `deploy/multi-region/check_temp_link.sh`
+- `deploy/multi-region/check_temp_topology.sh`
 - `deploy/multi-region/simulate_cluster_node.py`
 - `deploy/multi-region/simulate_multi_region.sh`
 - `deploy/multi-region/prune_cluster_nodes.py`
@@ -199,13 +204,48 @@ bash deploy/multi-region/check_mainland_controller.sh

 这条命令会直接检查：

- `/etc/default/domaincheck-worker` 是否存在
+- `/etc/default/domaincheck-api` 是否存在
 - `NODE_ROLE=control` 是否正确
 - `SYNC_PUSH_ENABLED` / `SYNC_TARGET_API_BASE_URL` 是否已配置
+- `domaincheck-api`
 - `domaincheck-worker`
 - `domaincheck-sync-agent`
  是否已启用并处于运行态

+如果要判断“在线 Worker 有几台”和“当前真正领任务执行的是哪台”是否一致，可以执行：
+
+```bash
+cd /opt/domaincheck/domain-api
+bash deploy/multi-region/check_worker_participation.sh
+```
+
+这条命令会直接输出：
+
+- `detect_worker_nodes` 当前在线节点概览
+- 最新活跃任务的 `claimed_by / status / count`
+- 一段压缩摘要 JSON
+
+如果要检查纯大陆 worker 节点自己的配置和服务状态，可以执行：
+
+```bash
+cd /opt/domaincheck/domain-api
+bash deploy/multi-region/check_mainland_worker.sh
+```
+
+如果当前是“大陆 controller + 大陆 worker + 临时海外控制面”的联调拓扑，还可以执行：
+
+```bash
+cd /opt/domaincheck/domain-api
+bash deploy/multi-region/check_temp_topology.sh http://127.0.0.1:8100 http://152.53.37.118:8100
+```
+
+这条命令会同时汇总：
+
+- 大陆控制面的 `runtime/cluster`
+- 大陆控制面的 `runtime/readiness`
+- 最新活跃任务到底由谁 `claimed_by`
+- 临时海外控制面的 `runtime/cluster`
+
 如果当前还无法真正部署到大陆机器，也可以先在国外测试机上做“单机模拟多节点联调”：

 ```bash
--- a/domain-api/deploy/multi-region/check_mainland_controller.sh
+++ b/domain-api/deploy/multi-region/check_mainland_controller.sh
@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail

-ENV_FILE="${1:-/etc/default/domaincheck-worker}"
+ENV_FILE="${1:-/etc/default/domaincheck-api}"
 WORKER_SERVICE="${WORKER_SERVICE:-domaincheck-worker}"
+API_SERVICE="${API_SERVICE:-domaincheck-api}"
 SYNC_AGENT_SERVICE="${SYNC_AGENT_SERVICE:-domaincheck-sync-agent}"
 PYTHON_BIN="${PYTHON_BIN:-python3}"

@@ -61,16 +62,22 @@ if errors:
 PY
 echo

-echo "[3/5] systemd status"
+echo "[3/6] systemd status"
+systemctl is-enabled "$API_SERVICE" || true
+systemctl is-active "$API_SERVICE"
 systemctl is-enabled "$WORKER_SERVICE" || true
 systemctl is-active "$WORKER_SERVICE"
 systemctl is-enabled "$SYNC_AGENT_SERVICE" || true
 systemctl is-active "$SYNC_AGENT_SERVICE"
 echo

-echo "[4/5] worker service detail"
+echo "[4/6] api service detail"
+systemctl status "$API_SERVICE" --no-pager -l | sed -n '1,25p'
+echo
+
+echo "[5/6] worker service detail"
 systemctl status "$WORKER_SERVICE" --no-pager -l | sed -n '1,25p'
 echo

-echo "[5/5] sync-agent detail"
+echo "[6/6] sync-agent detail"
 systemctl status "$SYNC_AGENT_SERVICE" --no-pager -l | sed -n '1,25p'
--- a/domain-api/deploy/multi-region/check_mainland_worker.sh
+++ b/domain-api/deploy/multi-region/check_mainland_worker.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ENV_FILE="${1:-/etc/default/domaincheck-worker}"
+WORKER_SERVICE="${WORKER_SERVICE:-domaincheck-worker}"
+PYTHON_BIN="${PYTHON_BIN:-python3}"
+
+echo "[1/4] environment file"
+if [ ! -f "$ENV_FILE" ]; then
+  echo "missing env file: $ENV_FILE"
+  exit 1
+fi
+echo "env file: $ENV_FILE"
+echo
+
+echo "[2/4] key env summary"
+"${PYTHON_BIN}" - <<'PY' "$ENV_FILE"
+import json
+import sys
+from pathlib import Path
+
+env_path = Path(sys.argv[1])
+values = {}
+for line in env_path.read_text(encoding="utf-8").splitlines():
+    text = line.strip()
+    if not text or text.startswith("#") or "=" not in text:
+        continue
+    key, value = text.split("=", 1)
+    values[key.strip()] = value.strip()
+
+required = {
+    "NODE_CODE": values.get("NODE_CODE", ""),
+    "NODE_REGION": values.get("NODE_REGION", ""),
+    "NODE_ROLE": values.get("NODE_ROLE", ""),
+    "DB_HOST": values.get("DB_HOST", ""),
+    "DB_PORT": values.get("DB_PORT", ""),
+    "REDIS_HOST": values.get("REDIS_HOST", ""),
+    "REDIS_PORT": values.get("REDIS_PORT", ""),
+    "SYNC_TARGET_API_BASE_URL": values.get("SYNC_TARGET_API_BASE_URL", ""),
+}
+print(json.dumps(required, ensure_ascii=False, indent=2))
+
+errors = []
+if required["NODE_REGION"] != "mainland":
+    errors.append("NODE_REGION 必须为 mainland")
+if required["NODE_ROLE"] != "worker":
+    errors.append("NODE_ROLE 必须为 worker")
+if not required["NODE_CODE"]:
+    errors.append("NODE_CODE 不能为空")
+if not required["DB_HOST"]:
+    errors.append("DB_HOST 不能为空")
+if not required["REDIS_HOST"]:
+    errors.append("REDIS_HOST 不能为空")
+
+if errors:
+    print()
+    print("env validation failed:")
+    for item in errors:
+        print(f"- {item}")
+    sys.exit(1)
+PY
+echo
+
+echo "[3/4] systemd status"
+systemctl is-enabled "$WORKER_SERVICE" || true
+systemctl is-active "$WORKER_SERVICE"
+echo
+
+echo "[4/4] worker service detail"
+systemctl status "$WORKER_SERVICE" --no-pager -l | sed -n '1,30p'
--- a/domain-api/deploy/multi-region/check_temp_link.sh
+++ b/domain-api/deploy/multi-region/check_temp_link.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MAINLAND_BASE_URL="${1:-http://127.0.0.1:8100}"
+OVERSEAS_BASE_URL="${2:-http://152.53.37.118:8100}"
+PYTHON_BIN="${PYTHON_BIN:-python3}"
+
+echo "[1/6] mainland readiness"
+MAINLAND_READINESS="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")"
+echo "${MAINLAND_READINESS}"
+echo
+echo
+
+echo "[2/6] mainland sync summary"
+MAINLAND_SYNC="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/sync-summary")"
+echo "${MAINLAND_SYNC}"
+echo
+echo
+
+echo "[3/6] mainland push sync"
+MAINLAND_PUSH="$(curl -fsS -X POST "${MAINLAND_BASE_URL}/api/v1/runtime/actions/push_sync")"
+echo "${MAINLAND_PUSH}"
+echo
+echo
+
+echo "[4/6] overseas cluster"
+OVERSEAS_CLUSTER="$(curl -fsS "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")"
+echo "${OVERSEAS_CLUSTER}"
+echo
+echo
+
+echo "[5/6] overseas sync summary"
+OVERSEAS_SYNC="$(curl -fsS "${OVERSEAS_BASE_URL}/api/v1/runtime/sync-summary")"
+echo "${OVERSEAS_SYNC}"
+echo
+echo
+
+echo "[6/6] condensed summary"
+"${PYTHON_BIN}" - <<'PY' "$MAINLAND_READINESS" "$MAINLAND_SYNC" "$MAINLAND_PUSH" "$OVERSEAS_CLUSTER" "$OVERSEAS_SYNC"
+import json
+import sys
+
+mainland_readiness = json.loads(sys.argv[1]).get("data", {})
+mainland_sync = json.loads(sys.argv[2]).get("data", {})
+mainland_push = json.loads(sys.argv[3])
+overseas_cluster = json.loads(sys.argv[4]).get("data", {})
+overseas_sync = json.loads(sys.argv[5]).get("data", {})
+
+cluster_nodes = overseas_cluster.get("nodes") or []
+cluster_summary = overseas_cluster.get("summary") or {}
+type_counts = overseas_sync.get("type_counts") or {}
+status_counts = overseas_sync.get("status_counts") or {}
+
+print(
+    json.dumps(
+        {
+            "mainland": {
+                "readiness_status": mainland_readiness.get("status", ""),
+                "readiness_summary": mainland_readiness.get("summary", ""),
+                "sync_enabled": bool(mainland_sync.get("enabled", False)),
+                "target_api_base_url": mainland_sync.get("target_api_base_url", ""),
+                "push_sync_code": mainland_push.get("code"),
+                "push_sync_message": mainland_push.get("message", ""),
+            },
+            "overseas": {
+                "nodes_total": overseas_cluster.get("nodes_total", 0),
+                "online_control_nodes": cluster_summary.get("online_control_nodes", 0),
+                "online_worker_nodes": cluster_summary.get("online_worker_nodes", 0),
+                "node_codes": [str(item.get("node_code") or "") for item in cluster_nodes],
+                "runtime_ingest": type_counts.get("runtime_ingest", 0),
+                "detect_result_ingest": type_counts.get("detect_result_ingest", 0),
+                "status_received": status_counts.get("received", 0),
+                "status_success": status_counts.get("success", 0),
+            },
+        },
+        ensure_ascii=False,
+        indent=2,
+    )
+)
+PY
--- a/domain-api/deploy/multi-region/check_temp_overseas_control.sh
+++ b/domain-api/deploy/multi-region/check_temp_overseas_control.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BASE_URL="${1:-http://127.0.0.1:8100}"
+PYTHON_BIN="${PYTHON_BIN:-python3}"
+
+echo "[1/5] health"
+curl -fsS "${BASE_URL}/health"
+echo
+echo
+
+echo "[2/5] runtime status"
+STATUS_JSON="$(curl -fsS "${BASE_URL}/api/v1/runtime/status")"
+echo "${STATUS_JSON}"
+echo
+echo
+
+echo "[3/5] runtime cluster"
+curl -fsS "${BASE_URL}/api/v1/runtime/cluster"
+echo
+echo
+
+echo "[4/5] runtime readiness"
+curl -fsS "${BASE_URL}/api/v1/runtime/readiness"
+echo
+echo
+
+echo "[5/5] condensed summary"
+"${PYTHON_BIN}" - <<'PY' "$STATUS_JSON"
+import json
+import sys
+
+data = json.loads(sys.argv[1]).get("data", {})
+node = data.get("node") or {}
+worker = data.get("worker") or {}
+sync_agent = data.get("sync_agent") or {}
+detect = data.get("detect") or {}
+cluster = data.get("cluster") or {}
+summary = cluster.get("summary") or {}
+
+print(
+    json.dumps(
+        {
+            "node": {
+                "code": node.get("code", ""),
+                "region": node.get("region", ""),
+                "role": node.get("role", ""),
+            },
+            "worker": {
+                "running": bool(worker.get("running", False)),
+                "expected_on_this_node": bool(worker.get("expected_on_this_node", True)),
+                "message": worker.get("message", ""),
+            },
+            "sync_agent": {
+                "running": bool(sync_agent.get("running", False)),
+                "expected_on_this_node": bool(sync_agent.get("expected_on_this_node", False)),
+                "message": sync_agent.get("message", ""),
+            },
+            "detect": {
+                "phase_label": detect.get("phase_label", ""),
+                "proxy_runtime_label": detect.get("proxy_runtime_label", ""),
+                "active_job": (detect.get("active_job") or {}).get("job_code", ""),
+            },
+            "cluster": {
+                "nodes_total": cluster.get("nodes_total", 0),
+                "online_control_nodes": summary.get("online_control_nodes", 0),
+                "online_worker_nodes": summary.get("online_worker_nodes", 0),
+                "dedicated_online_worker_nodes": summary.get("dedicated_online_worker_nodes", 0),
+            },
+        },
+        ensure_ascii=False,
+        indent=2,
+    )
+)
+PY
--- a/domain-api/deploy/multi-region/check_temp_topology.sh
+++ b/domain-api/deploy/multi-region/check_temp_topology.sh
@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MAINLAND_BASE_URL="${1:-http://127.0.0.1:8100}"
+OVERSEAS_BASE_URL="${2:-http://152.53.37.118:8100}"
+PYTHON_BIN="${PYTHON_BIN:-python3}"
+
+echo "[1/5] mainland runtime cluster"
+MAINLAND_CLUSTER="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/cluster")"
+echo "${MAINLAND_CLUSTER}"
+echo
+echo
+
+echo "[2/5] mainland runtime readiness"
+MAINLAND_READINESS="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")"
+echo "${MAINLAND_READINESS}"
+echo
+echo
+
+echo "[3/5] mainland worker participation"
+PARTICIPATION_JSON="$(/www/server/pgsql/bin/psql -U postgres -d domain -At <<'SQL'
+with latest_job as (
+  select id, job_code, status
+  from detect_jobs
+  where status in ('pending','running')
+  order by id desc
+  limit 1
+),
+node_stats as (
+  select
+    node_code,
+    role,
+    status,
+    coalesce(metadata_json->>'worker_online', '') as worker_online,
+    coalesce(metadata_json->>'detect_participating', '') as detect_participating
+  from detect_worker_nodes
+),
+item_stats as (
+  select
+    coalesce(claimed_by, '') as claimed_by,
+    status,
+    count(*) as cnt
+  from detect_job_items
+  where job_id = (select id from latest_job)
+  group by claimed_by, status
+)
+select json_build_object(
+  'job', (
+    select json_build_object(
+      'job_code', coalesce(job_code, ''),
+      'status', coalesce(status, '')
+    )
+    from latest_job
+  ),
+  'online_nodes', (
+    select coalesce(json_agg(json_build_object(
+      'node_code', node_code,
+      'role', role,
+      'status', status,
+      'worker_online', worker_online,
+      'detect_participating', detect_participating
+    ) order by node_code), '[]'::json)
+    from node_stats
+    where status in ('online', 'busy')
+  ),
+  'claimed_distribution', (
+    select coalesce(json_agg(json_build_object(
+      'claimed_by', claimed_by,
+      'status', status,
+      'count', cnt
+    ) order by claimed_by, status), '[]'::json)
+    from item_stats
+  )
+);
+SQL
+)"
+echo "${PARTICIPATION_JSON}"
+echo
+echo
+
+echo "[4/5] overseas runtime cluster"
+OVERSEAS_CLUSTER="$(curl -fsS "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")"
+echo "${OVERSEAS_CLUSTER}"
+echo
+echo
+
+echo "[5/5] condensed summary"
+"${PYTHON_BIN}" - <<'PY' "$MAINLAND_CLUSTER" "$MAINLAND_READINESS" "$PARTICIPATION_JSON" "$OVERSEAS_CLUSTER"
+import json
+import sys
+
+mainland_cluster = json.loads(sys.argv[1]).get("data", {})
+mainland_readiness = json.loads(sys.argv[2]).get("data", {})
+participation = json.loads(sys.argv[3])
+overseas_cluster = json.loads(sys.argv[4]).get("data", {})
+
+mainland_summary = mainland_cluster.get("summary") or {}
+overseas_summary = overseas_cluster.get("summary") or {}
+
+def node_codes(cluster):
+    return [str(item.get("node_code") or "") for item in (cluster.get("nodes") or [])]
+
+print(
+    json.dumps(
+        {
+            "mainland": {
+                "readiness_status": mainland_readiness.get("status", ""),
+                "readiness_summary": mainland_readiness.get("summary", ""),
+                "online_worker_nodes": mainland_summary.get("online_worker_nodes", 0),
+                "dedicated_online_worker_nodes": mainland_summary.get("dedicated_online_worker_nodes", 0),
+                "online_control_nodes": mainland_summary.get("online_control_nodes", 0),
+                "node_codes": node_codes(mainland_cluster),
+            },
+            "participation": participation,
+            "overseas": {
+                "online_worker_nodes": overseas_summary.get("online_worker_nodes", 0),
+                "dedicated_online_worker_nodes": overseas_summary.get("dedicated_online_worker_nodes", 0),
+                "online_control_nodes": overseas_summary.get("online_control_nodes", 0),
+                "node_codes": node_codes(overseas_cluster),
+            },
+        },
+        ensure_ascii=False,
+        indent=2,
+    )
+)
+PY
--- a/domain-api/deploy/multi-region/check_worker_participation.sh
+++ b/domain-api/deploy/multi-region/check_worker_participation.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+DB_NAME="${DB_NAME:-domain}"
+DB_USER="${DB_USER:-postgres}"
+PSQL_BIN="${PSQL_BIN:-/www/server/pgsql/bin/psql}"
+
+echo "[1/3] cluster nodes"
+"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc "
+select
+  node_code || '|' ||
+  role || '|' ||
+  status || '|' ||
+  coalesce(current_load, 0) || '|' ||
+  to_char(last_heartbeat_at, 'YYYY-MM-DD HH24:MI:SS') || '|' ||
+  coalesce(metadata_json->>'worker_online', '') || '|' ||
+  coalesce(metadata_json->>'detect_participating', '')
+from detect_worker_nodes
+order by node_code;
+"
+echo
+
+echo "[2/3] active job distribution"
+"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc "
+select
+  coalesce(claimed_by, '') || '|' ||
+  status || '|' ||
+  count(*)
+from detect_job_items
+where job_id = (
+  select id
+  from detect_jobs
+  where status in ('pending','running')
+  order by id desc
+  limit 1
+)
+group by claimed_by, status
+order by claimed_by, status;
+"
+echo
+
+echo "[3/3] condensed summary"
+"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -At <<'SQL'
+with latest_job as (
+  select id, job_code, status
+  from detect_jobs
+  where status in ('pending','running')
+  order by id desc
+  limit 1
+),
+node_stats as (
+  select
+    node_code,
+    role,
+    status,
+    coalesce(metadata_json->>'worker_online', '') as worker_online,
+    coalesce(metadata_json->>'detect_participating', '') as detect_participating
+  from detect_worker_nodes
+),
+item_stats as (
+  select
+    coalesce(claimed_by, '') as claimed_by,
+    status,
+    count(*) as cnt
+  from detect_job_items
+  where job_id = (select id from latest_job)
+  group by claimed_by, status
+)
+select json_build_object(
+  'job', (
+    select json_build_object(
+      'job_code', coalesce(job_code, ''),
+      'status', coalesce(status, '')
+    )
+    from latest_job
+  ),
+  'online_nodes', (
+    select coalesce(json_agg(json_build_object(
+      'node_code', node_code,
+      'role', role,
+      'status', status,
+      'worker_online', worker_online,
+      'detect_participating', detect_participating
+    ) order by node_code), '[]'::json)
+    from node_stats
+    where status in ('online', 'busy')
+  ),
+  'claimed_distribution', (
+    select coalesce(json_agg(json_build_object(
+      'claimed_by', claimed_by,
+      'status', status,
+      'count', cnt
+    ) order by claimed_by, status), '[]'::json)
+    from item_stats
+  )
+);
+SQL